Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- RELEASES.json +1 -1
- VERSION +1 -1
- api/agent_directory.py +20 -3
- api/amazon_quotes_loop.py +128 -0
- api/app_templates.py +10 -5
- api/application_registry.py +274 -17
- api/assistant_memory.py +69 -0
- api/assistant_mirror.py +19 -12
- api/automation_engine.py +383 -14
- api/connector_onboarding.py +6 -5
- api/deps.py +228 -58
- api/diag_aging.py +8 -13
- api/external_freshness.py +22 -9
- api/loopy_file_tools.py +3 -1
- api/loopy_tools.py +1 -1
- api/main.py +610 -62
- api/odoo_relational.py +144 -35
- api/pages.py +6 -4
- api/process_role.py +366 -161
- api/registry_source.py +193 -1
- api/rollup_sql.py +19 -7
- api/routes_account_health.py +141 -10
- api/routes_admin.py +107 -8
- api/routes_aged_stock.py +8 -5
- api/routes_amazon_dashboard.py +18 -43
- api/routes_amazon_operator.py +103 -15
- api/routes_amazon_profit.py +163 -19
- api/routes_amazon_projects.py +389 -0
- api/routes_app_dashboard.py +72 -26
- api/routes_applications.py +1 -1
- api/routes_assets.py +22 -4
- api/routes_auth.py +5 -1
- api/routes_competitor_watch.py +80 -16
- api/routes_competitors.py +203 -0
- api/routes_connectors.py +127 -28
- api/routes_customers.py +86 -16
- api/routes_external.py +517 -54
- api/routes_feedback.py +45 -4
- api/routes_grid.py +44 -2
- api/routes_keychain.py +282 -3
- api/routes_nav.py +12 -8
- api/routes_odoo_tables.py +177 -19
- api/routes_opportunity_finder.py +47 -24
- api/routes_payouts.py +38 -15
- api/routes_products.py +24 -2
- api/routes_query.py +10 -4
- api/routes_ranking_diagnostic.py +58 -34
- api/routes_reimbursements.py +60 -18
- api/routes_repeat_buyers.py +16 -11
- api/routes_restock_planner.py +96 -6
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "53755382a",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
53755382a
|
api/agent_directory.py
CHANGED
|
@@ -166,11 +166,28 @@ def _filter_sentences(tree, seen, label):
|
|
| 166 |
return [LIMITED if p is None else prefix + p for p in parts]
|
| 167 |
|
| 168 |
|
| 169 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
out = []
|
| 171 |
labels = users.allowed_bus_labels(rec)
|
| 172 |
if "All" not in labels and labels:
|
| 173 |
-
out.append("
|
| 174 |
agent = perms.scope_agent(rec)
|
| 175 |
if agent:
|
| 176 |
out.append(f"Sales agent is {agent}")
|
|
@@ -210,7 +227,7 @@ def _entry(session, uname, rec, ctx):
|
|
| 210 |
if unnamed:
|
| 211 |
named.append(f"{unnamed} database{'' if unnamed == 1 else 's'} you cannot open")
|
| 212 |
if not admin and not perm_scope.is_migrated(principal):
|
| 213 |
-
limits = _legacy_limits(rec) + limits
|
| 214 |
return {"username": str(rec.get("username_original") or uname),
|
| 215 |
"name": str(rec.get("name") or uname),
|
| 216 |
"role": "Administrator" if admin else "Member",
|
|
|
|
| 166 |
return [LIMITED if p is None else prefix + p for p in parts]
|
| 167 |
|
| 168 |
|
| 169 |
+
#: The Odoo field a legacy team scope narrows, and Odoo's own name for it (R66).
|
| 170 |
+
_TEAM_TABLE, _TEAM_FIELD, _TEAM_LABEL = "ut_odoo_orders", "team", "Sales team"
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _team_label(st, caller):
|
| 174 |
+
"""The caller-visible team label, with Odoo's name for an absent field."""
|
| 175 |
+
try:
|
| 176 |
+
from core import user_tables
|
| 177 |
+
fields = (user_tables.get_one(_TEAM_TABLE, st) or {}).get("fields") or ()
|
| 178 |
+
field = next((f for f in fields if isinstance(f, dict) and f.get("key") == _TEAM_FIELD), {})
|
| 179 |
+
if field:
|
| 180 |
+
return _name_field(_fields(caller, _TEAM_TABLE, st), _TEAM_FIELD)
|
| 181 |
+
return _TEAM_LABEL
|
| 182 |
+
except Exception: # noqa: BLE001
|
| 183 |
+
return _TEAM_LABEL
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _legacy_limits(rec, st=None, caller=None):
|
| 187 |
out = []
|
| 188 |
labels = users.allowed_bus_labels(rec)
|
| 189 |
if "All" not in labels and labels:
|
| 190 |
+
out.append(f"{_team_label(st, caller)} is " + " or ".join(labels))
|
| 191 |
agent = perms.scope_agent(rec)
|
| 192 |
if agent:
|
| 193 |
out.append(f"Sales agent is {agent}")
|
|
|
|
| 227 |
if unnamed:
|
| 228 |
named.append(f"{unnamed} database{'' if unnamed == 1 else 's'} you cannot open")
|
| 229 |
if not admin and not perm_scope.is_migrated(principal):
|
| 230 |
+
limits = _legacy_limits(rec, st, caller) + limits
|
| 231 |
return {"username": str(rec.get("username_original") or uname),
|
| 232 |
"name": str(rec.get("name") or uname),
|
| 233 |
"role": "Administrator" if admin else "Member",
|
api/amazon_quotes_loop.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Price and rank every rival once a day per tenant in the sync worker, outside the sync pass (C7).
|
| 2 |
+
|
| 3 |
+
The loop holds the SP-API client and passes its reads into `core.amazon_rivals.run_quotes`, because
|
| 4 |
+
core imports no harness module. It never joins `amazon_store.AFTER_SYNC` and never runs as an HF job.
|
| 5 |
+
"""
|
| 6 |
+
from contextlib import contextmanager
|
| 7 |
+
import importlib
|
| 8 |
+
import logging
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import threading
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
JOB = "amazon-rival-quotes"
|
| 14 |
+
#: The loop asks each tenant whether its daily run is due this often.
|
| 15 |
+
POLL_SECONDS = 600
|
| 16 |
+
|
| 17 |
+
#: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
|
| 18 |
+
#: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
|
| 19 |
+
#: Command Center patches the competitor cells of the rows those ASINs move (S82-T89).
|
| 20 |
+
AFTER_QUOTES = ("modules.amazon_profit:on_quotes", "modules.competitor_watch:on_quotes")
|
| 21 |
+
|
| 22 |
+
_log = logging.getLogger(__name__)
|
| 23 |
+
_missing = False
|
| 24 |
+
_unavailable = set()
|
| 25 |
+
_thread = None
|
| 26 |
+
_stop = None
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _say(message):
|
| 30 |
+
print(f"[{JOB}] {message}", flush=True)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def readers(fields):
|
| 34 |
+
"""The three SP-API reads of one tenant's credential. Each takes a list of at most 20 ASINs."""
|
| 35 |
+
from harness import amazon_client as cx
|
| 36 |
+
return {"offers": lambda asins: cx.item_offers(fields, asins),
|
| 37 |
+
# The pricing sync draws this same `competitive_summary` pacer in this worker process.
|
| 38 |
+
"summaries": lambda asins: cx.competitive_summaries(fields, asins),
|
| 39 |
+
"ranks": lambda asins: cx.catalog_sales_ranks(fields, asins)}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@contextmanager
|
| 43 |
+
def _mirror(runtime):
|
| 44 |
+
"""A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
|
| 45 |
+
from harness import runtime as R
|
| 46 |
+
if not Path(runtime.datastore_path).is_file():
|
| 47 |
+
yield None
|
| 48 |
+
return
|
| 49 |
+
cur = R.mirror_cursor(runtime, require_ready=False)
|
| 50 |
+
try:
|
| 51 |
+
yield cur
|
| 52 |
+
finally:
|
| 53 |
+
cur.close()
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _after_quotes(tenant, changed):
|
| 57 |
+
for target in AFTER_QUOTES:
|
| 58 |
+
try:
|
| 59 |
+
module_name, _, name = target.partition(":")
|
| 60 |
+
getattr(importlib.import_module(module_name), name)(tenant, changed)
|
| 61 |
+
_say(f"tenant={tenant} after-quotes callback {target} completed")
|
| 62 |
+
except Exception as exc: # noqa: BLE001
|
| 63 |
+
_say(f"tenant={tenant} after-quotes callback {target} failed ({type(exc).__name__}); "
|
| 64 |
+
"the run result is unchanged.")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
|
| 68 |
+
"""Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
|
| 69 |
+
from core import amazon_rivals as rivals
|
| 70 |
+
try:
|
| 71 |
+
if not rivals.due(runtime, now=now):
|
| 72 |
+
return None
|
| 73 |
+
except rivals.BackendUnavailable:
|
| 74 |
+
if tenant not in _unavailable:
|
| 75 |
+
_unavailable.add(tenant)
|
| 76 |
+
_say(f"tenant={tenant} has no PostgreSQL store, so no rival quotes run")
|
| 77 |
+
return None
|
| 78 |
+
if fields is None:
|
| 79 |
+
from harness import amazon_store
|
| 80 |
+
fields = amazon_store.credential(tenant, runtime)
|
| 81 |
+
if not fields:
|
| 82 |
+
return None
|
| 83 |
+
result = rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
|
| 84 |
+
seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
|
| 85 |
+
log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep)
|
| 86 |
+
if result["cause"] is None:
|
| 87 |
+
_after_quotes(tenant, result["changed"])
|
| 88 |
+
return result
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def run_once(*, for_each_tenant=None, now=None):
|
| 92 |
+
"""Admit one independent per-tenant job without waiting for its body."""
|
| 93 |
+
global _missing
|
| 94 |
+
if for_each_tenant is None:
|
| 95 |
+
try:
|
| 96 |
+
from tenant_loops import for_each_tenant
|
| 97 |
+
except ImportError:
|
| 98 |
+
if not _missing:
|
| 99 |
+
_log.warning("Rival quotes unavailable: tenant_loops is not installed.")
|
| 100 |
+
_missing = True
|
| 101 |
+
return {}
|
| 102 |
+
_missing = False
|
| 103 |
+
return for_each_tenant(lambda tenant, runtime: run_tenant(tenant, runtime, now=now),
|
| 104 |
+
name=JOB, logger=_log, kind="heavy")
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def start():
|
| 108 |
+
"""Start the daily loop only when process_role admits its heavy coordinator (the sync worker)."""
|
| 109 |
+
global _thread, _stop
|
| 110 |
+
import process_role
|
| 111 |
+
if _thread is not None and _thread.is_alive():
|
| 112 |
+
return _thread
|
| 113 |
+
stop_event = threading.Event()
|
| 114 |
+
_stop = stop_event
|
| 115 |
+
|
| 116 |
+
def run():
|
| 117 |
+
while not stop_event.is_set():
|
| 118 |
+
run_once()
|
| 119 |
+
stop_event.wait(POLL_SECONDS)
|
| 120 |
+
|
| 121 |
+
# A literal name: verify_background_loops classifies a job by it.
|
| 122 |
+
_thread = process_role.start("amazon-rival-quotes", "heavy", run)
|
| 123 |
+
return _thread
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def stop():
|
| 127 |
+
if _stop is not None:
|
| 128 |
+
_stop.set()
|
api/app_templates.py
CHANGED
|
@@ -78,11 +78,12 @@ TEMPLATES = (
|
|
| 78 |
"module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"),
|
| 79 |
"proposes": False},
|
| 80 |
_amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.",
|
| 81 |
-
"Acquire", ("ut_amzn_brand_analytics",), proposes=False),
|
| 82 |
_amazon("ranking_diagnostic", "Ranking diagnostic", "Find why a listing lost its rank.",
|
| 83 |
"Acquire", ("ut_amzn_pricing", "ut_amzn_catalog_items", "ut_amzn_brand_analytics",
|
| 84 |
"ut_amzn_economics", "ut_amzn_fba_inventory", "ut_amzn_suppressed_listings",
|
| 85 |
-
"ut_amzn_stranded_inventory", "ut_amzn_inventory_planning"), proposes=False
|
|
|
|
| 86 |
_amazon("amazon_operator", "Operator inbox", "Approve ad changes in one batch.",
|
| 87 |
"Acquire", ("ut_amzn_ads_sp_campaigns", "ut_amzn_ads_sb_campaigns", "ut_amzn_ads_sd_campaigns",
|
| 88 |
"ut_amzn_economics"), connector="amazon_ads", proposes=False),
|
|
@@ -100,11 +101,15 @@ TEMPLATES = (
|
|
| 100 |
"Create value", ("ut_amzn_brand_analytics", "ut_amzn_search_query_performance", "ut_amzn_listings"),
|
| 101 |
optional_connectors=("odoo",)),
|
| 102 |
_amazon("returns_refunds", "Returns and refunds", "See why buyers return and track each fix.",
|
| 103 |
-
"Retain", ("ut_amzn_fba_customer_returns", "ut_amzn_fba_shipped_sales", "
|
| 104 |
-
"
|
|
|
|
|
|
|
| 105 |
optional_connectors=("odoo",)),
|
| 106 |
_amazon("repeat_buyers", "Repeat buyers", "See who buys again and what they buy together.",
|
| 107 |
-
"Retain", ("ut_amzn_repeat_purchase", "ut_amzn_market_basket"), proposes=False
|
|
|
|
|
|
|
| 108 |
_amazon("payouts", "Payouts", "Tie each payout to its orders and fees.",
|
| 109 |
"Finance and cash", ("ut_amzn_settlements", "ut_amzn_economics", "ut_amzn_transactions"),
|
| 110 |
optional_connectors=("odoo",)),
|
|
|
|
| 78 |
"module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"),
|
| 79 |
"proposes": False},
|
| 80 |
_amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.",
|
| 81 |
+
"Acquire", ("ut_amzn_brand_analytics", "ut_amzn_listings"), proposes=False),
|
| 82 |
_amazon("ranking_diagnostic", "Ranking diagnostic", "Find why a listing lost its rank.",
|
| 83 |
"Acquire", ("ut_amzn_pricing", "ut_amzn_catalog_items", "ut_amzn_brand_analytics",
|
| 84 |
"ut_amzn_economics", "ut_amzn_fba_inventory", "ut_amzn_suppressed_listings",
|
| 85 |
+
"ut_amzn_stranded_inventory", "ut_amzn_inventory_planning"), proposes=False,
|
| 86 |
+
optional_sources=("ut_amzn_search_query_performance", "ut_amzn_listing_items")),
|
| 87 |
_amazon("amazon_operator", "Operator inbox", "Approve ad changes in one batch.",
|
| 88 |
"Acquire", ("ut_amzn_ads_sp_campaigns", "ut_amzn_ads_sb_campaigns", "ut_amzn_ads_sd_campaigns",
|
| 89 |
"ut_amzn_economics"), connector="amazon_ads", proposes=False),
|
|
|
|
| 101 |
"Create value", ("ut_amzn_brand_analytics", "ut_amzn_search_query_performance", "ut_amzn_listings"),
|
| 102 |
optional_connectors=("odoo",)),
|
| 103 |
_amazon("returns_refunds", "Returns and refunds", "See why buyers return and track each fix.",
|
| 104 |
+
"Retain", ("ut_amzn_fba_customer_returns", "ut_amzn_fba_shipped_sales", "ut_amzn_mfn_returns",
|
| 105 |
+
"ut_amzn_settlements", "ut_amzn_transactions", "ut_amzn_order_items"),
|
| 106 |
+
optional_sources=("ut_amzn_fba_inventory", "ut_amzn_removal_orders",
|
| 107 |
+
"ut_amzn_removal_shipments", "ut_amzn_storage_fees"),
|
| 108 |
optional_connectors=("odoo",)),
|
| 109 |
_amazon("repeat_buyers", "Repeat buyers", "See who buys again and what they buy together.",
|
| 110 |
+
"Retain", ("ut_amzn_repeat_purchase", "ut_amzn_market_basket"), proposes=False,
|
| 111 |
+
optional_sources=("ut_amzn_replenishment", "ut_amzn_subscribe_save_performance",
|
| 112 |
+
"ut_amzn_fba_shipped_sales", "ut_amzn_listings")),
|
| 113 |
_amazon("payouts", "Payouts", "Tie each payout to its orders and fees.",
|
| 114 |
"Finance and cash", ("ut_amzn_settlements", "ut_amzn_economics", "ut_amzn_transactions"),
|
| 115 |
optional_connectors=("odoo",)),
|
api/application_registry.py
CHANGED
|
@@ -185,6 +185,7 @@ def _connector_gate(connector):
|
|
| 185 |
|
| 186 |
# Competitor watch's gate already requires the Amazon connection; it adds brand analytics (R21).
|
| 187 |
_TEMPLATE_GATES = {kind: brand_analytics_offered if kind == "competitor_watch"
|
|
|
|
| 188 |
else _connector_gate(contract["connector"])
|
| 189 |
for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()}
|
| 190 |
#: S79-T02 (instr 20): Routing and Catalog are offered only to a reader with Odoo.
|
|
@@ -552,7 +553,8 @@ def load(runtime, *, strict=False):
|
|
| 552 |
_AVAILABILITY_MESSAGES = {
|
| 553 |
"router_unavailable": "This App cannot open because its routes are unavailable.",
|
| 554 |
"no_grant": "Your access limit prevents this App from reading its source.",
|
| 555 |
-
"not_connected": "Connect the App's connector to open this App.",
|
|
|
|
| 556 |
"no_rows": "This App has no source rows yet.",
|
| 557 |
"source_unavailable": "This App cannot read its source right now.",
|
| 558 |
"held": "This App is not available yet.",
|
|
@@ -565,7 +567,7 @@ def availability_reason(state, connector=None):
|
|
| 565 |
if state == "not_connected":
|
| 566 |
label = {"amazon": "Amazon", "amazon_ads": "Amazon Ads", "odoo": "Odoo"}.get(connector)
|
| 567 |
if label:
|
| 568 |
-
message = f"Connect {label} to open this App."
|
| 569 |
return {"state": state, "message": message,
|
| 570 |
"lock": "Access limit" if state == "no_grant" else None}
|
| 571 |
|
|
@@ -631,8 +633,116 @@ def _readable_template_app(session, app_id):
|
|
| 631 |
return app, catalogue
|
| 632 |
|
| 633 |
|
| 634 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 635 |
from core import perm_scope
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 636 |
from routes_connectors import _entry_types
|
| 637 |
|
| 638 |
kind = app["templateKind"]
|
|
@@ -643,28 +753,51 @@ def _availability_for_app(session, app, catalogue):
|
|
| 643 |
return mount
|
| 644 |
if not _app_templates.seams_ready(kind):
|
| 645 |
return availability_reason("router_unavailable")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 646 |
try:
|
| 647 |
sources = catalogue["sources"]
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 654 |
types, _scopes, failure = _entry_types(session.runtime, session.uname)
|
| 655 |
if failure is not None:
|
| 656 |
-
return
|
| 657 |
connected = _app_templates.entry_type(catalogue["connector"]) in set(types or ())
|
| 658 |
if not connected and catalogue["connector"] == "odoo":
|
| 659 |
from routes_keychain import env_odoo_available
|
| 660 |
connected = env_odoo_available(session.runtime)
|
| 661 |
if not connected:
|
| 662 |
-
return
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 668 |
|
| 669 |
|
| 670 |
def availability(session, app_id):
|
|
@@ -1179,7 +1312,8 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
|
|
| 1179 |
mount = _mount_reason(row.get("templateKind"))
|
| 1180 |
if mount:
|
| 1181 |
row["availability"] = mount
|
| 1182 |
-
elif
|
|
|
|
| 1183 |
row["availability"] = availability_reason("no_grant")
|
| 1184 |
if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
|
| 1185 |
# A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
|
|
@@ -1644,6 +1778,129 @@ def migrate_returns_split(runtime, tenant):
|
|
| 1644 |
return {"outcome": "migrated", "itemIds": copy.deepcopy(copied["itemIds"])}
|
| 1645 |
|
| 1646 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1647 |
def ensure_route_application(runtime, tenant, *, accounts=None, wall=None):
|
| 1648 |
"""R15: boot migrations, once per tenant, never from an Application read.
|
| 1649 |
|
|
|
|
| 185 |
|
| 186 |
# Competitor watch's gate already requires the Amazon connection; it adds brand analytics (R21).
|
| 187 |
_TEMPLATE_GATES = {kind: brand_analytics_offered if kind == "competitor_watch"
|
| 188 |
+
else _connector_gate("amazon") if kind == "amazon_operator"
|
| 189 |
else _connector_gate(contract["connector"])
|
| 190 |
for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()}
|
| 191 |
#: S79-T02 (instr 20): Routing and Catalog are offered only to a reader with Odoo.
|
|
|
|
| 553 |
_AVAILABILITY_MESSAGES = {
|
| 554 |
"router_unavailable": "This App cannot open because its routes are unavailable.",
|
| 555 |
"no_grant": "Your access limit prevents this App from reading its source.",
|
| 556 |
+
"not_connected": "Connect the App's connector in Connectors to open this App.",
|
| 557 |
+
"refreshing": "This App's source is refreshing and opens when the refresh ends",
|
| 558 |
"no_rows": "This App has no source rows yet.",
|
| 559 |
"source_unavailable": "This App cannot read its source right now.",
|
| 560 |
"held": "This App is not available yet.",
|
|
|
|
| 567 |
if state == "not_connected":
|
| 568 |
label = {"amazon": "Amazon", "amazon_ads": "Amazon Ads", "odoo": "Odoo"}.get(connector)
|
| 569 |
if label:
|
| 570 |
+
message = f"Connect {label} in Connectors to open this App."
|
| 571 |
return {"state": state, "message": message,
|
| 572 |
"lock": "Access limit" if state == "no_grant" else None}
|
| 573 |
|
|
|
|
| 633 |
return app, catalogue
|
| 634 |
|
| 635 |
|
| 636 |
+
#: R62: App kinds whose readers apply a source row and field wall to their own rows.
|
| 637 |
+
#: Any other kind refuses its App while such a wall applies, because it cannot narrow.
|
| 638 |
+
NARROWING_KINDS = frozenset({"account_health", "opportunity_finder", "ranking_diagnostic", "repeat_buyers"})
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
def app_access(user, runtime, app, sources):
|
| 642 |
+
"""R46, R62: the one source-wall answer for an App, its Dashboard and its readers.
|
| 643 |
+
|
| 644 |
+
Takes a user record, not a session, so a digest can ask for any recipient. Returns
|
| 645 |
+
`app` and `dashboard` ("ready" or refused), the Dashboard refusal's `status`, `code`
|
| 646 |
+
and one-line `message`, and `narrow`: per source, the row wall and hidden field keys.
|
| 647 |
+
"""
|
| 648 |
+
from core import perm_scope, perms, route_system, user_tables
|
| 649 |
+
from core.request_timing import remaining_budget
|
| 650 |
+
|
| 651 |
+
user = user if isinstance(user, dict) else {}
|
| 652 |
+
kind = app.get("templateKind") if isinstance(app, dict) else None
|
| 653 |
+
granted = may_read(app, user.get("username"), perms.is_admin(user))
|
| 654 |
+
refusal = []
|
| 655 |
+
|
| 656 |
+
def refuse(status, code, message):
|
| 657 |
+
refusal.append((status, code, message))
|
| 658 |
+
|
| 659 |
+
all_units = "All" in perms.allowed_bu_labels(user)
|
| 660 |
+
raw_units = user.get("bus", "all")
|
| 661 |
+
if raw_units != "all":
|
| 662 |
+
from core import users
|
| 663 |
+
by_label = {label: key for key, label in users.BU_LABELS.items()}
|
| 664 |
+
identities = []
|
| 665 |
+
valid_units = isinstance(raw_units, (tuple, list)) and bool(raw_units)
|
| 666 |
+
for unit in raw_units if valid_units else ():
|
| 667 |
+
if type(unit) is int and unit in users.BU_LABELS:
|
| 668 |
+
identities.append(unit)
|
| 669 |
+
elif isinstance(unit, str) and unit in by_label:
|
| 670 |
+
identities.append(by_label[unit])
|
| 671 |
+
elif unit != "All":
|
| 672 |
+
valid_units = False
|
| 673 |
+
all_units = bool(valid_units and identities) and "All" in perms.allowed_bu_labels(dict(user, bus=identities))
|
| 674 |
+
if not all_units:
|
| 675 |
+
refuse(403, "app_dashboard_scope", "The Dashboard needs access to every business unit.")
|
| 676 |
+
elif perms.scope_agent(user) is not None:
|
| 677 |
+
refuse(403, "app_dashboard_scope", "The Dashboard needs access to every salesperson's records.")
|
| 678 |
+
if not isinstance(sources, (tuple, list)) or not sources:
|
| 679 |
+
refuse(503, "app_dashboard_sources", "The Dashboard has no source to read.")
|
| 680 |
+
sources = ()
|
| 681 |
+
config = getattr(getattr(runtime, "tenant", None), "config", None) or {}
|
| 682 |
+
enabled = config.get("modules", "all")
|
| 683 |
+
narrow = {}
|
| 684 |
+
for source in sources:
|
| 685 |
+
remaining_budget()
|
| 686 |
+
label = _source_label(source) or "a source database"
|
| 687 |
+
definition = user_tables.get_definition(source, st=runtime)
|
| 688 |
+
fields = definition.get("fields") if isinstance(definition, dict) else None
|
| 689 |
+
declared = perm_scope.entry(user, source) or {}
|
| 690 |
+
walled = bool(perm_scope.wall_declared(user, source, st=runtime, defn=definition))
|
| 691 |
+
rows = bool(declared.get("filter")) or bool(
|
| 692 |
+
fields and perm_scope.row_scope_applies(user, source, st=runtime, defn=definition))
|
| 693 |
+
hidden = set(declared.get("hiddenFields") or ()) | set(
|
| 694 |
+
perm_scope.hidden_keys(user, source, fields, st=runtime) if fields else ())
|
| 695 |
+
narrow[source] = {"rows": rows or (walled and not hidden), "hidden": tuple(sorted(map(str, hidden)))}
|
| 696 |
+
# The Dashboard totals the raw source, so it keeps the source grant and refuses any narrowing.
|
| 697 |
+
if (enabled != "all" and source not in (enabled or ())) or not perm_scope.may_read(user, source, st=runtime):
|
| 698 |
+
refuse(403, "app_dashboard_source", f"The Dashboard needs read access to {label}.")
|
| 699 |
+
elif not fields:
|
| 700 |
+
refuse(503, "app_dashboard_source", f"The Dashboard cannot read {label} right now.")
|
| 701 |
+
elif (declared.get("access") is False or walled or rows or hidden
|
| 702 |
+
or route_system.is_managed(definition) or route_system.is_managed(source)):
|
| 703 |
+
refuse(403, "app_dashboard_scope", f"The Dashboard needs {label} without a row or field limit.")
|
| 704 |
+
narrowed = any(wall["rows"] or wall["hidden"] for wall in narrow.values())
|
| 705 |
+
status, code, message = refusal[0] if refusal else (None, None, None)
|
| 706 |
+
return {"app": "ready" if granted and (kind in NARROWING_KINDS or not narrowed) else "no_grant",
|
| 707 |
+
"dashboard": "ready" if not refusal else "refused",
|
| 708 |
+
"status": status, "code": code, "message": message, "narrow": narrow}
|
| 709 |
+
|
| 710 |
+
|
| 711 |
+
def app_source_scope(user, runtime, source, *, app, records=None):
|
| 712 |
+
"""Return source fields and row identities after live admission of the named App."""
|
| 713 |
from core import perm_scope
|
| 714 |
+
# Importing the API adapter registers the core admission callback.
|
| 715 |
+
import routes_app_dashboard
|
| 716 |
+
fields, rows = perm_scope.scoped_app_source(
|
| 717 |
+
user, source, st=runtime, app_id=app.get("id") if isinstance(app, dict) else None, records=records)
|
| 718 |
+
identity = "__scope_key" if records is not None else "pid"
|
| 719 |
+
return {str(field["key"]) for field in fields if field.get("key")}, {
|
| 720 |
+
str(row[identity]) for row in rows if identity in row}
|
| 721 |
+
|
| 722 |
+
|
| 723 |
+
def readable_app(runtime, tenant, uname, is_admin, kind):
|
| 724 |
+
"""The caller's live readable App of `kind` in `tenant`, their own copy first, or None."""
|
| 725 |
+
apps = [row for _key, row in sorted(load(runtime)["applications"].items())
|
| 726 |
+
if isinstance(row, dict) and row.get("templateKind") == kind and row.get("tenant") == tenant
|
| 727 |
+
and row.get("retired") is not True and may_read(row, uname, is_admin)]
|
| 728 |
+
return next((row for row in apps if row.get("createdBy") == uname), apps[0] if apps else None)
|
| 729 |
+
|
| 730 |
+
|
| 731 |
+
def admit_app(session, app, sources, *, code):
|
| 732 |
+
"""R62: admit an App read on the App grant plus `app_access`, never the source open grant."""
|
| 733 |
+
from deps import err
|
| 734 |
+
if (session.tenant != getattr(session.runtime, "key", None)
|
| 735 |
+
or session.user.get("tenant", session.tenant) != session.tenant
|
| 736 |
+
or not isinstance(app, dict) or app.get("tenant") != session.tenant
|
| 737 |
+
or app.get("retired") is True or not may_read(app, session.uname, session.admin)):
|
| 738 |
+
raise err(404, f"{code}_app_not_found", "This App is unavailable or is not shared with you.")
|
| 739 |
+
access = app_access(session.user, session.runtime, app, sources)
|
| 740 |
+
if access["app"] != "ready":
|
| 741 |
+
raise err(403, f"{code}_limited", "This App cannot apply your access limit on its source.")
|
| 742 |
+
return access
|
| 743 |
+
|
| 744 |
+
|
| 745 |
+
def _availability_for_app(session, app, catalogue):
|
| 746 |
from routes_connectors import _entry_types
|
| 747 |
|
| 748 |
kind = app["templateKind"]
|
|
|
|
| 753 |
return mount
|
| 754 |
if not _app_templates.seams_ready(kind):
|
| 755 |
return availability_reason("router_unavailable")
|
| 756 |
+
amazon = catalogue.get("connector") in ("amazon", "amazon_ads")
|
| 757 |
+
dashboard = {"state": "source_unavailable", "message": _AVAILABILITY_MESSAGES["source_unavailable"]}
|
| 758 |
+
|
| 759 |
+
def reason(state, connector=None):
|
| 760 |
+
result = availability_reason(state, connector)
|
| 761 |
+
return {**result, "dashboard": dashboard} if amazon else result
|
| 762 |
+
|
| 763 |
try:
|
| 764 |
sources = catalogue["sources"]
|
| 765 |
+
if amazon:
|
| 766 |
+
# Availability and the Dashboard use one source-wall answer.
|
| 767 |
+
access = app_access(session.user, session.runtime, app, sources)
|
| 768 |
+
dashboard = {"state": "ready" if access["dashboard"] == "ready" else
|
| 769 |
+
"no_grant" if access["status"] == 403 else "source_unavailable",
|
| 770 |
+
"message": access["message"]}
|
| 771 |
+
if access["app"] != "ready":
|
| 772 |
+
return reason("no_grant")
|
| 773 |
+
else:
|
| 774 |
+
from core import perm_scope
|
| 775 |
+
for source in sources:
|
| 776 |
+
if (not perm_scope.may_read(session.user, source, st=session.runtime)
|
| 777 |
+
or perm_scope.wall_declared(session.user, source, st=session.runtime)):
|
| 778 |
+
return reason("no_grant")
|
| 779 |
types, _scopes, failure = _entry_types(session.runtime, session.uname)
|
| 780 |
if failure is not None:
|
| 781 |
+
return reason("source_unavailable")
|
| 782 |
connected = _app_templates.entry_type(catalogue["connector"]) in set(types or ())
|
| 783 |
if not connected and catalogue["connector"] == "odoo":
|
| 784 |
from routes_keychain import env_odoo_available
|
| 785 |
connected = env_odoo_available(session.runtime)
|
| 786 |
if not connected:
|
| 787 |
+
return reason("not_connected", catalogue["connector"])
|
| 788 |
+
row_sources = ((*sources, *catalogue.get("optional_sources", ()))
|
| 789 |
+
if kind == "repeat_buyers" else sources)
|
| 790 |
+
if amazon:
|
| 791 |
+
from harness.mirror_store import restore_in_progress
|
| 792 |
+
if restore_in_progress(session.tenant, catalogue["connector"]):
|
| 793 |
+
return reason("refreshing")
|
| 794 |
+
if row_sources and not _source_rows_exist(session, row_sources):
|
| 795 |
+
return reason("no_rows")
|
| 796 |
+
except Exception as exc: # noqa: BLE001
|
| 797 |
+
if amazon:
|
| 798 |
+
raise_if_resource_error(exc)
|
| 799 |
+
return reason("source_unavailable")
|
| 800 |
+
return reason("ready")
|
| 801 |
|
| 802 |
|
| 803 |
def availability(session, app_id):
|
|
|
|
| 1312 |
mount = _mount_reason(row.get("templateKind"))
|
| 1313 |
if mount:
|
| 1314 |
row["availability"] = mount
|
| 1315 |
+
elif (catalogue.get("connector") not in ("amazon", "amazon_ads")
|
| 1316 |
+
and not _database_free(app) and not compatible):
|
| 1317 |
row["availability"] = availability_reason("no_grant")
|
| 1318 |
if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
|
| 1319 |
# A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
|
|
|
|
| 1778 |
return {"outcome": "migrated", "itemIds": copy.deepcopy(copied["itemIds"])}
|
| 1779 |
|
| 1780 |
|
| 1781 |
+
#: Marker of the one-copy step: each removed later copy and the kept copy that holds its slot now.
|
| 1782 |
+
TEMPLATE_ONE_COPY_MARKER = "templateOneCopy"
|
| 1783 |
+
#: The one-copy step deletes as the account Loopable seeds Apps under.
|
| 1784 |
+
_ONE_COPY_ACTOR = "loopable"
|
| 1785 |
+
|
| 1786 |
+
|
| 1787 |
+
def _template_double_plan(documents):
|
| 1788 |
+
"""`[(identity, (key, kept), [(key, later), ...])]` per tenant-wide template with two live copies.
|
| 1789 |
+
|
| 1790 |
+
Copies sort as `_refuse_template_held` sorts them. Routing is one private copy per account by design.
|
| 1791 |
+
"""
|
| 1792 |
+
groups = {}
|
| 1793 |
+
for key, doc in documents.items():
|
| 1794 |
+
for app_id, app in doc["applications"].items():
|
| 1795 |
+
if not isinstance(app, dict) or app.get("retired") is True:
|
| 1796 |
+
continue
|
| 1797 |
+
identity = template_identity(app)
|
| 1798 |
+
if identity and identity != "route":
|
| 1799 |
+
groups.setdefault(identity, {}).setdefault(app_id, (key, app))
|
| 1800 |
+
plan = []
|
| 1801 |
+
for identity, copies in sorted(groups.items()):
|
| 1802 |
+
rows = sorted(copies.values(), key=lambda row: (str(row[1].get("createdAt") or ""),
|
| 1803 |
+
str(row[1].get("id") or "")))
|
| 1804 |
+
if len(rows) > 1:
|
| 1805 |
+
plan.append((identity, rows[0], rows[1:]))
|
| 1806 |
+
return plan
|
| 1807 |
+
|
| 1808 |
+
|
| 1809 |
+
def _fold_template_copy(runtime, kept_key, kept_id, later):
|
| 1810 |
+
"""Give the kept copy the later copy's readers and stars, and repoint its defaults."""
|
| 1811 |
+
def fold(raw):
|
| 1812 |
+
doc = _clean_doc(raw)
|
| 1813 |
+
app = doc["applications"].get(kept_id)
|
| 1814 |
+
if not isinstance(app, dict) or app.get("retired") is True:
|
| 1815 |
+
raise ApplicationError("The kept App copy is unavailable.")
|
| 1816 |
+
before = copy.deepcopy(app)
|
| 1817 |
+
share = app.get("share") if isinstance(app.get("share"), dict) else {}
|
| 1818 |
+
other = later.get("share") if isinstance(later.get("share"), dict) else {}
|
| 1819 |
+
if share.get("mode") != "tenant":
|
| 1820 |
+
if other.get("mode") == "tenant":
|
| 1821 |
+
app["share"] = {"mode": "tenant", "users": []}
|
| 1822 |
+
else:
|
| 1823 |
+
users = [user for user in _users([
|
| 1824 |
+
*(share.get("users") or () if share.get("mode") == "users" else ()),
|
| 1825 |
+
*(other.get("users") or () if other.get("mode") == "users" else ()),
|
| 1826 |
+
later.get("createdBy")]) if user != app.get("createdBy")]
|
| 1827 |
+
if users:
|
| 1828 |
+
app["share"] = {"mode": "users", "users": users}
|
| 1829 |
+
stars = {*(app.get("starredBy") or ()), *(later.get("starredBy") or ())}
|
| 1830 |
+
if stars != set(app.get("starredBy") or ()):
|
| 1831 |
+
app["starredBy"] = sorted(stars)
|
| 1832 |
+
if app != before:
|
| 1833 |
+
app["revision"] = int(app.get("revision") or 0) + 1
|
| 1834 |
+
app["updatedAt"] = write_execution.utcnow().isoformat()
|
| 1835 |
+
return doc
|
| 1836 |
+
return raw
|
| 1837 |
+
_update_document(runtime, fold, key=kept_key)
|
| 1838 |
+
|
| 1839 |
+
def repoint(raw):
|
| 1840 |
+
doc = _clean_doc(raw)
|
| 1841 |
+
names = [name for name, value in doc["defaults"].items() if value == later.get("id")]
|
| 1842 |
+
if not names:
|
| 1843 |
+
return raw
|
| 1844 |
+
for name in names:
|
| 1845 |
+
doc["defaults"][name] = kept_id
|
| 1846 |
+
return doc
|
| 1847 |
+
if later.get("id") in _clean_doc(_read_strict(runtime, _KEY))["defaults"].values():
|
| 1848 |
+
_update_document(runtime, repoint)
|
| 1849 |
+
index = _read_strict(runtime, _APP_INDEX_KEY)
|
| 1850 |
+
owners = index.get("users") if isinstance(index, dict) else None
|
| 1851 |
+
if isinstance(owners, dict):
|
| 1852 |
+
_move_placements(runtime, {name: {later["id"]: kept_id} for name in owners})
|
| 1853 |
+
|
| 1854 |
+
|
| 1855 |
+
def remove_template_doubles(runtime, tenant):
|
| 1856 |
+
"""Keep the oldest live copy of each tenant-wide template and delete every later copy.
|
| 1857 |
+
|
| 1858 |
+
`delete` keeps the later copy's databases, Views and records. A second run writes nothing.
|
| 1859 |
+
Never raises: a failed copy is reported and retried at the next boot.
|
| 1860 |
+
"""
|
| 1861 |
+
if (getattr(runtime, "key", None) != tenant
|
| 1862 |
+
or getattr(getattr(runtime, "tenant", None), "key", tenant) != tenant):
|
| 1863 |
+
return {"outcome": "skipped", "removed": {}, "reason": "the runtime belongs to another tenant"}
|
| 1864 |
+
try:
|
| 1865 |
+
documents = {_KEY: _clean_doc(_read_strict(runtime, _KEY))}
|
| 1866 |
+
documents.update(_record_documents(runtime))
|
| 1867 |
+
except Exception as exc:
|
| 1868 |
+
return {"outcome": "skipped", "removed": {},
|
| 1869 |
+
"reason": f"registry read failed: {type(exc).__name__}: {exc}"}
|
| 1870 |
+
removed, failed = {}, []
|
| 1871 |
+
for identity, (kept_key, kept), later in _template_double_plan(documents):
|
| 1872 |
+
for _key, app in later:
|
| 1873 |
+
try:
|
| 1874 |
+
_fold_template_copy(runtime, kept_key, kept["id"], app)
|
| 1875 |
+
delete(runtime, app["id"], _ONE_COPY_ACTOR, True)
|
| 1876 |
+
except Exception as exc:
|
| 1877 |
+
failed.append(f"{app['id']}: {type(exc).__name__}")
|
| 1878 |
+
print(f"[aios-api] one template copy {tenant}: {app['id']} not removed: "
|
| 1879 |
+
f"{type(exc).__name__}")
|
| 1880 |
+
continue
|
| 1881 |
+
removed[app["id"]] = kept["id"]
|
| 1882 |
+
print(f"[aios-api] one template copy {tenant}: removed {app['id']}, "
|
| 1883 |
+
f"kept {kept['id']} ({identity})")
|
| 1884 |
+
if removed:
|
| 1885 |
+
def mark(raw):
|
| 1886 |
+
doc = _clean_doc(raw)
|
| 1887 |
+
migrations = doc.setdefault("migrations", {})
|
| 1888 |
+
marker = migrations.get(TEMPLATE_ONE_COPY_MARKER)
|
| 1889 |
+
marker = copy.deepcopy(marker) if isinstance(marker, dict) else {}
|
| 1890 |
+
held = marker.get("removed") if isinstance(marker.get("removed"), dict) else {}
|
| 1891 |
+
marker.update(removed={**held, **removed}, at=write_execution.utcnow().isoformat())
|
| 1892 |
+
migrations[TEMPLATE_ONE_COPY_MARKER] = marker
|
| 1893 |
+
return doc
|
| 1894 |
+
try:
|
| 1895 |
+
_update_document(runtime, mark)
|
| 1896 |
+
except Exception as exc:
|
| 1897 |
+
failed.append(f"marker: {type(exc).__name__}")
|
| 1898 |
+
outcome = "removed" if removed else ("failed" if failed else "unchanged")
|
| 1899 |
+
reason = ", ".join(f"{later_id} into {kept_id}" for later_id, kept_id in removed.items())
|
| 1900 |
+
return {"outcome": outcome, "removed": removed,
|
| 1901 |
+
"reason": "; ".join(part for part in (reason, ", ".join(failed)) if part)}
|
| 1902 |
+
|
| 1903 |
+
|
| 1904 |
def ensure_route_application(runtime, tenant, *, accounts=None, wall=None):
|
| 1905 |
"""R15: boot migrations, once per tenant, never from an Application read.
|
| 1906 |
|
api/assistant_memory.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
"""C13: a user's sourced facts, separate from the C12 conversation transcript."""
|
|
|
|
| 2 |
import copy
|
| 3 |
import datetime
|
| 4 |
import json
|
|
@@ -493,6 +494,74 @@ def recall(rt, principal, query=None, limit=20):
|
|
| 493 |
return result
|
| 494 |
|
| 495 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 496 |
#: One compaction reads and sends at most one log chunk. A tail that grew while compaction could
|
| 497 |
#: not run (no model, no credit) is worked through in order, one bounded batch per attempt, so
|
| 498 |
#: neither the store read nor the model prompt grows with the conversation (D-803).
|
|
|
|
| 1 |
"""C13: a user's sourced facts, separate from the C12 conversation transcript."""
|
| 2 |
+
import base64
|
| 3 |
import copy
|
| 4 |
import datetime
|
| 5 |
import json
|
|
|
|
| 494 |
return result
|
| 495 |
|
| 496 |
|
| 497 |
+
INSPECT_ORDERS = ("newest", "oldest")
|
| 498 |
+
|
| 499 |
+
|
| 500 |
+
class InspectRequestError(ValueError):
|
| 501 |
+
"""A page request the route answers with 400, raised before any store read."""
|
| 502 |
+
|
| 503 |
+
|
| 504 |
+
def _encode_cursor(key, id):
|
| 505 |
+
stamp, company, position = key
|
| 506 |
+
raw = json.dumps([None if stamp == float("-inf") else stamp, company, position, id], separators=(",", ":"))
|
| 507 |
+
return base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=")
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
def _decode_cursor(cursor):
|
| 511 |
+
try:
|
| 512 |
+
stamp, company, position, id = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)))
|
| 513 |
+
if ((stamp is not None and type(stamp) not in (int, float)) or company not in (0, 1)
|
| 514 |
+
or type(position) is not int or not isinstance(id, str)):
|
| 515 |
+
raise TypeError
|
| 516 |
+
return (float("-inf") if stamp is None else float(stamp), company, position), id
|
| 517 |
+
except Exception:
|
| 518 |
+
raise InspectRequestError("That memory page is out of date. Reload the list.") from None
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
def inspect_page(rt, principal, *, cursor=None, limit=MAX_ENTRIES, order="newest"):
|
| 522 |
+
"""Page the whole authorized memory for the inspect and erase panel. Turn context keeps recall."""
|
| 523 |
+
if order not in INSPECT_ORDERS:
|
| 524 |
+
raise InspectRequestError("Choose newest or oldest first.")
|
| 525 |
+
after = _decode_cursor(cursor) if cursor else None
|
| 526 |
+
limit = max(1, min(MAX_ENTRIES, int(limit)))
|
| 527 |
+
if principal["tenant"] != assistant_log._tenant_of(rt):
|
| 528 |
+
return {"entries": [], "total": 0, "next": None}
|
| 529 |
+
doc = document(rt, principal["username"])
|
| 530 |
+
company = company_document(rt)
|
| 531 |
+
entries = [(entry, "personal", doc["tombstones"], 0, n) for n, entry in enumerate(doc["entries"])]
|
| 532 |
+
entries.extend((entry, "company", company["tombstones"], 1, n) for n, entry in enumerate(company["entries"]))
|
| 533 |
+
# Same walls as recall: the prevailing note is chosen before the source filter.
|
| 534 |
+
superseded = _superseded(company["entries"], company["tombstones"])
|
| 535 |
+
readable, rows = {}, []
|
| 536 |
+
for entry, scope, tombstones, rank, position in entries:
|
| 537 |
+
if (not isinstance(entry, dict) or not all(key in entry for key in ("id", "text", "sources", "door", "at"))
|
| 538 |
+
or not isinstance(entry["text"], str) or not isinstance(entry["id"], str)):
|
| 539 |
+
continue
|
| 540 |
+
if scope == "company" and id(entry) in superseded:
|
| 541 |
+
continue
|
| 542 |
+
if _normalized(entry["text"]) in tombstones:
|
| 543 |
+
continue
|
| 544 |
+
# Permissions are fixed for one request, so one answer per source set is enough.
|
| 545 |
+
wall = (scope, json.dumps(entry["sources"], sort_keys=True, default=str))
|
| 546 |
+
if wall not in readable:
|
| 547 |
+
readable[wall] = _readable(rt, principal, entry["sources"], company=scope == "company")
|
| 548 |
+
if readable[wall]:
|
| 549 |
+
# Age, then document position: the order recall uses, so equal stamps never reorder.
|
| 550 |
+
rows.append(((_oldest(entry), rank, position), entry, scope))
|
| 551 |
+
newest = order == "newest"
|
| 552 |
+
rows.sort(key=lambda row: row[0], reverse=newest)
|
| 553 |
+
start = 0
|
| 554 |
+
if after is not None:
|
| 555 |
+
key, anchor = after
|
| 556 |
+
# Resume after the anchor id. If it was erased, resume after its sort key.
|
| 557 |
+
start = next((n + 1 for n, row in enumerate(rows) if row[1]["id"] == anchor and row[0][1] == key[1]),
|
| 558 |
+
next((n for n, row in enumerate(rows) if (row[0] < key if newest else row[0] > key)), len(rows)))
|
| 559 |
+
page = rows[start:start + limit]
|
| 560 |
+
more = start + limit < len(rows)
|
| 561 |
+
return {"entries": [{**copy.deepcopy(entry), "scope": scope} for _, entry, scope in page],
|
| 562 |
+
"total": len(rows), "next": _encode_cursor(page[-1][0], page[-1][1]["id"]) if more else None}
|
| 563 |
+
|
| 564 |
+
|
| 565 |
#: One compaction reads and sends at most one log chunk. A tail that grew while compaction could
|
| 566 |
#: not run (no model, no credit) is worked through in order, one bounded batch per attempt, so
|
| 567 |
#: neither the store read nor the model prompt grows with the conversation (D-803).
|
api/assistant_mirror.py
CHANGED
|
@@ -58,6 +58,12 @@ def _rot():
|
|
| 58 |
return routes_odoo_tables
|
| 59 |
|
| 60 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
def open_cursor(session, table_key=None):
|
| 62 |
"""`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
|
| 63 |
|
|
@@ -77,17 +83,21 @@ def open_cursor(session, table_key=None):
|
|
| 77 |
try:
|
| 78 |
cur, mismatch = rot._mirror_cur(session, table_key)
|
| 79 |
except RuntimeError as exc:
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
except Exception as exc: # noqa: BLE001
|
| 82 |
# ⚠ A DELIBERATE THIRD CLAUSE, and it is not laziness. This module is reached from
|
| 83 |
# `deps`, which every fixture and gate in the repo imports and drives with runtimes that
|
| 84 |
# are not a real `TenantRuntime`; `mirror_cursor` calls `assert_datastore_matches` on
|
| 85 |
# whatever it is handed. An unexpected failure there is a 500 on an assistant answer,
|
| 86 |
# which reads to a user as the product being broken rather than the mirror being shut.
|
| 87 |
-
# Named and fail-closed beats a stack trace.
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
if mismatch:
|
| 92 |
return None, MirrorRefusal(409, "assistant_cross_tenant_store", str(mismatch))
|
| 93 |
return cur, None
|
|
@@ -167,17 +177,14 @@ def unavailable_reason(table_key, definition=None, *, snapshot=None):
|
|
| 167 |
if key.startswith("ut_amzn_"):
|
| 168 |
import amazon_relational as amazon
|
| 169 |
if key in amazon.SOURCE:
|
| 170 |
-
if snapshot is not None
|
| 171 |
-
|
| 172 |
-
return amazon.missing_source(key)["missing"]
|
| 173 |
spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key)
|
| 174 |
if spec and not spec.get("missing"):
|
| 175 |
-
|
| 176 |
-
return (f"{label} is still loading into this server after a restart, "
|
| 177 |
-
"so ask again in an hour.")
|
| 178 |
except Exception: # noqa: BLE001
|
| 179 |
pass
|
| 180 |
-
return
|
| 181 |
|
| 182 |
|
| 183 |
def still_loading(table_key, *, cur=None, snapshot=None):
|
|
|
|
| 58 |
return routes_odoo_tables
|
| 59 |
|
| 60 |
|
| 61 |
+
def _cause(cause):
|
| 62 |
+
"""The closed-set refusal sentence `deps` owns, imported lazily to break the `deps` cycle."""
|
| 63 |
+
import deps
|
| 64 |
+
return deps._assistant_source_message(cause)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
def open_cursor(session, table_key=None):
|
| 68 |
"""`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
|
| 69 |
|
|
|
|
| 83 |
try:
|
| 84 |
cur, mismatch = rot._mirror_cur(session, table_key)
|
| 85 |
except RuntimeError as exc:
|
| 86 |
+
if "could not be restored" in str(exc):
|
| 87 |
+
return None, MirrorRefusal(409, "assistant_snapshot_unavailable", _cause("restore_blocked"))
|
| 88 |
+
if "has not been attached" in str(exc):
|
| 89 |
+
return None, MirrorRefusal(503, "assistant_store_not_ready", _cause("loading"))
|
| 90 |
+
return None, MirrorRefusal(503, "assistant_store_not_ready", _cause("syncing"))
|
| 91 |
except Exception as exc: # noqa: BLE001
|
| 92 |
# ⚠ A DELIBERATE THIRD CLAUSE, and it is not laziness. This module is reached from
|
| 93 |
# `deps`, which every fixture and gate in the repo imports and drives with runtimes that
|
| 94 |
# are not a real `TenantRuntime`; `mirror_cursor` calls `assert_datastore_matches` on
|
| 95 |
# whatever it is handed. An unexpected failure there is a 500 on an assistant answer,
|
| 96 |
# which reads to a user as the product being broken rather than the mirror being shut.
|
| 97 |
+
# Named and fail-closed beats a stack trace. The class goes to the server log, never the reply.
|
| 98 |
+
import deps
|
| 99 |
+
deps._assistant_log_cause("assistant mirror open", exc)
|
| 100 |
+
return None, MirrorRefusal(409, "assistant_snapshot_unavailable", _cause("mirror_open"))
|
| 101 |
if mismatch:
|
| 102 |
return None, MirrorRefusal(409, "assistant_cross_tenant_store", str(mismatch))
|
| 103 |
return cur, None
|
|
|
|
| 177 |
if key.startswith("ut_amzn_"):
|
| 178 |
import amazon_relational as amazon
|
| 179 |
if key in amazon.SOURCE:
|
| 180 |
+
pending = snapshot.amazon_pending() if snapshot is not None else None
|
| 181 |
+
return amazon.missing_source(key, pending=pending)["missing"]
|
|
|
|
| 182 |
spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key)
|
| 183 |
if spec and not spec.get("missing"):
|
| 184 |
+
return _cause("loading")
|
|
|
|
|
|
|
| 185 |
except Exception: # noqa: BLE001
|
| 186 |
pass
|
| 187 |
+
return _cause("no_binding")
|
| 188 |
|
| 189 |
|
| 190 |
def still_loading(table_key, *, cur=None, snapshot=None):
|
api/automation_engine.py
CHANGED
|
@@ -6804,14 +6804,22 @@ def migrate_ig_tables(rt, log=print, only=""):
|
|
| 6804 |
rows, merged = _merge_candidates(rows)
|
| 6805 |
stats["merged"] += merged
|
| 6806 |
|
|
|
|
|
|
|
| 6807 |
def _up(cur, _f=fields, _r=rows):
|
|
|
|
| 6808 |
cur = cur or {}
|
| 6809 |
t = dict(cur.get(table_key) or {})
|
|
|
|
| 6810 |
t["fields"], t["rows"] = _f, _r
|
|
|
|
|
|
|
|
|
|
| 6811 |
cur[table_key] = t
|
| 6812 |
return cur
|
| 6813 |
|
| 6814 |
rt.update(UT_STORE_KEY, _up)
|
|
|
|
| 6815 |
# ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION.
|
| 6816 |
#
|
| 6817 |
# W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early
|
|
@@ -14211,6 +14219,10 @@ def _module_table(rt, key, memo, want_rows=False):
|
|
| 14211 |
"check this tenant's Odoo connector under Settings -> Connectors; the linked column "
|
| 14212 |
"keeps its last resolved value until the source answers again")
|
| 14213 |
table["rows"] = {str(r.get("pid")): r for r in (pool or []) if r.get("pid") is not None}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14214 |
for rid, row in list(table['rows'].items()):
|
| 14215 |
table['rows'][rid] = {**row, **(table.get('_if_shared_cells', {}).get(rid) or {})}
|
| 14216 |
table["_rows_built"] = True
|
|
@@ -14360,24 +14372,52 @@ def _resolve_table(rt, store, key, memo, want_rows=False):
|
|
| 14360 |
return _module_table(rt, key, memo, want_rows=want_rows)
|
| 14361 |
|
| 14362 |
|
| 14363 |
-
def _link_from_key(fields, bag):
|
| 14364 |
"""Which column on THIS table supplies the join value.
|
| 14365 |
|
| 14366 |
Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That
|
| 14367 |
fallback chain is what makes an Instagram database link up with no configuration at all —
|
| 14368 |
the "automatically" in the owner's instruction — and it is the same chain the grid already
|
| 14369 |
uses to decide a table's identity column, rather than a second opinion about it.
|
|
|
|
|
|
|
| 14370 |
"""
|
| 14371 |
declared = str((bag or {}).get("from") or "").strip()
|
| 14372 |
if declared:
|
| 14373 |
return declared
|
| 14374 |
-
|
|
|
|
| 14375 |
if prof:
|
| 14376 |
return str(prof.get("key") or "")
|
| 14377 |
-
pin = next((f for f in
|
| 14378 |
return str(pin.get("key") or "") if pin else ""
|
| 14379 |
|
| 14380 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14381 |
def _join_norm(value):
|
| 14382 |
"""ONE normalisation of an `on` join value, applied to BOTH sides of the join.
|
| 14383 |
|
|
@@ -14564,6 +14604,18 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14564 |
# --- resolve every link field ONCE per table, not once per row.
|
| 14565 |
# `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]`
|
| 14566 |
resolved, linked_types, refused = {}, {}, set()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14567 |
for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]:
|
| 14568 |
lk_key = str((f or {}).get("key") or "")
|
| 14569 |
if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict):
|
|
@@ -14630,7 +14682,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14630 |
on_key = str(bag.get("on") or "")
|
| 14631 |
hop_in = str(bag.get("throughOn") or "")
|
| 14632 |
hop_out = str(bag.get("throughTo") or "")
|
| 14633 |
-
from_key =
|
| 14634 |
if not (on_key and hop_in and hop_out and from_key):
|
| 14635 |
# ⛔ REFUSED, NOT BLANK — the same law the one-hop arm below follows. A route with
|
| 14636 |
# a missing leg has no second hop to make; resolving it to `[]` would render a
|
|
@@ -14668,7 +14720,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14668 |
target_idx = _linked_rows_by_join(linked, on_key)
|
| 14669 |
cap = int(_ut().LINK_THROUGH_MAX_IDS)
|
| 14670 |
|
| 14671 |
-
def _walk(row, _bidx=bridge_idx, _tidx=target_idx, _cap=cap):
|
| 14672 |
"""One source row's DISTINCT target records, and whether the ceiling stopped it.
|
| 14673 |
|
| 14674 |
⚠ DE-DUPED ON THE TARGET ROW ID. A customer orders the same SKU on twenty lines;
|
|
@@ -14676,7 +14728,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14676 |
label of a set of PRODUCTS.
|
| 14677 |
"""
|
| 14678 |
seen, out = set(), []
|
| 14679 |
-
for _bid, brow in _bidx.get(_join_norm((row
|
| 14680 |
for tid, trow in _tidx.get(_join_norm((brow or {}).get(hop_out)), ()):
|
| 14681 |
if tid in seen:
|
| 14682 |
continue
|
|
@@ -14688,7 +14740,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14688 |
|
| 14689 |
two_hop, capped = {}, 0
|
| 14690 |
for rid, row in rows.items():
|
| 14691 |
-
hits_2, hit_cap = _walk(row)
|
| 14692 |
two_hop[str(rid)] = hits_2
|
| 14693 |
capped += 1 if hit_cap else 0
|
| 14694 |
resolved[lk_key] = two_hop
|
|
@@ -14712,7 +14764,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14712 |
effect="capped", limit=cap))
|
| 14713 |
elif bag.get("on"):
|
| 14714 |
idx = _linked_rows_by_join(linked, str(bag["on"]))
|
| 14715 |
-
from_key =
|
| 14716 |
if not from_key:
|
| 14717 |
# ⛔ REFUSED, NOT BLANK. `_link_from_key` falls back to the PROFILE-flagged column
|
| 14718 |
# and then the PINNED one; a table with neither — which a registry module can be,
|
|
@@ -14728,7 +14780,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14728 |
"database that is the id column, e.g. `partner_id` or `product_id`)"))
|
| 14729 |
continue
|
| 14730 |
resolved[lk_key] = {
|
| 14731 |
-
str(rid): idx.get(_join_norm((row
|
| 14732 |
for rid, row in rows.items()}
|
| 14733 |
else:
|
| 14734 |
# An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds
|
|
@@ -15024,6 +15076,136 @@ def related_product_metric_values(rt, customer_id, products, fields, offer, *,
|
|
| 15024 |
return out, list(dict.fromkeys(reasons))
|
| 15025 |
|
| 15026 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15027 |
def compute_relation_cells(rt, table_key, tables=None):
|
| 15028 |
"""Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
|
| 15029 |
|
|
@@ -15050,7 +15232,10 @@ def compute_relation_cells(rt, table_key, tables=None):
|
|
| 15050 |
another section. ⇒ Refusals ride out on `relation_cells`' SECOND RETURN VALUE, where the
|
| 15051 |
caller that renders them reads them, and nothing about this door changes shape.
|
| 15052 |
"""
|
| 15053 |
-
changes,
|
|
|
|
|
|
|
|
|
|
| 15054 |
if not changes:
|
| 15055 |
return 0
|
| 15056 |
if str(table_key) in LINKABLE_MODULES:
|
|
@@ -15093,24 +15278,36 @@ def _refresh_relations_inplace(tables, log=print, rt=None):
|
|
| 15093 |
keyword this function briefly added (see `compute_relation_cells`).
|
| 15094 |
"""
|
| 15095 |
touched = 0
|
|
|
|
| 15096 |
for tk, table in list((tables or {}).items()):
|
| 15097 |
fields = (table or {}).get("fields") or []
|
| 15098 |
if not any(f.get("type") in ('rollup', 'if') or _ut().is_derived_link(f) for f in fields):
|
|
|
|
|
|
|
| 15099 |
continue
|
|
|
|
| 15100 |
try:
|
| 15101 |
touched += compute_relation_cells(rt, tk, tables=tables)
|
|
|
|
| 15102 |
except Exception as e: # noqa: BLE001
|
| 15103 |
log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}")
|
|
|
|
|
|
|
|
|
|
| 15104 |
return touched
|
| 15105 |
|
| 15106 |
|
| 15107 |
def refresh_relations(rt, log=print, tables=None, persist=True):
|
| 15108 |
"""Compute relations outside the write lock and merge only unchanged derived cells."""
|
| 15109 |
-
snapshot = tables if tables is not None else
|
| 15110 |
# Keep authored inputs before computation, including nested cell values.
|
| 15111 |
before = copy.deepcopy(snapshot)
|
| 15112 |
local = _refresh_relations_inplace(snapshot, log=log, rt=rt)
|
| 15113 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15114 |
return local
|
| 15115 |
patches = {}
|
| 15116 |
for key, old in before.items():
|
|
@@ -15129,31 +15326,47 @@ def refresh_relations(rt, log=print, tables=None, persist=True):
|
|
| 15129 |
if changed:
|
| 15130 |
patches.setdefault(str(key), {})[str(rid)] = (row, changed)
|
| 15131 |
if not patches:
|
|
|
|
| 15132 |
return 0
|
| 15133 |
actual = [0]
|
|
|
|
|
|
|
|
|
|
| 15134 |
|
| 15135 |
def _up(cur):
|
| 15136 |
actual[0] = 0
|
|
|
|
| 15137 |
lock = getattr(cur, 'lock_records', None)
|
| 15138 |
if callable(lock):
|
| 15139 |
lock({key: list(rows) for key, rows in patches.items()})
|
| 15140 |
for key, rows in patches.items():
|
| 15141 |
table = (cur or {}).get(key)
|
| 15142 |
if not isinstance(table, dict) or table.get('fields') != before[key].get('fields'):
|
|
|
|
| 15143 |
continue
|
|
|
|
| 15144 |
current_rows = table.get('rows')
|
| 15145 |
if current_rows is None:
|
|
|
|
| 15146 |
continue
|
| 15147 |
for rid, (old_row, changes) in rows.items():
|
| 15148 |
current = current_rows.get(rid)
|
| 15149 |
# A changed authored input or a deleted row waits for the next refresh.
|
| 15150 |
if not isinstance(current, dict) or current != old_row:
|
|
|
|
| 15151 |
continue
|
| 15152 |
current.update(changes)
|
| 15153 |
actual[0] += 1
|
|
|
|
|
|
|
|
|
|
| 15154 |
return cur
|
| 15155 |
|
| 15156 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15157 |
return actual[0]
|
| 15158 |
|
| 15159 |
|
|
@@ -16256,13 +16469,153 @@ def _automation_slack_reply_text(result):
|
|
| 16256 |
return "An authorized automation completed."
|
| 16257 |
|
| 16258 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16259 |
def dispatch_authorized_slack_automation(rt, defn, result=None, *, post=None):
|
| 16260 |
"""Deliver one authorized event result, with readable linked records for an enters_view cohort."""
|
| 16261 |
definition = defn if isinstance(defn, dict) else {}
|
| 16262 |
aid = str(definition.get("id") or "").strip()
|
| 16263 |
if not aid:
|
| 16264 |
return {"sent": False, "reason": "unknown_automation"}
|
| 16265 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16266 |
if not isinstance(stored, dict):
|
| 16267 |
return {"sent": False, "reason": "unknown_automation"}
|
| 16268 |
trigger = stored.get("trigger") if isinstance(stored.get("trigger"), dict) else {}
|
|
@@ -16286,6 +16639,22 @@ def dispatch_authorized_slack_automation(rt, defn, result=None, *, post=None):
|
|
| 16286 |
context = routes_slack.automation_answer_context(rt, aid)
|
| 16287 |
if context is None:
|
| 16288 |
return {"sent": False, "reason": "not_authorized"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16289 |
provenance = {"basis": "model_guidance", "citationIds": [], "modelGuidance": False}
|
| 16290 |
if trigger_key == "enters_view":
|
| 16291 |
module = str(trigger.get("table") or _flow_table(stored) or "")
|
|
|
|
| 6804 |
rows, merged = _merge_candidates(rows)
|
| 6805 |
stats["merged"] += merged
|
| 6806 |
|
| 6807 |
+
field_changes = {}
|
| 6808 |
+
|
| 6809 |
def _up(cur, _f=fields, _r=rows):
|
| 6810 |
+
field_changes.clear()
|
| 6811 |
cur = cur or {}
|
| 6812 |
t = dict(cur.get(table_key) or {})
|
| 6813 |
+
old_keys = _ut()._field_list_keys(t)
|
| 6814 |
t["fields"], t["rows"] = _f, _r
|
| 6815 |
+
new_keys = _ut()._field_list_keys(t)
|
| 6816 |
+
field_changes.update(removed=[key for key in old_keys if key not in new_keys],
|
| 6817 |
+
added=[key for key in new_keys if key not in old_keys])
|
| 6818 |
cur[table_key] = t
|
| 6819 |
return cur
|
| 6820 |
|
| 6821 |
rt.update(UT_STORE_KEY, _up)
|
| 6822 |
+
_ut()._log_field_list_change(rt, table_key, 'automation', 'migrate_ig_tables', field_changes)
|
| 6823 |
# ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION.
|
| 6824 |
#
|
| 6825 |
# W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early
|
|
|
|
| 14219 |
"check this tenant's Odoo connector under Settings -> Connectors; the linked column "
|
| 14220 |
"keeps its last resolved value until the source answers again")
|
| 14221 |
table["rows"] = {str(r.get("pid")): r for r in (pool or []) if r.get("pid") is not None}
|
| 14222 |
+
if key == "product_data":
|
| 14223 |
+
from modules import product_data
|
| 14224 |
+
table["rows"] = {str(row["pid"]): row for row in product_data.apply_lead_overlay(
|
| 14225 |
+
list(table["rows"].values()), table.get("_if_shared_cells"))}
|
| 14226 |
for rid, row in list(table['rows'].items()):
|
| 14227 |
table['rows'][rid] = {**row, **(table.get('_if_shared_cells', {}).get(rid) or {})}
|
| 14228 |
table["_rows_built"] = True
|
|
|
|
| 14372 |
return _module_table(rt, key, memo, want_rows=want_rows)
|
| 14373 |
|
| 14374 |
|
| 14375 |
+
def _link_from_key(fields, bag, overlay_fields=None):
|
| 14376 |
"""Which column on THIS table supplies the join value.
|
| 14377 |
|
| 14378 |
Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That
|
| 14379 |
fallback chain is what makes an Instagram database link up with no configuration at all —
|
| 14380 |
the "automatically" in the owner's instruction — and it is the same chain the grid already
|
| 14381 |
uses to decide a table's identity column, rather than a second opinion about it.
|
| 14382 |
+
`overlay_fields` (`{key: Field}` from the shared overlay) joins the fallback after the
|
| 14383 |
+
definition's own fields: a typed overlay column can supply the join too (D-11171).
|
| 14384 |
"""
|
| 14385 |
declared = str((bag or {}).get("from") or "").strip()
|
| 14386 |
if declared:
|
| 14387 |
return declared
|
| 14388 |
+
pool = list(fields) + [dict(f, key=k) for k, f in (overlay_fields or {}).items() if isinstance(f, dict)]
|
| 14389 |
+
prof = next((f for f in pool if isinstance(f.get("profile"), dict)), None)
|
| 14390 |
if prof:
|
| 14391 |
return str(prof.get("key") or "")
|
| 14392 |
+
pin = next((f for f in pool if f.get("pinned") is True), None)
|
| 14393 |
return str(pin.get("key") or "") if pin else ""
|
| 14394 |
|
| 14395 |
|
| 14396 |
+
def _overlay_join_rows(rt, table_key, rows):
|
| 14397 |
+
"""`({key: Field}, {row_id: {key: value}})`: this table's shared overlay columns and their
|
| 14398 |
+
typed cells for `rows`. A join reads a typed overlay value from here (D-11171).
|
| 14399 |
+
|
| 14400 |
+
Lenient like every display read: an unreadable overlay is `({}, {})`, so the join reads the
|
| 14401 |
+
stored row value only, exactly as it did before."""
|
| 14402 |
+
try:
|
| 14403 |
+
from core import shared_overlay
|
| 14404 |
+
declared = shared_overlay.fields(table_key, st=rt)
|
| 14405 |
+
if not declared:
|
| 14406 |
+
return {}, {}
|
| 14407 |
+
pids = [str(rid) for rid in rows if str(rid).isdigit()]
|
| 14408 |
+
return dict(declared), shared_overlay.cells(table_key, pids, st=rt)
|
| 14409 |
+
except Exception: # noqa: BLE001
|
| 14410 |
+
return {}, {}
|
| 14411 |
+
|
| 14412 |
+
|
| 14413 |
+
def _join_value(row, rid, from_key, overlay_cells):
|
| 14414 |
+
"""The join value a source row supplies: the stored cell, else its typed overlay cell."""
|
| 14415 |
+
value = (row or {}).get(from_key)
|
| 14416 |
+
if value is None or value == "":
|
| 14417 |
+
value = ((overlay_cells or {}).get(str(rid)) or {}).get(from_key)
|
| 14418 |
+
return value
|
| 14419 |
+
|
| 14420 |
+
|
| 14421 |
def _join_norm(value):
|
| 14422 |
"""ONE normalisation of an `on` join value, applied to BOTH sides of the join.
|
| 14423 |
|
|
|
|
| 14604 |
# --- resolve every link field ONCE per table, not once per row.
|
| 14605 |
# `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]`
|
| 14606 |
resolved, linked_types, refused = {}, {}, set()
|
| 14607 |
+
overlay = []
|
| 14608 |
+
|
| 14609 |
+
def _overlay(bag):
|
| 14610 |
+
"""`(overlay fields, overlay cells, from key)` for one join. The shared overlay is read
|
| 14611 |
+
once, and only when the `from` column is not a field this table defines."""
|
| 14612 |
+
own = _link_from_key(fields, bag)
|
| 14613 |
+
if own and own in by_key:
|
| 14614 |
+
return {}, {}, own
|
| 14615 |
+
if not overlay:
|
| 14616 |
+
overlay.append(_overlay_join_rows(rt, table_key, rows))
|
| 14617 |
+
return overlay[0] + (_link_from_key(fields, bag, overlay[0][0]),)
|
| 14618 |
+
|
| 14619 |
for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]:
|
| 14620 |
lk_key = str((f or {}).get("key") or "")
|
| 14621 |
if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict):
|
|
|
|
| 14682 |
on_key = str(bag.get("on") or "")
|
| 14683 |
hop_in = str(bag.get("throughOn") or "")
|
| 14684 |
hop_out = str(bag.get("throughTo") or "")
|
| 14685 |
+
_of, overlay_cells, from_key = _overlay(bag)
|
| 14686 |
if not (on_key and hop_in and hop_out and from_key):
|
| 14687 |
# ⛔ REFUSED, NOT BLANK — the same law the one-hop arm below follows. A route with
|
| 14688 |
# a missing leg has no second hop to make; resolving it to `[]` would render a
|
|
|
|
| 14720 |
target_idx = _linked_rows_by_join(linked, on_key)
|
| 14721 |
cap = int(_ut().LINK_THROUGH_MAX_IDS)
|
| 14722 |
|
| 14723 |
+
def _walk(row, rid, _bidx=bridge_idx, _tidx=target_idx, _cap=cap, _cells=overlay_cells):
|
| 14724 |
"""One source row's DISTINCT target records, and whether the ceiling stopped it.
|
| 14725 |
|
| 14726 |
⚠ DE-DUPED ON THE TARGET ROW ID. A customer orders the same SKU on twenty lines;
|
|
|
|
| 14728 |
label of a set of PRODUCTS.
|
| 14729 |
"""
|
| 14730 |
seen, out = set(), []
|
| 14731 |
+
for _bid, brow in _bidx.get(_join_norm(_join_value(row, rid, from_key, _cells)), ()):
|
| 14732 |
for tid, trow in _tidx.get(_join_norm((brow or {}).get(hop_out)), ()):
|
| 14733 |
if tid in seen:
|
| 14734 |
continue
|
|
|
|
| 14740 |
|
| 14741 |
two_hop, capped = {}, 0
|
| 14742 |
for rid, row in rows.items():
|
| 14743 |
+
hits_2, hit_cap = _walk(row, rid)
|
| 14744 |
two_hop[str(rid)] = hits_2
|
| 14745 |
capped += 1 if hit_cap else 0
|
| 14746 |
resolved[lk_key] = two_hop
|
|
|
|
| 14764 |
effect="capped", limit=cap))
|
| 14765 |
elif bag.get("on"):
|
| 14766 |
idx = _linked_rows_by_join(linked, str(bag["on"]))
|
| 14767 |
+
_of, overlay_cells, from_key = _overlay(bag)
|
| 14768 |
if not from_key:
|
| 14769 |
# ⛔ REFUSED, NOT BLANK. `_link_from_key` falls back to the PROFILE-flagged column
|
| 14770 |
# and then the PINNED one; a table with neither — which a registry module can be,
|
|
|
|
| 14780 |
"database that is the id column, e.g. `partner_id` or `product_id`)"))
|
| 14781 |
continue
|
| 14782 |
resolved[lk_key] = {
|
| 14783 |
+
str(rid): idx.get(_join_norm(_join_value(row, rid, from_key, overlay_cells)), [])
|
| 14784 |
for rid, row in rows.items()}
|
| 14785 |
else:
|
| 14786 |
# An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds
|
|
|
|
| 15076 |
return out, list(dict.fromkeys(reasons))
|
| 15077 |
|
| 15078 |
|
| 15079 |
+
#: Compatibility memo for direct computation; GET reads the durable provenance document.
|
| 15080 |
+
_RELATION_LIMITS = {}
|
| 15081 |
+
_RELATION_LIMITS_LOCK = threading.Lock()
|
| 15082 |
+
#: The refusals of the table this thread just refreshed, for the refresh log.
|
| 15083 |
+
_RELATION_PASS = threading.local()
|
| 15084 |
+
RELATION_LIMITS_KEY = "relation_refusals"
|
| 15085 |
+
_RELATION_LIMIT_KEYS = ("subject", "effect", "cause", "recommendation", "code", "limit")
|
| 15086 |
+
|
| 15087 |
+
|
| 15088 |
+
def _relation_tenant(rt):
|
| 15089 |
+
rt = getattr(rt, "_runtime", None) or rt
|
| 15090 |
+
return str(getattr(rt, "key", "") or "")
|
| 15091 |
+
|
| 15092 |
+
|
| 15093 |
+
def remember_relation_limits(rt, table_key, limits):
|
| 15094 |
+
"""Keep this table's newest link refusals for its tenant; an empty list clears them."""
|
| 15095 |
+
tenant = _relation_tenant(rt)
|
| 15096 |
+
if not tenant:
|
| 15097 |
+
return
|
| 15098 |
+
with _RELATION_LIMITS_LOCK:
|
| 15099 |
+
if limits:
|
| 15100 |
+
_RELATION_LIMITS[(tenant, str(table_key))] = [dict(limit) for limit in limits]
|
| 15101 |
+
else:
|
| 15102 |
+
_RELATION_LIMITS.pop((tenant, str(table_key)), None)
|
| 15103 |
+
|
| 15104 |
+
|
| 15105 |
+
def relation_limits(rt, table_key):
|
| 15106 |
+
"""Read the last saved relation provenance without refreshing or writing cells."""
|
| 15107 |
+
rt = getattr(rt, "_runtime", None) or rt
|
| 15108 |
+
document = _relation_document(rt, _relation_read(rt, RELATION_LIMITS_KEY))
|
| 15109 |
+
return copy.deepcopy(document.get(str(table_key), []))
|
| 15110 |
+
|
| 15111 |
+
|
| 15112 |
+
def _relation_read(rt, name):
|
| 15113 |
+
"""Use the bound backend's strict read and its existing bounded store waits."""
|
| 15114 |
+
if not _relation_tenant(rt) or not rt.available():
|
| 15115 |
+
raise RuntimeError("relation_provenance_unavailable")
|
| 15116 |
+
resolver = getattr(rt, "_document_store", None)
|
| 15117 |
+
if callable(resolver):
|
| 15118 |
+
backend = resolver() # Validates the store handle's tenant before reading.
|
| 15119 |
+
reader = getattr(backend, "_read_strict", None)
|
| 15120 |
+
if callable(reader):
|
| 15121 |
+
return reader(rt.store_key(name))
|
| 15122 |
+
return backend.get(rt.store_key(name), fresh=True)
|
| 15123 |
+
return rt.get(name, fresh=True)
|
| 15124 |
+
|
| 15125 |
+
|
| 15126 |
+
def _relation_document(rt, raw):
|
| 15127 |
+
"""Reject foreign or malformed provenance instead of treating it as a successful pass."""
|
| 15128 |
+
if raw is None or raw == {}:
|
| 15129 |
+
return {}
|
| 15130 |
+
if (not isinstance(raw, dict) or raw.get("tenant") != _relation_tenant(rt)
|
| 15131 |
+
or type(raw.get("version")) is not int or raw.get("version") != 1
|
| 15132 |
+
or not isinstance(raw.get("tables"), dict)):
|
| 15133 |
+
raise RuntimeError("relation_provenance_unavailable")
|
| 15134 |
+
tables = raw["tables"]
|
| 15135 |
+
if any(not isinstance(limits, list) or any(not isinstance(limit, dict) for limit in limits)
|
| 15136 |
+
for limits in tables.values()):
|
| 15137 |
+
raise RuntimeError("relation_provenance_unavailable")
|
| 15138 |
+
for limits in tables.values():
|
| 15139 |
+
for limit in limits:
|
| 15140 |
+
if (any(not isinstance(limit.get(key), str) for key in
|
| 15141 |
+
("subject", "effect", "cause", "recommendation", "fieldSignature"))
|
| 15142 |
+
or not re.fullmatch(r"[0-9a-f]{64}", limit["fieldSignature"])):
|
| 15143 |
+
raise RuntimeError("relation_provenance_unavailable")
|
| 15144 |
+
return tables
|
| 15145 |
+
|
| 15146 |
+
|
| 15147 |
+
def relation_field_signature(field):
|
| 15148 |
+
"""Bind saved provenance to the field definition that produced it."""
|
| 15149 |
+
definition = {k: field[k] for k in ("key", "type", "link", "rollup", "if") if k in field}
|
| 15150 |
+
return hashlib.sha256(json.dumps(definition, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
| 15151 |
+
|
| 15152 |
+
|
| 15153 |
+
def _relation_provenance(table, limits):
|
| 15154 |
+
"""Keep only field-bound diagnostics, excluding row identities, cells and exception payloads."""
|
| 15155 |
+
fields = [f for f in (table or {}).get("fields", []) if f.get("key")]
|
| 15156 |
+
out = []
|
| 15157 |
+
for limit in limits:
|
| 15158 |
+
subject = str(limit.get("subject") or "")
|
| 15159 |
+
matched = [f for f in fields if str(f["key"]) == subject]
|
| 15160 |
+
if not matched:
|
| 15161 |
+
matched = [f for f in fields if str((f.get("link") or {}).get("table") or "") == subject]
|
| 15162 |
+
if not matched and subject == str((table or {}).get("key") or ""):
|
| 15163 |
+
matched = [f for f in fields if f.get("type") in ("rollup", "if") or _ut().is_derived_link(f)]
|
| 15164 |
+
for field in matched:
|
| 15165 |
+
clean = {k: str(limit[k]) for k in _RELATION_LIMIT_KEYS if k in limit}
|
| 15166 |
+
if type(limit.get("limit")) is int:
|
| 15167 |
+
clean["limit"] = limit["limit"]
|
| 15168 |
+
clean["subject"] = str(field["key"])
|
| 15169 |
+
# Source exceptions can contain row values, SQL operands or provider response bodies.
|
| 15170 |
+
if any(marker in clean.get("cause", "") for marker in
|
| 15171 |
+
("could not be read:", "could not answer for it:")):
|
| 15172 |
+
clean["cause"] = "The relation source could not be read. Its last resolved value is retained."
|
| 15173 |
+
clean["recommendation"] = "Check this tenant's source connector, then retry the relation refresh."
|
| 15174 |
+
clean["fieldSignature"] = relation_field_signature(field)
|
| 15175 |
+
out.append(clean)
|
| 15176 |
+
return out
|
| 15177 |
+
|
| 15178 |
+
|
| 15179 |
+
def _persist_relation_limits(rt, results):
|
| 15180 |
+
"""Save one small tenant-bound document per pass, including recovery with no changed cells."""
|
| 15181 |
+
if not results:
|
| 15182 |
+
return
|
| 15183 |
+
old = _relation_document(rt, _relation_read(rt, RELATION_LIMITS_KEY))
|
| 15184 |
+
def merged(tables):
|
| 15185 |
+
tables = copy.deepcopy(tables)
|
| 15186 |
+
for key, limits in results.items():
|
| 15187 |
+
if limits:
|
| 15188 |
+
tables[str(key)] = copy.deepcopy(limits)
|
| 15189 |
+
else:
|
| 15190 |
+
tables.pop(str(key), None)
|
| 15191 |
+
return tables
|
| 15192 |
+
if merged(old) == old:
|
| 15193 |
+
return
|
| 15194 |
+
def up(raw):
|
| 15195 |
+
return {"version": 1, "tenant": _relation_tenant(rt),
|
| 15196 |
+
"tables": merged(_relation_document(rt, raw))}
|
| 15197 |
+
rt.update(RELATION_LIMITS_KEY, up, flush="sync")
|
| 15198 |
+
|
| 15199 |
+
|
| 15200 |
+
def relation_limit_line(table_key, limit):
|
| 15201 |
+
"""One log line for one refused link: the field, the cause and the fix."""
|
| 15202 |
+
limit = limit or {}
|
| 15203 |
+
cause = str(limit.get("cause") or "").strip().rstrip(".")
|
| 15204 |
+
fix = str(limit.get("recommendation") or "").strip().rstrip(".")
|
| 15205 |
+
return (f"[aios-auto] relation refresh {table_key}: link {limit.get('subject')} refused: "
|
| 15206 |
+
f"{cause}. Fix: {fix}.")
|
| 15207 |
+
|
| 15208 |
+
|
| 15209 |
def compute_relation_cells(rt, table_key, tables=None):
|
| 15210 |
"""Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
|
| 15211 |
|
|
|
|
| 15232 |
another section. ⇒ Refusals ride out on `relation_cells`' SECOND RETURN VALUE, where the
|
| 15233 |
caller that renders them reads them, and nothing about this door changes shape.
|
| 15234 |
"""
|
| 15235 |
+
changes, limits = relation_cells(rt, table_key, tables=tables)
|
| 15236 |
+
# D-11172: the refusals leave through state, never through this signature (see above).
|
| 15237 |
+
_RELATION_PASS.limits = [dict(limit) for limit in limits]
|
| 15238 |
+
remember_relation_limits(rt, table_key, limits)
|
| 15239 |
if not changes:
|
| 15240 |
return 0
|
| 15241 |
if str(table_key) in LINKABLE_MODULES:
|
|
|
|
| 15278 |
keyword this function briefly added (see `compute_relation_cells`).
|
| 15279 |
"""
|
| 15280 |
touched = 0
|
| 15281 |
+
_RELATION_PASS.results = {}
|
| 15282 |
for tk, table in list((tables or {}).items()):
|
| 15283 |
fields = (table or {}).get("fields") or []
|
| 15284 |
if not any(f.get("type") in ('rollup', 'if') or _ut().is_derived_link(f) for f in fields):
|
| 15285 |
+
remember_relation_limits(rt, tk, [])
|
| 15286 |
+
_RELATION_PASS.results[str(tk)] = []
|
| 15287 |
continue
|
| 15288 |
+
_RELATION_PASS.limits = []
|
| 15289 |
try:
|
| 15290 |
touched += compute_relation_cells(rt, tk, tables=tables)
|
| 15291 |
+
_RELATION_PASS.results[str(tk)] = _relation_provenance(table, _RELATION_PASS.limits)
|
| 15292 |
except Exception as e: # noqa: BLE001
|
| 15293 |
log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}")
|
| 15294 |
+
# D-11172: one line per refused link, its cause and its fix (standing rule 1).
|
| 15295 |
+
for limit in getattr(_RELATION_PASS, "limits", None) or ():
|
| 15296 |
+
log(relation_limit_line(tk, limit))
|
| 15297 |
return touched
|
| 15298 |
|
| 15299 |
|
| 15300 |
def refresh_relations(rt, log=print, tables=None, persist=True):
|
| 15301 |
"""Compute relations outside the write lock and merge only unchanged derived cells."""
|
| 15302 |
+
snapshot = tables if tables is not None else (_relation_read(rt, UT_STORE_KEY) or {})
|
| 15303 |
# Keep authored inputs before computation, including nested cell values.
|
| 15304 |
before = copy.deepcopy(snapshot)
|
| 15305 |
local = _refresh_relations_inplace(snapshot, log=log, rt=rt)
|
| 15306 |
+
results = copy.deepcopy(_RELATION_PASS.results)
|
| 15307 |
+
if not persist:
|
| 15308 |
+
return local
|
| 15309 |
+
if not local:
|
| 15310 |
+
_persist_relation_limits(rt, results)
|
| 15311 |
return local
|
| 15312 |
patches = {}
|
| 15313 |
for key, old in before.items():
|
|
|
|
| 15326 |
if changed:
|
| 15327 |
patches.setdefault(str(key), {})[str(rid)] = (row, changed)
|
| 15328 |
if not patches:
|
| 15329 |
+
_persist_relation_limits(rt, results)
|
| 15330 |
return 0
|
| 15331 |
actual = [0]
|
| 15332 |
+
field_changes = {}
|
| 15333 |
+
previous_limits = _relation_document(rt, _relation_read(rt, RELATION_LIMITS_KEY))
|
| 15334 |
+
clearing_refusal = any(previous_limits.get(key) and not limits for key, limits in results.items())
|
| 15335 |
|
| 15336 |
def _up(cur):
|
| 15337 |
actual[0] = 0
|
| 15338 |
+
field_changes.clear()
|
| 15339 |
lock = getattr(cur, 'lock_records', None)
|
| 15340 |
if callable(lock):
|
| 15341 |
lock({key: list(rows) for key, rows in patches.items()})
|
| 15342 |
for key, rows in patches.items():
|
| 15343 |
table = (cur or {}).get(key)
|
| 15344 |
if not isinstance(table, dict) or table.get('fields') != before[key].get('fields'):
|
| 15345 |
+
results.pop(key, None)
|
| 15346 |
continue
|
| 15347 |
+
old_keys = _ut()._field_list_keys(table)
|
| 15348 |
current_rows = table.get('rows')
|
| 15349 |
if current_rows is None:
|
| 15350 |
+
results.pop(key, None)
|
| 15351 |
continue
|
| 15352 |
for rid, (old_row, changes) in rows.items():
|
| 15353 |
current = current_rows.get(rid)
|
| 15354 |
# A changed authored input or a deleted row waits for the next refresh.
|
| 15355 |
if not isinstance(current, dict) or current != old_row:
|
| 15356 |
+
results.pop(key, None)
|
| 15357 |
continue
|
| 15358 |
current.update(changes)
|
| 15359 |
actual[0] += 1
|
| 15360 |
+
new_keys = _ut()._field_list_keys(table)
|
| 15361 |
+
field_changes[key] = {'removed': [name for name in old_keys if name not in new_keys],
|
| 15362 |
+
'added': [name for name in new_keys if name not in old_keys]}
|
| 15363 |
return cur
|
| 15364 |
|
| 15365 |
+
# A cold process must see recovered cells before their old refusal disappears.
|
| 15366 |
+
rt.update(UT_STORE_KEY, _up, flush="sync" if clearing_refusal else "async")
|
| 15367 |
+
_persist_relation_limits(rt, results)
|
| 15368 |
+
for key, change in field_changes.items():
|
| 15369 |
+
_ut()._log_field_list_change(rt, key, 'automation', 'refresh_relations', change)
|
| 15370 |
return actual[0]
|
| 15371 |
|
| 15372 |
|
|
|
|
| 16469 |
return "An authorized automation completed."
|
| 16470 |
|
| 16471 |
|
| 16472 |
+
NOTIFICATION_PAYLOAD_VERSION = 1
|
| 16473 |
+
|
| 16474 |
+
|
| 16475 |
+
def _authorized_notification_text(rt, stored, payload, context):
|
| 16476 |
+
"""Authorize the current recipient, channel and every source before composing notification content."""
|
| 16477 |
+
from core import account_rows, notify_destinations, perm_scope, perms, user_tables, users
|
| 16478 |
+
if (not isinstance(payload, dict) or type(payload.get('version')) is not int
|
| 16479 |
+
or payload.get('version') != NOTIFICATION_PAYLOAD_VERSION):
|
| 16480 |
+
return None
|
| 16481 |
+
aid = stored.get('id')
|
| 16482 |
+
expected = {key: payload.get(key) for key in ('version', 'tenant', 'user', 'kind', 'channel')}
|
| 16483 |
+
if (expected != stored.get('notificationDestination') or payload.get('automationId') != aid
|
| 16484 |
+
or expected['tenant'] != getattr(rt, 'key', None) or expected['channel'] != context.get('channel')
|
| 16485 |
+
or not isinstance(expected['user'], str) or not expected['user']
|
| 16486 |
+
or stored.get('createdBy') != expected['user']
|
| 16487 |
+
or stored.get('trigger', {}).get('key') != 'notification'):
|
| 16488 |
+
return None
|
| 16489 |
+
if aid != notify_destinations.automation_id(rt.key, expected['user'], expected['kind'], expected['channel']):
|
| 16490 |
+
return None
|
| 16491 |
+
resolve = getattr(rt, '_document_store', None)
|
| 16492 |
+
record = account_rows.get(expected['user'], st=resolve() if callable(resolve) else rt, fresh=True)
|
| 16493 |
+
if not record or record.get('active', True) is False or record.get('tenant') != rt.key:
|
| 16494 |
+
return None
|
| 16495 |
+
recipient = users._public(expected['user'], record)
|
| 16496 |
+
principal = context.get('principal')
|
| 16497 |
+
if not isinstance(principal, dict) or not principal.get('username'):
|
| 16498 |
+
return None
|
| 16499 |
+
notify_destinations.kind_id(expected['kind'])
|
| 16500 |
+
raw_items = payload.get('items')
|
| 16501 |
+
if not isinstance(raw_items, list) or not raw_items:
|
| 16502 |
+
return None
|
| 16503 |
+
for item in raw_items:
|
| 16504 |
+
source = item.get('source') if isinstance(item, dict) else None
|
| 16505 |
+
if (not isinstance(source, dict) or set(source) != {'table', 'rowIds'}
|
| 16506 |
+
or not isinstance(source['table'], str) or not source['table']
|
| 16507 |
+
or not isinstance(source['rowIds'], list) or not source['rowIds']
|
| 16508 |
+
or any(not isinstance(rid, str) or not rid for rid in source['rowIds'])
|
| 16509 |
+
or not _notification_source_readable(rt, context, source)):
|
| 16510 |
+
return None
|
| 16511 |
+
items = notify_destinations.clean_items(raw_items)
|
| 16512 |
+
for item in items:
|
| 16513 |
+
source = item['source']
|
| 16514 |
+
table, wanted = source['table'], set(source['rowIds'])
|
| 16515 |
+
if table == 'applications':
|
| 16516 |
+
import application_registry
|
| 16517 |
+
import app_templates
|
| 16518 |
+
applications = application_registry.load(rt, strict=True)['applications']
|
| 16519 |
+
for app_id in wanted:
|
| 16520 |
+
app = applications.get(app_id)
|
| 16521 |
+
if (not isinstance(app, dict) or app.get('id') != app_id or app.get('tenant') != rt.key
|
| 16522 |
+
or app.get('retired') is True):
|
| 16523 |
+
return None
|
| 16524 |
+
template = app_templates.BY_KIND.get(app.get('templateKind'))
|
| 16525 |
+
if not template or app_templates.held(app.get('templateKind')):
|
| 16526 |
+
return None
|
| 16527 |
+
sources = (*template['sources'], *template.get('optional_sources', ()))
|
| 16528 |
+
enabled = (getattr(getattr(rt, 'tenant', None), 'config', None) or {}).get('modules', 'all')
|
| 16529 |
+
if any((enabled != 'all' and (not isinstance(enabled, (list, tuple, set, frozenset))
|
| 16530 |
+
or source not in enabled))
|
| 16531 |
+
or not (user_tables.get_definition(source, st=rt) or {}).get('fields')
|
| 16532 |
+
for source in sources):
|
| 16533 |
+
return None
|
| 16534 |
+
for user in (recipient, principal):
|
| 16535 |
+
if not application_registry.may_read(app, user['username'], perms.is_admin(user)):
|
| 16536 |
+
return None
|
| 16537 |
+
access = application_registry.app_access(user, rt, app, sources)
|
| 16538 |
+
if access['app'] != 'ready' or any(wall['rows'] or wall['hidden']
|
| 16539 |
+
for wall in access['narrow'].values()):
|
| 16540 |
+
return None
|
| 16541 |
+
else:
|
| 16542 |
+
for user in (recipient, principal):
|
| 16543 |
+
if not perm_scope.may_read(user, table, st=rt) or perm_scope.wall_declared(user, table, st=rt):
|
| 16544 |
+
return None
|
| 16545 |
+
missing = set(wanted)
|
| 16546 |
+
for row in perm_scope.scoped_table(user, table, st=rt):
|
| 16547 |
+
missing.discard(str(row.get('pid')))
|
| 16548 |
+
if not missing:
|
| 16549 |
+
break
|
| 16550 |
+
if missing:
|
| 16551 |
+
return None
|
| 16552 |
+
return '\n\n'.join('\n'.join(str(item[key]) for key in ('title', 'body', 'url') if item.get(key))
|
| 16553 |
+
for item in items)
|
| 16554 |
+
|
| 16555 |
+
|
| 16556 |
+
def _notification_source_readable(rt, context, source):
|
| 16557 |
+
"""Require destination read access to every source row and field before using supplied content."""
|
| 16558 |
+
import routes_slack
|
| 16559 |
+
from core import perm_scope
|
| 16560 |
+
agent = routes_slack._agents(rt).get(context["agentId"])
|
| 16561 |
+
module, ids = source["table"], set(source["rowIds"])
|
| 16562 |
+
if not agent or agent.get("active", True) is False:
|
| 16563 |
+
return False
|
| 16564 |
+
if module == "applications":
|
| 16565 |
+
import application_registry
|
| 16566 |
+
import app_templates
|
| 16567 |
+
apps = application_registry.load(rt, strict=True)["applications"]
|
| 16568 |
+
for rid in ids:
|
| 16569 |
+
app = apps.get(rid)
|
| 16570 |
+
if (not isinstance(app, dict) or app.get("tenant") != rt.key or app.get("retired")
|
| 16571 |
+
or not application_registry.may_read(app, context["principal"]["username"])):
|
| 16572 |
+
return False
|
| 16573 |
+
template = app_templates.BY_KIND.get(app.get("templateKind")) or {}
|
| 16574 |
+
sources = template.get("sources") or []
|
| 16575 |
+
if not sources or any(not _notification_source_readable(
|
| 16576 |
+
rt, context, {"table": key, "rowIds": []}) for key in sources):
|
| 16577 |
+
return False
|
| 16578 |
+
return bool(ids)
|
| 16579 |
+
if not routes_slack.agent_may_read(agent, module):
|
| 16580 |
+
return False
|
| 16581 |
+
definition = _ut().get_definition(module, st=rt)
|
| 16582 |
+
if not ids:
|
| 16583 |
+
table, refusal = definition, None
|
| 16584 |
+
elif definition is None:
|
| 16585 |
+
table, refusal = _module_table(rt, module, {}, want_rows=True)
|
| 16586 |
+
elif _ut().materialises(module, defn=definition):
|
| 16587 |
+
table, refusal = _ut().get_records(module, ids, st=rt), None
|
| 16588 |
+
else:
|
| 16589 |
+
table, refusal = _resolve_table(rt, {module: definition}, module, {}, want_rows=True)
|
| 16590 |
+
if refusal or not isinstance(table, dict):
|
| 16591 |
+
return False
|
| 16592 |
+
fields = table.get("fields") or []
|
| 16593 |
+
if not fields or perm_scope.hidden_keys(context["principal"], module, fields, st=rt):
|
| 16594 |
+
return False
|
| 16595 |
+
if not ids:
|
| 16596 |
+
return not perm_scope.row_scope_applies(context["principal"], module, st=rt, defn=definition)
|
| 16597 |
+
rows = table.get("rows") or {}
|
| 16598 |
+
if not ids <= set(rows):
|
| 16599 |
+
return False
|
| 16600 |
+
selected = [{**rows[rid], "pid": rid} for rid in ids]
|
| 16601 |
+
readable = routes_slack.agent_rows(agent, selected, module, fields, st=rt)
|
| 16602 |
+
return {str(row.get("pid")) for row in readable} == ids
|
| 16603 |
+
|
| 16604 |
+
|
| 16605 |
def dispatch_authorized_slack_automation(rt, defn, result=None, *, post=None):
|
| 16606 |
"""Deliver one authorized event result, with readable linked records for an enters_view cohort."""
|
| 16607 |
definition = defn if isinstance(defn, dict) else {}
|
| 16608 |
aid = str(definition.get("id") or "").strip()
|
| 16609 |
if not aid:
|
| 16610 |
return {"sent": False, "reason": "unknown_automation"}
|
| 16611 |
+
if re.fullmatch(r"nd_[0-9a-f]{48}", aid):
|
| 16612 |
+
from core import notify_destinations
|
| 16613 |
+
try:
|
| 16614 |
+
stored = notify_destinations.slack_automation(rt, aid)
|
| 16615 |
+
except Exception: # noqa: BLE001
|
| 16616 |
+
return {"sent": False, "reason": "unknown_automation"}
|
| 16617 |
+
else:
|
| 16618 |
+
stored = all_definitions(rt).get(aid)
|
| 16619 |
if not isinstance(stored, dict):
|
| 16620 |
return {"sent": False, "reason": "unknown_automation"}
|
| 16621 |
trigger = stored.get("trigger") if isinstance(stored.get("trigger"), dict) else {}
|
|
|
|
| 16639 |
context = routes_slack.automation_answer_context(rt, aid)
|
| 16640 |
if context is None:
|
| 16641 |
return {"sent": False, "reason": "not_authorized"}
|
| 16642 |
+
if isinstance(result, dict) and "notification" in result:
|
| 16643 |
+
try:
|
| 16644 |
+
text = _authorized_notification_text(rt, stored, result['notification'], context)
|
| 16645 |
+
except Exception:
|
| 16646 |
+
text = None
|
| 16647 |
+
if text is None:
|
| 16648 |
+
return {'sent': False, 'reason': 'notification_not_authorized'}
|
| 16649 |
+
provenance = {'basis': 'authorized_app_data',
|
| 16650 |
+
'citationIds': [item['id'] for item in result['notification']['items']],
|
| 16651 |
+
'modelGuidance': False}
|
| 16652 |
+
delivered = routes_slack.deliver_channel_reply(
|
| 16653 |
+
rt, context['channel'], text, principal=context['principal'], automation_id=aid,
|
| 16654 |
+
provenance=provenance, post=post)
|
| 16655 |
+
if delivered.get('sent') is True and delivered.get('channel') == context['channel']:
|
| 16656 |
+
delivered['notificationVersion'] = NOTIFICATION_PAYLOAD_VERSION
|
| 16657 |
+
return delivered
|
| 16658 |
provenance = {"basis": "model_guidance", "citationIds": [], "modelGuidance": False}
|
| 16659 |
if trigger_key == "enters_view":
|
| 16660 |
module = str(trigger.get("table") or _flow_table(stored) or "")
|
api/connector_onboarding.py
CHANGED
|
@@ -136,14 +136,15 @@ def mirror_cursor(rt, table_key=None):
|
|
| 136 |
if rt is None:
|
| 137 |
return datastore.ro_con()
|
| 138 |
rt.assert_datastore_matches()
|
|
|
|
| 139 |
entities = selected_mirror_entities(rt)
|
| 140 |
if entities is None:
|
| 141 |
import odoo_relational as rel
|
| 142 |
if (table_key is not None
|
| 143 |
and rel.base_table_key(table_key) in rel.stock_source_specs()
|
| 144 |
-
and not
|
| 145 |
raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
|
| 146 |
-
return
|
| 147 |
if table_key is not None:
|
| 148 |
import odoo_relational as rel
|
| 149 |
import routes_odoo_tables as route
|
|
@@ -152,13 +153,13 @@ def mirror_cursor(rt, table_key=None):
|
|
| 152 |
raise RuntimeError("Enable this database in connection settings before reading it.")
|
| 153 |
source = route._odoo_sources().get(table_key)
|
| 154 |
if source is None:
|
| 155 |
-
return
|
| 156 |
entities = set(source.get("tables", {}).values())
|
| 157 |
if source.get("table"):
|
| 158 |
entities.add(source["table"])
|
| 159 |
if source.get("needs_excluded"):
|
| 160 |
entities.add("res_partner")
|
| 161 |
-
if any(not
|
| 162 |
raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
|
| 163 |
# An empty first selection has no readers to run and need not create a mirror file.
|
| 164 |
-
return
|
|
|
|
| 136 |
if rt is None:
|
| 137 |
return datastore.ro_con()
|
| 138 |
rt.assert_datastore_matches()
|
| 139 |
+
mirror = rt.mirror_handle()
|
| 140 |
entities = selected_mirror_entities(rt)
|
| 141 |
if entities is None:
|
| 142 |
import odoo_relational as rel
|
| 143 |
if (table_key is not None
|
| 144 |
and rel.base_table_key(table_key) in rel.stock_source_specs()
|
| 145 |
+
and not mirror.entity_live("stock_quant")):
|
| 146 |
raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
|
| 147 |
+
return mirror.ro_con()
|
| 148 |
if table_key is not None:
|
| 149 |
import odoo_relational as rel
|
| 150 |
import routes_odoo_tables as route
|
|
|
|
| 153 |
raise RuntimeError("Enable this database in connection settings before reading it.")
|
| 154 |
source = route._odoo_sources().get(table_key)
|
| 155 |
if source is None:
|
| 156 |
+
return mirror.ro_con()
|
| 157 |
entities = set(source.get("tables", {}).values())
|
| 158 |
if source.get("table"):
|
| 159 |
entities.add(source["table"])
|
| 160 |
if source.get("needs_excluded"):
|
| 161 |
entities.add("res_partner")
|
| 162 |
+
if any(not mirror.entity_live(key) for key in entities):
|
| 163 |
raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
|
| 164 |
# An empty first selection has no readers to run and need not create a mirror file.
|
| 165 |
+
return mirror.ro_cursor() if entities else None
|
api/deps.py
CHANGED
|
@@ -180,21 +180,173 @@ def _assistant_refuse(status, code, message):
|
|
| 180 |
raise err(status, code, message)
|
| 181 |
|
| 182 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
def assistant_unknown_field_message(requested, visible_keys):
|
| 184 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 185 |
candidates = sorted({key for key in visible_keys if isinstance(key, str) and key})
|
| 186 |
requested = [requested] if isinstance(requested, str) else requested
|
| 187 |
-
matches = []
|
| 188 |
for key in requested if isinstance(requested, (list, tuple)) else ():
|
| 189 |
if isinstance(key, str) and key not in candidates:
|
|
|
|
|
|
|
|
|
|
| 190 |
for match in difflib.get_close_matches(key, candidates, n=5, cutoff=0.6):
|
| 191 |
if match not in matches:
|
| 192 |
matches.append(match)
|
| 193 |
-
message =
|
| 194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 195 |
if matches:
|
| 196 |
message += " Close visible field keys: " + ", ".join(matches[:5]) + "."
|
| 197 |
-
return message
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
|
| 199 |
|
| 200 |
def _assistant_tenant_allows(session: Session, database: str) -> bool:
|
|
@@ -217,13 +369,18 @@ def _assistant_preflight_request(session: Session, database: str, canonical_fiel
|
|
| 217 |
"your current permissions do not allow this database")
|
| 218 |
visible = (perm_scope.assistant_visible_fields(canonical_fields, session.user, database)
|
| 219 |
if grant is not None else list(canonical_fields or ()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
try:
|
| 221 |
-
perm_scope.validate_assistant_filter((grant or {}).get("filter"),
|
| 222 |
-
canonical_fields if permission_fields is None else permission_fields)
|
| 223 |
-
_assistant_requested_fields(requested_fields, visible)
|
| 224 |
perm_scope.validate_assistant_filter(filters, visible)
|
| 225 |
except ValueError as exc:
|
| 226 |
-
|
| 227 |
return grant, visible
|
| 228 |
|
| 229 |
|
|
@@ -263,14 +420,16 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
|
|
| 263 |
if field["key"] not in have)]
|
| 264 |
fields, _ = schema_overlay.for_grid(database, fields, st=session.runtime, contract=fields)
|
| 265 |
internal = grid.get("_permission_scope") or {"fields": fields, "rows": [], "row_fields": fields}
|
| 266 |
-
|
| 267 |
-
|
|
|
|
| 268 |
for member in members:
|
|
|
|
| 269 |
try:
|
| 270 |
-
perm_scope.validate_assistant_filter(
|
| 271 |
-
(perm_scope.assistant_entry(member, database) or {}).get("filter"), internal["fields"])
|
| 272 |
except ValueError as exc:
|
| 273 |
-
|
|
|
|
| 274 |
if mode in ("schema", "query_schema"):
|
| 275 |
if permission_scope is not None and "_permission_scope" in grid:
|
| 276 |
permission_scope.update(internal)
|
|
@@ -299,15 +458,13 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
|
|
| 299 |
except HTTPException:
|
| 300 |
raise
|
| 301 |
except Exception as exc: # noqa: BLE001
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
f"scope ({type(exc).__name__}); try again shortly")
|
| 305 |
entry = cache.get(cache_key)
|
| 306 |
if (not isinstance(entry, tuple) or len(entry) < 2
|
| 307 |
or not isinstance(entry[1], (list, tuple))
|
| 308 |
or any(not isinstance(row, dict) for row in entry[1])):
|
| 309 |
-
|
| 310 |
-
"this database has no app-stored snapshot for your current scope")
|
| 311 |
grid = grid_assembly(session, scope=grid_scope, consume_corrections=False, with_rows=True)
|
| 312 |
if permission_scope is not None and "_permission_scope" in grid:
|
| 313 |
permission_scope.update(grid["_permission_scope"])
|
|
@@ -389,12 +546,11 @@ def _assistant_table_defn(session: Session, database: str):
|
|
| 389 |
|
| 390 |
try:
|
| 391 |
definitions = session.runtime.get_projection('user_tables', drop=('rows',))
|
| 392 |
-
except Exception:
|
| 393 |
-
|
| 394 |
-
|
| 395 |
if not isinstance(definitions, dict):
|
| 396 |
-
|
| 397 |
-
"this database has no readable definition snapshot")
|
| 398 |
if database == "ut_activity":
|
| 399 |
definitions = user_tables._with_activity(definitions)
|
| 400 |
definition = definitions.get(database)
|
|
@@ -428,8 +584,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
|
|
| 428 |
# upstream fallback: it is the tenant's materialised document already held by the runtime.
|
| 429 |
table = user_tables.get(database, st=session.runtime)
|
| 430 |
if not isinstance(table, dict):
|
| 431 |
-
|
| 432 |
-
"this database changed before its stored snapshot could be read")
|
| 433 |
if not user_tables.materialises(database, st=session.runtime, defn=table):
|
| 434 |
# ⭐⭐ W47-T23 / R11 - THIS IS NO LONGER THE ANSWER FOR A MIRROR-SERVED DATABASE, and the
|
| 435 |
# seam this comment used to ask for now exists: `assistant_mirror.py`, reached through
|
|
@@ -442,9 +597,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
|
|
| 442 |
# ⛔ DO NOT "FIX" THIS BY FLIPPING `readThrough`. D-378 says these tables' classification
|
| 443 |
# is separately wrong; changing the flag to make a read pass would hide that defect and
|
| 444 |
# move ten grids onto the wrong reader.
|
| 445 |
-
|
| 446 |
-
"this database stopped storing its rows while it was being read; ask "
|
| 447 |
-
"again and it will be read from its connector mirror instead")
|
| 448 |
|
| 449 |
# A table's field contract was authorized from the rows-free definition. If it changed while
|
| 450 |
# the materialised document was fetched, fail rather than projecting a schema that was never
|
|
@@ -452,12 +605,10 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
|
|
| 452 |
actual_fields = [dict(field) for field in (table.get("fields") or [])
|
| 453 |
if isinstance(field, dict) and field.get("key")]
|
| 454 |
if actual_fields != fields:
|
| 455 |
-
|
| 456 |
-
"this database changed before its stored snapshot could be read")
|
| 457 |
raw_rows = table.get("rows") or {}
|
| 458 |
if not isinstance(raw_rows, dict):
|
| 459 |
-
|
| 460 |
-
"this database does not have a readable stored snapshot")
|
| 461 |
keys = {field["key"] for field in fields}
|
| 462 |
rows = []
|
| 463 |
for record_id, raw in raw_rows.items():
|
|
@@ -539,9 +690,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 539 |
# `ut_odoo_model_*` table answers from its own field contract, as it does on the grid.
|
| 540 |
declared = assistant_mirror.declared_fields(database, definition)
|
| 541 |
if declared is None:
|
| 542 |
-
|
| 543 |
-
"this database does not store its rows and it is not one of the "
|
| 544 |
-
"connected databases the assistant can read from the mirror")
|
| 545 |
# ⭐ S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's
|
| 546 |
# first sync. Passed only for an Amazon key, which is the one family it changes.
|
| 547 |
cur, refusal = (assistant_mirror.open_cursor(session, database)
|
|
@@ -554,9 +703,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 554 |
assistant_mirror.unavailable_reason(database, definition))
|
| 555 |
fields = assistant_mirror.answerable_fields(database, spec, declared)
|
| 556 |
if not fields:
|
| 557 |
-
|
| 558 |
-
"this database's connector mirror carries none of its declared columns "
|
| 559 |
-
"on this deployment, so there is nothing to answer from")
|
| 560 |
|
| 561 |
grant, visible = _assistant_preflight_request(session, database, fields, requested_fields,
|
| 562 |
filters, require_explicit_grant=False)
|
|
@@ -567,11 +714,17 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 567 |
# Pushing the raw input to SQL instead would mean the pushdown and the Python pass are two
|
| 568 |
# predicates that happen to agree, which is this repo's recorded defect class. Same input to
|
| 569 |
# both compilers, same answer by construction.
|
|
|
|
| 570 |
try:
|
| 571 |
wall_tree = perm_scope.validate_assistant_filter((grant or {}).get("filter"), fields)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 572 |
filter_tree = perm_scope.validate_assistant_filter(filters, visible)
|
| 573 |
except ValueError as exc:
|
| 574 |
-
|
| 575 |
raise # unreachable; keeps the names bound for a reader
|
| 576 |
# ⚠ NO GRANT MEANS NO ROW WALL ON THIS PATH, matching `assistant_read_scope` exactly: it
|
| 577 |
# applies `assistant_apply_row_scope` only when a grant resolved, because `may_open` above is
|
|
@@ -589,10 +742,11 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 589 |
hidden.update(perm_scope.hidden_keys(member, database, fields, st=session.runtime))
|
| 590 |
tree = ((perm_scope.entry(member, database) or {}).get("filter")
|
| 591 |
if not perms.is_admin(member) else None)
|
|
|
|
| 592 |
try:
|
| 593 |
tree = perm_scope.validate_assistant_filter(tree, fields)
|
| 594 |
except ValueError as exc:
|
| 595 |
-
|
| 596 |
if tree is not None:
|
| 597 |
by_key = {field["key"]: field for field in fields}
|
| 598 |
for leaf_key in perm_scope._wall_leaf_keys(tree["nodes"]):
|
|
@@ -602,13 +756,15 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 602 |
verdict = perm_scope.wall_leaf_verdict(
|
| 603 |
database, {"colId": leaf_key}, session.runtime)
|
| 604 |
if not verdict["enforceable"]:
|
| 605 |
-
_assistant_refuse(400, "assistant_bad_filter",
|
|
|
|
|
|
|
| 606 |
trees.append(tree)
|
| 607 |
visible = [field for field in visible if field.get("key") not in hidden]
|
| 608 |
try:
|
| 609 |
filter_tree = perm_scope.validate_assistant_filter(filters, visible)
|
| 610 |
except ValueError as exc:
|
| 611 |
-
|
| 612 |
wall_tree = {"conj": "and", "nodes": [
|
| 613 |
{"conj": tree["conj"], "children": tree["nodes"]} for tree in trees
|
| 614 |
]} if trees else None
|
|
@@ -788,7 +944,7 @@ def _assistant_activity_table(session, definition, fields, filters, mode, member
|
|
| 788 |
try:
|
| 789 |
perm_scope.validate_assistant_filter(filters, schema["fields"])
|
| 790 |
except ValueError as exc:
|
| 791 |
-
|
| 792 |
rows = []
|
| 793 |
if mode not in ("schema", "query_schema"):
|
| 794 |
cursor = None
|
|
@@ -804,16 +960,20 @@ def _assistant_activity_table(session, definition, fields, filters, mode, member
|
|
| 804 |
if cursor in seen:
|
| 805 |
break
|
| 806 |
seen.add(cursor)
|
| 807 |
-
|
| 808 |
-
"This activity history is too large for an Assistant snapshot. Read it in Table by page.")
|
| 809 |
return rows, canonical, {"read": "field contract"}, definition.get("label") or "Activity"
|
| 810 |
|
| 811 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 812 |
def _assistant_identity_request(database, filters, identity_pid, *, trusted_filters=False):
|
| 813 |
"""Separate one opaque identity equality from visible field operands."""
|
| 814 |
if database not in _ASSISTANT_BUILT_INS:
|
| 815 |
if identity_pid is not None:
|
| 816 |
-
_assistant_refuse(400, "assistant_bad_filter",
|
|
|
|
| 817 |
return filters, identity_pid
|
| 818 |
if (trusted_filters and isinstance(filters, dict) and set(filters) == {"conj", "nodes"}
|
| 819 |
and filters.get("conj") == "and" and isinstance(filters.get("nodes"), list)
|
|
@@ -824,14 +984,17 @@ def _assistant_identity_request(database, filters, identity_pid, *, trusted_filt
|
|
| 824 |
and not set(leaf) - {"id", "colId", "op", "value"}):
|
| 825 |
value = leaf["value"]
|
| 826 |
if value is None:
|
| 827 |
-
_assistant_refuse(400, "assistant_bad_filter",
|
|
|
|
| 828 |
if identity_pid is not None and (type(value) is not type(identity_pid) or value != identity_pid):
|
| 829 |
-
_assistant_refuse(400, "assistant_bad_filter",
|
|
|
|
| 830 |
identity_pid, filters = value, None
|
| 831 |
if identity_pid is not None and (type(identity_pid) not in (int, str)
|
| 832 |
or isinstance(identity_pid, str)
|
| 833 |
and (not identity_pid.strip() or len(identity_pid) > 200)):
|
| 834 |
-
_assistant_refuse(400, "assistant_bad_filter",
|
|
|
|
| 835 |
return filters, identity_pid
|
| 836 |
|
| 837 |
|
|
@@ -885,8 +1048,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 885 |
try:
|
| 886 |
access = perm_scope.min_access([session.user, *members], key, st=session.runtime)
|
| 887 |
except (perm_scope.UnknownTable, perm_scope.Unresolvable):
|
| 888 |
-
|
| 889 |
-
"The database field contract could not be read. Try again in a moment.")
|
| 890 |
if not access["readable"]:
|
| 891 |
blocked = access["blockedBy"]
|
| 892 |
names = dash_norm.plain(", ".join(blocked))
|
|
@@ -906,7 +1068,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 906 |
try:
|
| 907 |
perm_scope.validate_assistant_filter(filters, shared_fields)
|
| 908 |
except ValueError as exc:
|
| 909 |
-
|
| 910 |
if isinstance(mode, tuple) and mode and mode[0] == "aggregate":
|
| 911 |
operands = [spec.get("field") for spec in mode[1] if isinstance(spec, dict)]
|
| 912 |
if len(mode) > 2:
|
|
@@ -970,9 +1132,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 970 |
if perm_scope.audience_declared(key, st=session.runtime, defn=_defn):
|
| 971 |
# C16 on a mirror-served table would have to run inside the SQL before counting
|
| 972 |
# and paging; filtering the returned page afterwards gives a wrong total. Refuse.
|
| 973 |
-
|
| 974 |
-
"This database limits each record to its listed recipients, "
|
| 975 |
-
"and that limit cannot be applied to this kind of read yet.")
|
| 976 |
if members:
|
| 977 |
rows, canonical_fields, source_version, label = _assistant_mirror_table(
|
| 978 |
session, key, fields, filters, defn=_defn, lent=_lent, mode=mode,
|
|
@@ -1012,7 +1172,17 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 1012 |
try:
|
| 1013 |
perm_scope.validate_assistant_filter((grant or {}).get("filter"), canonical_fields)
|
| 1014 |
except ValueError as exc:
|
| 1015 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1016 |
# ⭐ OWNER I16 — the tenant handle rides into the row wall here too, so one stored rule means
|
| 1017 |
# the same thing in the Analyst's answer as it does on the grid (see
|
| 1018 |
# `perm_scope._enrich_for_wall`).
|
|
@@ -1079,7 +1249,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 1079 |
try:
|
| 1080 |
requested_filter = perm_scope.validate_assistant_filter(filters, visible)
|
| 1081 |
except ValueError as exc:
|
| 1082 |
-
|
| 1083 |
if requested_filter is not None:
|
| 1084 |
visible_keys = {field["key"] for field in visible}
|
| 1085 |
evaluation_visible = [field for field in evaluation_fields if field["key"] in visible_keys]
|
|
|
|
| 180 |
raise err(status, code, message)
|
| 181 |
|
| 182 |
|
| 183 |
+
def _assistant_source_causes():
|
| 184 |
+
"""Return each unavailable cause's sentence and whether waiting can make the read succeed."""
|
| 185 |
+
import assistant_mirror
|
| 186 |
+
return {
|
| 187 |
+
"loading": ("The copy of this database is still loading on this server.", True),
|
| 188 |
+
"syncing": ("A sync of this database's source is running.", True),
|
| 189 |
+
"store_silent": ("The store did not answer.", True),
|
| 190 |
+
"changed": ("This database changed while it was being read.", True),
|
| 191 |
+
"not_built": ("The copy of this database for your access could not be built.", False),
|
| 192 |
+
"contract_unread": ("The field list of this database could not be read.", False),
|
| 193 |
+
"no_binding": (assistant_mirror.NO_BINDING, False),
|
| 194 |
+
"no_copy": ("This database stores no rows and has no connector copy to read them from.", False),
|
| 195 |
+
"no_columns": ("The connector copy of this database holds none of its fields on this server.", False),
|
| 196 |
+
"unreadable": ("The stored copy of this database cannot be read.", False),
|
| 197 |
+
"audience": ("This database limits each record to its listed recipients, and that limit "
|
| 198 |
+
"cannot be applied to this kind of read yet.", False),
|
| 199 |
+
"too_large": ("This activity history is too large for an Assistant snapshot. "
|
| 200 |
+
"Read it in Table by page.", False),
|
| 201 |
+
"no_connector": ("This database names no connector, so its completed sync cannot be checked.", False),
|
| 202 |
+
"no_activity_store": ("Activity storage is not set up for this workspace.", False),
|
| 203 |
+
"invalid_publication": ("The completed sync identity of this database is unreadable.", False),
|
| 204 |
+
"restore_blocked": ("The saved copy of this database could not be restored. "
|
| 205 |
+
"Ask an Administrator to repair store access and restart the server.", False),
|
| 206 |
+
"mirror_open": ("The connector copy of this database could not be opened on this server.", False),
|
| 207 |
+
"not_supported": ("Selected record reads are not available on this deployment until it is updated.",
|
| 208 |
+
False),
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def _assistant_source_message(cause, verb="read", *, message=None):
|
| 213 |
+
"""The refusal text for one cause: its sentence, then a retry line only where a retry can succeed."""
|
| 214 |
+
if cause in {"amazon_restore", "amazon_sync"} and _assistant_source_cause_of(message) == cause:
|
| 215 |
+
return message
|
| 216 |
+
sentence, retry = _assistant_source_causes()[cause]
|
| 217 |
+
return f"{sentence} Try the same {verb} again shortly." if retry else sentence
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _assistant_source_cause_of(message):
|
| 221 |
+
"""The cause a refusal message names, or None for text outside the closed set."""
|
| 222 |
+
text = " ".join(str(message or "").split())
|
| 223 |
+
for cause, (sentence, _retry) in _assistant_source_causes().items():
|
| 224 |
+
if text.startswith(sentence):
|
| 225 |
+
return cause
|
| 226 |
+
if text.startswith("Amazon "):
|
| 227 |
+
import amazon_relational as amazon
|
| 228 |
+
# Recognize native instructions without another pending-state read.
|
| 229 |
+
for key in amazon.SOURCE:
|
| 230 |
+
for cause, pending in (("amazon_restore", True), ("amazon_sync", False)):
|
| 231 |
+
if text == amazon.missing_source(key, pending=pending)["missing"]:
|
| 232 |
+
return cause
|
| 233 |
+
return None
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def _assistant_refuse_source(cause, status=409, code="assistant_snapshot_unavailable"):
|
| 237 |
+
"""Refuse a read whose snapshot or source is unavailable, naming its cause."""
|
| 238 |
+
_assistant_refuse(status, code, _assistant_source_message(cause))
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def _assistant_log_cause(where, exc):
|
| 242 |
+
"""Log an exception's class, never its text, beside the cause the caller was told."""
|
| 243 |
+
import logging
|
| 244 |
+
logging.getLogger(__name__).warning("%s failed (%s)", where, type(exc).__name__)
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _assistant_display_key(key):
|
| 248 |
+
"""A caller-supplied key or operator as message text: one line, at most 80 characters."""
|
| 249 |
+
return " ".join(str(key).split())[:80]
|
| 250 |
+
|
| 251 |
+
|
| 252 |
def assistant_unknown_field_message(requested, visible_keys):
|
| 253 |
+
"""Name the requested keys that are not readable, then suggest at most five close visible keys.
|
| 254 |
+
|
| 255 |
+
S82-T124: only keys the caller sent are echoed, and suggestions come from `visible_keys` only.
|
| 256 |
+
A hidden key and an unknown key therefore read the same, so the message is no existence oracle.
|
| 257 |
+
"""
|
| 258 |
candidates = sorted({key for key in visible_keys if isinstance(key, str) and key})
|
| 259 |
requested = [requested] if isinstance(requested, str) else requested
|
| 260 |
+
matches, missing = [], []
|
| 261 |
for key in requested if isinstance(requested, (list, tuple)) else ():
|
| 262 |
if isinstance(key, str) and key not in candidates:
|
| 263 |
+
name = _assistant_display_key(key)
|
| 264 |
+
if name and name not in missing:
|
| 265 |
+
missing.append(name)
|
| 266 |
for match in difflib.get_close_matches(key, candidates, n=5, cutoff=0.6):
|
| 267 |
if match not in matches:
|
| 268 |
matches.append(match)
|
| 269 |
+
message = ""
|
| 270 |
+
if missing:
|
| 271 |
+
message = (("No readable field is named " if len(missing) == 1
|
| 272 |
+
else "No readable fields are named ") + ", ".join(missing[:5]) + ". ")
|
| 273 |
+
message += ("Every requested field must be readable in this database. "
|
| 274 |
+
"Use a field key returned by describe_database.")
|
| 275 |
if matches:
|
| 276 |
message += " Close visible field keys: " + ", ".join(matches[:5]) + "."
|
| 277 |
+
return dash_norm.plain(message)
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def _assistant_filter_leaves(tree, depth=1):
|
| 281 |
+
"""Every condition of a raw filter tree, in the order the strict validator reads them."""
|
| 282 |
+
nodes = tree if isinstance(tree, list) else (tree.get("nodes") if isinstance(tree, dict) else None)
|
| 283 |
+
out = []
|
| 284 |
+
for node in nodes if isinstance(nodes, list) else ():
|
| 285 |
+
if isinstance(node, dict) and "children" in node:
|
| 286 |
+
if depth < 3:
|
| 287 |
+
out.extend(_assistant_filter_leaves(node.get("children"), depth + 1))
|
| 288 |
+
else:
|
| 289 |
+
out.append(node)
|
| 290 |
+
return out
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def _assistant_rejected_condition(tree, fields):
|
| 294 |
+
"""The first condition the strict validator refuses on its own, or None for a shape fault."""
|
| 295 |
+
import core.perm_scope as perm_scope
|
| 296 |
+
for leaf in _assistant_filter_leaves(tree):
|
| 297 |
+
try:
|
| 298 |
+
perm_scope.validate_assistant_filter([leaf], fields)
|
| 299 |
+
except ValueError:
|
| 300 |
+
return leaf
|
| 301 |
+
except Exception: # noqa: BLE001 - a leaf the tree never reached
|
| 302 |
+
return None
|
| 303 |
+
return None
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def _assistant_bad_filter_message(reason, condition, *, visible_keys=(), wall=False):
|
| 307 |
+
"""The `assistant_bad_filter` text: the validator's reason plus the rejected field and operator.
|
| 308 |
+
|
| 309 |
+
S82-T124: every refusal names its field. A wall's field is named only when the caller may read
|
| 310 |
+
it, so the restriction an administrator set never names a hidden field.
|
| 311 |
+
"""
|
| 312 |
+
text = " ".join(str(reason or "").split()).rstrip(".") or "assistant filter cannot be read"
|
| 313 |
+
where = " in the restriction an administrator set on this database" if wall else ""
|
| 314 |
+
field = condition.get("colId") if isinstance(condition, dict) else None
|
| 315 |
+
name = _assistant_display_key(field) if isinstance(field, str) else ""
|
| 316 |
+
if condition is None:
|
| 317 |
+
text += ". The filter's structure is at fault, not one field"
|
| 318 |
+
elif not name:
|
| 319 |
+
text += f". The rejected condition{where} names no field"
|
| 320 |
+
elif wall and field not in set(visible_keys or ()):
|
| 321 |
+
text += f". The rejected condition{where} is on a field you cannot read"
|
| 322 |
+
else:
|
| 323 |
+
text += f'. The rejected condition{where} is on field "{name}"'
|
| 324 |
+
if "op" in condition:
|
| 325 |
+
op = condition.get("op")
|
| 326 |
+
text += (f' with operator "{_assistant_display_key(op)}"'
|
| 327 |
+
if isinstance(op, str) and op.strip() else " with no operator")
|
| 328 |
+
return dash_norm.plain(text + ".")
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def _assistant_refuse_filter(reason, tree, fields, *, visible_keys=None, wall=False):
|
| 332 |
+
"""Refuse a filter the strict validator rejected, naming the condition it rejected.
|
| 333 |
+
|
| 334 |
+
A caller's condition on a key outside its readable `fields` answers `assistant_unknown_field`,
|
| 335 |
+
as a field selection does, so hidden and unknown keys read alike. `pid` is the record identity
|
| 336 |
+
no filter may name, and a wall is the administrator's: both stay `assistant_bad_filter`.
|
| 337 |
+
"""
|
| 338 |
+
keys = {str(field.get("key")) for field in fields or ()
|
| 339 |
+
if isinstance(field, dict) and field.get("key")}
|
| 340 |
+
visible = keys if visible_keys is None else set(visible_keys)
|
| 341 |
+
condition = _assistant_rejected_condition(tree, fields)
|
| 342 |
+
if not wall and isinstance(condition, dict):
|
| 343 |
+
rhs = condition.get("rhs")
|
| 344 |
+
for key in (condition.get("colId"), rhs.get("colId") if isinstance(rhs, dict) else None):
|
| 345 |
+
if isinstance(key, str) and key.strip() and key not in keys and key != "pid":
|
| 346 |
+
_assistant_refuse(400, "assistant_unknown_field",
|
| 347 |
+
assistant_unknown_field_message(key, visible))
|
| 348 |
+
_assistant_refuse(400, "assistant_bad_filter",
|
| 349 |
+
_assistant_bad_filter_message(reason, condition, visible_keys=visible, wall=wall))
|
| 350 |
|
| 351 |
|
| 352 |
def _assistant_tenant_allows(session: Session, database: str) -> bool:
|
|
|
|
| 369 |
"your current permissions do not allow this database")
|
| 370 |
visible = (perm_scope.assistant_visible_fields(canonical_fields, session.user, database)
|
| 371 |
if grant is not None else list(canonical_fields or ()))
|
| 372 |
+
wall_fields = canonical_fields if permission_fields is None else permission_fields
|
| 373 |
+
visible_keys = [field.get("key") for field in visible if isinstance(field, dict)]
|
| 374 |
+
try:
|
| 375 |
+
perm_scope.validate_assistant_filter((grant or {}).get("filter"), wall_fields)
|
| 376 |
+
except ValueError as exc:
|
| 377 |
+
_assistant_refuse_filter(exc, (grant or {}).get("filter"), wall_fields,
|
| 378 |
+
visible_keys=visible_keys, wall=True)
|
| 379 |
+
_assistant_requested_fields(requested_fields, visible)
|
| 380 |
try:
|
|
|
|
|
|
|
|
|
|
| 381 |
perm_scope.validate_assistant_filter(filters, visible)
|
| 382 |
except ValueError as exc:
|
| 383 |
+
_assistant_refuse_filter(exc, filters, visible)
|
| 384 |
return grant, visible
|
| 385 |
|
| 386 |
|
|
|
|
| 420 |
if field["key"] not in have)]
|
| 421 |
fields, _ = schema_overlay.for_grid(database, fields, st=session.runtime, contract=fields)
|
| 422 |
internal = grid.get("_permission_scope") or {"fields": fields, "rows": [], "row_fields": fields}
|
| 423 |
+
_grant, visible = _assistant_preflight_request(
|
| 424 |
+
session, database, fields, requested_fields, filters,
|
| 425 |
+
require_explicit_grant=True, permission_fields=internal["fields"])
|
| 426 |
for member in members:
|
| 427 |
+
wall = (perm_scope.assistant_entry(member, database) or {}).get("filter")
|
| 428 |
try:
|
| 429 |
+
perm_scope.validate_assistant_filter(wall, internal["fields"])
|
|
|
|
| 430 |
except ValueError as exc:
|
| 431 |
+
_assistant_refuse_filter(exc, wall, internal["fields"], wall=True, visible_keys=[
|
| 432 |
+
field.get("key") for field in visible if isinstance(field, dict)])
|
| 433 |
if mode in ("schema", "query_schema"):
|
| 434 |
if permission_scope is not None and "_permission_scope" in grid:
|
| 435 |
permission_scope.update(internal)
|
|
|
|
| 458 |
except HTTPException:
|
| 459 |
raise
|
| 460 |
except Exception as exc: # noqa: BLE001
|
| 461 |
+
_assistant_log_cause("assistant pool build", exc)
|
| 462 |
+
_assistant_refuse_source("not_built")
|
|
|
|
| 463 |
entry = cache.get(cache_key)
|
| 464 |
if (not isinstance(entry, tuple) or len(entry) < 2
|
| 465 |
or not isinstance(entry[1], (list, tuple))
|
| 466 |
or any(not isinstance(row, dict) for row in entry[1])):
|
| 467 |
+
_assistant_refuse_source("not_built")
|
|
|
|
| 468 |
grid = grid_assembly(session, scope=grid_scope, consume_corrections=False, with_rows=True)
|
| 469 |
if permission_scope is not None and "_permission_scope" in grid:
|
| 470 |
permission_scope.update(grid["_permission_scope"])
|
|
|
|
| 546 |
|
| 547 |
try:
|
| 548 |
definitions = session.runtime.get_projection('user_tables', drop=('rows',))
|
| 549 |
+
except Exception as exc:
|
| 550 |
+
_assistant_log_cause("assistant definition read", exc)
|
| 551 |
+
_assistant_refuse_source("store_silent")
|
| 552 |
if not isinstance(definitions, dict):
|
| 553 |
+
_assistant_refuse_source("unreadable")
|
|
|
|
| 554 |
if database == "ut_activity":
|
| 555 |
definitions = user_tables._with_activity(definitions)
|
| 556 |
definition = definitions.get(database)
|
|
|
|
| 584 |
# upstream fallback: it is the tenant's materialised document already held by the runtime.
|
| 585 |
table = user_tables.get(database, st=session.runtime)
|
| 586 |
if not isinstance(table, dict):
|
| 587 |
+
_assistant_refuse_source("changed")
|
|
|
|
| 588 |
if not user_tables.materialises(database, st=session.runtime, defn=table):
|
| 589 |
# ⭐⭐ W47-T23 / R11 - THIS IS NO LONGER THE ANSWER FOR A MIRROR-SERVED DATABASE, and the
|
| 590 |
# seam this comment used to ask for now exists: `assistant_mirror.py`, reached through
|
|
|
|
| 597 |
# ⛔ DO NOT "FIX" THIS BY FLIPPING `readThrough`. D-378 says these tables' classification
|
| 598 |
# is separately wrong; changing the flag to make a read pass would hide that defect and
|
| 599 |
# move ten grids onto the wrong reader.
|
| 600 |
+
_assistant_refuse_source("changed")
|
|
|
|
|
|
|
| 601 |
|
| 602 |
# A table's field contract was authorized from the rows-free definition. If it changed while
|
| 603 |
# the materialised document was fetched, fail rather than projecting a schema that was never
|
|
|
|
| 605 |
actual_fields = [dict(field) for field in (table.get("fields") or [])
|
| 606 |
if isinstance(field, dict) and field.get("key")]
|
| 607 |
if actual_fields != fields:
|
| 608 |
+
_assistant_refuse_source("changed")
|
|
|
|
| 609 |
raw_rows = table.get("rows") or {}
|
| 610 |
if not isinstance(raw_rows, dict):
|
| 611 |
+
_assistant_refuse_source("unreadable")
|
|
|
|
| 612 |
keys = {field["key"] for field in fields}
|
| 613 |
rows = []
|
| 614 |
for record_id, raw in raw_rows.items():
|
|
|
|
| 690 |
# `ut_odoo_model_*` table answers from its own field contract, as it does on the grid.
|
| 691 |
declared = assistant_mirror.declared_fields(database, definition)
|
| 692 |
if declared is None:
|
| 693 |
+
_assistant_refuse_source("no_copy")
|
|
|
|
|
|
|
| 694 |
# ⭐ S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's
|
| 695 |
# first sync. Passed only for an Amazon key, which is the one family it changes.
|
| 696 |
cur, refusal = (assistant_mirror.open_cursor(session, database)
|
|
|
|
| 703 |
assistant_mirror.unavailable_reason(database, definition))
|
| 704 |
fields = assistant_mirror.answerable_fields(database, spec, declared)
|
| 705 |
if not fields:
|
| 706 |
+
_assistant_refuse_source("no_columns")
|
|
|
|
|
|
|
| 707 |
|
| 708 |
grant, visible = _assistant_preflight_request(session, database, fields, requested_fields,
|
| 709 |
filters, require_explicit_grant=False)
|
|
|
|
| 714 |
# Pushing the raw input to SQL instead would mean the pushdown and the Python pass are two
|
| 715 |
# predicates that happen to agree, which is this repo's recorded defect class. Same input to
|
| 716 |
# both compilers, same answer by construction.
|
| 717 |
+
visible_keys = [field.get("key") for field in visible if isinstance(field, dict)]
|
| 718 |
try:
|
| 719 |
wall_tree = perm_scope.validate_assistant_filter((grant or {}).get("filter"), fields)
|
| 720 |
+
except ValueError as exc:
|
| 721 |
+
_assistant_refuse_filter(exc, (grant or {}).get("filter"), fields,
|
| 722 |
+
visible_keys=visible_keys, wall=True)
|
| 723 |
+
raise # unreachable; keeps the names bound for a reader
|
| 724 |
+
try:
|
| 725 |
filter_tree = perm_scope.validate_assistant_filter(filters, visible)
|
| 726 |
except ValueError as exc:
|
| 727 |
+
_assistant_refuse_filter(exc, filters, visible)
|
| 728 |
raise # unreachable; keeps the names bound for a reader
|
| 729 |
# ⚠ NO GRANT MEANS NO ROW WALL ON THIS PATH, matching `assistant_read_scope` exactly: it
|
| 730 |
# applies `assistant_apply_row_scope` only when a grant resolved, because `may_open` above is
|
|
|
|
| 742 |
hidden.update(perm_scope.hidden_keys(member, database, fields, st=session.runtime))
|
| 743 |
tree = ((perm_scope.entry(member, database) or {}).get("filter")
|
| 744 |
if not perms.is_admin(member) else None)
|
| 745 |
+
raw_wall = tree
|
| 746 |
try:
|
| 747 |
tree = perm_scope.validate_assistant_filter(tree, fields)
|
| 748 |
except ValueError as exc:
|
| 749 |
+
_assistant_refuse_filter(exc, raw_wall, fields, visible_keys=visible_keys, wall=True)
|
| 750 |
if tree is not None:
|
| 751 |
by_key = {field["key"]: field for field in fields}
|
| 752 |
for leaf_key in perm_scope._wall_leaf_keys(tree["nodes"]):
|
|
|
|
| 756 |
verdict = perm_scope.wall_leaf_verdict(
|
| 757 |
database, {"colId": leaf_key}, session.runtime)
|
| 758 |
if not verdict["enforceable"]:
|
| 759 |
+
_assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
|
| 760 |
+
verdict["reason"], {"colId": leaf_key},
|
| 761 |
+
visible_keys=visible_keys, wall=True))
|
| 762 |
trees.append(tree)
|
| 763 |
visible = [field for field in visible if field.get("key") not in hidden]
|
| 764 |
try:
|
| 765 |
filter_tree = perm_scope.validate_assistant_filter(filters, visible)
|
| 766 |
except ValueError as exc:
|
| 767 |
+
_assistant_refuse_filter(exc, filters, visible)
|
| 768 |
wall_tree = {"conj": "and", "nodes": [
|
| 769 |
{"conj": tree["conj"], "children": tree["nodes"]} for tree in trees
|
| 770 |
]} if trees else None
|
|
|
|
| 944 |
try:
|
| 945 |
perm_scope.validate_assistant_filter(filters, schema["fields"])
|
| 946 |
except ValueError as exc:
|
| 947 |
+
_assistant_refuse_filter(exc, filters, schema["fields"])
|
| 948 |
rows = []
|
| 949 |
if mode not in ("schema", "query_schema"):
|
| 950 |
cursor = None
|
|
|
|
| 960 |
if cursor in seen:
|
| 961 |
break
|
| 962 |
seen.add(cursor)
|
| 963 |
+
_assistant_refuse_source("too_large")
|
|
|
|
| 964 |
return rows, canonical, {"read": "field contract"}, definition.get("label") or "Activity"
|
| 965 |
|
| 966 |
|
| 967 |
+
#: The one condition an opaque identity read carries; its refusals name this field and operator.
|
| 968 |
+
_IDENTITY_CONDITION = {"colId": "pid", "op": "eq"}
|
| 969 |
+
|
| 970 |
+
|
| 971 |
def _assistant_identity_request(database, filters, identity_pid, *, trusted_filters=False):
|
| 972 |
"""Separate one opaque identity equality from visible field operands."""
|
| 973 |
if database not in _ASSISTANT_BUILT_INS:
|
| 974 |
if identity_pid is not None:
|
| 975 |
+
_assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
|
| 976 |
+
"Opaque identity reads require a built-in database", _IDENTITY_CONDITION))
|
| 977 |
return filters, identity_pid
|
| 978 |
if (trusted_filters and isinstance(filters, dict) and set(filters) == {"conj", "nodes"}
|
| 979 |
and filters.get("conj") == "and" and isinstance(filters.get("nodes"), list)
|
|
|
|
| 984 |
and not set(leaf) - {"id", "colId", "op", "value"}):
|
| 985 |
value = leaf["value"]
|
| 986 |
if value is None:
|
| 987 |
+
_assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
|
| 988 |
+
"Use one non-null opaque record identity", _IDENTITY_CONDITION))
|
| 989 |
if identity_pid is not None and (type(value) is not type(identity_pid) or value != identity_pid):
|
| 990 |
+
_assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
|
| 991 |
+
"Opaque identity pins must agree", _IDENTITY_CONDITION))
|
| 992 |
identity_pid, filters = value, None
|
| 993 |
if identity_pid is not None and (type(identity_pid) not in (int, str)
|
| 994 |
or isinstance(identity_pid, str)
|
| 995 |
and (not identity_pid.strip() or len(identity_pid) > 200)):
|
| 996 |
+
_assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
|
| 997 |
+
"Use one opaque integer or string record identity", _IDENTITY_CONDITION))
|
| 998 |
return filters, identity_pid
|
| 999 |
|
| 1000 |
|
|
|
|
| 1048 |
try:
|
| 1049 |
access = perm_scope.min_access([session.user, *members], key, st=session.runtime)
|
| 1050 |
except (perm_scope.UnknownTable, perm_scope.Unresolvable):
|
| 1051 |
+
_assistant_refuse_source("contract_unread")
|
|
|
|
| 1052 |
if not access["readable"]:
|
| 1053 |
blocked = access["blockedBy"]
|
| 1054 |
names = dash_norm.plain(", ".join(blocked))
|
|
|
|
| 1068 |
try:
|
| 1069 |
perm_scope.validate_assistant_filter(filters, shared_fields)
|
| 1070 |
except ValueError as exc:
|
| 1071 |
+
_assistant_refuse_filter(exc, filters, shared_fields)
|
| 1072 |
if isinstance(mode, tuple) and mode and mode[0] == "aggregate":
|
| 1073 |
operands = [spec.get("field") for spec in mode[1] if isinstance(spec, dict)]
|
| 1074 |
if len(mode) > 2:
|
|
|
|
| 1132 |
if perm_scope.audience_declared(key, st=session.runtime, defn=_defn):
|
| 1133 |
# C16 on a mirror-served table would have to run inside the SQL before counting
|
| 1134 |
# and paging; filtering the returned page afterwards gives a wrong total. Refuse.
|
| 1135 |
+
_assistant_refuse_source("audience")
|
|
|
|
|
|
|
| 1136 |
if members:
|
| 1137 |
rows, canonical_fields, source_version, label = _assistant_mirror_table(
|
| 1138 |
session, key, fields, filters, defn=_defn, lent=_lent, mode=mode,
|
|
|
|
| 1172 |
try:
|
| 1173 |
perm_scope.validate_assistant_filter((grant or {}).get("filter"), canonical_fields)
|
| 1174 |
except ValueError as exc:
|
| 1175 |
+
# Only on the refusal path: the caller's readable keys decide whether the wall's field
|
| 1176 |
+
# is named. An unreadable list names no field, so the refusal stays a 400.
|
| 1177 |
+
try:
|
| 1178 |
+
withheld = set(perm_scope.hidden_keys(session.user, key, canonical_fields,
|
| 1179 |
+
st=session.runtime))
|
| 1180 |
+
readable = [field.get("key") for field in perm_scope.assistant_visible_fields(
|
| 1181 |
+
canonical_fields, session.user, key) if field.get("key") not in withheld]
|
| 1182 |
+
except Exception: # noqa: BLE001 - the refusal must still answer
|
| 1183 |
+
readable = []
|
| 1184 |
+
_assistant_refuse_filter(exc, (grant or {}).get("filter"), canonical_fields,
|
| 1185 |
+
visible_keys=readable, wall=True)
|
| 1186 |
# ⭐ OWNER I16 — the tenant handle rides into the row wall here too, so one stored rule means
|
| 1187 |
# the same thing in the Analyst's answer as it does on the grid (see
|
| 1188 |
# `perm_scope._enrich_for_wall`).
|
|
|
|
| 1249 |
try:
|
| 1250 |
requested_filter = perm_scope.validate_assistant_filter(filters, visible)
|
| 1251 |
except ValueError as exc:
|
| 1252 |
+
_assistant_refuse_filter(exc, filters, visible)
|
| 1253 |
if requested_filter is not None:
|
| 1254 |
visible_keys = {field["key"] for field in visible}
|
| 1255 |
evaluation_visible = [field for field in evaluation_fields if field["key"] in visible_keys]
|
api/diag_aging.py
CHANGED
|
@@ -26,22 +26,17 @@ def validate(session, rows, restrictions, *, today, documents=None):
|
|
| 26 |
scoped = ScopedOdoo(O.get_odoo(), restrictions)
|
| 27 |
helpers = scoped.helpers()
|
| 28 |
module = bind_functions(ar, {"O": helpers})
|
| 29 |
-
docs = module._open_docs(today)
|
|
|
|
| 30 |
exposures = {}
|
| 31 |
for doc in docs:
|
| 32 |
pid = helpers.m2o_id(doc["partner_id"])
|
| 33 |
-
row = exposures.setdefault(pid, {"open": 0
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
row["open"] += doc["open"]
|
| 40 |
-
# The unrelated settlement check remains false and is not part of this diagnostic.
|
| 41 |
-
existing = module.validate(today, exposure={"_all_rows": list(exposures.values())},
|
| 42 |
-
behavior={"violations": 0, "n_customers": 0})[0]
|
| 43 |
-
existing = {"check": "AR aging buckets equal open documents",
|
| 44 |
-
**{key: existing[key] for key in ("a", "b", "ok")}}
|
| 45 |
canonical = module.aging(today)
|
| 46 |
checks = [existing, {"check": "AR aging matches module buckets", "a": len(rows),
|
| 47 |
"b": len(canonical), "ok": rows == [
|
|
|
|
| 26 |
scoped = ScopedOdoo(O.get_odoo(), restrictions)
|
| 27 |
helpers = scoped.helpers()
|
| 28 |
module = bind_functions(ar, {"O": helpers})
|
| 29 |
+
docs = list(module._open_docs(today))
|
| 30 |
+
# Tie in place: ar.validate reads account.move.line, which the scoped client refuses.
|
| 31 |
exposures = {}
|
| 32 |
for doc in docs:
|
| 33 |
pid = helpers.m2o_id(doc["partner_id"])
|
| 34 |
+
row = exposures.setdefault(pid, {"open": Decimal(0), "overdue": Decimal(0)})
|
| 35 |
+
row["overdue" if doc["days_overdue"] > 5 else "open"] += amount(doc["open"])
|
| 36 |
+
exposed = sum((row["open"] + row["overdue"] for row in exposures.values()), Decimal(0))
|
| 37 |
+
population = sum((amount(doc["open"]) for doc in docs), Decimal(0))
|
| 38 |
+
existing = {"check": "AR aging buckets equal open documents", "a": float(exposed),
|
| 39 |
+
"b": float(population), "ok": abs(exposed - population) < Decimal("0.01")}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
canonical = module.aging(today)
|
| 41 |
checks = [existing, {"check": "AR aging matches module buckets", "a": len(rows),
|
| 42 |
"b": len(canonical), "ok": rows == [
|
api/external_freshness.py
CHANGED
|
@@ -82,8 +82,18 @@ def _amazon_raw(table_key, source=None):
|
|
| 82 |
return None
|
| 83 |
|
| 84 |
|
| 85 |
-
def amazon_sync_state(cur):
|
| 86 |
"""`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
return {str(entity): at for entity, at in cur.execute(
|
| 88 |
"SELECT entity, updated_at FROM _sync_state "
|
| 89 |
"WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
|
|
@@ -138,7 +148,7 @@ def _amazon_refreshes(session, keys):
|
|
| 138 |
cur = None
|
| 139 |
continue
|
| 140 |
if state is None:
|
| 141 |
-
state = amazon_sync_state(cur)
|
| 142 |
refreshed[key] = amazon_sync_stamp(cur, key, state=state)
|
| 143 |
except Exception:
|
| 144 |
continue
|
|
@@ -167,7 +177,7 @@ def amazon_nav(session, pages):
|
|
| 167 |
cur, refusal = assistant_mirror.open_cursor(session, rows[0]["key"])
|
| 168 |
if refusal is not None or cur is None:
|
| 169 |
return pages
|
| 170 |
-
state = amazon_sync_state(cur)
|
| 171 |
if not state:
|
| 172 |
# An unrestored or absent mirror holds no receipt at all: unknown, not never synced.
|
| 173 |
return pages
|
|
@@ -247,14 +257,17 @@ def snapshot(session, databases):
|
|
| 247 |
edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
|
| 248 |
edits = {}
|
| 249 |
if edited:
|
|
|
|
|
|
|
|
|
|
| 250 |
try:
|
| 251 |
-
|
|
|
|
| 252 |
except Exception:
|
| 253 |
-
|
| 254 |
-
for
|
| 255 |
-
if
|
| 256 |
-
key =
|
| 257 |
-
edits[key] = _latest(edits.get(key), event.get("at"))
|
| 258 |
receipts = {}
|
| 259 |
if connected:
|
| 260 |
try:
|
|
|
|
| 82 |
return None
|
| 83 |
|
| 84 |
|
| 85 |
+
def amazon_sync_state(cur, rt=None):
|
| 86 |
"""`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
|
| 87 |
+
if rt is not None:
|
| 88 |
+
from core import connector_sync, connector_spec
|
| 89 |
+
entry = connector_sync.status(rt, "amazon_sp", fresh=True)
|
| 90 |
+
if entry is not None:
|
| 91 |
+
last_pass = entry.get("pass") or {}
|
| 92 |
+
held = last_pass.get("held") or {}
|
| 93 |
+
stamps = last_pass.get("tableSync") or {}
|
| 94 |
+
tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES}
|
| 95 |
+
return {"amzn_sync_mark_" + name: stamp for name in tables
|
| 96 |
+
if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))}
|
| 97 |
return {str(entity): at for entity, at in cur.execute(
|
| 98 |
"SELECT entity, updated_at FROM _sync_state "
|
| 99 |
"WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
|
|
|
|
| 148 |
cur = None
|
| 149 |
continue
|
| 150 |
if state is None:
|
| 151 |
+
state = amazon_sync_state(cur, session.runtime)
|
| 152 |
refreshed[key] = amazon_sync_stamp(cur, key, state=state)
|
| 153 |
except Exception:
|
| 154 |
continue
|
|
|
|
| 177 |
cur, refusal = assistant_mirror.open_cursor(session, rows[0]["key"])
|
| 178 |
if refusal is not None or cur is None:
|
| 179 |
return pages
|
| 180 |
+
state = amazon_sync_state(cur, session.runtime)
|
| 181 |
if not state:
|
| 182 |
# An unrestored or absent mirror holds no receipt at all: unknown, not never synced.
|
| 183 |
return pages
|
|
|
|
| 257 |
edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
|
| 258 |
edits = {}
|
| 259 |
if edited:
|
| 260 |
+
# D-11777: edits append only to the history rows, so one SQL aggregate gives each table's
|
| 261 |
+
# newest edit. A store with no history backend (TypeError) or an unreachable one keeps the
|
| 262 |
+
# definition's `updated` stamp below and never fails discovery.
|
| 263 |
try:
|
| 264 |
+
from core import history_rows
|
| 265 |
+
latest = history_rows.backend(rt).latest(sorted(edited))
|
| 266 |
except Exception:
|
| 267 |
+
latest = {}
|
| 268 |
+
for key, at in latest.items() if isinstance(latest, dict) else ():
|
| 269 |
+
if key in edited:
|
| 270 |
+
edits[key] = _latest(edits.get(key), at)
|
|
|
|
| 271 |
receipts = {}
|
| 272 |
if connected:
|
| 273 |
try:
|
api/loopy_file_tools.py
CHANGED
|
@@ -55,7 +55,9 @@ def read_all(session, database, *, fields=None, filters=None, sorts=None, max_ro
|
|
| 55 |
or type(page.get("returned")) is not int or page["returned"] != len(rows)):
|
| 56 |
raise ValueError("The database did not return a complete readable page.")
|
| 57 |
page_keys = [field.get("key") for field in result.get("fields", []) if isinstance(field, dict)]
|
| 58 |
-
|
|
|
|
|
|
|
| 59 |
raise err(409, "query_cursor_stale", "The readable fields changed. Ask again to read the complete list.")
|
| 60 |
if "offset" in page and (type(page["offset"]) is not int or page["offset"] != len(records)):
|
| 61 |
raise ValueError("The database skipped rows. Read the complete result again. No file was stored.")
|
|
|
|
| 55 |
or type(page.get("returned")) is not int or page["returned"] != len(rows)):
|
| 56 |
raise ValueError("The database did not return a complete readable page.")
|
| 57 |
page_keys = [field.get("key") for field in result.get("fields", []) if isinstance(field, dict)]
|
| 58 |
+
# Field set and count, not order: query pages and the schema list fields in different orders (D-8381).
|
| 59 |
+
wanted = [key for key in by_key if key in selected]
|
| 60 |
+
if len(page_keys) != len(wanted) or set(page_keys) != set(wanted):
|
| 61 |
raise err(409, "query_cursor_stale", "The readable fields changed. Ask again to read the complete list.")
|
| 62 |
if "offset" in page and (type(page["offset"]) is not int or page["offset"] != len(records)):
|
| 63 |
raise ValueError("The database skipped rows. Read the complete result again. No file was stored.")
|
api/loopy_tools.py
CHANGED
|
@@ -4,7 +4,7 @@ import logging
|
|
| 4 |
|
| 5 |
MODULES = ("loopy_file_tools", "loopy_chat_link_tools", "loopy_skill_tools",
|
| 6 |
"loopy_catalog_tools", "loopy_diagnostic_tools", "loopy_worklist_tools",
|
| 7 |
-
"loopy_reminder_tools", "tool_groups.apps")
|
| 8 |
TOOLS = []
|
| 9 |
_LOG = logging.getLogger(__name__)
|
| 10 |
|
|
|
|
| 4 |
|
| 5 |
MODULES = ("loopy_file_tools", "loopy_chat_link_tools", "loopy_skill_tools",
|
| 6 |
"loopy_catalog_tools", "loopy_diagnostic_tools", "loopy_worklist_tools",
|
| 7 |
+
"loopy_reminder_tools", "tool_groups.apps", "tool_groups.live")
|
| 8 |
TOOLS = []
|
| 9 |
_LOG = logging.getLogger(__name__)
|
| 10 |
|
api/main.py
CHANGED
|
@@ -412,10 +412,11 @@ def _mirror_answers(rt=None):
|
|
| 412 |
|
| 413 |
def _tenant_readiness(rt):
|
| 414 |
"""Check this tenant's reads without requiring an Odoo mirror for tenants without Odoo."""
|
| 415 |
-
checks = {"mirror": False, "pools": False, "first_reads": False}
|
| 416 |
detail = {"mirror": "The tenant mirror check did not answer.",
|
| 417 |
"pools": "The tenant store did not answer.",
|
| 418 |
-
"first_reads": "The tenant's first read did not answer."
|
|
|
|
| 419 |
stage = "mirror"
|
| 420 |
try:
|
| 421 |
if rt.odoo_source() is not None:
|
|
@@ -432,6 +433,11 @@ def _tenant_readiness(rt):
|
|
| 432 |
# HF display reads hide outages, so readiness uses the tenant's strict reader.
|
| 433 |
rt._store()._read_strict(rt.store_key("sync_settings"))
|
| 434 |
checks["first_reads"] = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 435 |
except Exception as exc: # noqa: BLE001
|
| 436 |
detail[stage] += f" ({type(exc).__name__})"
|
| 437 |
failing = [name for name in checks if not checks[name]]
|
|
@@ -441,21 +447,52 @@ def _tenant_readiness(rt):
|
|
| 441 |
|
| 442 |
@app.get("/api/v1/ready")
|
| 443 |
def readiness_check(request: Request, response: Response):
|
| 444 |
-
"""Report tenant readiness publicly
|
|
|
|
| 445 |
session = (deps_require_session(request, response)
|
| 446 |
if request.cookies.get(aios_session.COOKIE_NAME) else None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 447 |
try:
|
| 448 |
from harness import runtime as _ready_runtime
|
| 449 |
if session is not None:
|
| 450 |
report = _tenant_readiness(session.runtime)
|
| 451 |
else:
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 455 |
try:
|
| 456 |
-
reports.append(_tenant_readiness(
|
| 457 |
except Exception:
|
| 458 |
reports.append({"checks": dict.fromkeys(names, False)})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 459 |
checks = {name: bool(reports) and all(r["checks"].get(name, False) for r in reports)
|
| 460 |
for name in names}
|
| 461 |
failing = [name for name in names if not checks[name]]
|
|
@@ -465,7 +502,9 @@ def readiness_check(request: Request, response: Response):
|
|
| 465 |
report = {"ready": False, "checks": {"tenant": False}, "failing": ["tenant"],
|
| 466 |
"detail": {"tenant": f"The tenant could not be resolved ({type(exc).__name__})."}}
|
| 467 |
body = {"ready": report["ready"], "checks": report["checks"]}
|
| 468 |
-
if
|
|
|
|
|
|
|
| 469 |
body["version"] = os.environ.get("AIOS_VERSION") or "unknown"
|
| 470 |
response.headers["Cache-Control"] = "no-store"
|
| 471 |
if report["ready"]:
|
|
@@ -704,6 +743,8 @@ app.include_router(routes_external.router)
|
|
| 704 |
import app_routes
|
| 705 |
_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
|
| 706 |
_mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
|
|
|
|
|
|
|
| 707 |
_mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
|
| 708 |
_mount_router("routes_sku_actions")
|
| 709 |
_mount_router("notify_destinations")
|
|
@@ -977,6 +1018,131 @@ def _migrate_env_odoo(why):
|
|
| 977 |
print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}")
|
| 978 |
|
| 979 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 980 |
def _pull_meta(why, tenant_key=None):
|
| 981 |
"""Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it.
|
| 982 |
|
|
@@ -996,22 +1162,35 @@ def _pull_meta(why, tenant_key=None):
|
|
| 996 |
outcomes = {}
|
| 997 |
try:
|
| 998 |
import process_role as _meta_role
|
|
|
|
| 999 |
# Each tenant is pulled by exactly one worker: its own, else the primary.
|
| 1000 |
tenants = [tenant_key] if tenant_key else [
|
| 1001 |
key for key in runtime.known_tenants() if _meta_role.syncs_here(key)]
|
| 1002 |
-
|
| 1003 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1004 |
try:
|
| 1005 |
-
rt = runtime.get_runtime(key)
|
| 1006 |
if not _meta._connection_rows(rt, key, {"problems": []}):
|
| 1007 |
continue
|
|
|
|
|
|
|
|
|
|
| 1008 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 1009 |
log=lambda *_a: None, insights_days=7)
|
| 1010 |
outcomes[key] = mirror_store._sync_result("meta", rep)[1]
|
|
|
|
| 1011 |
for problem in rep.get("problems") or []:
|
| 1012 |
print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {problem}")
|
| 1013 |
except Exception as exc:
|
| 1014 |
print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
|
|
|
|
|
|
|
|
|
|
| 1015 |
except Exception as exc:
|
| 1016 |
print(f"[aios-api] meta scheduling FAILED ({why}): {type(exc).__name__}")
|
| 1017 |
return outcomes.get(tenant_key or runtime.tenant_for_store_path(), False)
|
|
@@ -1101,6 +1280,9 @@ def _pull_amazon(why, vendor=True):
|
|
| 1101 |
print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential "
|
| 1102 |
f"for {tenant_key}; the connector is idle, not broken")
|
| 1103 |
return False
|
|
|
|
|
|
|
|
|
|
| 1104 |
# Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
|
| 1105 |
# bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
|
| 1106 |
# ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
|
|
@@ -1118,17 +1300,28 @@ def _pull_amazon(why, vendor=True):
|
|
| 1118 |
_rot.invalidate_sources()
|
| 1119 |
_rot._sources()
|
| 1120 |
if first_load_needed:
|
| 1121 |
-
|
| 1122 |
-
|
| 1123 |
-
|
| 1124 |
-
|
| 1125 |
if not vendor:
|
| 1126 |
print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
|
| 1127 |
return True
|
| 1128 |
from harness import mirror_store
|
| 1129 |
-
|
| 1130 |
-
|
| 1131 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1132 |
_rot.invalidate_sources()
|
| 1133 |
_rot._sources()
|
| 1134 |
for problem in rep.get("problems") or []:
|
|
@@ -1143,7 +1336,7 @@ def _pull_amazon(why, vendor=True):
|
|
| 1143 |
store=rt, quiet=_sa.amazon_standing(rep))
|
| 1144 |
except Exception: # noqa: BLE001
|
| 1145 |
pass
|
| 1146 |
-
return not bool(rep.get("
|
| 1147 |
except Exception as exc: # noqa: BLE001
|
| 1148 |
print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}")
|
| 1149 |
return False
|
|
@@ -1295,6 +1488,17 @@ def _sync_now_cause(cause):
|
|
| 1295 |
"""Classify actual provider failures without returning vendor text or exception reprs."""
|
| 1296 |
import re
|
| 1297 |
if isinstance(cause, dict):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1298 |
sentence = "The connector could not read an expected value in the sync response."
|
| 1299 |
if cause.get("classifier") == "unreadable_value":
|
| 1300 |
table = cause.get("table")
|
|
@@ -1311,6 +1515,8 @@ def _sync_now_cause(cause):
|
|
| 1311 |
raw = str(cause)
|
| 1312 |
text = raw.lower()
|
| 1313 |
causes = (
|
|
|
|
|
|
|
| 1314 |
("response_too_large", ("reduce the amount of data", "response too large"),
|
| 1315 |
"The provider refused the pull because the response contains too much data."),
|
| 1316 |
("meta_no_ad_accounts", ("no ad accounts",), "The Meta token cannot access any ad accounts."),
|
|
@@ -1389,7 +1595,7 @@ def _sync_now_failure(cause):
|
|
| 1389 |
return _sync_now_cause(cause)[1]
|
| 1390 |
|
| 1391 |
|
| 1392 |
-
def _amazon_sync_receipt(report):
|
| 1393 |
"""Project table windows and resync counts without vendor diagnostics or exception values."""
|
| 1394 |
import re
|
| 1395 |
details = []
|
|
@@ -1397,6 +1603,8 @@ def _amazon_sync_receipt(report):
|
|
| 1397 |
if not isinstance(row, dict) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", str(table)):
|
| 1398 |
continue
|
| 1399 |
parts = []
|
|
|
|
|
|
|
| 1400 |
progress = row.get("resync")
|
| 1401 |
if isinstance(progress, dict) and all(type(progress.get(k)) is int and progress[k] >= 0
|
| 1402 |
for k in ("landed", "left")):
|
|
@@ -1405,8 +1613,27 @@ def _amazon_sync_receipt(report):
|
|
| 1405 |
if isinstance(window, str):
|
| 1406 |
# Window descriptions are authored by the loader, never copied from its diagnostics.
|
| 1407 |
parts.append(" ".join(window.split()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1408 |
if parts:
|
| 1409 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1410 |
return " | ".join(details)
|
| 1411 |
|
| 1412 |
|
|
@@ -1415,6 +1642,10 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
|
|
| 1415 |
if ok:
|
| 1416 |
return detail
|
| 1417 |
classifier, sentence = _sync_now_cause(cause)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1418 |
print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
|
| 1419 |
if isinstance(cause, BaseException):
|
| 1420 |
import traceback
|
|
@@ -1512,6 +1743,31 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1512 |
|
| 1513 |
return log, lines
|
| 1514 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1515 |
odoo_requested = False
|
| 1516 |
# A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
|
| 1517 |
hold = during_odoo if hold_first_load is None else hold_first_load
|
|
@@ -1531,8 +1787,10 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1531 |
ok, detail = False, ""
|
| 1532 |
first_load_progress, continuing = None, False
|
| 1533 |
floor_wait = None
|
|
|
|
| 1534 |
progress = ()
|
| 1535 |
amazon_detail, amazon_failure = "", None
|
|
|
|
| 1536 |
try:
|
| 1537 |
from harness import runtime, datastore, mirror_store
|
| 1538 |
rt = runtime.get_runtime(tenant)
|
|
@@ -1544,6 +1802,12 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1544 |
datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()):
|
| 1545 |
raise _SyncNowRefused(
|
| 1546 |
"This tenant's mirror is not resident on this server. Sync requires its tenant worker.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1547 |
connector_sync.poll_remote_requests(rt)
|
| 1548 |
if provider == "odoo":
|
| 1549 |
import core.odoo as odoo
|
|
@@ -1574,6 +1838,8 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1574 |
problems = []
|
| 1575 |
elif provider == "amazon_sp":
|
| 1576 |
from harness import amazon_store as amazon
|
|
|
|
|
|
|
| 1577 |
_drain_amazon_connection_revocations(rt)
|
| 1578 |
if not amazon.credential(tenant, rt=rt):
|
| 1579 |
raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
|
|
@@ -1581,13 +1847,14 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1581 |
target_options = notification_sync_options(receipt)
|
| 1582 |
targeted = bool(target_options)
|
| 1583 |
first_load = False if targeted else amazon.needs_first_load(tenant, rt=rt)
|
| 1584 |
-
options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS
|
| 1585 |
-
"refresh_loaded": True} if first_load else {}
|
| 1586 |
_log, progress = progress_log(provider, tenant, receipt, rt)
|
| 1587 |
-
|
| 1588 |
-
|
| 1589 |
-
|
| 1590 |
-
|
|
|
|
|
|
|
| 1591 |
print("[aios-api] amazon sync done (sync desk): " + ", ".join(
|
| 1592 |
f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
|
| 1593 |
+ (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left"
|
|
@@ -1599,11 +1866,17 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1599 |
# HOTFIX 2026-09-29: a continuation that landed nothing only met refresh floors.
|
| 1600 |
if continuing and report.get("landedTasks") == 0 and report.get("nextDueSeconds"):
|
| 1601 |
floor_wait = float(report["nextDueSeconds"])
|
| 1602 |
-
|
|
|
|
| 1603 |
tables = report.get("tables") or {}
|
| 1604 |
-
amazon_detail = _amazon_sync_receipt(report)
|
| 1605 |
amazon_failure = next((row["failure"] for row in tables.values()
|
| 1606 |
if isinstance(row, dict) and isinstance(row.get("failure"), dict)), None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1607 |
if not problems and not tables and not continuing:
|
| 1608 |
raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.")
|
| 1609 |
if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()):
|
|
@@ -1621,7 +1894,15 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1621 |
raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.")
|
| 1622 |
else:
|
| 1623 |
raise _SyncNowRefused("This connector has no Sync now runner.")
|
| 1624 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1625 |
ok = True
|
| 1626 |
detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; "
|
| 1627 |
f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.")
|
|
@@ -1652,9 +1933,22 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1652 |
first_load=first_load_progress, request_id=receipt.get("id"))
|
| 1653 |
else:
|
| 1654 |
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, request_id=receipt.get("id"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1655 |
if outcome is not None:
|
| 1656 |
-
outcome[provider] =
|
| 1657 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1658 |
_project_desk({provider: "done"}, tenant)
|
| 1659 |
if continuing and ok:
|
| 1660 |
# The continuation takes this run's desk slot: publish the finish to its web hand-off first.
|
|
@@ -1706,7 +2000,7 @@ def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations
|
|
| 1706 |
if state["present"] and (state["changed"] or force_relations):
|
| 1707 |
try:
|
| 1708 |
import automation_engine as _engine
|
| 1709 |
-
_engine.refresh_relations(rt, log=
|
| 1710 |
except Exception as e: # noqa: BLE001
|
| 1711 |
print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}")
|
| 1712 |
return state
|
|
@@ -1959,7 +2253,7 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1959 |
# refusal silently cancel a relational pass that had already succeeded.
|
| 1960 |
try:
|
| 1961 |
import automation_engine as _engine
|
| 1962 |
-
_engine.refresh_relations(_rt, log=
|
| 1963 |
except Exception as e: # noqa: BLE001
|
| 1964 |
rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
|
| 1965 |
print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
|
|
@@ -2938,6 +3232,12 @@ def _sheet_warm_after_pass(tenant, good, generation_before, rebuilt):
|
|
| 2938 |
print(f"[sheet-warm] request failed: {type(exc).__name__}: {exc}")
|
| 2939 |
|
| 2940 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2941 |
def _store_resync_loop():
|
| 2942 |
"""Keep the analytical mirror current — the API-process stand-in for the re-sync the
|
| 2943 |
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
|
@@ -2984,15 +3284,18 @@ def _store_resync_loop():
|
|
| 2984 |
# H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
|
| 2985 |
slot_resume_checked = False
|
| 2986 |
_slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
|
|
|
|
| 2987 |
_slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
|
| 2988 |
_slot_interrupted_clear = globals().get("_resync_slot_interrupted_clear") or (lambda *_a, **_k: False)
|
| 2989 |
-
#
|
| 2990 |
_backfill_read = globals().get("_odoo_backfill_checkpoints")
|
| 2991 |
backfill_next_at = None
|
| 2992 |
backfill_continuations = 0
|
| 2993 |
backfill_spacing_seconds = 60.0
|
| 2994 |
-
backfill_cap =
|
| 2995 |
while True:
|
|
|
|
|
|
|
| 2996 |
# ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
|
| 2997 |
#
|
| 2998 |
# ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 —
|
|
@@ -3078,7 +3381,7 @@ def _store_resync_loop():
|
|
| 3078 |
# full pass too, so passes start a day apart however long the pass ran (a daily pass
|
| 3079 |
# that also refreshes every Amazon table can run an hour, and a wait counted from its
|
| 3080 |
# END drifted later every day and past the 26-hour stale line).
|
| 3081 |
-
_woken = bool(connector_sync.sync_wake_event()
|
| 3082 |
max(0.0, _every - (_t.time() - last_full_pass_at)),
|
| 3083 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _t.time()))))
|
| 3084 |
connector_sync.sync_wake_event().clear()
|
|
@@ -3090,7 +3393,7 @@ def _store_resync_loop():
|
|
| 3090 |
if passes or slot_restored:
|
| 3091 |
# Sleep to one second past the next slot; Sync now still interrupts the wait.
|
| 3092 |
_slot_now = _t.time()
|
| 3093 |
-
_woken = bool(connector_sync.sync_wake_event()
|
| 3094 |
0.0 if _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
|
| 3095 |
else max(0.0, _rel_slot.slot_after(_slot_now, _slot) + 1.0 - _slot_now),
|
| 3096 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _slot_now))))
|
|
@@ -3145,7 +3448,7 @@ def _store_resync_loop():
|
|
| 3145 |
if _woken and (_t.time() - last_full_pass_at < _every if _slot is None else not _slot_due):
|
| 3146 |
_desk = {}
|
| 3147 |
_desk_started = _t.time()
|
| 3148 |
-
if not _run_sync_requests(_desk, hold_first_load=
|
| 3149 |
_project_desk(_desk, _cad["tenant"])
|
| 3150 |
if "continuing" in _desk.values():
|
| 3151 |
# A floor between continuations, so a chunk that returns at once cannot spin.
|
|
@@ -3156,12 +3459,13 @@ def _store_resync_loop():
|
|
| 3156 |
_odoo_requested = None
|
| 3157 |
# H1: a queued backfill pass that is due, unless the slot, the cadence or a Sync now runs.
|
| 3158 |
_continuation = (backfill_next_at is not None and _t.time() >= backfill_next_at - 1.0
|
|
|
|
| 3159 |
and _odoo_requested is None and not _slot_due
|
| 3160 |
and (_slot is not None or _t.time() - last_full_pass_at < _every))
|
| 3161 |
if _slot is not None and not _slot_due and _odoo_requested is None and not _continuation:
|
| 3162 |
# H73: outside the slot no Odoo call is made. Serve the stored mirror, drain the desk.
|
| 3163 |
_desk = {}
|
| 3164 |
-
if _run_sync_requests(_desk, hold_first_load=
|
| 3165 |
_odoo_requested = True
|
| 3166 |
else:
|
| 3167 |
_project_desk(_desk, _cad["tenant"])
|
|
@@ -3184,10 +3488,20 @@ def _store_resync_loop():
|
|
| 3184 |
# H73: stamp the slot BEFORE any Odoo read. H2: `slotDoneAt` marks its end, any outcome.
|
| 3185 |
last_slot_pass_at = last_full_pass_at
|
| 3186 |
_slot_write(last_full_pass_at)
|
| 3187 |
-
|
| 3188 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3189 |
_warm_generation = _sheet_warm_generation()
|
| 3190 |
_backfill_before = _backfill_read(_cad["tenant"]) if _backfill_read else None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3191 |
try:
|
| 3192 |
selected_entities = _configured_mirror_entities(_cad["tenant"])
|
| 3193 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
|
|
@@ -3225,6 +3539,10 @@ def _store_resync_loop():
|
|
| 3225 |
print(f"[aios-api] store resync failed ({_err['kind']}, "
|
| 3226 |
f"{RESYNC_STATE['consecutiveFailures']} in a row): "
|
| 3227 |
f"{_err['type']}: {_err['message']}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3228 |
if _publication_held:
|
| 3229 |
print("[aios-api] store resync: publication held; no vendor pulls or relational rebuild; "
|
| 3230 |
"the next admitted pass resumes the unfinished Odoo cohort")
|
|
@@ -3234,8 +3552,8 @@ def _store_resync_loop():
|
|
| 3234 |
_backfill_after = _backfill_read(_cad["tenant"]) if _backfill_read else None
|
| 3235 |
_moved = ([row for row in _backfill_after if row not in _backfill_before]
|
| 3236 |
if _err is None and _backfill_after and _backfill_before is not None else [])
|
| 3237 |
-
if _moved and
|
| 3238 |
-
print(f"[aios-api] odoo backfill stops for today:
|
| 3239 |
f"the next slot pass or Sync now resumes it")
|
| 3240 |
elif _moved:
|
| 3241 |
backfill_continuations += 1
|
|
@@ -3243,7 +3561,7 @@ def _store_resync_loop():
|
|
| 3243 |
_row = max(_moved, key=lambda row: row[4] or 0)
|
| 3244 |
print(f"[aios-api] odoo backfill continues: {str(_row[0]).split('.', 1)[0]} at "
|
| 3245 |
f"{_row[4] or 0:,} rows; next bounded pass queued "
|
| 3246 |
-
f"({backfill_continuations}
|
| 3247 |
continue
|
| 3248 |
if _boot_catch_up and not _odoo_requested:
|
| 3249 |
# Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
|
|
@@ -3476,6 +3794,28 @@ def _boot_app_migrations(why):
|
|
| 3476 |
print(f"[aios-api] Returns split ({why}) {slug}: {split['outcome']}")
|
| 3477 |
except Exception as exc:
|
| 3478 |
print(f"[aios-api] Returns split ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3479 |
try:
|
| 3480 |
result = application_registry.ensure_route_application(rt, slug)
|
| 3481 |
reason = f": {result['reason']}" if result.get("reason") else ""
|
|
@@ -3493,6 +3833,97 @@ def _boot_app_migrations(why):
|
|
| 3493 |
print(f"[aios-api] Command Center App ({why}) {slug}: skipped: {type(exc).__name__}")
|
| 3494 |
|
| 3495 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3496 |
def _prewarm():
|
| 3497 |
if _freeze_holds("prewarm"):
|
| 3498 |
return
|
|
@@ -3565,17 +3996,35 @@ def _seed_mirror_if_absent():
|
|
| 3565 |
from harness import runtime as _runtime
|
| 3566 |
MIRROR_SEED["attempted"] = True
|
| 3567 |
try:
|
| 3568 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3569 |
try:
|
| 3570 |
-
rt = _runtime.get_runtime(tenant_key)
|
| 3571 |
result = _ds.restore_odoo_mirror(tenant_key, rt)
|
| 3572 |
except Exception as exc:
|
| 3573 |
-
result =
|
| 3574 |
-
f"Odoo mirror restore failed for {tenant_key} ({type(exc).__name__})"]}
|
| 3575 |
-
_ds.ODOO_MIRROR_STATUS[str(_ds.path_for(tenant_key).resolve())] = result
|
| 3576 |
if result["state"] == "refused":
|
| 3577 |
MIRROR_SEED["cause"] = "; ".join(result["problems"])
|
| 3578 |
MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3579 |
if _ds.ensure_seed():
|
| 3580 |
MIRROR_SEED["seeded"] = True
|
| 3581 |
print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
|
|
@@ -3681,7 +4130,7 @@ def _await_mirror_restore():
|
|
| 3681 |
|
| 3682 |
@app.on_event("startup")
|
| 3683 |
def _start_process_role():
|
| 3684 |
-
"""Web role:
|
| 3685 |
if data_binding.gate_process():
|
| 3686 |
return
|
| 3687 |
import process_role as _process_role
|
|
@@ -3694,12 +4143,22 @@ def _restore_sync_request_desks():
|
|
| 3694 |
import process_role as _process_role
|
| 3695 |
# Only the sync writer requeues in-flight claims: never the web role, never a worker without the lease.
|
| 3696 |
requeue = _process_role.may_write()
|
| 3697 |
-
|
|
|
|
|
|
|
|
|
|
| 3698 |
try:
|
| 3699 |
-
connector_sync.restore_sync_requests(
|
| 3700 |
except Exception as exc:
|
| 3701 |
print(f"[aios-api] sync request restore refused for {tenant_key} ({type(exc).__name__})")
|
| 3702 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3703 |
|
| 3704 |
def _boot_mirrors():
|
| 3705 |
import process_role as _role_boot
|
|
@@ -3737,6 +4196,53 @@ _READ_WARM_ATTEMPTS = 40
|
|
| 3737 |
_READ_WARM_RETRY_SECONDS = 15
|
| 3738 |
|
| 3739 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3740 |
def _probe_read_pools(rt):
|
| 3741 |
"""Open the mirror read cursor and the tenant store once. '' when both answer, else the cause."""
|
| 3742 |
from harness import datastore as _probe_ds
|
|
@@ -3745,11 +4251,18 @@ def _probe_read_pools(rt):
|
|
| 3745 |
physical = _probe_ds.serving_file(path)[0] if _probe_ds.versioned_mode() else path
|
| 3746 |
if not physical.exists():
|
| 3747 |
return "The mirror file is missing."
|
| 3748 |
-
|
| 3749 |
-
|
| 3750 |
-
|
| 3751 |
-
|
| 3752 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3753 |
if rt.available() is False:
|
| 3754 |
return "The tenant store did not answer."
|
| 3755 |
return ""
|
|
@@ -3764,6 +4277,7 @@ def _warm_read_pools(attempts=None, retry_seconds=None):
|
|
| 3764 |
import process_role as _warm_role
|
| 3765 |
attempts = _READ_WARM_ATTEMPTS if attempts is None else max(1, int(attempts))
|
| 3766 |
retry_seconds = _READ_WARM_RETRY_SECONDS if retry_seconds is None else retry_seconds
|
|
|
|
| 3767 |
for attempt in range(1, attempts + 1):
|
| 3768 |
t0 = _t.time()
|
| 3769 |
stage = "pools"
|
|
@@ -3776,13 +4290,23 @@ def _warm_read_pools(attempts=None, retry_seconds=None):
|
|
| 3776 |
raise RuntimeError(problem)
|
| 3777 |
_warm_role.mark_ready("pools", True)
|
| 3778 |
stage = "first_reads"
|
| 3779 |
-
|
| 3780 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3781 |
_warm_role.mark_ready("first_reads", True)
|
| 3782 |
routes_customers.warm_bu_pools(rt)
|
| 3783 |
print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
|
| 3784 |
return True
|
| 3785 |
except Exception as exc: # noqa: BLE001
|
|
|
|
|
|
|
|
|
|
| 3786 |
if not _warm_role._READINESS.get(stage, {}).get("ok"):
|
| 3787 |
_warm_role.mark_ready(stage, False, f"The {'read pools' if stage == 'pools' else 'first reads'} "
|
| 3788 |
f"failed ({type(exc).__name__}); attempt {attempt} of {attempts}.")
|
|
@@ -3797,14 +4321,16 @@ def _restore_declared_mirrors(why):
|
|
| 3797 |
"""Restore each tenant's declarations after the Odoo seed and Amazon policy adapters."""
|
| 3798 |
from core import connector_spec
|
| 3799 |
from harness import mirror_store, runtime
|
|
|
|
|
|
|
| 3800 |
errors = RESYNC_STATE.setdefault("mirrorRestoreErrors", {})
|
| 3801 |
-
|
|
|
|
| 3802 |
for connector in tuple(connector_spec.MIRRORS):
|
| 3803 |
if connector in {"odoo", "amazon"}:
|
| 3804 |
continue # Their existing adapters also enforce seed and deletion policies.
|
| 3805 |
identity = f"{tenant_key}/{connector}"
|
| 3806 |
try:
|
| 3807 |
-
rt = runtime.get_runtime(tenant_key)
|
| 3808 |
result = mirror_store.restore_mirror(connector, tenant_key, rt)
|
| 3809 |
if result["problems"]:
|
| 3810 |
errors[identity] = {"state": result["state"], "problems": list(result["problems"])}
|
|
@@ -3813,9 +4339,25 @@ def _restore_declared_mirrors(why):
|
|
| 3813 |
print(f"[aios-api] {connector} mirror {tenant_key} {result['state']} ({why})")
|
| 3814 |
except Exception as exc:
|
| 3815 |
errors[identity] = {"state": "refused", "problems": [type(exc).__name__]}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3816 |
return not errors
|
| 3817 |
|
| 3818 |
|
|
|
|
|
|
|
|
|
|
| 3819 |
try:
|
| 3820 |
from harness import datastore as _ds_boot
|
| 3821 |
if not data_binding.gate_process() and (data_binding.is_space() or not _ds_boot.DB_PATH.exists()):
|
|
@@ -3852,9 +4394,13 @@ if os.environ.get("AIOS_BOOT_SYNC") == "1" and freeze_notice.enabled():
|
|
| 3852 |
if not data_binding.gate_process() and not _freeze_holds("Amazon App workers"):
|
| 3853 |
import marketing_stream_loop
|
| 3854 |
import ads_schedule_loop
|
|
|
|
|
|
|
| 3855 |
import routes_returns_refunds
|
| 3856 |
marketing_stream_loop.start()
|
| 3857 |
ads_schedule_loop.start()
|
|
|
|
|
|
|
| 3858 |
routes_returns_refunds.start_auto_approve_worker()
|
| 3859 |
|
| 3860 |
|
|
@@ -3862,8 +4408,10 @@ if not data_binding.gate_process() and not _freeze_holds("Amazon App workers"):
|
|
| 3862 |
def _stop_amazon_app_workers():
|
| 3863 |
import marketing_stream_loop
|
| 3864 |
import ads_schedule_loop
|
|
|
|
| 3865 |
marketing_stream_loop.stop()
|
| 3866 |
ads_schedule_loop.stop()
|
|
|
|
| 3867 |
|
| 3868 |
|
| 3869 |
if (not data_binding.gate_process()
|
|
|
|
| 412 |
|
| 413 |
def _tenant_readiness(rt):
|
| 414 |
"""Check this tenant's reads without requiring an Odoo mirror for tenants without Odoo."""
|
| 415 |
+
checks = {"mirror": False, "pools": False, "first_reads": False, "tenant_keys": False}
|
| 416 |
detail = {"mirror": "The tenant mirror check did not answer.",
|
| 417 |
"pools": "The tenant store did not answer.",
|
| 418 |
+
"first_reads": "The tenant's first read did not answer.",
|
| 419 |
+
"tenant_keys": "The tenant data keys did not answer."}
|
| 420 |
stage = "mirror"
|
| 421 |
try:
|
| 422 |
if rt.odoo_source() is not None:
|
|
|
|
| 433 |
# HF display reads hide outages, so readiness uses the tenant's strict reader.
|
| 434 |
rt._store()._read_strict(rt.store_key("sync_settings"))
|
| 435 |
checks["first_reads"] = True
|
| 436 |
+
stage = "tenant_keys"
|
| 437 |
+
from core import tenant_keys as _tenant_keys
|
| 438 |
+
checks["tenant_keys"], _keys_detail = _tenant_keys.readiness_check([rt.key])
|
| 439 |
+
if not checks["tenant_keys"]:
|
| 440 |
+
detail["tenant_keys"] = _keys_detail
|
| 441 |
except Exception as exc: # noqa: BLE001
|
| 442 |
detail[stage] += f" ({type(exc).__name__})"
|
| 443 |
failing = [name for name in checks if not checks[name]]
|
|
|
|
| 447 |
|
| 448 |
@app.get("/api/v1/ready")
|
| 449 |
def readiness_check(request: Request, response: Response):
|
| 450 |
+
"""Report tenant readiness publicly; only a verified session or the deploy nonce sees the version."""
|
| 451 |
+
import hmac as _ready_hmac
|
| 452 |
session = (deps_require_session(request, response)
|
| 453 |
if request.cookies.get(aios_session.COOKIE_NAME) else None)
|
| 454 |
+
# A deploy probe proves itself with the deploy nonce; it sees the version, never a tenant's data.
|
| 455 |
+
nonce = os.environ.get("AIOS_DEPLOY_NONCE") or ""
|
| 456 |
+
probe = bool(nonce) and _ready_hmac.compare_digest(
|
| 457 |
+
request.headers.get("x-aios-deploy-nonce") or "", nonce)
|
| 458 |
+
skipped = {}
|
| 459 |
try:
|
| 460 |
from harness import runtime as _ready_runtime
|
| 461 |
if session is not None:
|
| 462 |
report = _tenant_readiness(session.runtime)
|
| 463 |
else:
|
| 464 |
+
import tenant_scan as _ready_scan
|
| 465 |
+
names = ("mirror", "pools", "first_reads", "tenant_keys")
|
| 466 |
+
reports, unmirrored, resolved = [], [], set()
|
| 467 |
+
|
| 468 |
+
def failed(slug, _exc):
|
| 469 |
+
# An active tenant whose runtime fails is judged, and fails closed.
|
| 470 |
+
resolved.add(slug)
|
| 471 |
+
reports.append({"checks": dict.fromkeys(names, False)})
|
| 472 |
+
|
| 473 |
+
# R70: only active tenants with a declared mirror are judged; the others are named.
|
| 474 |
+
for key, tenant_rt in _ready_scan.live_tenants(_ready_runtime, on_error=failed):
|
| 475 |
+
resolved.add(key)
|
| 476 |
+
try:
|
| 477 |
+
mirrored = tenant_rt.odoo_source() is not None
|
| 478 |
+
except Exception:
|
| 479 |
+
mirrored = True
|
| 480 |
+
if not mirrored:
|
| 481 |
+
unmirrored.append((key, tenant_rt))
|
| 482 |
+
skipped[key] = "no mirror"
|
| 483 |
+
continue
|
| 484 |
try:
|
| 485 |
+
reports.append(_tenant_readiness(tenant_rt))
|
| 486 |
except Exception:
|
| 487 |
reports.append({"checks": dict.fromkeys(names, False)})
|
| 488 |
+
for key in _ready_runtime.known_tenants():
|
| 489 |
+
if key not in resolved:
|
| 490 |
+
skipped[key] = "inactive"
|
| 491 |
+
if not reports:
|
| 492 |
+
# No tenant declares a mirror, so the active tenants' own stores answer for the process.
|
| 493 |
+
for key, tenant_rt in unmirrored:
|
| 494 |
+
skipped.pop(key, None)
|
| 495 |
+
reports.append(_tenant_readiness(tenant_rt))
|
| 496 |
checks = {name: bool(reports) and all(r["checks"].get(name, False) for r in reports)
|
| 497 |
for name in names}
|
| 498 |
failing = [name for name in names if not checks[name]]
|
|
|
|
| 502 |
report = {"ready": False, "checks": {"tenant": False}, "failing": ["tenant"],
|
| 503 |
"detail": {"tenant": f"The tenant could not be resolved ({type(exc).__name__})."}}
|
| 504 |
body = {"ready": report["ready"], "checks": report["checks"]}
|
| 505 |
+
if skipped:
|
| 506 |
+
body["skipped"] = skipped
|
| 507 |
+
if session is not None or probe:
|
| 508 |
body["version"] = os.environ.get("AIOS_VERSION") or "unknown"
|
| 509 |
response.headers["Cache-Control"] = "no-store"
|
| 510 |
if report["ready"]:
|
|
|
|
| 743 |
import app_routes
|
| 744 |
_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
|
| 745 |
_mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
|
| 746 |
+
_mount_router("routes_amazon_projects")
|
| 747 |
+
_mount_router("routes_competitors")
|
| 748 |
_mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
|
| 749 |
_mount_router("routes_sku_actions")
|
| 750 |
_mount_router("notify_destinations")
|
|
|
|
| 1018 |
print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}")
|
| 1019 |
|
| 1020 |
|
| 1021 |
+
def _publish_connector_status(rt, provider, outcome=None, *, detail="", connection_id=None, report=None):
|
| 1022 |
+
"""C-status: write a pass end and the mirror's completed sync to the tenant's status document.
|
| 1023 |
+
|
| 1024 |
+
`outcome=None` writes only the completed sync, so a restored mirror seeds the store. Odoo's
|
| 1025 |
+
good sync is the pass end itself. Never raises: a status write cannot fail a pass.
|
| 1026 |
+
"""
|
| 1027 |
+
try:
|
| 1028 |
+
from core import connector_sync as _cs
|
| 1029 |
+
good = None
|
| 1030 |
+
if provider == "odoo":
|
| 1031 |
+
good = _cs._utc_stamp() if outcome == "done" else None
|
| 1032 |
+
else:
|
| 1033 |
+
import routes_connectors as _rc
|
| 1034 |
+
stamp, known = _rc._mirror_completed_sync(rt, provider)
|
| 1035 |
+
good = stamp if known else None
|
| 1036 |
+
if outcome is None:
|
| 1037 |
+
return _cs.record_good_sync(rt, provider, good, connection_id=connection_id) if good else False
|
| 1038 |
+
extra = {}
|
| 1039 |
+
if (provider == "odoo" or provider == "amazon_sp" and RESYNC_STATE.get("slotPass")) and RESYNC_STATE.get("slotDate"):
|
| 1040 |
+
extra["slotDate"] = RESYNC_STATE["slotDate"]
|
| 1041 |
+
if provider == "amazon_sp" and isinstance(report, dict):
|
| 1042 |
+
from harness import amazon_store as _az
|
| 1043 |
+
holds = _az._MIRROR_STORE._publication_holds(report, ()) or {}
|
| 1044 |
+
extra["notOffered"] = len(holds.get("standing") or ())
|
| 1045 |
+
extra["notOfferedTables"] = sorted(holds.get("standing") or ())
|
| 1046 |
+
extra["held"] = {name: {"reason": "Amazon does not offer this report to this seller."
|
| 1047 |
+
if name in (holds.get("standing") or ())
|
| 1048 |
+
else "This table did not finish. Its last published rows are kept."}
|
| 1049 |
+
for name in holds.get("held") or {}}
|
| 1050 |
+
previous = _cs.status(rt, provider, connection_id, fresh=True) or {}
|
| 1051 |
+
stamps = dict((previous.get("pass") or {}).get("tableSync") or {})
|
| 1052 |
+
for name, row in (report.get("tables") or {}).items():
|
| 1053 |
+
if name not in extra["held"] and isinstance(row, dict) and row.get("state", "done") == "done":
|
| 1054 |
+
stamps[name] = good or _cs._utc_stamp()
|
| 1055 |
+
extra["tableSync"] = stamps
|
| 1056 |
+
if extra["held"]:
|
| 1057 |
+
from harness.runtime import mirror_cursor
|
| 1058 |
+
cur = None
|
| 1059 |
+
try:
|
| 1060 |
+
cur = mirror_cursor(rt, require_ready=False)
|
| 1061 |
+
for name, held in extra["held"].items():
|
| 1062 |
+
table = _az.table_name(name)
|
| 1063 |
+
if _az._MIRROR_STORE._valid_table(table):
|
| 1064 |
+
try:
|
| 1065 |
+
held["rows"] = int(cur.execute(f'SELECT count(*) FROM "{table}"').fetchone()[0])
|
| 1066 |
+
except Exception:
|
| 1067 |
+
held["rows"] = None
|
| 1068 |
+
except Exception:
|
| 1069 |
+
for held in extra["held"].values():
|
| 1070 |
+
held["rows"] = None
|
| 1071 |
+
finally:
|
| 1072 |
+
if cur is not None:
|
| 1073 |
+
cur.close()
|
| 1074 |
+
return _cs.record_pass_end(rt, provider, outcome, detail=detail, last_good_sync_at=good,
|
| 1075 |
+
connection_id=connection_id, extra=extra)
|
| 1076 |
+
except Exception as exc: # noqa: BLE001
|
| 1077 |
+
print(f"[aios-api] connector status not written for {provider} ({type(exc).__name__})", flush=True)
|
| 1078 |
+
return False
|
| 1079 |
+
|
| 1080 |
+
|
| 1081 |
+
#: The last resync snapshot written to the store, so an unchanged loop state is not rewritten.
|
| 1082 |
+
_RESYNC_PUBLISHED = [None]
|
| 1083 |
+
|
| 1084 |
+
|
| 1085 |
+
def _publish_resync_state(tenant_key, *, started=False, ended=False, outcome=None):
|
| 1086 |
+
"""C-status: copy RESYNC_STATE, and the Odoo pass it ran, to the tenant's status document."""
|
| 1087 |
+
try:
|
| 1088 |
+
tenant_key = str(tenant_key or "")
|
| 1089 |
+
if not tenant_key:
|
| 1090 |
+
return False
|
| 1091 |
+
from core import connector_sync as _cs
|
| 1092 |
+
from harness import runtime as _status_runtime
|
| 1093 |
+
rt = _status_runtime.get_runtime(tenant_key)
|
| 1094 |
+
cid = str(RESYNC_STATE.get("connectionId") or "default")
|
| 1095 |
+
if started:
|
| 1096 |
+
_cs.record_pass_start(rt, "odoo", connection_id=cid)
|
| 1097 |
+
if ended:
|
| 1098 |
+
error = RESYNC_STATE.get("lastError")
|
| 1099 |
+
_publish_connector_status(rt, "odoo", outcome, connection_id=cid,
|
| 1100 |
+
detail=str(error.get("message") or "") if isinstance(error, dict) else "")
|
| 1101 |
+
import json as _json
|
| 1102 |
+
snapshot = _json.dumps(RESYNC_STATE, sort_keys=True, default=str)
|
| 1103 |
+
if snapshot != _RESYNC_PUBLISHED[0] and _cs.record_resync_state(
|
| 1104 |
+
rt, "odoo", RESYNC_STATE, connection_id=cid):
|
| 1105 |
+
_RESYNC_PUBLISHED[0] = snapshot
|
| 1106 |
+
return True
|
| 1107 |
+
except Exception as exc: # noqa: BLE001
|
| 1108 |
+
print(f"[aios-api] resync status not written ({type(exc).__name__})", flush=True)
|
| 1109 |
+
return False
|
| 1110 |
+
|
| 1111 |
+
|
| 1112 |
+
def _connector_publication_status(connector, tenant_key, manifest, _rows, _paths):
|
| 1113 |
+
"""Close a running pass at publication, before Amazon queues its App callbacks."""
|
| 1114 |
+
if not process_role.may_write() or not manifest.get("snapshot_id"):
|
| 1115 |
+
return
|
| 1116 |
+
provider = {"odoo": "odoo", "amazon": "amazon_sp", "meta": "meta_ads"}.get(connector)
|
| 1117 |
+
if provider is None:
|
| 1118 |
+
return
|
| 1119 |
+
from core import connector_sync
|
| 1120 |
+
from harness import runtime
|
| 1121 |
+
rt = runtime.get_runtime(tenant_key)
|
| 1122 |
+
cid = None
|
| 1123 |
+
if provider == "odoo":
|
| 1124 |
+
import odoo_relational
|
| 1125 |
+
cid = str(odoo_relational.default_connection_id(rt) or "default")
|
| 1126 |
+
entry = connector_sync.status(rt, provider, cid, fresh=True) or {}
|
| 1127 |
+
last_pass = entry.get("pass") or {}
|
| 1128 |
+
if last_pass.get("outcome") != "running":
|
| 1129 |
+
return
|
| 1130 |
+
from datetime import datetime
|
| 1131 |
+
started = connector_sync._utc_stamp(last_pass["startedAt"]) if last_pass.get("startedAt") else None
|
| 1132 |
+
at = connector_sync._utc_stamp(manifest["completed_at"]) if manifest.get("completed_at") else None
|
| 1133 |
+
if not started or not at or datetime.fromisoformat(at) < datetime.fromisoformat(started):
|
| 1134 |
+
return
|
| 1135 |
+
extra = {"publishedAt": at}
|
| 1136 |
+
if RESYNC_STATE.get("slotDate") and (provider == "odoo" or RESYNC_STATE.get("slotPass")):
|
| 1137 |
+
extra["slotDate"] = RESYNC_STATE["slotDate"]
|
| 1138 |
+
connector_sync.record_pass_end(rt, provider, "done", at=at, last_good_sync_at=at,
|
| 1139 |
+
connection_id=cid, extra=extra)
|
| 1140 |
+
|
| 1141 |
+
|
| 1142 |
+
from harness import mirror_store as _status_mirrors
|
| 1143 |
+
_status_mirrors.register_publish_listener(_connector_publication_status)
|
| 1144 |
+
|
| 1145 |
+
|
| 1146 |
def _pull_meta(why, tenant_key=None):
|
| 1147 |
"""Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it.
|
| 1148 |
|
|
|
|
| 1162 |
outcomes = {}
|
| 1163 |
try:
|
| 1164 |
import process_role as _meta_role
|
| 1165 |
+
import tenant_scan as _meta_scan
|
| 1166 |
# Each tenant is pulled by exactly one worker: its own, else the primary.
|
| 1167 |
tenants = [tenant_key] if tenant_key else [
|
| 1168 |
key for key in runtime.known_tenants() if _meta_role.syncs_here(key)]
|
| 1169 |
+
outcomes.update(dict.fromkeys(tenants, False))
|
| 1170 |
+
|
| 1171 |
+
def runtime_failed(key, exc):
|
| 1172 |
+
print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
|
| 1173 |
+
|
| 1174 |
+
# S82-T18: an inactive tenant is skipped with one named line, never a KeyError per pass.
|
| 1175 |
+
for key, rt in _meta_scan.live_tenants(runtime, tenants=tenants, name="meta-pull",
|
| 1176 |
+
on_error=runtime_failed):
|
| 1177 |
try:
|
|
|
|
| 1178 |
if not _meta._connection_rows(rt, key, {"problems": []}):
|
| 1179 |
continue
|
| 1180 |
+
from core import connector_sync as _meta_status
|
| 1181 |
+
_meta_status.record_pass_start(rt, "meta_ads")
|
| 1182 |
+
_status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
|
| 1183 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 1184 |
log=lambda *_a: None, insights_days=7)
|
| 1185 |
outcomes[key] = mirror_store._sync_result("meta", rep)[1]
|
| 1186 |
+
_status(rt, "meta_ads", "done" if outcomes[key] else "failed")
|
| 1187 |
for problem in rep.get("problems") or []:
|
| 1188 |
print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {problem}")
|
| 1189 |
except Exception as exc:
|
| 1190 |
print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
|
| 1191 |
+
_failed = globals().get("_publish_connector_status")
|
| 1192 |
+
if _failed:
|
| 1193 |
+
_failed(rt, "meta_ads", "failed", detail="The Meta Ads pull failed.")
|
| 1194 |
except Exception as exc:
|
| 1195 |
print(f"[aios-api] meta scheduling FAILED ({why}): {type(exc).__name__}")
|
| 1196 |
return outcomes.get(tenant_key or runtime.tenant_for_store_path(), False)
|
|
|
|
| 1280 |
print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential "
|
| 1281 |
f"for {tenant_key}; the connector is idle, not broken")
|
| 1282 |
return False
|
| 1283 |
+
if _amazon.first_load_job_live(rt):
|
| 1284 |
+
print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
|
| 1285 |
+
return False
|
| 1286 |
# Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
|
| 1287 |
# bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
|
| 1288 |
# ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
|
|
|
|
| 1300 |
_rot.invalidate_sources()
|
| 1301 |
_rot._sources()
|
| 1302 |
if first_load_needed:
|
| 1303 |
+
print(f"[aios-api] amazon sync ({why}): history belongs to the independent first-load job")
|
| 1304 |
+
# C-status: the restored mirror seeds the store's last good sync; an older stamp never moves it back.
|
| 1305 |
+
_status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
|
| 1306 |
+
_status(rt, "amazon_sp")
|
| 1307 |
if not vendor:
|
| 1308 |
print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
|
| 1309 |
return True
|
| 1310 |
from harness import mirror_store
|
| 1311 |
+
from core import connector_sync as _amazon_status
|
| 1312 |
+
_amazon_status.record_pass_start(rt, "amazon_sp")
|
| 1313 |
+
try:
|
| 1314 |
+
with _amazon_role.connector_writer(rt, "amazon"):
|
| 1315 |
+
rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
|
| 1316 |
+
slot_pass=bool(RESYNC_STATE.get("slotPass")),
|
| 1317 |
+
log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}")))
|
| 1318 |
+
except Exception:
|
| 1319 |
+
_status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
|
| 1320 |
+
raise
|
| 1321 |
+
_holds = _amazon._MIRROR_STORE._publication_holds(rep, ()) or {}
|
| 1322 |
+
_problems = _holds.get("unsettled", rep.get("problems") or [])
|
| 1323 |
+
_status(rt, "amazon_sp", "failed" if _problems or rep.get("audit_problems") else
|
| 1324 |
+
"continuing" if rep.get("state") == "partial" else "done", report=rep)
|
| 1325 |
_rot.invalidate_sources()
|
| 1326 |
_rot._sources()
|
| 1327 |
for problem in rep.get("problems") or []:
|
|
|
|
| 1336 |
store=rt, quiet=_sa.amazon_standing(rep))
|
| 1337 |
except Exception: # noqa: BLE001
|
| 1338 |
pass
|
| 1339 |
+
return not bool(_problems or rep.get("audit_problems"))
|
| 1340 |
except Exception as exc: # noqa: BLE001
|
| 1341 |
print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}")
|
| 1342 |
return False
|
|
|
|
| 1488 |
"""Classify actual provider failures without returning vendor text or exception reprs."""
|
| 1489 |
import re
|
| 1490 |
if isinstance(cause, dict):
|
| 1491 |
+
if cause.get("amazonStatus") == "NOT_OFFERED" or cause.get("classifier") == "not_offered":
|
| 1492 |
+
return "not_offered", "Amazon does not offer this report to this seller. Its last published rows are kept."
|
| 1493 |
+
# S82-T04 zero-row class: a pass that moved no rows because nothing was due names when it reopens.
|
| 1494 |
+
if cause.get("classifier") == "nothing_due":
|
| 1495 |
+
import datetime as _dt
|
| 1496 |
+
at = cause.get("nextAt")
|
| 1497 |
+
sentence = "Amazon had nothing due in this pass, so no rows moved. Each table waits for its next report window."
|
| 1498 |
+
if isinstance(at, (int, float)) and not isinstance(at, bool) and 0 < at < 4102444800:
|
| 1499 |
+
when = _dt.datetime.fromtimestamp(at, _dt.timezone.utc)
|
| 1500 |
+
sentence += f" The next window opens at {when:%H:%M} UTC on {when.day} {when:%b}."
|
| 1501 |
+
return "nothing_due", sentence
|
| 1502 |
sentence = "The connector could not read an expected value in the sync response."
|
| 1503 |
if cause.get("classifier") == "unreadable_value":
|
| 1504 |
table = cause.get("table")
|
|
|
|
| 1515 |
raw = str(cause)
|
| 1516 |
text = raw.lower()
|
| 1517 |
causes = (
|
| 1518 |
+
("not_offered", ("not_offered", "not offered", "does not offer this report"),
|
| 1519 |
+
"Amazon does not offer this report to this seller. Its last published rows are kept."),
|
| 1520 |
("response_too_large", ("reduce the amount of data", "response too large"),
|
| 1521 |
"The provider refused the pull because the response contains too much data."),
|
| 1522 |
("meta_no_ad_accounts", ("no ad accounts",), "The Meta token cannot access any ad accounts."),
|
|
|
|
| 1595 |
return _sync_now_cause(cause)[1]
|
| 1596 |
|
| 1597 |
|
| 1598 |
+
def _amazon_sync_receipt(report, rt=None):
|
| 1599 |
"""Project table windows and resync counts without vendor diagnostics or exception values."""
|
| 1600 |
import re
|
| 1601 |
details = []
|
|
|
|
| 1603 |
if not isinstance(row, dict) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", str(table)):
|
| 1604 |
continue
|
| 1605 |
parts = []
|
| 1606 |
+
if row.get("state") == "refused" and row.get("amazonStatus") == "NOT_OFFERED":
|
| 1607 |
+
parts.append("report not offered; last published rows kept")
|
| 1608 |
progress = row.get("resync")
|
| 1609 |
if isinstance(progress, dict) and all(type(progress.get(k)) is int and progress[k] >= 0
|
| 1610 |
for k in ("landed", "left")):
|
|
|
|
| 1613 |
if isinstance(window, str):
|
| 1614 |
# Window descriptions are authored by the loader, never copied from its diagnostics.
|
| 1615 |
parts.append(" ".join(window.split()))
|
| 1616 |
+
operation, resume_at = row.get("operation"), row.get("resume_at")
|
| 1617 |
+
if isinstance(operation, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", operation):
|
| 1618 |
+
parts.append(f"operation {operation}")
|
| 1619 |
+
if isinstance(resume_at, str):
|
| 1620 |
+
from core.connector_sync import _utc_stamp
|
| 1621 |
+
stamp = _utc_stamp(resume_at)
|
| 1622 |
+
if stamp:
|
| 1623 |
+
parts.append(f"resume_at {stamp}")
|
| 1624 |
if parts:
|
| 1625 |
+
line = f"{table}: " + "; ".join(parts)
|
| 1626 |
+
if operation or resume_at:
|
| 1627 |
+
details.insert(0, line)
|
| 1628 |
+
else:
|
| 1629 |
+
details.append(line)
|
| 1630 |
+
if rt is not None and not any("operation " in line or "resume_at " in line for line in details):
|
| 1631 |
+
from harness import amazon_store
|
| 1632 |
+
job = amazon_store.first_load_job_summary(rt)
|
| 1633 |
+
if job:
|
| 1634 |
+
tail = _amazon_sync_receipt({"tables": {"first_load": job}})
|
| 1635 |
+
if tail:
|
| 1636 |
+
details.append(tail)
|
| 1637 |
return " | ".join(details)
|
| 1638 |
|
| 1639 |
|
|
|
|
| 1642 |
if ok:
|
| 1643 |
return detail
|
| 1644 |
classifier, sentence = _sync_now_cause(cause)
|
| 1645 |
+
if classifier == "nothing_due":
|
| 1646 |
+
# A wait with a named cause is not a failure: no FAILED line, no failure location.
|
| 1647 |
+
print(f"[aios-api] sync WAITS provider={provider} tenant={tenant} classifier={classifier}")
|
| 1648 |
+
return sentence
|
| 1649 |
print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
|
| 1650 |
if isinstance(cause, BaseException):
|
| 1651 |
import traceback
|
|
|
|
| 1743 |
|
| 1744 |
return log, lines
|
| 1745 |
|
| 1746 |
+
def nothing_due_at(report, problems, failure, tenant):
|
| 1747 |
+
"""S82-T04: the epoch the next Amazon window opens if a pass moved no rows and every table waited.
|
| 1748 |
+
|
| 1749 |
+
An unclassified problem on such a pass is a wait, not a failure. A named failure class keeps its FAILED.
|
| 1750 |
+
Nested on purpose: several gates compile this runner with a fixed set of main.py names.
|
| 1751 |
+
"""
|
| 1752 |
+
import time
|
| 1753 |
+
tables = report.get("tables") or {}
|
| 1754 |
+
rows = [row for row in tables.values() if isinstance(row, dict)]
|
| 1755 |
+
if (not rows or len(rows) != len(tables) or report.get("landedTasks")
|
| 1756 |
+
or any(row.get("state") != "deferred" or row.get("rows") for row in rows)):
|
| 1757 |
+
return None
|
| 1758 |
+
classify = globals().get("_sync_now_cause")
|
| 1759 |
+
if problems and classify and classify(failure or problems[0])[0] != "unclassified":
|
| 1760 |
+
return None
|
| 1761 |
+
now, idle = time.time(), 1800.0
|
| 1762 |
+
wait, held = report.get("nextDueSeconds"), connector_sync.sync_not_before("amazon_sp", tenant)
|
| 1763 |
+
if isinstance(wait, (int, float)) and not isinstance(wait, bool) and wait > 0:
|
| 1764 |
+
at = now + float(wait)
|
| 1765 |
+
elif held and held > now:
|
| 1766 |
+
at = float(held)
|
| 1767 |
+
else:
|
| 1768 |
+
at = now + idle # No floor is known: wait one desk cadence, never requeue at once.
|
| 1769 |
+
return min(max(at, now + 60.0), now + 86400.0)
|
| 1770 |
+
|
| 1771 |
odoo_requested = False
|
| 1772 |
# A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
|
| 1773 |
hold = during_odoo if hold_first_load is None else hold_first_load
|
|
|
|
| 1787 |
ok, detail = False, ""
|
| 1788 |
first_load_progress, continuing = None, False
|
| 1789 |
floor_wait = None
|
| 1790 |
+
nothing_due, first_load = None, False
|
| 1791 |
progress = ()
|
| 1792 |
amazon_detail, amazon_failure = "", None
|
| 1793 |
+
status_rt, status_cid = None, None
|
| 1794 |
try:
|
| 1795 |
from harness import runtime, datastore, mirror_store
|
| 1796 |
rt = runtime.get_runtime(tenant)
|
|
|
|
| 1802 |
datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()):
|
| 1803 |
raise _SyncNowRefused(
|
| 1804 |
"This tenant's mirror is not resident on this server. Sync requires its tenant worker.")
|
| 1805 |
+
# C-status: the pass starts on the resident writer; its end is written after the finish below.
|
| 1806 |
+
status_rt = rt
|
| 1807 |
+
if provider == "odoo":
|
| 1808 |
+
import routes_connectors as _status_routes
|
| 1809 |
+
status_cid = _status_routes._odoo_connection_slot(rt, None)
|
| 1810 |
+
connector_sync.record_pass_start(rt, provider, connection_id=status_cid)
|
| 1811 |
connector_sync.poll_remote_requests(rt)
|
| 1812 |
if provider == "odoo":
|
| 1813 |
import core.odoo as odoo
|
|
|
|
| 1838 |
problems = []
|
| 1839 |
elif provider == "amazon_sp":
|
| 1840 |
from harness import amazon_store as amazon
|
| 1841 |
+
if amazon.first_load_job_live(rt):
|
| 1842 |
+
raise _SyncNowRefused("The first-load job holds the Amazon writer lease.")
|
| 1843 |
_drain_amazon_connection_revocations(rt)
|
| 1844 |
if not amazon.credential(tenant, rt=rt):
|
| 1845 |
raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
|
|
|
|
| 1847 |
target_options = notification_sync_options(receipt)
|
| 1848 |
targeted = bool(target_options)
|
| 1849 |
first_load = False if targeted else amazon.needs_first_load(tenant, rt=rt)
|
| 1850 |
+
options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS} if first_load else {}
|
|
|
|
| 1851 |
_log, progress = progress_log(provider, tenant, receipt, rt)
|
| 1852 |
+
import process_role as _desk_role
|
| 1853 |
+
with _desk_role.connector_writer(rt, "amazon"):
|
| 1854 |
+
report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt,
|
| 1855 |
+
scheduled=False if targeted else receipt.get("requestedBy") == "first load",
|
| 1856 |
+
**target_options, **options,
|
| 1857 |
+
log=_log)
|
| 1858 |
print("[aios-api] amazon sync done (sync desk): " + ", ".join(
|
| 1859 |
f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
|
| 1860 |
+ (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left"
|
|
|
|
| 1866 |
# HOTFIX 2026-09-29: a continuation that landed nothing only met refresh floors.
|
| 1867 |
if continuing and report.get("landedTasks") == 0 and report.get("nextDueSeconds"):
|
| 1868 |
floor_wait = float(report["nextDueSeconds"])
|
| 1869 |
+
_holds = amazon._MIRROR_STORE._publication_holds(report, ()) or {}
|
| 1870 |
+
problems = list(_holds.get("unsettled", report.get("problems") or [])) + list(report.get("audit_problems") or [])
|
| 1871 |
tables = report.get("tables") or {}
|
| 1872 |
+
amazon_detail = _amazon_sync_receipt(report, rt)
|
| 1873 |
amazon_failure = next((row["failure"] for row in tables.values()
|
| 1874 |
if isinstance(row, dict) and isinstance(row.get("failure"), dict)), None)
|
| 1875 |
+
# The floor timer below already serves a clean 0-row continuation that names its floor.
|
| 1876 |
+
nothing_due = (None if targeted or (continuing and floor_wait and not problems)
|
| 1877 |
+
else nothing_due_at(report, problems, amazon_failure, tenant))
|
| 1878 |
+
if nothing_due:
|
| 1879 |
+
continuing, floor_wait = False, None
|
| 1880 |
if not problems and not tables and not continuing:
|
| 1881 |
raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.")
|
| 1882 |
if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()):
|
|
|
|
| 1894 |
raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.")
|
| 1895 |
else:
|
| 1896 |
raise _SyncNowRefused("This connector has no Sync now runner.")
|
| 1897 |
+
if nothing_due:
|
| 1898 |
+
# S82-T04: nothing was due. Finish with the named cause, then wait for the floor.
|
| 1899 |
+
if problems:
|
| 1900 |
+
print(f"[aios-api] amazon sync: {len(problems)} unclassified problem(s) on a pass "
|
| 1901 |
+
f"with nothing due; see the connector sync report", flush=True)
|
| 1902 |
+
ok, problems, amazon_detail = True, [], ""
|
| 1903 |
+
detail = _sync_now_detail(provider, tenant, ok=False,
|
| 1904 |
+
cause={"classifier": "nothing_due", "nextAt": nothing_due})
|
| 1905 |
+
elif continuing:
|
| 1906 |
ok = True
|
| 1907 |
detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; "
|
| 1908 |
f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.")
|
|
|
|
| 1933 |
first_load=first_load_progress, request_id=receipt.get("id"))
|
| 1934 |
else:
|
| 1935 |
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, request_id=receipt.get("id"))
|
| 1936 |
+
pass_outcome = ("waiting" if nothing_due and ok else "continuing" if (continuing and ok)
|
| 1937 |
+
else ("done" if ok else "failed"))
|
| 1938 |
+
_status_write = globals().get("_publish_connector_status")
|
| 1939 |
+
if status_rt is not None and _status_write:
|
| 1940 |
+
_status_write(status_rt, provider, pass_outcome, detail=detail, connection_id=status_cid,
|
| 1941 |
+
**({"report": report} if provider == "amazon_sp" else {}))
|
| 1942 |
if outcome is not None:
|
| 1943 |
+
outcome[provider] = pass_outcome
|
| 1944 |
+
if nothing_due and ok and first_load:
|
| 1945 |
+
# The next continuation waits on the desk for the floor; requeueing at once spins 0-row passes.
|
| 1946 |
+
connector_sync.request_sync("amazon_sp", tenant, requested_by="first load",
|
| 1947 |
+
not_before=nothing_due, wait_cause=detail)
|
| 1948 |
+
import time as _time
|
| 1949 |
+
print(f"[aios-api] amazon first load had nothing due; it waits "
|
| 1950 |
+
f"{int(nothing_due - _time.time())}s for the next report window", flush=True)
|
| 1951 |
+
if during_odoo and ok and not continuing and not nothing_due:
|
| 1952 |
_project_desk({provider: "done"}, tenant)
|
| 1953 |
if continuing and ok:
|
| 1954 |
# The continuation takes this run's desk slot: publish the finish to its web hand-off first.
|
|
|
|
| 2000 |
if state["present"] and (state["changed"] or force_relations):
|
| 2001 |
try:
|
| 2002 |
import automation_engine as _engine
|
| 2003 |
+
_engine.refresh_relations(rt, log=print)
|
| 2004 |
except Exception as e: # noqa: BLE001
|
| 2005 |
print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}")
|
| 2006 |
return state
|
|
|
|
| 2253 |
# refusal silently cancel a relational pass that had already succeeded.
|
| 2254 |
try:
|
| 2255 |
import automation_engine as _engine
|
| 2256 |
+
_engine.refresh_relations(_rt, log=print)
|
| 2257 |
except Exception as e: # noqa: BLE001
|
| 2258 |
rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
|
| 2259 |
print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
|
|
|
|
| 3232 |
print(f"[sheet-warm] request failed: {type(exc).__name__}: {exc}")
|
| 3233 |
|
| 3234 |
|
| 3235 |
+
def _sync_idle_wait(event, seconds):
|
| 3236 |
+
"""The resync loop's wait. A `sync_worker.py --once` job exits here when it is idle (S82-T10)."""
|
| 3237 |
+
import sync_runtime
|
| 3238 |
+
return sync_runtime.idle_wait(event, seconds)
|
| 3239 |
+
|
| 3240 |
+
|
| 3241 |
def _store_resync_loop():
|
| 3242 |
"""Keep the analytical mirror current — the API-process stand-in for the re-sync the
|
| 3243 |
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
|
|
|
| 3284 |
# H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
|
| 3285 |
slot_resume_checked = False
|
| 3286 |
_slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
|
| 3287 |
+
_idle_wait = globals().get("_sync_idle_wait") or (lambda event, seconds: event.wait(seconds))
|
| 3288 |
_slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
|
| 3289 |
_slot_interrupted_clear = globals().get("_resync_slot_interrupted_clear") or (lambda *_a, **_k: False)
|
| 3290 |
+
# Column fills resume only before this slot's load window ends.
|
| 3291 |
_backfill_read = globals().get("_odoo_backfill_checkpoints")
|
| 3292 |
backfill_next_at = None
|
| 3293 |
backfill_continuations = 0
|
| 3294 |
backfill_spacing_seconds = 60.0
|
| 3295 |
+
backfill_cap = None
|
| 3296 |
while True:
|
| 3297 |
+
if backfill_next_at is not None and backfill_cap is not None and _t.time() >= backfill_cap:
|
| 3298 |
+
backfill_next_at = None
|
| 3299 |
# ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
|
| 3300 |
#
|
| 3301 |
# ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 —
|
|
|
|
| 3381 |
# full pass too, so passes start a day apart however long the pass ran (a daily pass
|
| 3382 |
# that also refreshes every Amazon table can run an hour, and a wait counted from its
|
| 3383 |
# END drifted later every day and past the 26-hour stale line).
|
| 3384 |
+
_woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
|
| 3385 |
max(0.0, _every - (_t.time() - last_full_pass_at)),
|
| 3386 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _t.time()))))
|
| 3387 |
connector_sync.sync_wake_event().clear()
|
|
|
|
| 3393 |
if passes or slot_restored:
|
| 3394 |
# Sleep to one second past the next slot; Sync now still interrupts the wait.
|
| 3395 |
_slot_now = _t.time()
|
| 3396 |
+
_woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
|
| 3397 |
0.0 if _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
|
| 3398 |
else max(0.0, _rel_slot.slot_after(_slot_now, _slot) + 1.0 - _slot_now),
|
| 3399 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _slot_now))))
|
|
|
|
| 3448 |
if _woken and (_t.time() - last_full_pass_at < _every if _slot is None else not _slot_due):
|
| 3449 |
_desk = {}
|
| 3450 |
_desk_started = _t.time()
|
| 3451 |
+
if not _run_sync_requests(_desk, hold_first_load=True):
|
| 3452 |
_project_desk(_desk, _cad["tenant"])
|
| 3453 |
if "continuing" in _desk.values():
|
| 3454 |
# A floor between continuations, so a chunk that returns at once cannot spin.
|
|
|
|
| 3459 |
_odoo_requested = None
|
| 3460 |
# H1: a queued backfill pass that is due, unless the slot, the cadence or a Sync now runs.
|
| 3461 |
_continuation = (backfill_next_at is not None and _t.time() >= backfill_next_at - 1.0
|
| 3462 |
+
and backfill_cap is not None and _t.time() < backfill_cap
|
| 3463 |
and _odoo_requested is None and not _slot_due
|
| 3464 |
and (_slot is not None or _t.time() - last_full_pass_at < _every))
|
| 3465 |
if _slot is not None and not _slot_due and _odoo_requested is None and not _continuation:
|
| 3466 |
# H73: outside the slot no Odoo call is made. Serve the stored mirror, drain the desk.
|
| 3467 |
_desk = {}
|
| 3468 |
+
if _run_sync_requests(_desk, hold_first_load=True):
|
| 3469 |
_odoo_requested = True
|
| 3470 |
else:
|
| 3471 |
_project_desk(_desk, _cad["tenant"])
|
|
|
|
| 3488 |
# H73: stamp the slot BEFORE any Odoo read. H2: `slotDoneAt` marks its end, any outcome.
|
| 3489 |
last_slot_pass_at = last_full_pass_at
|
| 3490 |
_slot_write(last_full_pass_at)
|
| 3491 |
+
from harness import datastore as _fill_ds
|
| 3492 |
+
if not _continuation:
|
| 3493 |
+
_slot_epoch = (_rel_slot.slot_before(last_full_pass_at, _slot)
|
| 3494 |
+
if _slot is not None else last_full_pass_at)
|
| 3495 |
+
backfill_cap = _slot_epoch + _fill_ds.COL_BACKFILL_WINDOW_SECONDS
|
| 3496 |
+
RESYNC_STATE["slotDate"] = _t.strftime("%Y-%m-%d", _t.gmtime(_slot_epoch))
|
| 3497 |
+
_fill_ds.COL_BACKFILL_STOP_AT = backfill_cap
|
| 3498 |
+
_odoo_requested = bool(_odoo_requested)
|
| 3499 |
_warm_generation = _sheet_warm_generation()
|
| 3500 |
_backfill_before = _backfill_read(_cad["tenant"]) if _backfill_read else None
|
| 3501 |
+
# C-status: the loop's state and its Odoo pass reach the store at each pass start and end.
|
| 3502 |
+
_resync_status = globals().get("_publish_resync_state")
|
| 3503 |
+
if _resync_status:
|
| 3504 |
+
_resync_status(_cad["tenant"], started=True)
|
| 3505 |
try:
|
| 3506 |
selected_entities = _configured_mirror_entities(_cad["tenant"])
|
| 3507 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
|
|
|
|
| 3539 |
print(f"[aios-api] store resync failed ({_err['kind']}, "
|
| 3540 |
f"{RESYNC_STATE['consecutiveFailures']} in a row): "
|
| 3541 |
f"{_err['type']}: {_err['message']}")
|
| 3542 |
+
if _resync_status:
|
| 3543 |
+
_resync_status(_cad["tenant"], ended=True,
|
| 3544 |
+
outcome=("failed" if _err is not None else "waiting" if _manual
|
| 3545 |
+
else "held" if _publication_held else "done"))
|
| 3546 |
if _publication_held:
|
| 3547 |
print("[aios-api] store resync: publication held; no vendor pulls or relational rebuild; "
|
| 3548 |
"the next admitted pass resumes the unfinished Odoo cohort")
|
|
|
|
| 3552 |
_backfill_after = _backfill_read(_cad["tenant"]) if _backfill_read else None
|
| 3553 |
_moved = ([row for row in _backfill_after if row not in _backfill_before]
|
| 3554 |
if _err is None and _backfill_after and _backfill_before is not None else [])
|
| 3555 |
+
if _moved and _t.time() + backfill_spacing_seconds >= backfill_cap:
|
| 3556 |
+
print(f"[aios-api] odoo backfill stops for today: the load window ended; "
|
| 3557 |
f"the next slot pass or Sync now resumes it")
|
| 3558 |
elif _moved:
|
| 3559 |
backfill_continuations += 1
|
|
|
|
| 3561 |
_row = max(_moved, key=lambda row: row[4] or 0)
|
| 3562 |
print(f"[aios-api] odoo backfill continues: {str(_row[0]).split('.', 1)[0]} at "
|
| 3563 |
f"{_row[4] or 0:,} rows; next bounded pass queued "
|
| 3564 |
+
f"({backfill_continuations} continuations today)")
|
| 3565 |
continue
|
| 3566 |
if _boot_catch_up and not _odoo_requested:
|
| 3567 |
# Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
|
|
|
|
| 3794 |
print(f"[aios-api] Returns split ({why}) {slug}: {split['outcome']}")
|
| 3795 |
except Exception as exc:
|
| 3796 |
print(f"[aios-api] Returns split ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
|
| 3797 |
+
try:
|
| 3798 |
+
result = application_registry.remove_template_doubles(rt, slug)
|
| 3799 |
+
reason = f": {result['reason']}" if result.get("reason") else ""
|
| 3800 |
+
print(f"[aios-api] one template copy ({why}) {slug}: {result['outcome']}{reason}")
|
| 3801 |
+
except Exception as exc:
|
| 3802 |
+
print(f"[aios-api] one template copy ({why}) {slug}: {type(exc).__name__}")
|
| 3803 |
+
try:
|
| 3804 |
+
from modules import amazon_profit as _amazon_profit
|
| 3805 |
+
moved = _amazon_profit.migrate_projects(rt)
|
| 3806 |
+
print(f"[aios-api] Projects move ({why}) {slug}: copied {moved['copied']} of {moved['rows']}")
|
| 3807 |
+
except Exception as exc:
|
| 3808 |
+
print(f"[aios-api] Projects move ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
|
| 3809 |
+
try:
|
| 3810 |
+
import core.user_tables as _ut
|
| 3811 |
+
from modules import amazon_profit as _amazon_profit
|
| 3812 |
+
import routes_amazon_projects as _projects_routes
|
| 3813 |
+
if _ut.get_one(_amazon_profit.TABLE_KEY, rt) is not None:
|
| 3814 |
+
_amazon_profit.ensure_table(rt, _amazon_profit.WRITER)
|
| 3815 |
+
painted = _projects_routes.paint(rt)
|
| 3816 |
+
print(f"[aios-api] Projects cells ({why}) {slug}: {sum(len(ids) for ids in painted.values())} rows")
|
| 3817 |
+
except Exception as exc:
|
| 3818 |
+
print(f"[aios-api] Projects cells ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
|
| 3819 |
try:
|
| 3820 |
result = application_registry.ensure_route_application(rt, slug)
|
| 3821 |
reason = f": {result['reason']}" if result.get("reason") else ""
|
|
|
|
| 3833 |
print(f"[aios-api] Command Center App ({why}) {slug}: skipped: {type(exc).__name__}")
|
| 3834 |
|
| 3835 |
|
| 3836 |
+
def _rows_layouts():
|
| 3837 |
+
"""The rows layouts the boot step migrates, each `(name, ready(rt, slug), migrate(rt, slug))`.
|
| 3838 |
+
|
| 3839 |
+
Every module marker reader needs an entry here: verify_account_rows fails a new one without it.
|
| 3840 |
+
"""
|
| 3841 |
+
from core import account_rows, history_rows, sheets
|
| 3842 |
+
|
| 3843 |
+
def users_ready(_rt, slug):
|
| 3844 |
+
state = account_rows.verify(slug)
|
| 3845 |
+
return state["mode"] == "rows" or not state["records"]
|
| 3846 |
+
|
| 3847 |
+
def history_migrate(_rt, slug):
|
| 3848 |
+
# R58: history_rows refuses a production store unless it is isolated staging. Never widen it.
|
| 3849 |
+
result = history_rows.migrate(slug)
|
| 3850 |
+
return dict(result, records=(result.get("copied") or {}).get("events", 0))
|
| 3851 |
+
|
| 3852 |
+
return [*[(bucket,
|
| 3853 |
+
lambda _rt, slug, b=bucket: account_rows.record_rows_ready(slug, b),
|
| 3854 |
+
lambda _rt, slug, b=bucket: account_rows.records_action("migrate", slug, b))
|
| 3855 |
+
for bucket in account_rows.RECORD_BUCKETS],
|
| 3856 |
+
("users", users_ready, lambda _rt, slug: account_rows.migrate(slug)),
|
| 3857 |
+
("value_history", lambda _rt, slug: history_rows.rows_ready(slug), history_migrate),
|
| 3858 |
+
# Bounded per boot by sheets.BOOT_MIGRATION_LIMIT and BOOT_MIGRATION_SECONDS.
|
| 3859 |
+
("workbooks", lambda _rt, slug: sheets.rows_ready(slug),
|
| 3860 |
+
lambda _rt, slug: sheets.migrate_rows_batch(slug))]
|
| 3861 |
+
|
| 3862 |
+
|
| 3863 |
+
def _boot_rows_migrations(why):
|
| 3864 |
+
"""Migrate each live tenant's rows layouts before their writers run. A migrated layout is a no-op."""
|
| 3865 |
+
if _freeze_holds("rows migrations"):
|
| 3866 |
+
return
|
| 3867 |
+
import sync_alerts
|
| 3868 |
+
|
| 3869 |
+
def report(tenant, name, outcome):
|
| 3870 |
+
print(f"[aios-api] rows migration ({why}) {tenant} {name}: {outcome}")
|
| 3871 |
+
|
| 3872 |
+
def failure(exc):
|
| 3873 |
+
# Scrubbed, so a connection error never prints a credential.
|
| 3874 |
+
return f"failed: {type(exc).__name__}: {sync_alerts.scrub(exc)}"
|
| 3875 |
+
|
| 3876 |
+
def migrated(result):
|
| 3877 |
+
# A bounded layout names, by count, what it left for the next boot and what it could not copy.
|
| 3878 |
+
extra = [f"remaining={result['remaining']} for the next boot"] if result.get("remaining") else []
|
| 3879 |
+
extra += [f"failed {code}={count}" for code, count in sorted((result.get("failed") or {}).items())]
|
| 3880 |
+
if not (result.get("changed") or extra):
|
| 3881 |
+
return "already in rows mode"
|
| 3882 |
+
verb = "migrated" if result.get("changed") else "not migrated"
|
| 3883 |
+
return f"{verb} (" + ", ".join([f"records={result.get('records', 0)}", *extra]) + ")"
|
| 3884 |
+
|
| 3885 |
+
try:
|
| 3886 |
+
import tenant_scan
|
| 3887 |
+
from core import account_rows
|
| 3888 |
+
from harness import runtime as _runtime
|
| 3889 |
+
layouts = _rows_layouts()
|
| 3890 |
+
unresolved = []
|
| 3891 |
+
tenants = list(tenant_scan.live_tenants(
|
| 3892 |
+
_runtime, name="rows-migrations", on_error=lambda slug, exc: unresolved.append((slug, exc))))
|
| 3893 |
+
except Exception as exc:
|
| 3894 |
+
print(f"[aios-api] rows migrations ({why}) skipped: {failure(exc)}")
|
| 3895 |
+
return
|
| 3896 |
+
for tenant, exc in unresolved:
|
| 3897 |
+
for name, _ready, _migrate in layouts:
|
| 3898 |
+
report(tenant, name, failure(exc))
|
| 3899 |
+
for tenant, rt in tenants:
|
| 3900 |
+
try:
|
| 3901 |
+
backend = account_rows.records_backend(rt)
|
| 3902 |
+
except Exception as exc:
|
| 3903 |
+
for name, _ready, _migrate in layouts:
|
| 3904 |
+
report(tenant, name, failure(exc))
|
| 3905 |
+
continue
|
| 3906 |
+
for name, ready, migrate in layouts:
|
| 3907 |
+
# One tenant or layout failing never stops the others.
|
| 3908 |
+
try:
|
| 3909 |
+
if backend is None:
|
| 3910 |
+
report(tenant, name, "skipped, not a PostgreSQL rows store")
|
| 3911 |
+
elif ready(rt, backend[1]):
|
| 3912 |
+
report(tenant, name, "already in rows mode")
|
| 3913 |
+
else:
|
| 3914 |
+
report(tenant, name, migrated(migrate(rt, backend[1])))
|
| 3915 |
+
except Exception as exc:
|
| 3916 |
+
report(tenant, name, failure(exc))
|
| 3917 |
+
|
| 3918 |
+
|
| 3919 |
+
def _boot_rows_before_writers():
|
| 3920 |
+
"""Run the rows migration synchronously at import, before this web or worker process starts a writer."""
|
| 3921 |
+
flag = os.environ.get("AIOS_BOOT_ROWS_MIGRATIONS")
|
| 3922 |
+
if data_binding.gate_process() or flag == "0" or not (os.environ.get("SPACE_ID") or flag == "1"):
|
| 3923 |
+
return
|
| 3924 |
+
_boot_rows_migrations("boot")
|
| 3925 |
+
|
| 3926 |
+
|
| 3927 |
def _prewarm():
|
| 3928 |
if _freeze_holds("prewarm"):
|
| 3929 |
return
|
|
|
|
| 3996 |
from harness import runtime as _runtime
|
| 3997 |
MIRROR_SEED["attempted"] = True
|
| 3998 |
try:
|
| 3999 |
+
import tenant_loops
|
| 4000 |
+
import tenant_scan
|
| 4001 |
+
|
| 4002 |
+
def refused(tenant_key, exc):
|
| 4003 |
+
result = {"state": "refused", "problems": [
|
| 4004 |
+
f"Odoo mirror restore failed for {tenant_key} ({type(exc).__name__})"]}
|
| 4005 |
+
_ds.ODOO_MIRROR_STATUS[str(_ds.path_for(tenant_key).resolve())] = result
|
| 4006 |
+
return result
|
| 4007 |
+
|
| 4008 |
+
def restore(tenant_key, rt):
|
| 4009 |
try:
|
|
|
|
| 4010 |
result = _ds.restore_odoo_mirror(tenant_key, rt)
|
| 4011 |
except Exception as exc:
|
| 4012 |
+
result = refused(tenant_key, exc)
|
|
|
|
|
|
|
| 4013 |
if result["state"] == "refused":
|
| 4014 |
MIRROR_SEED["cause"] = "; ".join(result["problems"])
|
| 4015 |
MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
|
| 4016 |
+
|
| 4017 |
+
def runtime_failed(tenant_key, exc):
|
| 4018 |
+
MIRROR_SEED["cause"] = "; ".join(refused(tenant_key, exc)["problems"])
|
| 4019 |
+
MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
|
| 4020 |
+
|
| 4021 |
+
# S82-T18: each tenant restores in its own job; an inactive one is skipped with one named line.
|
| 4022 |
+
jobs = tenant_loops.for_each_tenant(
|
| 4023 |
+
restore, name="odoo-mirror-restore", tenants=_runtime.known_tenants(),
|
| 4024 |
+
get_runtime=tenant_scan.runtime_lookup(_runtime, "odoo-mirror-restore",
|
| 4025 |
+
on_error=runtime_failed), kind="light")
|
| 4026 |
+
for job in jobs.values():
|
| 4027 |
+
job.join() # The legacy seed below decides on the restored manifests.
|
| 4028 |
if _ds.ensure_seed():
|
| 4029 |
MIRROR_SEED["seeded"] = True
|
| 4030 |
print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
|
|
|
|
| 4130 |
|
| 4131 |
@app.on_event("startup")
|
| 4132 |
def _start_process_role():
|
| 4133 |
+
"""Web role: tick the sync runtime, forward Sync now, install the worker's snapshots."""
|
| 4134 |
if data_binding.gate_process():
|
| 4135 |
return
|
| 4136 |
import process_role as _process_role
|
|
|
|
| 4143 |
import process_role as _process_role
|
| 4144 |
# Only the sync writer requeues in-flight claims: never the web role, never a worker without the lease.
|
| 4145 |
requeue = _process_role.may_write()
|
| 4146 |
+
import tenant_loops
|
| 4147 |
+
import tenant_scan
|
| 4148 |
+
|
| 4149 |
+
def restore(tenant_key, rt):
|
| 4150 |
try:
|
| 4151 |
+
connector_sync.restore_sync_requests(rt, requeue=requeue)
|
| 4152 |
except Exception as exc:
|
| 4153 |
print(f"[aios-api] sync request restore refused for {tenant_key} ({type(exc).__name__})")
|
| 4154 |
|
| 4155 |
+
# S82-T18: an inactive tenant is skipped with one named line; one failure leaves the others restored.
|
| 4156 |
+
jobs = tenant_loops.for_each_tenant(
|
| 4157 |
+
restore, name="sync-request-restore", tenants=runtime.known_tenants(),
|
| 4158 |
+
get_runtime=tenant_scan.runtime_lookup(runtime, "sync-request-restore"), kind="light")
|
| 4159 |
+
for job in jobs.values():
|
| 4160 |
+
job.join() # The desks are restored before the boot goes on.
|
| 4161 |
+
|
| 4162 |
|
| 4163 |
def _boot_mirrors():
|
| 4164 |
import process_role as _role_boot
|
|
|
|
| 4196 |
_READ_WARM_RETRY_SECONDS = 15
|
| 4197 |
|
| 4198 |
|
| 4199 |
+
def _read_pool_state(rt):
|
| 4200 |
+
"""The D-12159 settling line: which connection the read warm holds, and whether it is shared."""
|
| 4201 |
+
from harness import datastore as _state_ds
|
| 4202 |
+
try:
|
| 4203 |
+
handle = rt.mirror_handle() if callable(getattr(rt, "mirror_handle", None)) else None
|
| 4204 |
+
shared = _state_ds._INSTANCE.get("con")
|
| 4205 |
+
return (f"instance={id(shared) if shared is not None else None} "
|
| 4206 |
+
f"handle_con_none={None if handle is None else handle._con is None} "
|
| 4207 |
+
f"handle_path_is_db_path={None if handle is None else handle.path == _state_ds.DB_PATH.resolve()}")
|
| 4208 |
+
except Exception as exc: # noqa: BLE001
|
| 4209 |
+
return f"unreadable ({type(exc).__name__})"
|
| 4210 |
+
|
| 4211 |
+
|
| 4212 |
+
def _connection_closed(con):
|
| 4213 |
+
try:
|
| 4214 |
+
con.cursor().close()
|
| 4215 |
+
return False
|
| 4216 |
+
except Exception: # noqa: BLE001
|
| 4217 |
+
return True
|
| 4218 |
+
|
| 4219 |
+
|
| 4220 |
+
def _reopen_read_pool(rt):
|
| 4221 |
+
"""Drop this tenant's closed mirror read connection so the next cursor opens a new one. True if dropped."""
|
| 4222 |
+
from harness import datastore as _reopen_ds
|
| 4223 |
+
try:
|
| 4224 |
+
handle = rt.mirror_handle() if callable(getattr(rt, "mirror_handle", None)) else None
|
| 4225 |
+
if handle is None or handle.path == _reopen_ds.DB_PATH.resolve():
|
| 4226 |
+
with _reopen_ds._INSTANCE_LOCK:
|
| 4227 |
+
con = _reopen_ds._INSTANCE.get("con")
|
| 4228 |
+
if con is None or not _connection_closed(con):
|
| 4229 |
+
return False
|
| 4230 |
+
_reopen_ds._INSTANCE["con"] = None
|
| 4231 |
+
_reopen_ds._close_instance(con)
|
| 4232 |
+
else:
|
| 4233 |
+
with _reopen_ds._HANDLES_LOCK:
|
| 4234 |
+
con = handle._con
|
| 4235 |
+
if con is None or not _connection_closed(con):
|
| 4236 |
+
return False
|
| 4237 |
+
handle._con = None
|
| 4238 |
+
_reopen_ds._close_instance(con)
|
| 4239 |
+
except Exception as exc: # noqa: BLE001
|
| 4240 |
+
print(f"[aios-api] read pool re-open refused ({type(exc).__name__})")
|
| 4241 |
+
return False
|
| 4242 |
+
print("[aios-api] read pool connection was closed; the next read opens a new one")
|
| 4243 |
+
return True
|
| 4244 |
+
|
| 4245 |
+
|
| 4246 |
def _probe_read_pools(rt):
|
| 4247 |
"""Open the mirror read cursor and the tenant store once. '' when both answer, else the cause."""
|
| 4248 |
from harness import datastore as _probe_ds
|
|
|
|
| 4251 |
physical = _probe_ds.serving_file(path)[0] if _probe_ds.versioned_mode() else path
|
| 4252 |
if not physical.exists():
|
| 4253 |
return "The mirror file is missing."
|
| 4254 |
+
for retry in (False, True):
|
| 4255 |
+
try:
|
| 4256 |
+
cur = handle.ro_cursor() if handle is not None else _probe_ds.ro_cursor()
|
| 4257 |
+
try:
|
| 4258 |
+
cur.execute("SELECT 1").fetchone()
|
| 4259 |
+
finally:
|
| 4260 |
+
cur.close()
|
| 4261 |
+
break
|
| 4262 |
+
except Exception: # noqa: BLE001
|
| 4263 |
+
# D-12159: a closed pool connection stays closed until something drops it.
|
| 4264 |
+
if retry or not _reopen_read_pool(rt):
|
| 4265 |
+
raise
|
| 4266 |
if rt.available() is False:
|
| 4267 |
return "The tenant store did not answer."
|
| 4268 |
return ""
|
|
|
|
| 4277 |
import process_role as _warm_role
|
| 4278 |
attempts = _READ_WARM_ATTEMPTS if attempts is None else max(1, int(attempts))
|
| 4279 |
retry_seconds = _READ_WARM_RETRY_SECONDS if retry_seconds is None else retry_seconds
|
| 4280 |
+
rt, diagnosed = None, False
|
| 4281 |
for attempt in range(1, attempts + 1):
|
| 4282 |
t0 = _t.time()
|
| 4283 |
stage = "pools"
|
|
|
|
| 4290 |
raise RuntimeError(problem)
|
| 4291 |
_warm_role.mark_ready("pools", True)
|
| 4292 |
stage = "first_reads"
|
| 4293 |
+
for retry in (False, True):
|
| 4294 |
+
try:
|
| 4295 |
+
routes_customers.warm_default(rt)
|
| 4296 |
+
routes_products._pool_for(rt, None)
|
| 4297 |
+
break
|
| 4298 |
+
except Exception: # noqa: BLE001
|
| 4299 |
+
# D-12159: a pool connection closed mid-warm is dropped, then the reads run once more.
|
| 4300 |
+
if retry or not _reopen_read_pool(rt):
|
| 4301 |
+
raise
|
| 4302 |
_warm_role.mark_ready("first_reads", True)
|
| 4303 |
routes_customers.warm_bu_pools(rt)
|
| 4304 |
print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
|
| 4305 |
return True
|
| 4306 |
except Exception as exc: # noqa: BLE001
|
| 4307 |
+
if not diagnosed and rt is not None:
|
| 4308 |
+
diagnosed = True
|
| 4309 |
+
print(f"[aios-api] read warm pool state at the first failure: {_read_pool_state(rt)}")
|
| 4310 |
if not _warm_role._READINESS.get(stage, {}).get("ok"):
|
| 4311 |
_warm_role.mark_ready(stage, False, f"The {'read pools' if stage == 'pools' else 'first reads'} "
|
| 4312 |
f"failed ({type(exc).__name__}); attempt {attempt} of {attempts}.")
|
|
|
|
| 4321 |
"""Restore each tenant's declarations after the Odoo seed and Amazon policy adapters."""
|
| 4322 |
from core import connector_spec
|
| 4323 |
from harness import mirror_store, runtime
|
| 4324 |
+
import tenant_loops
|
| 4325 |
+
import tenant_scan
|
| 4326 |
errors = RESYNC_STATE.setdefault("mirrorRestoreErrors", {})
|
| 4327 |
+
|
| 4328 |
+
def restore(tenant_key, rt):
|
| 4329 |
for connector in tuple(connector_spec.MIRRORS):
|
| 4330 |
if connector in {"odoo", "amazon"}:
|
| 4331 |
continue # Their existing adapters also enforce seed and deletion policies.
|
| 4332 |
identity = f"{tenant_key}/{connector}"
|
| 4333 |
try:
|
|
|
|
| 4334 |
result = mirror_store.restore_mirror(connector, tenant_key, rt)
|
| 4335 |
if result["problems"]:
|
| 4336 |
errors[identity] = {"state": result["state"], "problems": list(result["problems"])}
|
|
|
|
| 4339 |
print(f"[aios-api] {connector} mirror {tenant_key} {result['state']} ({why})")
|
| 4340 |
except Exception as exc:
|
| 4341 |
errors[identity] = {"state": "refused", "problems": [type(exc).__name__]}
|
| 4342 |
+
|
| 4343 |
+
def runtime_failed(tenant_key, exc):
|
| 4344 |
+
for connector in tuple(connector_spec.MIRRORS):
|
| 4345 |
+
if connector not in {"odoo", "amazon"}:
|
| 4346 |
+
errors[f"{tenant_key}/{connector}"] = {"state": "refused", "problems": [type(exc).__name__]}
|
| 4347 |
+
|
| 4348 |
+
# S82-T18: each tenant restores in its own job; an inactive one is skipped with one named line.
|
| 4349 |
+
jobs = tenant_loops.for_each_tenant(
|
| 4350 |
+
restore, name="declared-mirror-restore", tenants=runtime.known_tenants(),
|
| 4351 |
+
get_runtime=tenant_scan.runtime_lookup(runtime, "declared-mirror-restore",
|
| 4352 |
+
on_error=runtime_failed), kind="light")
|
| 4353 |
+
for job in jobs.values():
|
| 4354 |
+
job.join() # The caller reads the errors of a finished restore.
|
| 4355 |
return not errors
|
| 4356 |
|
| 4357 |
|
| 4358 |
+
# D-12554: the worker imports main and runs no startup hook, so the migration runs here, before any thread.
|
| 4359 |
+
_boot_rows_before_writers()
|
| 4360 |
+
|
| 4361 |
try:
|
| 4362 |
from harness import datastore as _ds_boot
|
| 4363 |
if not data_binding.gate_process() and (data_binding.is_space() or not _ds_boot.DB_PATH.exists()):
|
|
|
|
| 4394 |
if not data_binding.gate_process() and not _freeze_holds("Amazon App workers"):
|
| 4395 |
import marketing_stream_loop
|
| 4396 |
import ads_schedule_loop
|
| 4397 |
+
import amazon_quotes_loop
|
| 4398 |
+
from harness import amazon_store
|
| 4399 |
import routes_returns_refunds
|
| 4400 |
marketing_stream_loop.start()
|
| 4401 |
ads_schedule_loop.start()
|
| 4402 |
+
amazon_quotes_loop.start()
|
| 4403 |
+
process_role.start("amazon-after-sync", "heavy", amazon_store.run_after_sync_loop)
|
| 4404 |
routes_returns_refunds.start_auto_approve_worker()
|
| 4405 |
|
| 4406 |
|
|
|
|
| 4408 |
def _stop_amazon_app_workers():
|
| 4409 |
import marketing_stream_loop
|
| 4410 |
import ads_schedule_loop
|
| 4411 |
+
import amazon_quotes_loop
|
| 4412 |
marketing_stream_loop.stop()
|
| 4413 |
ads_schedule_loop.stop()
|
| 4414 |
+
amazon_quotes_loop.stop()
|
| 4415 |
|
| 4416 |
|
| 4417 |
if (not data_binding.gate_process()
|
api/odoo_relational.py
CHANGED
|
@@ -49,6 +49,7 @@ the oracle's predicate as an explicit `payment_state` condition pair.
|
|
| 49 |
"""
|
| 50 |
import datetime as _dt
|
| 51 |
import json as _json
|
|
|
|
| 52 |
import time as _time
|
| 53 |
|
| 54 |
#: ⛔ ROUTE ENTITLEMENT, NOT THE SPAWN GATE — `refresh()` stopped reading this at W53-T39 / R3,
|
|
@@ -159,6 +160,13 @@ def _preset(field, flow="odoo_relational"):
|
|
| 159 |
grid renders it grey and the preset walls refuse a rename or a delete."""
|
| 160 |
field = dict(field)
|
| 161 |
field["automation"] = {"flowId": flow, "preset": True}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 162 |
return field
|
| 163 |
|
| 164 |
|
|
@@ -773,6 +781,8 @@ def audit_mirror_additions():
|
|
| 773 |
# Customer tags migrate before their order projections and presets are mounted.
|
| 774 |
if "category_id" not in out["res_partner"]:
|
| 775 |
out["res_partner"].append("category_id")
|
|
|
|
|
|
|
| 776 |
return out
|
| 777 |
|
| 778 |
|
|
@@ -1124,6 +1134,14 @@ def invoice_fields():
|
|
| 1124 |
)] + audit_fields(INVOICES_KEY)
|
| 1125 |
|
| 1126 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1127 |
def order_fields():
|
| 1128 |
"""One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope."""
|
| 1129 |
return [_preset(f) for f in (
|
|
@@ -1151,8 +1169,8 @@ def order_fields():
|
|
| 1151 |
"description": "Odoo's fulfilment status: pending, started, partial, or full."},
|
| 1152 |
{"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay",
|
| 1153 |
"default": True, "agg": "sum"},
|
| 1154 |
-
{"key": "team", "label": "
|
| 1155 |
-
"default": True
|
| 1156 |
{"key": "pricelist", "label": "Pricelist", "type": "text", "source": "overlay",
|
| 1157 |
"default": True, "description": "The pricelist on this Odoo sale order."},
|
| 1158 |
{"key": "state", "label": "State", "type": "select", "source": "overlay",
|
|
@@ -1235,8 +1253,8 @@ def order_line_fields():
|
|
| 1235 |
"default": True},
|
| 1236 |
{"key": "customer_type", "label": "Customer type", "type": "text", "source": "odoo",
|
| 1237 |
"default": True, "description": "The customer's Odoo tags. Blank when no tag is set."},
|
| 1238 |
-
{"key": "team", "label": "
|
| 1239 |
-
"default": True,
|
| 1240 |
"description": "The sales team on this line's Odoo order."},
|
| 1241 |
{"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay",
|
| 1242 |
"default": False},
|
|
@@ -1428,6 +1446,9 @@ def landed_cost_line_fields():
|
|
| 1428 |
"format": {"decimals": 2}},
|
| 1429 |
{"key": "date", "label": "Date", "type": "date", "default": True},
|
| 1430 |
{"key": "state", "label": "State", "type": "text", "default": True},
|
|
|
|
|
|
|
|
|
|
| 1431 |
{"key": "write_date", "label": "Updated", "type": "date", "default": False},
|
| 1432 |
]] + system_link_fields(LANDED_COST_LINES_KEY)
|
| 1433 |
|
|
@@ -1532,7 +1553,8 @@ def customer_fields():
|
|
| 1532 |
{"key": "street2", "label": "Street 2", "type": "text", "source": "overlay",
|
| 1533 |
"default": False},
|
| 1534 |
{"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True},
|
| 1535 |
-
{"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True
|
|
|
|
| 1536 |
{"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False},
|
| 1537 |
{"key": "country", "label": "Country", "type": "text", "source": "overlay",
|
| 1538 |
"default": False},
|
|
@@ -2130,7 +2152,9 @@ def _as_date(value):
|
|
| 2130 |
STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort."""
|
| 2131 |
if not value:
|
| 2132 |
return ""
|
| 2133 |
-
|
|
|
|
|
|
|
| 2134 |
|
| 2135 |
|
| 2136 |
def _in_scope(pid, excluded):
|
|
@@ -4420,7 +4444,8 @@ def landed_cost_line_source_spec():
|
|
| 4420 |
"quantity": ("va.quantity", _stock_number), "former_cost": ("va.former_cost", _stock_number),
|
| 4421 |
"additional_landed_cost": ("va.additional_landed_cost", _stock_number),
|
| 4422 |
"final_cost": ("va.final_cost", _stock_number), "date": ("lc.date", _as_date),
|
| 4423 |
-
"state": ("lc.state", _stock_text), "
|
|
|
|
| 4424 |
},
|
| 4425 |
}
|
| 4426 |
|
|
@@ -4438,11 +4463,11 @@ def require_landed_cost_lines_mirror(cur):
|
|
| 4438 |
if not fields <= columns(cur, entity):
|
| 4439 |
raise Refused("The landed cost lines mirror is incomplete. Sync Odoo before reading it.")
|
| 4440 |
try:
|
| 4441 |
-
|
| 4442 |
except Exception:
|
| 4443 |
-
|
| 4444 |
spec = datastore.ENTITIES.get(entity)
|
| 4445 |
-
if (not
|
| 4446 |
or (spec.get("backfill_fields")
|
| 4447 |
and not datastore._columns_backfilled(cur, entity, spec))):
|
| 4448 |
raise Refused("The landed cost lines mirror is catching up. Try again shortly.")
|
|
@@ -4591,7 +4616,7 @@ def validate_customer_invoice_cadence(client, cur, today=None):
|
|
| 4591 |
observed = dict(cur.execute(
|
| 4592 |
f"SELECT {partner}, {sem._measure_sql(metric, store['alias'], today=today)} "
|
| 4593 |
f"FROM {store['table']} {store['alias']} {store.get('join', '')} "
|
| 4594 |
-
f"WHERE {sem.render_scope(store['scope_sql'])} GROUP BY {partner}").fetchall())
|
| 4595 |
pids = [int(row[JOIN_KEY]) for row in read_customers(cur)]
|
| 4596 |
# The oracle stays on invoice_date when a mutant redirects the served binding to orders.
|
| 4597 |
independent = sem.customer_rollup_oracle('customer_days_since_invoice', client,
|
|
@@ -4623,7 +4648,7 @@ def validate_customer_rollups(client, cur):
|
|
| 4623 |
value = cur.execute(
|
| 4624 |
f"SELECT {sem._measure_sql(metric, store['alias'])} FROM {store['table']} "
|
| 4625 |
f"{store['alias']} {store.get('join', '')} "
|
| 4626 |
-
f"WHERE {sem.render_scope(store['scope_sql'])}").fetchone()[0]
|
| 4627 |
independent = sem.customer_rollup_oracle(bag['measure'], client)
|
| 4628 |
ok = (str(value or '')[:10] == str(independent or '')[:10]
|
| 4629 |
if metric.get('format') == 'date' else
|
|
@@ -5873,8 +5898,9 @@ def slot_after(now, slot):
|
|
| 5873 |
#: A first-load continuation does not start this close to the next slot, so it cannot delay the slot.
|
| 5874 |
FIRST_LOAD_SLOT_QUIET_SECONDS = 2 * 3600
|
| 5875 |
|
| 5876 |
-
#:
|
| 5877 |
SLOT_LATE_SECONDS = 90 * 60
|
|
|
|
| 5878 |
|
| 5879 |
|
| 5880 |
def slot_near(now, slot, seconds=FIRST_LOAD_SLOT_QUIET_SECONDS):
|
|
@@ -6234,6 +6260,82 @@ def _effective_sync_seconds(preset):
|
|
| 6234 |
return max(secs, env)
|
| 6235 |
|
| 6236 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6237 |
def sync_freshness(rt, now=None):
|
| 6238 |
"""C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run
|
| 6239 |
fail. PURE over the store reads.
|
|
@@ -6242,6 +6344,7 @@ def sync_freshness(rt, now=None):
|
|
| 6242 |
`STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected
|
| 6243 |
(frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is
|
| 6244 |
the last run's own reason, its code prefix removed, when that run ended in error.
|
|
|
|
| 6245 |
"""
|
| 6246 |
now = now or _utc_now()
|
| 6247 |
if isinstance(now, str):
|
|
@@ -6261,10 +6364,15 @@ def sync_freshness(rt, now=None):
|
|
| 6261 |
except Exception: # noqa: BLE001
|
| 6262 |
status = None
|
| 6263 |
status = status if isinstance(status, dict) else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6264 |
phase = str(status.get("phase") or "unknown")
|
| 6265 |
-
last_good = raw.get("lastGoodSyncAt") or None
|
| 6266 |
# Before the first stamp this deployment writes, a status that last read `ok` IS a good sync.
|
| 6267 |
-
if not last_good and phase == "ok":
|
| 6268 |
last_good = status.get("lastRun") or None
|
| 6269 |
every = _effective_sync_seconds(cfg["syncEvery"])
|
| 6270 |
allowed = None if every is None else every + STALE_GRACE_SECONDS
|
|
@@ -6272,18 +6380,6 @@ def sync_freshness(rt, now=None):
|
|
| 6272 |
if allowed is not None and not cfg["frozen"]:
|
| 6273 |
good_at = _parse_iso(last_good)
|
| 6274 |
stale = good_at is None or (now - good_at).total_seconds() > allowed
|
| 6275 |
-
# A daily slot that passed SLOT_LATE_SECONDS ago with no good sync since is late, not fresh.
|
| 6276 |
-
late_slot = None
|
| 6277 |
-
slot = sync_slot(cfg)
|
| 6278 |
-
if slot is not None and not cfg["frozen"] and not stale:
|
| 6279 |
-
try:
|
| 6280 |
-
due = slot_before(now.timestamp(), slot)
|
| 6281 |
-
except Exception: # noqa: BLE001
|
| 6282 |
-
due = None
|
| 6283 |
-
good_at = _parse_iso(last_good)
|
| 6284 |
-
if (due is not None and now.timestamp() - due > SLOT_LATE_SECONDS
|
| 6285 |
-
and (good_at is None or good_at.timestamp() < due)):
|
| 6286 |
-
late_slot = slot_iso(due)
|
| 6287 |
failure = None
|
| 6288 |
errors = status.get("errors") if isinstance(status.get("errors"), list) else []
|
| 6289 |
if phase == "error" and errors:
|
|
@@ -6293,6 +6389,15 @@ def sync_freshness(rt, now=None):
|
|
| 6293 |
code, message = "sync_failed", text
|
| 6294 |
failure = {"code": code.strip()[:80], "message": message.strip()[:300],
|
| 6295 |
"at": status.get("lastRun") or None}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6296 |
# D-2679: a database that failed while its siblings synced is reported by itself.
|
| 6297 |
failed_databases = []
|
| 6298 |
for item in raw.get("failedDatabases") if isinstance(raw.get("failedDatabases"), list) else ():
|
|
@@ -6303,10 +6408,11 @@ def sync_freshness(rt, now=None):
|
|
| 6303 |
for name in ("key", "label", "code", "message", "stage", "at")
|
| 6304 |
+ (("cause",) if item.get("cause") else ())})
|
| 6305 |
return {"stale": bool(stale), "lastGoodSyncAt": last_good,
|
| 6306 |
-
|
| 6307 |
"failedDatabases": failed_databases,
|
|
|
|
| 6308 |
"schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
|
| 6309 |
-
"lastRunAt":
|
| 6310 |
"frozen": cfg["frozen"], "failure": failure,
|
| 6311 |
"syncAt": cfg["syncAt"], "syncTz": cfg["syncTz"],
|
| 6312 |
"nextSyncAt": None if cfg["frozen"] else next_sync_at(cfg, now.timestamp())}
|
|
@@ -6628,7 +6734,7 @@ def shipment_source_spec():
|
|
| 6628 |
"state": ("sp.state", _stock_text),
|
| 6629 |
"scheduled_date": (date_sql("sp.scheduled_date"), _as_date),
|
| 6630 |
"date_done": (date_sql("sp.date_done"), _as_date),
|
| 6631 |
-
"order_date": (
|
| 6632 |
"delivery_days": ("epoch(TRY_CAST(sp.date_done AS TIMESTAMPTZ) - "
|
| 6633 |
"TRY_CAST(so.date_order AS TIMESTAMPTZ)) / 86400.0", number),
|
| 6634 |
"refreshed": ("(SELECT substr(updated_at, 1, 10) FROM _sync_state "
|
|
@@ -6965,6 +7071,11 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
|
|
| 6965 |
# ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a
|
| 6966 |
# column a user added to a locked database is never rewritten.
|
| 6967 |
stored = have[fkey]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6968 |
# ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp,
|
| 6969 |
# same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the
|
| 6970 |
# loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on
|
|
@@ -6973,7 +7084,7 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
|
|
| 6973 |
continue
|
| 6974 |
if (stored.get("automation") or {}).get("preset"):
|
| 6975 |
for prop in ("label", "type", "options", "choicesFrom", "link", "rollup", "description",
|
| 6976 |
-
"agg", "pinned", "default"):
|
| 6977 |
# A subsequent row-only refresh must not replace authoritative selections with
|
| 6978 |
# an older compiled vocabulary while its metadata catalogue is unavailable.
|
| 6979 |
if (stored.get("choicesFrom") == "odoo" and field.get("choicesFrom") != "odoo"
|
|
@@ -7598,10 +7709,8 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
|
|
| 7598 |
|
| 7599 |
|
| 7600 |
def _canonical_mirror_ready(rt):
|
| 7601 |
-
#
|
| 7602 |
-
|
| 7603 |
-
from harness import datastore
|
| 7604 |
-
return rt is None or selected_mirror_entities(rt) is None or datastore.ready()
|
| 7605 |
|
| 7606 |
|
| 7607 |
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
|
|
|
|
| 49 |
"""
|
| 50 |
import datetime as _dt
|
| 51 |
import json as _json
|
| 52 |
+
import math as _math
|
| 53 |
import time as _time
|
| 54 |
|
| 55 |
#: ⛔ ROUTE ENTITLEMENT, NOT THE SPAWN GATE — `refresh()` stopped reading this at W53-T39 / R3,
|
|
|
|
| 160 |
grid renders it grey and the preset walls refuse a rename or a delete."""
|
| 161 |
field = dict(field)
|
| 162 |
field["automation"] = {"flowId": flow, "preset": True}
|
| 163 |
+
if field.get('key') in {'customer', 'vendor', 'email', 'phone', 'mobile', 'street',
|
| 164 |
+
'street2', 'city', 'zip', 'country', 'contact', 'contact_name'}:
|
| 165 |
+
field['personal'] = True
|
| 166 |
+
if field.get('key') == 'team':
|
| 167 |
+
field['label'] = 'Sales team'
|
| 168 |
+
field['options'] = _team_options()
|
| 169 |
+
field['automation']['renamable'] = True
|
| 170 |
return field
|
| 171 |
|
| 172 |
|
|
|
|
| 781 |
# Customer tags migrate before their order projections and presets are mounted.
|
| 782 |
if "category_id" not in out["res_partner"]:
|
| 783 |
out["res_partner"].append("category_id")
|
| 784 |
+
out["stock_landed_cost"] = list(LANDED_COST_HEADER_ADDITIONS)
|
| 785 |
+
out[BANK_LINE_TABLE] = list(BANK_LINE_ADDITIONS)
|
| 786 |
return out
|
| 787 |
|
| 788 |
|
|
|
|
| 1134 |
)] + audit_fields(INVOICES_KEY)
|
| 1135 |
|
| 1136 |
|
| 1137 |
+
def _team_options():
|
| 1138 |
+
from core import odoo
|
| 1139 |
+
try:
|
| 1140 |
+
return list(odoo.active_team_names().values())
|
| 1141 |
+
except odoo.TeamsUnknown:
|
| 1142 |
+
return []
|
| 1143 |
+
|
| 1144 |
+
|
| 1145 |
def order_fields():
|
| 1146 |
"""One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope."""
|
| 1147 |
return [_preset(f) for f in (
|
|
|
|
| 1169 |
"description": "Odoo's fulfilment status: pending, started, partial, or full."},
|
| 1170 |
{"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay",
|
| 1171 |
"default": True, "agg": "sum"},
|
| 1172 |
+
{"key": "team", "label": "Sales team", "type": "select", "source": "overlay",
|
| 1173 |
+
"default": True},
|
| 1174 |
{"key": "pricelist", "label": "Pricelist", "type": "text", "source": "overlay",
|
| 1175 |
"default": True, "description": "The pricelist on this Odoo sale order."},
|
| 1176 |
{"key": "state", "label": "State", "type": "select", "source": "overlay",
|
|
|
|
| 1253 |
"default": True},
|
| 1254 |
{"key": "customer_type", "label": "Customer type", "type": "text", "source": "odoo",
|
| 1255 |
"default": True, "description": "The customer's Odoo tags. Blank when no tag is set."},
|
| 1256 |
+
{"key": "team", "label": "Sales team", "type": "select", "source": "odoo",
|
| 1257 |
+
"default": True,
|
| 1258 |
"description": "The sales team on this line's Odoo order."},
|
| 1259 |
{"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay",
|
| 1260 |
"default": False},
|
|
|
|
| 1446 |
"format": {"decimals": 2}},
|
| 1447 |
{"key": "date", "label": "Date", "type": "date", "default": True},
|
| 1448 |
{"key": "state", "label": "State", "type": "text", "default": True},
|
| 1449 |
+
{"key": "amount_total", "label": "Landed cost total", "type": "currency", "default": False,
|
| 1450 |
+
"format": {"decimals": 2},
|
| 1451 |
+
"description": "The header total, repeated on each adjustment line."},
|
| 1452 |
{"key": "write_date", "label": "Updated", "type": "date", "default": False},
|
| 1453 |
]] + system_link_fields(LANDED_COST_LINES_KEY)
|
| 1454 |
|
|
|
|
| 1553 |
{"key": "street2", "label": "Street 2", "type": "text", "source": "overlay",
|
| 1554 |
"default": False},
|
| 1555 |
{"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True},
|
| 1556 |
+
{"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True,
|
| 1557 |
+
"personal": True},
|
| 1558 |
{"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False},
|
| 1559 |
{"key": "country", "label": "Country", "type": "text", "source": "overlay",
|
| 1560 |
"default": False},
|
|
|
|
| 2152 |
STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort."""
|
| 2153 |
if not value:
|
| 2154 |
return ""
|
| 2155 |
+
from core import periods
|
| 2156 |
+
day = periods.business_day(value)
|
| 2157 |
+
return day.isoformat() if day else str(value)[:10]
|
| 2158 |
|
| 2159 |
|
| 2160 |
def _in_scope(pid, excluded):
|
|
|
|
| 4444 |
"quantity": ("va.quantity", _stock_number), "former_cost": ("va.former_cost", _stock_number),
|
| 4445 |
"additional_landed_cost": ("va.additional_landed_cost", _stock_number),
|
| 4446 |
"final_cost": ("va.final_cost", _stock_number), "date": ("lc.date", _as_date),
|
| 4447 |
+
"state": ("lc.state", _stock_text), "amount_total": ("lc.amount_total", _stock_number),
|
| 4448 |
+
"write_date": ("va.write_date", _as_date),
|
| 4449 |
},
|
| 4450 |
}
|
| 4451 |
|
|
|
|
| 4463 |
if not fields <= columns(cur, entity):
|
| 4464 |
raise Refused("The landed cost lines mirror is incomplete. Sync Odoo before reading it.")
|
| 4465 |
try:
|
| 4466 |
+
state = datastore._state(cur, entity)
|
| 4467 |
except Exception:
|
| 4468 |
+
state = None
|
| 4469 |
spec = datastore.ENTITIES.get(entity)
|
| 4470 |
+
if (not state or state["phase"] != "live" or not spec
|
| 4471 |
or (spec.get("backfill_fields")
|
| 4472 |
and not datastore._columns_backfilled(cur, entity, spec))):
|
| 4473 |
raise Refused("The landed cost lines mirror is catching up. Try again shortly.")
|
|
|
|
| 4616 |
observed = dict(cur.execute(
|
| 4617 |
f"SELECT {partner}, {sem._measure_sql(metric, store['alias'], today=today)} "
|
| 4618 |
f"FROM {store['table']} {store['alias']} {store.get('join', '')} "
|
| 4619 |
+
f"WHERE {sem.render_scope(store['scope_sql'], cur=cur)} GROUP BY {partner}").fetchall())
|
| 4620 |
pids = [int(row[JOIN_KEY]) for row in read_customers(cur)]
|
| 4621 |
# The oracle stays on invoice_date when a mutant redirects the served binding to orders.
|
| 4622 |
independent = sem.customer_rollup_oracle('customer_days_since_invoice', client,
|
|
|
|
| 4648 |
value = cur.execute(
|
| 4649 |
f"SELECT {sem._measure_sql(metric, store['alias'])} FROM {store['table']} "
|
| 4650 |
f"{store['alias']} {store.get('join', '')} "
|
| 4651 |
+
f"WHERE {sem.render_scope(store['scope_sql'], cur=cur)}").fetchone()[0]
|
| 4652 |
independent = sem.customer_rollup_oracle(bag['measure'], client)
|
| 4653 |
ok = (str(value or '')[:10] == str(independent or '')[:10]
|
| 4654 |
if metric.get('format') == 'date' else
|
|
|
|
| 5898 |
#: A first-load continuation does not start this close to the next slot, so it cannot delay the slot.
|
| 5899 |
FIRST_LOAD_SLOT_QUIET_SECONDS = 2 * 3600
|
| 5900 |
|
| 5901 |
+
#: Retained for older scheduling callers. Freshness uses the stored pass budget.
|
| 5902 |
SLOT_LATE_SECONDS = 90 * 60
|
| 5903 |
+
PASS_BUDGET_SECONDS = 30 * 60
|
| 5904 |
|
| 5905 |
|
| 5906 |
def slot_near(now, slot, seconds=FIRST_LOAD_SLOT_QUIET_SECONDS):
|
|
|
|
| 6260 |
return max(secs, env)
|
| 6261 |
|
| 6262 |
|
| 6263 |
+
def _filling_databases(rt, cfg, now):
|
| 6264 |
+
"""Read column-fill progress from the tenant store, including after a worker restart."""
|
| 6265 |
+
from harness import datastore
|
| 6266 |
+
try:
|
| 6267 |
+
saved = rt.get(datastore.ODOO_FILL_STATUS_KEY, fresh=True) if rt is not None else None
|
| 6268 |
+
except Exception: # noqa: BLE001
|
| 6269 |
+
saved = None
|
| 6270 |
+
entities = saved.get("entities", {}) if isinstance(saved, dict) else {}
|
| 6271 |
+
if not isinstance(entities, dict):
|
| 6272 |
+
return []
|
| 6273 |
+
result = []
|
| 6274 |
+
for entity, connections in entities.items():
|
| 6275 |
+
if not isinstance(connections, dict):
|
| 6276 |
+
continue
|
| 6277 |
+
parts = [part for part in connections.values()
|
| 6278 |
+
if isinstance(part, dict) and part.get("state") in ("filling", "done")]
|
| 6279 |
+
if not any(part.get("state") == "filling" for part in parts):
|
| 6280 |
+
continue
|
| 6281 |
+
filling = [part for part in parts if part.get("state") == "filling"]
|
| 6282 |
+
active = any(part.get("active") is True for part in filling)
|
| 6283 |
+
resumes = [at for part in filling if (at := _parse_iso(part.get("resumeAt"))) is not None]
|
| 6284 |
+
resume_at = None
|
| 6285 |
+
if not cfg["frozen"] and not active:
|
| 6286 |
+
resume_at = (min(resumes).astimezone(_dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
| 6287 |
+
if resumes else next_sync_at(cfg, now.timestamp()))
|
| 6288 |
+
model = (datastore.ENTITIES.get(entity) or {}).get("model")
|
| 6289 |
+
tables = [(key, label) for _bucket, key, label, _fields in TABLES
|
| 6290 |
+
if model and ODOO_TABLE_MODELS.get(key) == model]
|
| 6291 |
+
tables.extend((key, spec["topicKey"].removeprefix("odoo_").replace("_", " ").capitalize())
|
| 6292 |
+
for key, spec in CANONICAL_RECEIPT_SPECS.items() if model and spec["model"] == model)
|
| 6293 |
+
for key, label in tables or [(entity, entity.replace("_", " "))]:
|
| 6294 |
+
result.append({"key": key, "label": label, "entity": entity, "state": "filling",
|
| 6295 |
+
"rowsDone": sum(max(0, int(part.get("rowsDone") or 0)) for part in parts),
|
| 6296 |
+
"rowsTotal": sum(max(0, int(part.get("rowsTotal") or 0)) for part in parts),
|
| 6297 |
+
"active": active, "resumeAt": resume_at})
|
| 6298 |
+
return result
|
| 6299 |
+
|
| 6300 |
+
|
| 6301 |
+
def _sync_pass_freshness(entry, cfg, now):
|
| 6302 |
+
"""Read duration, daily receipts and resync state from one stored C-status entry."""
|
| 6303 |
+
entry = entry if isinstance(entry, dict) else {}
|
| 6304 |
+
saved = entry.get("pass")
|
| 6305 |
+
saved = dict(saved) if isinstance(saved, dict) else {}
|
| 6306 |
+
|
| 6307 |
+
def number(value):
|
| 6308 |
+
if isinstance(value, (int, float)) and not isinstance(value, bool) and _math.isfinite(value) and value >= 0:
|
| 6309 |
+
return float(value)
|
| 6310 |
+
return None
|
| 6311 |
+
|
| 6312 |
+
budget = number(saved.get("budgetSeconds")) or PASS_BUDGET_SECONDS
|
| 6313 |
+
duration = number(saved.get("durationSeconds"))
|
| 6314 |
+
started = _parse_iso(saved.get("startedAt"))
|
| 6315 |
+
ended = _parse_iso(saved.get("endedAt"))
|
| 6316 |
+
if duration is None and started is not None:
|
| 6317 |
+
end = now if saved.get("outcome") == "running" else ended
|
| 6318 |
+
if end is not None:
|
| 6319 |
+
duration = max(0.0, (end - started).total_seconds())
|
| 6320 |
+
late = not cfg["frozen"] and (saved.get("outcome") == "failed"
|
| 6321 |
+
or duration is not None and duration > budget)
|
| 6322 |
+
late_slot = None
|
| 6323 |
+
slot = sync_slot(cfg)
|
| 6324 |
+
if late and slot is not None:
|
| 6325 |
+
late_slot = slot_iso(slot_before((started or ended or now).timestamp(), slot))
|
| 6326 |
+
receipts = entry.get("dailyPasses")
|
| 6327 |
+
receipts = dict(receipts) if isinstance(receipts, dict) else {}
|
| 6328 |
+
day = (slot_iso(slot_before(now.timestamp(), slot))[:10] if slot is not None
|
| 6329 |
+
else now.astimezone(_dt.timezone.utc).date().isoformat())
|
| 6330 |
+
daily = receipts.get(day)
|
| 6331 |
+
return {"pass": saved or None,
|
| 6332 |
+
"passMinutes": round(duration / 60, 3) if duration is not None else None,
|
| 6333 |
+
"budgetMinutes": budget / 60, "dailyPass": dict(daily) if isinstance(daily, dict) else None,
|
| 6334 |
+
"dailyPasses": receipts,
|
| 6335 |
+
"resync": dict(entry["resync"]) if isinstance(entry.get("resync"), dict) else {},
|
| 6336 |
+
"late": bool(late), "lateSlotAt": late_slot}
|
| 6337 |
+
|
| 6338 |
+
|
| 6339 |
def sync_freshness(rt, now=None):
|
| 6340 |
"""C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run
|
| 6341 |
fail. PURE over the store reads.
|
|
|
|
| 6344 |
`STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected
|
| 6345 |
(frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is
|
| 6346 |
the last run's own reason, its code prefix removed, when that run ended in error.
|
| 6347 |
+
`late` means a failed pass or a pass that exceeds its stored duration budget.
|
| 6348 |
"""
|
| 6349 |
now = now or _utc_now()
|
| 6350 |
if isinstance(now, str):
|
|
|
|
| 6364 |
except Exception: # noqa: BLE001
|
| 6365 |
status = None
|
| 6366 |
status = status if isinstance(status, dict) else {}
|
| 6367 |
+
from core import connector_sync
|
| 6368 |
+
reader = getattr(connector_sync, "status", None)
|
| 6369 |
+
canonical = reader(rt, "odoo", fresh=True) if reader is not None and rt is not None else None
|
| 6370 |
+
canonical = canonical if isinstance(canonical, dict) else None
|
| 6371 |
+
pass_status = _sync_pass_freshness(canonical, cfg, now)
|
| 6372 |
phase = str(status.get("phase") or "unknown")
|
| 6373 |
+
last_good = (canonical.get("lastGoodSyncAt") if canonical is not None else raw.get("lastGoodSyncAt")) or None
|
| 6374 |
# Before the first stamp this deployment writes, a status that last read `ok` IS a good sync.
|
| 6375 |
+
if canonical is None and not last_good and phase == "ok":
|
| 6376 |
last_good = status.get("lastRun") or None
|
| 6377 |
every = _effective_sync_seconds(cfg["syncEvery"])
|
| 6378 |
allowed = None if every is None else every + STALE_GRACE_SECONDS
|
|
|
|
| 6380 |
if allowed is not None and not cfg["frozen"]:
|
| 6381 |
good_at = _parse_iso(last_good)
|
| 6382 |
stale = good_at is None or (now - good_at).total_seconds() > allowed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6383 |
failure = None
|
| 6384 |
errors = status.get("errors") if isinstance(status.get("errors"), list) else []
|
| 6385 |
if phase == "error" and errors:
|
|
|
|
| 6389 |
code, message = "sync_failed", text
|
| 6390 |
failure = {"code": code.strip()[:80], "message": message.strip()[:300],
|
| 6391 |
"at": status.get("lastRun") or None}
|
| 6392 |
+
last_run = status.get("lastRun") or None
|
| 6393 |
+
if canonical is not None:
|
| 6394 |
+
saved_pass = pass_status["pass"] or {}
|
| 6395 |
+
outcome = saved_pass.get("outcome")
|
| 6396 |
+
phase = {"done": "ok", "failed": "error", "running": "syncing"}.get(outcome, outcome or "unknown")
|
| 6397 |
+
last_run = saved_pass.get("endedAt") or saved_pass.get("startedAt")
|
| 6398 |
+
detail = " ".join(str(saved_pass.get("detail") or "Sync failed.").replace("—", ",").replace("–", ",").split())
|
| 6399 |
+
failure = ({"code": "sync_failed", "message": detail[:300],
|
| 6400 |
+
"at": last_run} if outcome == "failed" else None)
|
| 6401 |
# D-2679: a database that failed while its siblings synced is reported by itself.
|
| 6402 |
failed_databases = []
|
| 6403 |
for item in raw.get("failedDatabases") if isinstance(raw.get("failedDatabases"), list) else ():
|
|
|
|
| 6408 |
for name in ("key", "label", "code", "message", "stage", "at")
|
| 6409 |
+ (("cause",) if item.get("cause") else ())})
|
| 6410 |
return {"stale": bool(stale), "lastGoodSyncAt": last_good,
|
| 6411 |
+
**pass_status,
|
| 6412 |
"failedDatabases": failed_databases,
|
| 6413 |
+
"fillingDatabases": _filling_databases(rt, cfg, now),
|
| 6414 |
"schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
|
| 6415 |
+
"lastRunAt": last_run, "phase": phase,
|
| 6416 |
"frozen": cfg["frozen"], "failure": failure,
|
| 6417 |
"syncAt": cfg["syncAt"], "syncTz": cfg["syncTz"],
|
| 6418 |
"nextSyncAt": None if cfg["frozen"] else next_sync_at(cfg, now.timestamp())}
|
|
|
|
| 6734 |
"state": ("sp.state", _stock_text),
|
| 6735 |
"scheduled_date": (date_sql("sp.scheduled_date"), _as_date),
|
| 6736 |
"date_done": (date_sql("sp.date_done"), _as_date),
|
| 6737 |
+
"order_date": (__import__('routes_odoo_tables')._business_day_sql("so.date_order"), _as_date),
|
| 6738 |
"delivery_days": ("epoch(TRY_CAST(sp.date_done AS TIMESTAMPTZ) - "
|
| 6739 |
"TRY_CAST(so.date_order AS TIMESTAMPTZ)) / 86400.0", number),
|
| 6740 |
"refreshed": ("(SELECT substr(updated_at, 1, 10) FROM _sync_state "
|
|
|
|
| 7071 |
# ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a
|
| 7072 |
# column a user added to a locked database is never rewritten.
|
| 7073 |
stored = have[fkey]
|
| 7074 |
+
if (field.get('automation') or {}).get('renamable'):
|
| 7075 |
+
repaired = ut.keep_user_edits(stored, {**stored, **field})
|
| 7076 |
+
stored.clear()
|
| 7077 |
+
stored.update(repaired)
|
| 7078 |
+
continue
|
| 7079 |
# ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp,
|
| 7080 |
# same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the
|
| 7081 |
# loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on
|
|
|
|
| 7084 |
continue
|
| 7085 |
if (stored.get("automation") or {}).get("preset"):
|
| 7086 |
for prop in ("label", "type", "options", "choicesFrom", "link", "rollup", "description",
|
| 7087 |
+
"agg", "pinned", "default", "personal"):
|
| 7088 |
# A subsequent row-only refresh must not replace authoritative selections with
|
| 7089 |
# an older compiled vocabulary while its metadata catalogue is unavailable.
|
| 7090 |
if (stored.get("choicesFrom") == "odoo" and field.get("choicesFrom") != "odoo"
|
|
|
|
| 7709 |
|
| 7710 |
|
| 7711 |
def _canonical_mirror_ready(rt):
|
| 7712 |
+
# Every tenant's canonical totals require its own ready mirror, regardless of selection.
|
| 7713 |
+
return rt is None or rt.mirror_handle().ready()
|
|
|
|
|
|
|
| 7714 |
|
| 7715 |
|
| 7716 |
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
|
api/pages.py
CHANGED
|
@@ -261,12 +261,13 @@ def bu_control(session: Session, team_id):
|
|
| 261 |
"""The BU picker, built from the user's OWN permitted labels — which is exactly why `controls`
|
| 262 |
is assembled per request and never cached on a scope key."""
|
| 263 |
labels = perms.allowed_bu_labels(session.user)
|
|
|
|
| 264 |
options = []
|
| 265 |
for lab in labels:
|
| 266 |
if lab == "All":
|
| 267 |
options.append({"value": "all", "label": "All"})
|
| 268 |
continue
|
| 269 |
-
tid =
|
| 270 |
if tid is not None:
|
| 271 |
options.append({"value": tid, "label": lab})
|
| 272 |
return {"key": "bu", "kind": "bu", "label": "Business Unit",
|
|
@@ -279,10 +280,10 @@ def choice_control(key, label, value, options):
|
|
| 279 |
|
| 280 |
|
| 281 |
def bu_label(team_id):
|
| 282 |
-
"""'All Business Units'
|
| 283 |
if team_id is None:
|
| 284 |
return "All Business Units"
|
| 285 |
-
for lab, tid in ctxlib.
|
| 286 |
if tid == team_id:
|
| 287 |
return lab
|
| 288 |
return str(team_id)
|
|
@@ -371,7 +372,8 @@ def _metrics_for(rt, page_key, team_id, granularity, builder):
|
|
| 371 |
"""
|
| 372 |
import scope_cache
|
| 373 |
|
| 374 |
-
|
|
|
|
| 375 |
key = ("page", page_key, team_id, granularity, today)
|
| 376 |
|
| 377 |
def _build():
|
|
|
|
| 261 |
"""The BU picker, built from the user's OWN permitted labels — which is exactly why `controls`
|
| 262 |
is assembled per request and never cached on a scope key."""
|
| 263 |
labels = perms.allowed_bu_labels(session.user)
|
| 264 |
+
brands = ctxlib.brands()
|
| 265 |
options = []
|
| 266 |
for lab in labels:
|
| 267 |
if lab == "All":
|
| 268 |
options.append({"value": "all", "label": "All"})
|
| 269 |
continue
|
| 270 |
+
tid = brands.get(lab)
|
| 271 |
if tid is not None:
|
| 272 |
options.append({"value": tid, "label": lab})
|
| 273 |
return {"key": "bu", "kind": "bu", "label": "Business Unit",
|
|
|
|
| 280 |
|
| 281 |
|
| 282 |
def bu_label(team_id):
|
| 283 |
+
"""'All Business Units' or the team's label, for prose, never for a permission decision."""
|
| 284 |
if team_id is None:
|
| 285 |
return "All Business Units"
|
| 286 |
+
for lab, tid in ctxlib.brands().items():
|
| 287 |
if tid == team_id:
|
| 288 |
return lab
|
| 289 |
return str(team_id)
|
|
|
|
| 372 |
"""
|
| 373 |
import scope_cache
|
| 374 |
|
| 375 |
+
from core import periods
|
| 376 |
+
today = periods.today().isoformat()
|
| 377 |
key = ("page", page_key, team_id, granularity, today)
|
| 378 |
|
| 379 |
def _build():
|
api/process_role.py
CHANGED
|
@@ -5,16 +5,24 @@ Roles: `web` serves reads and installs finished mirror snapshots, `worker` runs
|
|
| 5 |
never syncs (a second Space on the same store). Kinds: `heavy` moves source data or writes the
|
| 6 |
store in the background; `light` serves this process's own reads.
|
| 7 |
|
| 8 |
-
One
|
| 9 |
-
|
| 10 |
|
| 11 |
-
Several uvicorn workers (AIOS_UVICORN_WORKERS > 1): one web worker holds the primary lock
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
One sync worker per tenant: the primary worker serves AIOS_SYNC_WRITER_TENANT (royal-imports when
|
| 15 |
unset) and runs every job that loops all tenants. Each other tenant that holds a sync connector
|
| 16 |
gets a tenant worker (AIOS_WORKER_SCOPE=tenant) with its own DuckDB file and lease, which runs only
|
| 17 |
-
that tenant's sync jobs. AIOS_SYNC_WORKERS_MAX caps the
|
|
|
|
| 18 |
"""
|
| 19 |
from __future__ import annotations
|
| 20 |
|
|
@@ -25,9 +33,11 @@ import shutil
|
|
| 25 |
import socket
|
| 26 |
import subprocess
|
| 27 |
import sys
|
|
|
|
| 28 |
import threading
|
| 29 |
import time
|
| 30 |
import uuid
|
|
|
|
| 31 |
from pathlib import Path
|
| 32 |
|
| 33 |
ROLES = ("web", "worker", "all", "reader")
|
|
@@ -47,6 +57,10 @@ PRIMARY_LOCK = "web-primary.lock"
|
|
| 47 |
PRIMARY_FILE = "web-primary.json"
|
| 48 |
INSTALL_ATTEMPTS = 3
|
| 49 |
INCOMPLETE_GRACE_SECONDS = 3600
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
|
| 52 |
def resolve_role(env=None):
|
|
@@ -78,6 +92,9 @@ _INSTALLED = {}
|
|
| 78 |
_INSTALL_FAILURES = {}
|
| 79 |
LAST_INSTALL = {}
|
| 80 |
LAST_INSTALL_ERROR = {}
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
|
| 83 |
def role():
|
|
@@ -126,7 +143,11 @@ def heartbeat_name(env=None):
|
|
| 126 |
|
| 127 |
|
| 128 |
def fleet_tenants(root=None):
|
| 129 |
-
"""Tenants with their own tenant worker
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
state = _read_json(Path(root or handoff_dir()) / FLEET_FILE)
|
| 131 |
tenants = state.get("tenants") if isinstance(state, dict) else None
|
| 132 |
return {str(slug) for slug in tenants} if isinstance(tenants, dict) else set()
|
|
@@ -143,7 +164,7 @@ def syncs_here(slug, root=None):
|
|
| 143 |
|
| 144 |
|
| 145 |
def sync_worker_cap(env=None):
|
| 146 |
-
"""AIOS_SYNC_WORKERS_MAX: sync
|
| 147 |
env = os.environ if env is None else env
|
| 148 |
try:
|
| 149 |
return max(1, int(env.get("AIOS_SYNC_WORKERS_MAX") or 4))
|
|
@@ -249,7 +270,7 @@ def writer_holder(env=None):
|
|
| 249 |
return f"{env.get('SPACE_ID') or 'local'}@{socket.gethostname()}"
|
| 250 |
|
| 251 |
|
| 252 |
-
def acquire_writer(rt, holder=None, *, now=None, ttl=None):
|
| 253 |
"""Take or renew the store's writer lease. True when `holder` holds it after the call.
|
| 254 |
|
| 255 |
The store's update is the compare-and-set: Postgres locks the row for the read-modify-write.
|
|
@@ -268,13 +289,14 @@ def acquire_writer(rt, holder=None, *, now=None, ttl=None):
|
|
| 268 |
doc.update(holder=holder, renewedAt=stamp, pid=os.getpid())
|
| 269 |
return doc
|
| 270 |
|
| 271 |
-
|
|
|
|
| 272 |
if not isinstance(doc, dict):
|
| 273 |
-
doc = rt.get(
|
| 274 |
return isinstance(doc, dict) and doc.get("holder") == holder
|
| 275 |
|
| 276 |
|
| 277 |
-
def release_writer(rt, holder=None):
|
| 278 |
"""Give the lease up at a clean exit, so the next writer need not wait for it to expire."""
|
| 279 |
holder = holder or writer_holder()
|
| 280 |
|
|
@@ -284,7 +306,39 @@ def release_writer(rt, holder=None):
|
|
| 284 |
doc.update(holder=None, renewedAt=0, previous=holder)
|
| 285 |
return doc
|
| 286 |
|
| 287 |
-
rt.update(WRITER_LEASE_DOC, drop)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
|
| 289 |
|
| 290 |
def holds_writer():
|
|
@@ -312,7 +366,7 @@ def writer_lease_tick(rt_factory, holder, *, now=None, ttl=None):
|
|
| 312 |
stamp = time.time() if now is None else float(now)
|
| 313 |
ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl)
|
| 314 |
try:
|
| 315 |
-
held = acquire_writer(rt_factory(), holder, now=stamp, ttl=ttl)
|
| 316 |
except Exception as exc: # noqa: BLE001
|
| 317 |
held = None
|
| 318 |
if _WRITER.get("error") != type(exc).__name__:
|
|
@@ -378,10 +432,11 @@ def arm_writer_lease(rt_factory=None, holder=None, exit_fn=None):
|
|
| 378 |
|
| 379 |
|
| 380 |
# ----------------------------------------------------------------------------- readiness (C9)
|
| 381 |
-
READY_CHECKS = ("mirror", "pools", "first_reads")
|
| 382 |
_READY_DEFAULT = {"mirror": "The mirror is not installed yet.",
|
| 383 |
"pools": "The read pools are not open yet.",
|
| 384 |
-
"first_reads": "The first reads have not answered yet."
|
|
|
|
| 385 |
_READINESS = {}
|
| 386 |
_BOOT_RESTORE = {}
|
| 387 |
# The first full pass latches the boot conditions: a later worker restart or a fresh hand-off
|
|
@@ -415,8 +470,43 @@ def note_boot_restore(state="done"):
|
|
| 415 |
print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True)
|
| 416 |
|
| 417 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 418 |
def worker_boot_restored(root=None):
|
| 419 |
-
"""Web side: the
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 420 |
beat = _read_json(Path(root or handoff_dir()) / HEARTBEAT_FILE) or {}
|
| 421 |
restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
|
| 422 |
# A secondary web worker supervises nothing: the primary names the current sync worker.
|
|
@@ -432,10 +522,15 @@ def _installed_map(root=None):
|
|
| 432 |
|
| 433 |
|
| 434 |
def pending_handoffs(root=None):
|
| 435 |
-
"""`tenant/connector` for each
|
|
|
|
|
|
|
|
|
|
| 436 |
installed = _installed_map(root)
|
| 437 |
-
|
| 438 |
-
|
|
|
|
|
|
|
| 439 |
|
| 440 |
|
| 441 |
# ----------------------------------------------------------------------------- several web workers
|
|
@@ -476,7 +571,7 @@ def claim_primary(root=None):
|
|
| 476 |
os.close(fd)
|
| 477 |
return False
|
| 478 |
_PRIMARY["fd"] = fd
|
| 479 |
-
print(f"[process-role] web worker pid={os.getpid()} is the primary: it
|
| 480 |
f"and installs hand-offs", flush=True)
|
| 481 |
write_primary_state(root)
|
| 482 |
return True
|
|
@@ -493,7 +588,9 @@ def write_primary_state(root=None):
|
|
| 493 |
version = None
|
| 494 |
_write_json(Path(root or handoff_dir()) / PRIMARY_FILE, {
|
| 495 |
"pid": os.getpid(), "workerPid": _WORKER.get("pid"), "at": time.time(),
|
| 496 |
-
"installed": {f"{key[0]}/{key[1]}": snap for key, snap in _INSTALLED.items()},
|
|
|
|
|
|
|
| 497 |
"mirrorVersion": version})
|
| 498 |
return True
|
| 499 |
|
|
@@ -515,13 +612,21 @@ def readiness(mirror_live=None, role_name=None, root=None):
|
|
| 515 |
role_name = role_name or ROLE
|
| 516 |
checks, detail = {}, {}
|
| 517 |
for name in READY_CHECKS:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 518 |
row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")}
|
| 519 |
checks[name], detail[name] = bool(row["ok"]), row["detail"]
|
| 520 |
boot = role_name == "web" and _READY_LATCH["at"] is None
|
| 521 |
if "mirror" in checks and checks["mirror"]:
|
| 522 |
if mirror_live is False:
|
| 523 |
checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading."
|
| 524 |
-
elif boot and not worker_boot_restored(root):
|
| 525 |
checks["mirror"], detail["mirror"] = False, "The sync worker has not finished its boot restore."
|
| 526 |
elif boot and pending_handoffs(root):
|
| 527 |
checks["mirror"] = False
|
|
@@ -556,7 +661,11 @@ def _read_json(path):
|
|
| 556 |
|
| 557 |
|
| 558 |
def write_handoff(connector, tenant_key, manifest, rows, paths, *, root=None, now=None):
|
| 559 |
-
"""Keep one
|
|
|
|
|
|
|
|
|
|
|
|
|
| 560 |
root = Path(root or handoff_dir())
|
| 561 |
snapshot = str(manifest.get("snapshot_id") or uuid.uuid4().hex)
|
| 562 |
target = root / connector / snapshot
|
|
@@ -600,9 +709,9 @@ def ready_handoffs(root=None):
|
|
| 600 |
return groups
|
| 601 |
|
| 602 |
|
| 603 |
-
def _install_default(connector, tenant, ready):
|
| 604 |
from harness import mirror_store, runtime
|
| 605 |
-
rt = runtime.get_runtime(tenant)
|
| 606 |
from harness import datastore
|
| 607 |
paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
|
| 608 |
resident = Path(rt.datastore_path).resolve() == datastore.DB_PATH.resolve()
|
|
@@ -633,7 +742,10 @@ def _remove_dir(path):
|
|
| 633 |
|
| 634 |
|
| 635 |
def install_ready(root=None, installer=None, *, now=None):
|
| 636 |
-
"""Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
|
|
|
|
|
|
|
|
|
|
| 637 |
root = Path(root or handoff_dir())
|
| 638 |
installer = installer or _install_default
|
| 639 |
stamp = time.time() if now is None else now
|
|
@@ -678,6 +790,72 @@ def install_ready(root=None, installer=None, *, now=None):
|
|
| 678 |
return installed
|
| 679 |
|
| 680 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 681 |
# ----------------------------------------------------------------------------- cross-process desk
|
| 682 |
def _forward_request(receipt, connection_id):
|
| 683 |
"""Web side: hand an accepted Sync now to the worker through the store desk."""
|
|
@@ -693,34 +871,55 @@ def _forward_request(receipt, connection_id):
|
|
| 693 |
"The sync request could not be passed to the sync worker. Try again.")
|
| 694 |
|
| 695 |
|
| 696 |
-
def
|
| 697 |
-
|
| 698 |
from harness import runtime
|
| 699 |
-
return
|
| 700 |
|
| 701 |
|
| 702 |
-
def
|
| 703 |
-
"""
|
| 704 |
from core import connector_sync
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
|
| 708 |
-
|
| 709 |
-
|
| 710 |
-
|
| 711 |
-
|
| 712 |
-
|
| 713 |
-
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 724 |
|
| 725 |
|
| 726 |
def _worker_desk_loop():
|
|
@@ -731,43 +930,60 @@ def _worker_desk_loop():
|
|
| 731 |
print(f"[process-role] sync desk poll failed ({type(exc).__name__})", flush=True)
|
| 732 |
|
| 733 |
|
| 734 |
-
def _apply_worker_state():
|
| 735 |
-
"""Web side:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 736 |
from core import connector_sync
|
| 737 |
from harness import mirror_store
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
if isinstance(
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 747 |
app_main = _running_main()
|
| 748 |
-
if
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
connector_sync.apply_remote_progress(rt)
|
| 754 |
-
except Exception as exc: # noqa: BLE001
|
| 755 |
-
print(f"[process-role] sync progress read refused for {slug} ({type(exc).__name__})", flush=True)
|
| 756 |
-
if slug in catch_up:
|
| 757 |
-
mirror_store.apply_catch_up(rt, catch_up[slug])
|
| 758 |
|
| 759 |
|
| 760 |
def _web_watch_loop():
|
| 761 |
while not _STOP.wait(POLL_SECONDS):
|
| 762 |
-
try:
|
| 763 |
-
install_ready()
|
| 764 |
-
write_primary_state()
|
| 765 |
-
except Exception as exc: # noqa: BLE001
|
| 766 |
-
print(f"[process-role] hand-off scan failed ({type(exc).__name__}: {exc})", flush=True)
|
| 767 |
try:
|
| 768 |
_apply_worker_state()
|
| 769 |
except Exception as exc: # noqa: BLE001
|
| 770 |
print(f"[process-role] worker state read failed ({type(exc).__name__})", flush=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 771 |
|
| 772 |
|
| 773 |
# ----------------------------------------------------------------------------- the worker process
|
|
@@ -775,17 +991,52 @@ def _running_main():
|
|
| 775 |
return sys.modules.get("api.main") or sys.modules.get("main")
|
| 776 |
|
| 777 |
|
| 778 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 779 |
try:
|
| 780 |
from harness import mirror_store
|
| 781 |
catch_up = mirror_store.catch_up_snapshot()
|
| 782 |
except Exception: # noqa: BLE001
|
| 783 |
catch_up = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 784 |
app_main = _running_main()
|
| 785 |
-
|
| 786 |
-
|
| 787 |
-
|
| 788 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 789 |
|
| 790 |
|
| 791 |
def _heartbeat_loop(parent_pid):
|
|
@@ -803,9 +1054,9 @@ def _heartbeat_loop(parent_pid):
|
|
| 803 |
|
| 804 |
|
| 805 |
def start_worker_services():
|
| 806 |
-
"""Called by sync_worker.py: heartbeat,
|
| 807 |
from harness import mirror_store
|
| 808 |
-
mirror_store.register_publish_listener(
|
| 809 |
parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
|
| 810 |
start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
|
| 811 |
# Before the first heavy job: each one now waits until this worker holds the writer lease.
|
|
@@ -814,6 +1065,7 @@ def start_worker_services():
|
|
| 814 |
|
| 815 |
|
| 816 |
def _worker_env(tenant=None):
|
|
|
|
| 817 |
env = dict(os.environ)
|
| 818 |
env.update(AIOS_PROCESS_ROLE="worker", AIOS_PARENT_PID=str(os.getpid()),
|
| 819 |
AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
|
|
@@ -828,84 +1080,35 @@ def _worker_env(tenant=None):
|
|
| 828 |
|
| 829 |
|
| 830 |
def _supervise(tenant=None):
|
| 831 |
-
"""
|
| 832 |
-
|
| 833 |
-
label = "sync worker" if tenant is None else f"sync worker for {tenant}"
|
| 834 |
-
backoff = 5.0
|
| 835 |
-
while not _STOP.is_set():
|
| 836 |
-
started = time.time()
|
| 837 |
-
try:
|
| 838 |
-
proc = subprocess.Popen([sys.executable, "sync_worker.py"], cwd=str(API_DIR),
|
| 839 |
-
env=_worker_env(tenant))
|
| 840 |
-
except OSError as exc:
|
| 841 |
-
print(f"[process-role] {label} could not start ({type(exc).__name__}: {exc})", flush=True)
|
| 842 |
-
proc = None
|
| 843 |
-
if proc is not None:
|
| 844 |
-
record.update(proc=proc, pid=proc.pid, startedAt=started)
|
| 845 |
-
print(f"[process-role] {label} started pid={proc.pid}", flush=True)
|
| 846 |
-
if tenant is None:
|
| 847 |
-
write_primary_state()
|
| 848 |
-
code = proc.wait()
|
| 849 |
-
record["lastExit"] = code
|
| 850 |
-
else:
|
| 851 |
-
code = None
|
| 852 |
-
if _STOP.is_set():
|
| 853 |
-
return
|
| 854 |
-
if time.time() - started > 600:
|
| 855 |
-
backoff = 5.0
|
| 856 |
-
record["restarts"] += 1
|
| 857 |
-
print(f"[process-role] {label} exited code={code}; restart {record['restarts']} "
|
| 858 |
-
f"in {int(backoff)}s", flush=True)
|
| 859 |
-
if _STOP.wait(backoff):
|
| 860 |
-
return
|
| 861 |
-
backoff = min(backoff * 2, 300.0)
|
| 862 |
|
| 863 |
|
|
|
|
| 864 |
_FLEET = {}
|
| 865 |
_FLEET_STATE = {"overCap": [], "error": None, "at": None}
|
| 866 |
|
| 867 |
|
| 868 |
-
def _write_fleet(root=None):
|
| 869 |
-
_write_json(Path(root or handoff_dir()) / FLEET_FILE, {
|
| 870 |
-
"at": time.time(), "primary": writer_tenant(),
|
| 871 |
-
"tenants": {slug: {"pid": row.get("pid")} for slug, row in _FLEET.items()},
|
| 872 |
-
"overCap": list(_FLEET_STATE["overCap"])})
|
| 873 |
-
|
| 874 |
-
|
| 875 |
def _report_over_cap(over, cap):
|
| 876 |
"""Rule 1: a tenant past the cap is told, with the cause and the fix, never skipped in silence."""
|
| 877 |
-
cause = (f"its daily Odoo and Amazon sync
|
| 878 |
-
f"(AIOS_SYNC_WORKERS_MAX) and they are all
|
| 879 |
-
f"to a larger box.")
|
| 880 |
for slug in over:
|
| 881 |
-
print(f"[process-role] sync
|
| 882 |
try:
|
| 883 |
import sync_alerts
|
| 884 |
from harness import runtime
|
| 885 |
sync_alerts.notify(slug, "sync-worker", [{"key": "worker", "label": "Sync worker",
|
| 886 |
"code": "over_cap", "cause": cause}],
|
| 887 |
-
store=runtime.get_runtime(slug))
|
| 888 |
except Exception as exc: # noqa: BLE001
|
| 889 |
print(f"[process-role] over-cap alert for {slug} failed ({type(exc).__name__})", flush=True)
|
| 890 |
|
| 891 |
|
| 892 |
-
def fleet_tick(wanted=None, *
|
| 893 |
-
"""
|
| 894 |
-
|
| 895 |
-
|
| 896 |
-
new = [slug for slug in wanted if slug not in _FLEET]
|
| 897 |
-
room = max(0, cap - 1 - len(_FLEET))
|
| 898 |
-
starting, over = new[:room], new[room:]
|
| 899 |
-
for slug in starting:
|
| 900 |
-
_FLEET[slug] = {"proc": None, "pid": None, "restarts": 0, "startedAt": None, "lastExit": None}
|
| 901 |
-
_FLEET_STATE.update(overCap=over, at=time.time(), error=None)
|
| 902 |
-
# The fleet file names a tenant before its worker starts, so the primary stops serving it first.
|
| 903 |
-
_write_fleet(root)
|
| 904 |
-
for slug in starting:
|
| 905 |
-
(starter or (lambda s: start(f"sync-worker-supervisor-{s}", "light", _supervise, args=(s,))))(slug)
|
| 906 |
-
if over:
|
| 907 |
-
_report_over_cap(over, cap)
|
| 908 |
-
return starting
|
| 909 |
|
| 910 |
|
| 911 |
def _fleet_loop():
|
|
@@ -914,7 +1117,7 @@ def _fleet_loop():
|
|
| 914 |
fleet_tick()
|
| 915 |
except Exception as exc: # noqa: BLE001
|
| 916 |
_FLEET_STATE.update(error=type(exc).__name__, at=time.time())
|
| 917 |
-
print(f"[process-role]
|
| 918 |
if _STOP.wait(FLEET_SCAN_SECONDS):
|
| 919 |
return
|
| 920 |
|
|
@@ -933,7 +1136,7 @@ def stop_worker():
|
|
| 933 |
|
| 934 |
|
| 935 |
def start_web_services():
|
| 936 |
-
"""Web role only: forward Sync now,
|
| 937 |
|
| 938 |
With several uvicorn workers only the primary does this. The others stand by and read.
|
| 939 |
"""
|
|
@@ -946,35 +1149,37 @@ def start_web_services():
|
|
| 946 |
if _WORKER.get("standby") is None:
|
| 947 |
_WORKER["standby"] = start("web-primary-standby", "light", _standby_loop)
|
| 948 |
print(f"[process-role] web worker pid={os.getpid()} serves reads only; "
|
| 949 |
-
f"the primary
|
| 950 |
return False
|
| 951 |
-
|
| 952 |
-
start("sync-worker-
|
| 953 |
start("mirror-handoff-watcher", "light", _web_watch_loop)
|
| 954 |
atexit.register(stop_worker)
|
| 955 |
return True
|
| 956 |
|
| 957 |
|
| 958 |
def status():
|
| 959 |
-
"""{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
|
| 960 |
-
|
| 961 |
-
|
| 962 |
-
|
| 963 |
-
|
| 964 |
-
|
| 965 |
-
|
| 966 |
pid, alive = os.getpid(), True
|
|
|
|
|
|
|
|
|
|
|
|
|
| 967 |
else:
|
| 968 |
pid, alive = None, False
|
| 969 |
-
age = None
|
| 970 |
-
if isinstance(beat, dict) and beat.get("pid") == pid and beat.get("at"):
|
| 971 |
-
age = round(max(0.0, time.time() - float(beat["at"])), 1)
|
| 972 |
fleet = {}
|
| 973 |
for slug, row in _FLEET.items():
|
| 974 |
-
|
| 975 |
-
|
| 976 |
-
|
| 977 |
return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
|
|
|
|
| 978 |
"tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
|
| 979 |
"tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
|
| 980 |
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
|
|
|
| 5 |
never syncs (a second Space on the same store). Kinds: `heavy` moves source data or writes the
|
| 6 |
store in the background; `light` serves this process's own reads.
|
| 7 |
|
| 8 |
+
One writer per connector: resident jobs hold the Odoo lease, while each Amazon pass claims and
|
| 9 |
+
renews its own lease independently of the first-load job.
|
| 10 |
|
| 11 |
+
Several uvicorn workers (AIOS_UVICORN_WORKERS > 1): one web worker holds the primary lock, asks
|
| 12 |
+
the sync runtime for passes and installs mirrors. The others only read the mirror read-only.
|
| 13 |
+
|
| 14 |
+
The web never starts a sync worker child. sync_runtime picks where the worker runs: one HF job
|
| 15 |
+
per pass, per tenant (`hf-job`), a separately run resident worker (`external`), or nothing (`off`).
|
| 16 |
+
|
| 17 |
+
The web reads the worker's heartbeat, resync state and published mirror ids from the store's
|
| 18 |
+
C-status document (connector_sync.STATUS_KEY, provider WORKER_STATUS) and installs each mirror from
|
| 19 |
+
its stored manifest. No web read depends on a local child process or a hand-off file.
|
| 20 |
|
| 21 |
One sync worker per tenant: the primary worker serves AIOS_SYNC_WRITER_TENANT (royal-imports when
|
| 22 |
unset) and runs every job that loops all tenants. Each other tenant that holds a sync connector
|
| 23 |
gets a tenant worker (AIOS_WORKER_SCOPE=tenant) with its own DuckDB file and lease, which runs only
|
| 24 |
+
that tenant's sync jobs. AIOS_SYNC_WORKERS_MAX caps the sync jobs that run at once; a tenant past
|
| 25 |
+
the cap is reported.
|
| 26 |
"""
|
| 27 |
from __future__ import annotations
|
| 28 |
|
|
|
|
| 33 |
import socket
|
| 34 |
import subprocess
|
| 35 |
import sys
|
| 36 |
+
import tempfile
|
| 37 |
import threading
|
| 38 |
import time
|
| 39 |
import uuid
|
| 40 |
+
from contextlib import contextmanager
|
| 41 |
from pathlib import Path
|
| 42 |
|
| 43 |
ROLES = ("web", "worker", "all", "reader")
|
|
|
|
| 57 |
PRIMARY_FILE = "web-primary.json"
|
| 58 |
INSTALL_ATTEMPTS = 3
|
| 59 |
INCOMPLETE_GRACE_SECONDS = 3600
|
| 60 |
+
#: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under.
|
| 61 |
+
WORKER_STATUS = "sync_worker"
|
| 62 |
+
#: A stored heartbeat older than this reads as a dead worker. verify_live allows 120 s.
|
| 63 |
+
HEARTBEAT_STALE_SECONDS = 120
|
| 64 |
|
| 65 |
|
| 66 |
def resolve_role(env=None):
|
|
|
|
| 92 |
_INSTALL_FAILURES = {}
|
| 93 |
LAST_INSTALL = {}
|
| 94 |
LAST_INSTALL_ERROR = {}
|
| 95 |
+
# The web's last read of the store: the worker heartbeat and each tenant's published mirror ids.
|
| 96 |
+
_STORED = {"beat": None, "mirrors": {}}
|
| 97 |
+
_RESYNC_APPLIED = {"at": ""}
|
| 98 |
|
| 99 |
|
| 100 |
def role():
|
|
|
|
| 143 |
|
| 144 |
|
| 145 |
def fleet_tenants(root=None):
|
| 146 |
+
"""Tenants with their own tenant worker: AIOS_SYNC_FLEET_TENANTS in a job, else the fleet file."""
|
| 147 |
+
named = os.environ.get("AIOS_SYNC_FLEET_TENANTS")
|
| 148 |
+
if named is not None:
|
| 149 |
+
# The web names them when it submits the primary's job: a job has no shared fleet file.
|
| 150 |
+
return {slug.strip() for slug in named.split(",") if slug.strip()}
|
| 151 |
state = _read_json(Path(root or handoff_dir()) / FLEET_FILE)
|
| 152 |
tenants = state.get("tenants") if isinstance(state, dict) else None
|
| 153 |
return {str(slug) for slug in tenants} if isinstance(tenants, dict) else set()
|
|
|
|
| 164 |
|
| 165 |
|
| 166 |
def sync_worker_cap(env=None):
|
| 167 |
+
"""AIOS_SYNC_WORKERS_MAX: sync jobs that run at once, the primary's included. Default 4."""
|
| 168 |
env = os.environ if env is None else env
|
| 169 |
try:
|
| 170 |
return max(1, int(env.get("AIOS_SYNC_WORKERS_MAX") or 4))
|
|
|
|
| 270 |
return f"{env.get('SPACE_ID') or 'local'}@{socket.gethostname()}"
|
| 271 |
|
| 272 |
|
| 273 |
+
def acquire_writer(rt, holder=None, *, now=None, ttl=None, connector=None):
|
| 274 |
"""Take or renew the store's writer lease. True when `holder` holds it after the call.
|
| 275 |
|
| 276 |
The store's update is the compare-and-set: Postgres locks the row for the read-modify-write.
|
|
|
|
| 289 |
doc.update(holder=holder, renewedAt=stamp, pid=os.getpid())
|
| 290 |
return doc
|
| 291 |
|
| 292 |
+
lease_key = f"{WRITER_LEASE_DOC}:{connector or 'odoo'}"
|
| 293 |
+
doc = rt.update(lease_key, take)
|
| 294 |
if not isinstance(doc, dict):
|
| 295 |
+
doc = rt.get(lease_key, fresh=True)
|
| 296 |
return isinstance(doc, dict) and doc.get("holder") == holder
|
| 297 |
|
| 298 |
|
| 299 |
+
def release_writer(rt, holder=None, *, connector=None):
|
| 300 |
"""Give the lease up at a clean exit, so the next writer need not wait for it to expire."""
|
| 301 |
holder = holder or writer_holder()
|
| 302 |
|
|
|
|
| 306 |
doc.update(holder=None, renewedAt=0, previous=holder)
|
| 307 |
return doc
|
| 308 |
|
| 309 |
+
rt.update(f"{WRITER_LEASE_DOC}:{connector or 'odoo'}", drop)
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
@contextmanager
|
| 313 |
+
def connector_writer(rt, connector, *, exit_fn=None):
|
| 314 |
+
"""Hold and renew one connector lease for a pass, refusing a competing first-load writer."""
|
| 315 |
+
holder = f"{writer_holder()}:{connector}:{uuid.uuid4().hex}"
|
| 316 |
+
if not acquire_writer(rt, holder, connector=connector):
|
| 317 |
+
raise RuntimeError(f"Another job holds the {connector} writer lease.")
|
| 318 |
+
stopped = threading.Event()
|
| 319 |
+
|
| 320 |
+
def renew():
|
| 321 |
+
while not stopped.wait(WRITER_LEASE_SECONDS / 3):
|
| 322 |
+
try:
|
| 323 |
+
held = acquire_writer(rt, holder, connector=connector)
|
| 324 |
+
except Exception:
|
| 325 |
+
held = False
|
| 326 |
+
if not held:
|
| 327 |
+
print(f"[process-role] {connector} writer lease lost; this worker stops writing", flush=True)
|
| 328 |
+
(exit_fn or os._exit)(3)
|
| 329 |
+
return
|
| 330 |
+
|
| 331 |
+
job = None
|
| 332 |
+
try:
|
| 333 |
+
job = start("connector-writer-renew", "light", renew)
|
| 334 |
+
if job is None:
|
| 335 |
+
raise RuntimeError("The connector writer lease could not be renewed.")
|
| 336 |
+
yield
|
| 337 |
+
finally:
|
| 338 |
+
stopped.set()
|
| 339 |
+
if job is not None:
|
| 340 |
+
job.join()
|
| 341 |
+
release_writer(rt, holder, connector=connector)
|
| 342 |
|
| 343 |
|
| 344 |
def holds_writer():
|
|
|
|
| 366 |
stamp = time.time() if now is None else float(now)
|
| 367 |
ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl)
|
| 368 |
try:
|
| 369 |
+
held = acquire_writer(rt_factory(), holder, now=stamp, ttl=ttl, connector="odoo")
|
| 370 |
except Exception as exc: # noqa: BLE001
|
| 371 |
held = None
|
| 372 |
if _WRITER.get("error") != type(exc).__name__:
|
|
|
|
| 432 |
|
| 433 |
|
| 434 |
# ----------------------------------------------------------------------------- readiness (C9)
|
| 435 |
+
READY_CHECKS = ("mirror", "pools", "first_reads", "tenant_keys")
|
| 436 |
_READY_DEFAULT = {"mirror": "The mirror is not installed yet.",
|
| 437 |
"pools": "The read pools are not open yet.",
|
| 438 |
+
"first_reads": "The first reads have not answered yet.",
|
| 439 |
+
"tenant_keys": "The tenant data keys have not been checked yet."}
|
| 440 |
_READINESS = {}
|
| 441 |
_BOOT_RESTORE = {}
|
| 442 |
# The first full pass latches the boot conditions: a later worker restart or a fresh hand-off
|
|
|
|
| 470 |
print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True)
|
| 471 |
|
| 472 |
|
| 473 |
+
def _known_beat(root=None):
|
| 474 |
+
"""The stored worker heartbeat this web worker last read; a secondary takes the primary's read."""
|
| 475 |
+
beat = _STORED["beat"] if is_primary() else primary_state(root).get("workerBeat")
|
| 476 |
+
return beat if isinstance(beat, dict) else None
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
def _known_mirrors(root=None):
|
| 480 |
+
"""{(tenant, connector): snapshotId} the store says the worker published, as last read."""
|
| 481 |
+
if is_primary():
|
| 482 |
+
return dict(_STORED["mirrors"])
|
| 483 |
+
stored = primary_state(root).get("storeMirrors") or {}
|
| 484 |
+
return {tuple(key.split("/", 1)): snap for key, snap in stored.items() if "/" in key}
|
| 485 |
+
|
| 486 |
+
|
| 487 |
+
def _waits_for_worker_boot():
|
| 488 |
+
"""Only a resident worker reports a boot restore. A job per pass leaves the web to its own restore."""
|
| 489 |
+
import sync_runtime
|
| 490 |
+
return sync_runtime.runtime_name()[0] == "external"
|
| 491 |
+
|
| 492 |
+
|
| 493 |
+
def beat_age(beat, now=None):
|
| 494 |
+
"""Seconds since a heartbeat's stamp, or None when it carries none."""
|
| 495 |
+
try:
|
| 496 |
+
return round(max(0.0, (time.time() if now is None else now) - float(beat["at"])), 1)
|
| 497 |
+
except (KeyError, TypeError, ValueError):
|
| 498 |
+
return None
|
| 499 |
+
|
| 500 |
+
|
| 501 |
def worker_boot_restored(root=None):
|
| 502 |
+
"""Web side: the worker's stored heartbeat is fresh and says its boot restore finished."""
|
| 503 |
+
beat = _known_beat(root)
|
| 504 |
+
if beat is not None:
|
| 505 |
+
restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
|
| 506 |
+
age = beat_age(beat)
|
| 507 |
+
return (bool(restore.get("state")) and restore.get("pid") == beat.get("pid")
|
| 508 |
+
and age is not None and age < HEARTBEAT_STALE_SECONDS)
|
| 509 |
+
# Fallback for hand-off dir fixtures only (verify_readiness, verify_box_sizing): the store outranks it.
|
| 510 |
beat = _read_json(Path(root or handoff_dir()) / HEARTBEAT_FILE) or {}
|
| 511 |
restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
|
| 512 |
# A secondary web worker supervises nothing: the primary names the current sync worker.
|
|
|
|
| 522 |
|
| 523 |
|
| 524 |
def pending_handoffs(root=None):
|
| 525 |
+
"""`tenant/connector` for each stored publication the primary has not installed yet.
|
| 526 |
+
|
| 527 |
+
READY hand-offs count too, for the fixtures of verify_readiness and verify_box_sizing.
|
| 528 |
+
"""
|
| 529 |
installed = _installed_map(root)
|
| 530 |
+
pending = {f"{key[0]}/{key[1]}" for key, snap in _known_mirrors(root).items() if installed.get(key) != snap}
|
| 531 |
+
pending.update(f"{key[0]}/{key[1]}" for key, rows in ready_handoffs(root).items()
|
| 532 |
+
if installed.get(key) != (rows[-1][2].get("snapshotId") or rows[-1][1].name))
|
| 533 |
+
return sorted(pending)
|
| 534 |
|
| 535 |
|
| 536 |
# ----------------------------------------------------------------------------- several web workers
|
|
|
|
| 571 |
os.close(fd)
|
| 572 |
return False
|
| 573 |
_PRIMARY["fd"] = fd
|
| 574 |
+
print(f"[process-role] web worker pid={os.getpid()} is the primary: it ticks the sync runtime "
|
| 575 |
f"and installs hand-offs", flush=True)
|
| 576 |
write_primary_state(root)
|
| 577 |
return True
|
|
|
|
| 588 |
version = None
|
| 589 |
_write_json(Path(root or handoff_dir()) / PRIMARY_FILE, {
|
| 590 |
"pid": os.getpid(), "workerPid": _WORKER.get("pid"), "at": time.time(),
|
| 591 |
+
"installed": {f"{key[0]}/{key[1]}": snap for key, snap in dict(_INSTALLED).items()},
|
| 592 |
+
"workerBeat": _STORED["beat"],
|
| 593 |
+
"storeMirrors": {f"{key[0]}/{key[1]}": snap for key, snap in dict(_STORED["mirrors"]).items()},
|
| 594 |
"mirrorVersion": version})
|
| 595 |
return True
|
| 596 |
|
|
|
|
| 612 |
role_name = role_name or ROLE
|
| 613 |
checks, detail = {}, {}
|
| 614 |
for name in READY_CHECKS:
|
| 615 |
+
if name == "tenant_keys":
|
| 616 |
+
from core import tenant_keys
|
| 617 |
+
from harness import runtime
|
| 618 |
+
try:
|
| 619 |
+
checks[name], detail[name] = tenant_keys.readiness_check(runtime.known_tenants())
|
| 620 |
+
except Exception:
|
| 621 |
+
checks[name], detail[name] = False, "The tenant data keys did not answer."
|
| 622 |
+
continue
|
| 623 |
row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")}
|
| 624 |
checks[name], detail[name] = bool(row["ok"]), row["detail"]
|
| 625 |
boot = role_name == "web" and _READY_LATCH["at"] is None
|
| 626 |
if "mirror" in checks and checks["mirror"]:
|
| 627 |
if mirror_live is False:
|
| 628 |
checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading."
|
| 629 |
+
elif boot and _waits_for_worker_boot() and not worker_boot_restored(root):
|
| 630 |
checks["mirror"], detail["mirror"] = False, "The sync worker has not finished its boot restore."
|
| 631 |
elif boot and pending_handoffs(root):
|
| 632 |
checks["mirror"] = False
|
|
|
|
| 661 |
|
| 662 |
|
| 663 |
def write_handoff(connector, tenant_key, manifest, rows, paths, *, root=None, now=None):
|
| 664 |
+
"""Keep one snapshot's parquet in a hand-off dir. READY.json is written last.
|
| 665 |
+
|
| 666 |
+
No runtime caller: the web installs from the store (install_stored). verify_readiness and
|
| 667 |
+
verify_box_sizing build their fixtures with it.
|
| 668 |
+
"""
|
| 669 |
root = Path(root or handoff_dir())
|
| 670 |
snapshot = str(manifest.get("snapshot_id") or uuid.uuid4().hex)
|
| 671 |
target = root / connector / snapshot
|
|
|
|
| 709 |
return groups
|
| 710 |
|
| 711 |
|
| 712 |
+
def _install_default(connector, tenant, ready, rt=None):
|
| 713 |
from harness import mirror_store, runtime
|
| 714 |
+
rt = rt or runtime.get_runtime(tenant)
|
| 715 |
from harness import datastore
|
| 716 |
paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
|
| 717 |
resident = Path(rt.datastore_path).resolve() == datastore.DB_PATH.resolve()
|
|
|
|
| 742 |
|
| 743 |
|
| 744 |
def install_ready(root=None, installer=None, *, now=None):
|
| 745 |
+
"""Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
|
| 746 |
+
|
| 747 |
+
No runtime caller since the store install; kept for the verify_readiness fixture.
|
| 748 |
+
"""
|
| 749 |
root = Path(root or handoff_dir())
|
| 750 |
installer = installer or _install_default
|
| 751 |
stamp = time.time() if now is None else now
|
|
|
|
| 790 |
return installed
|
| 791 |
|
| 792 |
|
| 793 |
+
def _resident_snapshot(connector, tenant):
|
| 794 |
+
"""The snapshot id this process already serves, through its own read path. None when unreadable."""
|
| 795 |
+
from harness import datastore
|
| 796 |
+
try:
|
| 797 |
+
return datastore.snapshot_id(tenant, connector)
|
| 798 |
+
except Exception: # noqa: BLE001
|
| 799 |
+
return None
|
| 800 |
+
|
| 801 |
+
|
| 802 |
+
def _install_from_store(connector, tenant, rt, snapshot_id):
|
| 803 |
+
"""Download the store's published manifest and parquet, then install them. False if resident."""
|
| 804 |
+
from harness import datastore, mirror_store
|
| 805 |
+
store = mirror_store.for_connector(connector)
|
| 806 |
+
if snapshot_id and _resident_snapshot(store.connector, tenant) == snapshot_id:
|
| 807 |
+
return False
|
| 808 |
+
raw = rt.download_bytes(store.prefix + mirror_store._MANIFEST_KEY)
|
| 809 |
+
if raw is None:
|
| 810 |
+
raise LookupError(f"no published {connector} manifest in the store")
|
| 811 |
+
with tempfile.TemporaryDirectory(prefix="mirror-store-install-") as tmp:
|
| 812 |
+
# An in-memory DuckDB counts the parquet rows; the tenant file stays closed until install.
|
| 813 |
+
con = datastore.duckdb.connect()
|
| 814 |
+
try:
|
| 815 |
+
manifest, rows, paths, _total, _refused = store._validated_snapshot(tenant, rt, con, raw, tmp)
|
| 816 |
+
finally:
|
| 817 |
+
con.close()
|
| 818 |
+
return _install_default(connector, tenant, {"manifest": manifest, "stateRows": rows,
|
| 819 |
+
"paths": {name: list(files) for name, files in paths.items()}},
|
| 820 |
+
rt)
|
| 821 |
+
|
| 822 |
+
|
| 823 |
+
def install_stored(tenant, rt, mirrors, installer=None, *, now=None):
|
| 824 |
+
"""Install each mirror the store says the worker published and this process has not installed.
|
| 825 |
+
|
| 826 |
+
`mirrors` is the C-status `sync_worker.mirrors` section: {connector: {"snapshotId": ...}}.
|
| 827 |
+
"""
|
| 828 |
+
installer = installer or _install_from_store
|
| 829 |
+
stamp = time.time() if now is None else now
|
| 830 |
+
installed = []
|
| 831 |
+
for connector, row in sorted((mirrors or {}).items()):
|
| 832 |
+
snapshot = row.get("snapshotId") if isinstance(row, dict) else None
|
| 833 |
+
if not snapshot:
|
| 834 |
+
continue
|
| 835 |
+
key = (str(tenant), str(connector))
|
| 836 |
+
_STORED["mirrors"][key] = snapshot
|
| 837 |
+
if _INSTALLED.get(key) == snapshot or _INSTALL_FAILURES.get((key, snapshot), 0) >= INSTALL_ATTEMPTS:
|
| 838 |
+
continue
|
| 839 |
+
try:
|
| 840 |
+
changed = installer(key[1], key[0], rt, snapshot)
|
| 841 |
+
except Exception as exc: # noqa: BLE001
|
| 842 |
+
_INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
|
| 843 |
+
LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
|
| 844 |
+
attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
|
| 845 |
+
print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed "
|
| 846 |
+
f"({type(exc).__name__}: {exc})", flush=True)
|
| 847 |
+
continue
|
| 848 |
+
_INSTALLED[key] = snapshot
|
| 849 |
+
if changed is False:
|
| 850 |
+
continue
|
| 851 |
+
LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
|
| 852 |
+
if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
|
| 853 |
+
LAST_INSTALL_ERROR.clear()
|
| 854 |
+
installed.append(key)
|
| 855 |
+
print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True)
|
| 856 |
+
return installed
|
| 857 |
+
|
| 858 |
+
|
| 859 |
# ----------------------------------------------------------------------------- cross-process desk
|
| 860 |
def _forward_request(receipt, connection_id):
|
| 861 |
"""Web side: hand an accepted Sync now to the worker through the store desk."""
|
|
|
|
| 871 |
"The sync request could not be passed to the sync worker. Try again.")
|
| 872 |
|
| 873 |
|
| 874 |
+
def _status_runtime(tenant=None):
|
| 875 |
+
"""The runtime whose C-status document holds the sync worker's heartbeat."""
|
| 876 |
from harness import runtime
|
| 877 |
+
return runtime.get_runtime(tenant or writer_tenant())
|
| 878 |
|
| 879 |
|
| 880 |
+
def stored_worker(rt):
|
| 881 |
+
"""The `sync_worker` C-status entry of one tenant, read past this process's cache. {} when absent."""
|
| 882 |
from core import connector_sync
|
| 883 |
+
return connector_sync.status(rt, WORKER_STATUS, fresh=True) or {}
|
| 884 |
+
|
| 885 |
+
|
| 886 |
+
def stored_heartbeat(tenant=None):
|
| 887 |
+
"""The worker heartbeat in the store, or None. A store that does not answer reads as None."""
|
| 888 |
+
try:
|
| 889 |
+
beat = stored_worker(_status_runtime(tenant)).get("heartbeat")
|
| 890 |
+
except Exception: # noqa: BLE001
|
| 891 |
+
return None
|
| 892 |
+
return beat if isinstance(beat, dict) else None
|
| 893 |
+
|
| 894 |
+
|
| 895 |
+
def poll_worker_desk(tenants=None, get_runtime=None):
|
| 896 |
+
"""Worker side: pick up event receipts and Sync now hand-offs, one isolated job per tenant."""
|
| 897 |
+
import tenant_loops
|
| 898 |
+
# Light: the heavy desk loop that calls this already holds the writer lease.
|
| 899 |
+
return tenant_loops.for_each_tenant(_poll_tenant_desk, name="sync-desk-tenant", tenants=tenants,
|
| 900 |
+
get_runtime=get_runtime, kind="light")
|
| 901 |
+
|
| 902 |
+
|
| 903 |
+
def _poll_tenant_desk(slug, rt):
|
| 904 |
+
from core import connector_sync
|
| 905 |
+
if not syncs_here(slug):
|
| 906 |
+
return
|
| 907 |
+
try:
|
| 908 |
+
connector_sync.restore_sync_requests(rt)
|
| 909 |
+
admitted = connector_sync.poll_remote_requests(rt)
|
| 910 |
+
except Exception as exc: # noqa: BLE001
|
| 911 |
+
print(f"[process-role] sync desk poll refused for {slug} ({type(exc).__name__})", flush=True)
|
| 912 |
+
raise
|
| 913 |
+
for entry in admitted:
|
| 914 |
+
print(f"[process-role] sync request {entry['provider']} for {slug} admitted from the web", flush=True)
|
| 915 |
+
import routes_connectors
|
| 916 |
+
key = (entry["provider"], slug)
|
| 917 |
+
with routes_connectors._SYNC_REQUEST_CONNECTION_LOCK:
|
| 918 |
+
# No connection id means the default connection, never the previous request's.
|
| 919 |
+
if entry.get("connectionId"):
|
| 920 |
+
routes_connectors._SYNC_REQUEST_CONNECTIONS[key] = entry["connectionId"]
|
| 921 |
+
else:
|
| 922 |
+
routes_connectors._SYNC_REQUEST_CONNECTIONS.pop(key, None)
|
| 923 |
|
| 924 |
|
| 925 |
def _worker_desk_loop():
|
|
|
|
| 930 |
print(f"[process-role] sync desk poll failed ({type(exc).__name__})", flush=True)
|
| 931 |
|
| 932 |
|
| 933 |
+
def _apply_worker_state(tenants=None, get_runtime=None):
|
| 934 |
+
"""Web side: apply each tenant's stored worker state and install its mirrors, one job per tenant.
|
| 935 |
+
|
| 936 |
+
One tenant whose store read raises leaves the other tenants' state applied (D-11881).
|
| 937 |
+
"""
|
| 938 |
+
import tenant_loops
|
| 939 |
+
return tenant_loops.for_each_tenant(_apply_tenant_state, name="worker-state", tenants=tenants,
|
| 940 |
+
get_runtime=get_runtime, kind="light")
|
| 941 |
+
|
| 942 |
+
|
| 943 |
+
def _apply_tenant_state(slug, rt):
|
| 944 |
+
"""Read one tenant's C-status document once, then apply heartbeat, resync, progress and mirrors."""
|
| 945 |
from core import connector_sync
|
| 946 |
from harness import mirror_store
|
| 947 |
+
document = connector_sync.status_all(rt, fresh=True)
|
| 948 |
+
worker = (document.get(WORKER_STATUS) or {}).get("default") or {}
|
| 949 |
+
# The runtime tick reads this copy instead of a second store read.
|
| 950 |
+
_STORED.setdefault("workers", {})[slug] = (time.time(), dict(worker))
|
| 951 |
+
if slug == writer_tenant():
|
| 952 |
+
if isinstance(worker.get("heartbeat"), dict):
|
| 953 |
+
_STORED["beat"] = worker["heartbeat"]
|
| 954 |
+
_apply_resync(document.get("odoo") or {})
|
| 955 |
+
connector_sync.apply_remote_progress(rt)
|
| 956 |
+
if isinstance(worker.get("catchUp"), dict) and worker["catchUp"]:
|
| 957 |
+
mirror_store.apply_catch_up(rt, worker["catchUp"])
|
| 958 |
+
if isinstance(worker.get("mirrors"), dict):
|
| 959 |
+
install_stored(slug, rt, worker["mirrors"])
|
| 960 |
+
|
| 961 |
+
|
| 962 |
+
def _apply_resync(connections):
|
| 963 |
+
"""Copy the newest stored Odoo resync state into this process's RESYNC_STATE for the status routes."""
|
| 964 |
+
rows = [entry["resync"] for entry in connections.values()
|
| 965 |
+
if isinstance(entry, dict) and isinstance(entry.get("resync"), dict)]
|
| 966 |
+
if not rows:
|
| 967 |
+
return
|
| 968 |
+
newest = max(rows, key=lambda row: str(row.get("at") or ""))
|
| 969 |
app_main = _running_main()
|
| 970 |
+
if app_main is None or str(newest.get("at") or "") < _RESYNC_APPLIED["at"]:
|
| 971 |
+
return
|
| 972 |
+
_RESYNC_APPLIED["at"] = str(newest.get("at") or "")
|
| 973 |
+
# Restore errors stay this process's own.
|
| 974 |
+
app_main.RESYNC_STATE.update({k: v for k, v in newest.items() if "estore" not in k and k != "at"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 975 |
|
| 976 |
|
| 977 |
def _web_watch_loop():
|
| 978 |
while not _STOP.wait(POLL_SECONDS):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 979 |
try:
|
| 980 |
_apply_worker_state()
|
| 981 |
except Exception as exc: # noqa: BLE001
|
| 982 |
print(f"[process-role] worker state read failed ({type(exc).__name__})", flush=True)
|
| 983 |
+
try:
|
| 984 |
+
write_primary_state()
|
| 985 |
+
except Exception as exc: # noqa: BLE001
|
| 986 |
+
print(f"[process-role] primary state write failed ({type(exc).__name__}: {exc})", flush=True)
|
| 987 |
|
| 988 |
|
| 989 |
# ----------------------------------------------------------------------------- the worker process
|
|
|
|
| 991 |
return sys.modules.get("api.main") or sys.modules.get("main")
|
| 992 |
|
| 993 |
|
| 994 |
+
_CATCH_UP_SENT = {}
|
| 995 |
+
|
| 996 |
+
|
| 997 |
+
def write_heartbeat(now=None, get_runtime=None):
|
| 998 |
+
"""Worker side: write the heartbeat, each tenant's catch-up and RESYNC_STATE to the C-status store.
|
| 999 |
+
|
| 1000 |
+
Only the writer lease holder beats once the lease is armed, so a waiting worker never masks it.
|
| 1001 |
+
"""
|
| 1002 |
+
from core import connector_sync
|
| 1003 |
+
if _WRITER["armed"] and not _WRITER_HELD.is_set():
|
| 1004 |
+
return False
|
| 1005 |
+
get_runtime = get_runtime or _status_runtime
|
| 1006 |
+
beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
|
| 1007 |
+
"at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE)}
|
| 1008 |
+
written = connector_sync.update_status(get_runtime(writer_tenant()), WORKER_STATUS, "heartbeat", beat)
|
| 1009 |
try:
|
| 1010 |
from harness import mirror_store
|
| 1011 |
catch_up = mirror_store.catch_up_snapshot()
|
| 1012 |
except Exception: # noqa: BLE001
|
| 1013 |
catch_up = {}
|
| 1014 |
+
for tenant, sources in catch_up.items():
|
| 1015 |
+
text = json.dumps(sources, sort_keys=True, default=str)
|
| 1016 |
+
if _CATCH_UP_SENT.get(tenant) == text:
|
| 1017 |
+
continue
|
| 1018 |
+
try:
|
| 1019 |
+
if connector_sync.update_status(get_runtime(tenant), WORKER_STATUS, "catchUp", sources):
|
| 1020 |
+
_CATCH_UP_SENT[tenant] = text
|
| 1021 |
+
except Exception as exc: # noqa: BLE001
|
| 1022 |
+
print(f"[sync-worker] catch-up not stored for {tenant} ({type(exc).__name__})", flush=True)
|
| 1023 |
app_main = _running_main()
|
| 1024 |
+
publish = getattr(app_main, "_publish_resync_state", None)
|
| 1025 |
+
tenant = str((getattr(app_main, "RESYNC_STATE", None) or {}).get("tenant") or "")
|
| 1026 |
+
if callable(publish) and tenant:
|
| 1027 |
+
# Writes only when RESYNC_STATE changed since its last copy.
|
| 1028 |
+
publish(tenant)
|
| 1029 |
+
return written
|
| 1030 |
+
|
| 1031 |
+
|
| 1032 |
+
def record_publication(connector, tenant_key, manifest, _rows=None, _paths=None):
|
| 1033 |
+
"""Publish listener: name the stored snapshot in C-status, so every web process installs it."""
|
| 1034 |
+
from core import connector_sync
|
| 1035 |
+
snapshot = str((manifest or {}).get("snapshot_id") or "")
|
| 1036 |
+
if not snapshot:
|
| 1037 |
+
return False
|
| 1038 |
+
return connector_sync.update_status(_status_runtime(tenant_key), WORKER_STATUS, "mirrors", {
|
| 1039 |
+
str(connector): {"snapshotId": snapshot, "at": time.time()}})
|
| 1040 |
|
| 1041 |
|
| 1042 |
def _heartbeat_loop(parent_pid):
|
|
|
|
| 1054 |
|
| 1055 |
|
| 1056 |
def start_worker_services():
|
| 1057 |
+
"""Called by sync_worker.py: heartbeat, the stored publication record and the store desk poll."""
|
| 1058 |
from harness import mirror_store
|
| 1059 |
+
mirror_store.register_publish_listener(record_publication)
|
| 1060 |
parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
|
| 1061 |
start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
|
| 1062 |
# Before the first heavy job: each one now waits until this worker holds the writer lease.
|
|
|
|
| 1065 |
|
| 1066 |
|
| 1067 |
def _worker_env(tenant=None):
|
| 1068 |
+
"""A resident worker's env. No runtime caller since the web runs no child; gates read it."""
|
| 1069 |
env = dict(os.environ)
|
| 1070 |
env.update(AIOS_PROCESS_ROLE="worker", AIOS_PARENT_PID=str(os.getpid()),
|
| 1071 |
AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
|
|
|
|
| 1080 |
|
| 1081 |
|
| 1082 |
def _supervise(tenant=None):
|
| 1083 |
+
"""One sync runtime tick, for `tenant` or every tenant. The web never starts a sync worker child."""
|
| 1084 |
+
return fleet_tick(None if tenant is None else [tenant])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1085 |
|
| 1086 |
|
| 1087 |
+
#: {tenant: {job, jobId, jobs, alive, aliveWhy}} from this web process's runtime ticks.
|
| 1088 |
_FLEET = {}
|
| 1089 |
_FLEET_STATE = {"overCap": [], "error": None, "at": None}
|
| 1090 |
|
| 1091 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1092 |
def _report_over_cap(over, cap):
|
| 1093 |
"""Rule 1: a tenant past the cap is told, with the cause and the fix, never skipped in silence."""
|
| 1094 |
+
cause = (f"its daily Odoo and Amazon sync waits: {cap} sync job(s) may run at once "
|
| 1095 |
+
f"(AIOS_SYNC_WORKERS_MAX) and they are all running. Raise AIOS_SYNC_WORKERS_MAX.")
|
|
|
|
| 1096 |
for slug in over:
|
| 1097 |
+
print(f"[process-role] sync job for {slug} NOT submitted: {cause}", flush=True)
|
| 1098 |
try:
|
| 1099 |
import sync_alerts
|
| 1100 |
from harness import runtime
|
| 1101 |
sync_alerts.notify(slug, "sync-worker", [{"key": "worker", "label": "Sync worker",
|
| 1102 |
"code": "over_cap", "cause": cause}],
|
| 1103 |
+
observed={"worker"}, store=runtime.get_runtime(slug))
|
| 1104 |
except Exception as exc: # noqa: BLE001
|
| 1105 |
print(f"[process-role] over-cap alert for {slug} failed ({type(exc).__name__})", flush=True)
|
| 1106 |
|
| 1107 |
|
| 1108 |
+
def fleet_tick(wanted=None, **kwargs):
|
| 1109 |
+
"""Ask the sync runtime for each due tenant's pass inside the cap. Returns the tenants it submitted."""
|
| 1110 |
+
import sync_runtime
|
| 1111 |
+
return sync_runtime.tick(wanted, **kwargs)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1112 |
|
| 1113 |
|
| 1114 |
def _fleet_loop():
|
|
|
|
| 1117 |
fleet_tick()
|
| 1118 |
except Exception as exc: # noqa: BLE001
|
| 1119 |
_FLEET_STATE.update(error=type(exc).__name__, at=time.time())
|
| 1120 |
+
print(f"[process-role] sync runtime tick failed ({type(exc).__name__}: {exc})", flush=True)
|
| 1121 |
if _STOP.wait(FLEET_SCAN_SECONDS):
|
| 1122 |
return
|
| 1123 |
|
|
|
|
| 1136 |
|
| 1137 |
|
| 1138 |
def start_web_services():
|
| 1139 |
+
"""Web role only: forward Sync now, tick the sync runtime, install the worker's snapshots.
|
| 1140 |
|
| 1141 |
With several uvicorn workers only the primary does this. The others stand by and read.
|
| 1142 |
"""
|
|
|
|
| 1149 |
if _WORKER.get("standby") is None:
|
| 1150 |
_WORKER["standby"] = start("web-primary-standby", "light", _standby_loop)
|
| 1151 |
print(f"[process-role] web worker pid={os.getpid()} serves reads only; "
|
| 1152 |
+
f"the primary ticks the sync runtime", flush=True)
|
| 1153 |
return False
|
| 1154 |
+
# No worker child: the supervisor thread only ticks the sync runtime.
|
| 1155 |
+
_WORKER["supervisor"] = start("sync-worker-supervisor", "light", _fleet_loop)
|
| 1156 |
start("mirror-handoff-watcher", "light", _web_watch_loop)
|
| 1157 |
atexit.register(stop_worker)
|
| 1158 |
return True
|
| 1159 |
|
| 1160 |
|
| 1161 |
def status():
|
| 1162 |
+
"""{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
|
| 1163 |
+
|
| 1164 |
+
A serving role reads the worker from its stored heartbeat, never from a child process (D-11183).
|
| 1165 |
+
"""
|
| 1166 |
+
beat = stored_heartbeat() if ROLE != "all" else None
|
| 1167 |
+
age = beat_age(beat) if beat is not None else None
|
| 1168 |
+
if ROLE == "worker":
|
| 1169 |
pid, alive = os.getpid(), True
|
| 1170 |
+
age = age if beat is not None and beat.get("pid") == pid else None
|
| 1171 |
+
elif ROLE in ("web", "reader") and beat is not None:
|
| 1172 |
+
import sync_runtime
|
| 1173 |
+
pid, alive = beat.get("pid"), sync_runtime.beat_alive(beat)
|
| 1174 |
else:
|
| 1175 |
pid, alive = None, False
|
|
|
|
|
|
|
|
|
|
| 1176 |
fleet = {}
|
| 1177 |
for slug, row in _FLEET.items():
|
| 1178 |
+
fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0),
|
| 1179 |
+
"lastExit": None, "jobId": row.get("jobId")}
|
| 1180 |
+
import sync_runtime
|
| 1181 |
return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
|
| 1182 |
+
"syncRuntime": sync_runtime.runtime_name()[0],
|
| 1183 |
"tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
|
| 1184 |
"tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
|
| 1185 |
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
api/registry_source.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""Bounded stored-source reads before canonical grid projection."""
|
| 2 |
import copy
|
| 3 |
-
from contextlib import closing
|
| 4 |
from dataclasses import is_dataclass, replace
|
| 5 |
|
| 6 |
from deps import err
|
|
@@ -530,6 +530,198 @@ def canonical_metadata(session, table_key, *, metadata=None, cur=None):
|
|
| 530 |
consume_corrections=False, with_rows=False, needed=frozenset())
|
| 531 |
|
| 532 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 533 |
def canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None):
|
| 534 |
"""Refuse a failed compiled source by name without publishing an empty population."""
|
| 535 |
from fastapi import HTTPException
|
|
|
|
| 1 |
"""Bounded stored-source reads before canonical grid projection."""
|
| 2 |
import copy
|
| 3 |
+
from contextlib import closing, contextmanager
|
| 4 |
from dataclasses import is_dataclass, replace
|
| 5 |
|
| 6 |
from deps import err
|
|
|
|
| 530 |
consume_corrections=False, with_rows=False, needed=frozenset())
|
| 531 |
|
| 532 |
|
| 533 |
+
def _selection_teams(session):
|
| 534 |
+
"""Validate the explicit runtime identity and its configured teams before SQL."""
|
| 535 |
+
runtime = session.runtime
|
| 536 |
+
key = str(getattr(runtime, 'key', '') or '').strip().lower()
|
| 537 |
+
tenant = getattr(runtime, 'tenant', None)
|
| 538 |
+
owner = str(getattr(tenant, 'key', '') or '').strip().lower()
|
| 539 |
+
actor = str(session.user.get('tenant') or key).strip().lower()
|
| 540 |
+
if not key or key != owner or actor != key:
|
| 541 |
+
raise err(403, 'tenant_mismatch', 'Canonical selection requires a bound tenant identity.')
|
| 542 |
+
config = getattr(tenant, 'config', None) or {}
|
| 543 |
+
units = config.get('business_units')
|
| 544 |
+
if (not isinstance(units, dict) or not units
|
| 545 |
+
or any(not isinstance(label, str) or not label.strip()
|
| 546 |
+
or type(tid) is not int or tid <= 0 for label, tid in units.items())
|
| 547 |
+
or len(set(units.values())) != len(units)):
|
| 548 |
+
raise err(403, 'bu_scope_denied', 'Canonical selection requires known tenant teams.')
|
| 549 |
+
labels = {tid: label for label, tid in units.items()}
|
| 550 |
+
bus = session.user.get('bus', 'all')
|
| 551 |
+
if bus != 'all' and (not isinstance(bus, (list, tuple, set, frozenset)) or not bus
|
| 552 |
+
or any(type(tid) is not int or tid not in labels for tid in bus)):
|
| 553 |
+
raise err(403, 'bu_scope_denied', 'Account teams are empty or unknown for this tenant.')
|
| 554 |
+
return labels
|
| 555 |
+
|
| 556 |
+
|
| 557 |
+
@contextmanager
|
| 558 |
+
def canonical_selection(session, cur, table_key, metadata, needed, *, ids=None):
|
| 559 |
+
"""Lend SQL identities and proven native operands before canonical page hydration."""
|
| 560 |
+
from core import perm_scope
|
| 561 |
+
from harness import canonical_source as cs
|
| 562 |
+
from modules import ar, customer_data, product_data
|
| 563 |
+
import routes_keychain
|
| 564 |
+
import routes_odoo_tables as rot
|
| 565 |
+
import time
|
| 566 |
+
|
| 567 |
+
team_labels = _selection_teams(session)
|
| 568 |
+
if (needed is None or getattr(session.runtime, 'key', None) != 'royal-imports'
|
| 569 |
+
or routes_keychain.odoo_paused(session.runtime)):
|
| 570 |
+
yield None
|
| 571 |
+
return
|
| 572 |
+
session.require(table_key)
|
| 573 |
+
team, agent = perm_scope.derive_pool_scope(session.user, table_key)
|
| 574 |
+
cache_key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
|
| 575 |
+
cached = session.runtime.pool_cache.get(cache_key)
|
| 576 |
+
if cached and time.time() - cached[0] < 900:
|
| 577 |
+
yield None
|
| 578 |
+
return # The canonical page must read the same source as the selection.
|
| 579 |
+
fields = metadata['fields']
|
| 580 |
+
scope_fields = (metadata.get('_permission_scope') or {}).get('fields', fields)
|
| 581 |
+
tree = {} if session.admin else (perm_scope.entry(session.user, table_key) or {}).get('filter') or {}
|
| 582 |
+
from harness import filter_eval
|
| 583 |
+
nodes, _conj = filter_eval.tree_parts(tree)
|
| 584 |
+
required = set(needed) | rot._leaf_cols(nodes)
|
| 585 |
+
# Set-dependent and authored permission operands retain their canonical evaluator.
|
| 586 |
+
def complex_wall(nodes):
|
| 587 |
+
return any(isinstance(n, dict) and (n.get('window') is not None
|
| 588 |
+
or n.get('op') in rot._fs().RANK_OPS or complex_wall(n.get('children') or [])) for n in nodes)
|
| 589 |
+
if complex_wall(nodes) or perm_scope._row_audience_field(table_key, st=session.runtime) is not None:
|
| 590 |
+
yield None
|
| 591 |
+
return
|
| 592 |
+
whitespace = "'" + ''.join(chr(c) for c in (*range(9, 14), *range(28, 33), 133, 160,
|
| 593 |
+
5760, *range(8192, 8203), 8232, 8233, 8239, 8287, 12288)) + "'"
|
| 594 |
+
if table_key == 'customer_data':
|
| 595 |
+
expressions = {
|
| 596 |
+
'archived': "CASE WHEN NOT live THEN true WHEN p.id IS NULL THEN false ELSE NOT coalesce(p.active, false) END",
|
| 597 |
+
'dba': "CASE WHEN live THEN coalesce(brands.brand, '') END",
|
| 598 |
+
'agent': "CASE WHEN live THEN coalesce((SELECT string_agg(a.name, ', ' ORDER BY cast(member.key AS BIGINT)) "
|
| 599 |
+
"FROM json_each(p.agent_ids) member JOIN res_partner a ON a.id=try_cast(member.value AS BIGINT) "
|
| 600 |
+
"WHERE coalesce(a.active, false) AND a.name IS NOT NULL AND a.name<>''), '(none)') ELSE '(none)' END"}
|
| 601 |
+
# These source attributes have no authored display override or case transformation.
|
| 602 |
+
for key in ('street', 'street2', 'zip'):
|
| 603 |
+
expressions[key] = f"CASE WHEN live THEN coalesce(nullif(trim(p.{key}, {whitespace}), ''), '(none)') ELSE '(none)' END"
|
| 604 |
+
for key, column in (('state', 'state_name'), ('country', 'country_name'),
|
| 605 |
+
('payment_terms', 'property_payment_term_name'),
|
| 606 |
+
('pricelist', 'property_product_pricelist_name')):
|
| 607 |
+
expressions[key] = f"CASE WHEN live THEN coalesce(nullif(p.{column}, ''), '(none)') ELSE '(none)' END"
|
| 608 |
+
for key in ('customer_since', 'created_on'):
|
| 609 |
+
expressions[key] = "CASE WHEN live THEN coalesce(substr(p.create_date, 1, 10), '') ELSE '' END"
|
| 610 |
+
else:
|
| 611 |
+
expressions = {'code': 'code'}
|
| 612 |
+
vocabulary = {f['key']: f for f in scope_fields}
|
| 613 |
+
visible = {f['key'] for f in fields}
|
| 614 |
+
if (not set(needed) <= visible or not required <= expressions.keys()
|
| 615 |
+
or any((vocabulary.get(k) or {}).get('source') != 'odoo'
|
| 616 |
+
or (vocabulary.get(k) or {}).get('derived')
|
| 617 |
+
or (vocabulary.get(k) or {}).get('type') not in {
|
| 618 |
+
'text', 'long_text', 'status', 'select', 'multiselect', 'date', 'checkbox'} for k in required)):
|
| 619 |
+
yield None
|
| 620 |
+
return
|
| 621 |
+
kind = 'product' if table_key == 'product_data' else 'customer'
|
| 622 |
+
missing = cs.mirror_missing(cur, kind)
|
| 623 |
+
if missing:
|
| 624 |
+
raise err(503, 'canonical_source_unavailable', cs._unready(missing))
|
| 625 |
+
hash_function = None
|
| 626 |
+
with cs.mirrored(cur, session.runtime, kind=kind) as source:
|
| 627 |
+
try:
|
| 628 |
+
if table_key == 'customer_data':
|
| 629 |
+
params, legs = [], []
|
| 630 |
+
domains = ([('sale.order', [('state', 'in', ('sale', 'done')), ('team_id', '=', team)], 'partner_id')]
|
| 631 |
+
if team else [
|
| 632 |
+
('res.partner', [('customer_rank', '>', 0), ('active', 'in', [True, False])], 'id'),
|
| 633 |
+
('sale.order', [('state', 'in', ('sale', 'done'))], 'partner_id'),
|
| 634 |
+
('account.move', [('state', '=', 'posted'), ('move_type', 'in', ('out_invoice', 'out_refund'))], 'partner_id'),
|
| 635 |
+
('account.move.line', list(ar.AGED_LINE_DOMAIN), 'partner_id')])
|
| 636 |
+
for model, domain, key in domains:
|
| 637 |
+
query = cs._Query(source, model, params)
|
| 638 |
+
value, field_kind, _ = query.field(key)
|
| 639 |
+
value = value[0] if field_kind == 'm2o' else value
|
| 640 |
+
where = query.where(domain if ids is None else [*domain, (key, 'in', list(ids))])
|
| 641 |
+
legs.append(f'SELECT {value} pid FROM {query.from_sql()} WHERE {where}')
|
| 642 |
+
cur.execute('CREATE TEMP TABLE canonical_live AS SELECT DISTINCT pid FROM ('
|
| 643 |
+
+ ' UNION ALL '.join(legs) + ') source WHERE pid IS NOT NULL', params)
|
| 644 |
+
if agent and agent not in {'All', 'All agents'}:
|
| 645 |
+
members = source.agent_members(agent)
|
| 646 |
+
cur.execute('DELETE FROM canonical_live WHERE pid NOT IN '
|
| 647 |
+
f'(SELECT try_cast(json_extract_string(body, \'/id\') AS BIGINT) FROM "{members.table}")')
|
| 648 |
+
cur.execute('CREATE TEMP TABLE canonical_retained (pid BIGINT)')
|
| 649 |
+
if not agent and team is None and session.runtime.exists(customer_data.LEDGER_KEY):
|
| 650 |
+
with closing(items(session.runtime, customer_data.LEDGER_KEY, selected=ids)) as batches:
|
| 651 |
+
for batch in batches:
|
| 652 |
+
retained = []
|
| 653 |
+
for key, _entry in batch:
|
| 654 |
+
try:
|
| 655 |
+
retained.append(int(key))
|
| 656 |
+
except (TypeError, ValueError):
|
| 657 |
+
continue
|
| 658 |
+
if retained:
|
| 659 |
+
cur.execute('INSERT INTO canonical_retained SELECT unnest(?::BIGINT[])', [retained])
|
| 660 |
+
population = ('SELECT pid, true live FROM canonical_live UNION ALL '
|
| 661 |
+
'SELECT DISTINCT pid, false live FROM canonical_retained '
|
| 662 |
+
'WHERE pid NOT IN (SELECT pid FROM canonical_live)')
|
| 663 |
+
projected = ', '.join(f'{expressions[k]} AS "{k}"' for k in sorted(required))
|
| 664 |
+
brand_join = ''
|
| 665 |
+
if 'dba' in required:
|
| 666 |
+
import datetime
|
| 667 |
+
start = (datetime.date.fromisoformat(str(metadata['today'])) - datetime.timedelta(days=731)).isoformat()
|
| 668 |
+
query = cs._Query(source, 'sale.order')
|
| 669 |
+
where = query.where([('state', 'in', ('sale', 'done')), ('date_order', '>=', start),
|
| 670 |
+
('team_id', 'in', list(team_labels))])
|
| 671 |
+
cases = ' '.join('WHEN ? THEN ?' for _ in team_labels)
|
| 672 |
+
label_params = [value for pair in team_labels.items() for value in pair]
|
| 673 |
+
cur.execute('CREATE TEMP TABLE canonical_brands AS SELECT b.partner_id pid, '
|
| 674 |
+
f'CASE WHEN count(DISTINCT b.team_id)>1 THEN ? ELSE CASE min(b.team_id) {cases} END END AS brand '
|
| 675 |
+
f'FROM {query.from_sql()} WHERE {where} AND b.partner_id<>0 GROUP BY b.partner_id',
|
| 676 |
+
[customer_data._multi_team_label(len(team_labels)), *label_params, *query.params])
|
| 677 |
+
brand_join = ' LEFT JOIN canonical_brands brands ON brands.pid=u.pid'
|
| 678 |
+
# Absent optional columns retain the canonical path's blank/refusal behavior.
|
| 679 |
+
try:
|
| 680 |
+
cur.execute('CREATE TEMP TABLE canonical_selection AS SELECT u.pid'
|
| 681 |
+
+ (', ' + projected if projected else '')
|
| 682 |
+
+ f' FROM ({population}) u LEFT JOIN res_partner p ON p.id=u.pid' + brand_join)
|
| 683 |
+
except Exception as exc:
|
| 684 |
+
from core.request_timing import raise_if_resource_error
|
| 685 |
+
raise_if_resource_error(exc)
|
| 686 |
+
import duckdb
|
| 687 |
+
if not isinstance(exc, duckdb.BinderException):
|
| 688 |
+
raise
|
| 689 |
+
yield None
|
| 690 |
+
return
|
| 691 |
+
else:
|
| 692 |
+
import uuid
|
| 693 |
+
function_name = 'canonical_pid_' + uuid.uuid4().hex
|
| 694 |
+
cur.create_function(function_name, lambda code: product_data.sku_pid(code), ['VARCHAR'], 'BIGINT')
|
| 695 |
+
hash_function = function_name
|
| 696 |
+
cur.execute('CREATE TEMP TABLE canonical_selection AS SELECT '
|
| 697 |
+
f'{hash_function}(code) pid, code FROM (SELECT DISTINCT '
|
| 698 |
+
"CASE WHEN default_code IS NULL OR default_code='' THEN 'pid:'||CAST(id AS VARCHAR) "
|
| 699 |
+
f'ELSE trim(default_code, {whitespace}) END code FROM product_product WHERE active=true) codes')
|
| 700 |
+
if cur.execute('SELECT pid FROM canonical_selection GROUP BY pid HAVING count(*)>1 LIMIT 1').fetchone():
|
| 701 |
+
raise err(503, 'canonical_source_unavailable', 'product_data: pid collision. Refusing the catalogue.')
|
| 702 |
+
bindings = {k: (f'"{k}"', str) for k in required}
|
| 703 |
+
spec = {'table': 'canonical_selection', 'id': 'pid', 'cols': bindings}
|
| 704 |
+
wall_cols = rot._filter_columns(spec, scope_fields)
|
| 705 |
+
# Permission booleans retain their source spelling in the canonical payload.
|
| 706 |
+
wall, params = perm_scope._window_wall(session.user, table_key, scope_fields,
|
| 707 |
+
wall_cols, id_sql='pid', st=session.runtime)
|
| 708 |
+
if ids is not None:
|
| 709 |
+
wall = f'({wall}) AND pid IN (SELECT unnest(?::BIGINT[]))'
|
| 710 |
+
params.append(list(ids))
|
| 711 |
+
cur.execute(f'DELETE FROM canonical_selection WHERE NOT coalesce(({wall}), false)', params)
|
| 712 |
+
yield spec
|
| 713 |
+
except Exception as exc:
|
| 714 |
+
from fastapi import HTTPException
|
| 715 |
+
from core.request_timing import raise_if_resource_error
|
| 716 |
+
if isinstance(exc, HTTPException):
|
| 717 |
+
raise
|
| 718 |
+
raise_if_resource_error(exc)
|
| 719 |
+
raise err(503, 'canonical_source_unavailable', f'{table_key}: {type(exc).__name__}: {str(exc)[:180]}') from exc
|
| 720 |
+
finally:
|
| 721 |
+
if hash_function:
|
| 722 |
+
cur.remove_function(hash_function)
|
| 723 |
+
|
| 724 |
+
|
| 725 |
def canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None):
|
| 726 |
"""Refuse a failed compiled source by name without publishing an empty population."""
|
| 727 |
from fastapi import HTTPException
|
api/rollup_sql.py
CHANGED
|
@@ -59,7 +59,7 @@ def source_fields(table_def):
|
|
| 59 |
return out
|
| 60 |
|
| 61 |
|
| 62 |
-
def group_values(bag, today=None):
|
| 63 |
"""`{group_id: value}` for one source bag — ONE grouped query over the whole scope.
|
| 64 |
|
| 65 |
⚠ Keyed by the dim's `_id` column, never its label. Two customers can share a display name;
|
|
@@ -67,6 +67,7 @@ def group_values(bag, today=None):
|
|
| 67 |
"""
|
| 68 |
from harness import semantic as sem
|
| 69 |
from harness import windows as W
|
|
|
|
| 70 |
|
| 71 |
topic = str(bag.get("topic") or "")
|
| 72 |
measure = str(bag.get("measure") or "")
|
|
@@ -86,9 +87,20 @@ def group_values(bag, today=None):
|
|
| 86 |
date_from, date_to = rng
|
| 87 |
|
| 88 |
try:
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
except Exception as e: # noqa: BLE001
|
| 93 |
raise RollupSourceError(f"{type(e).__name__}: {e}") from e
|
| 94 |
|
|
@@ -121,8 +133,8 @@ def group_values(bag, today=None):
|
|
| 121 |
|
| 122 |
|
| 123 |
def _today():
|
| 124 |
-
|
| 125 |
-
return
|
| 126 |
|
| 127 |
|
| 128 |
def compute(rt, table_key, today=None, tables=None, persist=False):
|
|
@@ -146,7 +158,7 @@ def compute(rt, table_key, today=None, tables=None, persist=False):
|
|
| 146 |
bag = f["rollup"]["source"]
|
| 147 |
# ⚠ Resolved BEFORE anything is written. A rollup that refuses must leave every cell as it
|
| 148 |
# was — a partially applied pass would mix two vintages of the same column.
|
| 149 |
-
plans.append((f["key"], str(bag.get("on") or ""), group_values(bag, today=stamp)))
|
| 150 |
|
| 151 |
written = {}
|
| 152 |
|
|
|
|
| 59 |
return out
|
| 60 |
|
| 61 |
|
| 62 |
+
def group_values(rt, bag, today=None):
|
| 63 |
"""`{group_id: value}` for one source bag — ONE grouped query over the whole scope.
|
| 64 |
|
| 65 |
⚠ Keyed by the dim's `_id` column, never its label. Two customers can share a display name;
|
|
|
|
| 67 |
"""
|
| 68 |
from harness import semantic as sem
|
| 69 |
from harness import windows as W
|
| 70 |
+
from harness.runtime import mirror_cursor
|
| 71 |
|
| 72 |
topic = str(bag.get("topic") or "")
|
| 73 |
measure = str(bag.get("measure") or "")
|
|
|
|
| 87 |
date_from, date_to = rng
|
| 88 |
|
| 89 |
try:
|
| 90 |
+
if rt is None:
|
| 91 |
+
raise RollupSourceError("A tenant runtime is required for source rollups")
|
| 92 |
+
handle = rt.mirror_handle()
|
| 93 |
+
if handle is None:
|
| 94 |
+
raise RollupSourceError("The runtime has no tenant-owned mirror handle")
|
| 95 |
+
cur = mirror_cursor(rt)
|
| 96 |
+
try:
|
| 97 |
+
if getattr(cur, "_mirror_handle", None) is not handle:
|
| 98 |
+
raise RollupSourceError("The runtime has no tenant-owned mirror cursor")
|
| 99 |
+
res = sem.store_query(topic, [measure], group_by=[dim],
|
| 100 |
+
date_from=date_from, date_to=date_to,
|
| 101 |
+
limit=sem.MAX_GROUPS, today=today, runtime=rt, cur=cur)
|
| 102 |
+
finally:
|
| 103 |
+
cur.close()
|
| 104 |
except Exception as e: # noqa: BLE001
|
| 105 |
raise RollupSourceError(f"{type(e).__name__}: {e}") from e
|
| 106 |
|
|
|
|
| 133 |
|
| 134 |
|
| 135 |
def _today():
|
| 136 |
+
from core import periods
|
| 137 |
+
return periods.today().isoformat()
|
| 138 |
|
| 139 |
|
| 140 |
def compute(rt, table_key, today=None, tables=None, persist=False):
|
|
|
|
| 158 |
bag = f["rollup"]["source"]
|
| 159 |
# ⚠ Resolved BEFORE anything is written. A rollup that refuses must leave every cell as it
|
| 160 |
# was — a partially applied pass would mix two vintages of the same column.
|
| 161 |
+
plans.append((f["key"], str(bag.get("on") or ""), group_values(rt, bag, today=stamp)))
|
| 162 |
|
| 163 |
written = {}
|
| 164 |
|
api/routes_account_health.py
CHANGED
|
@@ -14,13 +14,15 @@ def _module():
|
|
| 14 |
return account_health
|
| 15 |
|
| 16 |
|
| 17 |
-
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 18 |
-
"""
|
| 19 |
-
|
| 20 |
if session.tenant != getattr(session.runtime, "key", None):
|
| 21 |
raise err(403, "account_health_forbidden", "This workspace does not match your session.")
|
| 22 |
-
|
| 23 |
-
|
|
|
|
|
|
|
| 24 |
return session
|
| 25 |
|
| 26 |
|
|
@@ -104,8 +106,12 @@ def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
|
|
| 104 |
offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 105 |
metric: str | None = Query(None), session: Session = Depends(_app_reader)):
|
| 106 |
ah = _module()
|
| 107 |
-
|
| 108 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 110 |
|
| 111 |
|
|
@@ -113,6 +119,17 @@ def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
|
|
| 113 |
def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 114 |
limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
|
| 115 |
ah = _module()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
|
| 117 |
|
| 118 |
|
|
@@ -122,8 +139,13 @@ def list_items(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 122 |
cursor: str | None = Query(None), limit: int = Query(50, ge=1),
|
| 123 |
session: Session = Depends(_app_reader)):
|
| 124 |
ah = _module()
|
| 125 |
-
|
| 126 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 128 |
|
| 129 |
|
|
@@ -131,8 +153,10 @@ def list_items(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 131 |
def read_item(item_id: str, app: str = Query(..., min_length=1),
|
| 132 |
session: Session = Depends(_app_reader)):
|
| 133 |
ah = _module()
|
|
|
|
| 134 |
alert = _call(ah.alert_record, _tenant(session), item_id, st=session.runtime)
|
| 135 |
-
context = _call(
|
|
|
|
| 136 |
if not _source_allowed(alert, context):
|
| 137 |
raise err(403, "account_health_source_forbidden", "Your account cannot read this alert's source records and fields.")
|
| 138 |
return {"item": ah._public(alert, session.user), "owners": owners(session),
|
|
@@ -144,6 +168,10 @@ def patch_item(item_id: str, owner: str | None = Body(None), status: str | None
|
|
| 144 |
due: str | None = Body(None), note: str | None = Body(None),
|
| 145 |
session: Session = Depends(_app_writer)):
|
| 146 |
ah = _module()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
return _call(ah.update_item, _tenant(session), session.user, item_id, owner=owner, status=status,
|
| 148 |
due=due, note=note, st=session.runtime)
|
| 149 |
|
|
@@ -168,6 +196,10 @@ def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 168 |
search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 169 |
session: Session = Depends(_app_reader)):
|
| 170 |
ah = _module()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
out = _call(ah.risks, _tenant(session), status=status, search=search, offset=offset, limit=limit,
|
| 172 |
st=session.runtime)
|
| 173 |
return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
|
@@ -179,10 +211,109 @@ def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None
|
|
| 179 |
outcome: str | None = Body(None), note: str | None = Body(None),
|
| 180 |
session: Session = Depends(_app_writer)):
|
| 181 |
ah = _module()
|
|
|
|
|
|
|
|
|
|
| 182 |
return _call(ah.update_risk, _tenant(session), session.user, item_id, owner=owner, status=status,
|
| 183 |
case_id=caseId, appeal_date=appealDate, outcome=outcome, note=note, st=session.runtime)
|
| 184 |
|
| 185 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
# --------------------------------------------------------------------------- C5 worker bridge
|
| 187 |
|
| 188 |
def _source_context(rt, user):
|
|
|
|
| 14 |
return account_health
|
| 15 |
|
| 16 |
|
| 17 |
+
def _reader(app: str | None = Query(None), session: Session = Depends(require_session)) -> Session:
|
| 18 |
+
"""R62: reads need the App grant plus the shared source check, never the source open grant."""
|
| 19 |
+
import application_registry
|
| 20 |
if session.tenant != getattr(session.runtime, "key", None):
|
| 21 |
raise err(403, "account_health_forbidden", "This workspace does not match your session.")
|
| 22 |
+
record = _app(session, app) if app else _call(application_registry.readable_app, session.runtime,
|
| 23 |
+
_tenant(session), session.uname, session.admin, TEMPLATE_KIND)
|
| 24 |
+
application_registry.admit_app(session, record, (SOURCE,), code="account_health")
|
| 25 |
+
session.source_app = record
|
| 26 |
return session
|
| 27 |
|
| 28 |
|
|
|
|
| 106 |
offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 107 |
metric: str | None = Query(None), session: Session = Depends(_app_reader)):
|
| 108 |
ah = _module()
|
| 109 |
+
context = _call(_narrowed, session)
|
| 110 |
+
if context is None:
|
| 111 |
+
out = _call(ah.summary_page, _tenant(session), search=search, offset=offset, limit=limit,
|
| 112 |
+
metric=metric, st=session.runtime)
|
| 113 |
+
else:
|
| 114 |
+
out = _narrow_summary(session, context, search=search, offset=offset, limit=limit, metric=metric)
|
| 115 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 116 |
|
| 117 |
|
|
|
|
| 119 |
def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 120 |
limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
|
| 121 |
ah = _module()
|
| 122 |
+
context = _call(_narrowed, session)
|
| 123 |
+
if context is not None:
|
| 124 |
+
kept = _allowed_metrics(session, context)["metrics"]
|
| 125 |
+
row = next((row for row in kept if row["id"] == metric), None) or next(
|
| 126 |
+
(row for row in kept if str(row["id"]).split("@", 1)[0] == metric), None)
|
| 127 |
+
if row is None:
|
| 128 |
+
raise err(404, "account_health_not_found", "That alert, risk or metric does not exist.")
|
| 129 |
+
out = _call(ah.history, _tenant(session), row["id"], cursor=cursor, limit=limit, st=session.runtime)
|
| 130 |
+
out["points"] = [point for point in out["points"] if _source_allowed({"sourceEvidence": _call(
|
| 131 |
+
ah._alert_source, _tenant(session), session.runtime, row, point["day"])}, context)]
|
| 132 |
+
return out
|
| 133 |
return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
|
| 134 |
|
| 135 |
|
|
|
|
| 139 |
cursor: str | None = Query(None), limit: int = Query(50, ge=1),
|
| 140 |
session: Session = Depends(_app_reader)):
|
| 141 |
ah = _module()
|
| 142 |
+
context = _call(_narrowed, session)
|
| 143 |
+
if context is None:
|
| 144 |
+
listed = _call(ah.items, _tenant(session), session.user, status=status, search=search,
|
| 145 |
+
offset=offset, cursor=cursor, limit=limit, metric=metric, st=session.runtime)
|
| 146 |
+
else:
|
| 147 |
+
listed = _narrow_items(session, context, offset, limit, status=status, search=search,
|
| 148 |
+
cursor=cursor, metric=metric)
|
| 149 |
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 150 |
|
| 151 |
|
|
|
|
| 153 |
def read_item(item_id: str, app: str = Query(..., min_length=1),
|
| 154 |
session: Session = Depends(_app_reader)):
|
| 155 |
ah = _module()
|
| 156 |
+
import application_registry
|
| 157 |
alert = _call(ah.alert_record, _tenant(session), item_id, st=session.runtime)
|
| 158 |
+
context = _call(application_registry.app_source_scope, session.user, session.runtime, SOURCE,
|
| 159 |
+
app=session.source_app)
|
| 160 |
if not _source_allowed(alert, context):
|
| 161 |
raise err(403, "account_health_source_forbidden", "Your account cannot read this alert's source records and fields.")
|
| 162 |
return {"item": ah._public(alert, session.user), "owners": owners(session),
|
|
|
|
| 168 |
due: str | None = Body(None), note: str | None = Body(None),
|
| 169 |
session: Session = Depends(_app_writer)):
|
| 170 |
ah = _module()
|
| 171 |
+
context = _call(_narrowed, session)
|
| 172 |
+
if context is not None and not _source_allowed(
|
| 173 |
+
_call(ah.alert_record, _tenant(session), item_id, st=session.runtime), context):
|
| 174 |
+
raise err(404, "account_health_not_found", "That alert, risk or metric does not exist.")
|
| 175 |
return _call(ah.update_item, _tenant(session), session.user, item_id, owner=owner, status=status,
|
| 176 |
due=due, note=note, st=session.runtime)
|
| 177 |
|
|
|
|
| 196 |
search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 197 |
session: Session = Depends(_app_reader)):
|
| 198 |
ah = _module()
|
| 199 |
+
context = _call(_narrowed, session)
|
| 200 |
+
if context is not None:
|
| 201 |
+
out = _narrow_risks(session, context, status=status, search=search, offset=offset, limit=limit)
|
| 202 |
+
return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 203 |
out = _call(ah.risks, _tenant(session), status=status, search=search, offset=offset, limit=limit,
|
| 204 |
st=session.runtime)
|
| 205 |
return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
|
|
|
| 211 |
outcome: str | None = Body(None), note: str | None = Body(None),
|
| 212 |
session: Session = Depends(_app_writer)):
|
| 213 |
ah = _module()
|
| 214 |
+
context = _call(_narrowed, session)
|
| 215 |
+
if context is not None:
|
| 216 |
+
raise err(403, "account_health_source_forbidden", "Risk changes need unrestricted source access.")
|
| 217 |
return _call(ah.update_risk, _tenant(session), session.user, item_id, owner=owner, status=status,
|
| 218 |
case_id=caseId, appeal_date=appealDate, outcome=outcome, note=note, st=session.runtime)
|
| 219 |
|
| 220 |
|
| 221 |
+
# --------------------------------------------------------------------------- R62 source narrowing
|
| 222 |
+
|
| 223 |
+
def _narrowed(session):
|
| 224 |
+
"""The caller's (visible fields, permitted row ids) on the source, or None when no wall applies."""
|
| 225 |
+
import application_registry
|
| 226 |
+
wall = application_registry.app_access(session.user, session.runtime, None, (SOURCE,))["narrow"][SOURCE]
|
| 227 |
+
if not wall["rows"] and not wall["hidden"]:
|
| 228 |
+
return None
|
| 229 |
+
return application_registry.app_source_scope(session.user, session.runtime, SOURCE, app=session.source_app)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _allowed_metrics(session, context, *, search="", metric=None):
|
| 233 |
+
"""Every metric row whose source row and fields the caller's source wall admits."""
|
| 234 |
+
ah = _module()
|
| 235 |
+
tenant, rt = _tenant(session), session.runtime
|
| 236 |
+
first = _call(ah.summary_page, tenant, search=search, offset=0, limit=1, metric=metric, st=rt)
|
| 237 |
+
full = first if first["metricTotal"] <= 1 else _call(
|
| 238 |
+
ah.summary_page, tenant, search=search, offset=0, limit=first["metricTotal"], metric=metric, st=rt)
|
| 239 |
+
kept = [row for row in full["metrics"] if _source_allowed(
|
| 240 |
+
{"sourceEvidence": _call(ah._alert_source, tenant, rt, row, full["snapshotDate"])}, context)]
|
| 241 |
+
return {**full, "metrics": kept}
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def _page(rows, offset, limit):
|
| 245 |
+
start = min(max(0, offset), max(0, (len(rows) - 1) // limit * limit))
|
| 246 |
+
return start, rows[start:start + limit]
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _narrow_summary(session, context, *, search, offset, limit, metric):
|
| 250 |
+
"""A source row filter narrows the metric rows; a hidden source field hides its metric."""
|
| 251 |
+
ah = _module()
|
| 252 |
+
out = _allowed_metrics(session, context, search=search, metric=metric)
|
| 253 |
+
kept = out["metrics"]
|
| 254 |
+
if metric is not None and not kept:
|
| 255 |
+
raise err(404, "account_health_not_found", "That alert, risk or metric does not exist.")
|
| 256 |
+
markets = {row.get("marketplaceId") for row in kept}
|
| 257 |
+
if "groups" in out:
|
| 258 |
+
scoped = _allowed_metrics(session, context)["metrics"] if search or metric is not None else kept
|
| 259 |
+
groups, gauge = ah.group_scores(ah.count_groups(scoped))
|
| 260 |
+
out.update(groups=groups, gauge=gauge,
|
| 261 |
+
foldedCount=sum(row.get("status") in ah.FOLDED for row in scoped))
|
| 262 |
+
start, page = _page(kept, offset, limit)
|
| 263 |
+
out.update(metrics=page, metricTotal=len(kept), metricOffset=start, metricLimit=limit,
|
| 264 |
+
marketCount=len(markets), cursor=str(start + limit) if start + limit < len(kept) else None)
|
| 265 |
+
if out.get("marketplaceId") not in markets or "account_health_rating_ahr_score" not in context[0]:
|
| 266 |
+
out.update(rating=None, ratingStatus=None, marketplaceId=None)
|
| 267 |
+
elif "account_health_rating_ahr_status" not in context[0]:
|
| 268 |
+
out["ratingStatus"] = None
|
| 269 |
+
if not kept:
|
| 270 |
+
out.update(state="empty", message=ah.EMPTY, snapshotDate=None)
|
| 271 |
+
return out
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def _narrow_items(session, context, offset, limit, **options):
|
| 275 |
+
"""Keep the alerts whose source rows and fields the caller's source wall admits."""
|
| 276 |
+
ah = _module()
|
| 277 |
+
rows, scan = [], 0
|
| 278 |
+
while True:
|
| 279 |
+
listed = _call(ah.items, _tenant(session), session.user, offset=scan, limit=200,
|
| 280 |
+
st=session.runtime, **options)
|
| 281 |
+
for row in listed["items"]:
|
| 282 |
+
alert = _call(ah.alert_record, _tenant(session), row["id"], st=session.runtime)
|
| 283 |
+
if _source_allowed(alert, context):
|
| 284 |
+
rows.append(row)
|
| 285 |
+
scan += len(listed["items"])
|
| 286 |
+
if not listed["items"] or scan >= listed["total"]:
|
| 287 |
+
break
|
| 288 |
+
start, page = _page(rows, offset, limit)
|
| 289 |
+
more = start + len(page) < len(rows)
|
| 290 |
+
return {**listed, "items": page, "cursor": page[-1]["id"] if more and page else None,
|
| 291 |
+
"total": len(rows), "offset": start, "limit": limit}
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def _narrow_risks(session, context, *, status, search, offset, limit):
|
| 295 |
+
"""Serve only risks whose source evidence fits the Account health wall."""
|
| 296 |
+
ah = _module()
|
| 297 |
+
metrics = {row["id"]: row for row in _allowed_metrics(session, context)["metrics"]}
|
| 298 |
+
kept, scan = [], 0
|
| 299 |
+
while True:
|
| 300 |
+
out = _call(ah.risks, _tenant(session), status=status, search=search,
|
| 301 |
+
offset=scan, limit=200, st=session.runtime)
|
| 302 |
+
for risk in out["items"]:
|
| 303 |
+
metric = metrics.get(risk.get("metricId"))
|
| 304 |
+
if metric is not None and _source_allowed({"sourceEvidence": _call(
|
| 305 |
+
ah._alert_source, _tenant(session), session.runtime, metric, risk.get("snapshotDate"))}, context):
|
| 306 |
+
kept.append(risk)
|
| 307 |
+
scan += len(out["items"])
|
| 308 |
+
if not out["items"] or scan >= out["total"]:
|
| 309 |
+
break
|
| 310 |
+
start, page = _page(kept, offset, limit)
|
| 311 |
+
return {**out, "items": page, "total": len(kept), "offset": start, "limit": limit,
|
| 312 |
+
"cursor": str(start + limit) if start + limit < len(kept) else None,
|
| 313 |
+
"reserveSummary": {"available": False, "total": None, "byCurrency": {}, "lineCount": 0,
|
| 314 |
+
"unknownReasons": ["Reserve figures need unrestricted source access."]}}
|
| 315 |
+
|
| 316 |
+
|
| 317 |
# --------------------------------------------------------------------------- C5 worker bridge
|
| 318 |
|
| 319 |
def _source_context(rt, user):
|
api/routes_admin.py
CHANGED
|
@@ -1464,6 +1464,8 @@ _VOCAB_ARM_CACHE = {}
|
|
| 1464 |
#: The one arm whose answer belongs to the CALLER's tenant, named once so both the memo and the
|
| 1465 |
#: stamp order can refuse it rather than each restating the rule.
|
| 1466 |
_TENANT_OWNED_ARM = "_product_shared_choices"
|
|
|
|
|
|
|
| 1467 |
|
| 1468 |
|
| 1469 |
def _server_side_vocabularies(session=None):
|
|
@@ -1560,15 +1562,31 @@ def _server_side_vocabularies(session=None):
|
|
| 1560 |
"occassion", "shape_style", "size", "sub_category")
|
| 1561 |
return {key: (stored.get(key) or {}).get("options") or () for key in keys}
|
| 1562 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1563 |
if _VOCAB_CACHE is not None:
|
| 1564 |
out = dict(_VOCAB_CACHE)
|
| 1565 |
-
|
| 1566 |
-
|
| 1567 |
-
|
| 1568 |
-
|
| 1569 |
-
|
| 1570 |
-
|
| 1571 |
-
|
|
|
|
| 1572 |
return out
|
| 1573 |
|
| 1574 |
out = {}
|
|
@@ -1582,7 +1600,7 @@ def _server_side_vocabularies(session=None):
|
|
| 1582 |
normalisation rather than two that can drift.
|
| 1583 |
"""
|
| 1584 |
name = getattr(fn, "__name__", "")
|
| 1585 |
-
memoable = name and name
|
| 1586 |
try:
|
| 1587 |
got = _VOCAB_ARM_CACHE[name] if memoable and name in _VOCAB_ARM_CACHE else fn()
|
| 1588 |
for key, values in (got or {}).items():
|
|
@@ -1708,6 +1726,7 @@ def _server_side_vocabularies(session=None):
|
|
| 1708 |
# them beside Odoo's shared Royal vocabulary would leak one tenant's Product taxonomy into
|
| 1709 |
# another tenant's permission editor for the life of the worker.
|
| 1710 |
arm(_product_shared_choices)
|
|
|
|
| 1711 |
return out
|
| 1712 |
|
| 1713 |
|
|
@@ -2543,6 +2562,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2543 |
return {"username": uname,
|
| 2544 |
"perms": perms_out,
|
| 2545 |
"stored_perms": deepcopy(stored),
|
|
|
|
|
|
|
| 2546 |
# S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a
|
| 2547 |
# migrated record and LEGACY on an un-migrated one: opposite meanings for identical
|
| 2548 |
# JSON, so the client cannot render honestly without knowing which world it is in.
|
|
@@ -2867,6 +2888,84 @@ def grant_database_access(session, username, database, role):
|
|
| 2867 |
return {'username': uname, 'database': key, 'role': role}
|
| 2868 |
|
| 2869 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2870 |
@router.put("/admin/users/{username}/perms")
|
| 2871 |
def put_perms(username: str, body: dict = Body(default=None),
|
| 2872 |
session: Session = Depends(admin_gate)):
|
|
|
|
| 1464 |
#: The one arm whose answer belongs to the CALLER's tenant, named once so both the memo and the
|
| 1465 |
#: stamp order can refuse it rather than each restating the rule.
|
| 1466 |
_TENANT_OWNED_ARM = "_product_shared_choices"
|
| 1467 |
+
#: Every arm whose answer depends on the session's tenant: never memoed, never in the process cache.
|
| 1468 |
+
_TENANT_OWNED_ARMS = frozenset({_TENANT_OWNED_ARM, "_dba_choices"})
|
| 1469 |
|
| 1470 |
|
| 1471 |
def _server_side_vocabularies(session=None):
|
|
|
|
| 1562 |
"occassion", "shape_style", "size", "sub_category")
|
| 1563 |
return {key: (stored.get(key) or {}).get("options") or () for key in keys}
|
| 1564 |
|
| 1565 |
+
def _dba_choices():
|
| 1566 |
+
"""The customer `dba` choices: the session tenant's team labels plus one multi-team value.
|
| 1567 |
+
|
| 1568 |
+
Read under the tenant's own Odoo binding, so its configured teams answer. A tenant with
|
| 1569 |
+
no Odoo source supplies nothing rather than the process default tenant's teams."""
|
| 1570 |
+
if session is None:
|
| 1571 |
+
return {}
|
| 1572 |
+
import modules.customer_data as customer_data
|
| 1573 |
+
|
| 1574 |
+
src = session.runtime.odoo_source()
|
| 1575 |
+
if src is None:
|
| 1576 |
+
return {}
|
| 1577 |
+
with src._bound():
|
| 1578 |
+
return {"dba": customer_data.dba_options()}
|
| 1579 |
+
|
| 1580 |
if _VOCAB_CACHE is not None:
|
| 1581 |
out = dict(_VOCAB_CACHE)
|
| 1582 |
+
for tenant_arm in (_product_shared_choices, _dba_choices):
|
| 1583 |
+
try:
|
| 1584 |
+
for key, values in tenant_arm().items():
|
| 1585 |
+
clean = sorted({str(v) for v in values if str(v or "").strip()})
|
| 1586 |
+
if clean:
|
| 1587 |
+
out[key] = clean
|
| 1588 |
+
except Exception: # noqa: BLE001
|
| 1589 |
+
pass
|
| 1590 |
return out
|
| 1591 |
|
| 1592 |
out = {}
|
|
|
|
| 1600 |
normalisation rather than two that can drift.
|
| 1601 |
"""
|
| 1602 |
name = getattr(fn, "__name__", "")
|
| 1603 |
+
memoable = name and name not in _TENANT_OWNED_ARMS
|
| 1604 |
try:
|
| 1605 |
got = _VOCAB_ARM_CACHE[name] if memoable and name in _VOCAB_ARM_CACHE else fn()
|
| 1606 |
for key, values in (got or {}).items():
|
|
|
|
| 1726 |
# them beside Odoo's shared Royal vocabulary would leak one tenant's Product taxonomy into
|
| 1727 |
# another tenant's permission editor for the life of the worker.
|
| 1728 |
arm(_product_shared_choices)
|
| 1729 |
+
arm(_dba_choices)
|
| 1730 |
return out
|
| 1731 |
|
| 1732 |
|
|
|
|
| 2562 |
return {"username": uname,
|
| 2563 |
"perms": perms_out,
|
| 2564 |
"stored_perms": deepcopy(stored),
|
| 2565 |
+
# The `expect` that `PUT /admin/users/{u}/perms/{database}` compares before writing.
|
| 2566 |
+
"access_state": database_access_token(rec),
|
| 2567 |
# S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a
|
| 2568 |
# migrated record and LEGACY on an un-migrated one: opposite meanings for identical
|
| 2569 |
# JSON, so the client cannot render honestly without knowing which world it is in.
|
|
|
|
| 2888 |
return {'username': uname, 'database': key, 'role': role}
|
| 2889 |
|
| 2890 |
|
| 2891 |
+
def database_access_token(record):
|
| 2892 |
+
"""A stable digest of `users.database_access_state(record)` that a client sends back as `expect`."""
|
| 2893 |
+
import hashlib
|
| 2894 |
+
state = users.database_access_state(record or {})
|
| 2895 |
+
raw = json.dumps(state, sort_keys=True, separators=(",", ":"), default=str)
|
| 2896 |
+
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
| 2897 |
+
|
| 2898 |
+
|
| 2899 |
+
def set_database_block(session, username, database, entry, *, expect, confirm_widen=False):
|
| 2900 |
+
"""Write one database's access, row filter and hidden fields in one account compare-and-set.
|
| 2901 |
+
|
| 2902 |
+
`expect` is the `users.database_access_state` dict, or its `database_access_token`, the caller
|
| 2903 |
+
read. A changed account answers 409 and writes nothing.
|
| 2904 |
+
"""
|
| 2905 |
+
import copy
|
| 2906 |
+
from core import user_tables
|
| 2907 |
+
|
| 2908 |
+
admin_gate(session)
|
| 2909 |
+
uname = str(username or "").strip().lower()
|
| 2910 |
+
key = str(database or "").strip()
|
| 2911 |
+
if not isinstance(entry, dict) or type(entry.get("access")) is not bool:
|
| 2912 |
+
raise err(400, "bad_perms", "The entry needs access set to true or false.")
|
| 2913 |
+
if not isinstance(expect, (dict, str)) or not expect:
|
| 2914 |
+
raise err(400, "missing_expect", "Read the account permissions first, then send its access state.")
|
| 2915 |
+
tenant = str(session.runtime.tenant.slug).strip().lower()
|
| 2916 |
+
if _tenant_of(session.user) != tenant:
|
| 2917 |
+
raise err(403, "forbidden", "Database access could not be changed.")
|
| 2918 |
+
# Read the account store strictly rather than the directory's fail-silent projection.
|
| 2919 |
+
reg = store.get("users", fresh=True)
|
| 2920 |
+
rec = (reg or {}).get(uname)
|
| 2921 |
+
if not isinstance(rec, dict) or not rec.get("active", True) or _tenant_of(rec) != tenant:
|
| 2922 |
+
raise err(404, "no_such_user", "That account is not active in this workspace.")
|
| 2923 |
+
if uname == session.uname:
|
| 2924 |
+
raise err(400, "self_perms", "Ask another administrator to change your access.")
|
| 2925 |
+
governed = {m["key"] for m in _perm_modules(session)}
|
| 2926 |
+
if key not in governed:
|
| 2927 |
+
raise err(404, "no_database", "That database is not available in this workspace.")
|
| 2928 |
+
state = users.database_access_state(rec)
|
| 2929 |
+
if (state if isinstance(expect, dict) else database_access_token(rec)) != expect:
|
| 2930 |
+
raise err(409, "permissions_changed", "Account access changed. Read it again and retry.")
|
| 2931 |
+
submitted = {field: copy.deepcopy(entry[field]) for field in users.DATABASE_ENTRY_KEYS
|
| 2932 |
+
if field in entry}
|
| 2933 |
+
cleaned = _clean_perms({key: submitted}, governed_keys=governed, session=session,
|
| 2934 |
+
stored=rec.get("perms"))
|
| 2935 |
+
folded = _fold_legacy_scope(rec, cleaned)
|
| 2936 |
+
_guard_bu_widening(rec, folded, bool(confirm_widen))
|
| 2937 |
+
candidate = {field: copy.deepcopy(folded[key][field]) for field in users.DATABASE_ENTRY_KEYS}
|
| 2938 |
+
prior = copy.deepcopy((rec.get("perms") or {}).get(key))
|
| 2939 |
+
result = users.set_database_access(uname, key, candidate, actor=session.uname,
|
| 2940 |
+
tenant=tenant, expect=state)
|
| 2941 |
+
status = result.get("status")
|
| 2942 |
+
if status == "conflict":
|
| 2943 |
+
raise err(409, "permissions_changed", "Account access changed. Read it again and retry.")
|
| 2944 |
+
if status == "missing":
|
| 2945 |
+
raise err(404, "no_such_user", "That account is not active in this workspace.")
|
| 2946 |
+
if status != "updated":
|
| 2947 |
+
raise err(403, "forbidden", "Database access could not be changed.")
|
| 2948 |
+
written = ((result.get("after") or {}).get("perms") or {}).get(key) or candidate
|
| 2949 |
+
if key.startswith(user_tables.KEY_PREFIX):
|
| 2950 |
+
_sync_database_shares(session, uname, {key: prior} if prior else {}, {key: written})
|
| 2951 |
+
return {"username": uname, "database": key, "entry": copy.deepcopy(written),
|
| 2952 |
+
"access_state": database_access_token(result.get("after"))}
|
| 2953 |
+
|
| 2954 |
+
|
| 2955 |
+
@router.put("/admin/users/{username}/perms/{database}")
|
| 2956 |
+
def put_database_perms(username: str, database: str, body: dict = Body(default=None),
|
| 2957 |
+
session: Session = Depends(admin_gate)):
|
| 2958 |
+
"""Replace one database's access, row filter and hidden fields if the account is unchanged.
|
| 2959 |
+
|
| 2960 |
+
Body: `{entry: {access, filter?, hiddenFields?}, expect: <access_state from GET perms>,
|
| 2961 |
+
confirm_widen?}`. A stale `expect` answers 409 and writes nothing.
|
| 2962 |
+
"""
|
| 2963 |
+
body = body if isinstance(body, dict) else {}
|
| 2964 |
+
return set_database_block(session, username, database, body.get("entry"),
|
| 2965 |
+
expect=body.get("expect"),
|
| 2966 |
+
confirm_widen=bool(body.get("confirm_widen")))
|
| 2967 |
+
|
| 2968 |
+
|
| 2969 |
@router.put("/admin/users/{username}/perms")
|
| 2970 |
def put_perms(username: str, body: dict = Body(default=None),
|
| 2971 |
session: Session = Depends(admin_gate)):
|
api/routes_aged_stock.py
CHANGED
|
@@ -19,10 +19,9 @@ def _module():
|
|
| 19 |
|
| 20 |
|
| 21 |
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 22 |
-
"""
|
| 23 |
-
from
|
| 24 |
-
|
| 25 |
-
raise err(403, "aged_stock_forbidden", "Your account cannot read Amazon storage fees.")
|
| 26 |
return session
|
| 27 |
|
| 28 |
|
|
@@ -60,6 +59,8 @@ def _app(session, app_id):
|
|
| 60 |
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 61 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 62 |
raise err(404, "aged_stock_app_not_found", "That Aged stock App does not exist.")
|
|
|
|
|
|
|
| 63 |
return app
|
| 64 |
|
| 65 |
|
|
@@ -140,9 +141,11 @@ def read_item(item_id: str, app: str = Query(..., min_length=1), offset: int = Q
|
|
| 140 |
@router.patch("/items/{item_id}")
|
| 141 |
def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 142 |
due: str | None = Body(None), action: str | None = Body(None), note: str | None = Body(None),
|
|
|
|
| 143 |
session: Session = Depends(_app_writer)):
|
|
|
|
| 144 |
return _call(_module().update_item, _tenant(session), item_id, owner=owner, status=status, due=due,
|
| 145 |
-
action=action, note=note, user=session.uname or "unknown", st=session.runtime)
|
| 146 |
|
| 147 |
|
| 148 |
@router.get("/settings")
|
|
|
|
| 19 |
|
| 20 |
|
| 21 |
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 22 |
+
"""Admit a live App holder and refuse unsupported source row or field limits."""
|
| 23 |
+
from routes_app_dashboard import admit_reader
|
| 24 |
+
admit_reader(session, 'aged_stock')
|
|
|
|
| 25 |
return session
|
| 26 |
|
| 27 |
|
|
|
|
| 59 |
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 60 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 61 |
raise err(404, "aged_stock_app_not_found", "That Aged stock App does not exist.")
|
| 62 |
+
from routes_app_dashboard import admit_reader
|
| 63 |
+
admit_reader(session, 'aged_stock', app)
|
| 64 |
return app
|
| 65 |
|
| 66 |
|
|
|
|
| 141 |
@router.patch("/items/{item_id}")
|
| 142 |
def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 143 |
due: str | None = Body(None), action: str | None = Body(None), note: str | None = Body(None),
|
| 144 |
+
price: StrictFloat | StrictInt | StrictStr | None = Body(None),
|
| 145 |
session: Session = Depends(_app_writer)):
|
| 146 |
+
"""A price is stored as a proposal on the item and never sent to Amazon. A blank price clears it."""
|
| 147 |
return _call(_module().update_item, _tenant(session), item_id, owner=owner, status=status, due=due,
|
| 148 |
+
action=action, note=note, price=price, user=session.uname or "unknown", st=session.runtime)
|
| 149 |
|
| 150 |
|
| 151 |
@router.get("/settings")
|
api/routes_amazon_dashboard.py
CHANGED
|
@@ -13,57 +13,28 @@ from modules import amazon_dashboard as dashboard_module
|
|
| 13 |
router = APIRouter(prefix="/api/v1/app-kinds/amazon_dashboard", tags=["amazon-dashboard"])
|
| 14 |
|
| 15 |
|
| 16 |
-
def _unrestricted_bu(user):
|
| 17 |
-
from core import perms, users
|
| 18 |
-
raw = user.get("bus", "all")
|
| 19 |
-
if raw == "all":
|
| 20 |
-
return "All" in perms.allowed_bu_labels(user)
|
| 21 |
-
if not isinstance(raw, (list, tuple)) or not raw:
|
| 22 |
-
return False
|
| 23 |
-
by_label = {label: key for key, label in users.BU_LABELS.items()}
|
| 24 |
-
identities = []
|
| 25 |
-
for value in raw:
|
| 26 |
-
if type(value) is int and value in users.BU_LABELS:
|
| 27 |
-
identities.append(value)
|
| 28 |
-
elif isinstance(value, str) and value in by_label:
|
| 29 |
-
identities.append(by_label[value])
|
| 30 |
-
elif value != "All":
|
| 31 |
-
return False
|
| 32 |
-
return bool(identities) and "All" in perms.allowed_bu_labels(dict(user, bus=identities))
|
| 33 |
-
|
| 34 |
-
|
| 35 |
def admit_sources(session):
|
| 36 |
"""Resolve source, recipient, field and BU walls before any mirror metadata or row I/O."""
|
| 37 |
-
|
|
|
|
| 38 |
runtime, user = session.runtime, session.user
|
| 39 |
if (not getattr(runtime, "key", None) or session.tenant != runtime.key
|
| 40 |
or user.get("tenant", session.tenant) != runtime.key):
|
| 41 |
raise err(403, "amazon_dashboard_forbidden", "The account does not belong to this workspace.")
|
| 42 |
-
if not _unrestricted_bu(user) or perms.scope_agent(user) is not None:
|
| 43 |
-
raise err(403, "amazon_dashboard_scope_unsupported", "This dashboard cannot apply your business unit or agent scope.")
|
| 44 |
allowed, refused = {}, {}
|
| 45 |
-
config = getattr(getattr(runtime, "tenant", None), "config", None) or {}
|
| 46 |
-
tenant_modules = config.get("modules", "all")
|
| 47 |
for table, key in dashboard_module.SOURCE_DATABASES.items():
|
| 48 |
# No production cost mapping is available, so this door never opens the Odoo cost mirror.
|
| 49 |
if table == "product_product":
|
| 50 |
refused[table] = "No proven Amazon SKU and Odoo unit mapping is available for this period."
|
| 51 |
continue
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
| 55 |
continue
|
| 56 |
definition = user_tables.get_definition(key, st=runtime)
|
| 57 |
-
if not isinstance(definition, dict) or not definition.get("fields"):
|
| 58 |
-
refused[table] = "A required source database has no readable field definition."
|
| 59 |
-
continue
|
| 60 |
-
wall = perm_scope.entry(user, key) or {}
|
| 61 |
-
# Explicit source restrictions remain binding on aggregate reads, including admin records.
|
| 62 |
-
if (wall.get("access") is False or wall.get("filter") or wall.get("hiddenFields")
|
| 63 |
-
or perm_scope.row_scope_applies(user, key, st=runtime, defn=definition)
|
| 64 |
-
or perm_scope.hidden_keys(user, key, definition["fields"], st=runtime)
|
| 65 |
-
or route_system.is_managed(definition) or route_system.is_managed(key)):
|
| 66 |
-
raise err(403, "amazon_dashboard_scope_unsupported", "This dashboard cannot apply a source row or field access limit.")
|
| 67 |
allowed[table] = frozenset(str(field["key"]) for field in definition["fields"]
|
| 68 |
if isinstance(field, dict) and field.get("key"))
|
| 69 |
if "amzn_order_metrics" not in allowed:
|
|
@@ -86,14 +57,16 @@ class AdmittedReader(dashboard_module.MirrorReader):
|
|
| 86 |
|
| 87 |
|
| 88 |
def _app(session, app_id):
|
| 89 |
-
if not app_id:
|
| 90 |
-
if not session.admin:
|
| 91 |
-
raise err(403, "amazon_dashboard_app_required", "Open this dashboard from an App you can read.")
|
| 92 |
-
return
|
| 93 |
import application_registry
|
| 94 |
-
|
| 95 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
raise err(404, "amazon_dashboard_app_not_found", "That Amazon dashboard App does not exist.")
|
|
|
|
| 97 |
|
| 98 |
|
| 99 |
@router.get("/dashboard")
|
|
@@ -101,5 +74,7 @@ def read_dashboard(start: str | None = None, end: str | None = None, compare: st
|
|
| 101 |
app: str | None = None, session: Session = Depends(require_session)):
|
| 102 |
"""Serve the published Amazon route through canonical admission and figure validation."""
|
| 103 |
from routes_app_dashboard import read_dashboard as canonical_dashboard
|
|
|
|
|
|
|
| 104 |
return canonical_dashboard(kind="amazon_dashboard", app=app, start=start, end=end,
|
| 105 |
compare=compare, session=session)
|
|
|
|
| 13 |
router = APIRouter(prefix="/api/v1/app-kinds/amazon_dashboard", tags=["amazon-dashboard"])
|
| 14 |
|
| 15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
def admit_sources(session):
|
| 17 |
"""Resolve source, recipient, field and BU walls before any mirror metadata or row I/O."""
|
| 18 |
+
import application_registry
|
| 19 |
+
from core import user_tables
|
| 20 |
runtime, user = session.runtime, session.user
|
| 21 |
if (not getattr(runtime, "key", None) or session.tenant != runtime.key
|
| 22 |
or user.get("tenant", session.tenant) != runtime.key):
|
| 23 |
raise err(403, "amazon_dashboard_forbidden", "The account does not belong to this workspace.")
|
|
|
|
|
|
|
| 24 |
allowed, refused = {}, {}
|
|
|
|
|
|
|
| 25 |
for table, key in dashboard_module.SOURCE_DATABASES.items():
|
| 26 |
# No production cost mapping is available, so this door never opens the Odoo cost mirror.
|
| 27 |
if table == "product_product":
|
| 28 |
refused[table] = "No proven Amazon SKU and Odoo unit mapping is available for this period."
|
| 29 |
continue
|
| 30 |
+
# R46: the shared check; explicit source restrictions bind aggregate reads, admin records too.
|
| 31 |
+
access = application_registry.app_access(user, runtime, None, (key,))
|
| 32 |
+
if access["code"] == "app_dashboard_scope":
|
| 33 |
+
raise err(403, "amazon_dashboard_scope_unsupported", access["message"])
|
| 34 |
+
if access["dashboard"] != "ready":
|
| 35 |
+
refused[table] = access["message"]
|
| 36 |
continue
|
| 37 |
definition = user_tables.get_definition(key, st=runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
allowed[table] = frozenset(str(field["key"]) for field in definition["fields"]
|
| 39 |
if isinstance(field, dict) and field.get("key"))
|
| 40 |
if "amzn_order_metrics" not in allowed:
|
|
|
|
| 57 |
|
| 58 |
|
| 59 |
def _app(session, app_id):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
import application_registry
|
| 61 |
+
record = (application_registry.load(session.runtime, strict=True)["applications"].get(app_id) if app_id else
|
| 62 |
+
application_registry.readable_app(session.runtime, session.tenant, session.uname,
|
| 63 |
+
session.admin, dashboard_module.KIND))
|
| 64 |
+
if (session.tenant != getattr(session.runtime, "key", None) or not isinstance(record, dict)
|
| 65 |
+
or record.get("tenant") != session.tenant or record.get("retired") is True
|
| 66 |
+
or record.get("templateKind") != dashboard_module.KIND
|
| 67 |
+
or not application_registry.may_read(record, session.uname, session.admin)):
|
| 68 |
raise err(404, "amazon_dashboard_app_not_found", "That Amazon dashboard App does not exist.")
|
| 69 |
+
return record["id"]
|
| 70 |
|
| 71 |
|
| 72 |
@router.get("/dashboard")
|
|
|
|
| 74 |
app: str | None = None, session: Session = Depends(require_session)):
|
| 75 |
"""Serve the published Amazon route through canonical admission and figure validation."""
|
| 76 |
from routes_app_dashboard import read_dashboard as canonical_dashboard
|
| 77 |
+
if not app:
|
| 78 |
+
app = _app(session, app)
|
| 79 |
return canonical_dashboard(kind="amazon_dashboard", app=app, start=start, end=end,
|
| 80 |
compare=compare, session=session)
|
api/routes_amazon_operator.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
"""Amazon operator App routes (C3): today's batch, history, settings, and the batch approve and dismiss.
|
| 2 |
|
| 3 |
-
Every route names its App with `?app=<id>`. Approve and dismiss call `amazon_operator.set_status`
|
| 4 |
-
|
| 5 |
"""
|
| 6 |
from fastapi import APIRouter, Body, Depends, Query
|
| 7 |
|
|
@@ -77,13 +77,52 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
|
|
| 77 |
return session
|
| 78 |
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
@router.get("/today")
|
| 81 |
def read_today(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 82 |
app_cursor: str | None = Query(None), held_cursor: str | None = Query(None),
|
| 83 |
limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 84 |
op = _module()
|
| 85 |
out = _call(op.screen_today, _tenant(session), st=session.runtime, cursor=cursor,
|
| 86 |
-
app_cursor=app_cursor, held_cursor=held_cursor, limit=limit, q=q
|
|
|
|
| 87 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 88 |
|
| 89 |
|
|
@@ -110,7 +149,7 @@ def read_history(status: str | None = Query(None), cursor: str | None = Query(No
|
|
| 110 |
limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 111 |
op = _module()
|
| 112 |
return _call(op.screen_history, _tenant(session), status=status, cursor=cursor, limit=limit, q=q,
|
| 113 |
-
st=session.runtime)
|
| 114 |
|
| 115 |
|
| 116 |
@router.get("/by-hour")
|
|
@@ -241,12 +280,16 @@ def approve_wasted(group_id: str, digest: str = Body(..., embed=True, min_length
|
|
| 241 |
@router.get("/results")
|
| 242 |
def read_results(proposal_id: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
|
| 243 |
op = _module()
|
| 244 |
-
return _call(op.results, _tenant(session), proposal_id, st=session.runtime
|
|
|
|
| 245 |
|
| 246 |
|
| 247 |
@router.post("/undo")
|
| 248 |
def undo(proposal_id: str = Body(..., embed=True, min_length=1), session: Session = Depends(_app_writer)):
|
| 249 |
op = _module()
|
|
|
|
|
|
|
|
|
|
| 250 |
return _call(op.propose_undo, _tenant(session), session.user, proposal_id, st=session.runtime)
|
| 251 |
|
| 252 |
|
|
@@ -286,12 +329,20 @@ def _weekly_context(st):
|
|
| 286 |
config = getattr(getattr(st, "tenant", None), "config", {}) or {}
|
| 287 |
timezone = config.get("timezone") or _module()._doc(st, "tenant_settings").get("timezone")
|
| 288 |
|
|
|
|
|
|
|
| 289 |
def may_read(app, owner):
|
| 290 |
-
|
| 291 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
and str(user.get("tenant") or getattr(st, "key", "")) == str(getattr(st, "key", ""))
|
| 293 |
and application_registry.may_read(
|
| 294 |
-
app, owner, is_admin=str(user.get("role") or "").lower() == "admin")
|
|
|
|
| 295 |
return {"apps": apps, "may_read": may_read, "timezone": timezone,
|
| 296 |
"base_url": os.environ.get("APP_BASE_URL") or ""}
|
| 297 |
|
|
@@ -302,7 +353,20 @@ def _weekly_delivery():
|
|
| 302 |
from notify_destinations import deliver
|
| 303 |
except ImportError:
|
| 304 |
return None
|
| 305 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 306 |
|
| 307 |
|
| 308 |
def _weekly_destinations(session):
|
|
@@ -352,6 +416,7 @@ def read_weekly(session: Session = Depends(_app_reader)):
|
|
| 352 |
if row.get("source_module") in op.PROPOSAL_SOURCES})}
|
| 353 |
summary = _call(weekly.compose, _tenant(session), session.uname, week, st=st, now=now,
|
| 354 |
source_state=source_state, **context)
|
|
|
|
| 355 |
return {**summary, "inbox_basis": "Open items in the saved Operator inbox.",
|
| 356 |
"destinations": _weekly_destinations(session)}
|
| 357 |
|
|
@@ -362,15 +427,38 @@ def send_weekly_now(owner: str | None = Body(None, embed=True), session: Session
|
|
| 362 |
if not session.admin:
|
| 363 |
raise err(403, "amazon_operator_admin_only", "Only an Administrator can send a weekly summary now.")
|
| 364 |
from modules import amazon_operator_weekly as weekly
|
| 365 |
-
|
| 366 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 367 |
|
| 368 |
|
| 369 |
@router.post("/approve")
|
| 370 |
-
def approve(item_ids: list[str] = Body(..., embed=True),
|
| 371 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 372 |
op = _module()
|
| 373 |
-
return _call(op.set_status, _tenant(session), session.user, item_ids, "approved", st=session.runtime
|
|
|
|
| 374 |
|
| 375 |
|
| 376 |
@router.post("/dismiss")
|
|
@@ -378,7 +466,7 @@ def dismiss(item_ids: list[str] = Body(...), reason: str = Body(..., min_length=
|
|
| 378 |
session: Session = Depends(_app_writer)):
|
| 379 |
op = _module()
|
| 380 |
return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason,
|
| 381 |
-
st=session.runtime)
|
| 382 |
|
| 383 |
|
| 384 |
def _refresh_operator_tenant(tenant, st, *, chat=None):
|
|
|
|
| 1 |
"""Amazon operator App routes (C3): today's batch, history, settings, and the batch approve and dismiss.
|
| 2 |
|
| 3 |
+
Every route names its App with `?app=<id>`. Approve and dismiss call `amazon_operator.set_status`.
|
| 4 |
+
MCP approvals are refused. The router never calls the Ads client.
|
| 5 |
"""
|
| 6 |
from fastapi import APIRouter, Body, Depends, Query
|
| 7 |
|
|
|
|
| 77 |
return session
|
| 78 |
|
| 79 |
|
| 80 |
+
def _source_reader(user, st, *, weekly=False):
|
| 81 |
+
"""Cache Dashboard admission per source kind for one viewer and request."""
|
| 82 |
+
import application_registry
|
| 83 |
+
import app_templates
|
| 84 |
+
cache = {}
|
| 85 |
+
|
| 86 |
+
def allowed(source):
|
| 87 |
+
if str(source or "").startswith("modules.") and source not in _module().PROPOSAL_SOURCES:
|
| 88 |
+
return False
|
| 89 |
+
kind = str(source or "").removeprefix("modules.")
|
| 90 |
+
if kind not in cache:
|
| 91 |
+
cache[kind] = False
|
| 92 |
+
try:
|
| 93 |
+
template = app_templates.BY_KIND.get(kind)
|
| 94 |
+
if not isinstance(template, dict):
|
| 95 |
+
return False
|
| 96 |
+
required = template.get("sources")
|
| 97 |
+
optional = template.get("optional_sources", ())
|
| 98 |
+
if not isinstance(required, (list, tuple)) or not required or not isinstance(optional, (list, tuple)):
|
| 99 |
+
return False
|
| 100 |
+
sources = list(dict.fromkeys((*required, *optional)))
|
| 101 |
+
# Removal prose includes ship-to details even when these sources become optional.
|
| 102 |
+
if kind == "returns_refunds":
|
| 103 |
+
sources = list(dict.fromkeys((*sources, "ut_amzn_removal_orders", "ut_amzn_removal_shipments")))
|
| 104 |
+
if weekly:
|
| 105 |
+
sources = list(dict.fromkeys((*sources, "ut_amazon_sku_profit", "ut_amzn_inventory_planning")))
|
| 106 |
+
if any(not isinstance(key, str) or not key for key in sources):
|
| 107 |
+
return False
|
| 108 |
+
access = application_registry.app_access(user, st, {"templateKind": kind}, sources)
|
| 109 |
+
cache[kind] = access.get("dashboard") == "ready"
|
| 110 |
+
except Exception:
|
| 111 |
+
import logging
|
| 112 |
+
logging.getLogger(__name__).exception("Operator source access check failed for %s", kind)
|
| 113 |
+
return cache[kind]
|
| 114 |
+
|
| 115 |
+
return allowed
|
| 116 |
+
|
| 117 |
+
|
| 118 |
@router.get("/today")
|
| 119 |
def read_today(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 120 |
app_cursor: str | None = Query(None), held_cursor: str | None = Query(None),
|
| 121 |
limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 122 |
op = _module()
|
| 123 |
out = _call(op.screen_today, _tenant(session), st=session.runtime, cursor=cursor,
|
| 124 |
+
app_cursor=app_cursor, held_cursor=held_cursor, limit=limit, q=q,
|
| 125 |
+
source_allowed=_source_reader(session.user, session.runtime))
|
| 126 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 127 |
|
| 128 |
|
|
|
|
| 149 |
limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 150 |
op = _module()
|
| 151 |
return _call(op.screen_history, _tenant(session), status=status, cursor=cursor, limit=limit, q=q,
|
| 152 |
+
st=session.runtime, source_allowed=_source_reader(session.user, session.runtime))
|
| 153 |
|
| 154 |
|
| 155 |
@router.get("/by-hour")
|
|
|
|
| 280 |
@router.get("/results")
|
| 281 |
def read_results(proposal_id: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
|
| 282 |
op = _module()
|
| 283 |
+
return _call(op.results, _tenant(session), proposal_id, st=session.runtime,
|
| 284 |
+
source_allowed=_source_reader(session.user, session.runtime))
|
| 285 |
|
| 286 |
|
| 287 |
@router.post("/undo")
|
| 288 |
def undo(proposal_id: str = Body(..., embed=True, min_length=1), session: Session = Depends(_app_writer)):
|
| 289 |
op = _module()
|
| 290 |
+
allowed = _source_reader(session.user, session.runtime)
|
| 291 |
+
if not allowed(TEMPLATE_KIND):
|
| 292 |
+
return op.withheld(1)
|
| 293 |
return _call(op.propose_undo, _tenant(session), session.user, proposal_id, st=session.runtime)
|
| 294 |
|
| 295 |
|
|
|
|
| 329 |
config = getattr(getattr(st, "tenant", None), "config", {}) or {}
|
| 330 |
timezone = config.get("timezone") or _module()._doc(st, "tenant_settings").get("timezone")
|
| 331 |
|
| 332 |
+
readers = {}
|
| 333 |
+
|
| 334 |
def may_read(app, owner):
|
| 335 |
+
account = _module().account_get(owner, st=st) or {}
|
| 336 |
+
user = {**account, "username": owner} if account else {}
|
| 337 |
+
if owner not in readers:
|
| 338 |
+
readers[owner] = _source_reader(user, st, weekly=True)
|
| 339 |
+
return bool(isinstance(app, dict) and not app.get("retired")
|
| 340 |
+
and app.get("tenant") == str(getattr(st, "key", ""))
|
| 341 |
+
and user and user.get("active", True)
|
| 342 |
and str(user.get("tenant") or getattr(st, "key", "")) == str(getattr(st, "key", ""))
|
| 343 |
and application_registry.may_read(
|
| 344 |
+
app, owner, is_admin=str(user.get("role") or "").lower() == "admin")
|
| 345 |
+
and readers[owner](app.get("templateKind")))
|
| 346 |
return {"apps": apps, "may_read": may_read, "timezone": timezone,
|
| 347 |
"base_url": os.environ.get("APP_BASE_URL") or ""}
|
| 348 |
|
|
|
|
| 353 |
from notify_destinations import deliver
|
| 354 |
except ImportError:
|
| 355 |
return None
|
| 356 |
+
def guarded(st, kind, items, *, user, **kwargs):
|
| 357 |
+
context = _weekly_context(st)
|
| 358 |
+
apps = {str(app.get("id")): app for app in context["apps"]}
|
| 359 |
+
kept = []
|
| 360 |
+
for item in items:
|
| 361 |
+
source = item.get("source") or {}
|
| 362 |
+
ids = source.get("rowIds") or []
|
| 363 |
+
if (source.get("table") == "applications" and ids
|
| 364 |
+
and all(str(key) in apps and context["may_read"](apps[str(key)], user) for key in ids)):
|
| 365 |
+
kept.append(item)
|
| 366 |
+
if not kept:
|
| 367 |
+
return {"state": "empty", "withheldCount": len(items), "results": []}
|
| 368 |
+
return deliver(st, kind, kept, user=user, **kwargs)
|
| 369 |
+
return guarded
|
| 370 |
|
| 371 |
|
| 372 |
def _weekly_destinations(session):
|
|
|
|
| 416 |
if row.get("source_module") in op.PROPOSAL_SOURCES})}
|
| 417 |
summary = _call(weekly.compose, _tenant(session), session.uname, week, st=st, now=now,
|
| 418 |
source_state=source_state, **context)
|
| 419 |
+
# Compose independently validates the count for this recipient's owned items.
|
| 420 |
return {**summary, "inbox_basis": "Open items in the saved Operator inbox.",
|
| 421 |
"destinations": _weekly_destinations(session)}
|
| 422 |
|
|
|
|
| 427 |
if not session.admin:
|
| 428 |
raise err(403, "amazon_operator_admin_only", "Only an Administrator can send a weekly summary now.")
|
| 429 |
from modules import amazon_operator_weekly as weekly
|
| 430 |
+
result = _call(weekly.send_tenant, _tenant(session), session.runtime, force=True, owner=owner,
|
| 431 |
+
deliver=_weekly_delivery(), **_call(_weekly_context, session.runtime))
|
| 432 |
+
allowed = _source_reader(session.user, session.runtime, weekly=True)
|
| 433 |
+
rows = []
|
| 434 |
+
for row in result.get("rows", []):
|
| 435 |
+
row = dict(row)
|
| 436 |
+
row.pop("delivery", None)
|
| 437 |
+
if isinstance(row.get("summary"), dict):
|
| 438 |
+
summary = dict(row["summary"])
|
| 439 |
+
entries = summary.get("apps", [])
|
| 440 |
+
summary["apps"] = [app for app in entries if allowed(app.get("kind"))]
|
| 441 |
+
# Validation contains App names and figures too.
|
| 442 |
+
summary.pop("validation", None)
|
| 443 |
+
row["summary"] = summary
|
| 444 |
+
rows.append(row)
|
| 445 |
+
# Recipient counts survive caller text filtering without an all-owner recount.
|
| 446 |
+
return {**result, "rows": rows}
|
| 447 |
|
| 448 |
|
| 449 |
@router.post("/approve")
|
| 450 |
+
def approve(item_ids: list[str] = Body(..., embed=True), reviews: dict | None = Body(None, embed=True),
|
| 451 |
+
session: Session = Depends(_app_writer)):
|
| 452 |
+
"""Approve the ticked proposals as one batch. Returns each row's outcome.
|
| 453 |
+
|
| 454 |
+
`reviews` maps each removal's inbox id to the `review.reviewed` value shown in its drawer. Only a person
|
| 455 |
+
signed in to the App approves: an API key or OAuth session is refused before anything is stored.
|
| 456 |
+
"""
|
| 457 |
+
if (session.claims or {}).get("external"):
|
| 458 |
+
raise err(403, "amazon_operator_internal_approval", "Approve these inside the Operator inbox.")
|
| 459 |
op = _module()
|
| 460 |
+
return _call(op.set_status, _tenant(session), session.user, item_ids, "approved", st=session.runtime,
|
| 461 |
+
reviews=reviews, source_allowed=_source_reader(session.user, session.runtime))
|
| 462 |
|
| 463 |
|
| 464 |
@router.post("/dismiss")
|
|
|
|
| 466 |
session: Session = Depends(_app_writer)):
|
| 467 |
op = _module()
|
| 468 |
return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason,
|
| 469 |
+
st=session.runtime, source_allowed=_source_reader(session.user, session.runtime))
|
| 470 |
|
| 471 |
|
| 472 |
def _refresh_operator_tenant(tenant, st, *, chat=None):
|
api/routes_amazon_profit.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""Command Center doors: read the settings, switch the period, recompute, set the owner,
|
| 2 |
-
|
| 3 |
|
| 4 |
The table itself (`ut_amazon_sku_profit`) is read and edited through the ordinary table doors.
|
| 5 |
A refresh rewrites only computed columns, so owner, status and due date survive a period switch.
|
|
@@ -12,7 +12,7 @@ from typing import Any
|
|
| 12 |
from fastapi import APIRouter, Body, Depends, HTTPException
|
| 13 |
|
| 14 |
from deps import Session, err, require_session
|
| 15 |
-
from modules import amazon_profit, amazon_profit_account
|
| 16 |
|
| 17 |
router = APIRouter(prefix="/api/v1")
|
| 18 |
|
|
@@ -96,6 +96,8 @@ def profit_status(session: Session = Depends(require_session)):
|
|
| 96 |
"linked": settings.get("linked"),
|
| 97 |
"verifiedMonth": settings.get("verifiedMonth"),
|
| 98 |
"refusal": settings.get("refusal"),
|
|
|
|
|
|
|
| 99 |
"defaultOwner": settings["defaultOwner"],
|
| 100 |
"periods": amazon_profit.period_choices(),
|
| 101 |
"statuses": list(amazon_profit.STATUSES),
|
|
@@ -109,6 +111,26 @@ def profit_status(session: Session = Depends(require_session)):
|
|
| 109 |
"canAdmin": bool(session.admin)}
|
| 110 |
|
| 111 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
@router.get("/amazon-profit/account")
|
| 113 |
def profit_account(period: str | None = None, session: Session = Depends(require_session)):
|
| 114 |
"""Read this tenant's account P&L without refreshing costs, caches or history."""
|
|
@@ -158,7 +180,8 @@ def profit_what_if(sku: str, price: float | None = None, landed_cost: float | No
|
|
| 158 |
|
| 159 |
|
| 160 |
def _recompute(session, period=None, thin=None):
|
| 161 |
-
"""Run one refresh through the tenant's mirror receipts and return 409 on a failed tie.
|
|
|
|
| 162 |
try:
|
| 163 |
if period is not None:
|
| 164 |
amazon_profit.period_bounds(period)
|
|
@@ -207,16 +230,6 @@ def profit_period(body: Any = Body(default=None), session: Session = Depends(req
|
|
| 207 |
return _recompute(session, period)
|
| 208 |
|
| 209 |
|
| 210 |
-
@router.post("/amazon-profit/projects/relink")
|
| 211 |
-
def projects_relink(session: Session = Depends(require_session)):
|
| 212 |
-
"""Write every project's row id onto its SKU, ASIN and parent ASIN rows; editors only."""
|
| 213 |
-
if not _may_edit(session):
|
| 214 |
-
raise err(403, "forbidden", "Only an editor of Command Center can relink projects.")
|
| 215 |
-
with amazon_profit._LOCK:
|
| 216 |
-
linked = amazon_profit.relink_projects(session.runtime)
|
| 217 |
-
return {"ok": True, "linked": linked}
|
| 218 |
-
|
| 219 |
-
|
| 220 |
@router.put("/amazon-profit/settings")
|
| 221 |
def profit_settings(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 222 |
"""Administrator settings: the default owner of new items and the thin margin."""
|
|
@@ -238,12 +251,13 @@ def profit_settings(body: Any = Body(default=None), session: Session = Depends(r
|
|
| 238 |
|
| 239 |
@router.post("/amazon-profit/matches/{sku:path}/{decision}")
|
| 240 |
def profit_match(sku: str, decision: str, session: Session = Depends(require_session)):
|
| 241 |
-
"""
|
| 242 |
|
| 243 |
-
Confirm copies the suggestion into the link and recomputes
|
| 244 |
-
|
|
|
|
| 245 |
import core.user_tables as ut
|
| 246 |
-
states = {"confirm": amazon_profit.
|
| 247 |
if decision not in states:
|
| 248 |
raise err(404, "not_found", "Choose confirm or reject.")
|
| 249 |
if not _may_edit(session):
|
|
@@ -266,12 +280,142 @@ def profit_match(sku: str, decision: str, session: Session = Depends(require_ses
|
|
| 266 |
"unitsPerSku": (result["row"] or {}).get("units_per_sku")}
|
| 267 |
if decision == "confirm":
|
| 268 |
try:
|
| 269 |
-
|
| 270 |
-
|
|
|
|
| 271 |
except HTTPException as exc:
|
| 272 |
out["refreshError"] = ((exc.detail or {}).get("error") or {}).get("message")
|
|
|
|
|
|
|
| 273 |
current = next((r for r in amazon_profit.table_rows(session.runtime).values()
|
| 274 |
if str(r.get("sku") or "").strip() == sku), {})
|
| 275 |
out.update(cogsPerUnit=current.get("cogs_per_unit"), netProfit=current.get("net_profit"),
|
| 276 |
margin=current.get("margin"), band=current.get("band"))
|
| 277 |
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""Command Center doors: read the settings, switch the period, recompute, set the owner,
|
| 2 |
+
link or reject a suggested Odoo match at once.
|
| 3 |
|
| 4 |
The table itself (`ut_amazon_sku_profit`) is read and edited through the ordinary table doors.
|
| 5 |
A refresh rewrites only computed columns, so owner, status and due date survive a period switch.
|
|
|
|
| 12 |
from fastapi import APIRouter, Body, Depends, HTTPException
|
| 13 |
|
| 14 |
from deps import Session, err, require_session
|
| 15 |
+
from modules import amazon_data_entry, amazon_profit, amazon_profit_account
|
| 16 |
|
| 17 |
router = APIRouter(prefix="/api/v1")
|
| 18 |
|
|
|
|
| 96 |
"linked": settings.get("linked"),
|
| 97 |
"verifiedMonth": settings.get("verifiedMonth"),
|
| 98 |
"refusal": settings.get("refusal"),
|
| 99 |
+
"verified": settings.get("verified"),
|
| 100 |
+
"lastRefusal": settings.get("lastRefusal"),
|
| 101 |
"defaultOwner": settings["defaultOwner"],
|
| 102 |
"periods": amazon_profit.period_choices(),
|
| 103 |
"statuses": list(amazon_profit.STATUSES),
|
|
|
|
| 111 |
"canAdmin": bool(session.admin)}
|
| 112 |
|
| 113 |
|
| 114 |
+
COMMAND_CENTER_KEYS = (amazon_profit.TABLE_KEY, amazon_profit.ASIN_KEY, amazon_profit.PARENT_KEY,
|
| 115 |
+
amazon_profit.PROJECTS_KEY, amazon_data_entry.TABLE_KEY)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
@router.get("/amazon-profit/tables")
|
| 119 |
+
def profit_tables(session: Session = Depends(require_session)):
|
| 120 |
+
"""Whether each Command Center table this session may open exists.
|
| 121 |
+
|
| 122 |
+
Each table is walled by its own `session.require`, never by `_account_access`, and no row is read."""
|
| 123 |
+
import core.user_tables as ut
|
| 124 |
+
tables = {}
|
| 125 |
+
for key in COMMAND_CENTER_KEYS:
|
| 126 |
+
try:
|
| 127 |
+
session.require(key)
|
| 128 |
+
except HTTPException:
|
| 129 |
+
continue
|
| 130 |
+
tables[key] = ut.get_definition(key, session.runtime) is not None
|
| 131 |
+
return {"tables": tables}
|
| 132 |
+
|
| 133 |
+
|
| 134 |
@router.get("/amazon-profit/account")
|
| 135 |
def profit_account(period: str | None = None, session: Session = Depends(require_session)):
|
| 136 |
"""Read this tenant's account P&L without refreshing costs, caches or history."""
|
|
|
|
| 180 |
|
| 181 |
|
| 182 |
def _recompute(session, period=None, thin=None):
|
| 183 |
+
"""Run one refresh through the tenant's mirror receipts and return 409 on a failed tie.
|
| 184 |
+
A 409 left the stored rows and period as they were."""
|
| 185 |
try:
|
| 186 |
if period is not None:
|
| 187 |
amazon_profit.period_bounds(period)
|
|
|
|
| 230 |
return _recompute(session, period)
|
| 231 |
|
| 232 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 233 |
@router.put("/amazon-profit/settings")
|
| 234 |
def profit_settings(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 235 |
"""Administrator settings: the default owner of new items and the thin margin."""
|
|
|
|
| 251 |
|
| 252 |
@router.post("/amazon-profit/matches/{sku:path}/{decision}")
|
| 253 |
def profit_match(sku: str, decision: str, session: Session = Depends(require_session)):
|
| 254 |
+
"""Link or reject the suggested Odoo product for one SKU; editors only.
|
| 255 |
|
| 256 |
+
Confirm copies the suggestion into the link and recomputes that SKU's cost and profit in the
|
| 257 |
+
same request (`recompute_sku`), so a refused refresh never blocks it. Reject clears the
|
| 258 |
+
suggestion, keeps the SKU unlinked and remembers the code."""
|
| 259 |
import core.user_tables as ut
|
| 260 |
+
states = {"confirm": amazon_profit.LINKED, "reject": amazon_profit.WRONG_MATCH}
|
| 261 |
if decision not in states:
|
| 262 |
raise err(404, "not_found", "Choose confirm or reject.")
|
| 263 |
if not _may_edit(session):
|
|
|
|
| 280 |
"unitsPerSku": (result["row"] or {}).get("units_per_sku")}
|
| 281 |
if decision == "confirm":
|
| 282 |
try:
|
| 283 |
+
linked = link_now(session.runtime, sku, by=session.uname)
|
| 284 |
+
if linked.get("refusal"):
|
| 285 |
+
out["refreshError"] = linked["refusal"]
|
| 286 |
except HTTPException as exc:
|
| 287 |
out["refreshError"] = ((exc.detail or {}).get("error") or {}).get("message")
|
| 288 |
+
except (amazon_profit.SourceUnavailable, amazon_profit.TableRefused) as exc:
|
| 289 |
+
out["refreshError"] = str(exc)
|
| 290 |
current = next((r for r in amazon_profit.table_rows(session.runtime).values()
|
| 291 |
if str(r.get("sku") or "").strip() == sku), {})
|
| 292 |
out.update(cogsPerUnit=current.get("cogs_per_unit"), netProfit=current.get("net_profit"),
|
| 293 |
margin=current.get("margin"), band=current.get("band"))
|
| 294 |
return out
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def _runtime_of(st):
|
| 298 |
+
"""The tenant runtime behind a lent store snapshot; the mirror and the writes need it."""
|
| 299 |
+
import core.user_tables as ut
|
| 300 |
+
lent = getattr(ut, "_Lent", None)
|
| 301 |
+
while lent is not None and isinstance(st, lent):
|
| 302 |
+
st = st._runtime
|
| 303 |
+
return st
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def link_now(runtime, sku, *, by=""):
|
| 307 |
+
"""Recompute one SKU's cost and profit from its Odoo link on this tenant's mirror."""
|
| 308 |
+
con = _mirror(runtime)
|
| 309 |
+
try:
|
| 310 |
+
result = amazon_profit.recompute_sku(runtime, con, sku, by=by, cost_reader=_cost_reader(runtime))
|
| 311 |
+
finally:
|
| 312 |
+
try:
|
| 313 |
+
con.close()
|
| 314 |
+
except Exception: # noqa: BLE001
|
| 315 |
+
pass
|
| 316 |
+
# S82-T71: a linked SKU that is now complete leaves Data Entry in the same request.
|
| 317 |
+
amazon_data_entry.sync(runtime, skus=[sku])
|
| 318 |
+
return result
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
#: Cell edits on the SKU table that change its Odoo link.
|
| 322 |
+
LINK_FIELDS = ("match_state", "odoo_product", "units_per_sku", "amazon_cogs")
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def link_hook(evt):
|
| 326 |
+
"""A person's Link status, Odoo product or Units per SKU edit links that SKU at once.
|
| 327 |
+
|
| 328 |
+
Registered on `core.user_tables.ROW_HOOKS`, which only the human write doors emit into.
|
| 329 |
+
Its own writes go through `update_with_activity`, so it never fires itself."""
|
| 330 |
+
if str(evt.get("type") or "") != "event_field" or str(evt.get("table") or "") != amazon_profit.TABLE_KEY:
|
| 331 |
+
return
|
| 332 |
+
field = str(evt.get("field") or "")
|
| 333 |
+
if field not in LINK_FIELDS or evt.get("st") is None:
|
| 334 |
+
return
|
| 335 |
+
runtime = _runtime_of(evt["st"])
|
| 336 |
+
row = amazon_profit.table_rows(runtime).get(str(evt.get("rowId") or "")) or {}
|
| 337 |
+
sku = str(row.get("sku") or "").strip()
|
| 338 |
+
if not sku:
|
| 339 |
+
return
|
| 340 |
+
by = str(evt.get("user") or "") or amazon_profit.WRITER
|
| 341 |
+
if field == "match_state":
|
| 342 |
+
state = str(evt.get("after") or "").strip()
|
| 343 |
+
state = amazon_profit.LEGACY_MATCH_STATES.get(state, state)
|
| 344 |
+
if state not in (amazon_profit.LINKED, amazon_profit.WRONG_MATCH):
|
| 345 |
+
return
|
| 346 |
+
with amazon_profit._LOCK:
|
| 347 |
+
result = amazon_profit.apply_decisions(runtime, by=by, sku=sku, state=state)
|
| 348 |
+
if state != amazon_profit.LINKED or not result.get("confirmed"):
|
| 349 |
+
return
|
| 350 |
+
link_now(runtime, sku, by=by)
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
# One registration per process: a reimport replaces the hook rather than adding a second one.
|
| 354 |
+
import core.user_tables as _ut_hooks # noqa: E402
|
| 355 |
+
|
| 356 |
+
_ut_hooks.ROW_HOOKS[:] = [h for h in _ut_hooks.ROW_HOOKS
|
| 357 |
+
if (getattr(h, "__module__", None), getattr(h, "__name__", None)) != (__name__, "link_hook")]
|
| 358 |
+
_ut_hooks.ROW_HOOKS.append(link_hook)
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def data_entry_hook(evt):
|
| 362 |
+
"""A person's Data Entry cell edit writes that value to the SKU table in the same request.
|
| 363 |
+
|
| 364 |
+
Registered on `core.user_tables.ROW_HOOKS` and filtered on the Data Entry key. The write-back
|
| 365 |
+
goes through `patch_cells`, which emits no row event, so it never fires itself or `link_hook`.
|
| 366 |
+
A link edit then links the SKU at once, and the row is rebuilt: a complete SKU leaves (R8)."""
|
| 367 |
+
if (str(evt.get("type") or "") != "event_field"
|
| 368 |
+
or str(evt.get("table") or "") != amazon_data_entry.TABLE_KEY or evt.get("st") is None):
|
| 369 |
+
return
|
| 370 |
+
field = str(evt.get("field") or "")
|
| 371 |
+
if field not in amazon_data_entry.ENTRY_KEYS:
|
| 372 |
+
return
|
| 373 |
+
runtime = _runtime_of(evt["st"])
|
| 374 |
+
user = str(evt.get("user") or "")
|
| 375 |
+
result = amazon_data_entry.write_back(runtime, str(evt.get("rowId") or ""), field, evt.get("after"), user=user)
|
| 376 |
+
sku = result.get("sku")
|
| 377 |
+
if not sku:
|
| 378 |
+
return
|
| 379 |
+
try:
|
| 380 |
+
if result["written"] and field in (*amazon_data_entry.LINK_KEYS, "amazon_cogs"):
|
| 381 |
+
state = result["changes"].get("match_state")
|
| 382 |
+
if state in (amazon_profit.LINKED, amazon_profit.WRONG_MATCH):
|
| 383 |
+
with amazon_profit._LOCK:
|
| 384 |
+
amazon_profit.apply_decisions(runtime, by=user or amazon_profit.WRITER, sku=sku, state=state)
|
| 385 |
+
row = next((r for r in amazon_profit.table_rows(runtime).values()
|
| 386 |
+
if str(r.get("sku") or "").strip() == sku), {})
|
| 387 |
+
if str(row.get("odoo_product") or "").strip() or field == "amazon_cogs":
|
| 388 |
+
try:
|
| 389 |
+
link_now(runtime, sku, by=user)
|
| 390 |
+
except (HTTPException, amazon_profit.SourceUnavailable, amazon_profit.TableRefused) as exc:
|
| 391 |
+
print(f"[amazon-profit] Data Entry link for {sku} not recomputed: {exc}")
|
| 392 |
+
finally:
|
| 393 |
+
amazon_data_entry.sync(runtime, skus=[sku])
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
_ut_hooks.ROW_HOOKS[:] = [h for h in _ut_hooks.ROW_HOOKS
|
| 397 |
+
if (getattr(h, "__module__", None), getattr(h, "__name__", None)) != (__name__, "data_entry_hook")]
|
| 398 |
+
_ut_hooks.ROW_HOOKS.append(data_entry_hook)
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
@router.get("/amazon-profit/fba-capacity")
|
| 402 |
+
def fba_capacity_read(session: Session = Depends(require_session)):
|
| 403 |
+
"""The one typed FBA capacity Command Center and Restock share."""
|
| 404 |
+
from core import amazon_inputs
|
| 405 |
+
session.require(amazon_profit.TABLE_KEY)
|
| 406 |
+
return {"fbaCapacity": amazon_inputs.fba_capacity(session.runtime), "units": list(amazon_inputs.UNITS),
|
| 407 |
+
"canEdit": _may_edit(session)}
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
@router.put("/amazon-profit/fba-capacity")
|
| 411 |
+
def fba_capacity_write(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 412 |
+
"""Save the typed FBA capacity `{value, unit}`, or clear it with null; editors only."""
|
| 413 |
+
from core import amazon_inputs
|
| 414 |
+
if not _may_edit(session):
|
| 415 |
+
raise err(403, "forbidden", "Only an editor of Command Center can change FBA capacity.")
|
| 416 |
+
body = body if isinstance(body, dict) else {}
|
| 417 |
+
try:
|
| 418 |
+
stored = amazon_inputs.set_fba_capacity(session.runtime, body.get("fbaCapacity"), by=session.uname)
|
| 419 |
+
except ValueError as exc:
|
| 420 |
+
raise err(400, "invalid_setting", str(exc)) from None
|
| 421 |
+
return {"fbaCapacity": stored}
|
api/routes_amazon_projects.py
ADDED
|
@@ -0,0 +1,389 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Projects doors: a row's projects, one project with its targets and activity, edits, and target search.
|
| 2 |
+
|
| 3 |
+
Editors of Command Center write and a viewer reads (R16, R17). Project rows live in the per-tenant
|
| 4 |
+
store (C6). Search and target details read the Amazon mirror, never `user_tables`. Each write
|
| 5 |
+
repaints the `projects` and `projects_label` cells of only the Command Center rows it touches.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from contextlib import contextmanager
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
import sys
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
from fastapi import APIRouter, Body, Depends, HTTPException
|
| 16 |
+
|
| 17 |
+
from core import amazon_projects as projects
|
| 18 |
+
from deps import Session, err, require_session
|
| 19 |
+
from modules import amazon_profit
|
| 20 |
+
import routes_amazon_profit
|
| 21 |
+
|
| 22 |
+
router = APIRouter(prefix="/api/v1/amazon-projects")
|
| 23 |
+
|
| 24 |
+
FORBIDDEN = "Only an editor of Command Center can change projects."
|
| 25 |
+
NO_MIRROR = "This workspace has no Amazon data yet."
|
| 26 |
+
UNREADABLE = "Amazon data could not be read. Try again shortly."
|
| 27 |
+
NOT_FOUND = "That project does not exist."
|
| 28 |
+
#: Contract field name to store field.
|
| 29 |
+
FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": "owner",
|
| 30 |
+
"startOn": "start_on", "targetOn": "target_on", "progress": "progress", "notes": "notes"}
|
| 31 |
+
#: Project JSON keys a client may echo back. They are ignored, never written.
|
| 32 |
+
READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
|
| 33 |
+
"completedOn", "rollup", "impact", "via"))
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _call(fn, *args, **kwargs):
|
| 37 |
+
try:
|
| 38 |
+
return fn(*args, **kwargs)
|
| 39 |
+
except HTTPException:
|
| 40 |
+
raise
|
| 41 |
+
except KeyError:
|
| 42 |
+
raise err(404, "project_not_found", NOT_FOUND) from None
|
| 43 |
+
except projects.BackendUnavailable:
|
| 44 |
+
raise err(503, "projects_store_unavailable", "Projects need the workspace database.") from None
|
| 45 |
+
except ValueError as exc:
|
| 46 |
+
raise err(400, "invalid_project", str(exc)) from None
|
| 47 |
+
except Exception as exc: # noqa: BLE001
|
| 48 |
+
print(f"[amazon-projects] {getattr(fn, '__name__', fn)} failed: {type(exc).__name__}: {exc}",
|
| 49 |
+
file=sys.stderr)
|
| 50 |
+
raise err(503, "projects_unavailable", "Projects are unavailable. Try again shortly.") from None
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _reader(session):
|
| 54 |
+
session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
|
| 55 |
+
return session.runtime
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _writer(session):
|
| 59 |
+
"""Editors and Administrators of Command Center, through the Command Center edit check."""
|
| 60 |
+
if not routes_amazon_profit._may_edit(session):
|
| 61 |
+
raise err(403, "forbidden", FORBIDDEN)
|
| 62 |
+
return session.runtime
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _by(session):
|
| 66 |
+
return session.uname or "unknown"
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
@contextmanager
|
| 70 |
+
def _open_mirror(runtime):
|
| 71 |
+
"""A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
|
| 72 |
+
from harness import runtime as R
|
| 73 |
+
if not Path(runtime.datastore_path).is_file():
|
| 74 |
+
yield None
|
| 75 |
+
return
|
| 76 |
+
cur = R.mirror_cursor(runtime, require_ready=False)
|
| 77 |
+
try:
|
| 78 |
+
yield cur
|
| 79 |
+
finally:
|
| 80 |
+
cur.close()
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def _mirror(runtime, read, empty):
|
| 84 |
+
"""(read(cursor), None), or (empty, the cause) when the mirror is absent or unreadable."""
|
| 85 |
+
try:
|
| 86 |
+
with _open_mirror(runtime) as cur:
|
| 87 |
+
if cur is None:
|
| 88 |
+
return empty, NO_MIRROR
|
| 89 |
+
return read(cur), None
|
| 90 |
+
except Exception as exc: # noqa: BLE001
|
| 91 |
+
print(f"[amazon-projects] mirror read failed: {type(exc).__name__}: {exc}", file=sys.stderr)
|
| 92 |
+
return empty, UNREADABLE
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _pictures(runtime):
|
| 96 |
+
"""`{ASIN: picture ref}`; empty when no picture is stored or the index is unreadable."""
|
| 97 |
+
try:
|
| 98 |
+
from harness import amazon_pictures
|
| 99 |
+
return amazon_pictures.picture_refs(runtime)
|
| 100 |
+
except Exception: # noqa: BLE001
|
| 101 |
+
return {}
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def _project(record, today):
|
| 105 |
+
return {"id": record["id"], "name": record["name"], "status": record["status"],
|
| 106 |
+
"priority": record["priority"], "owner": record["owner"], "startOn": record["start_on"],
|
| 107 |
+
"targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"],
|
| 108 |
+
"completedOn": record["completed_on"], "health": projects.health(record, today),
|
| 109 |
+
"targetCount": record.get("target_count", 0), "createdAt": record["created_at"],
|
| 110 |
+
"updatedAt": record["updated_at"], "createdBy": record["created_by"],
|
| 111 |
+
"rollup": record.get("rollup"), "impact": record.get("impact")}
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _item(grain, key, listing, pictures):
|
| 115 |
+
asin, parent = listing.get("asin"), listing.get("parent_asin")
|
| 116 |
+
return {"grain": grain, "key": key, "sku": listing.get("sku"), "asin": asin, "parentAsin": parent,
|
| 117 |
+
"title": listing.get("title"), "picture": pictures.get(asin) or pictures.get(parent)}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def _pair(grain, key):
|
| 121 |
+
try:
|
| 122 |
+
return projects.target(grain, key)
|
| 123 |
+
except (TypeError, ValueError):
|
| 124 |
+
raise err(400, "invalid_target", "Name a grain of sku, asin or parent_asin and its key.") from None
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _pairs(values):
|
| 128 |
+
if not isinstance(values, list) or not values or not all(isinstance(v, dict) for v in values):
|
| 129 |
+
raise err(400, "invalid_target", "Send targets as a list of grain and key.")
|
| 130 |
+
return [_pair(v.get("grain"), v.get("key")) for v in values]
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _fields(body, *, partial):
|
| 134 |
+
if not isinstance(body, dict):
|
| 135 |
+
raise err(400, "invalid_project", "Send the project fields as an object.")
|
| 136 |
+
unknown = sorted(set(body) - set(FIELDS) - READ_ONLY - {"targets"})
|
| 137 |
+
if unknown:
|
| 138 |
+
raise err(400, "invalid_project", f"Unknown project fields: {', '.join(unknown)}.")
|
| 139 |
+
values = {FIELDS[name]: body[name] for name in FIELDS if name in body}
|
| 140 |
+
if not partial and "name" not in values:
|
| 141 |
+
raise err(400, "invalid_project", "Name the project.")
|
| 142 |
+
return values
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _targets(runtime, project_id):
|
| 146 |
+
"""The project's targets with their listing details, and a cause when the mirror is out."""
|
| 147 |
+
listed = _call(projects.targets, runtime, project_id)
|
| 148 |
+
pairs = [(t["grain"], t["key"]) for t in listed]
|
| 149 |
+
details, cause = _mirror(runtime, lambda cur: projects.describe(cur, pairs), {})
|
| 150 |
+
pictures = _pictures(runtime) if pairs else {}
|
| 151 |
+
out = [_item(grain, key, details.get((grain, key)) or {"sku": key if grain == "sku" else None,
|
| 152 |
+
"asin": key if grain == "asin" else None,
|
| 153 |
+
"parent_asin": key if grain == "parent_asin" else None}, pictures)
|
| 154 |
+
for grain, key in pairs]
|
| 155 |
+
return out, cause
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def _with_cause(body, cause):
|
| 159 |
+
if cause:
|
| 160 |
+
body["cause"] = cause
|
| 161 |
+
return body
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
# --------------------------------------------------------------------------- Command Center cells
|
| 165 |
+
|
| 166 |
+
PROGRESS_KEY, LABEL_KEY = "projects", "projects_label"
|
| 167 |
+
#: (table, the grains one row counts, in order): a SKU row counts its ASIN's and parent's projects too.
|
| 168 |
+
CELL_TABLES = ((amazon_profit.TABLE_KEY, ("sku", "asin", "parent_asin")),
|
| 169 |
+
(amazon_profit.ASIN_KEY, ("asin", "parent_asin")),
|
| 170 |
+
(amazon_profit.PARENT_KEY, ("parent_asin",)))
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _distinct(found, pairs):
|
| 174 |
+
"""`[(via pair, project)]` with each project once, in the order of `pairs`."""
|
| 175 |
+
seen, out = set(), []
|
| 176 |
+
for via in pairs:
|
| 177 |
+
for record in found.get(via, []):
|
| 178 |
+
if record["id"] not in seen:
|
| 179 |
+
seen.add(record["id"])
|
| 180 |
+
out.append((via, record))
|
| 181 |
+
return out
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def cell_values(records):
|
| 185 |
+
"""R19 and R67: the average progress as a stored pct and "N of M completed", blank with no counted project."""
|
| 186 |
+
counted = projects.summary(records)
|
| 187 |
+
if not counted["total"]:
|
| 188 |
+
return {PROGRESS_KEY: None, LABEL_KEY: None}
|
| 189 |
+
return {PROGRESS_KEY: amazon_profit.stored_cell({"type": "pct"}, counted["average"] / 100),
|
| 190 |
+
LABEL_KEY: f"{counted['completed']} of {counted['total']} completed"}
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _row_pairs(row, grains):
|
| 194 |
+
out = []
|
| 195 |
+
for grain in grains:
|
| 196 |
+
value = str(row.get(grain) or "").strip()
|
| 197 |
+
if value:
|
| 198 |
+
pair = projects.target(grain, value)
|
| 199 |
+
if pair not in out:
|
| 200 |
+
out.append(pair)
|
| 201 |
+
return out
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def _cells_store(runtime):
|
| 205 |
+
"""The store that holds the Command Center tables: the tenant runtime itself."""
|
| 206 |
+
return runtime
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def _row_ids(cells):
|
| 210 |
+
"""The write names only the rows whose cells change."""
|
| 211 |
+
return sorted(cells)
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def _write_cells(st, table_key, cells):
|
| 215 |
+
import core.user_tables as ut
|
| 216 |
+
|
| 217 |
+
def _set(cur):
|
| 218 |
+
rows = ((cur or {}).get(table_key) or {}).get("rows") or {}
|
| 219 |
+
for rid, values in cells.items():
|
| 220 |
+
row = rows.get(rid)
|
| 221 |
+
if row is None:
|
| 222 |
+
continue
|
| 223 |
+
for key, value in values.items():
|
| 224 |
+
if value is None:
|
| 225 |
+
row.pop(key, None)
|
| 226 |
+
else:
|
| 227 |
+
row[key] = value
|
| 228 |
+
return cur
|
| 229 |
+
|
| 230 |
+
ut.update_with_activity(table_key, _set, st=st, by=amazon_profit.WRITER, via=amazon_profit.VIA,
|
| 231 |
+
row_ids=_row_ids(cells))
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def paint(runtime, pairs=None):
|
| 235 |
+
"""Rewrite the progress cells of the rows that count a touched pair; every row when `pairs` is None.
|
| 236 |
+
|
| 237 |
+
Returns `{table: [row id]}` for the rows whose cells changed. Never runs under `amazon_profit._LOCK`."""
|
| 238 |
+
touched = None if pairs is None else {projects.target(*pair) for pair in pairs}
|
| 239 |
+
if touched is not None and not touched:
|
| 240 |
+
return {}
|
| 241 |
+
st = _cells_store(runtime)
|
| 242 |
+
source = {grains[0]: amazon_profit.table_rows(st, key) for key, grains in CELL_TABLES}
|
| 243 |
+
own = {g: {rid: {g: row.get(g)} for rid, row in rows.items()} for g, rows in source.items()}
|
| 244 |
+
lineage, _cause = _mirror(runtime, lambda cur: amazon_profit.M.lineage_rows(cur, source), own)
|
| 245 |
+
plan = []
|
| 246 |
+
for table_key, grains in CELL_TABLES:
|
| 247 |
+
rows = {}
|
| 248 |
+
for rid, row in source[grains[0]].items():
|
| 249 |
+
chain = _row_pairs(lineage[grains[0]][rid], grains)
|
| 250 |
+
if touched is None or touched.intersection(chain + _row_pairs(row, grains)):
|
| 251 |
+
rows[str(rid)] = (chain, row)
|
| 252 |
+
plan.append((table_key, rows))
|
| 253 |
+
wanted = sorted({pair for _key, rows in plan for chain, _row in rows.values() for pair in chain})
|
| 254 |
+
found = projects.projects_for(runtime, wanted) if wanted else {}
|
| 255 |
+
written = {}
|
| 256 |
+
for table_key, rows in plan:
|
| 257 |
+
cells = {}
|
| 258 |
+
for rid, (chain, row) in rows.items():
|
| 259 |
+
values = cell_values([record for _via, record in _distinct(found, chain)])
|
| 260 |
+
if any(row.get(key) != value for key, value in values.items()):
|
| 261 |
+
cells[rid] = values
|
| 262 |
+
if cells:
|
| 263 |
+
_write_cells(st, table_key, cells)
|
| 264 |
+
written[table_key] = sorted(cells)
|
| 265 |
+
return written
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def _repaint(runtime, pairs):
|
| 269 |
+
"""Paint after a committed project write. A failure is logged: the write itself stands."""
|
| 270 |
+
try:
|
| 271 |
+
return paint(runtime, pairs)
|
| 272 |
+
except Exception as exc: # noqa: BLE001
|
| 273 |
+
print(f"[amazon-projects] cell repaint failed: {type(exc).__name__}: {exc}", file=sys.stderr)
|
| 274 |
+
return None
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def _target_pairs(runtime, project_id):
|
| 278 |
+
return [(t["grain"], t["key"]) for t in _call(projects.targets, runtime, project_id)]
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
@router.get("")
|
| 282 |
+
def row_projects(grain: str | None = None, key: str | None = None,
|
| 283 |
+
session: Session = Depends(require_session)):
|
| 284 |
+
"""A row's projects with its ASIN's and Parent ASIN's, each once; every project without a row."""
|
| 285 |
+
runtime = _reader(session)
|
| 286 |
+
can_edit = bool(routes_amazon_profit._may_edit(session))
|
| 287 |
+
today = projects._today()
|
| 288 |
+
if grain is None and key is None:
|
| 289 |
+
records = _call(projects.list_projects, runtime)
|
| 290 |
+
return {"grain": None, "key": None, "canEdit": can_edit,
|
| 291 |
+
"projects": [_project(r, today) for r in records], "summary": projects.summary(records)}
|
| 292 |
+
pair = _pair(grain, key)
|
| 293 |
+
above, cause = _mirror(runtime, lambda cur: projects.lineage(cur, *pair), [])
|
| 294 |
+
pairs = [pair] + [p for p in above if p != pair]
|
| 295 |
+
listed = _distinct(_call(projects.projects_for, runtime, pairs), pairs)
|
| 296 |
+
out = [{**_project(record, today), "via": {"grain": via[0], "key": via[1]}} for via, record in listed]
|
| 297 |
+
return _with_cause({"grain": pair[0], "key": pair[1], "canEdit": can_edit, "projects": out,
|
| 298 |
+
"summary": projects.summary([record for _via, record in listed])}, cause)
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
@router.get("/search")
|
| 302 |
+
def search_targets(q: str = "", limit: int = 25, session: Session = Depends(require_session), offset: int = 0):
|
| 303 |
+
"""A SKU-ordered page from the Amazon mirror; total counts every match, including past the end."""
|
| 304 |
+
runtime = _reader(session)
|
| 305 |
+
if not 1 <= limit <= projects.SEARCH_MAX:
|
| 306 |
+
raise err(400, "invalid_limit", f"Limit must be from 1 to {projects.SEARCH_MAX}.")
|
| 307 |
+
if type(offset) is not int or offset < 0:
|
| 308 |
+
raise err(400, "invalid_offset", "Offset must be a nonnegative whole number.")
|
| 309 |
+
(listings, total), cause = _mirror(runtime, lambda cur: projects.search(cur, q, limit, offset), ([], 0))
|
| 310 |
+
pictures = _pictures(runtime) if listings else {}
|
| 311 |
+
results = [_item("sku", item["sku"], item, pictures) for item in listings]
|
| 312 |
+
return _with_cause({"results": results, "total": total}, cause)
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
@router.post("")
|
| 316 |
+
def create_project(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 317 |
+
"""Create one project, with targets when the body names them."""
|
| 318 |
+
runtime = _writer(session)
|
| 319 |
+
values = _fields(body, partial=False)
|
| 320 |
+
targets = _pairs(body["targets"]) if body.get("targets") else []
|
| 321 |
+
record = _call(projects.create, runtime, targets=targets, by=_by(session), **values)
|
| 322 |
+
_repaint(runtime, targets)
|
| 323 |
+
return {"project": _project(record, projects._today())}
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
@router.get("/{project_id}")
|
| 327 |
+
def one_project(project_id: str, session: Session = Depends(require_session)):
|
| 328 |
+
"""One project with its targets and its activity, newest first."""
|
| 329 |
+
runtime = _reader(session)
|
| 330 |
+
record = _call(projects.get, runtime, project_id)
|
| 331 |
+
targets, cause = _targets(runtime, project_id)
|
| 332 |
+
activity = _call(projects.activity, runtime, project_id)
|
| 333 |
+
return _with_cause({"canEdit": bool(routes_amazon_profit._may_edit(session)),
|
| 334 |
+
"project": _project(record, projects._today()), "targets": targets,
|
| 335 |
+
"activity": activity}, cause)
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
@router.patch("/{project_id}")
|
| 339 |
+
def edit_project(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 340 |
+
"""Change the named fields. Targets named here are added."""
|
| 341 |
+
runtime = _writer(session)
|
| 342 |
+
values = _fields(body, partial=True)
|
| 343 |
+
targets = _pairs(body["targets"]) if body.get("targets") else []
|
| 344 |
+
if not values and not targets:
|
| 345 |
+
raise err(400, "invalid_project", "Change at least one project field.")
|
| 346 |
+
if values:
|
| 347 |
+
record = _call(projects.update, runtime, project_id, values, by=_by(session))
|
| 348 |
+
if targets:
|
| 349 |
+
record = _call(projects.add_targets, runtime, project_id, targets, by=_by(session))["project"]
|
| 350 |
+
# Only status and progress move a cell: a name or note edit repaints only the targets it adds.
|
| 351 |
+
_repaint(runtime, _target_pairs(runtime, project_id) if {"status", "progress"} & set(values) else targets)
|
| 352 |
+
return {"project": _project(record, projects._today())}
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
@router.delete("/{project_id}")
|
| 356 |
+
def delete_project(project_id: str, session: Session = Depends(require_session)):
|
| 357 |
+
"""Delete one project and its targets."""
|
| 358 |
+
runtime = _writer(session)
|
| 359 |
+
pairs = _target_pairs(runtime, project_id)
|
| 360 |
+
_call(projects.delete, runtime, project_id, by=_by(session))
|
| 361 |
+
_repaint(runtime, pairs)
|
| 362 |
+
return {"ok": True, "id": project_id}
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def _target_answer(runtime, project_id, result):
|
| 366 |
+
targets, cause = _targets(runtime, project_id)
|
| 367 |
+
return _with_cause({"project": _project(result["project"], projects._today()), "targets": targets}, cause)
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
@router.post("/{project_id}/targets")
|
| 371 |
+
def add_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 372 |
+
"""Assign the project to more SKUs, ASINs or Parent ASINs."""
|
| 373 |
+
runtime = _writer(session)
|
| 374 |
+
pairs = _pairs((body or {}).get("targets") if isinstance(body, dict) else None)
|
| 375 |
+
result = _call(projects.add_targets, runtime, project_id, pairs, by=_by(session))
|
| 376 |
+
_repaint(runtime, pairs)
|
| 377 |
+
return _target_answer(runtime, project_id, result)
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
@router.post("/{project_id}/targets/remove")
|
| 381 |
+
def remove_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 382 |
+
"""Unassign one SKU, ASIN or Parent ASIN from the project."""
|
| 383 |
+
runtime = _writer(session)
|
| 384 |
+
if not isinstance(body, dict):
|
| 385 |
+
raise err(400, "invalid_target", "Name the grain and key to remove.")
|
| 386 |
+
pair = _pair(body.get("grain"), body.get("key"))
|
| 387 |
+
result = _call(projects.remove_targets, runtime, project_id, [pair], by=_by(session))
|
| 388 |
+
_repaint(runtime, [pair])
|
| 389 |
+
return _target_answer(runtime, project_id, result)
|
api/routes_app_dashboard.py
CHANGED
|
@@ -3,21 +3,83 @@ from __future__ import annotations
|
|
| 3 |
|
| 4 |
from datetime import date, timedelta
|
| 5 |
from importlib import import_module
|
|
|
|
| 6 |
|
| 7 |
from fastapi import APIRouter, Depends, HTTPException, Query
|
| 8 |
|
| 9 |
from deps import Session, err, require_session
|
| 10 |
-
from core.app_dashboard import checked_dashboard
|
| 11 |
from core.request_timing import raise_if_resource_error, remaining_budget
|
| 12 |
|
| 13 |
router = APIRouter(prefix="/api/v1/app-kinds", tags=["app-dashboard"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
|
| 16 |
def admit(session, kind, app_id):
|
| 17 |
"""Refuse global saved aggregates when any source has a row or field restriction."""
|
| 18 |
import app_templates
|
| 19 |
import application_registry
|
| 20 |
-
from core import perm_scope, perms, route_system, user_tables
|
| 21 |
|
| 22 |
runtime, user = session.runtime, session.user
|
| 23 |
if (not session.tenant or getattr(runtime, "key", None) != session.tenant
|
|
@@ -32,28 +94,10 @@ def admit(session, kind, app_id):
|
|
| 32 |
or app.get("retired") is True or app.get("templateKind") != kind
|
| 33 |
or not application_registry.may_read(app, session.uname, session.admin)):
|
| 34 |
raise err(404, "app_dashboard_app", "This App is unavailable or is not shared with you.")
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
raise err(503, "app_dashboard_sources", "The dashboard source descriptor is unavailable.")
|
| 40 |
-
config = getattr(getattr(runtime, "tenant", None), "config", None) or {}
|
| 41 |
-
enabled = config.get("modules", "all")
|
| 42 |
-
for source in sources:
|
| 43 |
-
remaining_budget()
|
| 44 |
-
if ((enabled != "all" and source not in (enabled or ()))
|
| 45 |
-
or not perm_scope.may_read(user, source, st=runtime)):
|
| 46 |
-
raise err(403, "app_dashboard_source", "Your account cannot read a required dashboard source.")
|
| 47 |
-
definition = user_tables.get_definition(source, st=runtime)
|
| 48 |
-
if not isinstance(definition, dict) or not definition.get("fields"):
|
| 49 |
-
raise err(503, "app_dashboard_source", "A dashboard source is unavailable.")
|
| 50 |
-
declared = perm_scope.entry(user, source) or {}
|
| 51 |
-
if (declared.get("access") is False or declared.get("filter") or declared.get("hiddenFields")
|
| 52 |
-
or perm_scope.wall_declared(user, source, st=runtime, defn=definition)
|
| 53 |
-
or perm_scope.row_scope_applies(user, source, st=runtime, defn=definition)
|
| 54 |
-
or perm_scope.hidden_keys(user, source, definition["fields"], st=runtime)
|
| 55 |
-
or route_system.is_managed(definition) or route_system.is_managed(source)):
|
| 56 |
-
raise err(403, "app_dashboard_scope", "This dashboard needs unrestricted source access.")
|
| 57 |
return dict(template)
|
| 58 |
|
| 59 |
|
|
@@ -78,6 +122,7 @@ def read_dashboard(kind: str, app: str = Query(..., min_length=1), start: str |
|
|
| 78 |
raise ValueError
|
| 79 |
except ValueError:
|
| 80 |
raise err(400, "app_dashboard_period", "Choose valid dates and a supported comparison.") from None
|
|
|
|
| 81 |
try:
|
| 82 |
remaining_budget()
|
| 83 |
descriptor = admit(session, kind, app)
|
|
@@ -86,7 +131,7 @@ def read_dashboard(kind: str, app: str = Query(..., min_length=1), start: str |
|
|
| 86 |
options = {}
|
| 87 |
if kind == "amazon_dashboard":
|
| 88 |
options.update(compare=compare, allowed_sources=tuple(descriptor["sources"]), cost_links=None)
|
| 89 |
-
if kind in ("payouts", "reimbursements"):
|
| 90 |
options["user"] = dict(session.user)
|
| 91 |
if kind == "opportunity_finder":
|
| 92 |
options.update(opener=_opener(session.runtime, "ut_amzn_brand_analytics"),
|
|
@@ -99,4 +144,5 @@ def read_dashboard(kind: str, app: str = Query(..., min_length=1), start: str |
|
|
| 99 |
raise_if_resource_error(exc)
|
| 100 |
if isinstance(exc, HTTPException):
|
| 101 |
raise
|
| 102 |
-
|
|
|
|
|
|
| 3 |
|
| 4 |
from datetime import date, timedelta
|
| 5 |
from importlib import import_module
|
| 6 |
+
import logging
|
| 7 |
|
| 8 |
from fastapi import APIRouter, Depends, HTTPException, Query
|
| 9 |
|
| 10 |
from deps import Session, err, require_session
|
| 11 |
+
from core.app_dashboard import checked_dashboard, log_failure, plain_cause
|
| 12 |
from core.request_timing import raise_if_resource_error, remaining_budget
|
| 13 |
|
| 14 |
router = APIRouter(prefix="/api/v1/app-kinds", tags=["app-dashboard"])
|
| 15 |
+
_log = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _app_source_admission(user, runtime, app_id, source):
|
| 19 |
+
"""Bind core delegation to a fresh readable App and its declared sources."""
|
| 20 |
+
import application_registry
|
| 21 |
+
import app_templates
|
| 22 |
+
from core import perms
|
| 23 |
+
tenant = getattr(runtime, 'key', None)
|
| 24 |
+
if not tenant or user.get('tenant') != tenant or not user.get('username'):
|
| 25 |
+
return False
|
| 26 |
+
enabled = (getattr(getattr(runtime, 'tenant', None), 'config', None) or {}).get('modules', 'all')
|
| 27 |
+
if enabled != 'all' and (not isinstance(enabled, (list, tuple, set, frozenset)) or source not in enabled):
|
| 28 |
+
return False
|
| 29 |
+
app = application_registry.load(runtime, strict=True)['applications'].get(app_id)
|
| 30 |
+
if (not isinstance(app, dict) or app.get('id') != app_id or app.get('tenant') != tenant
|
| 31 |
+
or app.get('retired') is True
|
| 32 |
+
or not application_registry.may_read(app, user['username'], perms.is_admin(user))):
|
| 33 |
+
return False
|
| 34 |
+
template = app_templates.BY_KIND.get(app.get('templateKind'))
|
| 35 |
+
return bool(template and not app_templates.held(app.get('templateKind'))
|
| 36 |
+
and source in (*template['sources'], *template.get('optional_sources', ())))
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
from core.perm_scope import register_app_source_admission
|
| 40 |
+
register_app_source_admission(_app_source_admission)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def source_scope_for(session, app):
|
| 44 |
+
"""Bind a provider callback to the live App's tenant, principal and declared sources."""
|
| 45 |
+
import application_registry
|
| 46 |
+
|
| 47 |
+
def scope(source, records=None):
|
| 48 |
+
return application_registry.app_source_scope(session.user, session.runtime, source,
|
| 49 |
+
app=app, records=records)
|
| 50 |
+
|
| 51 |
+
return scope
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def admit_reader(session, kind, app=None, *, sources=None):
|
| 55 |
+
"""Admit an App holder while unsupported source narrowing remains refused."""
|
| 56 |
+
import application_registry
|
| 57 |
+
import app_templates
|
| 58 |
+
descriptor = app_templates.BY_KIND[kind]
|
| 59 |
+
if app is None:
|
| 60 |
+
app = application_registry.readable_app(session.runtime, session.tenant, session.uname,
|
| 61 |
+
session.admin, kind)
|
| 62 |
+
if not isinstance(app, dict) or app.get('templateKind') != kind:
|
| 63 |
+
raise err(404, kind + '_app_not_found', 'This App is unavailable or is not shared with you.')
|
| 64 |
+
required = tuple(descriptor['sources'] if sources is None else sources)
|
| 65 |
+
checked = tuple(dict.fromkeys((*required, *descriptor.get('optional_sources', ()))))
|
| 66 |
+
access = application_registry.admit_app(session, app, checked, code=kind)
|
| 67 |
+
from core import user_tables
|
| 68 |
+
enabled = (getattr(getattr(session.runtime, 'tenant', None), 'config', None) or {}).get('modules', 'all')
|
| 69 |
+
for source in required:
|
| 70 |
+
if not _app_source_admission(session.user, session.runtime, app.get('id'), source):
|
| 71 |
+
raise err(403, kind + '_source_unavailable', 'This App cannot read that source.')
|
| 72 |
+
if enabled != 'all' and (not isinstance(enabled, (list, tuple, set, frozenset)) or source not in enabled):
|
| 73 |
+
raise err(403, kind + '_source_unavailable', 'This App source is unavailable in this workspace.')
|
| 74 |
+
if not (user_tables.get_definition(source, st=session.runtime) or {}).get('fields'):
|
| 75 |
+
raise err(503, kind + '_source_contract', f'The field contract for {source} is unavailable. Sync its connector.')
|
| 76 |
+
return access
|
| 77 |
|
| 78 |
|
| 79 |
def admit(session, kind, app_id):
|
| 80 |
"""Refuse global saved aggregates when any source has a row or field restriction."""
|
| 81 |
import app_templates
|
| 82 |
import application_registry
|
|
|
|
| 83 |
|
| 84 |
runtime, user = session.runtime, session.user
|
| 85 |
if (not session.tenant or getattr(runtime, "key", None) != session.tenant
|
|
|
|
| 94 |
or app.get("retired") is True or app.get("templateKind") != kind
|
| 95 |
or not application_registry.may_read(app, session.uname, session.admin)):
|
| 96 |
raise err(404, "app_dashboard_app", "This App is unavailable or is not shared with you.")
|
| 97 |
+
# R46: the source walls live once, in the check availability also reads.
|
| 98 |
+
access = application_registry.app_access(user, runtime, app, template.get("sources"))
|
| 99 |
+
if access["dashboard"] != "ready":
|
| 100 |
+
raise err(access["status"], access["code"], access["message"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
return dict(template)
|
| 102 |
|
| 103 |
|
|
|
|
| 122 |
raise ValueError
|
| 123 |
except ValueError:
|
| 124 |
raise err(400, "app_dashboard_period", "Choose valid dates and a supported comparison.") from None
|
| 125 |
+
descriptor = {}
|
| 126 |
try:
|
| 127 |
remaining_budget()
|
| 128 |
descriptor = admit(session, kind, app)
|
|
|
|
| 131 |
options = {}
|
| 132 |
if kind == "amazon_dashboard":
|
| 133 |
options.update(compare=compare, allowed_sources=tuple(descriptor["sources"]), cost_links=None)
|
| 134 |
+
if kind in ("payouts", "reimbursements", "returns_refunds", "repeat_buyers"):
|
| 135 |
options["user"] = dict(session.user)
|
| 136 |
if kind == "opportunity_finder":
|
| 137 |
options.update(opener=_opener(session.runtime, "ut_amzn_brand_analytics"),
|
|
|
|
| 144 |
raise_if_resource_error(exc)
|
| 145 |
if isinstance(exc, HTTPException):
|
| 146 |
raise
|
| 147 |
+
log_failure(_log, exc, "read", kind, session.tenant)
|
| 148 |
+
raise err(503, "app_dashboard_unavailable", plain_cause(exc, "read", descriptor.get("sources"))) from None
|
api/routes_applications.py
CHANGED
|
@@ -311,7 +311,7 @@ def get_customer_card(app_id: str, customer_id: int,
|
|
| 311 |
if section != "customer":
|
| 312 |
_customer_card_admit(session, section, records)
|
| 313 |
|
| 314 |
-
result = customer_card.card(customer_id, cursor, rt=session.runtime, team_id=team_id,
|
| 315 |
invoice_limit=limit, invoice_offset=offset, authorize=authorize,
|
| 316 |
include_agents=agent_field in visible)
|
| 317 |
result["customer"]["name"] = selected.get(app["fieldMap"]["name"])
|
|
|
|
| 311 |
if section != "customer":
|
| 312 |
_customer_card_admit(session, section, records)
|
| 313 |
|
| 314 |
+
result = customer_card.card(customer_id, cursor, rt=session.runtime, user=session.user, team_id=team_id,
|
| 315 |
invoice_limit=limit, invoice_offset=offset, authorize=authorize,
|
| 316 |
include_agents=agent_field in visible)
|
| 317 |
result["customer"]["name"] = selected.get(app["fieldMap"]["name"])
|
api/routes_assets.py
CHANGED
|
@@ -894,14 +894,32 @@ def amazon_picture_bytes(session, ref):
|
|
| 894 |
return bytes(blob) if blob else None
|
| 895 |
|
| 896 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 897 |
@router.get("/assets/amazon/{ref}")
|
| 898 |
def amazon_asset(ref: str, q: str = "web",
|
| 899 |
session: Session = Depends(require_session_or_bearer)):
|
| 900 |
-
"""Serve one stored Amazon listing picture for THIS session's tenant
|
| 901 |
-
|
| 902 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 903 |
if blob is None:
|
| 904 |
-
# One answer for a bad ref, a missing picture and another tenant's picture.
|
| 905 |
raise err(404, "asset_not_found", "no image for that reference")
|
| 906 |
from harness import amazon_pictures
|
| 907 |
# Only a sha-stamped ref is immutable: the sha changes when the picture changes.
|
|
|
|
| 894 |
return bytes(blob) if blob else None
|
| 895 |
|
| 896 |
|
| 897 |
+
def amazon_full_picture_bytes(session, ref):
|
| 898 |
+
"""`(status, jpeg)` for the full-size copy of `<ASIN>.<sha>` in THIS session's tenant store."""
|
| 899 |
+
from harness import amazon_pictures
|
| 900 |
+
parsed = amazon_pictures.parse_ref(ref)
|
| 901 |
+
if parsed is None:
|
| 902 |
+
return "missing", None
|
| 903 |
+
try:
|
| 904 |
+
return amazon_pictures.full_picture(session.runtime, parsed[0])
|
| 905 |
+
except Exception: # noqa: BLE001
|
| 906 |
+
return "failed", None
|
| 907 |
+
|
| 908 |
+
|
| 909 |
@router.get("/assets/amazon/{ref}")
|
| 910 |
def amazon_asset(ref: str, q: str = "web",
|
| 911 |
session: Session = Depends(require_session_or_bearer)):
|
| 912 |
+
"""Serve one stored Amazon listing picture for THIS session's tenant: the 320 px thumbnail,
|
| 913 |
+
or with `q=full` the large copy the server fetched once and stored. The page never loads an
|
| 914 |
+
Amazon host."""
|
| 915 |
+
if q == "full":
|
| 916 |
+
status, blob = amazon_full_picture_bytes(session, ref)
|
| 917 |
+
if status == "failed":
|
| 918 |
+
raise err(502, "picture_unavailable", "the full picture could not be loaded, try again later")
|
| 919 |
+
else:
|
| 920 |
+
blob = amazon_picture_bytes(session, ref)
|
| 921 |
if blob is None:
|
| 922 |
+
# One answer for a bad ref, a missing picture, a refused link and another tenant's picture.
|
| 923 |
raise err(404, "asset_not_found", "no image for that reference")
|
| 924 |
from harness import amazon_pictures
|
| 925 |
# Only a sha-stamped ref is immutable: the sha changes when the picture changes.
|
api/routes_auth.py
CHANGED
|
@@ -311,7 +311,8 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 311 |
raise err(401, "mfa_required", "Enter the 6 digit code from your authenticator app.")
|
| 312 |
# TOTP FIRST: redemption consumes a recovery code. Do not spend one when the app's
|
| 313 |
# ordinary code already proves the factor, and never reset the guess budget on password.
|
| 314 |
-
factor_valid = bool(remembered) or users.totp_valid(rec.get("mfa_secret"), code
|
|
|
|
| 315 |
if not factor_valid:
|
| 316 |
recovery_code_used = users.consume_recovery_code(user["username"], code)
|
| 317 |
if not (factor_valid or recovery_code_used or remembered):
|
|
@@ -321,6 +322,9 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 321 |
except audit.AuditRefused as exc:
|
| 322 |
print(f"[aios-api] {exc}")
|
| 323 |
raise err(401, "mfa_invalid", _MFA_INVALID)
|
|
|
|
|
|
|
|
|
|
| 324 |
|
| 325 |
# ⭐ WAVE 40, R2 — SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this
|
| 326 |
# account already holds, so a forgotten sign-in on someone else's computer cannot lock the
|
|
|
|
| 311 |
raise err(401, "mfa_required", "Enter the 6 digit code from your authenticator app.")
|
| 312 |
# TOTP FIRST: redemption consumes a recovery code. Do not spend one when the app's
|
| 313 |
# ordinary code already proves the factor, and never reset the guess budget on password.
|
| 314 |
+
factor_valid = bool(remembered) or users.totp_valid(rec.get("mfa_secret"), code,
|
| 315 |
+
username=user["username"])
|
| 316 |
if not factor_valid:
|
| 317 |
recovery_code_used = users.consume_recovery_code(user["username"], code)
|
| 318 |
if not (factor_valid or recovery_code_used or remembered):
|
|
|
|
| 322 |
except audit.AuditRefused as exc:
|
| 323 |
print(f"[aios-api] {exc}")
|
| 324 |
raise err(401, "mfa_invalid", _MFA_INVALID)
|
| 325 |
+
if factor_valid and not remembered:
|
| 326 |
+
# S82-T134: a verified plaintext secret is sealed in place. Fail silent, no epoch bump.
|
| 327 |
+
users.reseal_mfa(user["username"], rec.get("mfa_secret"))
|
| 328 |
|
| 329 |
# ⭐ WAVE 40, R2 — SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this
|
| 330 |
# account already holds, so a forgotten sign-in on someone else's computer cannot lock the
|
api/routes_competitor_watch.py
CHANGED
|
@@ -1,4 +1,6 @@
|
|
| 1 |
"""Competitor watch App routes: watched terms, term detail, settings and owned alerts."""
|
|
|
|
|
|
|
| 2 |
from fastapi import APIRouter, Body, Depends, Query
|
| 3 |
|
| 4 |
from deps import Session, err, require_session
|
|
@@ -13,11 +15,11 @@ def _module():
|
|
| 13 |
return competitor_watch
|
| 14 |
|
| 15 |
|
| 16 |
-
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 17 |
-
"""
|
| 18 |
-
from
|
| 19 |
-
|
| 20 |
-
|
| 21 |
return session
|
| 22 |
|
| 23 |
|
|
@@ -33,19 +35,20 @@ def _tenant(session):
|
|
| 33 |
|
| 34 |
|
| 35 |
def _seller_alert_access(session):
|
| 36 |
-
"""Cached seller alerts
|
| 37 |
-
|
| 38 |
try:
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
|
|
|
| 42 |
except Exception: # noqa: BLE001
|
| 43 |
return False
|
| 44 |
|
| 45 |
|
| 46 |
def _call(fn, *args, **kwargs):
|
| 47 |
try:
|
| 48 |
-
return fn(*args, **kwargs)
|
| 49 |
except ValueError as exc:
|
| 50 |
raise err(400, "competitor_watch_invalid", str(exc)) from exc
|
| 51 |
except KeyError as exc:
|
|
@@ -71,6 +74,21 @@ def _call(fn, *args, **kwargs):
|
|
| 71 |
|
| 72 |
# --------------------------------------------------------------------------- S78-T17 App binding
|
| 73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
TEMPLATE_KIND = "competitor_watch"
|
| 75 |
|
| 76 |
|
|
@@ -81,9 +99,12 @@ def _app(session, app_id):
|
|
| 81 |
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 82 |
except Exception as exc: # noqa: BLE001
|
| 83 |
raise err(503, "competitor_watch_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 84 |
-
if (not isinstance(app, dict) or app.get("
|
|
|
|
| 85 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 86 |
raise err(404, "competitor_watch_app_not_found", "That Competitor watch App does not exist.")
|
|
|
|
|
|
|
| 87 |
return app
|
| 88 |
|
| 89 |
|
|
@@ -108,9 +129,11 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
|
|
| 108 |
|
| 109 |
@router.get("/terms")
|
| 110 |
def list_terms(q: str = "", offset: int = 0, limit: int = 50,
|
|
|
|
| 111 |
session: Session = Depends(_app_reader)):
|
| 112 |
cw = _module()
|
| 113 |
-
return _call(cw.term_page, _tenant(session), st=session.runtime, search=q, offset=offset, limit=limit
|
|
|
|
| 114 |
|
| 115 |
|
| 116 |
@router.get("/terms/detail")
|
|
@@ -122,13 +145,19 @@ def term_detail(term: str = Query(..., min_length=1), session: Session = Depends
|
|
| 122 |
@router.post("/terms")
|
| 123 |
def add_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 124 |
cw = _module()
|
| 125 |
-
return {"terms": _call(cw.add_terms, _tenant(session), terms, st=session.runtime, page_only=True)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
|
| 127 |
|
| 128 |
@router.post("/terms/remove")
|
| 129 |
def remove_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 130 |
cw = _module()
|
| 131 |
-
return {"terms": _call(cw.remove_terms, _tenant(session), terms, st=session.runtime, page_only=True)}
|
| 132 |
|
| 133 |
|
| 134 |
@router.get("/settings")
|
|
@@ -188,6 +217,33 @@ def _not_ready_message(cw, periods):
|
|
| 188 |
return cw.NOT_READY
|
| 189 |
|
| 190 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
|
| 192 |
"""Project `term_detail` to the screen: shares in percent points, previous values, causes, trend."""
|
| 193 |
periods = list(detail.get("periods") or ())
|
|
@@ -231,7 +287,13 @@ def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
|
|
| 231 |
comparison = detail.get("comparison") or {}
|
| 232 |
ready = comparison.get("ready") is True
|
| 233 |
return {"canEdit": bool(can_edit), "terms": list(terms), "term": term,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
"termsReady": (page or {}).get("termsReady", True),
|
|
|
|
|
|
|
| 235 |
"week": {"start": periods[0], "end": _week_end(periods[0])} if periods else None,
|
| 236 |
"previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
|
| 237 |
"comparisonReady": ready,
|
|
@@ -244,13 +306,15 @@ def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
|
|
| 244 |
@router.get("/screen")
|
| 245 |
def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(None),
|
| 246 |
q: str = "", offset: int = 0, limit: int = 50,
|
|
|
|
| 247 |
session: Session = Depends(_app_reader)):
|
| 248 |
"""S78-T17: the Terms view in one read (watched terms, the picked term's top 3 and trend)."""
|
| 249 |
cw = _module()
|
| 250 |
tenant = _tenant(session)
|
| 251 |
record = _app(session, app)
|
| 252 |
page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
|
| 253 |
-
limit=limit, selected=term)
|
| 254 |
terms, picked = page["terms"], page["term"]
|
|
|
|
| 255 |
detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
|
| 256 |
return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
|
|
|
|
| 1 |
"""Competitor watch App routes: watched terms, term detail, settings and owned alerts."""
|
| 2 |
+
from typing import Literal
|
| 3 |
+
|
| 4 |
from fastapi import APIRouter, Body, Depends, Query
|
| 5 |
|
| 6 |
from deps import Session, err, require_session
|
|
|
|
| 15 |
return competitor_watch
|
| 16 |
|
| 17 |
|
| 18 |
+
def _reader(session: Session = Depends(require_session), app: str | None = Query(None)) -> Session:
|
| 19 |
+
"""Bind the requested App before admitting its declared source contracts and limits."""
|
| 20 |
+
from routes_app_dashboard import admit_reader
|
| 21 |
+
record = _app(session, app) if isinstance(app, str) and app else None
|
| 22 |
+
admit_reader(session, 'competitor_watch', record)
|
| 23 |
return session
|
| 24 |
|
| 25 |
|
|
|
|
| 35 |
|
| 36 |
|
| 37 |
def _seller_alert_access(session):
|
| 38 |
+
"""Cached seller alerts use the App grant and refuse offer or ownership limits."""
|
| 39 |
+
import application_registry
|
| 40 |
try:
|
| 41 |
+
record = application_registry.readable_app(session.runtime, _tenant(session), session.uname,
|
| 42 |
+
session.admin, TEMPLATE_KIND)
|
| 43 |
+
access = application_registry.admit_app(session, record, SELLER_SOURCES, code="competitor_watch")
|
| 44 |
+
return not any(wall["rows"] or wall["hidden"] for wall in access["narrow"].values())
|
| 45 |
except Exception: # noqa: BLE001
|
| 46 |
return False
|
| 47 |
|
| 48 |
|
| 49 |
def _call(fn, *args, **kwargs):
|
| 50 |
try:
|
| 51 |
+
return _term_labels(fn(*args, **kwargs))
|
| 52 |
except ValueError as exc:
|
| 53 |
raise err(400, "competitor_watch_invalid", str(exc)) from exc
|
| 54 |
except KeyError as exc:
|
|
|
|
| 74 |
|
| 75 |
# --------------------------------------------------------------------------- S78-T17 App binding
|
| 76 |
|
| 77 |
+
def _term_labels(value):
|
| 78 |
+
"""Attach decoded labels while retaining stored term keys in every route payload."""
|
| 79 |
+
cw = _module()
|
| 80 |
+
if isinstance(value, list):
|
| 81 |
+
return [_term_labels(item) for item in value]
|
| 82 |
+
if isinstance(value, dict):
|
| 83 |
+
result = {key: _term_labels(item) for key, item in value.items()}
|
| 84 |
+
if isinstance(value.get("term"), str):
|
| 85 |
+
result["termLabel"] = cw.display_term(value["term"])
|
| 86 |
+
if isinstance(value.get("terms"), list) and all(isinstance(term, str) for term in value["terms"]):
|
| 87 |
+
result["termLabels"] = {term: cw.display_term(term) for term in value["terms"]}
|
| 88 |
+
return result
|
| 89 |
+
return value
|
| 90 |
+
|
| 91 |
+
|
| 92 |
TEMPLATE_KIND = "competitor_watch"
|
| 93 |
|
| 94 |
|
|
|
|
| 99 |
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 100 |
except Exception as exc: # noqa: BLE001
|
| 101 |
raise err(503, "competitor_watch_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 102 |
+
if (not isinstance(app, dict) or app.get("tenant") != _tenant(session)
|
| 103 |
+
or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 104 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 105 |
raise err(404, "competitor_watch_app_not_found", "That Competitor watch App does not exist.")
|
| 106 |
+
from routes_app_dashboard import admit_reader
|
| 107 |
+
admit_reader(session, 'competitor_watch', app)
|
| 108 |
return app
|
| 109 |
|
| 110 |
|
|
|
|
| 129 |
|
| 130 |
@router.get("/terms")
|
| 131 |
def list_terms(q: str = "", offset: int = 0, limit: int = 50,
|
| 132 |
+
tracked: bool = False, termOrder: Literal["asc", "desc"] | None = None,
|
| 133 |
session: Session = Depends(_app_reader)):
|
| 134 |
cw = _module()
|
| 135 |
+
return _call(cw.term_page, _tenant(session), st=session.runtime, search=q, offset=offset, limit=limit,
|
| 136 |
+
tracked=tracked, termOrder=termOrder)
|
| 137 |
|
| 138 |
|
| 139 |
@router.get("/terms/detail")
|
|
|
|
| 145 |
@router.post("/terms")
|
| 146 |
def add_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 147 |
cw = _module()
|
| 148 |
+
return _term_labels({"terms": _call(cw.add_terms, _tenant(session), terms, st=session.runtime, page_only=True)})
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
@router.patch("/terms/star")
|
| 152 |
+
def star_term(term: str = Body(...), starred: bool = Body(...), session: Session = Depends(_app_writer)):
|
| 153 |
+
cw = _module()
|
| 154 |
+
return _call(cw.star_term, _tenant(session), term, starred, st=session.runtime)
|
| 155 |
|
| 156 |
|
| 157 |
@router.post("/terms/remove")
|
| 158 |
def remove_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 159 |
cw = _module()
|
| 160 |
+
return _term_labels({"terms": _call(cw.remove_terms, _tenant(session), terms, st=session.runtime, page_only=True)})
|
| 161 |
|
| 162 |
|
| 163 |
@router.get("/settings")
|
|
|
|
| 217 |
return cw.NOT_READY
|
| 218 |
|
| 219 |
|
| 220 |
+
def _term_rows(cw, tenant, st, page):
|
| 221 |
+
"""Read dated-week counts for this server page without loading its ranking history."""
|
| 222 |
+
terms = page["terms"]
|
| 223 |
+
counts = {}
|
| 224 |
+
if terms:
|
| 225 |
+
try:
|
| 226 |
+
with cw._reader(tenant, st) as cur:
|
| 227 |
+
cols = cw._ba_columns(cur)
|
| 228 |
+
if {"search_term", "_period_start", "_period_end"} <= cols:
|
| 229 |
+
key = cw._term_sql("search_term")
|
| 230 |
+
rows = cur.execute(
|
| 231 |
+
f'SELECT {key}, COUNT(DISTINCT CAST("_period_start" AS DATE)) '
|
| 232 |
+
f'FROM {cw.BA_TABLE} WHERE {key} IN (' + ','.join('?' for _ in terms) + ') '
|
| 233 |
+
f'AND ({cw._weekly_period_sql()}) GROUP BY {key}', terms).fetchall()
|
| 234 |
+
counts = dict(rows)
|
| 235 |
+
except Exception as exc: # noqa: BLE001
|
| 236 |
+
cause = str(exc) if isinstance(exc, cw._ReadFailed) else type(exc).__name__
|
| 237 |
+
raise cw.SourceNotReady(cause, page=page) from exc
|
| 238 |
+
flags = page.get("termFlags") or {}
|
| 239 |
+
checked = page.get("trackingReady") is True
|
| 240 |
+
return [{"term": term, "termLabel": cw.display_term(term), "datedWeeks": counts.get(term, 0),
|
| 241 |
+
"starred": flags.get(term, {}).get("starred") is True,
|
| 242 |
+
"type": "Ours" if checked and flags.get(term, {}).get("ours") is True else
|
| 243 |
+
"Starred" if flags.get(term, {}).get("starred") is True else "Watched"}
|
| 244 |
+
for term in terms]
|
| 245 |
+
|
| 246 |
+
|
| 247 |
def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
|
| 248 |
"""Project `term_detail` to the screen: shares in percent points, previous values, causes, trend."""
|
| 249 |
periods = list(detail.get("periods") or ())
|
|
|
|
| 287 |
comparison = detail.get("comparison") or {}
|
| 288 |
ready = comparison.get("ready") is True
|
| 289 |
return {"canEdit": bool(can_edit), "terms": list(terms), "term": term,
|
| 290 |
+
"termLabel": cw.display_term(term) if term else None,
|
| 291 |
+
"termLabels": {key: cw.display_term(key) for key in [*terms, *([term] if term else [])]},
|
| 292 |
+
"termFlags": (page or {}).get("termFlags", {}),
|
| 293 |
+
"termRows": (page or {}).get("termRows", []),
|
| 294 |
"termsReady": (page or {}).get("termsReady", True),
|
| 295 |
+
"trackingReady": (page or {}).get("trackingReady", False),
|
| 296 |
+
"trackingCause": (page or {}).get("trackingCause"),
|
| 297 |
"week": {"start": periods[0], "end": _week_end(periods[0])} if periods else None,
|
| 298 |
"previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
|
| 299 |
"comparisonReady": ready,
|
|
|
|
| 306 |
@router.get("/screen")
|
| 307 |
def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(None),
|
| 308 |
q: str = "", offset: int = 0, limit: int = 50,
|
| 309 |
+
tracked: bool = False, termOrder: Literal["asc", "desc"] | None = None,
|
| 310 |
session: Session = Depends(_app_reader)):
|
| 311 |
"""S78-T17: the Terms view in one read (watched terms, the picked term's top 3 and trend)."""
|
| 312 |
cw = _module()
|
| 313 |
tenant = _tenant(session)
|
| 314 |
record = _app(session, app)
|
| 315 |
page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
|
| 316 |
+
limit=limit, selected=term, tracked=tracked, termOrder=termOrder)
|
| 317 |
terms, picked = page["terms"], page["term"]
|
| 318 |
+
page = {**page, "termRows": _call(_term_rows, cw, tenant, session.runtime, page)}
|
| 319 |
detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
|
| 320 |
return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
|
api/routes_competitors.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Competitors doors: list, add and remove an ASIN's rivals, with each rival's quote and the medians.
|
| 2 |
+
|
| 3 |
+
A SKU reads its ASIN's list. A Parent ASIN has no list (R74). Editors of Command Center write.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
from contextlib import contextmanager
|
| 9 |
+
import logging
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
from fastapi import APIRouter, Body, Depends
|
| 14 |
+
|
| 15 |
+
from core import amazon_rivals as rivals
|
| 16 |
+
from core import amazon_projects as projects
|
| 17 |
+
from deps import Session, err, require_session
|
| 18 |
+
import routes_amazon_profit
|
| 19 |
+
|
| 20 |
+
router = APIRouter(prefix="/api/v1")
|
| 21 |
+
_log = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
+
GRAINS = ("sku", "asin", "parent_asin")
|
| 24 |
+
FORBIDDEN = "Only an editor of Command Center can change competitors."
|
| 25 |
+
NO_MIRROR = "This workspace has no Amazon data yet."
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@contextmanager
|
| 29 |
+
def _open_mirror(runtime):
|
| 30 |
+
"""A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
|
| 31 |
+
from harness import runtime as R
|
| 32 |
+
if not Path(runtime.datastore_path).is_file():
|
| 33 |
+
yield None
|
| 34 |
+
return
|
| 35 |
+
cur = R.mirror_cursor(runtime, require_ready=False)
|
| 36 |
+
try:
|
| 37 |
+
yield cur
|
| 38 |
+
finally:
|
| 39 |
+
cur.close()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _rt(session):
|
| 43 |
+
session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
|
| 44 |
+
try:
|
| 45 |
+
rivals._backend(session.runtime)
|
| 46 |
+
except rivals.BackendUnavailable:
|
| 47 |
+
raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
|
| 48 |
+
return session.runtime
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _grain(grain, key):
|
| 52 |
+
if grain not in GRAINS:
|
| 53 |
+
raise err(400, "bad_grain", "Grain must be sku, asin or parent_asin.")
|
| 54 |
+
if not isinstance(key, str) or not key.strip():
|
| 55 |
+
raise err(400, "bad_key", "Name the SKU, ASIN or Parent ASIN.")
|
| 56 |
+
return key.strip()
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _target(cur, grain, key):
|
| 60 |
+
"""(asin, parent answer or None) for a grain and key."""
|
| 61 |
+
if grain == "sku":
|
| 62 |
+
try:
|
| 63 |
+
chain = projects.lineage(cur, "sku", key) if cur is not None else []
|
| 64 |
+
except projects.MirrorUnreadable as exc:
|
| 65 |
+
raise rivals.MirrorUnreadable(str(exc)) from exc
|
| 66 |
+
found = next((value for grain, value in chain if grain == "asin"), None)
|
| 67 |
+
if found is None:
|
| 68 |
+
raise err(404, "sku_not_listed", "This SKU has no ASIN in your Amazon listings.")
|
| 69 |
+
return found, None
|
| 70 |
+
try:
|
| 71 |
+
target = rivals.asin(key)
|
| 72 |
+
except ValueError as exc:
|
| 73 |
+
raise err(400, "bad_asin", str(exc)) from None
|
| 74 |
+
try:
|
| 75 |
+
kids = routes_amazon_profit.amazon_profit.M.parent_children(cur, target) if cur is not None else []
|
| 76 |
+
except projects.MirrorUnreadable as exc:
|
| 77 |
+
raise rivals.MirrorUnreadable(str(exc)) from exc
|
| 78 |
+
if grain == "parent_asin" or (kids and not rivals.our_asins(cur, [target])):
|
| 79 |
+
return target, {"children": kids}
|
| 80 |
+
return target, None
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def _payload(runtime, grain, key, target, parent, can_edit, discovery_error=None):
|
| 84 |
+
if parent is not None:
|
| 85 |
+
# R74: the rollup and each child's standing, never a parent rival list or a parent status.
|
| 86 |
+
kids = parent["children"] or [target]
|
| 87 |
+
found = rivals.standing(runtime, kids)
|
| 88 |
+
return {"grain": grain, "key": key, "asin": target, "parent": True, "rivals": [],
|
| 89 |
+
"children": parent["children"], "cause": rivals.PARENT_ROLLS_UP, "canEdit": False,
|
| 90 |
+
"rollup": rivals.rollup([found.get(a) for a in kids]),
|
| 91 |
+
"childStanding": [found.get(a) or {"asin": a, "competitors_cause": rivals.NO_QUOTE} for a in kids]}
|
| 92 |
+
listed = rivals.rivals(runtime, target)
|
| 93 |
+
quotes = rivals.newest_quotes(runtime, [r["rival_asin"] for r in listed["rivals"]])
|
| 94 |
+
mine = rivals.standing(runtime, [target]).get(target)
|
| 95 |
+
group = (mine or {}).get("rank_group")
|
| 96 |
+
return {"grain": grain, "key": key, "asin": target, "parent": False,
|
| 97 |
+
"rivals": [{"asin": r["rival_asin"], "title": r["title"], "source": r["source"],
|
| 98 |
+
"clickShare": r["click_share"], "term": r["term"], "week": r["week"],
|
| 99 |
+
"addedBy": r["added_by"], "quote": quotes.get(r["rival_asin"]),
|
| 100 |
+
"sameGroup": bool(group) and (quotes.get(r["rival_asin"]) or {}).get("display_group") == group}
|
| 101 |
+
for r in listed["rivals"]],
|
| 102 |
+
"standing": mine,
|
| 103 |
+
"autoCount": sum(1 for r in listed["rivals"] if r["source"] == "auto"),
|
| 104 |
+
"removed": listed["removed"], "week": listed["week"], "discoveredAt": listed["discoveredAt"],
|
| 105 |
+
"cause": listed["cause"], "discoveryError": discovery_error, "canEdit": can_edit}
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _rival_set(runtime, target):
|
| 109 |
+
return {r["rival_asin"] for r in rivals.rivals(runtime, target)["rivals"]}
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def _notify_rival_change(runtime, target, before):
|
| 113 |
+
"""Repaint affected cells from stored quotes after a committed rival membership change."""
|
| 114 |
+
try:
|
| 115 |
+
if _rival_set(runtime, target) != before:
|
| 116 |
+
from amazon_quotes_loop import _after_quotes
|
| 117 |
+
_after_quotes(runtime.key, (target,))
|
| 118 |
+
except Exception as exc: # noqa: BLE001
|
| 119 |
+
_log.warning("tenant=%s rival-change notification failed (%s); the rival change remains saved.",
|
| 120 |
+
runtime.key, type(exc).__name__)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
@router.get("/competitors")
|
| 124 |
+
def competitors_list(grain: str, key: str, session: Session = Depends(require_session)):
|
| 125 |
+
"""One SKU's, ASIN's or Parent ASIN's rivals, rediscovered when the newest week moved on."""
|
| 126 |
+
runtime = _rt(session)
|
| 127 |
+
key = _grain(grain, key)
|
| 128 |
+
problem = None
|
| 129 |
+
before = None
|
| 130 |
+
try:
|
| 131 |
+
with _open_mirror(runtime) as cur:
|
| 132 |
+
target, parent = _target(cur, grain, key)
|
| 133 |
+
if cur is None:
|
| 134 |
+
problem = NO_MIRROR
|
| 135 |
+
elif parent is None and rivals.stale(runtime, cur, target):
|
| 136 |
+
previous = _rival_set(runtime, target)
|
| 137 |
+
rivals.discover(runtime, cur, [target])
|
| 138 |
+
before = previous
|
| 139 |
+
except rivals.MirrorUnreadable as exc:
|
| 140 |
+
if grain == "sku":
|
| 141 |
+
raise err(503, "amazon_mirror_unreadable",
|
| 142 |
+
f"The Amazon data could not be read ({exc}). Try again shortly.") from None
|
| 143 |
+
target, parent = rivals.asin(key), (None if grain == "asin" else {"children": []})
|
| 144 |
+
problem = f"The Amazon data could not be read ({exc}), so the stored rivals show."
|
| 145 |
+
except rivals.BackendUnavailable:
|
| 146 |
+
raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
|
| 147 |
+
except RuntimeError:
|
| 148 |
+
raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
|
| 149 |
+
if before is not None:
|
| 150 |
+
_notify_rival_change(runtime, target, before)
|
| 151 |
+
return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def _writable(session, body):
|
| 155 |
+
runtime = _rt(session)
|
| 156 |
+
if not routes_amazon_profit._may_edit(session):
|
| 157 |
+
raise err(403, "forbidden", FORBIDDEN)
|
| 158 |
+
body = body if isinstance(body, dict) else {}
|
| 159 |
+
grain = body.get("grain")
|
| 160 |
+
key = _grain(grain, body.get("key"))
|
| 161 |
+
try:
|
| 162 |
+
with _open_mirror(runtime) as cur:
|
| 163 |
+
target, parent = _target(cur, grain, key)
|
| 164 |
+
except rivals.MirrorUnreadable as exc:
|
| 165 |
+
raise err(503, "amazon_mirror_unreadable",
|
| 166 |
+
f"The Amazon data could not be read ({exc}). Try again shortly.") from None
|
| 167 |
+
except rivals.BackendUnavailable:
|
| 168 |
+
raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
|
| 169 |
+
except RuntimeError:
|
| 170 |
+
raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
|
| 171 |
+
if parent is not None:
|
| 172 |
+
raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
|
| 173 |
+
try:
|
| 174 |
+
rival = rivals.asin(body.get("rival"), "rival")
|
| 175 |
+
except ValueError as exc:
|
| 176 |
+
raise err(400, "bad_asin", str(exc)) from None
|
| 177 |
+
return runtime, grain, key, target, rival
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
@router.post("/competitors/rivals")
|
| 181 |
+
def competitors_add(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 182 |
+
"""Add a rival ASIN by hand. It sits above the auto-found rivals."""
|
| 183 |
+
runtime, grain, key, target, rival = _writable(session, body)
|
| 184 |
+
before = _rival_set(runtime, target)
|
| 185 |
+
try:
|
| 186 |
+
rivals.add(runtime, target, rival, user=session.uname)
|
| 187 |
+
except ValueError as exc:
|
| 188 |
+
raise err(400, "bad_rival", str(exc)) from None
|
| 189 |
+
_notify_rival_change(runtime, target, before)
|
| 190 |
+
return _payload(runtime, grain, key, target, None, True)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
@router.post("/competitors/rivals/remove")
|
| 194 |
+
def competitors_remove(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 195 |
+
"""Remove a wrong rival. It stays removed after the next discovery."""
|
| 196 |
+
runtime, grain, key, target, rival = _writable(session, body)
|
| 197 |
+
before = _rival_set(runtime, target)
|
| 198 |
+
try:
|
| 199 |
+
rivals.remove(runtime, target, rival, user=session.uname)
|
| 200 |
+
except KeyError:
|
| 201 |
+
raise err(404, "rival_not_found", "This ASIN is not a rival of that product.") from None
|
| 202 |
+
_notify_rival_change(runtime, target, before)
|
| 203 |
+
return _payload(runtime, grain, key, target, None, True)
|
api/routes_connectors.py
CHANGED
|
@@ -2069,8 +2069,13 @@ def odoo_health_receipt(rt, connection_id=None, *, now=None, jobs=None):
|
|
| 2069 |
# that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
|
| 2070 |
cadence = stored.get("cadenceSeconds")
|
| 2071 |
try:
|
| 2072 |
-
|
| 2073 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2074 |
resident_cid = str(resident.get("connectionId") or "")
|
| 2075 |
if resident_cid == cid:
|
| 2076 |
cadence = resident.get("resolvedSeconds") or cadence
|
|
@@ -2150,21 +2155,66 @@ def _connection_jobs(rt, provider, connection_id):
|
|
| 2150 |
return connector_sync.jobs(rt, provider, cid)
|
| 2151 |
|
| 2152 |
|
| 2153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2154 |
"""One wire shape for every registered connector, including one not yet run."""
|
| 2155 |
meta = connector_sync.providers()[connector_key]
|
| 2156 |
slot = str(connection_id or "").strip() or "default"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2157 |
if row is None:
|
| 2158 |
-
|
| 2159 |
"connectionId": slot, "recorded": False, "state": None,
|
| 2160 |
"at": None, "stateAt": None, "lastSyncAt": None, "from": None,
|
| 2161 |
"detail": "no sync has been recorded", "errors": [], "errorsTotal": 0,
|
| 2162 |
-
"databases": [], "retryable": False, "retryHint": meta["retryHint"]}
|
| 2163 |
answer = dict(row)
|
|
|
|
| 2164 |
answer.update({"provider": connector_key, "connectionId": slot,
|
| 2165 |
-
"label": meta["label"], "recorded":
|
| 2166 |
"stateAt": row.get("at"), "lastSyncAt": None,
|
| 2167 |
"retryHint": meta["retryHint"]})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2168 |
return _safe_connector_status(connector_key, answer)
|
| 2169 |
|
| 2170 |
|
|
@@ -2710,7 +2760,8 @@ def _start_database_jobs():
|
|
| 2710 |
_materialization_wake.clear()
|
| 2711 |
try:
|
| 2712 |
for tenant, rt in tenant_scan.live_tenants(
|
| 2713 |
-
runtime,
|
|
|
|
| 2714 |
"connector database runtime failed: tenant=%s type=%s",
|
| 2715 |
tenant, type(exc).__name__)):
|
| 2716 |
if _materialization_stop.is_set():
|
|
@@ -2878,7 +2929,7 @@ def _odoo_status_reads(rt):
|
|
| 2878 |
import odoo_relational
|
| 2879 |
return _ConnectorStatusReads(rt, (
|
| 2880 |
keychain.KEY, keychain.SCOPE_KEY, odoo_relational.CONFIG_KEY,
|
| 2881 |
-
odoo_relational.MODEL_CATALOG_KEY, connector_sync.JOBS_KEY, connector_sync.STATE_KEY,
|
| 2882 |
_ODOO_HEALTH_STATUS, _DATABASE_DISPATCH_STATUS, _ODOO_SYNC_STATUS))
|
| 2883 |
|
| 2884 |
|
|
@@ -2958,7 +3009,22 @@ def odoo_status(connection_id: str = None, session: Session = Depends(require_se
|
|
| 2958 |
"receipt": receipt, "lastSyncAt": last_sync_at}
|
| 2959 |
|
| 2960 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2961 |
def _amazon_completed_sync(rt):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2962 |
"""Project completion markers without counting business rows or building database detail."""
|
| 2963 |
import duckdb
|
| 2964 |
from harness import amazon_store as az, datastore
|
|
@@ -2991,9 +3057,19 @@ def _amazon_completed_sync(rt):
|
|
| 2991 |
|
| 2992 |
|
| 2993 |
def _daily_completed_sync(rt, connector_key):
|
| 2994 |
-
"""Read
|
| 2995 |
if connector_key == "amazon_sp":
|
| 2996 |
return _amazon_completed_sync(rt)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2997 |
if connector_key == "meta_ads":
|
| 2998 |
from pathlib import Path
|
| 2999 |
from harness import datastore
|
|
@@ -3038,7 +3114,10 @@ def _amazon_restore_pending(rt):
|
|
| 3038 |
|
| 3039 |
|
| 3040 |
def _amazon_saved_completion(rt):
|
| 3041 |
-
"""The saved mirror manifest's completed_at
|
|
|
|
|
|
|
|
|
|
| 3042 |
key = str(getattr(rt, "key", "") or "")
|
| 3043 |
now = time.monotonic()
|
| 3044 |
cached = _SAVED_COMPLETION_CACHE.get(key)
|
|
@@ -3047,7 +3126,7 @@ def _amazon_saved_completion(rt):
|
|
| 3047 |
stamp = None
|
| 3048 |
try:
|
| 3049 |
from harness import amazon_store as az
|
| 3050 |
-
# Read the manifest object only: never
|
| 3051 |
download = getattr(rt, "download_bytes", None)
|
| 3052 |
raw = download(az.MIRROR_PREFIX + "manifest.json") if callable(download) else None
|
| 3053 |
manifest = json.loads(raw) if raw else None
|
|
@@ -3076,19 +3155,23 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
|
|
| 3076 |
raise err(404, "sync_provider_unknown",
|
| 3077 |
f"{connector_key} is not registered for connector sync")
|
| 3078 |
rt = (_odoo_status_reads(session.runtime) if connector_key == "odoo"
|
| 3079 |
-
else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY)
|
| 3080 |
if connector_key == "amazon_sp" else ()))
|
| 3081 |
resolved_connection = (_odoo_connection_slot(rt, connection_id)
|
| 3082 |
if connector_key == "odoo" else connection_id)
|
| 3083 |
jobs = _connection_jobs(rt, connector_key, resolved_connection)
|
| 3084 |
row = (connector_sync.job_summary(jobs)
|
| 3085 |
or connector_sync.read(rt, connector_key, resolved_connection))
|
| 3086 |
-
|
|
|
|
| 3087 |
if connector_key == "odoo":
|
| 3088 |
receipt = odoo_health_receipt(rt, resolved_connection, jobs=jobs)
|
| 3089 |
-
|
|
|
|
| 3090 |
if connector_key in ("amazon_sp", "meta_ads"):
|
| 3091 |
-
if
|
|
|
|
|
|
|
| 3092 |
completed, known = None, False
|
| 3093 |
else:
|
| 3094 |
completed, known = _daily_completed_sync(rt, connector_key)
|
|
@@ -4019,7 +4102,9 @@ def _amazon_database_sync_health(raw, enabled=True, *, now=None, database_key=""
|
|
| 4019 |
refused = (raw.get("refused") is True or is_refusal_reason(raw.get("unknownCause")) or any(item.get("table") == table for item in
|
| 4020 |
((raw.get("firstLoad") or {}).get("refused") or ()) if isinstance(item, dict))
|
| 4021 |
)
|
| 4022 |
-
if
|
|
|
|
|
|
|
| 4023 |
state = "partial" if success else "failed"
|
| 4024 |
error = {"code": "amazon_report_refused"}
|
| 4025 |
elif raw.get("stuck") is True:
|
|
@@ -4034,9 +4119,11 @@ def _amazon_database_sync_health(raw, enabled=True, *, now=None, database_key=""
|
|
| 4034 |
state, error = "queued", None
|
| 4035 |
else:
|
| 4036 |
state, error = "never_run", None
|
| 4037 |
-
|
| 4038 |
state=state, last_success=success, materialized_timestamp=success,
|
| 4039 |
row_count=raw.get("rowsLoaded"), error=error, now=now)
|
|
|
|
|
|
|
| 4040 |
|
| 4041 |
|
| 4042 |
def _stored_database_sync_health(raw, enabled=True, *, now=None):
|
|
@@ -4834,6 +4921,12 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 4834 |
raise_if_resource_error(exc)
|
| 4835 |
states = {}
|
| 4836 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4837 |
# Which OTHER declared tables are filled from the SAME report document. Derived from the
|
| 4838 |
# report map so a table added to `REPORT_TYPES` gets this disclosure for free.
|
| 4839 |
by_report = {}
|
|
@@ -4872,6 +4965,13 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 4872 |
# what `native` carries: the vendor's own name for the thing, never one we invented. It is
|
| 4873 |
# empty for a table whose report type is unknown, and the tab says so rather than guessing.
|
| 4874 |
raw_sync = (states or {}).get(name) if is_admin else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4875 |
if isinstance(raw_sync, dict):
|
| 4876 |
from harness.amazon_store import plain_refusal_reason, is_refusal_reason
|
| 4877 |
raw_sync = dict(raw_sync)
|
|
@@ -5085,9 +5185,11 @@ def _databases_sync_health(databases):
|
|
| 5085 |
"succeeded", "disabled") if state in states), "never_run")
|
| 5086 |
successes = [_sync_stamp(row.get("lastSuccess")) for row in health]
|
| 5087 |
failure = next((row.get("error") for row in health if row.get("error")), None)
|
| 5088 |
-
|
| 5089 |
last_success=max((stamp for stamp in successes if stamp), default=None),
|
| 5090 |
error=failure)
|
|
|
|
|
|
|
| 5091 |
|
| 5092 |
|
| 5093 |
def _detail_overview(row, databases, graph, *, sync_health=None):
|
|
@@ -5650,16 +5752,12 @@ class _ReplacementConflict(Exception):
|
|
| 5650 |
"""The addressed keychain row changed between replacement preparation and its store update."""
|
| 5651 |
|
| 5652 |
|
| 5653 |
-
def _encrypted_replacement_payload(kc, label, entry_type, fields, username):
|
| 5654 |
-
"""Validate and
|
| 5655 |
-
|
| 5656 |
-
``add_entry`` is the keychain's one validation, preview and Fernet implementation. A tiny
|
| 5657 |
-
memory-only runtime lets this route reuse it while discarding the temporary identity it mints;
|
| 5658 |
-
the tenant store sees only the later, single-row payload swap.
|
| 5659 |
-
"""
|
| 5660 |
class _Stage:
|
| 5661 |
def __init__(self):
|
| 5662 |
self.bucket = {}
|
|
|
|
| 5663 |
|
| 5664 |
def get(self, name):
|
| 5665 |
return self.bucket if name == kc.KEY else {}
|
|
@@ -5676,8 +5774,9 @@ def _encrypted_replacement_payload(kc, label, entry_type, fields, username):
|
|
| 5676 |
row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {})
|
| 5677 |
if not row.get("enc"):
|
| 5678 |
raise RuntimeError("the keychain encryptor returned no encrypted payload")
|
| 5679 |
-
|
| 5680 |
-
|
|
|
|
| 5681 |
if entry_type == "amazon_sp":
|
| 5682 |
payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD]
|
| 5683 |
return payload
|
|
@@ -5768,7 +5867,7 @@ def _replace_connector_key(connector_key, entry_id, body, session):
|
|
| 5768 |
raise err(400, "bad_replacement", refusal)
|
| 5769 |
try:
|
| 5770 |
encrypted = _encrypted_replacement_payload(
|
| 5771 |
-
kc, entry.get("label"), entry_type, merged_fields, session.uname)
|
| 5772 |
except kc.KeychainLocked as exc:
|
| 5773 |
raise err(503, "keychain_locked",
|
| 5774 |
f"the keychain is locked: {exc}. Nothing was changed.")
|
|
|
|
| 2069 |
# that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
|
| 2070 |
cadence = stored.get("cadenceSeconds")
|
| 2071 |
try:
|
| 2072 |
+
canonical = connector_sync.status(rt, "odoo", cid, fresh=True)
|
| 2073 |
+
if canonical and isinstance(canonical.get("resync"), dict):
|
| 2074 |
+
resident = dict(canonical["resync"])
|
| 2075 |
+
resident["connectionId"] = cid
|
| 2076 |
+
else:
|
| 2077 |
+
_main = running_main() # D-781: resolve, never import
|
| 2078 |
+
resident = dict(_main.RESYNC_STATE)
|
| 2079 |
resident_cid = str(resident.get("connectionId") or "")
|
| 2080 |
if resident_cid == cid:
|
| 2081 |
cadence = resident.get("resolvedSeconds") or cadence
|
|
|
|
| 2155 |
return connector_sync.jobs(rt, provider, cid)
|
| 2156 |
|
| 2157 |
|
| 2158 |
+
_PASS_STATES = {"running": "syncing", "continuing": "syncing", "done": "ready", "waiting": "ready",
|
| 2159 |
+
"held": "partial", "failed": "error"}
|
| 2160 |
+
|
| 2161 |
+
|
| 2162 |
+
def _stored_sync_row(rt, connector_key, slot):
|
| 2163 |
+
"""C-status: a recorded lifecycle row from the store's last pass and last good sync, or None."""
|
| 2164 |
+
entry = connector_sync.status(rt, connector_key, slot, fresh=True) if rt is not None else None
|
| 2165 |
+
if not entry:
|
| 2166 |
+
return None
|
| 2167 |
+
last_pass = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
|
| 2168 |
+
good = _sync_stamp(entry.get("lastGoodSyncAt"))
|
| 2169 |
+
state = _PASS_STATES.get(last_pass.get("outcome")) or ("ready" if good else None)
|
| 2170 |
+
at = _sync_stamp(last_pass.get("endedAt") or last_pass.get("startedAt")) or good
|
| 2171 |
+
detail = str(last_pass.get("detail") or "") or ("Sync finished." if state == "ready" else "")
|
| 2172 |
+
return {"state": state, "at": at, "from": None, "detail": detail,
|
| 2173 |
+
"errors": [detail] if state == "error" and detail else [],
|
| 2174 |
+
"errorsTotal": 1 if state == "error" and detail else 0, "databases": [],
|
| 2175 |
+
"retryable": state in ("error", "partial"), "storedStatus": entry}
|
| 2176 |
+
|
| 2177 |
+
|
| 2178 |
+
def _sync_payload(connector_key, connection_id, row=None, *, rt=None):
|
| 2179 |
"""One wire shape for every registered connector, including one not yet run."""
|
| 2180 |
meta = connector_sync.providers()[connector_key]
|
| 2181 |
slot = str(connection_id or "").strip() or "default"
|
| 2182 |
+
stored_row = _stored_sync_row(rt, connector_key, slot)
|
| 2183 |
+
entry = (stored_row or {}).get("storedStatus") or {}
|
| 2184 |
+
if row is None:
|
| 2185 |
+
row = stored_row
|
| 2186 |
+
recorded = row is not None
|
| 2187 |
if row is None:
|
| 2188 |
+
row = {"provider": connector_key, "label": meta["label"],
|
| 2189 |
"connectionId": slot, "recorded": False, "state": None,
|
| 2190 |
"at": None, "stateAt": None, "lastSyncAt": None, "from": None,
|
| 2191 |
"detail": "no sync has been recorded", "errors": [], "errorsTotal": 0,
|
| 2192 |
+
"databases": [], "retryable": False, "retryHint": meta["retryHint"]}
|
| 2193 |
answer = dict(row)
|
| 2194 |
+
answer.pop("storedStatus", None)
|
| 2195 |
answer.update({"provider": connector_key, "connectionId": slot,
|
| 2196 |
+
"label": meta["label"], "recorded": recorded,
|
| 2197 |
"stateAt": row.get("at"), "lastSyncAt": None,
|
| 2198 |
"retryHint": meta["retryHint"]})
|
| 2199 |
+
if entry:
|
| 2200 |
+
last_pass = entry.get("pass") or {}
|
| 2201 |
+
duration, budget = last_pass.get("durationSeconds"), last_pass.get("budgetSeconds")
|
| 2202 |
+
import math
|
| 2203 |
+
def minutes(value, *, positive=False):
|
| 2204 |
+
return (round(value / 60, 3) if type(value) in (int, float) and math.isfinite(value)
|
| 2205 |
+
and (value > 0 if positive else value >= 0) else None)
|
| 2206 |
+
days = entry.get("dailyPasses") or {}
|
| 2207 |
+
answer.update(lastGoodSyncAt=entry.get("lastGoodSyncAt"), lastSyncAt=entry.get("lastGoodSyncAt"),
|
| 2208 |
+
lastCompletedKnown=True,
|
| 2209 |
+
**{"pass": last_pass}, passMinutes=minutes(duration), budgetMinutes=minutes(budget, positive=True),
|
| 2210 |
+
dailyPasses=days, dailyPass=days[max(days)] if days else None,
|
| 2211 |
+
resync=entry.get("resync"), held=last_pass.get("held") or {},
|
| 2212 |
+
notOffered=last_pass.get("notOffered") or 0)
|
| 2213 |
+
if connector_key == "amazon_sp" and rt is not None:
|
| 2214 |
+
from harness import amazon_store
|
| 2215 |
+
job = amazon_store.first_load_job_summary(rt)
|
| 2216 |
+
if job:
|
| 2217 |
+
answer["amazonFirstLoad"] = job
|
| 2218 |
return _safe_connector_status(connector_key, answer)
|
| 2219 |
|
| 2220 |
|
|
|
|
| 2760 |
_materialization_wake.clear()
|
| 2761 |
try:
|
| 2762 |
for tenant, rt in tenant_scan.live_tenants(
|
| 2763 |
+
runtime, name="connector-materialization",
|
| 2764 |
+
on_error=lambda tenant, exc: _DATABASE_JOBS_LOG.error(
|
| 2765 |
"connector database runtime failed: tenant=%s type=%s",
|
| 2766 |
tenant, type(exc).__name__)):
|
| 2767 |
if _materialization_stop.is_set():
|
|
|
|
| 2929 |
import odoo_relational
|
| 2930 |
return _ConnectorStatusReads(rt, (
|
| 2931 |
keychain.KEY, keychain.SCOPE_KEY, odoo_relational.CONFIG_KEY,
|
| 2932 |
+
odoo_relational.MODEL_CATALOG_KEY, connector_sync.JOBS_KEY, connector_sync.STATE_KEY, connector_sync.STATUS_KEY,
|
| 2933 |
_ODOO_HEALTH_STATUS, _DATABASE_DISPATCH_STATUS, _ODOO_SYNC_STATUS))
|
| 2934 |
|
| 2935 |
|
|
|
|
| 3009 |
"receipt": receipt, "lastSyncAt": last_sync_at}
|
| 3010 |
|
| 3011 |
|
| 3012 |
+
def _stored_completed_sync(rt, connector_key):
|
| 3013 |
+
"""C-status: the store's lastGoodSyncAt as (stamp, True), or None when the store holds no entry."""
|
| 3014 |
+
entry = connector_sync.status(rt, connector_key, fresh=True)
|
| 3015 |
+
if entry is None:
|
| 3016 |
+
return None
|
| 3017 |
+
stamp = _sync_stamp(entry.get("lastGoodSyncAt"))
|
| 3018 |
+
return stamp, True
|
| 3019 |
+
|
| 3020 |
+
|
| 3021 |
def _amazon_completed_sync(rt):
|
| 3022 |
+
"""The store's last good Amazon sync; the mirror markers answer only before the store holds one."""
|
| 3023 |
+
stored = _stored_completed_sync(rt, "amazon_sp")
|
| 3024 |
+
return stored if stored is not None else _amazon_mirror_completion(rt)
|
| 3025 |
+
|
| 3026 |
+
|
| 3027 |
+
def _amazon_mirror_completion(rt):
|
| 3028 |
"""Project completion markers without counting business rows or building database detail."""
|
| 3029 |
import duckdb
|
| 3030 |
from harness import amazon_store as az, datastore
|
|
|
|
| 3057 |
|
| 3058 |
|
| 3059 |
def _daily_completed_sync(rt, connector_key):
|
| 3060 |
+
"""Read the stored last good sync; never create a mirror or contact a provider."""
|
| 3061 |
if connector_key == "amazon_sp":
|
| 3062 |
return _amazon_completed_sync(rt)
|
| 3063 |
+
if connector_key == "meta_ads":
|
| 3064 |
+
stored = _stored_completed_sync(rt, "meta_ads")
|
| 3065 |
+
return stored if stored is not None else _mirror_completed_sync(rt, "meta_ads")
|
| 3066 |
+
return None, False
|
| 3067 |
+
|
| 3068 |
+
|
| 3069 |
+
def _mirror_completed_sync(rt, connector_key):
|
| 3070 |
+
"""The container mirror's completed markers. Writers seed C-status from it; cards read the store."""
|
| 3071 |
+
if connector_key == "amazon_sp":
|
| 3072 |
+
return _amazon_mirror_completion(rt)
|
| 3073 |
if connector_key == "meta_ads":
|
| 3074 |
from pathlib import Path
|
| 3075 |
from harness import datastore
|
|
|
|
| 3114 |
|
| 3115 |
|
| 3116 |
def _amazon_saved_completion(rt):
|
| 3117 |
+
"""The stored last good sync, else the saved mirror manifest's completed_at, or None."""
|
| 3118 |
+
stored = _stored_completed_sync(rt, "amazon_sp")
|
| 3119 |
+
if stored is not None:
|
| 3120 |
+
return stored[0]
|
| 3121 |
key = str(getattr(rt, "key", "") or "")
|
| 3122 |
now = time.monotonic()
|
| 3123 |
cached = _SAVED_COMPLETION_CACHE.get(key)
|
|
|
|
| 3126 |
stamp = None
|
| 3127 |
try:
|
| 3128 |
from harness import amazon_store as az
|
| 3129 |
+
# Read the manifest object only: never a duckdb connection.
|
| 3130 |
download = getattr(rt, "download_bytes", None)
|
| 3131 |
raw = download(az.MIRROR_PREFIX + "manifest.json") if callable(download) else None
|
| 3132 |
manifest = json.loads(raw) if raw else None
|
|
|
|
| 3155 |
raise err(404, "sync_provider_unknown",
|
| 3156 |
f"{connector_key} is not registered for connector sync")
|
| 3157 |
rt = (_odoo_status_reads(session.runtime) if connector_key == "odoo"
|
| 3158 |
+
else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY, connector_sync.STATUS_KEY)
|
| 3159 |
if connector_key == "amazon_sp" else ()))
|
| 3160 |
resolved_connection = (_odoo_connection_slot(rt, connection_id)
|
| 3161 |
if connector_key == "odoo" else connection_id)
|
| 3162 |
jobs = _connection_jobs(rt, connector_key, resolved_connection)
|
| 3163 |
row = (connector_sync.job_summary(jobs)
|
| 3164 |
or connector_sync.read(rt, connector_key, resolved_connection))
|
| 3165 |
+
# Odoo's lifecycle comes from its job receipts; the other connectors read C-status beside no row.
|
| 3166 |
+
answer = _sync_payload(connector_key, resolved_connection, row, rt=rt)
|
| 3167 |
if connector_key == "odoo":
|
| 3168 |
receipt = odoo_health_receipt(rt, resolved_connection, jobs=jobs)
|
| 3169 |
+
if not answer.get("lastGoodSyncAt"):
|
| 3170 |
+
answer["lastSyncAt"] = _odoo_last_sync_at(rt, receipt, connection_id)
|
| 3171 |
if connector_key in ("amazon_sp", "meta_ads"):
|
| 3172 |
+
if answer.get("lastCompletedKnown") is True:
|
| 3173 |
+
completed, known = answer.get("lastGoodSyncAt"), True
|
| 3174 |
+
elif connector_key == "amazon_sp" and (str(resolved_connection or "").strip() or "default") != "default":
|
| 3175 |
completed, known = None, False
|
| 3176 |
else:
|
| 3177 |
completed, known = _daily_completed_sync(rt, connector_key)
|
|
|
|
| 4102 |
refused = (raw.get("refused") is True or is_refusal_reason(raw.get("unknownCause")) or any(item.get("table") == table for item in
|
| 4103 |
((raw.get("firstLoad") or {}).get("refused") or ()) if isinstance(item, dict))
|
| 4104 |
)
|
| 4105 |
+
if raw.get("notOffered") is True:
|
| 4106 |
+
state, error = "succeeded", None
|
| 4107 |
+
elif refused:
|
| 4108 |
state = "partial" if success else "failed"
|
| 4109 |
error = {"code": "amazon_report_refused"}
|
| 4110 |
elif raw.get("stuck") is True:
|
|
|
|
| 4119 |
state, error = "queued", None
|
| 4120 |
else:
|
| 4121 |
state, error = "never_run", None
|
| 4122 |
+
answer = _database_sync_health(
|
| 4123 |
state=state, last_success=success, materialized_timestamp=success,
|
| 4124 |
row_count=raw.get("rowsLoaded"), error=error, now=now)
|
| 4125 |
+
answer["notOffered"] = int(raw.get("notOffered") is True)
|
| 4126 |
+
return answer
|
| 4127 |
|
| 4128 |
|
| 4129 |
def _stored_database_sync_health(raw, enabled=True, *, now=None):
|
|
|
|
| 4921 |
raise_if_resource_error(exc)
|
| 4922 |
states = {}
|
| 4923 |
|
| 4924 |
+
canonical = connector_sync.status(rt, "amazon_sp", fresh=True) or {}
|
| 4925 |
+
canonical_pass = canonical.get("pass") or {}
|
| 4926 |
+
held_tables = canonical_pass.get("held") or {}
|
| 4927 |
+
from harness import amazon_store as _load_store
|
| 4928 |
+
first_load_job = _load_store.first_load_job_summary(rt) if is_admin else None
|
| 4929 |
+
|
| 4930 |
# Which OTHER declared tables are filled from the SAME report document. Derived from the
|
| 4931 |
# report map so a table added to `REPORT_TYPES` gets this disclosure for free.
|
| 4932 |
by_report = {}
|
|
|
|
| 4965 |
# what `native` carries: the vendor's own name for the thing, never one we invented. It is
|
| 4966 |
# empty for a table whose report type is unknown, and the tab says so rather than guessing.
|
| 4967 |
raw_sync = (states or {}).get(name) if is_admin else None
|
| 4968 |
+
if is_admin and (name in held_tables or first_load_job):
|
| 4969 |
+
raw_sync = dict(raw_sync or {})
|
| 4970 |
+
if name in held_tables:
|
| 4971 |
+
raw_sync["held"] = held_tables[name]
|
| 4972 |
+
raw_sync["notOffered"] = name in (canonical_pass.get("notOfferedTables") or ())
|
| 4973 |
+
if first_load_job:
|
| 4974 |
+
raw_sync["firstLoad"] = {**(raw_sync.get("firstLoad") or {}), **first_load_job}
|
| 4975 |
if isinstance(raw_sync, dict):
|
| 4976 |
from harness.amazon_store import plain_refusal_reason, is_refusal_reason
|
| 4977 |
raw_sync = dict(raw_sync)
|
|
|
|
| 5185 |
"succeeded", "disabled") if state in states), "never_run")
|
| 5186 |
successes = [_sync_stamp(row.get("lastSuccess")) for row in health]
|
| 5187 |
failure = next((row.get("error") for row in health if row.get("error")), None)
|
| 5188 |
+
answer = _database_sync_health(state=state,
|
| 5189 |
last_success=max((stamp for stamp in successes if stamp), default=None),
|
| 5190 |
error=failure)
|
| 5191 |
+
answer["notOffered"] = sum(row.get("notOffered") or 0 for row in health)
|
| 5192 |
+
return answer
|
| 5193 |
|
| 5194 |
|
| 5195 |
def _detail_overview(row, databases, graph, *, sync_health=None):
|
|
|
|
| 5752 |
"""The addressed keychain row changed between replacement preparation and its store update."""
|
| 5753 |
|
| 5754 |
|
| 5755 |
+
def _encrypted_replacement_payload(kc, label, entry_type, fields, username, rt, entry_id):
|
| 5756 |
+
"""Validate a temporary keychain row and reseal its fields for the retained tenant and entry."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5757 |
class _Stage:
|
| 5758 |
def __init__(self):
|
| 5759 |
self.bucket = {}
|
| 5760 |
+
self.key = rt.key
|
| 5761 |
|
| 5762 |
def get(self, name):
|
| 5763 |
return self.bucket if name == kc.KEY else {}
|
|
|
|
| 5774 |
row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {})
|
| 5775 |
if not row.get("enc"):
|
| 5776 |
raise RuntimeError("the keychain encryptor returned no encrypted payload")
|
| 5777 |
+
validated_fields = kc.read_fields(stage, temporary["id"])
|
| 5778 |
+
envelope = kc.seal_fields(rt.key, str(entry_id), validated_fields)
|
| 5779 |
+
payload = {**envelope, "preview": str(row.get("preview") or "")}
|
| 5780 |
if entry_type == "amazon_sp":
|
| 5781 |
payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD]
|
| 5782 |
return payload
|
|
|
|
| 5867 |
raise err(400, "bad_replacement", refusal)
|
| 5868 |
try:
|
| 5869 |
encrypted = _encrypted_replacement_payload(
|
| 5870 |
+
kc, entry.get("label"), entry_type, merged_fields, session.uname, session.runtime, entry_id)
|
| 5871 |
except kc.KeychainLocked as exc:
|
| 5872 |
raise err(503, "keychain_locked",
|
| 5873 |
f"the keychain is locked: {exc}. Nothing was changed.")
|
api/routes_customers.py
CHANGED
|
@@ -278,10 +278,11 @@ def warm_default(rt):
|
|
| 278 |
def warm_bu_pools(rt):
|
| 279 |
"""Warm each permitted business-unit pool outside the request path."""
|
| 280 |
from core import context, perms
|
|
|
|
| 281 |
for label in perms.allowed_bu_labels({"bus": "all", "role": "admin"}):
|
| 282 |
-
if label == "All" or label not in
|
| 283 |
continue
|
| 284 |
-
team = perms.scope_team_id({"bus": [
|
| 285 |
if team is None:
|
| 286 |
continue
|
| 287 |
try:
|
|
@@ -1303,7 +1304,16 @@ def allowed_pids(session: Session):
|
|
| 1303 |
|
| 1304 |
@router.get("/customers")
|
| 1305 |
def customers(session: Session = Depends(module_gate(MODULE)), related: str = None):
|
| 1306 |
-
return _payload(session, related=related)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1307 |
|
| 1308 |
|
| 1309 |
@router.post("/customers/aggregates")
|
|
@@ -1339,10 +1349,10 @@ def assigned_view_rows(view_id: str, session: Session = Depends(module_gate(MODU
|
|
| 1339 |
session, MODULE, rows, g['fields'], g['measures'], None)
|
| 1340 |
rows = _project_cell_notes(MODULE, rows, g['fields'], session.runtime)
|
| 1341 |
rows = _exclude_sourceless(rows, g['fields'])
|
| 1342 |
-
return {"fields": g["fields"], "rows": rows, "viewId": view_id,
|
| 1343 |
"locked": True, "today": g["today"],
|
| 1344 |
"refusals": metric_refusals,
|
| 1345 |
-
"limits": field_limits + linked_metric_limits(metric_refusals)}
|
| 1346 |
|
| 1347 |
|
| 1348 |
# ── Wave 44: normalized, Loopable-owned Route/Zone APIs ─────────────────────
|
|
@@ -1649,6 +1659,26 @@ def _visible_routes(session, records=None, application=None):
|
|
| 1649 |
return [route for route, _mark in _visible_normalized_routes(session, records, application)]
|
| 1650 |
|
| 1651 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1652 |
def _legacy_route_navigation(session):
|
| 1653 |
"""Wave 42 per-reader Route folders, used only as a no-write migration fallback."""
|
| 1654 |
try:
|
|
@@ -1668,7 +1698,7 @@ def _route_navigation(session, routes, application=None):
|
|
| 1668 |
legacy_folders=folders, legacy_placements=placements, application=application)
|
| 1669 |
|
| 1670 |
|
| 1671 |
-
def _route_dto(route, allowed, active, session, share_mark=None):
|
| 1672 |
from core import route_system
|
| 1673 |
allowed = {int(pid) for pid in allowed}
|
| 1674 |
stops = [stop for stop in route_system.ordered_stops(route)
|
|
@@ -1696,6 +1726,9 @@ def _route_dto(route, allowed, active, session, share_mark=None):
|
|
| 1696 |
# ⭐ `not compatibility and ...` led this expression until the legacy projection was retired.
|
| 1697 |
# Only a projected Route ever carried that marker, and nothing mints one now.
|
| 1698 |
can_edit = bool(owns_route or mark.get("sharedRole") == "edit" or source_edit)
|
|
|
|
|
|
|
|
|
|
| 1699 |
dto = {"id": route["id"], "name": route["name"], "color": route["color"],
|
| 1700 |
"application": route.get("application"),
|
| 1701 |
"number": route["number"], "visibleStopIds": visible,
|
|
@@ -1754,6 +1787,32 @@ def _route_or_404(session, route_id, application=None):
|
|
| 1754 |
return route
|
| 1755 |
|
| 1756 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1757 |
def _route_mutation_allowed(session, route, requested_source=None):
|
| 1758 |
"""W47-T08 / RULING R6 — OWNERSHIP SHORT-CIRCUITS, and that is the whole of instruction 22.
|
| 1759 |
|
|
@@ -1896,15 +1955,19 @@ def routes(session: Session = Depends(module_gate(MODULE)), application: str = N
|
|
| 1896 |
50,000 visible records sends exactly the same request as an empty grid; a future annotation
|
| 1897 |
door may accept a compact saved-View/filter reference without reopening that row-sized wire.
|
| 1898 |
"""
|
| 1899 |
-
application = _entry_route_application(session, application)
|
| 1900 |
allowed = allowed_pids(session)
|
| 1901 |
records = _route_share_records(session)
|
| 1902 |
navigation = _route_navigation(session, _visible_routes(session, records, application), application)
|
| 1903 |
-
|
| 1904 |
-
|
| 1905 |
-
|
| 1906 |
-
|
| 1907 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1908 |
|
| 1909 |
|
| 1910 |
@router.get("/customers/routes/delivery-recipients")
|
|
@@ -2079,10 +2142,13 @@ def remove_route(route_id: str, session: Session = Depends(module_gate(MODULE)),
|
|
| 2079 |
def move_route_to_folder(route_id: str, body: dict = Body(default=None),
|
| 2080 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2081 |
from core import route_system
|
| 2082 |
-
application = _entry_route_application(session, application, write=True)
|
| 2083 |
_route_application_body(body or {}, application)
|
| 2084 |
routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
|
| 2085 |
if str(route_id) not in {str(route.get("id")) for route in routes}:
|
|
|
|
|
|
|
|
|
|
| 2086 |
raise err(404, "route_not_found", "that Route does not exist")
|
| 2087 |
folder_id = (body or {}).get("folderId")
|
| 2088 |
if folder_id is not None and not isinstance(folder_id, str):
|
|
@@ -2102,12 +2168,16 @@ def move_route_to_folder(route_id: str, body: dict = Body(default=None),
|
|
| 2102 |
def reorder_route_navigation(body: dict = Body(default=None),
|
| 2103 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2104 |
from core import route_system
|
| 2105 |
-
application = _entry_route_application(session, application, write=True)
|
| 2106 |
_route_application_body(body or {}, application)
|
| 2107 |
routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
|
| 2108 |
order = (body or {}).get("order")
|
| 2109 |
if not isinstance(order, list):
|
| 2110 |
raise err(400, "invalid_route_order", "order must be a list of visible Route ids")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2111 |
result = route_system.reorder_routes(
|
| 2112 |
session.runtime, session.uname, order, routes,
|
| 2113 |
legacy_folders=legacy_folders, legacy_placements=legacy_placements, application=application)
|
|
@@ -2715,8 +2785,8 @@ def route_delivery_list(session: Session = Depends(module_gate(MODULE)), applica
|
|
| 2715 |
def normalized_route_delivery_list(route_id: str,
|
| 2716 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2717 |
"""Deliveries for the one normalized Route whose detail is open."""
|
| 2718 |
-
application = _entry_route_application(session, application)
|
| 2719 |
-
|
| 2720 |
return {"deliveries": _route_delivery_definitions(session, route_id, application)}
|
| 2721 |
|
| 2722 |
|
|
|
|
| 278 |
def warm_bu_pools(rt):
|
| 279 |
"""Warm each permitted business-unit pool outside the request path."""
|
| 280 |
from core import context, perms
|
| 281 |
+
brands = context.brands()
|
| 282 |
for label in perms.allowed_bu_labels({"bus": "all", "role": "admin"}):
|
| 283 |
+
if label == "All" or label not in brands:
|
| 284 |
continue
|
| 285 |
+
team = perms.scope_team_id({"bus": [brands[label]], "role": "user"})
|
| 286 |
if team is None:
|
| 287 |
continue
|
| 288 |
try:
|
|
|
|
| 1304 |
|
| 1305 |
@router.get("/customers")
|
| 1306 |
def customers(session: Session = Depends(module_gate(MODULE)), related: str = None):
|
| 1307 |
+
return _audit_customer_response(_payload(session, related=related), session)
|
| 1308 |
+
|
| 1309 |
+
|
| 1310 |
+
def _audit_customer_response(payload, session):
|
| 1311 |
+
if not isinstance(payload, dict) or payload.get('ok') is False:
|
| 1312 |
+
return payload
|
| 1313 |
+
from core import personal_reads
|
| 1314 |
+
personal_reads.record_rows(MODULE, payload.get('rows') or [], st=session.runtime,
|
| 1315 |
+
user=session.user, definition={'fields': payload.get('fields') or []})
|
| 1316 |
+
return payload
|
| 1317 |
|
| 1318 |
|
| 1319 |
@router.post("/customers/aggregates")
|
|
|
|
| 1349 |
session, MODULE, rows, g['fields'], g['measures'], None)
|
| 1350 |
rows = _project_cell_notes(MODULE, rows, g['fields'], session.runtime)
|
| 1351 |
rows = _exclude_sourceless(rows, g['fields'])
|
| 1352 |
+
return _audit_customer_response({"fields": g["fields"], "rows": rows, "viewId": view_id,
|
| 1353 |
"locked": True, "today": g["today"],
|
| 1354 |
"refusals": metric_refusals,
|
| 1355 |
+
"limits": field_limits + linked_metric_limits(metric_refusals)}, session)
|
| 1356 |
|
| 1357 |
|
| 1358 |
# ── Wave 44: normalized, Loopable-owned Route/Zone APIs ─────────────────────
|
|
|
|
| 1659 |
return [route for route, _mark in _visible_normalized_routes(session, records, application)]
|
| 1660 |
|
| 1661 |
|
| 1662 |
+
def _cross_partition_routes(session, records, application):
|
| 1663 |
+
"""R11: the no-App list also reads Routes in other partitions shared to the caller, plus the
|
| 1664 |
+
no-App partition for an admin whose own Routing App took the request. Never writable here."""
|
| 1665 |
+
from core import route_system
|
| 1666 |
+
records = records if isinstance(records, dict) else _route_share_records(session)
|
| 1667 |
+
home = str(application or "").strip() or None
|
| 1668 |
+
out = []
|
| 1669 |
+
for route in route_system.read(session.runtime).get("routes") or []:
|
| 1670 |
+
if not isinstance(route, dict):
|
| 1671 |
+
continue
|
| 1672 |
+
partition = str(route.get("application") or "").strip() or None
|
| 1673 |
+
if partition == home:
|
| 1674 |
+
continue
|
| 1675 |
+
mark = _route_share_mark(session, route, records)
|
| 1676 |
+
if mark["shared"] or (session.admin and home is not None and partition is None):
|
| 1677 |
+
out.append((deepcopy(route), mark))
|
| 1678 |
+
out.sort(key=lambda item: (int(item[0].get("number") or 0), str(item[0].get("id") or "")))
|
| 1679 |
+
return out
|
| 1680 |
+
|
| 1681 |
+
|
| 1682 |
def _legacy_route_navigation(session):
|
| 1683 |
"""Wave 42 per-reader Route folders, used only as a no-write migration fallback."""
|
| 1684 |
try:
|
|
|
|
| 1698 |
legacy_folders=folders, legacy_placements=placements, application=application)
|
| 1699 |
|
| 1700 |
|
| 1701 |
+
def _route_dto(route, allowed, active, session, share_mark=None, *, writable=True):
|
| 1702 |
from core import route_system
|
| 1703 |
allowed = {int(pid) for pid in allowed}
|
| 1704 |
stops = [stop for stop in route_system.ordered_stops(route)
|
|
|
|
| 1726 |
# ⭐ `not compatibility and ...` led this expression until the legacy projection was retired.
|
| 1727 |
# Only a projected Route ever carried that marker, and nothing mints one now.
|
| 1728 |
can_edit = bool(owns_route or mark.get("sharedRole") == "edit" or source_edit)
|
| 1729 |
+
if not writable:
|
| 1730 |
+
# R11: a Route listed from another partition offers no write; the client hides each one.
|
| 1731 |
+
can_edit = source_edit = False
|
| 1732 |
dto = {"id": route["id"], "name": route["name"], "color": route["color"],
|
| 1733 |
"application": route.get("application"),
|
| 1734 |
"number": route["number"], "visibleStopIds": visible,
|
|
|
|
| 1787 |
return route
|
| 1788 |
|
| 1789 |
|
| 1790 |
+
def _route_for_read_or_404(session, route_id, application=None, requested=None):
|
| 1791 |
+
"""A read door's Route lookup. On the no-App door it also finds a Route the no-App list shows
|
| 1792 |
+
from another partition (R11). Every write door keeps `_route_or_404`."""
|
| 1793 |
+
try:
|
| 1794 |
+
return _route_or_404(session, route_id, application)
|
| 1795 |
+
except Exception as exc:
|
| 1796 |
+
if requested is not None or getattr(exc, "status_code", None) != 404:
|
| 1797 |
+
raise
|
| 1798 |
+
found = next((route for route, _mark in _cross_partition_routes(session, None, application)
|
| 1799 |
+
if route.get("id") == str(route_id)), None)
|
| 1800 |
+
if found is None:
|
| 1801 |
+
raise
|
| 1802 |
+
return found
|
| 1803 |
+
|
| 1804 |
+
|
| 1805 |
+
def _cross_partition_refusal(session, route_id, application):
|
| 1806 |
+
"""R11: the 409 a no-App write door answers for a Route listed from another partition."""
|
| 1807 |
+
found = next((mark for route, mark in _cross_partition_routes(session, None, application)
|
| 1808 |
+
if route.get("id") == str(route_id)), None)
|
| 1809 |
+
if found is None:
|
| 1810 |
+
return None
|
| 1811 |
+
message = ("This route is shared with you. Only its owner can move it." if found["shared"]
|
| 1812 |
+
else "This route belongs to another Routing App. Only its owner can move it.")
|
| 1813 |
+
return err(409, "route_read_only_here", message)
|
| 1814 |
+
|
| 1815 |
+
|
| 1816 |
def _route_mutation_allowed(session, route, requested_source=None):
|
| 1817 |
"""W47-T08 / RULING R6 — OWNERSHIP SHORT-CIRCUITS, and that is the whole of instruction 22.
|
| 1818 |
|
|
|
|
| 1955 |
50,000 visible records sends exactly the same request as an empty grid; a future annotation
|
| 1956 |
door may accept a compact saved-View/filter reference without reopening that row-sized wire.
|
| 1957 |
"""
|
| 1958 |
+
requested, application = application, _entry_route_application(session, application)
|
| 1959 |
allowed = allowed_pids(session)
|
| 1960 |
records = _route_share_records(session)
|
| 1961 |
navigation = _route_navigation(session, _visible_routes(session, records, application), application)
|
| 1962 |
+
listed = [_route_dto(route, allowed, (), session, _route_share_mark(session, route, records))
|
| 1963 |
+
for route in navigation["routes"]]
|
| 1964 |
+
if requested is None:
|
| 1965 |
+
# R11: a no-App reader also sees what was shared to them from another partition.
|
| 1966 |
+
rank = len(listed)
|
| 1967 |
+
for index, (route, mark) in enumerate(_cross_partition_routes(session, records, application)):
|
| 1968 |
+
route.update(folderId=None, navOrder=rank + index)
|
| 1969 |
+
listed.append(_route_dto(route, allowed, (), session, mark, writable=False))
|
| 1970 |
+
return {"routes": listed, "folders": navigation["folders"]}
|
| 1971 |
|
| 1972 |
|
| 1973 |
@router.get("/customers/routes/delivery-recipients")
|
|
|
|
| 2142 |
def move_route_to_folder(route_id: str, body: dict = Body(default=None),
|
| 2143 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2144 |
from core import route_system
|
| 2145 |
+
requested, application = application, _entry_route_application(session, application, write=True)
|
| 2146 |
_route_application_body(body or {}, application)
|
| 2147 |
routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
|
| 2148 |
if str(route_id) not in {str(route.get("id")) for route in routes}:
|
| 2149 |
+
refusal = _cross_partition_refusal(session, route_id, application) if requested is None else None
|
| 2150 |
+
if refusal is not None:
|
| 2151 |
+
raise refusal
|
| 2152 |
raise err(404, "route_not_found", "that Route does not exist")
|
| 2153 |
folder_id = (body or {}).get("folderId")
|
| 2154 |
if folder_id is not None and not isinstance(folder_id, str):
|
|
|
|
| 2168 |
def reorder_route_navigation(body: dict = Body(default=None),
|
| 2169 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2170 |
from core import route_system
|
| 2171 |
+
requested, application = application, _entry_route_application(session, application, write=True)
|
| 2172 |
_route_application_body(body or {}, application)
|
| 2173 |
routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
|
| 2174 |
order = (body or {}).get("order")
|
| 2175 |
if not isinstance(order, list):
|
| 2176 |
raise err(400, "invalid_route_order", "order must be a list of visible Route ids")
|
| 2177 |
+
if requested is None:
|
| 2178 |
+
# R11: the no-App rail lists other partitions' Routes read-only. They are not ranked here.
|
| 2179 |
+
foreign = {str(route.get("id")) for route, _mark in _cross_partition_routes(session, None, application)}
|
| 2180 |
+
order = [value for value in order if str(value) not in foreign]
|
| 2181 |
result = route_system.reorder_routes(
|
| 2182 |
session.runtime, session.uname, order, routes,
|
| 2183 |
legacy_folders=legacy_folders, legacy_placements=legacy_placements, application=application)
|
|
|
|
| 2785 |
def normalized_route_delivery_list(route_id: str,
|
| 2786 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2787 |
"""Deliveries for the one normalized Route whose detail is open."""
|
| 2788 |
+
requested, application = application, _entry_route_application(session, application)
|
| 2789 |
+
_route_for_read_or_404(session, route_id, application, requested)
|
| 2790 |
return {"deliveries": _route_delivery_definitions(session, route_id, application)}
|
| 2791 |
|
| 2792 |
|
api/routes_external.py
CHANGED
|
@@ -198,6 +198,10 @@ class _PreviewRuntime:
|
|
| 198 |
return self.get(name)
|
| 199 |
|
| 200 |
def get_projection(self, name, drop=()):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
value = self.get(name)
|
| 202 |
if isinstance(value, dict):
|
| 203 |
for key in drop or ():
|
|
@@ -220,6 +224,12 @@ class _PreviewRuntime:
|
|
| 220 |
return sorted(key for key in keys if key.startswith(prefix))
|
| 221 |
|
| 222 |
def get_entry(self, name, entry):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
return copy.deepcopy((self.get(name) or {}).get(entry))
|
| 224 |
|
| 225 |
def get_projections(self, names, drop=(), timeout_s=None):
|
|
@@ -487,6 +497,8 @@ def _database_preview_documents(session, before, after):
|
|
| 487 |
pair[side] = None
|
| 488 |
continue
|
| 489 |
rt = _PreviewRuntime(session.runtime)
|
|
|
|
|
|
|
| 490 |
rt.put(user_tables.STORE_KEY, bucket)
|
| 491 |
reader = Session(session.tenant, session.user, session.claims, rt)
|
| 492 |
try:
|
|
@@ -738,14 +750,33 @@ def _query_publication(session: Session, database: str):
|
|
| 738 |
source = {"meta ads": "meta", "amazon sp-api": "amazon"}.get(source, source)
|
| 739 |
source = connector_spec.CONNECTOR_SPEC_KEYS.get(source, source)
|
| 740 |
if not source or source in {"blank", "loopable"}:
|
| 741 |
-
raise
|
| 742 |
identity = provider(session.tenant, source)
|
| 743 |
if not isinstance(identity, str) or not identity.strip():
|
| 744 |
-
raise
|
| 745 |
return {"source": source, "snapshot_id": identity}
|
|
|
|
|
|
|
| 746 |
except Exception as exc:
|
| 747 |
-
raise
|
| 748 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 749 |
|
| 750 |
|
| 751 |
def _assert_query_publication(previous, current):
|
|
@@ -972,6 +1003,85 @@ def _assert_database_scope(session, database):
|
|
| 972 |
{"database": named})
|
| 973 |
|
| 974 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 975 |
def _external_read_scope(session: Session, database, *args, **kwargs):
|
| 976 |
"""Attach external refusal details at the shared Assistant read boundary."""
|
| 977 |
try:
|
|
@@ -981,7 +1091,7 @@ def _external_read_scope(session: Session, database, *args, **kwargs):
|
|
| 981 |
if database == "ut_activity":
|
| 982 |
scoped = _native_activity_scope(session, *args, **kwargs)
|
| 983 |
else:
|
| 984 |
-
scoped =
|
| 985 |
if pin is not None and pin.source == database:
|
| 986 |
mode = kwargs.get("mode")
|
| 987 |
if scoped.get("source_kind") == "mirror_user_table":
|
|
@@ -1146,7 +1256,7 @@ def _native_activity_scope(session, fields=None, filters=None, *, mode=None):
|
|
| 1146 |
try:
|
| 1147 |
activity_log._backend(session.runtime)
|
| 1148 |
except (TypeError, ValueError) as exc:
|
| 1149 |
-
raise
|
| 1150 |
grid = routes_tables._compute_ut_assembly(session, "ut_activity", with_rows=False, fields_only=True)
|
| 1151 |
visible = list(grid["fields"])
|
| 1152 |
keys = {field["key"] for field in visible}
|
|
@@ -2569,9 +2679,10 @@ def query_as(rt, tenant, username, database, payload, scope=None) -> dict:
|
|
| 2569 |
if not message.endswith((".", "!", "?")):
|
| 2570 |
message += "."
|
| 2571 |
return {"ok": False, "error": {"code": code, "message": message}}
|
| 2572 |
-
except Exception:
|
| 2573 |
-
|
| 2574 |
-
|
|
|
|
| 2575 |
|
| 2576 |
|
| 2577 |
def _query_database(session: Session, database: str, payload: dict):
|
|
@@ -2590,10 +2701,11 @@ def _query_database(session: Session, database: str, payload: dict):
|
|
| 2590 |
return result
|
| 2591 |
except HTTPException as exc:
|
| 2592 |
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 2593 |
-
|
| 2594 |
-
|
| 2595 |
-
|
| 2596 |
-
|
|
|
|
| 2597 |
if info.get("code") == "assistant_bad_filter":
|
| 2598 |
# S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules.
|
| 2599 |
info["message"] = bad_filter_message(info.get("message"))
|
|
@@ -2607,6 +2719,8 @@ def _query_database(session: Session, database: str, payload: dict):
|
|
| 2607 |
info["retry_query"] = {"database": database, **copy.deepcopy(payload)}
|
| 2608 |
info["retry_query"].pop("cursor", None)
|
| 2609 |
raise
|
|
|
|
|
|
|
| 2610 |
|
| 2611 |
|
| 2612 |
def _query_activity_page(session, payload, scoped, selected_keys, sorts, aggregates, groups, limit, cursor):
|
|
@@ -2741,8 +2855,7 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 2741 |
if identity is not None and (
|
| 2742 |
scoped.get("source_kind") != "runtime_cache"
|
| 2743 |
or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters):
|
| 2744 |
-
raise err(409, "assistant_snapshot_unavailable",
|
| 2745 |
-
"Selected record reads are unavailable in this deployment. Retry after an update.")
|
| 2746 |
mirror = scoped.get("source_kind") == "mirror_user_table"
|
| 2747 |
visible_fields = list(scoped.get("fields") or [])
|
| 2748 |
visible_keys = {
|
|
@@ -3575,10 +3688,232 @@ def _database_record_write_guard(session, database, *, field_keys=None):
|
|
| 3575 |
from core import user_tables
|
| 3576 |
|
| 3577 |
guard = routes_tables._records_or_refuse
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3578 |
if field_keys is not None and not user_tables.records_mutable(database, st=session.runtime):
|
| 3579 |
# The shared wall admits a locked database's update only when every key is an authored field.
|
| 3580 |
-
return guard(session, database, cell_updates=dict.fromkeys(field_keys))
|
| 3581 |
-
return guard(session, database)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3582 |
|
| 3583 |
|
| 3584 |
def _plan_database_write(session, args):
|
|
@@ -3595,19 +3930,18 @@ def _plan_database_write(session, args):
|
|
| 3595 |
if action == "rename":
|
| 3596 |
return _plan_database_rename(session, args)
|
| 3597 |
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 3598 |
-
|
| 3599 |
-
|
| 3600 |
-
|
| 3601 |
-
table = copy.deepcopy(bucket_before.get(database) or {})
|
| 3602 |
if not table or not user_tables.is_user_table(database, st=session.runtime):
|
| 3603 |
raise err(403, "connected_source_read_only",
|
| 3604 |
"Records from connected sources are read only.")
|
| 3605 |
-
revision = _bucket_revision(session.runtime, user_tables.STORE_KEY,
|
|
|
|
| 3606 |
if type(revision.get("rev")) is not int:
|
| 3607 |
raise err(503, "confirmation_revision_unavailable",
|
| 3608 |
"The database version could not be checked. Nothing was saved.")
|
| 3609 |
warnings = []
|
| 3610 |
-
bucket_after = copy.deepcopy(bucket_before)
|
| 3611 |
|
| 3612 |
if action.startswith("record_"):
|
| 3613 |
field_keys = None
|
|
@@ -3629,6 +3963,11 @@ def _plan_database_write(session, args):
|
|
| 3629 |
record_id = str(args.get("record_id") or "").strip()
|
| 3630 |
if not record_id.isdigit() or int(record_id) not in g["pids"]:
|
| 3631 |
raise err(403, "record_unavailable", "That record is not available to this account.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3632 |
before = copy.deepcopy(rows.get(record_id))
|
| 3633 |
if before is None:
|
| 3634 |
raise err(403, "record_unavailable", "That record is not available to this account.")
|
|
@@ -3645,7 +3984,8 @@ def _plan_database_write(session, args):
|
|
| 3645 |
after[field["key"]] = ""
|
| 3646 |
else:
|
| 3647 |
after = None
|
| 3648 |
-
|
|
|
|
| 3649 |
if action == "record_delete":
|
| 3650 |
(target.get("rows") or {}).pop(record_id, None)
|
| 3651 |
else:
|
|
@@ -3659,13 +3999,12 @@ def _plan_database_write(session, args):
|
|
| 3659 |
# This is part of the UI row writer's durable result, not an optional callback.
|
| 3660 |
# Include newly encountered select values in the preview and the one guarded commit.
|
| 3661 |
user_tables._hydrate_choice_options(target.get("fields") or [], [after])
|
| 3662 |
-
changed, truncated = _diff_documents(_database_preview_documents(session,
|
| 3663 |
if truncated:
|
| 3664 |
warnings.append("The preview diff reached its display cap; split this change.")
|
| 3665 |
return {"diff": changed, "affected": {"ids": [record_id], "count": 1},
|
| 3666 |
"warnings": warnings, "revision": revision,
|
| 3667 |
-
|
| 3668 |
-
"bucket_after": bucket_after,
|
| 3669 |
"planned": {"database": database, "action": action, "record_id": record_id}}
|
| 3670 |
|
| 3671 |
field_key = str(args.get("field_key") or "").strip()
|
|
@@ -3719,8 +4058,11 @@ def _plan_database_write(session, args):
|
|
| 3719 |
# Build the exact schema document through the core schema mechanics on an isolated runtime.
|
| 3720 |
# This includes reciprocal-link cleanup/creation while keeping preview free of relation
|
| 3721 |
# refreshes, audit calls, automation events and every other out-of-store side effect.
|
|
|
|
|
|
|
|
|
|
| 3722 |
preview = _PreviewRuntime(session.runtime)
|
| 3723 |
-
|
| 3724 |
if action == "field_create":
|
| 3725 |
stored = user_tables.add_field(
|
| 3726 |
database, after, st=preview, recurrence_validator=recurrence_validator)
|
|
@@ -3739,8 +4081,8 @@ def _plan_database_write(session, args):
|
|
| 3739 |
warnings.append(str(synced.get("reason") or "A reciprocal field was not created."))
|
| 3740 |
elif not user_tables.delete_field(database, field_key, st=preview):
|
| 3741 |
raise err(403, "field_refused", "That field cannot be removed.")
|
| 3742 |
-
|
| 3743 |
-
changed, truncated = _diff_documents(_database_preview_documents(session,
|
| 3744 |
if truncated:
|
| 3745 |
warnings.append("The preview diff reached its display cap; split this change.")
|
| 3746 |
affected_ids = [field_key]
|
|
@@ -3752,8 +4094,7 @@ def _plan_database_write(session, args):
|
|
| 3752 |
return {"diff": changed,
|
| 3753 |
"affected": {"ids": affected_ids, "count": len(affected_ids)},
|
| 3754 |
"warnings": warnings, "revision": revision,
|
| 3755 |
-
|
| 3756 |
-
"bucket_after": bucket_after,
|
| 3757 |
"planned": {"database": database, "action": action, "field_key": field_key}}
|
| 3758 |
|
| 3759 |
|
|
@@ -3782,28 +4123,47 @@ def _execute_database_object(session, args, *, plan=None):
|
|
| 3782 |
"The database metadata changed after the preview, " + _STALE_FIX) from exc
|
| 3783 |
return {"database": database, "action": action, "meta": result["meta"],
|
| 3784 |
"affected": copy.deepcopy(plan.get("affected") or {})}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3785 |
expected_rev = plan.get("revision", {}).get("rev")
|
| 3786 |
expected_digest = plan.get("bucket_before_digest")
|
| 3787 |
replacement = copy.deepcopy(plan.get("bucket_after"))
|
| 3788 |
-
if type(expected_rev) is not int or not isinstance(replacement, dict):
|
| 3789 |
raise err(503, "confirmation_revision_unavailable",
|
| 3790 |
"The database version could not be checked. Nothing was saved.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3791 |
|
| 3792 |
def apply_exact(current):
|
| 3793 |
-
|
|
|
|
| 3794 |
raise view_audit.ViewRevisionStale(
|
| 3795 |
rev_sent=expected_rev, rev_current=expected_rev + 1,
|
| 3796 |
message="The database changed after preview.")
|
| 3797 |
-
|
|
|
|
|
|
|
| 3798 |
|
| 3799 |
try:
|
| 3800 |
-
# One
|
| 3801 |
# inside the PG lock where available; the closure's digest check also protects a backend
|
| 3802 |
# that can only provide the runtime's documented guarded fallback.
|
| 3803 |
user_tables.update_with_activity(
|
| 3804 |
database, apply_exact, st=session.runtime, by=session.uname,
|
| 3805 |
-
via=_trusted_activity_via(session), row_ids=
|
| 3806 |
-
flush="sync", expect_rev=expected_rev)
|
| 3807 |
except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
|
| 3808 |
raise err(409, "confirmation_stale",
|
| 3809 |
"The database changed after the preview, " + _STALE_FIX) from exc
|
|
@@ -4204,7 +4564,15 @@ def _tool_context(session: Session, *, preview=False) -> external_tools.ToolCont
|
|
| 4204 |
)
|
| 4205 |
|
| 4206 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4207 |
def _strong_confirmation_required(name, args):
|
|
|
|
|
|
|
|
|
|
| 4208 |
if name not in external_tools.TOOLS:
|
| 4209 |
import tool_groups
|
| 4210 |
resolved = tool_groups.resolve(name, args)
|
|
@@ -4429,7 +4797,7 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
|
|
| 4429 |
"Part of this change could not be accepted. Nothing was previewed.")
|
| 4430 |
|
| 4431 |
remote_note = _is_website_note(name, normalised)
|
| 4432 |
-
plan = result if name == "database_object" or remote_note else None
|
| 4433 |
if plan is not None:
|
| 4434 |
diff = copy.deepcopy(plan.get("diff") or [])
|
| 4435 |
warnings = list(plan.get("warnings") or [])
|
|
@@ -4462,6 +4830,9 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
|
|
| 4462 |
normalised["arguments"]["expect"] = [
|
| 4463 |
{"user": row["user"], "role": row["role"]}
|
| 4464 |
for row in result["current_entries"]]
|
|
|
|
|
|
|
|
|
|
| 4465 |
|
| 4466 |
_canonical_json(diff, capped=False)
|
| 4467 |
diff_digest = _json_digest(diff)
|
|
@@ -4520,6 +4891,8 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
|
|
| 4520 |
response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
|
| 4521 |
if name == "apps" and normalised.get("action") == "catalog_build":
|
| 4522 |
response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
|
|
|
|
|
|
|
| 4523 |
if payload["strong"]:
|
| 4524 |
response["confirmation_phrase"] = _CONFIRMATION_PHRASE
|
| 4525 |
if issue_token:
|
|
@@ -5055,6 +5428,9 @@ def _confirm_change(session, args):
|
|
| 5055 |
if tool == "database_object":
|
| 5056 |
result = _execute_database_object(
|
| 5057 |
fresh_session, arguments, plan=recomputed.get("_plan"))
|
|
|
|
|
|
|
|
|
|
| 5058 |
else:
|
| 5059 |
from datetime import datetime, timezone
|
| 5060 |
commit_started = datetime.now(timezone.utc)
|
|
@@ -5085,11 +5461,13 @@ def _confirm_change(session, args):
|
|
| 5085 |
from core import user_tables
|
| 5086 |
import routes_nav
|
| 5087 |
bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY
|
| 5088 |
-
|
| 5089 |
-
|
| 5090 |
produced_rev = before_revision["rev"] + 1
|
| 5091 |
if after_revision.get("rev") != produced_rev:
|
| 5092 |
after_revision = {"rev": produced_rev}
|
|
|
|
|
|
|
| 5093 |
else:
|
| 5094 |
after_revision = copy.deepcopy(guarded_session.runtime._expected)
|
| 5095 |
after_revision_digest = _json_digest(after_revision)
|
|
@@ -5119,10 +5497,14 @@ def _confirm_change(session, args):
|
|
| 5119 |
response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
|
| 5120 |
if tool == "apps" and arguments.get("action") == "catalog_build":
|
| 5121 |
response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
|
|
|
|
|
|
|
|
|
|
| 5122 |
_confirmation_status(fresh_session, operation_id, "committed", response)
|
| 5123 |
if isinstance(stored.get("arguments_ref"), dict):
|
| 5124 |
_drop_operation_documents(fresh_session, [stored["arguments_ref"].get("key")])
|
| 5125 |
-
if tool != "database_object" and
|
|
|
|
| 5126 |
try:
|
| 5127 |
targets, native_ids = _committed_build_evidence(fresh_session, guarded_session.runtime, arguments,
|
| 5128 |
build_kind=payload["buildKind"])
|
|
@@ -5254,7 +5636,7 @@ MCP_RESULT_CODES.update({
|
|
| 5254 |
"query_filter_limit": "Filters allow 3 levels, 100 nodes and 50 conditions per group to bound query validation work. To reduce that work, " + _MCP_FIXES["query_filter_limit"] + ".",
|
| 5255 |
"query_cursor_stale": "The query, data or permissions changed. To restart, " + _MCP_FIXES["query_cursor_stale"] + ".",
|
| 5256 |
"query_cursor_invalid": "The query cursor is invalid. Repeat the original query without cursor for a fresh first page.",
|
| 5257 |
-
"query_source_unavailable": "This database
|
| 5258 |
"test_connection_production": "Test connections cannot write to production. Use a staging environment.",
|
| 5259 |
"test_connection_environment_unknown": "This environment is unknown. Test connections need a named non-production environment.",
|
| 5260 |
"external_test_connection_invalid": "Set test_connection to true or false.",
|
|
@@ -5288,7 +5670,8 @@ _MCP_INTERNAL_PROSE = re.compile(
|
|
| 5288 |
r"Traceback|stack trace|\bengine\b|\bmanifest\b|\brevision-guard\b|\bgate\b|"
|
| 5289 |
r"\b(?:tenant store|dispatcher|document cleaner)\b|"
|
| 5290 |
r"(?:[A-Za-z]:[\\/]|/(?:tmp|root|work|home|srv|app)/)|"
|
| 5291 |
-
|
|
|
|
| 5292 |
r"\.(?:claude|agents|codex)/|~[/\\]", re.I)
|
| 5293 |
_MCP_PROSE_REPLACEMENTS = (
|
| 5294 |
(r"strict Assistant scope wall|scope wall|live UI walls|UI walls", "permissions"),
|
|
@@ -5893,6 +6276,8 @@ def _mcp_capabilities(session, groups, rows):
|
|
| 5893 |
required.append("a named non-production environment for this test connection")
|
| 5894 |
if group == "websites":
|
| 5895 |
required.extend(_website_requirements(session, note=action == "add_internal_note"))
|
|
|
|
|
|
|
| 5896 |
if legacy in {"describe_database", "query_database", "view_upsert", "view_delete", "view_share_preview", "view_object", "app_upsert", "app_delete"} or (
|
| 5897 |
legacy == "database_object" and entry.get("legacy_action") != "list") or legacy == "sheet_set_binding":
|
| 5898 |
if not rows:
|
|
@@ -5993,10 +6378,37 @@ def _guide_unread_count(session: Session) -> int:
|
|
| 5993 |
return count if type(count) is int and count >= 0 else 0
|
| 5994 |
|
| 5995 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5996 |
def _mcp_guide(session: Session, group=None) -> str:
|
| 5997 |
"""Return the group index or one group's action schemas without reading records."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5998 |
import tool_groups
|
| 5999 |
groups = tool_groups.groups()
|
|
|
|
| 6000 |
if isinstance(group, str) and group not in groups and group != "data":
|
| 6001 |
# A cased, plural or dotted name (Sheets, sheet.write_cells) reads the one group it names.
|
| 6002 |
key = group.strip().lower().partition(".")[0]
|
|
@@ -6004,9 +6416,10 @@ def _mcp_guide(session: Session, group=None) -> str:
|
|
| 6004 |
if name in groups or name == "data"), group)
|
| 6005 |
alias = group == "data"
|
| 6006 |
group = tool_groups.canonical_name(group)
|
|
|
|
| 6007 |
if group is not None and group not in groups:
|
| 6008 |
-
|
| 6009 |
-
|
| 6010 |
# One rows-free definitions read per guide: a live runtime pays one pool checkout per database.
|
| 6011 |
# The guide reads schemas only, so no reader here needs `rows`. On a failed read, keep the old path.
|
| 6012 |
try:
|
|
@@ -6024,7 +6437,7 @@ def _mcp_guide(session: Session, group=None) -> str:
|
|
| 6024 |
rows = [row for row in rows if row["permitted"]]
|
| 6025 |
capabilities = _mcp_capabilities(session, groups, [row for row in rows if row.get("schema")])
|
| 6026 |
unread = _guide_unread_count(session)
|
| 6027 |
-
lines = ["# Loopable guide", "", f"Signed in as {session.uname}.", "", MCP_INSTRUCTIONS, "",
|
| 6028 |
f"Scheduled results you have not read: {unread}.",
|
| 6029 |
*(["Read schedules.inbox before answering."] if unread > 0 else []), "",
|
| 6030 |
"Call a group with action and arguments. Each action keeps its own read or write rules.",
|
|
@@ -6176,7 +6589,7 @@ def _dispatch_external_tool(ctx, name, arguments, *, about=None):
|
|
| 6176 |
import tool_groups
|
| 6177 |
resolved = tool_groups.resolve(name, arguments)
|
| 6178 |
if resolved is None:
|
| 6179 |
-
raise sheets.SheetRefusal("tool_not_found",
|
| 6180 |
entry, args = resolved
|
| 6181 |
if call is not None and call["tool"] == name:
|
| 6182 |
external_activity.note_action(arguments["action"])
|
|
@@ -6191,6 +6604,30 @@ def _dispatch_external_tool(ctx, name, arguments, *, about=None):
|
|
| 6191 |
return result
|
| 6192 |
|
| 6193 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6194 |
def _record_mcp_method(session, req, reply):
|
| 6195 |
"""Retain bounded method names for admitted protocol requests without request values."""
|
| 6196 |
try:
|
|
@@ -6249,8 +6686,9 @@ def _handle_mcp(req, session: Session, base_url: str = ""):
|
|
| 6249 |
code = error.get("code", "tool_refused") if error else ("tool_refused" if reply.get("error") else "")
|
| 6250 |
if code and not call["limitation_recorded"]:
|
| 6251 |
from routes_feedback import record_limitation
|
| 6252 |
-
|
| 6253 |
-
|
|
|
|
| 6254 |
except Exception:
|
| 6255 |
pass
|
| 6256 |
finally:
|
|
@@ -6463,6 +6901,9 @@ def _mcp_next_call(code, message):
|
|
| 6463 |
if code == "assistant_unknown_database":
|
| 6464 |
# The read path's lowercase reason is replaced: the fix sentence names the missing key itself.
|
| 6465 |
return fix
|
|
|
|
|
|
|
|
|
|
| 6466 |
return text if not fix or fix in text else (text + " " + fix).strip()
|
| 6467 |
|
| 6468 |
|
|
@@ -6680,9 +7121,11 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 6680 |
out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry)
|
| 6681 |
except (TypeError, ValueError, KeyError, AttributeError):
|
| 6682 |
out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
|
| 6683 |
-
except Exception:
|
|
|
|
| 6684 |
out = _tool_refusal(name, "tool_unavailable",
|
| 6685 |
"The tool could not complete this operation, so try the same call again shortly.", 503)
|
|
|
|
| 6686 |
return _mcp_content(rid, out, session)
|
| 6687 |
|
| 6688 |
|
|
@@ -6736,22 +7179,42 @@ _WRITE_TOOLS = frozenset({
|
|
| 6736 |
"database_object", "view_object", "workbook_object", "worksheet_object",
|
| 6737 |
"range_object", "calculation_object", "validated_batch",
|
| 6738 |
"confirm_change",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6739 |
})
|
| 6740 |
|
| 6741 |
|
| 6742 |
def _assert_write_allow_list(*, check_groups=False) -> None:
|
| 6743 |
declared = {name for name, tool in external_tools.TOOLS.items() if tool.get("write")}
|
| 6744 |
-
|
| 6745 |
-
|
|
|
|
| 6746 |
if undeclared or stale:
|
| 6747 |
raise RuntimeError("External write allow-list mismatch: undeclared="
|
| 6748 |
+ repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
|
| 6749 |
if check_groups:
|
| 6750 |
import tool_groups
|
| 6751 |
-
|
|
|
|
| 6752 |
for entry in group["actions"].values()}
|
| 6753 |
if set(external_tools.TOOLS) - mapped - {"confirm_change"}:
|
| 6754 |
raise RuntimeError("Every legacy tool must have a grouped action")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6755 |
|
| 6756 |
|
| 6757 |
_assert_write_allow_list()
|
|
|
|
| 198 |
return self.get(name)
|
| 199 |
|
| 200 |
def get_projection(self, name, drop=()):
|
| 201 |
+
project = getattr(self._real, "get_projection", None)
|
| 202 |
+
if str(name) not in self._docs and callable(project):
|
| 203 |
+
# D-11174: an unstaged document is projected by its source, never copied whole here.
|
| 204 |
+
return project(name, drop=drop)
|
| 205 |
value = self.get(name)
|
| 206 |
if isinstance(value, dict):
|
| 207 |
for key in drop or ():
|
|
|
|
| 224 |
return sorted(key for key in keys if key.startswith(prefix))
|
| 225 |
|
| 226 |
def get_entry(self, name, entry):
|
| 227 |
+
if str(name) not in self._docs:
|
| 228 |
+
# D-11174: an unstaged document answers one entry from its source's entry door.
|
| 229 |
+
from core import user_tables
|
| 230 |
+
reader = user_tables._entry_reader(self._real)
|
| 231 |
+
if reader is not None:
|
| 232 |
+
return copy.deepcopy(reader(name, entry))
|
| 233 |
return copy.deepcopy((self.get(name) or {}).get(entry))
|
| 234 |
|
| 235 |
def get_projections(self, names, drop=(), timeout_s=None):
|
|
|
|
| 497 |
pair[side] = None
|
| 498 |
continue
|
| 499 |
rt = _PreviewRuntime(session.runtime)
|
| 500 |
+
# Stage the planned bucket without reading the source's whole document first.
|
| 501 |
+
_seed_preview(rt, user_tables.STORE_KEY, {})
|
| 502 |
rt.put(user_tables.STORE_KEY, bucket)
|
| 503 |
reader = Session(session.tenant, session.user, session.claims, rt)
|
| 504 |
try:
|
|
|
|
| 750 |
source = {"meta ads": "meta", "amazon sp-api": "amazon"}.get(source, source)
|
| 751 |
source = connector_spec.CONNECTOR_SPEC_KEYS.get(source, source)
|
| 752 |
if not source or source in {"blank", "loopable"}:
|
| 753 |
+
raise _query_source_unavailable("no_connector")
|
| 754 |
identity = provider(session.tenant, source)
|
| 755 |
if not isinstance(identity, str) or not identity.strip():
|
| 756 |
+
raise _query_source_unavailable("invalid_publication")
|
| 757 |
return {"source": source, "snapshot_id": identity}
|
| 758 |
+
except HTTPException:
|
| 759 |
+
raise
|
| 760 |
except Exception as exc:
|
| 761 |
+
raise _query_failure(exc, "query publication") from exc
|
| 762 |
+
|
| 763 |
+
|
| 764 |
+
def _query_source_unavailable(cause, *, message=None):
|
| 765 |
+
"""S82-T125: a 503 `query_source_unavailable` naming one cause from `deps`' closed set."""
|
| 766 |
+
return err(503, "query_source_unavailable", deps._assistant_source_message(cause, "query", message=message))
|
| 767 |
+
|
| 768 |
+
|
| 769 |
+
def _query_failure(exc, where):
|
| 770 |
+
"""Name transport failures and log unexpected exceptions as unavailable tools."""
|
| 771 |
+
import logging
|
| 772 |
+
logging.getLogger(__name__).warning("%s failed (%s)", where, type(exc).__name__)
|
| 773 |
+
transport = isinstance(exc, (ConnectionError, TimeoutError, OSError)) or any(
|
| 774 |
+
cls.__module__.split(".")[0] in {"duckdb", "_duckdb", "psycopg", "psycopg2", "psycopg_pool"}
|
| 775 |
+
and cls.__name__ in {"IOException", "ConnectionException", "OperationalError", "InterfaceError", "PoolTimeout"}
|
| 776 |
+
for cls in type(exc).__mro__)
|
| 777 |
+
if transport:
|
| 778 |
+
return _query_source_unavailable("store_silent")
|
| 779 |
+
return err(503, "tool_unavailable", "This query could not be completed because of an error on the server.")
|
| 780 |
|
| 781 |
|
| 782 |
def _assert_query_publication(previous, current):
|
|
|
|
| 1003 |
{"database": named})
|
| 1004 |
|
| 1005 |
|
| 1006 |
+
def _external_shared_cells(session, database, rows):
|
| 1007 |
+
"""Read overlay values for exactly the records returned by the permissioned source reader."""
|
| 1008 |
+
from core import shared_overlay
|
| 1009 |
+
return shared_overlay.cells(database, [row["pid"] for row in rows], st=session.runtime)
|
| 1010 |
+
|
| 1011 |
+
|
| 1012 |
+
def _external_shared_read_scope(session, database, fields=None, filters=None, *, mode=None, **kwargs):
|
| 1013 |
+
"""Project shared user cells over permitted connector pages with the existing field walls."""
|
| 1014 |
+
from core import shared_overlay
|
| 1015 |
+
|
| 1016 |
+
shared = {}
|
| 1017 |
+
if str(database).startswith("ut_") and database != "ut_activity":
|
| 1018 |
+
definition, _ = deps._assistant_table_defn(session, database)
|
| 1019 |
+
declared = {field.get("key") for field in definition.get("fields") or ()}
|
| 1020 |
+
shared = {key: dict(field, key=key, source="overlay", shared=True)
|
| 1021 |
+
for key, field in shared_overlay.fields(database, st=session.runtime).items()
|
| 1022 |
+
if key not in declared and isinstance(field, dict)}
|
| 1023 |
+
if not shared:
|
| 1024 |
+
return deps.assistant_read_scope(session, database, fields=fields, filters=filters,
|
| 1025 |
+
mode=mode, **kwargs)
|
| 1026 |
+
# The source reader still resolves every row permission before any overlay cell is fetched.
|
| 1027 |
+
schema = deps.assistant_read_scope(session, database, fields=None, mode="schema", **kwargs)
|
| 1028 |
+
contract = list(definition.get("fields") or ()) + list(shared.values())
|
| 1029 |
+
available = list(schema["fields"]) + list(shared.values())
|
| 1030 |
+
withheld = set()
|
| 1031 |
+
for user in [session.user, *perm_scope._read_users(kwargs.get("also_as", ()))]:
|
| 1032 |
+
hidden = perm_scope.hidden_keys(user, database, contract, st=session.runtime)
|
| 1033 |
+
visible = (perm_scope.assistant_visible_fields(available, user, database)
|
| 1034 |
+
if perm_scope.assistant_entry(user, database) is not None else available)
|
| 1035 |
+
allowed = {field["key"] for field in visible if field.get("key") not in hidden}
|
| 1036 |
+
withheld.update(field["key"] for field in contract if field.get("key") not in allowed)
|
| 1037 |
+
available = [field for field in available if field.get("key") in allowed]
|
| 1038 |
+
selected = deps._assistant_requested_fields(fields, available)
|
| 1039 |
+
try:
|
| 1040 |
+
perm_scope.validate_assistant_filter(filters, available)
|
| 1041 |
+
except ValueError as exc:
|
| 1042 |
+
deps._assistant_refuse_filter(exc, filters, available)
|
| 1043 |
+
|
| 1044 |
+
def refuse(key, code, verb):
|
| 1045 |
+
raise err(400, code, f'Field "{deps._assistant_display_key(key)}" is stored beside the connector rows, '
|
| 1046 |
+
f"so it can be read but not {verb}.")
|
| 1047 |
+
|
| 1048 |
+
for leaf in deps._assistant_filter_leaves(filters):
|
| 1049 |
+
rhs = leaf.get("rhs") if isinstance(leaf, dict) else None
|
| 1050 |
+
for key in (leaf.get("colId") if isinstance(leaf, dict) else None,
|
| 1051 |
+
rhs.get("colId") if isinstance(rhs, dict) else None):
|
| 1052 |
+
if key in shared:
|
| 1053 |
+
refuse(key, "assistant_bad_filter", "filtered")
|
| 1054 |
+
if isinstance(mode, tuple) and mode:
|
| 1055 |
+
if mode[0] == "page" and len(mode) > 3:
|
| 1056 |
+
for rule in mode[3] or ():
|
| 1057 |
+
if rule.get("field") in shared:
|
| 1058 |
+
refuse(rule["field"], "query_sorts_invalid", "sorted")
|
| 1059 |
+
if mode[0] == "aggregate":
|
| 1060 |
+
for spec in mode[1] or ():
|
| 1061 |
+
if spec.get("field") in shared:
|
| 1062 |
+
refuse(spec["field"], "query_aggregates_invalid", "aggregated")
|
| 1063 |
+
for group in (mode[2].get("groupBy") or ()) if len(mode) > 2 else ():
|
| 1064 |
+
if group.get("field") in shared:
|
| 1065 |
+
refuse(group["field"], "query_groups_invalid", "grouped")
|
| 1066 |
+
source_fields = None if fields is None else [key for key in selected if key not in shared]
|
| 1067 |
+
scoped = deps.assistant_read_scope(session, database, fields=source_fields, filters=filters,
|
| 1068 |
+
mode=mode, **kwargs)
|
| 1069 |
+
# Query schemas may omit declared fields that have no mirror expression.
|
| 1070 |
+
source_keys = {field["key"] for field in scoped["fields"]}
|
| 1071 |
+
available = [field for field in available if field["key"] in shared or field["key"] in source_keys]
|
| 1072 |
+
selected = deps._assistant_requested_fields(fields, available)
|
| 1073 |
+
overlay_keys = set(selected) & shared.keys()
|
| 1074 |
+
rows = scoped.get("records") or []
|
| 1075 |
+
cells = _external_shared_cells(session, database, rows) if overlay_keys else {}
|
| 1076 |
+
scoped["records"] = [{"pid": row["pid"], **{
|
| 1077 |
+
key: copy.deepcopy((cells.get(str(row["pid"])) or {}).get(key) if key in overlay_keys else row.get(key))
|
| 1078 |
+
for key in selected}} for row in rows]
|
| 1079 |
+
from core import if_field
|
| 1080 |
+
scoped["fields"] = if_field.project_fields(
|
| 1081 |
+
[copy.deepcopy(field) for field in available if field["key"] in selected], withheld)
|
| 1082 |
+
return scoped
|
| 1083 |
+
|
| 1084 |
+
|
| 1085 |
def _external_read_scope(session: Session, database, *args, **kwargs):
|
| 1086 |
"""Attach external refusal details at the shared Assistant read boundary."""
|
| 1087 |
try:
|
|
|
|
| 1091 |
if database == "ut_activity":
|
| 1092 |
scoped = _native_activity_scope(session, *args, **kwargs)
|
| 1093 |
else:
|
| 1094 |
+
scoped = _external_shared_read_scope(session, database, *args, **kwargs)
|
| 1095 |
if pin is not None and pin.source == database:
|
| 1096 |
mode = kwargs.get("mode")
|
| 1097 |
if scoped.get("source_kind") == "mirror_user_table":
|
|
|
|
| 1256 |
try:
|
| 1257 |
activity_log._backend(session.runtime)
|
| 1258 |
except (TypeError, ValueError) as exc:
|
| 1259 |
+
raise _query_source_unavailable("no_activity_store") from exc
|
| 1260 |
grid = routes_tables._compute_ut_assembly(session, "ut_activity", with_rows=False, fields_only=True)
|
| 1261 |
visible = list(grid["fields"])
|
| 1262 |
keys = {field["key"] for field in visible}
|
|
|
|
| 2679 |
if not message.endswith((".", "!", "?")):
|
| 2680 |
message += "."
|
| 2681 |
return {"ok": False, "error": {"code": code, "message": message}}
|
| 2682 |
+
except Exception as exc:
|
| 2683 |
+
refused = _query_failure(exc, "saved query").detail["error"]
|
| 2684 |
+
message = re.split(r"(?<=[.!?])\s+", refused["message"], maxsplit=1)[0]
|
| 2685 |
+
return {"ok": False, "error": {"code": refused["code"], "message": message}}
|
| 2686 |
|
| 2687 |
|
| 2688 |
def _query_database(session: Session, database: str, payload: dict):
|
|
|
|
| 2701 |
return result
|
| 2702 |
except HTTPException as exc:
|
| 2703 |
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 2704 |
+
cause = {"assistant_store_not_ready": deps._assistant_source_cause_of(info.get("message")) or "syncing",
|
| 2705 |
+
"store_not_ready": "syncing",
|
| 2706 |
+
"amazon_mirror_not_loaded": deps._assistant_source_cause_of(info.get("message")) or "loading"}.get(info.get("code"))
|
| 2707 |
+
if exc.status_code == 503 and cause:
|
| 2708 |
+
raise _query_source_unavailable(cause, message=info.get("message")) from exc
|
| 2709 |
if info.get("code") == "assistant_bad_filter":
|
| 2710 |
# S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules.
|
| 2711 |
info["message"] = bad_filter_message(info.get("message"))
|
|
|
|
| 2719 |
info["retry_query"] = {"database": database, **copy.deepcopy(payload)}
|
| 2720 |
info["retry_query"].pop("cursor", None)
|
| 2721 |
raise
|
| 2722 |
+
except Exception as exc:
|
| 2723 |
+
raise _query_failure(exc, "database query") from exc
|
| 2724 |
|
| 2725 |
|
| 2726 |
def _query_activity_page(session, payload, scoped, selected_keys, sorts, aggregates, groups, limit, cursor):
|
|
|
|
| 2855 |
if identity is not None and (
|
| 2856 |
scoped.get("source_kind") != "runtime_cache"
|
| 2857 |
or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters):
|
| 2858 |
+
raise err(409, "assistant_snapshot_unavailable", deps._assistant_source_message("not_supported"))
|
|
|
|
| 2859 |
mirror = scoped.get("source_kind") == "mirror_user_table"
|
| 2860 |
visible_fields = list(scoped.get("fields") or [])
|
| 2861 |
visible_keys = {
|
|
|
|
| 3688 |
from core import user_tables
|
| 3689 |
|
| 3690 |
guard = routes_tables._records_or_refuse
|
| 3691 |
+
# D-11174: the wall reads this one definition, never a lend of the whole bucket.
|
| 3692 |
+
definition = user_tables.get_definition(database, st=session.runtime)
|
| 3693 |
+
lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: (
|
| 3694 |
+
{database: definition} if isinstance(definition, dict) else {})})
|
| 3695 |
if field_keys is not None and not user_tables.records_mutable(database, st=session.runtime):
|
| 3696 |
# The shared wall admits a locked database's update only when every key is an authored field.
|
| 3697 |
+
return guard(session, database, st=lent, cell_updates=dict.fromkeys(field_keys))
|
| 3698 |
+
return guard(session, database, st=lent)
|
| 3699 |
+
|
| 3700 |
+
|
| 3701 |
+
def _database_write_source(runtime):
|
| 3702 |
+
"""The runtime that owns the unstaged `user_tables` document beneath any preview layers."""
|
| 3703 |
+
from core import user_tables
|
| 3704 |
+
|
| 3705 |
+
source = runtime
|
| 3706 |
+
while isinstance(source, _PreviewRuntime) and user_tables.STORE_KEY not in source._docs:
|
| 3707 |
+
source = source._real
|
| 3708 |
+
return source
|
| 3709 |
+
|
| 3710 |
+
|
| 3711 |
+
def _database_write_tables(runtime, keys):
|
| 3712 |
+
"""D-11174: copy only the named tables of `user_tables`, never the whole bucket."""
|
| 3713 |
+
from core import user_tables
|
| 3714 |
+
|
| 3715 |
+
source = _database_write_source(runtime)
|
| 3716 |
+
reader = user_tables._entry_reader(source)
|
| 3717 |
+
if reader is None:
|
| 3718 |
+
# Only a store without an entry door reads the document; the plan still keeps the named tables.
|
| 3719 |
+
whole = source.get(user_tables.STORE_KEY)
|
| 3720 |
+
if not isinstance(whole, dict):
|
| 3721 |
+
raise err(503, "store_unavailable", "The database is unavailable. Nothing was saved.")
|
| 3722 |
+
reader = lambda _name, key: whole.get(key)
|
| 3723 |
+
tables = {}
|
| 3724 |
+
for key in dict.fromkeys(str(key) for key in keys if key):
|
| 3725 |
+
table = reader(user_tables.STORE_KEY, key)
|
| 3726 |
+
if isinstance(table, dict):
|
| 3727 |
+
tables[key] = copy.deepcopy(table)
|
| 3728 |
+
return tables
|
| 3729 |
+
|
| 3730 |
+
|
| 3731 |
+
def _seed_preview(preview, key, value, revision=None):
|
| 3732 |
+
"""Stage the known part of a document in a preview without reading the rest from its source."""
|
| 3733 |
+
preview._initial[key] = copy.deepcopy(value)
|
| 3734 |
+
preview._docs[key] = copy.deepcopy(value)
|
| 3735 |
+
preview._base_revisions[key] = copy.deepcopy(revision)
|
| 3736 |
+
preview._local_revisions[key] = 0
|
| 3737 |
+
|
| 3738 |
+
|
| 3739 |
+
def _database_row_maps():
|
| 3740 |
+
from core import user_table_rows
|
| 3741 |
+
return ("rows",) + tuple(user_table_rows.ROW_MAPS)
|
| 3742 |
+
|
| 3743 |
+
|
| 3744 |
+
def _database_table_delta(before, after):
|
| 3745 |
+
"""The definition keys and row-map entries that turn one table into the other, by name."""
|
| 3746 |
+
maps = _database_row_maps()
|
| 3747 |
+
before = before if isinstance(before, dict) else {}
|
| 3748 |
+
after = after if isinstance(after, dict) else {}
|
| 3749 |
+
definition, records = [], {}
|
| 3750 |
+
missing = object()
|
| 3751 |
+
for key in sorted(set(before) | set(after), key=str):
|
| 3752 |
+
old, new = before.get(key, missing), after.get(key, missing)
|
| 3753 |
+
if (key in maps and (old is missing or isinstance(old, dict))
|
| 3754 |
+
and (new is missing or isinstance(new, dict))):
|
| 3755 |
+
old = {} if old is missing else old
|
| 3756 |
+
new = {} if new is missing else new
|
| 3757 |
+
ids = sorted((rid for rid in set(old) | set(new)
|
| 3758 |
+
if old.get(rid, missing) != new.get(rid, missing)), key=str)
|
| 3759 |
+
if ids:
|
| 3760 |
+
records[str(key)] = [str(rid) for rid in ids]
|
| 3761 |
+
elif old != new:
|
| 3762 |
+
definition.append(str(key))
|
| 3763 |
+
return {"definition": definition, "records": records}
|
| 3764 |
+
|
| 3765 |
+
|
| 3766 |
+
def _database_table_guard(table, delta):
|
| 3767 |
+
"""One table's definition and the records a delta names: what a confirm must find unchanged."""
|
| 3768 |
+
if not isinstance(table, dict):
|
| 3769 |
+
return None
|
| 3770 |
+
maps = _database_row_maps()
|
| 3771 |
+
# A row map is read only at the ids the delta names, so a record-mode table stays per record.
|
| 3772 |
+
view = {"definition": {str(key): value for key, value in table.items()
|
| 3773 |
+
if not (key in maps and isinstance(value, dict))},
|
| 3774 |
+
"records": {}}
|
| 3775 |
+
for name, ids in delta["records"].items():
|
| 3776 |
+
rows = table.get(name)
|
| 3777 |
+
view["records"][name] = (None if not isinstance(rows, dict)
|
| 3778 |
+
else {rid: rows[rid] for rid in ids if rid in rows})
|
| 3779 |
+
return view
|
| 3780 |
+
|
| 3781 |
+
|
| 3782 |
+
def _database_tables_digest(tables, deltas):
|
| 3783 |
+
return _json_digest({key: _database_table_guard(tables.get(key), deltas[key])
|
| 3784 |
+
for key in sorted(deltas)})
|
| 3785 |
+
|
| 3786 |
+
|
| 3787 |
+
def _apply_database_table_delta(document, key, delta, after):
|
| 3788 |
+
"""Write one table's delta in place, so a row-mode store commits only those rows."""
|
| 3789 |
+
if not isinstance(after, dict):
|
| 3790 |
+
if key in document:
|
| 3791 |
+
del document[key]
|
| 3792 |
+
return
|
| 3793 |
+
table = document.get(key)
|
| 3794 |
+
if not isinstance(table, dict):
|
| 3795 |
+
document[key] = copy.deepcopy(after)
|
| 3796 |
+
return
|
| 3797 |
+
for name in delta["definition"]:
|
| 3798 |
+
if name in after:
|
| 3799 |
+
table[name] = copy.deepcopy(after[name])
|
| 3800 |
+
else:
|
| 3801 |
+
table.pop(name, None)
|
| 3802 |
+
for name, ids in delta["records"].items():
|
| 3803 |
+
new = after.get(name)
|
| 3804 |
+
if not isinstance(new, dict):
|
| 3805 |
+
table.pop(name, None)
|
| 3806 |
+
continue
|
| 3807 |
+
rows = table.get(name)
|
| 3808 |
+
if not isinstance(rows, dict):
|
| 3809 |
+
table[name] = {rid: copy.deepcopy(new[rid]) for rid in ids if rid in new}
|
| 3810 |
+
continue
|
| 3811 |
+
for rid in ids:
|
| 3812 |
+
if rid in new:
|
| 3813 |
+
rows[rid] = copy.deepcopy(new[rid])
|
| 3814 |
+
else:
|
| 3815 |
+
rows.pop(rid, None)
|
| 3816 |
+
|
| 3817 |
+
|
| 3818 |
+
def _database_write_plan(tables_before, tables_after, database):
|
| 3819 |
+
"""The per-table commit plan: deltas, the before digest, and the target's changed record ids."""
|
| 3820 |
+
deltas = {key: _database_table_delta(tables_before.get(key), tables_after.get(key))
|
| 3821 |
+
for key in sorted(set(tables_before) | set(tables_after))}
|
| 3822 |
+
row_ids = sorted({rid for ids in deltas.get(database, {"records": {}})["records"].values()
|
| 3823 |
+
for rid in ids}, key=str)
|
| 3824 |
+
# `bucket_after` keeps its name for existing readers; it now holds the planned tables only.
|
| 3825 |
+
return {"bucket_before_digest": _database_tables_digest(tables_before, deltas),
|
| 3826 |
+
"table_deltas": deltas, "bucket_after": tables_after, "row_ids": row_ids}
|
| 3827 |
+
|
| 3828 |
+
|
| 3829 |
+
def _plan_database_shared_update(session, database, record_id, raw_values, g, revision):
|
| 3830 |
+
"""D-11337: plan a record_update of shared fields on their overlay cell, or None for row fields."""
|
| 3831 |
+
from core import shared_overlay, user_tables
|
| 3832 |
+
import aios_grid
|
| 3833 |
+
|
| 3834 |
+
definitions = shared_overlay.fields(database, st=session.runtime)
|
| 3835 |
+
keys = [str(key) for key in raw_values]
|
| 3836 |
+
shared = [key for key in keys if shared_overlay.is_shared(database, key, st=session.runtime)]
|
| 3837 |
+
if not shared:
|
| 3838 |
+
return None
|
| 3839 |
+
if len(shared) != len(keys):
|
| 3840 |
+
raise err(400, "tool_arguments_invalid",
|
| 3841 |
+
"Change shared fields and other fields in separate calls.")
|
| 3842 |
+
# The grid's shared-cell door refuses an unresolved pid scope before any cell is read.
|
| 3843 |
+
if g.get("limits"):
|
| 3844 |
+
raise err(409, "pid_scope_unresolved",
|
| 3845 |
+
"The permitted record set could not be resolved. Nothing was saved.")
|
| 3846 |
+
by_key = {str(field.get("key")): field for field in g["fields"] if isinstance(field, dict)}
|
| 3847 |
+
from core import shares
|
| 3848 |
+
if not shares.may_edit("database", database, session.uname, is_admin=session.admin,
|
| 3849 |
+
st=session.runtime):
|
| 3850 |
+
raise err(403, "forbidden", "An edit share on this database is required to change a shared cell.")
|
| 3851 |
+
values = _database_value_plan(session, database, int(record_id), raw_values, g["fields"])
|
| 3852 |
+
for key in shared:
|
| 3853 |
+
# D-9735 item 3: `put_cells` stores any scalar, so a server-assigned kind refuses here.
|
| 3854 |
+
if any((field or {}).get("type") == "autonumber"
|
| 3855 |
+
or (field or {}).get("type") in user_tables.AUDIT_FIELD_TYPES
|
| 3856 |
+
for field in (definitions.get(key), by_key.get(key))):
|
| 3857 |
+
raise err(403, "computed_field", "Autonumber and record audit values are assigned by the server.")
|
| 3858 |
+
try:
|
| 3859 |
+
for value in values.values():
|
| 3860 |
+
shared_overlay._value(value)
|
| 3861 |
+
except ValueError as exc:
|
| 3862 |
+
raise err(400, "bad_value", str(exc)[:300]) from exc
|
| 3863 |
+
choices = {}
|
| 3864 |
+
for key, value in values.items():
|
| 3865 |
+
field = by_key.get(key) or definitions.get(key) or {}
|
| 3866 |
+
if field.get("type") in ("select", "multiselect") and value:
|
| 3867 |
+
known = {option.casefold() for option in aios_grid.merge_choice_options(
|
| 3868 |
+
(definitions.get(key) or {}).get("options") or field.get("options"))}
|
| 3869 |
+
parts = [value] if field["type"] == "select" else value.split(",")
|
| 3870 |
+
added = [part for part in parts if part.casefold() not in known]
|
| 3871 |
+
if added:
|
| 3872 |
+
choices[key] = added
|
| 3873 |
+
held = shared_overlay.cells(database, [int(record_id)], st=session.runtime).get(record_id) or {}
|
| 3874 |
+
before = {key: copy.deepcopy(held.get(key)) for key in values}
|
| 3875 |
+
changed, _ = _diff_documents({"databases/" + database: {
|
| 3876 |
+
"before": {"records": {record_id: before}}, "after": {"records": {record_id: values}}}})
|
| 3877 |
+
return {"diff": changed, "affected": {"ids": [record_id], "count": 1},
|
| 3878 |
+
"warnings": [], "revision": {**revision, "shared_fields": _json_digest({
|
| 3879 |
+
key: definitions[key] for key in values})},
|
| 3880 |
+
"shared_cells": {"record_id": record_id, "before": before, "after": values,
|
| 3881 |
+
"fields": {key: copy.deepcopy(definitions[key]) for key in values},
|
| 3882 |
+
"choices": choices},
|
| 3883 |
+
"planned": {"database": database, "action": "record_update", "record_id": record_id}}
|
| 3884 |
+
|
| 3885 |
+
|
| 3886 |
+
def _commit_database_shared_update(session, database, shared):
|
| 3887 |
+
"""Write a planned shared-cell update after checking the cell still holds its previewed value."""
|
| 3888 |
+
from core import shared_overlay
|
| 3889 |
+
|
| 3890 |
+
record_id = str(shared.get("record_id") or "")
|
| 3891 |
+
before, after = shared.get("before"), shared.get("after")
|
| 3892 |
+
if not record_id.isdigit() or not isinstance(before, dict) or not isinstance(after, dict) or not after:
|
| 3893 |
+
raise err(503, "confirmation_revision_unavailable",
|
| 3894 |
+
"The database version could not be checked. Nothing was saved.")
|
| 3895 |
+
held = shared_overlay.cells(database, [int(record_id)], st=session.runtime).get(record_id) or {}
|
| 3896 |
+
if any(held.get(key) != value for key, value in before.items()):
|
| 3897 |
+
raise err(409, "confirmation_stale", "The database changed after the preview, " + _STALE_FIX)
|
| 3898 |
+
try:
|
| 3899 |
+
expected = {"cells": before, "fields": shared["fields"]}
|
| 3900 |
+
if shared.get("choices"):
|
| 3901 |
+
shared_overlay.put_cells_and_choice_options(
|
| 3902 |
+
database, int(record_id), after, shared["choices"], st=session.runtime, expected=expected)
|
| 3903 |
+
else:
|
| 3904 |
+
shared_overlay.put_cells(database, int(record_id), after, st=session.runtime, expected=expected)
|
| 3905 |
+
except ValueError as exc:
|
| 3906 |
+
raise err(400, getattr(exc, "code", None) or "bad_value", str(exc)[:300]) from exc
|
| 3907 |
+
|
| 3908 |
+
|
| 3909 |
+
def _field_link_tables(*fields):
|
| 3910 |
+
"""The tables a link field's reciprocal lives on, before and after a schema write."""
|
| 3911 |
+
tables = []
|
| 3912 |
+
for field in fields:
|
| 3913 |
+
bag = field.get("link") if isinstance(field, dict) else None
|
| 3914 |
+
if isinstance(bag, dict) and bag.get("table"):
|
| 3915 |
+
tables.append(str(bag["table"]))
|
| 3916 |
+
return tables
|
| 3917 |
|
| 3918 |
|
| 3919 |
def _plan_database_write(session, args):
|
|
|
|
| 3930 |
if action == "rename":
|
| 3931 |
return _plan_database_rename(session, args)
|
| 3932 |
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 3933 |
+
# D-11174: a database write reads, copies and diffs only the tables it changes.
|
| 3934 |
+
tables_before = _database_write_tables(session.runtime, (database,))
|
| 3935 |
+
table = copy.deepcopy(tables_before.get(database) or {})
|
|
|
|
| 3936 |
if not table or not user_tables.is_user_table(database, st=session.runtime):
|
| 3937 |
raise err(403, "connected_source_read_only",
|
| 3938 |
"Records from connected sources are read only.")
|
| 3939 |
+
revision = _bucket_revision(_database_write_source(session.runtime), user_tables.STORE_KEY,
|
| 3940 |
+
tables_before)
|
| 3941 |
if type(revision.get("rev")) is not int:
|
| 3942 |
raise err(503, "confirmation_revision_unavailable",
|
| 3943 |
"The database version could not be checked. Nothing was saved.")
|
| 3944 |
warnings = []
|
|
|
|
| 3945 |
|
| 3946 |
if action.startswith("record_"):
|
| 3947 |
field_keys = None
|
|
|
|
| 3963 |
record_id = str(args.get("record_id") or "").strip()
|
| 3964 |
if not record_id.isdigit() or int(record_id) not in g["pids"]:
|
| 3965 |
raise err(403, "record_unavailable", "That record is not available to this account.")
|
| 3966 |
+
if action == "record_update":
|
| 3967 |
+
shared_plan = _plan_database_shared_update(
|
| 3968 |
+
session, database, record_id, raw_values, g, revision)
|
| 3969 |
+
if shared_plan is not None:
|
| 3970 |
+
return shared_plan
|
| 3971 |
before = copy.deepcopy(rows.get(record_id))
|
| 3972 |
if before is None:
|
| 3973 |
raise err(403, "record_unavailable", "That record is not available to this account.")
|
|
|
|
| 3984 |
after[field["key"]] = ""
|
| 3985 |
else:
|
| 3986 |
after = None
|
| 3987 |
+
tables_after = copy.deepcopy(tables_before)
|
| 3988 |
+
target = tables_after[database]
|
| 3989 |
if action == "record_delete":
|
| 3990 |
(target.get("rows") or {}).pop(record_id, None)
|
| 3991 |
else:
|
|
|
|
| 3999 |
# This is part of the UI row writer's durable result, not an optional callback.
|
| 4000 |
# Include newly encountered select values in the preview and the one guarded commit.
|
| 4001 |
user_tables._hydrate_choice_options(target.get("fields") or [], [after])
|
| 4002 |
+
changed, truncated = _diff_documents(_database_preview_documents(session, tables_before, tables_after))
|
| 4003 |
if truncated:
|
| 4004 |
warnings.append("The preview diff reached its display cap; split this change.")
|
| 4005 |
return {"diff": changed, "affected": {"ids": [record_id], "count": 1},
|
| 4006 |
"warnings": warnings, "revision": revision,
|
| 4007 |
+
**_database_write_plan(tables_before, tables_after, database),
|
|
|
|
| 4008 |
"planned": {"database": database, "action": action, "record_id": record_id}}
|
| 4009 |
|
| 4010 |
field_key = str(args.get("field_key") or "").strip()
|
|
|
|
| 4058 |
# Build the exact schema document through the core schema mechanics on an isolated runtime.
|
| 4059 |
# This includes reciprocal-link cleanup/creation while keeping preview free of relation
|
| 4060 |
# refreshes, audit calls, automation events and every other out-of-store side effect.
|
| 4061 |
+
# A link field's reciprocal lives on its target table, so that table joins the plan.
|
| 4062 |
+
tables_before.update(_database_write_tables(
|
| 4063 |
+
session.runtime, [key for key in _field_link_tables(before, after) if key not in tables_before]))
|
| 4064 |
preview = _PreviewRuntime(session.runtime)
|
| 4065 |
+
_seed_preview(preview, user_tables.STORE_KEY, tables_before, revision)
|
| 4066 |
if action == "field_create":
|
| 4067 |
stored = user_tables.add_field(
|
| 4068 |
database, after, st=preview, recurrence_validator=recurrence_validator)
|
|
|
|
| 4081 |
warnings.append(str(synced.get("reason") or "A reciprocal field was not created."))
|
| 4082 |
elif not user_tables.delete_field(database, field_key, st=preview):
|
| 4083 |
raise err(403, "field_refused", "That field cannot be removed.")
|
| 4084 |
+
tables_after = preview.get(user_tables.STORE_KEY)
|
| 4085 |
+
changed, truncated = _diff_documents(_database_preview_documents(session, tables_before, tables_after))
|
| 4086 |
if truncated:
|
| 4087 |
warnings.append("The preview diff reached its display cap; split this change.")
|
| 4088 |
affected_ids = [field_key]
|
|
|
|
| 4094 |
return {"diff": changed,
|
| 4095 |
"affected": {"ids": affected_ids, "count": len(affected_ids)},
|
| 4096 |
"warnings": warnings, "revision": revision,
|
| 4097 |
+
**_database_write_plan(tables_before, tables_after, database),
|
|
|
|
| 4098 |
"planned": {"database": database, "action": action, "field_key": field_key}}
|
| 4099 |
|
| 4100 |
|
|
|
|
| 4123 |
"The database metadata changed after the preview, " + _STALE_FIX) from exc
|
| 4124 |
return {"database": database, "action": action, "meta": result["meta"],
|
| 4125 |
"affected": copy.deepcopy(plan.get("affected") or {})}
|
| 4126 |
+
if plan.get("shared_cells") is not None:
|
| 4127 |
+
# D-11337: a shared field's value lives in the overlay, never in the table's rows.
|
| 4128 |
+
_commit_database_shared_update(session, database, plan["shared_cells"])
|
| 4129 |
+
try:
|
| 4130 |
+
import routes_tables
|
| 4131 |
+
routes_tables._refresh_relations(session)
|
| 4132 |
+
except Exception as exc: # noqa: BLE001
|
| 4133 |
+
print(f"[external] relation refresh deferred: {type(exc).__name__}: {exc}")
|
| 4134 |
+
return {"database": database, **copy.deepcopy(plan.get("planned") or {}),
|
| 4135 |
+
"affected": copy.deepcopy(plan.get("affected") or {})}
|
| 4136 |
expected_rev = plan.get("revision", {}).get("rev")
|
| 4137 |
expected_digest = plan.get("bucket_before_digest")
|
| 4138 |
replacement = copy.deepcopy(plan.get("bucket_after"))
|
| 4139 |
+
if type(expected_rev) is not int or not isinstance(replacement, dict) or not expected_digest:
|
| 4140 |
raise err(503, "confirmation_revision_unavailable",
|
| 4141 |
"The database version could not be checked. Nothing was saved.")
|
| 4142 |
+
deltas = copy.deepcopy(plan.get("table_deltas"))
|
| 4143 |
+
if not isinstance(deltas, dict):
|
| 4144 |
+
# A plan without deltas replaces its planned tables whole, under the same digest guard.
|
| 4145 |
+
deltas = {key: _database_table_delta(None, table) for key, table in replacement.items()}
|
| 4146 |
+
# The changed record ids keep a record-mode table per record; an older plan selects every record.
|
| 4147 |
+
row_ids = None if plan.get("row_ids") is None else list(plan["row_ids"])
|
| 4148 |
|
| 4149 |
def apply_exact(current):
|
| 4150 |
+
# D-11174: read and write only the planned tables, in place, so no whole document moves.
|
| 4151 |
+
if _database_tables_digest(current, deltas) != expected_digest:
|
| 4152 |
raise view_audit.ViewRevisionStale(
|
| 4153 |
rev_sent=expected_rev, rev_current=expected_rev + 1,
|
| 4154 |
message="The database changed after preview.")
|
| 4155 |
+
for key in sorted(deltas):
|
| 4156 |
+
_apply_database_table_delta(current, key, deltas[key], replacement.get(key))
|
| 4157 |
+
return current
|
| 4158 |
|
| 4159 |
try:
|
| 4160 |
+
# One guarded read-modify-write over the planned tables. The expected revision is checked
|
| 4161 |
# inside the PG lock where available; the closure's digest check also protects a backend
|
| 4162 |
# that can only provide the runtime's documented guarded fallback.
|
| 4163 |
user_tables.update_with_activity(
|
| 4164 |
database, apply_exact, st=session.runtime, by=session.uname,
|
| 4165 |
+
via=_trusted_activity_via(session), row_ids=row_ids,
|
| 4166 |
+
field_keys=None, flush="sync", expect_rev=expected_rev)
|
| 4167 |
except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
|
| 4168 |
raise err(409, "confirmation_stale",
|
| 4169 |
"The database changed after the preview, " + _STALE_FIX) from exc
|
|
|
|
| 4564 |
)
|
| 4565 |
|
| 4566 |
|
| 4567 |
+
def _is_members_write(name, arguments):
|
| 4568 |
+
"""A members permission write previews from the global account store, not the tenant runtime."""
|
| 4569 |
+
return name == "members" and isinstance(arguments, dict) and arguments.get("action") == "set_permissions"
|
| 4570 |
+
|
| 4571 |
+
|
| 4572 |
def _strong_confirmation_required(name, args):
|
| 4573 |
+
if _is_members_write(name, args):
|
| 4574 |
+
# A business unit widening needs the strong phrase in place of the editor's confirm_widen.
|
| 4575 |
+
return bool((args.get("arguments") or {}).get("confirm_widen"))
|
| 4576 |
if name not in external_tools.TOOLS:
|
| 4577 |
import tool_groups
|
| 4578 |
resolved = tool_groups.resolve(name, args)
|
|
|
|
| 4797 |
"Part of this change could not be accepted. Nothing was previewed.")
|
| 4798 |
|
| 4799 |
remote_note = _is_website_note(name, normalised)
|
| 4800 |
+
plan = result if name == "database_object" or remote_note or _is_members_write(name, normalised) else None
|
| 4801 |
if plan is not None:
|
| 4802 |
diff = copy.deepcopy(plan.get("diff") or [])
|
| 4803 |
warnings = list(plan.get("warnings") or [])
|
|
|
|
| 4830 |
normalised["arguments"]["expect"] = [
|
| 4831 |
{"user": row["user"], "role": row["role"]}
|
| 4832 |
for row in result["current_entries"]]
|
| 4833 |
+
if _is_members_write(name, normalised):
|
| 4834 |
+
# Pin the account state the preview read, and the widening it found, into the signed call.
|
| 4835 |
+
normalised["arguments"].update(expect=result["expect"], confirm_widen=bool(result["widening"]))
|
| 4836 |
|
| 4837 |
_canonical_json(diff, capped=False)
|
| 4838 |
diff_digest = _json_digest(diff)
|
|
|
|
| 4891 |
response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
|
| 4892 |
if name == "apps" and normalised.get("action") == "catalog_build":
|
| 4893 |
response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
|
| 4894 |
+
if _is_members_write(name, normalised):
|
| 4895 |
+
response.update({key: copy.deepcopy(result[key]) for key in ("username", "database", "before", "after")})
|
| 4896 |
if payload["strong"]:
|
| 4897 |
response["confirmation_phrase"] = _CONFIRMATION_PHRASE
|
| 4898 |
if issue_token:
|
|
|
|
| 5428 |
if tool == "database_object":
|
| 5429 |
result = _execute_database_object(
|
| 5430 |
fresh_session, arguments, plan=recomputed.get("_plan"))
|
| 5431 |
+
elif _is_members_write(tool, arguments):
|
| 5432 |
+
# The account writer compares the pinned expect itself, so no runtime guard is needed.
|
| 5433 |
+
result = _dispatch_external_tool(_tool_context(fresh_session), tool, arguments, about=payload["about"])
|
| 5434 |
else:
|
| 5435 |
from datetime import datetime, timezone
|
| 5436 |
commit_started = datetime.now(timezone.utc)
|
|
|
|
| 5461 |
from core import user_tables
|
| 5462 |
import routes_nav
|
| 5463 |
bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY
|
| 5464 |
+
# A failed revision read is replaced by the produced rev below, so no document is read here.
|
| 5465 |
+
after_revision = _bucket_revision(fresh_session.runtime, bucket, None)
|
| 5466 |
produced_rev = before_revision["rev"] + 1
|
| 5467 |
if after_revision.get("rev") != produced_rev:
|
| 5468 |
after_revision = {"rev": produced_rev}
|
| 5469 |
+
elif _is_members_write(tool, arguments):
|
| 5470 |
+
after_revision = copy.deepcopy(result["revision"])
|
| 5471 |
else:
|
| 5472 |
after_revision = copy.deepcopy(guarded_session.runtime._expected)
|
| 5473 |
after_revision_digest = _json_digest(after_revision)
|
|
|
|
| 5497 |
response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
|
| 5498 |
if tool == "apps" and arguments.get("action") == "catalog_build":
|
| 5499 |
response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
|
| 5500 |
+
if _is_members_write(tool, arguments):
|
| 5501 |
+
response.update(username=result["username"], database=result["database"],
|
| 5502 |
+
after=copy.deepcopy(result["entry"]))
|
| 5503 |
_confirmation_status(fresh_session, operation_id, "committed", response)
|
| 5504 |
if isinstance(stored.get("arguments_ref"), dict):
|
| 5505 |
_drop_operation_documents(fresh_session, [stored["arguments_ref"].get("key")])
|
| 5506 |
+
if (tool != "database_object" and not _is_members_write(tool, arguments)
|
| 5507 |
+
and receipt.get("persisted") is True):
|
| 5508 |
try:
|
| 5509 |
targets, native_ids = _committed_build_evidence(fresh_session, guarded_session.runtime, arguments,
|
| 5510 |
build_kind=payload["buildKind"])
|
|
|
|
| 5636 |
"query_filter_limit": "Filters allow 3 levels, 100 nodes and 50 conditions per group to bound query validation work. To reduce that work, " + _MCP_FIXES["query_filter_limit"] + ".",
|
| 5637 |
"query_cursor_stale": "The query, data or permissions changed. To restart, " + _MCP_FIXES["query_cursor_stale"] + ".",
|
| 5638 |
"query_cursor_invalid": "The query cursor is invalid. Repeat the original query without cursor for a fresh first page.",
|
| 5639 |
+
"query_source_unavailable": "This database's source could not be read for this query.",
|
| 5640 |
"test_connection_production": "Test connections cannot write to production. Use a staging environment.",
|
| 5641 |
"test_connection_environment_unknown": "This environment is unknown. Test connections need a named non-production environment.",
|
| 5642 |
"external_test_connection_invalid": "Set test_connection to true or false.",
|
|
|
|
| 5670 |
r"Traceback|stack trace|\bengine\b|\bmanifest\b|\brevision-guard\b|\bgate\b|"
|
| 5671 |
r"\b(?:tenant store|dispatcher|document cleaner)\b|"
|
| 5672 |
r"(?:[A-Za-z]:[\\/]|/(?:tmp|root|work|home|srv|app)/)|"
|
| 5673 |
+
# A file suffix ends at a word boundary: the views action view.share is no shell script.
|
| 5674 |
+
r"(?:[\w.-]+[/\\])*[\w.-]+\.(?:py|tsx?|pyc|ps1|sh|json)\b(?::\d+)?|"
|
| 5675 |
r"\.(?:claude|agents|codex)/|~[/\\]", re.I)
|
| 5676 |
_MCP_PROSE_REPLACEMENTS = (
|
| 5677 |
(r"strict Assistant scope wall|scope wall|live UI walls|UI walls", "permissions"),
|
|
|
|
| 6276 |
required.append("a named non-production environment for this test connection")
|
| 6277 |
if group == "websites":
|
| 6278 |
required.extend(_website_requirements(session, note=action == "add_internal_note"))
|
| 6279 |
+
if group == "members" and not deps.perms.is_admin(session.user):
|
| 6280 |
+
required.append("the Administrator role")
|
| 6281 |
if legacy in {"describe_database", "query_database", "view_upsert", "view_delete", "view_share_preview", "view_object", "app_upsert", "app_delete"} or (
|
| 6282 |
legacy == "database_object" and entry.get("legacy_action") != "list") or legacy == "sheet_set_binding":
|
| 6283 |
if not rows:
|
|
|
|
| 6378 |
return count if type(count) is int and count >= 0 else 0
|
| 6379 |
|
| 6380 |
|
| 6381 |
+
def _mcp_guide_group_note(requested, groups) -> str:
|
| 6382 |
+
"""Name an unknown guide group and the groups that exist at call time, in one line."""
|
| 6383 |
+
shown = requested.strip() if isinstance(requested, str) else ""
|
| 6384 |
+
# Only a short identifier is echoed: a credential-shaped name would withhold the whole guide.
|
| 6385 |
+
named = (f"No guide group is named `{shown}`."
|
| 6386 |
+
if re.fullmatch(r"[A-Za-z][A-Za-z0-9_.-]{0,59}", shown) and not _contains_credential(shown)
|
| 6387 |
+
else "No guide group has that name.")
|
| 6388 |
+
return (named + " The groups are " + ", ".join(f"`{name}`" for name in groups)
|
| 6389 |
+
+ ". Read one with group, or use the index below.")
|
| 6390 |
+
|
| 6391 |
+
|
| 6392 |
def _mcp_guide(session: Session, group=None) -> str:
|
| 6393 |
"""Return the group index or one group's action schemas without reading records."""
|
| 6394 |
+
try:
|
| 6395 |
+
return _mcp_guide_text(session, group)
|
| 6396 |
+
except HTTPException as exc:
|
| 6397 |
+
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 6398 |
+
if exc.status_code in (401, 403, 404, 429) or exc.status_code >= 500 or info.get("code") in MCP_RESULT_CODES:
|
| 6399 |
+
raise
|
| 6400 |
+
except sheets.SheetRefusal as exc:
|
| 6401 |
+
if exc.code in MCP_RESULT_CODES:
|
| 6402 |
+
raise
|
| 6403 |
+
# An unpublished refusal from a guide read would collapse to tool_refused, which names a rejected change.
|
| 6404 |
+
raise err(503, "tool_unavailable", "The guide could not be read right now. Try again shortly.")
|
| 6405 |
+
|
| 6406 |
+
|
| 6407 |
+
def _mcp_guide_text(session: Session, group=None) -> str:
|
| 6408 |
+
"""Render the group index or one group's action schemas without reading records."""
|
| 6409 |
import tool_groups
|
| 6410 |
groups = tool_groups.groups()
|
| 6411 |
+
requested = group
|
| 6412 |
if isinstance(group, str) and group not in groups and group != "data":
|
| 6413 |
# A cased, plural or dotted name (Sheets, sheet.write_cells) reads the one group it names.
|
| 6414 |
key = group.strip().lower().partition(".")[0]
|
|
|
|
| 6416 |
if name in groups or name == "data"), group)
|
| 6417 |
alias = group == "data"
|
| 6418 |
group = tool_groups.canonical_name(group)
|
| 6419 |
+
note = None
|
| 6420 |
if group is not None and group not in groups:
|
| 6421 |
+
# An unknown group answers the index, led by one line naming it and the real groups.
|
| 6422 |
+
note, group = _mcp_guide_group_note(requested, groups), None
|
| 6423 |
# One rows-free definitions read per guide: a live runtime pays one pool checkout per database.
|
| 6424 |
# The guide reads schemas only, so no reader here needs `rows`. On a failed read, keep the old path.
|
| 6425 |
try:
|
|
|
|
| 6437 |
rows = [row for row in rows if row["permitted"]]
|
| 6438 |
capabilities = _mcp_capabilities(session, groups, [row for row in rows if row.get("schema")])
|
| 6439 |
unread = _guide_unread_count(session)
|
| 6440 |
+
lines = [*([note, ""] if note else []), "# Loopable guide", "", f"Signed in as {session.uname}.", "", MCP_INSTRUCTIONS, "",
|
| 6441 |
f"Scheduled results you have not read: {unread}.",
|
| 6442 |
*(["Read schedules.inbox before answering."] if unread > 0 else []), "",
|
| 6443 |
"Call a group with action and arguments. Each action keeps its own read or write rules.",
|
|
|
|
| 6589 |
import tool_groups
|
| 6590 |
resolved = tool_groups.resolve(name, arguments)
|
| 6591 |
if resolved is None:
|
| 6592 |
+
raise sheets.SheetRefusal("tool_not_found", _unknown_tool_message(name))
|
| 6593 |
entry, args = resolved
|
| 6594 |
if call is not None and call["tool"] == name:
|
| 6595 |
external_activity.note_action(arguments["action"])
|
|
|
|
| 6604 |
return result
|
| 6605 |
|
| 6606 |
|
| 6607 |
+
def _unknown_tool_message(name):
|
| 6608 |
+
"""Name the group's real actions for a dotted name such as table.call, never echoing the caller's text."""
|
| 6609 |
+
import tool_groups
|
| 6610 |
+
prefix = name.partition(".")[0] if isinstance(name, str) else ""
|
| 6611 |
+
group = tool_groups.groups().get(tool_groups.canonical_name(prefix)) if prefix else None
|
| 6612 |
+
if group is None:
|
| 6613 |
+
return "The requested tool is not available."
|
| 6614 |
+
return ("That is not a tool name. Call the " + group["name"] + " tool and set action to one of the "
|
| 6615 |
+
+ group["name"] + " actions: " + ", ".join(group["actions"]) + ". Put that action's inputs "
|
| 6616 |
+
"inside arguments.")
|
| 6617 |
+
|
| 6618 |
+
|
| 6619 |
+
def _note_internal_code(out):
|
| 6620 |
+
"""Keep the refusal code before _mcp_public_code collapses it, for the limitation mailbox."""
|
| 6621 |
+
call = external_activity._CALL.get()
|
| 6622 |
+
if call is None or not isinstance(out, dict):
|
| 6623 |
+
return
|
| 6624 |
+
data = out.get("data") if isinstance(out.get("data"), dict) else {}
|
| 6625 |
+
error = out.get("error") if not out.get("ok") else data.get("error")
|
| 6626 |
+
error = error if isinstance(error, dict) else {}
|
| 6627 |
+
code = error.get("code") or (data.get("code") if out.get("ok") and data.get("ok") is False else "")
|
| 6628 |
+
call["internal_code"] = code if isinstance(code, str) else ""
|
| 6629 |
+
|
| 6630 |
+
|
| 6631 |
def _record_mcp_method(session, req, reply):
|
| 6632 |
"""Retain bounded method names for admitted protocol requests without request values."""
|
| 6633 |
try:
|
|
|
|
| 6686 |
code = error.get("code", "tool_refused") if error else ("tool_refused" if reply.get("error") else "")
|
| 6687 |
if code and not call["limitation_recorded"]:
|
| 6688 |
from routes_feedback import record_limitation
|
| 6689 |
+
# The reply's code is already public. The mailbox also keeps the one noted at dispatch.
|
| 6690 |
+
record_limitation(session, tool=limitation_tool, action=limitation_action, code=code,
|
| 6691 |
+
question=call["question"], internal_code=call.get("internal_code") or None)
|
| 6692 |
except Exception:
|
| 6693 |
pass
|
| 6694 |
finally:
|
|
|
|
| 6901 |
if code == "assistant_unknown_database":
|
| 6902 |
# The read path's lowercase reason is replaced: the fix sentence names the missing key itself.
|
| 6903 |
return fix
|
| 6904 |
+
if code in {"assistant_snapshot_unavailable", "assistant_store_not_ready"} and deps._assistant_source_cause_of(text):
|
| 6905 |
+
# S82-T125: a named cause carries its own retry line, or none where no retry can succeed.
|
| 6906 |
+
return text
|
| 6907 |
return text if not fix or fix in text else (text + " " + fix).strip()
|
| 6908 |
|
| 6909 |
|
|
|
|
| 7121 |
out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry)
|
| 7122 |
except (TypeError, ValueError, KeyError, AttributeError):
|
| 7123 |
out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
|
| 7124 |
+
except Exception as exc:
|
| 7125 |
+
deps._assistant_log_cause("external tool", exc)
|
| 7126 |
out = _tool_refusal(name, "tool_unavailable",
|
| 7127 |
"The tool could not complete this operation, so try the same call again shortly.", 503)
|
| 7128 |
+
_note_internal_code(out)
|
| 7129 |
return _mcp_content(rid, out, session)
|
| 7130 |
|
| 7131 |
|
|
|
|
| 7179 |
"database_object", "view_object", "workbook_object", "worksheet_object",
|
| 7180 |
"range_object", "calculation_object", "validated_batch",
|
| 7181 |
"confirm_change",
|
| 7182 |
+
# Dotted names declare every grouped write with no legacy tool.
|
| 7183 |
+
"apps.catalog_build", "apps.apply_template", "apps.save_code", "apps.set_status",
|
| 7184 |
+
"automations.create", "automations.pause", "automations.resume", "automations.delete",
|
| 7185 |
+
"intelligence.propose", "intelligence.create_wiki",
|
| 7186 |
+
"members.set_permissions",
|
| 7187 |
+
"requests.send_message", "requests.ask_user", "requests.send", "requests.answer", "requests.complete",
|
| 7188 |
+
"schedules.create", "schedules.pause", "schedules.resume", "schedules.delete",
|
| 7189 |
+
"sheet.worksheet.rename", "sheet.worksheet.reorder", "sheet.worksheet.duplicate", "sheet.refresh_scope",
|
| 7190 |
+
"views.share", "websites.add_internal_note",
|
| 7191 |
})
|
| 7192 |
|
| 7193 |
|
| 7194 |
def _assert_write_allow_list(*, check_groups=False) -> None:
|
| 7195 |
declared = {name for name, tool in external_tools.TOOLS.items() if tool.get("write")}
|
| 7196 |
+
legacy = {name for name in _WRITE_TOOLS if "." not in name}
|
| 7197 |
+
undeclared = declared - legacy
|
| 7198 |
+
stale = legacy - declared
|
| 7199 |
if undeclared or stale:
|
| 7200 |
raise RuntimeError("External write allow-list mismatch: undeclared="
|
| 7201 |
+ repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
|
| 7202 |
if check_groups:
|
| 7203 |
import tool_groups
|
| 7204 |
+
found = tool_groups.groups()
|
| 7205 |
+
mapped = {entry.get("legacy_name") for group in found.values()
|
| 7206 |
for entry in group["actions"].values()}
|
| 7207 |
if set(external_tools.TOOLS) - mapped - {"confirm_change"}:
|
| 7208 |
raise RuntimeError("Every legacy tool must have a grouped action")
|
| 7209 |
+
standalone = _WRITE_TOOLS - legacy
|
| 7210 |
+
grouped = {group + "." + action for group, definition in found.items()
|
| 7211 |
+
for action, entry in definition["actions"].items()
|
| 7212 |
+
if entry.get("read") is False and not entry.get("legacy_name")}
|
| 7213 |
+
undeclared = grouped - standalone
|
| 7214 |
+
stale = standalone - grouped
|
| 7215 |
+
if undeclared or stale:
|
| 7216 |
+
raise RuntimeError("Grouped write allow-list mismatch: undeclared="
|
| 7217 |
+
+ repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
|
| 7218 |
|
| 7219 |
|
| 7220 |
_assert_write_allow_list()
|
api/routes_feedback.py
CHANGED
|
@@ -72,6 +72,9 @@ MAX_CHARS = 4000
|
|
| 72 |
#: the newest, which is the one direction this store must never move in.
|
| 73 |
MAX_ROWS = 5000
|
| 74 |
QUESTION_MAX = 500
|
|
|
|
|
|
|
|
|
|
| 75 |
|
| 76 |
|
| 77 |
def _limitation_address(tool, action):
|
|
@@ -97,19 +100,55 @@ def _limitation_address(tool, action):
|
|
| 97 |
return ("loopable_guide", "read") if tool == "loopable_guide" else (tool, action)
|
| 98 |
|
| 99 |
|
| 100 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
"""Coalesce value-free AI limitations in the existing platform feedback document."""
|
| 102 |
try:
|
| 103 |
# Protocol discovery belongs in method telemetry, outside the product limitation mailbox.
|
| 104 |
if code == "request_method_unknown":
|
| 105 |
return False
|
| 106 |
from core import audit, freeze_notice
|
| 107 |
-
from routes_external import _contains_credential, _mcp_public_code
|
| 108 |
if (freeze_notice.enabled() or getattr(session.runtime, "preview_only", False)
|
| 109 |
or not store.available()):
|
| 110 |
return False
|
| 111 |
tool, action = _limitation_address(tool, action)
|
|
|
|
| 112 |
code = _mcp_public_code(code)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
fallback = f"I could not complete {tool}.{action}."
|
| 114 |
# The caller supplies prose explicitly. Never derive it from arguments or results.
|
| 115 |
explicit = (isinstance(question, str) and 0 < len(question.strip()) <= QUESTION_MAX
|
|
@@ -128,7 +167,7 @@ def record_limitation(session, *, tool, action, code, question=None):
|
|
| 128 |
report = {"user": user, "ts": ts, "question": question}
|
| 129 |
identity = hashlib.sha256(json.dumps([tenant, tool, action, code]).encode()).hexdigest()
|
| 130 |
row = {"id": "lim_" + identity, "category": "limitation", "tenant": tenant,
|
| 131 |
-
"user": user, "tool": tool, "action": action, "code": code,
|
| 132 |
"question": question, "ts": ts, "firstTs": ts, "count": 1,
|
| 133 |
"reports": [report], "filers": {user: {"count": 1, "lastTs": ts}}}
|
| 134 |
|
|
@@ -145,7 +184,9 @@ def record_limitation(session, *, tool, action, code, question=None):
|
|
| 145 |
filers = {old["user"]: {"count": int(old.get("count", 1)), "lastTs": old["ts"]}}
|
| 146 |
filer = filers.get(user, {})
|
| 147 |
# Reports have no cap. The feedback document remains one JSON value under the 128 MB store ceiling.
|
| 148 |
-
|
|
|
|
|
|
|
| 149 |
"count": int(old.get("count", 1)) + 1, "reports": reports + [report],
|
| 150 |
"filers": {**filers, user: {"count": int(filer.get("count", 0)) + 1,
|
| 151 |
"lastTs": ts}}}
|
|
|
|
| 72 |
#: the newest, which is the one direction this store must never move in.
|
| 73 |
MAX_ROWS = 5000
|
| 74 |
QUESTION_MAX = 500
|
| 75 |
+
#: A report without a code borrows the filer's refusal at the same address from this recent window.
|
| 76 |
+
REFUSAL_WINDOW_SECONDS = 3600
|
| 77 |
+
_CODE = re.compile(r"[a-z][a-z0-9_]{0,99}")
|
| 78 |
|
| 79 |
|
| 80 |
def _limitation_address(tool, action):
|
|
|
|
| 100 |
return ("loopable_guide", "read") if tool == "loopable_guide" else (tool, action)
|
| 101 |
|
| 102 |
|
| 103 |
+
def refusal_code(session, tool, action, code=None):
|
| 104 |
+
"""Return (internal, public) of this filer's latest recent refusal at the address, or None."""
|
| 105 |
+
try:
|
| 106 |
+
import tool_groups
|
| 107 |
+
from routes_external import _mcp_public_code
|
| 108 |
+
tool, action = _limitation_address(tool, action)
|
| 109 |
+
grouped = tool in tool_groups.groups()
|
| 110 |
+
user, tenant = str(session.uname or "").lower(), str(session.tenant or "").lower()
|
| 111 |
+
public = _mcp_public_code(code) if code else None
|
| 112 |
+
since = time.time() - REFUSAL_WINDOW_SECONDS
|
| 113 |
+
best = {}
|
| 114 |
+
for row in _rows():
|
| 115 |
+
filers = row.get("filers") if isinstance(row.get("filers"), dict) else {}
|
| 116 |
+
filer = {str(name).lower(): value for name, value in filers.items()}.get(user)
|
| 117 |
+
stamp = filer.get("lastTs") if isinstance(filer, dict) else (
|
| 118 |
+
row.get("ts") if str(row.get("user", "")).lower() == user else None)
|
| 119 |
+
if (row.get("category") != "limitation" or str(row.get("tenant", "")).lower() != tenant
|
| 120 |
+
or row.get("tool") != tool or grouped and row.get("action") != action
|
| 121 |
+
or type(stamp) not in (int, float) or stamp < since
|
| 122 |
+
or row.get("code") in (None, "capability_unavailable")
|
| 123 |
+
or public is not None and row.get("code") != public):
|
| 124 |
+
continue
|
| 125 |
+
# A group matches its exact action. A single tool such as confirm_change prefers it.
|
| 126 |
+
rank = (row.get("action") == action, stamp)
|
| 127 |
+
if not best or rank > best["rank"]:
|
| 128 |
+
best = {"rank": rank, "found": (row.get("internal_code") or row["code"], row["code"])}
|
| 129 |
+
return best.get("found")
|
| 130 |
+
except Exception:
|
| 131 |
+
return None
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def record_limitation(session, *, tool, action, code, question=None, internal_code=None):
|
| 135 |
"""Coalesce value-free AI limitations in the existing platform feedback document."""
|
| 136 |
try:
|
| 137 |
# Protocol discovery belongs in method telemetry, outside the product limitation mailbox.
|
| 138 |
if code == "request_method_unknown":
|
| 139 |
return False
|
| 140 |
from core import audit, freeze_notice
|
| 141 |
+
from routes_external import MCP_RESULT_CODES, _contains_credential, _mcp_public_code
|
| 142 |
if (freeze_notice.enabled() or getattr(session.runtime, "preview_only", False)
|
| 143 |
or not store.available()):
|
| 144 |
return False
|
| 145 |
tool, action = _limitation_address(tool, action)
|
| 146 |
+
internal = internal_code if internal_code is not None else code
|
| 147 |
code = _mcp_public_code(code)
|
| 148 |
+
# The row keeps the refusal's own code beside the public one. The id stays over the public code.
|
| 149 |
+
if (not isinstance(internal, str) or not _CODE.fullmatch(internal)
|
| 150 |
+
or internal in MCP_RESULT_CODES and internal != code):
|
| 151 |
+
internal = code
|
| 152 |
fallback = f"I could not complete {tool}.{action}."
|
| 153 |
# The caller supplies prose explicitly. Never derive it from arguments or results.
|
| 154 |
explicit = (isinstance(question, str) and 0 < len(question.strip()) <= QUESTION_MAX
|
|
|
|
| 167 |
report = {"user": user, "ts": ts, "question": question}
|
| 168 |
identity = hashlib.sha256(json.dumps([tenant, tool, action, code]).encode()).hexdigest()
|
| 169 |
row = {"id": "lim_" + identity, "category": "limitation", "tenant": tenant,
|
| 170 |
+
"user": user, "tool": tool, "action": action, "code": code, "internal_code": internal,
|
| 171 |
"question": question, "ts": ts, "firstTs": ts, "count": 1,
|
| 172 |
"reports": [report], "filers": {user: {"count": 1, "lastTs": ts}}}
|
| 173 |
|
|
|
|
| 184 |
filers = {old["user"]: {"count": int(old.get("count", 1)), "lastTs": old["ts"]}}
|
| 185 |
filer = filers.get(user, {})
|
| 186 |
# Reports have no cap. The feedback document remains one JSON value under the 128 MB store ceiling.
|
| 187 |
+
# A bare public code never overwrites a refusal code the row already holds.
|
| 188 |
+
kept = internal if internal != code else old.get("internal_code") or internal
|
| 189 |
+
updated = {**old, "user": user, "ts": ts, "internal_code": kept,
|
| 190 |
"count": int(old.get("count", 1)) + 1, "reports": reports + [report],
|
| 191 |
"filers": {**filers, user: {"count": int(filer.get("count", 0)) + 1,
|
| 192 |
"lastTs": ts}}}
|
api/routes_grid.py
CHANGED
|
@@ -405,6 +405,38 @@ def _workspace_values(values, session, module, fields):
|
|
| 405 |
return {str(pid): cells(raw) for pid, raw in values.items()}
|
| 406 |
|
| 407 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 408 |
def _workspace_filter_removals(workspace, fields, measures, hidden=()):
|
| 409 |
"""Clean saved display filters on a detached wire copy, retaining removal reasons."""
|
| 410 |
from copy import deepcopy
|
|
@@ -534,6 +566,8 @@ def workspace(scope: str = "customer",
|
|
| 534 |
workspace["limits"] = g.get("limits") or []
|
| 535 |
if fields_only:
|
| 536 |
workspace["fieldsOnly"] = True
|
|
|
|
|
|
|
| 537 |
return {"workspace": _workspace_filter_removals(
|
| 538 |
workspace, g['fields'], g['measures'], g.get('hidden') or ())}
|
| 539 |
|
|
@@ -594,6 +628,8 @@ def workspace(scope: str = "customer",
|
|
| 594 |
workspace["mapSetup"] = map_setup
|
| 595 |
if fields_only:
|
| 596 |
workspace["fieldsOnly"] = True
|
|
|
|
|
|
|
| 597 |
return {"workspace": _workspace_filter_removals(
|
| 598 |
workspace, g['fields'], g['measures'], g.get('_note_hidden') or ())}
|
| 599 |
|
|
@@ -698,6 +734,8 @@ def workspace(scope: str = "customer",
|
|
| 698 |
workspace["cohortMode"] = True
|
| 699 |
# The create pane offers "this cohort table only" (default) vs "all customer tables".
|
| 700 |
workspace["scopeChoice"] = True
|
|
|
|
|
|
|
| 701 |
return {"workspace": _workspace_filter_removals(
|
| 702 |
workspace, g['fields'], g['measures'], g.get('hidden') or ())}
|
| 703 |
|
|
@@ -2076,7 +2114,7 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2076 |
Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the
|
| 2077 |
same sessions the read door (`/workspace`) admits, or a tenant without the customer module
|
| 2078 |
can SEE its own user tables and not write to them."""
|
| 2079 |
-
from core import grid_events, shared_overlay
|
| 2080 |
from routes_customers import grid_assembly
|
| 2081 |
from routes_tables import _app_write_via, _managed_or_refuse
|
| 2082 |
|
|
@@ -2180,7 +2218,11 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2180 |
# answer "the batch was refused" and leave the caller to guess which member, which is
|
| 2181 |
# the same class of unfalsifiable answer the channel exists to end.
|
| 2182 |
_before = len(ctx.out.refusals)
|
| 2183 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2184 |
row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid,
|
| 2185 |
"rerender": bool(rerender)}
|
| 2186 |
mine = ctx.out.refusals[_before:]
|
|
|
|
| 405 |
return {str(pid): cells(raw) for pid, raw in values.items()}
|
| 406 |
|
| 407 |
|
| 408 |
+
def _published_rev(ws):
|
| 409 |
+
"""S82-T156: the store revision the workspace was read at, which a view write stamps."""
|
| 410 |
+
rev = ws.get("rev") if isinstance(ws, dict) else None
|
| 411 |
+
return rev if isinstance(rev, int) and not isinstance(rev, bool) else None
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
def _settle_refused_view_write(ctx, event):
|
| 415 |
+
"""Store a no-op receipt for a refused stale view write, so its resend re-reads and stops.
|
| 416 |
+
|
| 417 |
+
The refusal rolls back the event's own receipt, and the outbox resends the same id. Without
|
| 418 |
+
this receipt every resend is refused again and the browser's save queue never moves (D-634).
|
| 419 |
+
"""
|
| 420 |
+
from core import edit_receipts, grid_events
|
| 421 |
+
|
| 422 |
+
if not getattr(ctx, "durable_receipts", False) or not isinstance(event, dict):
|
| 423 |
+
return
|
| 424 |
+
identity = event.get("opId", event.get("id"))
|
| 425 |
+
if not identity:
|
| 426 |
+
return
|
| 427 |
+
try:
|
| 428 |
+
identity = edit_receipts.op_id(identity)
|
| 429 |
+
dropped = {"rerender": True, "write": {"opId": identity}, "refusals": [],
|
| 430 |
+
"doc": None, "toast": None}
|
| 431 |
+
edit_receipts.run_once(
|
| 432 |
+
grid_events._store_of(ctx), ctx.uname,
|
| 433 |
+
edit_receipts.operation(ctx.scope_key, event.get("type"), event.get("pid")),
|
| 434 |
+
identity, lambda: dropped)
|
| 435 |
+
except Exception as exc: # noqa: BLE001
|
| 436 |
+
print(f"[grid/events] refused view write {identity!r}: resend receipt not saved "
|
| 437 |
+
f"({type(exc).__name__})")
|
| 438 |
+
|
| 439 |
+
|
| 440 |
def _workspace_filter_removals(workspace, fields, measures, hidden=()):
|
| 441 |
"""Clean saved display filters on a detached wire copy, retaining removal reasons."""
|
| 442 |
from copy import deepcopy
|
|
|
|
| 566 |
workspace["limits"] = g.get("limits") or []
|
| 567 |
if fields_only:
|
| 568 |
workspace["fieldsOnly"] = True
|
| 569 |
+
if _published_rev(g["ws"]) is not None:
|
| 570 |
+
workspace["rev"] = _published_rev(g["ws"])
|
| 571 |
return {"workspace": _workspace_filter_removals(
|
| 572 |
workspace, g['fields'], g['measures'], g.get('hidden') or ())}
|
| 573 |
|
|
|
|
| 628 |
workspace["mapSetup"] = map_setup
|
| 629 |
if fields_only:
|
| 630 |
workspace["fieldsOnly"] = True
|
| 631 |
+
if _published_rev(g["ws"]) is not None:
|
| 632 |
+
workspace["rev"] = _published_rev(g["ws"])
|
| 633 |
return {"workspace": _workspace_filter_removals(
|
| 634 |
workspace, g['fields'], g['measures'], g.get('_note_hidden') or ())}
|
| 635 |
|
|
|
|
| 734 |
workspace["cohortMode"] = True
|
| 735 |
# The create pane offers "this cohort table only" (default) vs "all customer tables".
|
| 736 |
workspace["scopeChoice"] = True
|
| 737 |
+
if _published_rev(g["ws"]) is not None:
|
| 738 |
+
workspace["rev"] = _published_rev(g["ws"])
|
| 739 |
return {"workspace": _workspace_filter_removals(
|
| 740 |
workspace, g['fields'], g['measures'], g.get('hidden') or ())}
|
| 741 |
|
|
|
|
| 2114 |
Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the
|
| 2115 |
same sessions the read door (`/workspace`) admits, or a tenant without the customer module
|
| 2116 |
can SEE its own user tables and not write to them."""
|
| 2117 |
+
from core import grid_events, shared_overlay, view_audit
|
| 2118 |
from routes_customers import grid_assembly
|
| 2119 |
from routes_tables import _app_write_via, _managed_or_refuse
|
| 2120 |
|
|
|
|
| 2218 |
# answer "the batch was refused" and leave the caller to guess which member, which is
|
| 2219 |
# the same class of unfalsifiable answer the channel exists to end.
|
| 2220 |
_before = len(ctx.out.refusals)
|
| 2221 |
+
try:
|
| 2222 |
+
rerender = grid_events.handle_one(one, ctx)
|
| 2223 |
+
except view_audit.ViewRevisionStale:
|
| 2224 |
+
_settle_refused_view_write(ctx, one)
|
| 2225 |
+
raise
|
| 2226 |
row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid,
|
| 2227 |
"rerender": bool(rerender)}
|
| 2228 |
mine = ctx.out.refusals[_before:]
|
api/routes_keychain.py
CHANGED
|
@@ -12,7 +12,7 @@ the keychain cutover wave R3 staged.
|
|
| 12 |
import os
|
| 13 |
import time
|
| 14 |
|
| 15 |
-
from fastapi import Body, Depends
|
| 16 |
from fastapi import APIRouter
|
| 17 |
|
| 18 |
from deps import Session, err, require_session
|
|
@@ -1066,6 +1066,8 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 1066 |
raise err(503, "store_unavailable",
|
| 1067 |
"the key was not saved; its sharing setting could not be stored, so "
|
| 1068 |
"nothing was kept. Try again.")
|
|
|
|
|
|
|
| 1069 |
return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}, **lifecycle}
|
| 1070 |
|
| 1071 |
|
|
@@ -1275,8 +1277,18 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
|
|
| 1275 |
|
| 1276 |
@router.post("/admin/keychain/{entry_id}/test")
|
| 1277 |
def test_key(entry_id: str, session: Session = Depends(require_session)):
|
| 1278 |
-
_own_row(session, entry_id)
|
| 1279 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1280 |
|
| 1281 |
|
| 1282 |
def _unsynced_customer_records(session):
|
|
@@ -1486,6 +1498,144 @@ def _odoo_admin(session):
|
|
| 1486 |
"configure it")
|
| 1487 |
|
| 1488 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1489 |
@router.get("/admin/connectors/odoo/config")
|
| 1490 |
def odoo_config(session: Session = Depends(require_session)):
|
| 1491 |
"""Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a
|
|
@@ -1495,7 +1645,16 @@ def odoo_config(session: Session = Depends(require_session)):
|
|
| 1495 |
entry = _odoo_entry(session)
|
| 1496 |
server_db, server_url, api_user, editable = _odoo_source_fields(session, entry)
|
| 1497 |
cfg = rel.read_config(session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1498 |
return {
|
|
|
|
|
|
|
|
|
|
| 1499 |
"applicable": bool(rel.is_royal(session.tenant)),
|
| 1500 |
"source": "keychain" if entry else ("env" if env_odoo_available(session.runtime)
|
| 1501 |
else "none"),
|
|
@@ -1520,6 +1679,74 @@ def odoo_config(session: Session = Depends(require_session)):
|
|
| 1520 |
}
|
| 1521 |
|
| 1522 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1523 |
@router.put("/admin/connectors/odoo/config")
|
| 1524 |
def odoo_config_put(body: dict = Body(default=None),
|
| 1525 |
session: Session = Depends(require_session)):
|
|
@@ -1584,12 +1811,22 @@ def odoo_config_put(body: dict = Body(default=None),
|
|
| 1584 |
raise err(400, "missing_sync_tz",
|
| 1585 |
"choose a time zone for the daily sync time")
|
| 1586 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1587 |
server_db = body.get("serverDb")
|
| 1588 |
if server_db is not None:
|
| 1589 |
server_db = " ".join(str(server_db).split())[:80]
|
| 1590 |
|
| 1591 |
def _up(cur):
|
| 1592 |
cur = cur if isinstance(cur, dict) else {}
|
|
|
|
|
|
|
|
|
|
| 1593 |
if clean_grids is not None:
|
| 1594 |
cur.setdefault("grids", {}).update(clean_grids)
|
| 1595 |
if clean_every is not None:
|
|
@@ -1944,6 +2181,48 @@ def pause_connector(key: str, body: dict = Body(default=None),
|
|
| 1944 |
return {"key": key, "paused": paused, "snapshots": snapshots}
|
| 1945 |
|
| 1946 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1947 |
from routes_amazon_deletion import router as amazon_deletion_router # noqa: E402
|
| 1948 |
|
| 1949 |
router.include_router(amazon_deletion_router)
|
|
|
|
| 12 |
import os
|
| 13 |
import time
|
| 14 |
|
| 15 |
+
from fastapi import Body, Depends, HTTPException
|
| 16 |
from fastapi import APIRouter
|
| 17 |
|
| 18 |
from deps import Session, err, require_session
|
|
|
|
| 1066 |
raise err(503, "store_unavailable",
|
| 1067 |
"the key was not saved; its sharing setting could not be stored, so "
|
| 1068 |
"nothing was kept. Try again.")
|
| 1069 |
+
if etype == "odoo":
|
| 1070 |
+
_business_tz_on_connect(session) # S82-T107: default the zone on connect
|
| 1071 |
return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}, **lifecycle}
|
| 1072 |
|
| 1073 |
|
|
|
|
| 1277 |
|
| 1278 |
@router.post("/admin/keychain/{entry_id}/test")
|
| 1279 |
def test_key(entry_id: str, session: Session = Depends(require_session)):
|
| 1280 |
+
row = _own_row(session, entry_id) # 404 for an entry this caller may not see
|
| 1281 |
+
result = _kc().test_entry(session.runtime, entry_id)
|
| 1282 |
+
# S82-T107: a passing test of the connection that serves this tenant re-defaults the zone.
|
| 1283 |
+
if (isinstance(result, dict) and result.get("ok") and str(row.get("type") or "") == "odoo"
|
| 1284 |
+
and session.admin):
|
| 1285 |
+
try:
|
| 1286 |
+
serving = _odoo_entry(session)
|
| 1287 |
+
except Exception: # noqa: BLE001
|
| 1288 |
+
serving = None
|
| 1289 |
+
if serving is not None and str(serving.get("id")) == str(entry_id):
|
| 1290 |
+
_business_tz_on_connect(session)
|
| 1291 |
+
return result
|
| 1292 |
|
| 1293 |
|
| 1294 |
def _unsynced_customer_records(session):
|
|
|
|
| 1498 |
"configure it")
|
| 1499 |
|
| 1500 |
|
| 1501 |
+
# ═════════════════════════════════ S82-T107: THE TENANT'S BUSINESS TIMEZONE (`businessTz`, C10)
|
| 1502 |
+
#
|
| 1503 |
+
# One zone per tenant, stored beside the sync slot in the Odoo connector config. The business day
|
| 1504 |
+
# reads it. It is NEVER `syncTz`: that zone says when the sync runs, not where the business is.
|
| 1505 |
+
# It defaults from the connected Odoo user's own `tz` on connect, on test and on the first admin
|
| 1506 |
+
# config read. A blank Odoo `tz` leaves it unset, and the panel asks the admin to pick one. An
|
| 1507 |
+
# admin's pick wins over every later Odoo default.
|
| 1508 |
+
BUSINESS_TZ_KEY = "businessTz"
|
| 1509 |
+
#: Where the stored zone came from: "odoo", "admin", or "odoo-blank" (Odoo was asked, said nothing).
|
| 1510 |
+
BUSINESS_TZ_SOURCE_KEY = "businessTzSource"
|
| 1511 |
+
|
| 1512 |
+
|
| 1513 |
+
def _zone_ok(name):
|
| 1514 |
+
"""True when `name` is an IANA zone this process can load."""
|
| 1515 |
+
if not isinstance(name, str) or not name.strip():
|
| 1516 |
+
return False
|
| 1517 |
+
try:
|
| 1518 |
+
from zoneinfo import ZoneInfo
|
| 1519 |
+
ZoneInfo(name.strip())
|
| 1520 |
+
return True
|
| 1521 |
+
except Exception: # noqa: BLE001
|
| 1522 |
+
return False
|
| 1523 |
+
|
| 1524 |
+
|
| 1525 |
+
def _odoo_cfg(rt):
|
| 1526 |
+
try:
|
| 1527 |
+
cur = rt.get(_rel().CONFIG_KEY) if rt is not None else None
|
| 1528 |
+
except Exception: # noqa: BLE001
|
| 1529 |
+
cur = None
|
| 1530 |
+
return cur if isinstance(cur, dict) else {}
|
| 1531 |
+
|
| 1532 |
+
|
| 1533 |
+
def stored_business_tz(rt):
|
| 1534 |
+
"""The tenant's stored business timezone, or None. Reads `businessTz` only, never `syncTz`."""
|
| 1535 |
+
name = str(_odoo_cfg(rt).get(BUSINESS_TZ_KEY) or "").strip()
|
| 1536 |
+
return name if _zone_ok(name) else None
|
| 1537 |
+
|
| 1538 |
+
|
| 1539 |
+
def odoo_user_tz(creds, timeout=8):
|
| 1540 |
+
"""The connected Odoo user's `tz` from `res.users`.
|
| 1541 |
+
|
| 1542 |
+
Returns the zone name, "" when Odoo holds none (or one this process cannot load), and None when
|
| 1543 |
+
Odoo could not be asked. One admin act over read-only XML-RPC: `authenticate`, then one `read`
|
| 1544 |
+
of the user's own row. It goes outside every limiter and the shared client on purpose."""
|
| 1545 |
+
import xmlrpc.client
|
| 1546 |
+
creds = creds or {}
|
| 1547 |
+
url = str(creds.get("url") or "").strip().rstrip("/")
|
| 1548 |
+
db, user, key = (str(creds.get(k) or "") for k in ("db", "user", "api_key"))
|
| 1549 |
+
if not (url.startswith(("http://", "https://")) and db and user and key):
|
| 1550 |
+
return None
|
| 1551 |
+
base = xmlrpc.client.SafeTransport if url.startswith("https") else xmlrpc.client.Transport
|
| 1552 |
+
|
| 1553 |
+
class _Timed(base):
|
| 1554 |
+
def make_connection(self, host):
|
| 1555 |
+
conn = super().make_connection(host)
|
| 1556 |
+
conn.timeout = timeout
|
| 1557 |
+
return conn
|
| 1558 |
+
|
| 1559 |
+
try:
|
| 1560 |
+
common = xmlrpc.client.ServerProxy(f"{url}/xmlrpc/2/common", transport=_Timed(),
|
| 1561 |
+
allow_none=True)
|
| 1562 |
+
uid = common.authenticate(db, user, key, {})
|
| 1563 |
+
if not uid:
|
| 1564 |
+
return None
|
| 1565 |
+
models = xmlrpc.client.ServerProxy(f"{url}/xmlrpc/2/object", transport=_Timed(),
|
| 1566 |
+
allow_none=True)
|
| 1567 |
+
rows = models.execute_kw(db, uid, key, "res.users", "read", [[uid]], {"fields": ["tz"]})
|
| 1568 |
+
except Exception: # noqa: BLE001
|
| 1569 |
+
return None
|
| 1570 |
+
first = rows[0] if isinstance(rows, list) and rows and isinstance(rows[0], dict) else {}
|
| 1571 |
+
zone = str(first.get("tz") or "").strip() # Odoo sends False for a blank field
|
| 1572 |
+
return zone if _zone_ok(zone) else ""
|
| 1573 |
+
|
| 1574 |
+
|
| 1575 |
+
def _odoo_tz_creds(session, entry):
|
| 1576 |
+
"""The fields of the connection serving this tenant's Odoo, or None. Never returned to a client."""
|
| 1577 |
+
if entry is None:
|
| 1578 |
+
if not env_odoo_available(session.runtime):
|
| 1579 |
+
return None
|
| 1580 |
+
got, _why = _env_odoo_fields()
|
| 1581 |
+
return got
|
| 1582 |
+
try:
|
| 1583 |
+
return _kc().read_fields(session.runtime, entry["id"]) or None
|
| 1584 |
+
except Exception: # noqa: BLE001
|
| 1585 |
+
return None
|
| 1586 |
+
|
| 1587 |
+
|
| 1588 |
+
def default_business_tz(session, entry=None, refresh=False):
|
| 1589 |
+
"""Default `businessTz` from the connected Odoo user's `tz`, and return the zone in force.
|
| 1590 |
+
|
| 1591 |
+
`refresh=False` (the admin config read) asks Odoo only when no zone is stored and Odoo was not
|
| 1592 |
+
already asked. `refresh=True` (connect, test) asks again. An admin's pick is never replaced, a
|
| 1593 |
+
blank answer never clears a stored zone, and the default is never `syncTz` nor a literal."""
|
| 1594 |
+
rt = session.runtime
|
| 1595 |
+
cur = _odoo_cfg(rt)
|
| 1596 |
+
have = stored_business_tz(rt)
|
| 1597 |
+
source = str(cur.get(BUSINESS_TZ_SOURCE_KEY) or "")
|
| 1598 |
+
if have and source == "admin":
|
| 1599 |
+
return have
|
| 1600 |
+
if not refresh and (have or source == "odoo-blank"):
|
| 1601 |
+
return have
|
| 1602 |
+
creds = _odoo_tz_creds(session, entry)
|
| 1603 |
+
if not creds:
|
| 1604 |
+
return have
|
| 1605 |
+
zone = odoo_user_tz(creds)
|
| 1606 |
+
if zone is None or (not zone and have):
|
| 1607 |
+
return have
|
| 1608 |
+
|
| 1609 |
+
def _up(c):
|
| 1610 |
+
c = c if isinstance(c, dict) else {}
|
| 1611 |
+
if str(c.get(BUSINESS_TZ_SOURCE_KEY) or "") == "admin" and _zone_ok(c.get(BUSINESS_TZ_KEY)):
|
| 1612 |
+
return c # an admin picked a zone meanwhile
|
| 1613 |
+
if zone:
|
| 1614 |
+
c[BUSINESS_TZ_KEY] = zone
|
| 1615 |
+
c[BUSINESS_TZ_SOURCE_KEY] = "odoo"
|
| 1616 |
+
else:
|
| 1617 |
+
c.pop(BUSINESS_TZ_KEY, None)
|
| 1618 |
+
c[BUSINESS_TZ_SOURCE_KEY] = "odoo-blank"
|
| 1619 |
+
return c
|
| 1620 |
+
|
| 1621 |
+
try:
|
| 1622 |
+
rt.update(_rel().CONFIG_KEY, _up, flush="sync")
|
| 1623 |
+
except Exception: # noqa: BLE001
|
| 1624 |
+
return have
|
| 1625 |
+
return stored_business_tz(rt)
|
| 1626 |
+
|
| 1627 |
+
|
| 1628 |
+
def _business_tz_on_connect(session):
|
| 1629 |
+
"""Connect and test: re-default the zone from the serving Odoo user. Never raises."""
|
| 1630 |
+
try:
|
| 1631 |
+
entry = _odoo_entry(session)
|
| 1632 |
+
if entry is None and not env_odoo_available(session.runtime):
|
| 1633 |
+
return None
|
| 1634 |
+
return default_business_tz(session, entry, refresh=True)
|
| 1635 |
+
except Exception: # noqa: BLE001
|
| 1636 |
+
return None
|
| 1637 |
+
|
| 1638 |
+
|
| 1639 |
@router.get("/admin/connectors/odoo/config")
|
| 1640 |
def odoo_config(session: Session = Depends(require_session)):
|
| 1641 |
"""Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a
|
|
|
|
| 1645 |
entry = _odoo_entry(session)
|
| 1646 |
server_db, server_url, api_user, editable = _odoo_source_fields(session, entry)
|
| 1647 |
cfg = rel.read_config(session.runtime)
|
| 1648 |
+
# S82-T107: the first admin read of a connection with no stored zone takes the Odoo user's tz.
|
| 1649 |
+
business_tz = cfg.get(BUSINESS_TZ_KEY) if _zone_ok(cfg.get(BUSINESS_TZ_KEY)) else None
|
| 1650 |
+
if business_tz is None:
|
| 1651 |
+
live = not cfg["frozen"] and (entry is not None or env_odoo_available(session.runtime))
|
| 1652 |
+
business_tz = (default_business_tz(session, entry) if live
|
| 1653 |
+
else stored_business_tz(session.runtime))
|
| 1654 |
return {
|
| 1655 |
+
"businessTz": business_tz,
|
| 1656 |
+
"businessTzSource": str(_odoo_cfg(session.runtime).get(BUSINESS_TZ_SOURCE_KEY) or "")
|
| 1657 |
+
if business_tz else "",
|
| 1658 |
"applicable": bool(rel.is_royal(session.tenant)),
|
| 1659 |
"source": "keychain" if entry else ("env" if env_odoo_available(session.runtime)
|
| 1660 |
else "none"),
|
|
|
|
| 1679 |
}
|
| 1680 |
|
| 1681 |
|
| 1682 |
+
def _odoo_field_name_tables(session):
|
| 1683 |
+
"""Read only this connector's visible database definitions in the session's tenant."""
|
| 1684 |
+
from core import perm_scope, user_tables
|
| 1685 |
+
tables = {}
|
| 1686 |
+
for grid in _rel().grid_choices(session.runtime):
|
| 1687 |
+
key = grid["key"]
|
| 1688 |
+
if not perm_scope.may_read(session.user, key, st=session.runtime):
|
| 1689 |
+
continue
|
| 1690 |
+
definition = user_tables.get_one(key, st=session.runtime)
|
| 1691 |
+
if definition is not None:
|
| 1692 |
+
tables[key] = (definition, str(definition.get("name") or grid["label"]))
|
| 1693 |
+
return tables
|
| 1694 |
+
|
| 1695 |
+
|
| 1696 |
+
@router.get("/admin/connectors/odoo/field-names")
|
| 1697 |
+
def odoo_field_names(session: Session = Depends(require_session)):
|
| 1698 |
+
"""List the connector fields this administrator may rename without changing their definitions."""
|
| 1699 |
+
_odoo_admin(session)
|
| 1700 |
+
from core import perm_scope, user_tables
|
| 1701 |
+
try:
|
| 1702 |
+
fields = []
|
| 1703 |
+
for key, (definition, name) in _odoo_field_name_tables(session).items():
|
| 1704 |
+
hidden = perm_scope.hidden_keys(session.user, key, definition.get("fields") or [],
|
| 1705 |
+
st=session.runtime)
|
| 1706 |
+
for field in user_tables.renamable_fields([key], st=session.runtime):
|
| 1707 |
+
if field["key"] not in hidden and user_tables.may_rename_field(
|
| 1708 |
+
key, field["key"], session.uname, session.admin, st=session.runtime):
|
| 1709 |
+
fields.append({**field, "tableLabel": name})
|
| 1710 |
+
return {"fields": fields}
|
| 1711 |
+
except HTTPException:
|
| 1712 |
+
raise
|
| 1713 |
+
except Exception:
|
| 1714 |
+
raise err(503, "store_unavailable", "Field names could not be read. Try again.") from None
|
| 1715 |
+
|
| 1716 |
+
|
| 1717 |
+
@router.put("/admin/connectors/odoo/field-names")
|
| 1718 |
+
def odoo_field_names_put(body: dict = Body(default=None),
|
| 1719 |
+
session: Session = Depends(require_session)):
|
| 1720 |
+
"""Rename one connector field through the shared label-only authority."""
|
| 1721 |
+
_odoo_admin(session)
|
| 1722 |
+
from core import perm_scope, user_tables
|
| 1723 |
+
body = body or {}
|
| 1724 |
+
if not all(isinstance(body.get(key), str) and body[key].strip() for key in ("table", "key", "label")):
|
| 1725 |
+
raise err(400, "invalid_field_name", "Choose a database, column and nonblank field name.")
|
| 1726 |
+
table_key, fkey = body["table"], body["key"]
|
| 1727 |
+
try:
|
| 1728 |
+
tables = _odoo_field_name_tables(session)
|
| 1729 |
+
if table_key not in tables:
|
| 1730 |
+
raise err(404, "not_found", "This connector has no database by that name you may edit.")
|
| 1731 |
+
definition, name = tables[table_key]
|
| 1732 |
+
if fkey in perm_scope.hidden_keys(session.user, table_key, definition.get("fields") or [],
|
| 1733 |
+
st=session.runtime):
|
| 1734 |
+
raise err(403, "forbidden", "this database has no column by that name that you may edit")
|
| 1735 |
+
raw = {key: value for key, value in body.items() if key not in ("table", "key")}
|
| 1736 |
+
if not user_tables.may_rename_field(table_key, fkey, session.uname, session.admin,
|
| 1737 |
+
st=session.runtime, raw=raw):
|
| 1738 |
+
raise err(403, "preset_field_locked", "Only the name of a renamable pre-set column may change here.")
|
| 1739 |
+
saved = user_tables.rename_field(table_key, fkey, body["label"], session.uname,
|
| 1740 |
+
session.admin, st=session.runtime, by=session.uname, via="web")
|
| 1741 |
+
if saved is None:
|
| 1742 |
+
raise err(503, "store_unavailable", "The field name was not saved. Try again.")
|
| 1743 |
+
return {"field": {"table": table_key, "key": fkey, "label": saved["label"], "tableLabel": name}}
|
| 1744 |
+
except HTTPException:
|
| 1745 |
+
raise
|
| 1746 |
+
except Exception:
|
| 1747 |
+
raise err(503, "store_unavailable", "The field name was not saved. Try again.") from None
|
| 1748 |
+
|
| 1749 |
+
|
| 1750 |
@router.put("/admin/connectors/odoo/config")
|
| 1751 |
def odoo_config_put(body: dict = Body(default=None),
|
| 1752 |
session: Session = Depends(require_session)):
|
|
|
|
| 1811 |
raise err(400, "missing_sync_tz",
|
| 1812 |
"choose a time zone for the daily sync time")
|
| 1813 |
|
| 1814 |
+
# S82-T107: the business timezone. An admin's pick, kept apart from the sync slot zone.
|
| 1815 |
+
clean_business_tz = None
|
| 1816 |
+
if BUSINESS_TZ_KEY in body:
|
| 1817 |
+
clean_business_tz = str(body.get(BUSINESS_TZ_KEY) or "").strip()
|
| 1818 |
+
if not _zone_ok(clean_business_tz):
|
| 1819 |
+
raise err(400, "invalid_business_tz", "choose a business timezone from the list")
|
| 1820 |
+
|
| 1821 |
server_db = body.get("serverDb")
|
| 1822 |
if server_db is not None:
|
| 1823 |
server_db = " ".join(str(server_db).split())[:80]
|
| 1824 |
|
| 1825 |
def _up(cur):
|
| 1826 |
cur = cur if isinstance(cur, dict) else {}
|
| 1827 |
+
if clean_business_tz:
|
| 1828 |
+
cur[BUSINESS_TZ_KEY] = clean_business_tz
|
| 1829 |
+
cur[BUSINESS_TZ_SOURCE_KEY] = "admin"
|
| 1830 |
if clean_grids is not None:
|
| 1831 |
cur.setdefault("grids", {}).update(clean_grids)
|
| 1832 |
if clean_every is not None:
|
|
|
|
| 2181 |
return {"key": key, "paused": paused, "snapshots": snapshots}
|
| 2182 |
|
| 2183 |
|
| 2184 |
+
def _live_limits_admin(session):
|
| 2185 |
+
"""S82-T101: live call limits are workspace wide, so only an administrator reads or sets them."""
|
| 2186 |
+
if not session.admin:
|
| 2187 |
+
raise err(403, "not_admin", "only an administrator can see or change live call limits")
|
| 2188 |
+
|
| 2189 |
+
|
| 2190 |
+
def _live_limits_payload(limits):
|
| 2191 |
+
from core import live_limiter
|
| 2192 |
+
return {**limits, "defaults": live_limiter.defaults(),
|
| 2193 |
+
"callsPerMinuteMin": live_limiter.CALLS_PER_MINUTE_MIN,
|
| 2194 |
+
"callsPerMinuteMax": live_limiter.CALLS_PER_MINUTE_MAX}
|
| 2195 |
+
|
| 2196 |
+
|
| 2197 |
+
@router.get("/admin/connectors/live-limits")
|
| 2198 |
+
def live_limits(session: Session = Depends(require_session)):
|
| 2199 |
+
"""S82-T101 (R65): the tenant's AI and metric live call limits."""
|
| 2200 |
+
_live_limits_admin(session)
|
| 2201 |
+
from core import live_limiter
|
| 2202 |
+
try:
|
| 2203 |
+
limits = live_limiter.read_limits(session.runtime)
|
| 2204 |
+
except Exception:
|
| 2205 |
+
raise err(503, "store_unavailable", "the live call limits could not be read; try again")
|
| 2206 |
+
return _live_limits_payload(limits)
|
| 2207 |
+
|
| 2208 |
+
|
| 2209 |
+
@router.put("/admin/connectors/live-limits")
|
| 2210 |
+
def live_limits_put(body: dict = Body(default=None),
|
| 2211 |
+
session: Session = Depends(require_session)):
|
| 2212 |
+
"""S82-T101 (R65): set calls per minute, one in flight, and max rows per call (null is no limit)."""
|
| 2213 |
+
_live_limits_admin(session)
|
| 2214 |
+
from core import live_limiter
|
| 2215 |
+
clean, errors = live_limiter.validate_limits(body if body is not None else {})
|
| 2216 |
+
if errors:
|
| 2217 |
+
raise err(400, "invalid_live_limits", "some live call limits are not valid",
|
| 2218 |
+
details={"fields": errors})
|
| 2219 |
+
try:
|
| 2220 |
+
limits = live_limiter.write_limits(session.runtime, clean, by=session.uname)
|
| 2221 |
+
except Exception:
|
| 2222 |
+
raise err(503, "store_unavailable", "the change was not saved; try again")
|
| 2223 |
+
return _live_limits_payload(limits)
|
| 2224 |
+
|
| 2225 |
+
|
| 2226 |
from routes_amazon_deletion import router as amazon_deletion_router # noqa: E402
|
| 2227 |
|
| 2228 |
router.include_router(amazon_deletion_router)
|
api/routes_nav.py
CHANGED
|
@@ -596,18 +596,18 @@ def _granted_view_ids(session):
|
|
| 596 |
#: ⚠ COUNTED ON THE CONNECTIONS, NOT ON THE USABLE LABELS. A second connection whose label is one
|
| 597 |
#: of the reserved words below is still a second connection, and its sibling must still be named.
|
| 598 |
#:
|
| 599 |
-
#: These
|
| 600 |
#: `dataPath.ts::dataDatabaseOriginKey` folds `custom` / `loopable` / `blank` / `automation` /
|
| 601 |
-
#: empty into the one
|
| 602 |
#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
|
| 603 |
#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
|
| 604 |
-
_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
|
| 605 |
|
| 606 |
-
#: S79-T60 (QI27): the stable key of the native origin group. The client shows it as "
|
| 607 |
#: no logic compares that label. Must equal `dataPath.ts::CUSTOM_ORIGIN_KEY`.
|
| 608 |
CUSTOM_SOURCE_GROUP = "custom"
|
| 609 |
-
#: Native provenance tokens
|
| 610 |
-
_NATIVE_GROUP_TOKENS = frozenset({"", CUSTOM_SOURCE_GROUP, "loopable", "blank", "automation"})
|
| 611 |
|
| 612 |
|
| 613 |
def _native_group(group):
|
|
@@ -1165,7 +1165,7 @@ def _user_pref_now():
|
|
| 1165 |
|
| 1166 |
@router.get("/nav/prefs/{key}")
|
| 1167 |
def get_user_pref(key: str, session: Session = Depends(require_session)):
|
| 1168 |
-
"""C5: this user's stored value for one preference key
|
| 1169 |
key = _user_pref_key(key)
|
| 1170 |
try:
|
| 1171 |
bucket = (session.runtime.get(_NAV_USER_PREFS_KEY) or {}).get(session.uname) or {}
|
|
@@ -1173,7 +1173,7 @@ def get_user_pref(key: str, session: Session = Depends(require_session)):
|
|
| 1173 |
raise err(503, "store_unavailable", "Your saved preferences could not be loaded.")
|
| 1174 |
row = bucket.get(key) if isinstance(bucket, dict) else None
|
| 1175 |
if not isinstance(row, dict) or "value" not in row:
|
| 1176 |
-
|
| 1177 |
return {"key": key, "value": row.get("value"), "updated_at": row.get("updated_at")}
|
| 1178 |
|
| 1179 |
|
|
@@ -1427,6 +1427,10 @@ def nav_schema(key: str, session: Session = Depends(require_session)):
|
|
| 1427 |
"means": means.get(f["key"], "")}
|
| 1428 |
if f.get("options"):
|
| 1429 |
entry["options"] = list(f["options"])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1430 |
fields.append(entry)
|
| 1431 |
except Exception:
|
| 1432 |
fields = []
|
|
|
|
| 596 |
#: ⚠ COUNTED ON THE CONNECTIONS, NOT ON THE USABLE LABELS. A second connection whose label is one
|
| 597 |
#: of the reserved words below is still a second connection, and its sibling must still be named.
|
| 598 |
#:
|
| 599 |
+
#: These reserved words are the rail's OWN vocabulary, not a connection's to take:
|
| 600 |
#: `dataPath.ts::dataDatabaseOriginKey` folds `custom` / `loopable` / `blank` / `automation` /
|
| 601 |
+
#: `system` / empty into the one System group (S79-T60, key `CUSTOM_SOURCE_GROUP`). A connection someone named
|
| 602 |
#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
|
| 603 |
#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
|
| 604 |
+
_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "system", "odoo"})
|
| 605 |
|
| 606 |
+
#: S79-T60 (QI27): the stable key of the native origin group. The client shows it as "System";
|
| 607 |
#: no logic compares that label. Must equal `dataPath.ts::CUSTOM_ORIGIN_KEY`.
|
| 608 |
CUSTOM_SOURCE_GROUP = "custom"
|
| 609 |
+
#: Native provenance tokens fold into the key. Must equal `dataPath.ts::CUSTOM_SOURCE_TOKENS`.
|
| 610 |
+
_NATIVE_GROUP_TOKENS = frozenset({"", CUSTOM_SOURCE_GROUP, "loopable", "blank", "automation", "system"})
|
| 611 |
|
| 612 |
|
| 613 |
def _native_group(group):
|
|
|
|
| 1165 |
|
| 1166 |
@router.get("/nav/prefs/{key}")
|
| 1167 |
def get_user_pref(key: str, session: Session = Depends(require_session)):
|
| 1168 |
+
"""C5: this user's stored value for one preference key; an unset key answers 200 with null."""
|
| 1169 |
key = _user_pref_key(key)
|
| 1170 |
try:
|
| 1171 |
bucket = (session.runtime.get(_NAV_USER_PREFS_KEY) or {}).get(session.uname) or {}
|
|
|
|
| 1173 |
raise err(503, "store_unavailable", "Your saved preferences could not be loaded.")
|
| 1174 |
row = bucket.get(key) if isinstance(bucket, dict) else None
|
| 1175 |
if not isinstance(row, dict) or "value" not in row:
|
| 1176 |
+
return {"key": key, "value": None, "updated_at": None}
|
| 1177 |
return {"key": key, "value": row.get("value"), "updated_at": row.get("updated_at")}
|
| 1178 |
|
| 1179 |
|
|
|
|
| 1427 |
"means": means.get(f["key"], "")}
|
| 1428 |
if f.get("options"):
|
| 1429 |
entry["options"] = list(f["options"])
|
| 1430 |
+
elif f["key"] == "dba":
|
| 1431 |
+
# The dba vocabulary is the tenant's teams plus one multi-team value, never a contract list.
|
| 1432 |
+
from modules.customer_data import dba_options
|
| 1433 |
+
entry["options"] = dba_options()
|
| 1434 |
fields.append(entry)
|
| 1435 |
except Exception:
|
| 1436 |
fields = []
|
api/routes_odoo_tables.py
CHANGED
|
@@ -929,6 +929,7 @@ def _odoo_sources():
|
|
| 929 |
"id": "so.id",
|
| 930 |
"needs_excluded": True, # `wholesale_scope` is resolved against the excluded set
|
| 931 |
"default_wholesale_scope": True,
|
|
|
|
| 932 |
"cols": {
|
| 933 |
"order_no": ("so.name", _s),
|
| 934 |
"odoo_id": ("so.id", _i),
|
|
@@ -990,6 +991,7 @@ def _odoo_sources():
|
|
| 990 |
"id": "sol.id",
|
| 991 |
"needs_excluded": True,
|
| 992 |
"default_wholesale_scope": True,
|
|
|
|
| 993 |
"cols": {
|
| 994 |
"odoo_id": ("sol.id", _i),
|
| 995 |
"order_no": ("so.name", _s),
|
|
@@ -1259,6 +1261,7 @@ def _odoo_sources():
|
|
| 1259 |
},
|
| 1260 |
}
|
| 1261 |
GRID_SOURCES.update(rel.stock_source_specs())
|
|
|
|
| 1262 |
GRID_SOURCES.update(rel.purchase_source_specs())
|
| 1263 |
GRID_SOURCES.update(rel.payment_source_specs())
|
| 1264 |
GRID_SOURCES[rel.SHIPMENTS_KEY] = rel.shipment_source_spec()
|
|
@@ -1658,6 +1661,11 @@ def _source_for(cur, table_key, *, allow_lookups=True):
|
|
| 1658 |
import amazon_relational as amazon
|
| 1659 |
if key in amazon.SOURCE:
|
| 1660 |
return _amazon_page_source(cur, key, binding=True, allow_lookups=allow_lookups)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1661 |
spec = _sources().get(key)
|
| 1662 |
if not spec:
|
| 1663 |
return None
|
|
@@ -1717,6 +1725,10 @@ def _source_for(cur, table_key, *, allow_lookups=True):
|
|
| 1717 |
expr = spec['id']
|
| 1718 |
cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr,
|
| 1719 |
have_by_alias), cast)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1720 |
if str(table_key).startswith("ut_odoo_") and spec.get("table") and not spec.get("from_sql"):
|
| 1721 |
spec = {**spec, "page_key": spec["id"]}
|
| 1722 |
return {**spec, "cols": cols}
|
|
@@ -1898,11 +1910,76 @@ def _filter_columns(spec, fields):
|
|
| 1898 |
continue
|
| 1899 |
ftype = str((by_key.get(key) or {}).get("type") or "text")
|
| 1900 |
out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False}
|
|
|
|
|
|
|
| 1901 |
if key == "customer_type" and (by_key.get(key) or {}).get("source") == "odoo":
|
| 1902 |
out[key]["json_tags"] = True
|
| 1903 |
return out
|
| 1904 |
|
| 1905 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1906 |
def _leaf_cols(nodes):
|
| 1907 |
"""Every `colId` a filter tree names, at any depth."""
|
| 1908 |
seen = set()
|
|
@@ -4015,7 +4092,8 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 4015 |
try:
|
| 4016 |
cur, mismatch = (_mirror_cur(session, table_key)
|
| 4017 |
if (table_key.startswith(("ut_amzn_", "ut_meta_"))
|
| 4018 |
-
or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY
|
|
|
|
| 4019 |
else _mirror_cur(session))
|
| 4020 |
except RuntimeError as e:
|
| 4021 |
# ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the
|
|
@@ -4058,6 +4136,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 4058 |
fields = list(declared.values())
|
| 4059 |
spec = _if_read_spec(spec, fields, table_key, st)
|
| 4060 |
cols = _filter_columns(spec, fields)
|
|
|
|
| 4061 |
|
| 4062 |
# ── THE PERMANENT WALL (W36-T21 / R6), COMPILED FIRST AND FAIL-CLOSED ────────────────────
|
| 4063 |
# An administrator's row filter is a `FilterTree` in the SAME vocabulary the client sends, so
|
|
@@ -4273,7 +4352,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 4273 |
where = f"({where}) AND {pred.sql}" if where else pred.sql
|
| 4274 |
params.extend(pred.params)
|
| 4275 |
if str(search or "").strip():
|
| 4276 |
-
got =
|
| 4277 |
if got is not None:
|
| 4278 |
where = f"({where}) AND {got.sql}" if where else got.sql
|
| 4279 |
params.extend(got.params)
|
|
@@ -4288,7 +4367,8 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 4288 |
|
| 4289 |
|
| 4290 |
@contextmanager
|
| 4291 |
-
def _materialised_read(table_key, session, query, related, ids=None, *, payload=None, needed=None
|
|
|
|
| 4292 |
"""One compiler for native rows and their aggregate sibling, over the walled payload.
|
| 4293 |
|
| 4294 |
The temporary connection contains only this request's already stripped, pinned rows and is
|
|
@@ -4334,6 +4414,36 @@ def _materialised_read(table_key, session, query, related, ids=None, *, payload=
|
|
| 4334 |
else:
|
| 4335 |
connection = datastore.bounded_connection(duckdb.connect(":memory:"))
|
| 4336 |
with connection as cur:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4337 |
cur.execute("CREATE TEMP TABLE pinned_rows (_row_index BIGINT, payload JSON)")
|
| 4338 |
wanted = set(ids) if ids is not None else None
|
| 4339 |
if streamed:
|
|
@@ -4386,6 +4496,7 @@ def _materialised_read(table_key, session, query, related, ids=None, *, payload=
|
|
| 4386 |
spec = {"table": "pinned_rows", "id": "CAST(json_extract_string(payload, '/pid') AS BIGINT)",
|
| 4387 |
"cols": bindings, "where": None}
|
| 4388 |
cols = _filter_columns(spec, fields)
|
|
|
|
| 4389 |
limits = list(payload.get("limits") or [])
|
| 4390 |
_had_cohort = bool(_active_cohort_leaves(query["filters"]))
|
| 4391 |
tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
|
|
@@ -4405,7 +4516,7 @@ def _materialised_read(table_key, session, query, related, ids=None, *, payload=
|
|
| 4405 |
if pred is not None and pred.uses_aggregate:
|
| 4406 |
raise err(400, "filter_unsupported", "This condition cannot be evaluated for individual rows.")
|
| 4407 |
where, params = (pred.sql, list(pred.params)) if pred is not None else (None, [])
|
| 4408 |
-
search =
|
| 4409 |
if search is not None:
|
| 4410 |
where = f"({where}) AND {search.sql}" if where else search.sql
|
| 4411 |
params.extend(search.params)
|
|
@@ -4439,7 +4550,10 @@ def _group_pin(cols, group_by, group_key):
|
|
| 4439 |
key = json.loads(group_key) if group_key not in (None, "") else None
|
| 4440 |
except ValueError:
|
| 4441 |
raise err(400, "bad_group", "groupKey must be the JSON of a group key")
|
| 4442 |
-
gexpr =
|
|
|
|
|
|
|
|
|
|
| 4443 |
if key is None:
|
| 4444 |
return f"({gexpr}) IS NULL", []
|
| 4445 |
if gspec["type"] in _fs().NUMERIC_TYPES:
|
|
@@ -4672,15 +4786,16 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
|
|
| 4672 |
needed.add(group_field)
|
| 4673 |
if query['search'] or _leaf_cols(query['filters']) & _PSEUDO_COLS:
|
| 4674 |
needed = None
|
| 4675 |
-
with _materialised_read(table_key, session, query, related, ids=ids, payload=payload,
|
|
|
|
| 4676 |
spec = read["spec"]
|
| 4677 |
-
order =
|
| 4678 |
where, params = read["where"], list(read["params"])
|
| 4679 |
gsql, gparams = _group_pin(read["cols"], groupBy, groupKey)
|
| 4680 |
if gsql:
|
| 4681 |
where = f"({where}) AND {gsql}" if where else gsql
|
| 4682 |
params.extend(gparams)
|
| 4683 |
-
options = dict(select='_row_index', where=where, params=tuple(params),
|
| 4684 |
order_by=order or spec['id'], offset=window_offset, limit=limit or None,
|
| 4685 |
cur=read['cur'], **read['frm'])
|
| 4686 |
if table_key == 'ut_activity':
|
|
@@ -4689,6 +4804,8 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
|
|
| 4689 |
else:
|
| 4690 |
import registry_source
|
| 4691 |
revision = registry_source.generation(session, table_key)
|
|
|
|
|
|
|
| 4692 |
question = _window_query(table_key, filters, filterConj, sorts, search, related,
|
| 4693 |
groupBy, groupKey, ids, revision=revision)
|
| 4694 |
win = _window_page(session, question, cursor, page_first=True,
|
|
@@ -4696,7 +4813,8 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
|
|
| 4696 |
payload = read["payload"]
|
| 4697 |
indexes = [int(row[0]) for row in win['rows']]
|
| 4698 |
value_sql = read['spec']['id'] if read['streamed'] else 'payload'
|
| 4699 |
-
values = (
|
|
|
|
| 4700 |
'WHERE _row_index IN (SELECT unnest(?::BIGINT[]))', [indexes]).fetchall()
|
| 4701 |
if indexes else [])
|
| 4702 |
if read['streamed']:
|
|
@@ -4705,6 +4823,8 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
|
|
| 4705 |
page_ids = [by_index[index] for index in indexes]
|
| 4706 |
if table_key in {'customer_data', 'product_data'}:
|
| 4707 |
payload = registry_source.canonical_page(session, read['cur'], table_key, page_ids, related=related)
|
|
|
|
|
|
|
| 4708 |
else:
|
| 4709 |
payload = registry_source.stored_page(session, table_key, page_ids, related=related)
|
| 4710 |
rows = payload['rows']
|
|
@@ -4822,8 +4942,9 @@ def odoo_table_rows(table_key: str,
|
|
| 4822 |
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 4823 |
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
| 4824 |
if (admitted is None if page is not None else _materialised_table(table_key)):
|
| 4825 |
-
return
|
| 4826 |
-
|
|
|
|
| 4827 |
|
| 4828 |
# ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
|
| 4829 |
# hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
|
|
@@ -4850,7 +4971,7 @@ def odoo_table_rows(table_key: str,
|
|
| 4850 |
# ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one
|
| 4851 |
# row on two pages and drop another entirely — a duplicate the user sees with no error
|
| 4852 |
# anywhere. `compile_order_by`'s own docstring says so.
|
| 4853 |
-
order =
|
| 4854 |
|
| 4855 |
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
|
| 4856 |
revision = _mirror_page_revision(cur, page_rev)
|
|
@@ -4977,14 +5098,23 @@ def odoo_table_rows(table_key: str,
|
|
| 4977 |
for row in rows:
|
| 4978 |
if str(row["pid"]) in notes:
|
| 4979 |
row[user_tables.CELL_NOTES_KEY] = notes[str(row["pid"])]
|
| 4980 |
-
return {"ok": True, "fields": fields, "rows": rows,
|
| 4981 |
# ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`.
|
| 4982 |
"total": win["total"], "totalUnfiltered": unfiltered,
|
| 4983 |
"offset": win["offset"], "limit": win["limit"], "limits": limits,
|
| 4984 |
"next_cursor": win['next_cursor'],
|
| 4985 |
"today": _today(), "identity": {"pid": "pid"},
|
| 4986 |
"scope": {"table": table_key, "readThrough": True},
|
| 4987 |
-
"recordsMutable": bool(defn.get("recordMode") != _ut().AUTOMATION_RECORD_MODE)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4988 |
|
| 4989 |
|
| 4990 |
# ══════════════════════════════════════════════════════════
|
|
@@ -5297,7 +5427,7 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
|
|
| 5297 |
"recommendation": "group by a column the grid reads from the connection, or ask "
|
| 5298 |
"an administrator for access to this one"})
|
| 5299 |
else:
|
| 5300 |
-
gexpr =
|
| 5301 |
|
| 5302 |
# ⛔⛔ THE OVERALL SUMMARY IS COMPUTED WHETHER OR NOT THIS REQUEST GROUPS, AND IT IS **NOT**
|
| 5303 |
# RECOVERED FROM THE GROUPS. A page may hold only some groups, and a `median` or a
|
|
@@ -5324,7 +5454,9 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
|
|
| 5324 |
select=(f"count(DISTINCT {gexpr}) + CASE WHEN count(*) - count({gexpr}) > 0 "
|
| 5325 |
f"THEN 1 ELSE 0 END"),
|
| 5326 |
where=where, params=tuple(params), order_by="1", limit=1, cur=cur, **frm)["rows"][0][0] or 0)
|
| 5327 |
-
|
|
|
|
|
|
|
| 5328 |
group_limit = min(group_limit if group_limit is not None else MAX_GROUPS, MAX_GROUPS)
|
| 5329 |
rows = datastore.window(
|
| 5330 |
select=projection, where=where, params=tuple(params), order_by="2 DESC, 1 ASC",
|
|
@@ -5333,7 +5465,8 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
|
|
| 5333 |
for r in rows:
|
| 5334 |
vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None)
|
| 5335 |
for idx, (_slot, col_id, _expr) in enumerate(plan)}
|
| 5336 |
-
|
|
|
|
| 5337 |
"count": int(r[1] or 0), "aggregates": vals})
|
| 5338 |
return {"total_rows": total_rows, "aggregates": answers, "groups": groups,
|
| 5339 |
"total_groups": distinct, "group_offset": group_offset,
|
|
@@ -5344,6 +5477,8 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
|
|
| 5344 |
def _mirror_aggregate(read, tenant, aggs, group_by, group_offset, group_limit):
|
| 5345 |
"""Use an exact published summary or retain the full compiled SQL scan."""
|
| 5346 |
from harness import summary_tables
|
|
|
|
|
|
|
| 5347 |
|
| 5348 |
limits = list(read["limits"])
|
| 5349 |
plan = _agg_plan(read["cols"], aggs, limits)
|
|
@@ -5691,8 +5826,31 @@ def _ut():
|
|
| 5691 |
|
| 5692 |
|
| 5693 |
def _today():
|
| 5694 |
-
|
| 5695 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5696 |
|
| 5697 |
|
| 5698 |
def _register_c1_window():
|
|
|
|
| 929 |
"id": "so.id",
|
| 930 |
"needs_excluded": True, # `wholesale_scope` is resolved against the excluded set
|
| 931 |
"default_wholesale_scope": True,
|
| 932 |
+
"business_days": ("order_date",), # stored UTC instants read as tenant days (`_source_for`)
|
| 933 |
"cols": {
|
| 934 |
"order_no": ("so.name", _s),
|
| 935 |
"odoo_id": ("so.id", _i),
|
|
|
|
| 991 |
"id": "sol.id",
|
| 992 |
"needs_excluded": True,
|
| 993 |
"default_wholesale_scope": True,
|
| 994 |
+
"business_days": ("order_date",),
|
| 995 |
"cols": {
|
| 996 |
"odoo_id": ("sol.id", _i),
|
| 997 |
"order_no": ("so.name", _s),
|
|
|
|
| 1261 |
},
|
| 1262 |
}
|
| 1263 |
GRID_SOURCES.update(rel.stock_source_specs())
|
| 1264 |
+
GRID_SOURCES[rel.LANDED_COST_LINES_KEY] = rel.landed_cost_line_source_spec()
|
| 1265 |
GRID_SOURCES.update(rel.purchase_source_specs())
|
| 1266 |
GRID_SOURCES.update(rel.payment_source_specs())
|
| 1267 |
GRID_SOURCES[rel.SHIPMENTS_KEY] = rel.shipment_source_spec()
|
|
|
|
| 1661 |
import amazon_relational as amazon
|
| 1662 |
if key in amazon.SOURCE:
|
| 1663 |
return _amazon_page_source(cur, key, binding=True, allow_lookups=allow_lookups)
|
| 1664 |
+
if _rel().base_table_key(key) == _rel().LANDED_COST_LINES_KEY:
|
| 1665 |
+
try:
|
| 1666 |
+
_rel().require_landed_cost_lines_mirror(cur)
|
| 1667 |
+
except _rel().Refused as exc:
|
| 1668 |
+
raise err(503, 'store_not_ready', str(exc)) from exc
|
| 1669 |
spec = _sources().get(key)
|
| 1670 |
if not spec:
|
| 1671 |
return None
|
|
|
|
| 1725 |
expr = spec['id']
|
| 1726 |
cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr,
|
| 1727 |
have_by_alias), cast)
|
| 1728 |
+
# Resolved per request: the zone is the bound tenant's, and the registry is shared.
|
| 1729 |
+
for name in (spec.get("business_days") or ()):
|
| 1730 |
+
if name in cols:
|
| 1731 |
+
cols[name] = (_business_day_sql(cols[name][0]), cols[name][1])
|
| 1732 |
if str(table_key).startswith("ut_odoo_") and spec.get("table") and not spec.get("from_sql"):
|
| 1733 |
spec = {**spec, "page_key": spec["id"]}
|
| 1734 |
return {**spec, "cols": cols}
|
|
|
|
| 1910 |
continue
|
| 1911 |
ftype = str((by_key.get(key) or {}).get("type") or "text")
|
| 1912 |
out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False}
|
| 1913 |
+
if (by_key.get(key) or {}).get('personal') is True:
|
| 1914 |
+
out[key].update(personal=True, column=key, encrypted=False)
|
| 1915 |
if key == "customer_type" and (by_key.get(key) or {}).get("source") == "odoo":
|
| 1916 |
out[key]["json_tags"] = True
|
| 1917 |
return out
|
| 1918 |
|
| 1919 |
|
| 1920 |
+
def _query_compile(method, *args, **kwargs):
|
| 1921 |
+
try:
|
| 1922 |
+
return getattr(_fs(), method)(*args, **kwargs)
|
| 1923 |
+
except ValueError as exc:
|
| 1924 |
+
raise err(400, 'query_unsupported', str(exc)) from exc
|
| 1925 |
+
|
| 1926 |
+
|
| 1927 |
+
def _bind_personal_columns(cols, spec, cur, session, table_key):
|
| 1928 |
+
"""Lend SQL only ciphertext indexes; legacy mirrors remain readable until keys are enabled."""
|
| 1929 |
+
from core import row_cipher, tenant_keys, user_table_rows, store_pg
|
| 1930 |
+
personal = [key for key, value in cols.items() if value.get('personal')]
|
| 1931 |
+
if not personal or tenant_keys.subkey(session.runtime.key, 'index') is None:
|
| 1932 |
+
return cols
|
| 1933 |
+
tenant = session.runtime.key
|
| 1934 |
+
lease = _RelationLease(cur)
|
| 1935 |
+
loaded = {}
|
| 1936 |
+
|
| 1937 |
+
def expression(column):
|
| 1938 |
+
if column in loaded:
|
| 1939 |
+
return loaded[column]
|
| 1940 |
+
import csv
|
| 1941 |
+
import tempfile
|
| 1942 |
+
from pathlib import Path
|
| 1943 |
+
from core.request_timing import remaining_budget
|
| 1944 |
+
name = lease.table('_personal')
|
| 1945 |
+
# Fetch ciphertext in bounded batches; DuckDB receives identities and HMACs only.
|
| 1946 |
+
with tempfile.TemporaryDirectory(prefix='personal-index-') as directory:
|
| 1947 |
+
path = Path(directory) / 'indexes.csv'
|
| 1948 |
+
with path.open('w', newline='', encoding='utf-8') as output:
|
| 1949 |
+
writer = csv.writer(output)
|
| 1950 |
+
with store_pg._pool().connection() as pg, pg.transaction():
|
| 1951 |
+
pg.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY')
|
| 1952 |
+
after = ''
|
| 1953 |
+
while True:
|
| 1954 |
+
batch = pg.execute(user_table_rows._sql(
|
| 1955 |
+
'SELECT row_id, (value -> %s)::text FROM {recs} '
|
| 1956 |
+
'WHERE table_key = %s AND row_id > %s ORDER BY row_id LIMIT 2048', tenant),
|
| 1957 |
+
(column, table_key, after)).fetchall()
|
| 1958 |
+
if not batch:
|
| 1959 |
+
break
|
| 1960 |
+
remaining_budget()
|
| 1961 |
+
for identity, raw in batch:
|
| 1962 |
+
value = json.loads(raw) if isinstance(raw, str) else raw
|
| 1963 |
+
if value is not None and not row_cipher.is_sealed(value):
|
| 1964 |
+
row_cipher.refuse(column, 'query before personal-field migration')
|
| 1965 |
+
writer.writerow((str(identity), value['bi'] if value is not None else None))
|
| 1966 |
+
after = str(batch[-1][0])
|
| 1967 |
+
cur.execute(f'CREATE TEMP TABLE "{name}" (pid VARCHAR, bi VARCHAR)')
|
| 1968 |
+
if path.stat().st_size:
|
| 1969 |
+
cur.execute(f'INSERT INTO "{name}" SELECT * FROM read_csv(?, header=false, '
|
| 1970 |
+
"columns={'pid':'VARCHAR','bi':'VARCHAR'})", [str(path)])
|
| 1971 |
+
identity = f'CAST({spec["id"]} AS VARCHAR)'
|
| 1972 |
+
found = f'SELECT bi FROM "{name}" pi WHERE pi.pid = {identity}'
|
| 1973 |
+
loaded[column] = (f'(CASE WHEN EXISTS (SELECT 1 FROM "{name}" pi WHERE pi.pid = {identity}) '
|
| 1974 |
+
f'THEN ({found}) ELSE error(\'Finish personal-field migration before querying this column.\') END)')
|
| 1975 |
+
return loaded[column]
|
| 1976 |
+
|
| 1977 |
+
for column in personal:
|
| 1978 |
+
cols[column].update(tenant=tenant, encrypted=True, _personal_lease=lease,
|
| 1979 |
+
personal_index_sql=lambda column=column: expression(column))
|
| 1980 |
+
return cols
|
| 1981 |
+
|
| 1982 |
+
|
| 1983 |
def _leaf_cols(nodes):
|
| 1984 |
"""Every `colId` a filter tree names, at any depth."""
|
| 1985 |
seen = set()
|
|
|
|
| 4092 |
try:
|
| 4093 |
cur, mismatch = (_mirror_cur(session, table_key)
|
| 4094 |
if (table_key.startswith(("ut_amzn_", "ut_meta_"))
|
| 4095 |
+
or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY
|
| 4096 |
+
or rel.base_table_key(table_key) in rel.stock_source_specs())
|
| 4097 |
else _mirror_cur(session))
|
| 4098 |
except RuntimeError as e:
|
| 4099 |
# ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the
|
|
|
|
| 4136 |
fields = list(declared.values())
|
| 4137 |
spec = _if_read_spec(spec, fields, table_key, st)
|
| 4138 |
cols = _filter_columns(spec, fields)
|
| 4139 |
+
cols = _bind_personal_columns(cols, spec, cur, session, table_key)
|
| 4140 |
|
| 4141 |
# ── THE PERMANENT WALL (W36-T21 / R6), COMPILED FIRST AND FAIL-CLOSED ────────────────────
|
| 4142 |
# An administrator's row filter is a `FilterTree` in the SAME vocabulary the client sends, so
|
|
|
|
| 4352 |
where = f"({where}) AND {pred.sql}" if where else pred.sql
|
| 4353 |
params.extend(pred.params)
|
| 4354 |
if str(search or "").strip():
|
| 4355 |
+
got = _query_compile('compile_search', search.strip(), cols)
|
| 4356 |
if got is not None:
|
| 4357 |
where = f"({where}) AND {got.sql}" if where else got.sql
|
| 4358 |
params.extend(got.params)
|
|
|
|
| 4367 |
|
| 4368 |
|
| 4369 |
@contextmanager
|
| 4370 |
+
def _materialised_read(table_key, session, query, related, ids=None, *, payload=None, needed=None,
|
| 4371 |
+
sql_page=False):
|
| 4372 |
"""One compiler for native rows and their aggregate sibling, over the walled payload.
|
| 4373 |
|
| 4374 |
The temporary connection contains only this request's already stripped, pinned rows and is
|
|
|
|
| 4414 |
else:
|
| 4415 |
connection = datastore.bounded_connection(duckdb.connect(":memory:"))
|
| 4416 |
with connection as cur:
|
| 4417 |
+
if compiled and sql_page and not related:
|
| 4418 |
+
mirror_revision = _mirror_page_revision(cur, source_generation)
|
| 4419 |
+
with registry_source.canonical_selection(session, cur, table_key, metadata, needed, ids=ids) as spec:
|
| 4420 |
+
if spec is not None:
|
| 4421 |
+
cols = _filter_columns(dict(spec, cols={k: v for k, v in spec['cols'].items()
|
| 4422 |
+
if k in {f['key'] for f in fields}}), fields)
|
| 4423 |
+
limits = list(payload.get('limits') or [])
|
| 4424 |
+
had_cohort = bool(_active_cohort_leaves(query['filters']))
|
| 4425 |
+
tree, cohorts = _views_as_cohorts(query['filters'], table_key, session,
|
| 4426 |
+
spec, cols, None, [], cur, limits)
|
| 4427 |
+
tree, cohorts = _ranks_as_cohorts(tree, query['filterConj'], spec, cols,
|
| 4428 |
+
None, [], cur, cohorts, had_cohort, payload.get('today') or _today(), limits)
|
| 4429 |
+
if _leaf_cols(tree) - _PSEUDO_COLS - set(cols):
|
| 4430 |
+
raise err(400, 'filter_unsupported', 'A filter names a column unavailable to this account.')
|
| 4431 |
+
try:
|
| 4432 |
+
pred = _fs().compile_filter_tree(tree, conj=query['filterConj'], columns=cols,
|
| 4433 |
+
id_sql=spec['id'], today=payload.get('today') or _today(), cohort_sets=cohorts)
|
| 4434 |
+
except ValueError as exc:
|
| 4435 |
+
raise err(400, 'filter_unsupported', str(exc))
|
| 4436 |
+
if pred is not None and pred.uses_aggregate:
|
| 4437 |
+
raise err(400, 'filter_unsupported', 'This condition cannot be evaluated for individual rows.')
|
| 4438 |
+
yield {'cur': cur, 'spec': spec, 'cols': cols,
|
| 4439 |
+
'where': pred.sql if pred else None, 'params': list(pred.params) if pred else [],
|
| 4440 |
+
'frm': {'table': spec['table']}, 'limits': limits, 'payload': payload,
|
| 4441 |
+
'source_count': cur.execute(f'SELECT count(*) FROM "{spec["table"]}"').fetchone()[0],
|
| 4442 |
+
'streamed': True, 'selection': True}
|
| 4443 |
+
registry_source.require_generation(session, table_key, source_generation)
|
| 4444 |
+
if mirror_revision != _mirror_page_revision(cur, source_generation):
|
| 4445 |
+
raise err(409, 'window_population_changed', 'These records changed while loading. Reload this database.')
|
| 4446 |
+
return
|
| 4447 |
cur.execute("CREATE TEMP TABLE pinned_rows (_row_index BIGINT, payload JSON)")
|
| 4448 |
wanted = set(ids) if ids is not None else None
|
| 4449 |
if streamed:
|
|
|
|
| 4496 |
spec = {"table": "pinned_rows", "id": "CAST(json_extract_string(payload, '/pid') AS BIGINT)",
|
| 4497 |
"cols": bindings, "where": None}
|
| 4498 |
cols = _filter_columns(spec, fields)
|
| 4499 |
+
cols = _bind_personal_columns(cols, spec, cur, session, table_key)
|
| 4500 |
limits = list(payload.get("limits") or [])
|
| 4501 |
_had_cohort = bool(_active_cohort_leaves(query["filters"]))
|
| 4502 |
tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
|
|
|
|
| 4516 |
if pred is not None and pred.uses_aggregate:
|
| 4517 |
raise err(400, "filter_unsupported", "This condition cannot be evaluated for individual rows.")
|
| 4518 |
where, params = (pred.sql, list(pred.params)) if pred is not None else (None, [])
|
| 4519 |
+
search = _query_compile('compile_search', query["search"], cols) if query["search"] else None
|
| 4520 |
if search is not None:
|
| 4521 |
where = f"({where}) AND {search.sql}" if where else search.sql
|
| 4522 |
params.extend(search.params)
|
|
|
|
| 4550 |
key = json.loads(group_key) if group_key not in (None, "") else None
|
| 4551 |
except ValueError:
|
| 4552 |
raise err(400, "bad_group", "groupKey must be the JSON of a group key")
|
| 4553 |
+
gexpr = _query_compile('group_sql', gspec)
|
| 4554 |
+
if gspec.get('personal') and gspec.get('encrypted', True) and key is not None:
|
| 4555 |
+
from core import row_cipher
|
| 4556 |
+
key = row_cipher.blind_index(gspec['tenant'], key)
|
| 4557 |
if key is None:
|
| 4558 |
return f"({gexpr}) IS NULL", []
|
| 4559 |
if gspec["type"] in _fs().NUMERIC_TYPES:
|
|
|
|
| 4786 |
needed.add(group_field)
|
| 4787 |
if query['search'] or _leaf_cols(query['filters']) & _PSEUDO_COLS:
|
| 4788 |
needed = None
|
| 4789 |
+
with _materialised_read(table_key, session, query, related, ids=ids, payload=payload,
|
| 4790 |
+
needed=needed, sql_page=True) as read:
|
| 4791 |
spec = read["spec"]
|
| 4792 |
+
order = _query_compile('compile_order_by', query["sorts"], read["cols"], tiebreak_sql=spec["id"])
|
| 4793 |
where, params = read["where"], list(read["params"])
|
| 4794 |
gsql, gparams = _group_pin(read["cols"], groupBy, groupKey)
|
| 4795 |
if gsql:
|
| 4796 |
where = f"({where}) AND {gsql}" if where else gsql
|
| 4797 |
params.extend(gparams)
|
| 4798 |
+
options = dict(select=spec['id'] if read.get('selection') else '_row_index', where=where, params=tuple(params),
|
| 4799 |
order_by=order or spec['id'], offset=window_offset, limit=limit or None,
|
| 4800 |
cur=read['cur'], **read['frm'])
|
| 4801 |
if table_key == 'ut_activity':
|
|
|
|
| 4804 |
else:
|
| 4805 |
import registry_source
|
| 4806 |
revision = registry_source.generation(session, table_key)
|
| 4807 |
+
if read.get('selection'):
|
| 4808 |
+
revision = _mirror_page_revision(read['cur'], revision)
|
| 4809 |
question = _window_query(table_key, filters, filterConj, sorts, search, related,
|
| 4810 |
groupBy, groupKey, ids, revision=revision)
|
| 4811 |
win = _window_page(session, question, cursor, page_first=True,
|
|
|
|
| 4813 |
payload = read["payload"]
|
| 4814 |
indexes = [int(row[0]) for row in win['rows']]
|
| 4815 |
value_sql = read['spec']['id'] if read['streamed'] else 'payload'
|
| 4816 |
+
values = ([(pid, pid) for pid in indexes] if read.get('selection') else
|
| 4817 |
+
read['cur'].execute(f'SELECT _row_index, {value_sql} FROM pinned_rows '
|
| 4818 |
'WHERE _row_index IN (SELECT unnest(?::BIGINT[]))', [indexes]).fetchall()
|
| 4819 |
if indexes else [])
|
| 4820 |
if read['streamed']:
|
|
|
|
| 4823 |
page_ids = [by_index[index] for index in indexes]
|
| 4824 |
if table_key in {'customer_data', 'product_data'}:
|
| 4825 |
payload = registry_source.canonical_page(session, read['cur'], table_key, page_ids, related=related)
|
| 4826 |
+
if read.get('selection') and [int(row['pid']) for row in payload['rows']] != page_ids:
|
| 4827 |
+
raise err(409, 'window_population_changed', 'These records changed while loading. Reload this database.')
|
| 4828 |
else:
|
| 4829 |
payload = registry_source.stored_page(session, table_key, page_ids, related=related)
|
| 4830 |
rows = payload['rows']
|
|
|
|
| 4942 |
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 4943 |
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
| 4944 |
if (admitted is None if page is not None else _materialised_table(table_key)):
|
| 4945 |
+
return _audit_personal_response(table_key, _materialised_window(
|
| 4946 |
+
table_key, session, offset, limit, filters, filterConj,
|
| 4947 |
+
sorts, search, related, groupBy, groupKey, ids=ids, cursor=cursor), session)
|
| 4948 |
|
| 4949 |
# ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
|
| 4950 |
# hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
|
|
|
|
| 4971 |
# ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one
|
| 4972 |
# row on two pages and drop another entirely — a duplicate the user sees with no error
|
| 4973 |
# anywhere. `compile_order_by`'s own docstring says so.
|
| 4974 |
+
order = _query_compile('compile_order_by', sort_spec, cols, tiebreak_sql=spec["id"]) or spec["id"]
|
| 4975 |
|
| 4976 |
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
|
| 4977 |
revision = _mirror_page_revision(cur, page_rev)
|
|
|
|
| 5098 |
for row in rows:
|
| 5099 |
if str(row["pid"]) in notes:
|
| 5100 |
row[user_tables.CELL_NOTES_KEY] = notes[str(row["pid"])]
|
| 5101 |
+
return _audit_personal_response(table_key, {"ok": True, "fields": fields, "rows": rows,
|
| 5102 |
# ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`.
|
| 5103 |
"total": win["total"], "totalUnfiltered": unfiltered,
|
| 5104 |
"offset": win["offset"], "limit": win["limit"], "limits": limits,
|
| 5105 |
"next_cursor": win['next_cursor'],
|
| 5106 |
"today": _today(), "identity": {"pid": "pid"},
|
| 5107 |
"scope": {"table": table_key, "readThrough": True},
|
| 5108 |
+
"recordsMutable": bool(defn.get("recordMode") != _ut().AUTOMATION_RECORD_MODE)}, session)
|
| 5109 |
+
|
| 5110 |
+
|
| 5111 |
+
def _audit_personal_response(table_key, payload, session):
|
| 5112 |
+
if not isinstance(payload, dict) or payload.get('ok') is False:
|
| 5113 |
+
return payload
|
| 5114 |
+
from core import personal_reads
|
| 5115 |
+
personal_reads.record_rows(table_key, payload.get('rows') or [], st=session.runtime,
|
| 5116 |
+
user=session.user, definition={'fields': payload.get('fields') or []})
|
| 5117 |
+
return payload
|
| 5118 |
|
| 5119 |
|
| 5120 |
# ══════════════════════════════════════════════════════════
|
|
|
|
| 5427 |
"recommendation": "group by a column the grid reads from the connection, or ask "
|
| 5428 |
"an administrator for access to this one"})
|
| 5429 |
else:
|
| 5430 |
+
gexpr = _query_compile('group_sql', gspec)
|
| 5431 |
|
| 5432 |
# ⛔⛔ THE OVERALL SUMMARY IS COMPUTED WHETHER OR NOT THIS REQUEST GROUPS, AND IT IS **NOT**
|
| 5433 |
# RECOVERED FROM THE GROUPS. A page may hold only some groups, and a `median` or a
|
|
|
|
| 5454 |
select=(f"count(DISTINCT {gexpr}) + CASE WHEN count(*) - count({gexpr}) > 0 "
|
| 5455 |
f"THEN 1 ELSE 0 END"),
|
| 5456 |
where=where, params=tuple(params), order_by="1", limit=1, cur=cur, **frm)["rows"][0][0] or 0)
|
| 5457 |
+
personal_group = bool(gspec.get('personal') and gspec.get('encrypted', True))
|
| 5458 |
+
label_projection = [f'min({gspec["sql"]}) AS "_label"'] if personal_group else []
|
| 5459 |
+
projection = ", ".join([f'{gexpr} AS "_k"', 'count(*) AS "_c"'] + agg_proj + label_projection)
|
| 5460 |
group_limit = min(group_limit if group_limit is not None else MAX_GROUPS, MAX_GROUPS)
|
| 5461 |
rows = datastore.window(
|
| 5462 |
select=projection, where=where, params=tuple(params), order_by="2 DESC, 1 ASC",
|
|
|
|
| 5465 |
for r in rows:
|
| 5466 |
vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None)
|
| 5467 |
for idx, (_slot, col_id, _expr) in enumerate(plan)}
|
| 5468 |
+
value = r[-1] if personal_group else r[0]
|
| 5469 |
+
groups.append({"key": value if personal_group else _group_key(value), "label": _group_label(value),
|
| 5470 |
"count": int(r[1] or 0), "aggregates": vals})
|
| 5471 |
return {"total_rows": total_rows, "aggregates": answers, "groups": groups,
|
| 5472 |
"total_groups": distinct, "group_offset": group_offset,
|
|
|
|
| 5477 |
def _mirror_aggregate(read, tenant, aggs, group_by, group_offset, group_limit):
|
| 5478 |
"""Use an exact published summary or retain the full compiled SQL scan."""
|
| 5479 |
from harness import summary_tables
|
| 5480 |
+
if any(col.get('personal') and col.get('encrypted', True) for col in read['cols'].values()):
|
| 5481 |
+
return None
|
| 5482 |
|
| 5483 |
limits = list(read["limits"])
|
| 5484 |
plan = _agg_plan(read["cols"], aggs, limits)
|
|
|
|
| 5826 |
|
| 5827 |
|
| 5828 |
def _today():
|
| 5829 |
+
"""The bound tenant's business day, ISO; UTC when the tenant has no zone."""
|
| 5830 |
+
from core import periods
|
| 5831 |
+
return periods.today().isoformat()
|
| 5832 |
+
|
| 5833 |
+
|
| 5834 |
+
def _business_day_sql(expr):
|
| 5835 |
+
"""`expr`, a stored UTC instant, as its ISO business day in the bound tenant's zone.
|
| 5836 |
+
|
| 5837 |
+
⛔ ONE expression serves the projection, the filter and the sort, so an `order_date` filter
|
| 5838 |
+
pushes down the same tenant-time bounds the grid shows. A bare date passes through. No zone
|
| 5839 |
+
or a degraded NULL keeps `expr` unchanged."""
|
| 5840 |
+
from core import periods
|
| 5841 |
+
zone = periods.business_tz()
|
| 5842 |
+
if not zone or str(expr).strip().upper() == "NULL":
|
| 5843 |
+
return expr
|
| 5844 |
+
try:
|
| 5845 |
+
from zoneinfo import ZoneInfo
|
| 5846 |
+
ZoneInfo(zone)
|
| 5847 |
+
except Exception: # noqa: BLE001
|
| 5848 |
+
return expr # `periods` reads an unknown zone as UTC; so does this
|
| 5849 |
+
name = zone.replace("'", "''")
|
| 5850 |
+
text = f"CAST({expr} AS VARCHAR)"
|
| 5851 |
+
return (f"(CASE WHEN LENGTH({text}) <= 10 THEN {text} ELSE COALESCE(CAST(CAST(timezone("
|
| 5852 |
+
f"'{name}', timezone('UTC', TRY_CAST({text} AS TIMESTAMP))) AS DATE) AS VARCHAR), "
|
| 5853 |
+
f"SUBSTR({text}, 1, 10)) END)")
|
| 5854 |
|
| 5855 |
|
| 5856 |
def _register_c1_window():
|
api/routes_opportunity_finder.py
CHANGED
|
@@ -7,8 +7,6 @@ from deps import Session, err, require_session
|
|
| 7 |
router = APIRouter(prefix="/api/v1/app-kinds/opportunity_finder", tags=["opportunity_finder"])
|
| 8 |
SOURCE = "ut_amzn_brand_analytics"
|
| 9 |
TEMPLATE_KIND = "opportunity_finder"
|
| 10 |
-
#: Weeks in a list row's sparkline. Each week is one store read per /screen call.
|
| 11 |
-
SPARK_WEEKS = 8
|
| 12 |
SETTING_NAMES = {"min_rise_pct": "min_rise_pct", "minRisePct": "min_rise_pct", "departments": "departments"}
|
| 13 |
|
| 14 |
|
|
@@ -17,13 +15,14 @@ def _module():
|
|
| 17 |
return opportunity_finder
|
| 18 |
|
| 19 |
|
| 20 |
-
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 21 |
-
"""
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
|
|
|
| 27 |
return session
|
| 28 |
|
| 29 |
|
|
@@ -31,6 +30,8 @@ def _writer(session: Session = Depends(_reader)) -> Session:
|
|
| 31 |
"""Writes also need a role that may edit."""
|
| 32 |
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 33 |
raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.")
|
|
|
|
|
|
|
| 34 |
return session
|
| 35 |
|
| 36 |
|
|
@@ -46,7 +47,7 @@ def _call(fn, *args, **kwargs):
|
|
| 46 |
try:
|
| 47 |
return fn(*args, **kwargs)
|
| 48 |
except PermissionError as exc:
|
| 49 |
-
raise err(403, "
|
| 50 |
except KeyError as exc:
|
| 51 |
raise err(404, "opportunity_finder_not_found", "That search term is not in the opportunity finder.") from exc
|
| 52 |
except ValueError as exc:
|
|
@@ -94,12 +95,32 @@ def _mirror_opener(session):
|
|
| 94 |
return lambda: routes_odoo_tables._table_cursor(session.runtime, SOURCE)
|
| 95 |
|
| 96 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
def _detail_options(session):
|
| 98 |
"""Auxiliary sources fail closed before any raw mirror projection or stored-row read."""
|
|
|
|
| 99 |
from core import perm_scope
|
| 100 |
of = _module()
|
|
|
|
|
|
|
| 101 |
|
| 102 |
def admitted(key):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
return (perm_scope.may_read(session.user, key, st=session.runtime)
|
| 104 |
and not perm_scope.wall_declared(session.user, key, st=session.runtime))
|
| 105 |
|
|
@@ -119,13 +140,10 @@ def _detail_options(session):
|
|
| 119 |
"monthly_allowed": admitted("ut_amzn_search_query_performance")}
|
| 120 |
|
| 121 |
|
| 122 |
-
def _with_spark(of, st, items):
|
| 123 |
-
"""
|
| 124 |
-
weeks = of.weeks(None, st=st)[-SPARK_WEEKS:] if SPARK_WEEKS > 0 else []
|
| 125 |
-
held = [(week, of.week_point(None, week, st=st).get("terms") or {}) for week in weeks]
|
| 126 |
for item in items:
|
| 127 |
-
|
| 128 |
-
item["spark"] = [{"week": week, "rank": terms[term]} for week, terms in held if term in terms]
|
| 129 |
return items
|
| 130 |
|
| 131 |
|
|
@@ -136,10 +154,12 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 136 |
of = _module()
|
| 137 |
record = _app(session, app)
|
| 138 |
st = session.runtime
|
|
|
|
| 139 |
listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st,
|
| 140 |
-
opener=_mirror_opener(session))
|
| 141 |
-
listed["items"] = _call(_with_spark, of, st, listed["items"])
|
| 142 |
-
return {"canEdit": _may_edit(session, record),
|
|
|
|
| 143 |
"weeks": listed["weeks"], "windows": listed["windows"], "periodType": listed["periodType"],
|
| 144 |
"departments": listed["departments"], "counts": listed["counts"],
|
| 145 |
"total": listed["total"], "items": listed["items"], "cursor": listed["cursor"],
|
|
@@ -148,10 +168,13 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 148 |
|
| 149 |
|
| 150 |
@router.get("/items/{item_id}")
|
| 151 |
-
def item_detail(item_id: str, session: Session = Depends(_app_reader)):
|
| 152 |
of = _module()
|
| 153 |
-
|
| 154 |
-
|
|
|
|
|
|
|
|
|
|
| 155 |
|
| 156 |
|
| 157 |
@router.patch("/items/{item_id}")
|
|
@@ -168,7 +191,7 @@ def term_trend(item_id: str, session: Session = Depends(_app_reader)):
|
|
| 168 |
"""A term's weekly rank trend and its newest top 3 clicked ASINs."""
|
| 169 |
of = _module()
|
| 170 |
return _call(of.term_detail, _tenant(session), item_id, st=session.runtime, opener=_mirror_opener(session),
|
| 171 |
-
**_detail_options(session))
|
| 172 |
|
| 173 |
|
| 174 |
@router.get("/settings")
|
|
@@ -195,4 +218,4 @@ def validate(week: str | None = Query(None), start: str | None = Query(None), en
|
|
| 195 |
session: Session = Depends(_app_reader)):
|
| 196 |
of = _module()
|
| 197 |
return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime,
|
| 198 |
-
opener=_mirror_opener(session))
|
|
|
|
| 7 |
router = APIRouter(prefix="/api/v1/app-kinds/opportunity_finder", tags=["opportunity_finder"])
|
| 8 |
SOURCE = "ut_amzn_brand_analytics"
|
| 9 |
TEMPLATE_KIND = "opportunity_finder"
|
|
|
|
|
|
|
| 10 |
SETTING_NAMES = {"min_rise_pct": "min_rise_pct", "minRisePct": "min_rise_pct", "departments": "departments"}
|
| 11 |
|
| 12 |
|
|
|
|
| 15 |
return opportunity_finder
|
| 16 |
|
| 17 |
|
| 18 |
+
def _reader(app: str | None = Query(None), session: Session = Depends(require_session)) -> Session:
|
| 19 |
+
"""R62: reads need the App grant plus the shared source check, never the source open grant."""
|
| 20 |
+
import application_registry
|
| 21 |
+
record = _app(session, app) if app else _call(application_registry.readable_app, session.runtime,
|
| 22 |
+
_tenant(session), session.uname, session.admin, TEMPLATE_KIND)
|
| 23 |
+
application_registry.admit_app(session, record, (SOURCE, "ut_amzn_listings", "ut_amzn_search_query_performance"),
|
| 24 |
+
code="opportunity_finder")
|
| 25 |
+
session.source_app = record
|
| 26 |
return session
|
| 27 |
|
| 28 |
|
|
|
|
| 30 |
"""Writes also need a role that may edit."""
|
| 31 |
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 32 |
raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.")
|
| 33 |
+
if _source_scope(session) is not None:
|
| 34 |
+
raise err(403, "opportunity_finder_limited", "Saved workflow changes need unrestricted source rows and fields.")
|
| 35 |
return session
|
| 36 |
|
| 37 |
|
|
|
|
| 47 |
try:
|
| 48 |
return fn(*args, **kwargs)
|
| 49 |
except PermissionError as exc:
|
| 50 |
+
raise err(403, "opportunity_finder_limited", str(exc)) from exc
|
| 51 |
except KeyError as exc:
|
| 52 |
raise err(404, "opportunity_finder_not_found", "That search term is not in the opportunity finder.") from exc
|
| 53 |
except ValueError as exc:
|
|
|
|
| 95 |
return lambda: routes_odoo_tables._table_cursor(session.runtime, SOURCE)
|
| 96 |
|
| 97 |
|
| 98 |
+
def _source_scope(session):
|
| 99 |
+
"""Lend the shared source evaluator to the provider only while a source wall applies."""
|
| 100 |
+
import application_registry
|
| 101 |
+
sources = (SOURCE, "ut_amzn_listings", "ut_amzn_search_query_performance")
|
| 102 |
+
walls = application_registry.app_access(session.user, session.runtime, None, sources)["narrow"]
|
| 103 |
+
active = {source for source, wall in walls.items() if wall["rows"] or wall["hidden"]}
|
| 104 |
+
if not active:
|
| 105 |
+
return None
|
| 106 |
+
from routes_app_dashboard import source_scope_for
|
| 107 |
+
scope = source_scope_for(session, session.source_app)
|
| 108 |
+
return lambda source, records: scope(source, records) if source in active else None
|
| 109 |
+
|
| 110 |
+
|
| 111 |
def _detail_options(session):
|
| 112 |
"""Auxiliary sources fail closed before any raw mirror projection or stored-row read."""
|
| 113 |
+
import application_registry
|
| 114 |
from core import perm_scope
|
| 115 |
of = _module()
|
| 116 |
+
record = application_registry.readable_app(session.runtime, _tenant(session), session.uname,
|
| 117 |
+
session.admin, TEMPLATE_KIND)
|
| 118 |
|
| 119 |
def admitted(key):
|
| 120 |
+
if key.startswith("ut_amzn_"):
|
| 121 |
+
access = application_registry.app_access(session.user, session.runtime, record, (key,))
|
| 122 |
+
wall = access["narrow"][key]
|
| 123 |
+
return access["app"] == "ready" and not wall["rows"] and not wall["hidden"]
|
| 124 |
return (perm_scope.may_read(session.user, key, st=session.runtime)
|
| 125 |
and not perm_scope.wall_declared(session.user, key, st=session.runtime))
|
| 126 |
|
|
|
|
| 140 |
"monthly_allowed": admitted("ut_amzn_search_query_performance")}
|
| 141 |
|
| 142 |
|
| 143 |
+
def _with_spark(of, st, items, source_scope=None):
|
| 144 |
+
"""Keep the legacy spark wire as an alias of the provider's complete, scoped page history."""
|
|
|
|
|
|
|
| 145 |
for item in items:
|
| 146 |
+
item["spark"] = item["trendPoints"]
|
|
|
|
| 147 |
return items
|
| 148 |
|
| 149 |
|
|
|
|
| 154 |
of = _module()
|
| 155 |
record = _app(session, app)
|
| 156 |
st = session.runtime
|
| 157 |
+
scope = _call(_source_scope, session)
|
| 158 |
listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st,
|
| 159 |
+
opener=_mirror_opener(session), source_scope=scope)
|
| 160 |
+
listed["items"] = _call(_with_spark, of, st, listed["items"], source_scope=scope)
|
| 161 |
+
return {**listed, "canEdit": scope is None and _may_edit(session, record),
|
| 162 |
+
"week": listed["week"], "priorWeek": listed["priorWeek"],
|
| 163 |
"weeks": listed["weeks"], "windows": listed["windows"], "periodType": listed["periodType"],
|
| 164 |
"departments": listed["departments"], "counts": listed["counts"],
|
| 165 |
"total": listed["total"], "items": listed["items"], "cursor": listed["cursor"],
|
|
|
|
| 168 |
|
| 169 |
|
| 170 |
@router.get("/items/{item_id}")
|
| 171 |
+
def item_detail(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
|
| 172 |
of = _module()
|
| 173 |
+
scope = _call(_source_scope, session)
|
| 174 |
+
detail = _call(of.item_detail, _tenant(session), session.uname, item_id, st=session.runtime,
|
| 175 |
+
opener=_mirror_opener(session), source_scope=scope, **_detail_options(session))
|
| 176 |
+
_call(_with_spark, of, session.runtime, [detail["item"]], source_scope=scope)
|
| 177 |
+
return {**detail, "canEdit": scope is None and _may_edit(session, _app(session, app))}
|
| 178 |
|
| 179 |
|
| 180 |
@router.patch("/items/{item_id}")
|
|
|
|
| 191 |
"""A term's weekly rank trend and its newest top 3 clicked ASINs."""
|
| 192 |
of = _module()
|
| 193 |
return _call(of.term_detail, _tenant(session), item_id, st=session.runtime, opener=_mirror_opener(session),
|
| 194 |
+
source_scope=_call(_source_scope, session), **_detail_options(session))
|
| 195 |
|
| 196 |
|
| 197 |
@router.get("/settings")
|
|
|
|
| 218 |
session: Session = Depends(_app_reader)):
|
| 219 |
of = _module()
|
| 220 |
return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime,
|
| 221 |
+
opener=_mirror_opener(session), source_scope=_call(_source_scope, session))
|
api/routes_payouts.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""Payouts App routes (C3): the overview, settlements with their waterfall and tie, owned items and settings.
|
| 2 |
|
| 3 |
-
Every route names the App (`?app=`) and checks
|
| 4 |
Writes also need edit rights on that App and a role other than viewer. `main.py` does not mount this
|
| 5 |
router: `app_routes.mount_app_routers` does, from the App contract.
|
| 6 |
"""
|
|
@@ -18,11 +18,11 @@ def _module():
|
|
| 18 |
return payouts
|
| 19 |
|
| 20 |
|
| 21 |
-
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 22 |
-
"""
|
| 23 |
-
from
|
| 24 |
-
|
| 25 |
-
|
| 26 |
return session
|
| 27 |
|
| 28 |
|
|
@@ -52,9 +52,12 @@ def _app(session, app_id):
|
|
| 52 |
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 53 |
except Exception as exc: # noqa: BLE001
|
| 54 |
raise err(503, "payouts_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 55 |
-
if (not isinstance(app, dict) or app.get("
|
|
|
|
| 56 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 57 |
raise err(404, "payouts_app_not_found", "That Payouts App does not exist.")
|
|
|
|
|
|
|
| 58 |
return app
|
| 59 |
|
| 60 |
|
|
@@ -65,9 +68,9 @@ def _may_edit(session, app):
|
|
| 65 |
|
| 66 |
|
| 67 |
def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
| 68 |
-
"""A read
|
| 69 |
_app(session, app)
|
| 70 |
-
return _reader(session)
|
| 71 |
|
| 72 |
|
| 73 |
def _settings_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
|
@@ -92,7 +95,7 @@ def _forecast_access(session):
|
|
| 92 |
def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
| 93 |
"""A write names the App, and only an editor of that App may change it."""
|
| 94 |
found = _app(session, app)
|
| 95 |
-
_writer(_reader(session))
|
| 96 |
if not _may_edit(session, found):
|
| 97 |
raise err(403, "payouts_read_only", "You can view this App but not change it.")
|
| 98 |
return session
|
|
@@ -104,6 +107,8 @@ def _call(fn, *args, missing="payouts_not_found", missing_message="That settleme
|
|
| 104 |
return fn(*args, **kwargs)
|
| 105 |
except PermissionError as exc:
|
| 106 |
raise err(403, "payouts_read_only", str(exc)) from exc
|
|
|
|
|
|
|
| 107 |
except pay.SourceUnavailable as exc:
|
| 108 |
raise err(503, "payouts_no_source", str(exc)) from exc
|
| 109 |
except ValueError as exc:
|
|
@@ -157,24 +162,34 @@ def update_item(item_id: str, status: str = Body(..., embed=True), note: str | N
|
|
| 157 |
|
| 158 |
|
| 159 |
def _fee_access(session):
|
| 160 |
-
"""Fee comparisons
|
| 161 |
from core import perm_scope
|
|
|
|
| 162 |
for key in (SOURCE, _module().FEE_ESTIMATE_SOURCE):
|
| 163 |
team, agent = perm_scope.derive_pool_scope(session.user, key)
|
| 164 |
-
if (
|
| 165 |
-
or perm_scope.wall_declared(session.user, key, st=session.runtime)
|
| 166 |
or team is not None or agent is not None):
|
| 167 |
raise err(403, "payouts_fee_forbidden", "Fee checks need access to settlements and fee estimates.")
|
| 168 |
|
| 169 |
|
| 170 |
@router.get("/fee-checks")
|
| 171 |
def read_fee_checks(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 172 |
-
limit: int = Query(50, ge=1), q: str = Query(""),
|
|
|
|
| 173 |
_fee_access(session)
|
| 174 |
-
page = _call(_module().fee_checks, _tenant(session), st=session.runtime, cursor=cursor, limit=limit, q=q
|
|
|
|
| 175 |
return {**page, "canEdit": _may_edit(session, _app(session, app))}
|
| 176 |
|
| 177 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
@router.patch("/fee-checks/{item_id}")
|
| 179 |
def update_fee_check(item_id: str, status: str = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 180 |
_fee_access(session)
|
|
@@ -400,6 +415,14 @@ def read_screen(app: str = Query(..., min_length=1), currency: str | None = Quer
|
|
| 400 |
return {**base, **page, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page["settlements"]]}
|
| 401 |
|
| 402 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 403 |
@router.get("/screen/settlements/{settlement_id}")
|
| 404 |
def read_screen_settlement(settlement_id: str, app: str = Query(..., min_length=1),
|
| 405 |
bucket: str | None = Query(None), cursor: str | None = Query(None),
|
|
|
|
| 1 |
"""Payouts App routes (C3): the overview, settlements with their waterfall and tie, owned items and settings.
|
| 2 |
|
| 3 |
+
Every route names the App (`?app=`) and checks its grant and source limits.
|
| 4 |
Writes also need edit rights on that App and a role other than viewer. `main.py` does not mount this
|
| 5 |
router: `app_routes.mount_app_routers` does, from the App contract.
|
| 6 |
"""
|
|
|
|
| 18 |
return payouts
|
| 19 |
|
| 20 |
|
| 21 |
+
def _reader(session: Session = Depends(require_session), app: str | None = Query(None)) -> Session:
|
| 22 |
+
"""Bind the requested App before admitting its declared source contracts and limits."""
|
| 23 |
+
from routes_app_dashboard import admit_reader
|
| 24 |
+
record = _app(session, app) if isinstance(app, str) and app else None
|
| 25 |
+
admit_reader(session, 'payouts', record)
|
| 26 |
return session
|
| 27 |
|
| 28 |
|
|
|
|
| 52 |
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 53 |
except Exception as exc: # noqa: BLE001
|
| 54 |
raise err(503, "payouts_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 55 |
+
if (not isinstance(app, dict) or app.get("tenant") != _tenant(session)
|
| 56 |
+
or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 57 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 58 |
raise err(404, "payouts_app_not_found", "That Payouts App does not exist.")
|
| 59 |
+
from routes_app_dashboard import admit_reader
|
| 60 |
+
admit_reader(session, 'payouts', app)
|
| 61 |
return app
|
| 62 |
|
| 63 |
|
|
|
|
| 68 |
|
| 69 |
|
| 70 |
def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
| 71 |
+
"""A read checks App identity before admitting its source scope."""
|
| 72 |
_app(session, app)
|
| 73 |
+
return _reader(session, app=app)
|
| 74 |
|
| 75 |
|
| 76 |
def _settings_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
|
|
|
| 95 |
def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
| 96 |
"""A write names the App, and only an editor of that App may change it."""
|
| 97 |
found = _app(session, app)
|
| 98 |
+
_writer(_reader(session, app=app))
|
| 99 |
if not _may_edit(session, found):
|
| 100 |
raise err(403, "payouts_read_only", "You can view this App but not change it.")
|
| 101 |
return session
|
|
|
|
| 107 |
return fn(*args, **kwargs)
|
| 108 |
except PermissionError as exc:
|
| 109 |
raise err(403, "payouts_read_only", str(exc)) from exc
|
| 110 |
+
except pay.TableRefused as exc:
|
| 111 |
+
raise err(409, "payouts_table_refused", str(exc)) from exc
|
| 112 |
except pay.SourceUnavailable as exc:
|
| 113 |
raise err(503, "payouts_no_source", str(exc)) from exc
|
| 114 |
except ValueError as exc:
|
|
|
|
| 162 |
|
| 163 |
|
| 164 |
def _fee_access(session):
|
| 165 |
+
"""Fee comparisons use the App grant and refuse source or pool limits."""
|
| 166 |
from core import perm_scope
|
| 167 |
+
_reader(session)
|
| 168 |
for key in (SOURCE, _module().FEE_ESTIMATE_SOURCE):
|
| 169 |
team, agent = perm_scope.derive_pool_scope(session.user, key)
|
| 170 |
+
if (perm_scope.wall_declared(session.user, key, st=session.runtime)
|
|
|
|
| 171 |
or team is not None or agent is not None):
|
| 172 |
raise err(403, "payouts_fee_forbidden", "Fee checks need access to settlements and fee estimates.")
|
| 173 |
|
| 174 |
|
| 175 |
@router.get("/fee-checks")
|
| 176 |
def read_fee_checks(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 177 |
+
limit: int = Query(50, ge=1), q: str = Query(""), item: str | None = Query(None),
|
| 178 |
+
session: Session = Depends(_app_reader)):
|
| 179 |
_fee_access(session)
|
| 180 |
+
page = _call(_module().fee_checks, _tenant(session), st=session.runtime, cursor=cursor, limit=limit, q=q,
|
| 181 |
+
item_id=item or None)
|
| 182 |
return {**page, "canEdit": _may_edit(session, _app(session, app))}
|
| 183 |
|
| 184 |
|
| 185 |
+
@router.post("/fee-table")
|
| 186 |
+
def build_fee_table(session: Session = Depends(_app_reader)):
|
| 187 |
+
"""Build the fee checks table when its projection moved; the Fee checks tab calls this before its Grid reads."""
|
| 188 |
+
_fee_access(session)
|
| 189 |
+
built = _call(_module().fee_table, _tenant(session), st=session.runtime)
|
| 190 |
+
return {k: built.get(k) for k in ("table", "rows", "stamp")}
|
| 191 |
+
|
| 192 |
+
|
| 193 |
@router.patch("/fee-checks/{item_id}")
|
| 194 |
def update_fee_check(item_id: str, status: str = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 195 |
_fee_access(session)
|
|
|
|
| 415 |
return {**base, **page, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page["settlements"]]}
|
| 416 |
|
| 417 |
|
| 418 |
+
@router.post("/table")
|
| 419 |
+
def build_table(session: Session = Depends(_app_reader)):
|
| 420 |
+
"""Build the settlements table when its source moved; the Settlements tab calls this before its Grid reads."""
|
| 421 |
+
pay = _module()
|
| 422 |
+
built = _call(pay.settlement_table, _tenant(session), st=session.runtime)
|
| 423 |
+
return {k: built.get(k) for k in ("table", "rows", "stamp")}
|
| 424 |
+
|
| 425 |
+
|
| 426 |
@router.get("/screen/settlements/{settlement_id}")
|
| 427 |
def read_screen_settlement(settlement_id: str, app: str = Query(..., min_length=1),
|
| 428 |
bucket: str | None = Query(None), cursor: str | None = Query(None),
|
api/routes_products.py
CHANGED
|
@@ -442,7 +442,7 @@ def _duplicate_code_limits(rows_src, fields):
|
|
| 442 |
codes = ', '.join(report['codes'])
|
| 443 |
return [{'subject': 'code', 'effect': 'duplicate_codes',
|
| 444 |
'cause': f"Shared SKU codes {codes}: {report['gap']} fewer rows than active Odoo products for these codes.",
|
| 445 |
-
'recommendation': 'Ask the owner to
|
| 446 |
|
| 447 |
|
| 448 |
def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
|
@@ -515,6 +515,10 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_
|
|
| 515 |
for _pid, _cells in _shared.items():
|
| 516 |
_ov[_pid] = {**(_ov.get(_pid) or {}), **_cells}
|
| 517 |
ws["overlays"] = _ov
|
|
|
|
|
|
|
|
|
|
|
|
|
| 518 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 519 |
ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key,
|
| 520 |
fields_base=fields_base, st=session.runtime)
|
|
@@ -704,13 +708,31 @@ def products(session: Session = Depends(module_gate(MODULE)), related: str = Non
|
|
| 704 |
# report in there would red a gate outside this ticket's fence for a purely cosmetic nesting.
|
| 705 |
# The envelope check beside it filters to the four `/customers` keys, so an additive key is
|
| 706 |
# the shape this route already established with `identity` and `scope`.
|
| 707 |
-
|
| 708 |
"refusals": metric_refusals,
|
| 709 |
"limits": field_limits + _duplicate_code_limits(g['rows_src'], g['fields']) + linked_metric_limits(metric_refusals),
|
| 710 |
"today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
|
| 711 |
"identity": {"pid": "pid", "businessKey": "code"},
|
| 712 |
"identity_report": pd.identity_report(g["rows_src"]),
|
| 713 |
"scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 714 |
|
| 715 |
|
| 716 |
@router.post("/products/aggregates")
|
|
|
|
| 442 |
codes = ', '.join(report['codes'])
|
| 443 |
return [{'subject': 'code', 'effect': 'duplicate_codes',
|
| 444 |
'cause': f"Shared SKU codes {codes}: {report['gap']} fewer rows than active Odoo products for these codes.",
|
| 445 |
+
'recommendation': 'Ask the owner to choose the canonical Odoo record for each shared SKU.'}]
|
| 446 |
|
| 447 |
|
| 448 |
def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
|
|
|
| 515 |
for _pid, _cells in _shared.items():
|
| 516 |
_ov[_pid] = {**(_ov.get(_pid) or {}), **_cells}
|
| 517 |
ws["overlays"] = _ov
|
| 518 |
+
# D-7088: a saved lead overlay is the served lead, so coverage is recomputed from it here.
|
| 519 |
+
# This runs before the IF cells and the hidden strip, so every reader sees one coverage.
|
| 520 |
+
if with_rows:
|
| 521 |
+
rows_src = pd.apply_lead_overlay(rows_src, ws.get("overlays"))
|
| 522 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 523 |
ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key,
|
| 524 |
fields_base=fields_base, st=session.runtime)
|
|
|
|
| 708 |
# report in there would red a gate outside this ticket's fence for a purely cosmetic nesting.
|
| 709 |
# The envelope check beside it filters to the four `/customers` keys, so an additive key is
|
| 710 |
# the shape this route already established with `identity` and `scope`.
|
| 711 |
+
body = {"fields": g["fields"], "rows": rows,
|
| 712 |
"refusals": metric_refusals,
|
| 713 |
"limits": field_limits + _duplicate_code_limits(g['rows_src'], g['fields']) + linked_metric_limits(metric_refusals),
|
| 714 |
"today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
|
| 715 |
"identity": {"pid": "pid", "businessKey": "code"},
|
| 716 |
"identity_report": pd.identity_report(g["rows_src"]),
|
| 717 |
"scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}}
|
| 718 |
+
# D-11176: a paused Odoo serves the cached pool at any age, so the envelope names the pause.
|
| 719 |
+
# frozenAt is the served pool's epoch-seconds stamp. The keychain's frozenAt is naive local
|
| 720 |
+
# ISO text and never reaches the client. No stamp (a bounded pool) omits frozenAt.
|
| 721 |
+
import routes_keychain
|
| 722 |
+
rt = session.runtime
|
| 723 |
+
# A runtime that cannot answer the pause question (a gate double) serves no paused marks.
|
| 724 |
+
if callable(getattr(rt, "odoo_paused", None)) and routes_keychain.odoo_paused(rt):
|
| 725 |
+
body["paused"] = True
|
| 726 |
+
stamp = (_measure_stamp(rt, g["team_id"])
|
| 727 |
+
if getattr(rt, '_canonical_product_rows', None) is None else 0)
|
| 728 |
+
if stamp:
|
| 729 |
+
body["frozenAt"] = int(stamp)
|
| 730 |
+
# ProductsFrozenNotice renders only beside a frozen limit, and writes its own time.
|
| 731 |
+
body["limits"] = body["limits"] + [{
|
| 732 |
+
"subject": "products", "effect": "frozen",
|
| 733 |
+
"cause": "Odoo is paused, so these products show data as of the last sync.",
|
| 734 |
+
"recommendation": "Resume Odoo to refresh these products."}]
|
| 735 |
+
return body
|
| 736 |
|
| 737 |
|
| 738 |
@router.post("/products/aggregates")
|
api/routes_query.py
CHANGED
|
@@ -2381,11 +2381,17 @@ def conversation_log(before: str = None, limit: int = 50,
|
|
| 2381 |
|
| 2382 |
|
| 2383 |
@router.get("/assistant/memory")
|
| 2384 |
-
def conversation_memory(session: Session = Depends(require_session)
|
| 2385 |
-
|
| 2386 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2387 |
doc = assistant_memory.document(session.runtime, session.uname)
|
| 2388 |
-
return {"entries": entries, "total":
|
|
|
|
| 2389 |
|
| 2390 |
|
| 2391 |
@router.post("/assistant/confirm")
|
|
|
|
| 2381 |
|
| 2382 |
|
| 2383 |
@router.get("/assistant/memory")
|
| 2384 |
+
def conversation_memory(session: Session = Depends(require_session), cursor: str | None = None,
|
| 2385 |
+
order: str = "newest", limit: int = assistant_memory.MAX_ENTRIES):
|
| 2386 |
+
"""Page every authorized personal and company entry. `total` counts them all, `next` continues."""
|
| 2387 |
+
try:
|
| 2388 |
+
page = assistant_memory.inspect_page(session.runtime, {"tenant": session.tenant, "username": session.uname},
|
| 2389 |
+
cursor=cursor, limit=limit, order=order)
|
| 2390 |
+
except assistant_memory.InspectRequestError as exc:
|
| 2391 |
+
raise err(400, "bad_request", str(exc)) from exc
|
| 2392 |
doc = assistant_memory.document(session.runtime, session.uname)
|
| 2393 |
+
return {"entries": page["entries"], "total": page["total"], "next": page["next"], "order": order,
|
| 2394 |
+
"dropped": doc["dropped"]}
|
| 2395 |
|
| 2396 |
|
| 2397 |
@router.post("/assistant/confirm")
|
api/routes_ranking_diagnostic.py
CHANGED
|
@@ -13,11 +13,14 @@ def _module():
|
|
| 13 |
return ranking_diagnostic
|
| 14 |
|
| 15 |
|
| 16 |
-
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 17 |
-
"""
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
| 21 |
return session
|
| 22 |
|
| 23 |
|
|
@@ -25,6 +28,8 @@ def _writer(session: Session = Depends(_reader)) -> Session:
|
|
| 25 |
"""Writes also need a role that may edit."""
|
| 26 |
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 27 |
raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.")
|
|
|
|
|
|
|
| 28 |
return session
|
| 29 |
|
| 30 |
|
|
@@ -40,7 +45,7 @@ def _call(fn, *args, **kwargs):
|
|
| 40 |
try:
|
| 41 |
return fn(*args, **kwargs)
|
| 42 |
except PermissionError as exc:
|
| 43 |
-
raise err(403, "
|
| 44 |
except ValueError as exc:
|
| 45 |
raise err(400, "ranking_diagnostic_invalid", str(exc)) from exc
|
| 46 |
except KeyError as exc:
|
|
@@ -82,6 +87,19 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
|
|
| 82 |
return session
|
| 83 |
|
| 84 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
@router.get("/screen")
|
| 86 |
def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 87 |
week: str | None = Query(None), cursor: str | None = Query(None),
|
|
@@ -91,20 +109,25 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 91 |
rd = _module()
|
| 92 |
record = _app(session, app)
|
| 93 |
st = session.runtime
|
|
|
|
| 94 |
held = _call(rd.weeks, _tenant(session), st=st)
|
| 95 |
listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor, limit=limit, q=q,
|
| 96 |
-
week=week or (held[-1] if held else None), st=st)
|
| 97 |
-
return {"canEdit": _may_edit(session, record)
|
|
|
|
|
|
|
|
|
|
| 98 |
"counts": listed["counts"], "items": listed["items"], "cursor": listed["cursor"],
|
| 99 |
"problems": listed["problems"], "statuses": list(rd.STATUSES), "total": listed["total"],
|
| 100 |
-
"sourceTotals": _call(rd.screen_totals, _tenant(session), listed["week"], st=st),
|
| 101 |
"settings": _call(rd.settings, _tenant(session), st=st)}
|
| 102 |
|
| 103 |
|
| 104 |
@router.get("/items/{item_id}")
|
| 105 |
def item_detail(item_id: str, session: Session = Depends(_app_reader)):
|
| 106 |
rd = _module()
|
| 107 |
-
return _call(rd.item_detail, _tenant(session), session.uname, item_id, st=session.runtime
|
|
|
|
| 108 |
|
| 109 |
|
| 110 |
@router.patch("/items/{item_id}")
|
|
@@ -123,7 +146,7 @@ def asin_picker(week: str | None = Query(None), cursor: str | None = Query(None)
|
|
| 123 |
"""The ASIN picker page. A flagged ASIN carries its cause label and item id for the cause card."""
|
| 124 |
rd = _module()
|
| 125 |
page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor, limit=limit, q=q,
|
| 126 |
-
st=session.runtime)
|
| 127 |
labels = {key: label for key, label, _fix in (*rd.CAUSES, rd.UNEXPLAINED)}
|
| 128 |
for row in page["asins"]:
|
| 129 |
flagged = bool(row.get("cause") and page["week"])
|
|
@@ -135,34 +158,34 @@ def asin_picker(week: str | None = Query(None), cursor: str | None = Query(None)
|
|
| 135 |
@router.get("/trend/{asin}")
|
| 136 |
def asin_trend(asin: str, session: Session = Depends(_app_reader)):
|
| 137 |
rd = _module()
|
| 138 |
-
return _call(rd.trend, _tenant(session), asin, st=session.runtime)
|
| 139 |
|
| 140 |
|
| 141 |
@router.get("/content-history/{asin}")
|
| 142 |
-
def listing_content_history(asin: str, cursor: str | None = Query(None),
|
| 143 |
limit: int = Query(200, ge=1, le=200), session: Session = Depends(_app_reader)):
|
| 144 |
-
"""Read one page of saved content changes behind the App and source
|
| 145 |
-
from
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
def _benchmark_reader(session: Session = Depends(_app_reader)) -> Session:
|
| 155 |
-
"""Require the SQP source grant and refuse unscoped row or field access limits."""
|
| 156 |
-
from core import perm_scope
|
| 157 |
from modules import ranking_benchmark
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
raise err(403, "ranking_benchmark_limited", "The benchmark is unavailable with your access limit.")
|
| 163 |
return session
|
| 164 |
|
| 165 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
@router.get("/benchmark")
|
| 167 |
def benchmark(week: str | None = Query(None), cursor: str | None = Query(None),
|
| 168 |
limit: int = Query(200, ge=1, le=200), below_market: bool = Query(False),
|
|
@@ -170,7 +193,7 @@ def benchmark(week: str | None = Query(None), cursor: str | None = Query(None),
|
|
| 170 |
"""Read one weekly benchmark page, filtering below market before selecting the page."""
|
| 171 |
from modules import ranking_benchmark
|
| 172 |
return _call(ranking_benchmark.benchmark, _tenant(session), week, cursor=cursor, limit=limit,
|
| 173 |
-
below_market=below_market, q=q, st=session.runtime)
|
| 174 |
|
| 175 |
|
| 176 |
@router.get("/benchmark/validate")
|
|
@@ -180,7 +203,7 @@ def validate_benchmark(week: str | None = Query(None), cursor: str | None = Quer
|
|
| 180 |
"""Reconcile the selected benchmark page behind the same App and source permission walls."""
|
| 181 |
from modules import ranking_benchmark
|
| 182 |
return _call(ranking_benchmark.validate, _tenant(session), week, cursor=cursor, limit=limit,
|
| 183 |
-
below_market=below_market, q=q, st=session.runtime)
|
| 184 |
|
| 185 |
|
| 186 |
@router.get("/settings")
|
|
@@ -204,4 +227,5 @@ def write_settings(body: dict = Body(...), session: Session = Depends(_app_write
|
|
| 204 |
def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
|
| 205 |
session: Session = Depends(_app_reader)):
|
| 206 |
rd = _module()
|
| 207 |
-
return _call(rd.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime
|
|
|
|
|
|
| 13 |
return ranking_diagnostic
|
| 14 |
|
| 15 |
|
| 16 |
+
def _reader(app: str | None = Query(None), session: Session = Depends(require_session)) -> Session:
|
| 17 |
+
"""R62: reads need the App grant plus the shared source check, never the source open grant."""
|
| 18 |
+
import application_registry
|
| 19 |
+
record = _app(session, app) if app else _call(application_registry.readable_app, session.runtime,
|
| 20 |
+
_tenant(session), session.uname, session.admin, TEMPLATE_KIND)
|
| 21 |
+
from routes_app_dashboard import admit_reader
|
| 22 |
+
admit_reader(session, TEMPLATE_KIND, record)
|
| 23 |
+
session.source_app = record
|
| 24 |
return session
|
| 25 |
|
| 26 |
|
|
|
|
| 28 |
"""Writes also need a role that may edit."""
|
| 29 |
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 30 |
raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.")
|
| 31 |
+
if _source_scope(session) is not None:
|
| 32 |
+
raise err(403, "ranking_diagnostic_limited", "Saved workflow changes need unrestricted source rows and fields.")
|
| 33 |
return session
|
| 34 |
|
| 35 |
|
|
|
|
| 45 |
try:
|
| 46 |
return fn(*args, **kwargs)
|
| 47 |
except PermissionError as exc:
|
| 48 |
+
raise err(403, "ranking_diagnostic_limited", str(exc)) from exc
|
| 49 |
except ValueError as exc:
|
| 50 |
raise err(400, "ranking_diagnostic_invalid", str(exc)) from exc
|
| 51 |
except KeyError as exc:
|
|
|
|
| 87 |
return session
|
| 88 |
|
| 89 |
|
| 90 |
+
def _source_scope(session):
|
| 91 |
+
"""Pass the shared evaluator for every narrowed diagnostic source to the provider."""
|
| 92 |
+
import application_registry
|
| 93 |
+
sources = tuple("ut_" + source for source in _module().SOURCES)
|
| 94 |
+
walls = application_registry.app_access(session.user, session.runtime, None, sources)["narrow"]
|
| 95 |
+
active = {source for source, wall in walls.items() if wall["rows"] or wall["hidden"]}
|
| 96 |
+
if not active:
|
| 97 |
+
return None
|
| 98 |
+
from routes_app_dashboard import source_scope_for
|
| 99 |
+
scope = source_scope_for(session, session.source_app)
|
| 100 |
+
return lambda source, records: scope(source, records) if source in active else None
|
| 101 |
+
|
| 102 |
+
|
| 103 |
@router.get("/screen")
|
| 104 |
def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 105 |
week: str | None = Query(None), cursor: str | None = Query(None),
|
|
|
|
| 109 |
rd = _module()
|
| 110 |
record = _app(session, app)
|
| 111 |
st = session.runtime
|
| 112 |
+
scope = _call(_source_scope, session)
|
| 113 |
held = _call(rd.weeks, _tenant(session), st=st)
|
| 114 |
listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor, limit=limit, q=q,
|
| 115 |
+
week=week or (held[-1] if held else None), st=st, source_scope=scope)
|
| 116 |
+
return {"canEdit": _may_edit(session, record) and scope is None,
|
| 117 |
+
"readiness": listed["readiness"], "columns": listed["columns"],
|
| 118 |
+
**{key: listed["readiness"][key] for key in ("heldWeeks", "weeksNeeded", "nextSync")},
|
| 119 |
+
"week": listed["week"], "weeks": listed["weeks"],
|
| 120 |
"counts": listed["counts"], "items": listed["items"], "cursor": listed["cursor"],
|
| 121 |
"problems": listed["problems"], "statuses": list(rd.STATUSES), "total": listed["total"],
|
| 122 |
+
"sourceTotals": _call(rd.screen_totals, _tenant(session), listed["week"], st=st, source_scope=scope),
|
| 123 |
"settings": _call(rd.settings, _tenant(session), st=st)}
|
| 124 |
|
| 125 |
|
| 126 |
@router.get("/items/{item_id}")
|
| 127 |
def item_detail(item_id: str, session: Session = Depends(_app_reader)):
|
| 128 |
rd = _module()
|
| 129 |
+
return _call(rd.item_detail, _tenant(session), session.uname, item_id, st=session.runtime,
|
| 130 |
+
source_scope=_call(_source_scope, session))
|
| 131 |
|
| 132 |
|
| 133 |
@router.patch("/items/{item_id}")
|
|
|
|
| 146 |
"""The ASIN picker page. A flagged ASIN carries its cause label and item id for the cause card."""
|
| 147 |
rd = _module()
|
| 148 |
page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor, limit=limit, q=q,
|
| 149 |
+
st=session.runtime, source_scope=_call(_source_scope, session))
|
| 150 |
labels = {key: label for key, label, _fix in (*rd.CAUSES, rd.UNEXPLAINED)}
|
| 151 |
for row in page["asins"]:
|
| 152 |
flagged = bool(row.get("cause") and page["week"])
|
|
|
|
| 158 |
@router.get("/trend/{asin}")
|
| 159 |
def asin_trend(asin: str, session: Session = Depends(_app_reader)):
|
| 160 |
rd = _module()
|
| 161 |
+
return _call(rd.trend, _tenant(session), asin, st=session.runtime, source_scope=_call(_source_scope, session))
|
| 162 |
|
| 163 |
|
| 164 |
@router.get("/content-history/{asin}")
|
| 165 |
+
def listing_content_history(asin: str, app: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 166 |
limit: int = Query(200, ge=1, le=200), session: Session = Depends(_app_reader)):
|
| 167 |
+
"""Read one page of saved content changes behind the App grant and the shared source check (R62)."""
|
| 168 |
+
from routes_app_dashboard import admit_reader
|
| 169 |
+
admit_reader(session, TEMPLATE_KIND, _app(session, app), sources=(SOURCE, "ut_amzn_listing_items"))
|
| 170 |
+
return _call(_module().content_history, _tenant(session), asin, cursor=cursor, limit=limit, st=session.runtime,
|
| 171 |
+
source_scope=_call(_source_scope, session))
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def _benchmark_reader(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)) -> Session:
|
| 175 |
+
"""R62: the App grant plus the shared source check over SQP; a wall it cannot narrow refuses."""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
from modules import ranking_benchmark
|
| 177 |
+
from routes_app_dashboard import admit_reader
|
| 178 |
+
record = _app(session, app)
|
| 179 |
+
admit_reader(session, TEMPLATE_KIND, record, sources=(SOURCE, ranking_benchmark.SOURCE))
|
| 180 |
+
session.source_app = record
|
|
|
|
| 181 |
return session
|
| 182 |
|
| 183 |
|
| 184 |
+
def _benchmark_scope(session):
|
| 185 |
+
from routes_app_dashboard import source_scope_for
|
| 186 |
+
return source_scope_for(session, session.source_app)
|
| 187 |
+
|
| 188 |
+
|
| 189 |
@router.get("/benchmark")
|
| 190 |
def benchmark(week: str | None = Query(None), cursor: str | None = Query(None),
|
| 191 |
limit: int = Query(200, ge=1, le=200), below_market: bool = Query(False),
|
|
|
|
| 193 |
"""Read one weekly benchmark page, filtering below market before selecting the page."""
|
| 194 |
from modules import ranking_benchmark
|
| 195 |
return _call(ranking_benchmark.benchmark, _tenant(session), week, cursor=cursor, limit=limit,
|
| 196 |
+
below_market=below_market, q=q, st=session.runtime, source_scope=_benchmark_scope(session))
|
| 197 |
|
| 198 |
|
| 199 |
@router.get("/benchmark/validate")
|
|
|
|
| 203 |
"""Reconcile the selected benchmark page behind the same App and source permission walls."""
|
| 204 |
from modules import ranking_benchmark
|
| 205 |
return _call(ranking_benchmark.validate, _tenant(session), week, cursor=cursor, limit=limit,
|
| 206 |
+
below_market=below_market, q=q, st=session.runtime, source_scope=_benchmark_scope(session))
|
| 207 |
|
| 208 |
|
| 209 |
@router.get("/settings")
|
|
|
|
| 227 |
def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
|
| 228 |
session: Session = Depends(_app_reader)):
|
| 229 |
rd = _module()
|
| 230 |
+
return _call(rd.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime,
|
| 231 |
+
source_scope=_call(_source_scope, session))
|
api/routes_reimbursements.py
CHANGED
|
@@ -1,4 +1,7 @@
|
|
| 1 |
-
"""Reimbursements App routes (C3): claim items, one item with its case text, status, paid rows, validate.
|
|
|
|
|
|
|
|
|
|
| 2 |
import copy
|
| 3 |
import hashlib
|
| 4 |
import json
|
|
@@ -44,19 +47,53 @@ def _module():
|
|
| 44 |
return reimbursements
|
| 45 |
|
| 46 |
|
| 47 |
-
def
|
| 48 |
-
"""
|
| 49 |
-
|
| 50 |
-
if
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
try:
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
return session
|
| 61 |
|
| 62 |
|
|
@@ -95,9 +132,12 @@ def _app(session, app_id):
|
|
| 95 |
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 96 |
except Exception as exc: # noqa: BLE001
|
| 97 |
raise err(503, "reimbursements_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 98 |
-
if (not isinstance(app, dict) or app.get("
|
|
|
|
| 99 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 100 |
raise err(404, "reimbursements_app_not_found", "That Reimbursements App does not exist.")
|
|
|
|
|
|
|
| 101 |
return app
|
| 102 |
|
| 103 |
|
|
@@ -127,12 +167,13 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 127 |
session: Session = Depends(_app_reader)):
|
| 128 |
rb = _module()
|
| 129 |
record = _app(session, app)
|
| 130 |
-
|
|
|
|
| 131 |
held_cursor=held_cursor, limit=limit, st=session.runtime)
|
| 132 |
return {"canEdit": _may_edit(session, record), "counts": listed["counts"], "items": listed["items"],
|
| 133 |
"cursor": listed["cursor"], "total": listed["total"], "offset": listed["offset"],
|
| 134 |
-
"limit": listed["limit"], "paidTotal": listed["paidTotal"],
|
| 135 |
-
"
|
| 136 |
|
| 137 |
|
| 138 |
@router.get("/items/{item_id}")
|
|
@@ -151,7 +192,7 @@ def case_text(item_id: str, app: str = Query(..., min_length=1), session: Sessio
|
|
| 151 |
rb = _module()
|
| 152 |
|
| 153 |
def facts_for_actor():
|
| 154 |
-
_reader(session)
|
| 155 |
_app(session, app)
|
| 156 |
return _call(rb.case_facts, _tenant(session), _user(session), item_id,
|
| 157 |
budget=case_drafts.MAX_FACT_CHARS, st=session.runtime)
|
|
@@ -186,6 +227,7 @@ def update_item(item_id: str, status: str = Body(...), caseId: str | None = Body
|
|
| 186 |
def paid_rows(cursor: str | None = Query(None), search: str = Query(""), limit: int = Query(200, ge=1),
|
| 187 |
session: Session = Depends(_app_reader)):
|
| 188 |
rb = _module()
|
|
|
|
| 189 |
return _call(rb.paid, _tenant(session), session.uname, search=search, cursor=cursor, limit=limit, st=session.runtime)
|
| 190 |
|
| 191 |
|
|
|
|
| 1 |
+
"""Reimbursements App routes (C3): claim items, one item with its case text, status, paid rows, validate.
|
| 2 |
+
|
| 3 |
+
/screen and /paid refresh the App's module tables first (S82-T29), so both Grids read current rows.
|
| 4 |
+
/screen reads its counts from the stored claims table through the reader's row scope (S82-T30)."""
|
| 5 |
import copy
|
| 6 |
import hashlib
|
| 7 |
import json
|
|
|
|
| 47 |
return reimbursements
|
| 48 |
|
| 49 |
|
| 50 |
+
def _on_claim_event(evt):
|
| 51 |
+
"""A Processed tick in the claims Grid becomes set_status as the person who ticked it (S82-T30)."""
|
| 52 |
+
rb = _module()
|
| 53 |
+
if (evt.get("type") != "event_field" or evt.get("table") != rb.CLAIMS_TABLE
|
| 54 |
+
or evt.get("field") != rb.PROCESSED or evt.get("st") is None):
|
| 55 |
+
return
|
| 56 |
+
rb.apply_processed(evt["st"], evt.get("rowId"), evt.get("after"), evt.get("user"),
|
| 57 |
+
allowed=_may_tick(evt["st"], evt.get("user")))
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _may_tick(st, username):
|
| 61 |
+
"""A tick needs what the status PATCH needs: an editor of a live Reimbursements App."""
|
| 62 |
+
import application_registry
|
| 63 |
+
from core import perms
|
| 64 |
+
user = _module()._principal(st, username)
|
| 65 |
+
if not user or str(user.get("role") or "user").strip().lower() == "viewer":
|
| 66 |
+
return False
|
| 67 |
+
admin = perms.is_admin(user)
|
| 68 |
try:
|
| 69 |
+
apps = application_registry.load(st)["applications"].values()
|
| 70 |
+
except Exception: # noqa: BLE001
|
| 71 |
+
return False
|
| 72 |
+
return any(isinstance(app, dict) and app.get("templateKind") == TEMPLATE_KIND and app.get("retired") is not True
|
| 73 |
+
and application_registry.may_edit(app, str(username), is_admin=admin) for app in apps)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _register_row_hook():
|
| 77 |
+
"""Register once, also across a module reload, which makes a new function object."""
|
| 78 |
+
from core import user_tables
|
| 79 |
+
mine = (_on_claim_event.__module__, _on_claim_event.__name__)
|
| 80 |
+
if not any((getattr(h, "__module__", None), getattr(h, "__name__", None)) == mine
|
| 81 |
+
for h in user_tables.ROW_HOOKS):
|
| 82 |
+
user_tables.ROW_HOOKS.append(_on_claim_event)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
_register_row_hook()
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def _reader(session: Session = Depends(require_session), app: str | None = Query(None)) -> Session:
|
| 89 |
+
"""Bind the requested App before admitting its declared source contracts and limits."""
|
| 90 |
+
from routes_app_dashboard import admit_reader
|
| 91 |
+
record = _app(session, app) if isinstance(app, str) and app else None
|
| 92 |
+
admit_reader(session, 'reimbursements', record)
|
| 93 |
+
from core import user_tables
|
| 94 |
+
fields = (user_tables.get_definition(SOURCE, st=session.runtime) or {}).get('fields') or []
|
| 95 |
+
if not set(_module().REIMBURSEMENT_COLS) <= {field.get('key') for field in fields}:
|
| 96 |
+
raise err(503, 'reimbursements_source_contract', 'The reimbursement source fields are unavailable. Sync Amazon.')
|
| 97 |
return session
|
| 98 |
|
| 99 |
|
|
|
|
| 132 |
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 133 |
except Exception as exc: # noqa: BLE001
|
| 134 |
raise err(503, "reimbursements_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 135 |
+
if (not isinstance(app, dict) or app.get("tenant") != _tenant(session)
|
| 136 |
+
or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 137 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 138 |
raise err(404, "reimbursements_app_not_found", "That Reimbursements App does not exist.")
|
| 139 |
+
from routes_app_dashboard import admit_reader
|
| 140 |
+
admit_reader(session, 'reimbursements', app)
|
| 141 |
return app
|
| 142 |
|
| 143 |
|
|
|
|
| 167 |
session: Session = Depends(_app_reader)):
|
| 168 |
rb = _module()
|
| 169 |
record = _app(session, app)
|
| 170 |
+
# rb.screen refreshes the stored tables when stale, then reads them as the Grid does.
|
| 171 |
+
listed = _call(rb.screen, _tenant(session), _user(session), status=status, search=search, cursor=cursor,
|
| 172 |
held_cursor=held_cursor, limit=limit, st=session.runtime)
|
| 173 |
return {"canEdit": _may_edit(session, record), "counts": listed["counts"], "items": listed["items"],
|
| 174 |
"cursor": listed["cursor"], "total": listed["total"], "offset": listed["offset"],
|
| 175 |
+
"limit": listed["limit"], "paidTotal": listed["paidTotal"], "problems": listed["problems"],
|
| 176 |
+
"statuses": list(rb.STATUSES), "held": listed["held"], "toFile": listed["toFile"]}
|
| 177 |
|
| 178 |
|
| 179 |
@router.get("/items/{item_id}")
|
|
|
|
| 192 |
rb = _module()
|
| 193 |
|
| 194 |
def facts_for_actor():
|
| 195 |
+
_reader(session, app=app)
|
| 196 |
_app(session, app)
|
| 197 |
return _call(rb.case_facts, _tenant(session), _user(session), item_id,
|
| 198 |
budget=case_drafts.MAX_FACT_CHARS, st=session.runtime)
|
|
|
|
| 227 |
def paid_rows(cursor: str | None = Query(None), search: str = Query(""), limit: int = Query(200, ge=1),
|
| 228 |
session: Session = Depends(_app_reader)):
|
| 229 |
rb = _module()
|
| 230 |
+
rb.refresh_tables(_tenant(session), session.runtime)
|
| 231 |
return _call(rb.paid, _tenant(session), session.uname, search=search, cursor=cursor, limit=limit, st=session.runtime)
|
| 232 |
|
| 233 |
|
api/routes_repeat_buyers.py
CHANGED
|
@@ -55,17 +55,20 @@ def _app(session, app_id):
|
|
| 55 |
|
| 56 |
|
| 57 |
def _read_sources(session, app_id, sources):
|
|
|
|
| 58 |
_tenant(session)
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
raise err(403, "repeat_buyers_forbidden", "Your account cannot read this App source.")
|
| 64 |
-
if perm_scope.wall_declared(session.user, source, st=session.runtime):
|
| 65 |
-
raise err(403, "repeat_buyers_scoped", "This App cannot read a source with field or row restrictions.")
|
| 66 |
return session
|
| 67 |
|
| 68 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
def _app_reader(app_id: str = Path(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
| 70 |
"""A read names the App and requires both Brand Analytics sources."""
|
| 71 |
return _read_sources(session, app_id, SOURCES)
|
|
@@ -81,7 +84,8 @@ def read_summary(start: str | None = Query(None), end: str | None = Query(None),
|
|
| 81 |
cursor: str | None = Query(None), limit: int = Query(24, ge=1),
|
| 82 |
session: Session = Depends(_app_reader)):
|
| 83 |
rr = _module()
|
| 84 |
-
out = _call(rr.summary, _tenant(session), start=start, end=end, cursor=cursor, limit=limit,
|
|
|
|
| 85 |
return {**out, "canEdit": False}
|
| 86 |
|
| 87 |
|
|
@@ -91,7 +95,7 @@ def list_skus(segment: str = Query("repeat"), start: str | None = Query(None), e
|
|
| 91 |
session: Session = Depends(_app_reader)):
|
| 92 |
rr = _module()
|
| 93 |
return _call(rr.skus, _tenant(session), segment=segment, start=start, end=end, cursor=cursor,
|
| 94 |
-
limit=limit, q=q, st=session.runtime)
|
| 95 |
|
| 96 |
|
| 97 |
@router.get("/{app_id}/sku")
|
|
@@ -99,7 +103,8 @@ def sku_detail(key: str = Query(..., min_length=1), start: str | None = Query(No
|
|
| 99 |
end: str | None = Query(None), cursor: str | None = Query(None), limit: int = Query(25, ge=1),
|
| 100 |
session: Session = Depends(_app_reader)):
|
| 101 |
rr = _module()
|
| 102 |
-
return _call(rr.detail, _tenant(session), key, start=start, end=end, cursor=cursor, limit=limit,
|
|
|
|
| 103 |
|
| 104 |
|
| 105 |
@router.get("/{app_id}/subscribe-save")
|
|
@@ -107,5 +112,5 @@ def subscribe_save(start: str | None = Query(None), end: str | None = Query(None
|
|
| 107 |
cursor: str | None = Query(None), limit: int = Query(100, ge=1), q: str | None = Query(None),
|
| 108 |
session: Session = Depends(_sns_reader)):
|
| 109 |
return _call(_sns_module().per_sku, _tenant(session), start=start, end=end, cursor=cursor,
|
| 110 |
-
limit=limit, q=q, st=session.runtime)
|
| 111 |
|
|
|
|
| 55 |
|
| 56 |
|
| 57 |
def _read_sources(session, app_id, sources):
|
| 58 |
+
import application_registry
|
| 59 |
_tenant(session)
|
| 60 |
+
# R62: the App grant plus the shared source check, never the source open grant.
|
| 61 |
+
record = _app(session, app_id)
|
| 62 |
+
application_registry.admit_app(session, record, sources, code="repeat_buyers")
|
| 63 |
+
session.source_app = record
|
|
|
|
|
|
|
|
|
|
| 64 |
return session
|
| 65 |
|
| 66 |
|
| 67 |
+
def _source_scope(session):
|
| 68 |
+
from routes_app_dashboard import source_scope_for
|
| 69 |
+
return source_scope_for(session, session.source_app)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
def _app_reader(app_id: str = Path(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
| 73 |
"""A read names the App and requires both Brand Analytics sources."""
|
| 74 |
return _read_sources(session, app_id, SOURCES)
|
|
|
|
| 84 |
cursor: str | None = Query(None), limit: int = Query(24, ge=1),
|
| 85 |
session: Session = Depends(_app_reader)):
|
| 86 |
rr = _module()
|
| 87 |
+
out = _call(rr.summary, _tenant(session), start=start, end=end, cursor=cursor, limit=limit,
|
| 88 |
+
st=session.runtime, source_scope=_source_scope(session))
|
| 89 |
return {**out, "canEdit": False}
|
| 90 |
|
| 91 |
|
|
|
|
| 95 |
session: Session = Depends(_app_reader)):
|
| 96 |
rr = _module()
|
| 97 |
return _call(rr.skus, _tenant(session), segment=segment, start=start, end=end, cursor=cursor,
|
| 98 |
+
limit=limit, q=q, st=session.runtime, source_scope=_source_scope(session))
|
| 99 |
|
| 100 |
|
| 101 |
@router.get("/{app_id}/sku")
|
|
|
|
| 103 |
end: str | None = Query(None), cursor: str | None = Query(None), limit: int = Query(25, ge=1),
|
| 104 |
session: Session = Depends(_app_reader)):
|
| 105 |
rr = _module()
|
| 106 |
+
return _call(rr.detail, _tenant(session), key, start=start, end=end, cursor=cursor, limit=limit,
|
| 107 |
+
st=session.runtime, source_scope=_source_scope(session))
|
| 108 |
|
| 109 |
|
| 110 |
@router.get("/{app_id}/subscribe-save")
|
|
|
|
| 112 |
cursor: str | None = Query(None), limit: int = Query(100, ge=1), q: str | None = Query(None),
|
| 113 |
session: Session = Depends(_sns_reader)):
|
| 114 |
return _call(_sns_module().per_sku, _tenant(session), start=start, end=end, cursor=cursor,
|
| 115 |
+
limit=limit, q=q, st=session.runtime, source_scope=_source_scope(session))
|
| 116 |
|
api/routes_restock_planner.py
CHANGED
|
@@ -50,10 +50,9 @@ def _module():
|
|
| 50 |
|
| 51 |
|
| 52 |
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 53 |
-
"""
|
| 54 |
-
from
|
| 55 |
-
|
| 56 |
-
raise err(403, "restock_planner_forbidden", "Your account cannot read Amazon inventory planning.")
|
| 57 |
return session
|
| 58 |
|
| 59 |
|
|
@@ -91,6 +90,8 @@ def _app(session, app_id):
|
|
| 91 |
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 92 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 93 |
raise err(404, "restock_planner_app_not_found", "That Restock planner App does not exist.")
|
|
|
|
|
|
|
| 94 |
return app
|
| 95 |
|
| 96 |
|
|
@@ -147,7 +148,7 @@ def list_items(app: str = Query(..., min_length=1), band: str | None = Query(Non
|
|
| 147 |
rp = _module()
|
| 148 |
listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor,
|
| 149 |
limit=min(limit, rp.MAX_PAGE_SIZE), band=band, q=q, low_inventory_fee=low_inventory_fee,
|
| 150 |
-
st=session.runtime)
|
| 151 |
return {**listed, "owners": owners(session), "thresholds": thresholds(session),
|
| 152 |
"canEdit": _may_edit(session, _app(session, app))}
|
| 153 |
|
|
@@ -204,6 +205,18 @@ def read_capacity(session: Session = Depends(_app_reader)):
|
|
| 204 |
return _call(_module().capacity, _tenant(session), st=session.runtime)
|
| 205 |
|
| 206 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
@router.post("/items/{item_id}/cancel-plan")
|
| 208 |
def cancel_plan(item_id: str, request: CancelPlanRequest | None = Body(None),
|
| 209 |
session: Session = Depends(_app_writer)):
|
|
@@ -248,9 +261,86 @@ def _digest_module():
|
|
| 248 |
return restock_digest
|
| 249 |
|
| 250 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
def _deliver_digest(rt, kind, items):
|
| 252 |
"""Resolve C5 only when dispatching so a missing adapter cannot break API boot."""
|
| 253 |
-
return importlib.import_module("notify_destinations").deliver(rt, kind, items)
|
| 254 |
|
| 255 |
|
| 256 |
def _send_digest(rt, app_id, day=None, *, deliver=None):
|
|
|
|
| 50 |
|
| 51 |
|
| 52 |
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 53 |
+
"""Admit a live App holder and refuse unsupported source row or field limits."""
|
| 54 |
+
from routes_app_dashboard import admit_reader
|
| 55 |
+
admit_reader(session, 'restock_planner')
|
|
|
|
| 56 |
return session
|
| 57 |
|
| 58 |
|
|
|
|
| 90 |
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 91 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 92 |
raise err(404, "restock_planner_app_not_found", "That Restock planner App does not exist.")
|
| 93 |
+
from routes_app_dashboard import admit_reader
|
| 94 |
+
admit_reader(session, 'restock_planner', app)
|
| 95 |
return app
|
| 96 |
|
| 97 |
|
|
|
|
| 148 |
rp = _module()
|
| 149 |
listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor,
|
| 150 |
limit=min(limit, rp.MAX_PAGE_SIZE), band=band, q=q, low_inventory_fee=low_inventory_fee,
|
| 151 |
+
st=session.runtime, adopt=True)
|
| 152 |
return {**listed, "owners": owners(session), "thresholds": thresholds(session),
|
| 153 |
"canEdit": _may_edit(session, _app(session, app))}
|
| 154 |
|
|
|
|
| 205 |
return _call(_module().capacity, _tenant(session), st=session.runtime)
|
| 206 |
|
| 207 |
|
| 208 |
+
@router.get("/tiles")
|
| 209 |
+
def read_tiles(days: int = Query(30, ge=1, le=60), session: Session = Depends(_app_reader)):
|
| 210 |
+
from core import perm_scope
|
| 211 |
+
from odoo_relational import PRODUCTS_KEY, PURCHASE_ORDERS_KEY, PURCHASE_LINES_KEY
|
| 212 |
+
sources = (SOURCE, "ut_amzn_restock_recommendations", _module().LINK_TABLE,
|
| 213 |
+
PRODUCTS_KEY, PURCHASE_ORDERS_KEY, PURCHASE_LINES_KEY)
|
| 214 |
+
if any(not perm_scope.may_read(session.user, source, st=session.runtime)
|
| 215 |
+
or perm_scope.wall_declared(session.user, source, st=session.runtime) for source in sources):
|
| 216 |
+
raise err(403, "restock_tiles_forbidden", "These tiles need access to their full source rows and fields")
|
| 217 |
+
return _call(_module().tiles, _tenant(session), st=session.runtime, days=days)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
@router.post("/items/{item_id}/cancel-plan")
|
| 221 |
def cancel_plan(item_id: str, request: CancelPlanRequest | None = Body(None),
|
| 222 |
session: Session = Depends(_app_writer)):
|
|
|
|
| 261 |
return restock_digest
|
| 262 |
|
| 263 |
|
| 264 |
+
DIGEST_BODY_LIMIT = 8000
|
| 265 |
+
DIGEST_NO_DESTINATION = "The Restock owner has chosen no destination for the weekly digest."
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def _digest_rows(rt, skus):
|
| 269 |
+
"""{sku: Restock table row id} for the listed SKUs, so Slack can check the reader's access."""
|
| 270 |
+
from core import user_tables
|
| 271 |
+
want = set(skus)
|
| 272 |
+
if not want:
|
| 273 |
+
return {}
|
| 274 |
+
try:
|
| 275 |
+
table = user_tables.get_one(_module().TABLE_KEY, rt) or {}
|
| 276 |
+
except Exception: # noqa: BLE001
|
| 277 |
+
# Without row ids Slack refuses with its own named reason; email still delivers.
|
| 278 |
+
return {}
|
| 279 |
+
return {str(row.get("sku") or "").strip(): str(rid) for rid, row in (table.get("rows") or {}).items()
|
| 280 |
+
if isinstance(row, dict) and str(row.get("sku") or "").strip() in want}
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def _digest_items(rt, envelope):
|
| 284 |
+
"""D-12074: the digest envelope as C5 items that pass `clean_items`, split so no body is cut."""
|
| 285 |
+
lines = [("".join(c if c == "\t" or ord(c) >= 32 else " " for c in line))
|
| 286 |
+
for line in str(envelope.get("body") or "").split("\n")]
|
| 287 |
+
skus = [row.get("sku") for row in ((envelope.get("digest") or {}).get("items") or ())]
|
| 288 |
+
rows = _digest_rows(rt, [s for s in skus if s])
|
| 289 |
+
# A long line splits into pieces and a long body into numbered items: nothing is cut.
|
| 290 |
+
pieces = [(index, line[start:start + DIGEST_BODY_LIMIT]) for index, line in enumerate(lines)
|
| 291 |
+
for start in range(0, max(len(line), 1), DIGEST_BODY_LIMIT)]
|
| 292 |
+
chunks, current, size = [], [], 0
|
| 293 |
+
for index, text in pieces:
|
| 294 |
+
extra = len(text) + (1 if current else 0)
|
| 295 |
+
if current and size + extra > DIGEST_BODY_LIMIT:
|
| 296 |
+
chunks.append(current)
|
| 297 |
+
current, size, extra = [], 0, len(text)
|
| 298 |
+
current.append((index, text))
|
| 299 |
+
size += extra
|
| 300 |
+
chunks.append(current)
|
| 301 |
+
out = []
|
| 302 |
+
for number, chunk in enumerate(chunks, 1):
|
| 303 |
+
body = "\n".join(line for _i, line in chunk).strip() or "Nothing urgent"
|
| 304 |
+
item = {"id": str(envelope["id"]) if len(chunks) == 1 else f"{envelope['id']}-{number}",
|
| 305 |
+
"title": str(envelope.get("title") or "Weekly Restock digest")
|
| 306 |
+
+ ("" if len(chunks) == 1 else f" ({number} of {len(chunks)})"),
|
| 307 |
+
"body": body}
|
| 308 |
+
if envelope.get("link"):
|
| 309 |
+
item["url"] = str(envelope["link"])
|
| 310 |
+
ids = [rows[skus[i]] for i, _line in chunk if i < len(skus) and skus[i] in rows]
|
| 311 |
+
if ids:
|
| 312 |
+
item["source"] = {"table": _module().TABLE_KEY, "rowIds": list(dict.fromkeys(ids))}
|
| 313 |
+
out.append(item)
|
| 314 |
+
return out
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def _digest_receipt(delivery):
|
| 318 |
+
"""D-12074: C5's delivery result in the shape `restock_digest._outcome` reads."""
|
| 319 |
+
if not isinstance(delivery, dict):
|
| 320 |
+
return delivery
|
| 321 |
+
results = [row for row in delivery.get("results") or () if isinstance(row, dict)]
|
| 322 |
+
destinations = [{"id": row.get("destination"), "state": row.get("state"),
|
| 323 |
+
"reason": row.get("message") or row.get("reason")} for row in results]
|
| 324 |
+
sent = sum(row["state"] == "sent" for row in destinations)
|
| 325 |
+
state = "sent" if destinations and sent == len(destinations) else "partial" if sent else "failed"
|
| 326 |
+
out = {"state": state, "destinations": destinations, "sent": sent,
|
| 327 |
+
"refused": len(destinations) - sent, "itemCount": delivery.get("itemCount")}
|
| 328 |
+
if not destinations:
|
| 329 |
+
out["reason"] = DIGEST_NO_DESTINATION
|
| 330 |
+
return out
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def _deliver_digest_with(deliver):
|
| 334 |
+
"""`restock_digest.send`'s deliver contract over one C5 `deliver` function."""
|
| 335 |
+
def _deliver(rt, kind, items):
|
| 336 |
+
envelope = items[0]
|
| 337 |
+
return _digest_receipt(deliver(rt, kind, _digest_items(rt, envelope), user=envelope["recipient"]))
|
| 338 |
+
return _deliver
|
| 339 |
+
|
| 340 |
+
|
| 341 |
def _deliver_digest(rt, kind, items):
|
| 342 |
"""Resolve C5 only when dispatching so a missing adapter cannot break API boot."""
|
| 343 |
+
return _deliver_digest_with(importlib.import_module("notify_destinations").deliver)(rt, kind, items)
|
| 344 |
|
| 345 |
|
| 346 |
def _send_digest(rt, app_id, day=None, *, deliver=None):
|