Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- RELEASES.json +1 -1
- VERSION +1 -1
- api/amazon_relational.py +2 -15
- api/application_registry.py +1 -11
- api/automation_engine.py +7 -26
- api/deps.py +4 -60
- api/loopy_catalog_tools.py +1 -10
- api/main.py +64 -146
- api/notify_destinations.py +3 -111
- api/odoo_relational.py +3 -74
- api/routes_account_health.py +10 -27
- api/routes_admin.py +179 -10
- api/routes_amazon_projects.py +21 -160
- api/routes_assets.py +1 -14
- api/routes_connectors.py +9 -110
- api/routes_external.py +10 -119
- api/routes_odoo_tables.py +13 -71
- api/routes_tables.py +0 -2
- api/sync_worker.py +2 -4
- api/tool_groups/apps.py +1 -9
- api/whatsapp_identity.py +3 -13
- platform/core/activity_log.py +6 -7
- platform/core/amazon_project_results.py +1 -116
- platform/core/amazon_projects.py +2 -3
- platform/core/connector_sync.py +2 -28
- platform/core/external_access.py +27 -77
- platform/core/notify_destinations.py +13 -92
- platform/core/sheets.py +2 -3
- platform/core/table_store.py +30 -49
- platform/core/test_accounts.py +398 -0
- platform/core/user_tables.py +0 -2
- platform/core/users.py +112 -2
- platform/core/windows.py +1 -4
- platform/harness/amazon_ads_store.py +14 -68
- platform/harness/amazon_client.py +5 -43
- platform/harness/amazon_store.py +67 -384
- platform/harness/canonical_source.py +2 -2
- platform/harness/datastore.py +2 -21
- platform/modules/account_health.py +7 -161
- platform/modules/amazon_profit.py +21 -210
- platform/modules/amazon_profit_account.py +5 -9
- platform/modules/procurement.py +10 -27
- platform/modules/returns_refunds.py +31 -80
- platform/modules/warehouse.py +2 -8
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/apps/AppRail.tsx +1 -3
- web/src/apps/account-health/AppView.tsx +10 -175
- web/src/apps/amazon-profit/AppView.tsx +22 -47
- web/src/apps/amazon-profit/ProjectResults.tsx +137 -122
- web/src/apps/amazon-profit/Projects.tsx +135 -332
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "b876ebd5f145",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
b876ebd5f145
|
api/amazon_relational.py
CHANGED
|
@@ -734,7 +734,7 @@ def _read_through(cur, key, count, *, columns=None):
|
|
| 734 |
if key in READ_THROUGH or count > ut.MAX_ROWS:
|
| 735 |
return True
|
| 736 |
if not count:
|
| 737 |
-
return
|
| 738 |
names = [name for name, _dtype in (columns if columns is not None
|
| 739 |
else mirror_columns(cur, SOURCE[key]))]
|
| 740 |
sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
|
|
@@ -1106,13 +1106,6 @@ def plan(cur, rt=None):
|
|
| 1106 |
n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 1107 |
except Exception: # noqa: BLE001
|
| 1108 |
continue # absent = never synced
|
| 1109 |
-
if key == PREFIX + "inventory" and not n:
|
| 1110 |
-
# A measured empty inventory report remains a declared database with a source reason.
|
| 1111 |
-
status = amazon_store.inventory_table_status(cur)
|
| 1112 |
-
built[key] = {"readThrough": True,
|
| 1113 |
-
"fields": _fields_for(cur, key, read_through=True), "count": n,
|
| 1114 |
-
"sourceStatus": {name: status[name] for name in ("empty", "code", "reason")}}
|
| 1115 |
-
continue
|
| 1116 |
if _read_through(cur, key, n):
|
| 1117 |
built[key] = {"readThrough": True,
|
| 1118 |
"fields": _fields_for(cur, key, read_through=True), "count": n}
|
|
@@ -1419,7 +1412,7 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1419 |
not_stored.clear()
|
| 1420 |
live_columns = {key: {field["key"] for field in table.get("fields", ())}
|
| 1421 |
for key, table in doc.items() if key in SOURCE and isinstance(table, dict)}
|
| 1422 |
-
#
|
| 1423 |
unborn = {key for key, _label, _fields, rows in plans
|
| 1424 |
if isinstance(rows, list) and not rows and not isinstance(doc.get(key), dict)}
|
| 1425 |
live_columns.update({key: {field["key"] for field in fields}
|
|
@@ -1513,12 +1506,6 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1513 |
saved = stored.get(row["_id"])
|
| 1514 |
if numeric and isinstance(saved, dict):
|
| 1515 |
saved.update({name: row[name] for name in numeric if name in row})
|
| 1516 |
-
if key == PREFIX + "inventory":
|
| 1517 |
-
status = rows.get("sourceStatus") if isinstance(rows, dict) else None
|
| 1518 |
-
if status is not None:
|
| 1519 |
-
doc[key]["sourceStatus"] = dict(status)
|
| 1520 |
-
else:
|
| 1521 |
-
doc[key].pop("sourceStatus", None)
|
| 1522 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 1523 |
# mutable display label or from the process having imported this module.
|
| 1524 |
doc[key]["connector"] = source_attribution(key)
|
|
|
|
| 734 |
if key in READ_THROUGH or count > ut.MAX_ROWS:
|
| 735 |
return True
|
| 736 |
if not count:
|
| 737 |
+
return False
|
| 738 |
names = [name for name, _dtype in (columns if columns is not None
|
| 739 |
else mirror_columns(cur, SOURCE[key]))]
|
| 740 |
sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
|
|
|
|
| 1106 |
n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 1107 |
except Exception: # noqa: BLE001
|
| 1108 |
continue # absent = never synced
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1109 |
if _read_through(cur, key, n):
|
| 1110 |
built[key] = {"readThrough": True,
|
| 1111 |
"fields": _fields_for(cur, key, read_through=True), "count": n}
|
|
|
|
| 1412 |
not_stored.clear()
|
| 1413 |
live_columns = {key: {field["key"] for field in table.get("fields", ())}
|
| 1414 |
for key, table in doc.items() if key in SOURCE and isinstance(table, dict)}
|
| 1415 |
+
# S78-T02: an empty population reconciles an EXISTING definition only; it creates none.
|
| 1416 |
unborn = {key for key, _label, _fields, rows in plans
|
| 1417 |
if isinstance(rows, list) and not rows and not isinstance(doc.get(key), dict)}
|
| 1418 |
live_columns.update({key: {field["key"] for field in fields}
|
|
|
|
| 1506 |
saved = stored.get(row["_id"])
|
| 1507 |
if numeric and isinstance(saved, dict):
|
| 1508 |
saved.update({name: row[name] for name in numeric if name in row})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1509 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 1510 |
# mutable display label or from the process having imported this module.
|
| 1511 |
doc[key]["connector"] = source_attribution(key)
|
api/application_registry.py
CHANGED
|
@@ -1415,16 +1415,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 1415 |
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
|
| 1416 |
|
| 1417 |
|
| 1418 |
-
def _nav_placement(app, index):
|
| 1419 |
-
"""Put default Project management entries first without rewriting stored placements."""
|
| 1420 |
-
placement = app.get("navPlacement")
|
| 1421 |
-
if (app.get("templateKind") == "amazon_profit"
|
| 1422 |
-
and (not placement or placement == {"section": "app", "order": 0})):
|
| 1423 |
-
# A zero placement already asks for first place. Break its default tie in the rail.
|
| 1424 |
-
return {"section": "app", "order": -1}
|
| 1425 |
-
return placement or {"section": "app", "order": index}
|
| 1426 |
-
|
| 1427 |
-
|
| 1428 |
def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_databases=None):
|
| 1429 |
"""The App home read: stored records the caller owns or was shared, never a code projection."""
|
| 1430 |
return [{"id": app["id"], "name": app.get("name"),
|
|
@@ -1435,7 +1425,7 @@ def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_da
|
|
| 1435 |
"origin": "tenant",
|
| 1436 |
"compatibleDatabases": app.get("compatibleDatabases") or [],
|
| 1437 |
"openTarget": app.get("openTarget"),
|
| 1438 |
-
"navPlacement":
|
| 1439 |
"capabilities": app.get("capabilities"), "starred": app.get("starred"),
|
| 1440 |
"isDefault": app.get("isDefault")}
|
| 1441 |
for index, app in enumerate(list_apps(runtime, user, is_admin,
|
|
|
|
| 1415 |
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
|
| 1416 |
|
| 1417 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1418 |
def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_databases=None):
|
| 1419 |
"""The App home read: stored records the caller owns or was shared, never a code projection."""
|
| 1420 |
return [{"id": app["id"], "name": app.get("name"),
|
|
|
|
| 1425 |
"origin": "tenant",
|
| 1426 |
"compatibleDatabases": app.get("compatibleDatabases") or [],
|
| 1427 |
"openTarget": app.get("openTarget"),
|
| 1428 |
+
"navPlacement": app.get("navPlacement") or {"section": "app", "order": index},
|
| 1429 |
"capabilities": app.get("capabilities"), "starred": app.get("starred"),
|
| 1430 |
"isDefault": app.get("isDefault")}
|
| 1431 |
for index, app in enumerate(list_apps(runtime, user, is_admin,
|
api/automation_engine.py
CHANGED
|
@@ -2808,12 +2808,9 @@ def alert_principal(rt, username):
|
|
| 2808 |
from core import account_rows, users
|
| 2809 |
resolve = getattr(rt, '_document_store', None)
|
| 2810 |
record = account_rows.get(username, st=resolve() if callable(resolve) else rt, fresh=True)
|
| 2811 |
-
if not record or record.get('active', True) is False:
|
| 2812 |
-
raise PermissionError('This alert owner is no longer active in this workspace.')
|
| 2813 |
-
principal = users._public(username, record)
|
| 2814 |
-
if principal['tenant'] != rt.key:
|
| 2815 |
raise PermissionError('This alert owner is no longer active in this workspace.')
|
| 2816 |
-
return
|
| 2817 |
|
| 2818 |
|
| 2819 |
def alert_schema(rt, principal, database):
|
|
@@ -14611,16 +14608,6 @@ def _mirror_table(rt, key, defn):
|
|
| 14611 |
return table, None
|
| 14612 |
|
| 14613 |
|
| 14614 |
-
def _relation_native_rows(rt, key):
|
| 14615 |
-
"""Read omitted native rows only through the caller's tenant runtime."""
|
| 14616 |
-
if not _relation_tenant(rt):
|
| 14617 |
-
raise RuntimeError('A tenant runtime is required to read relation rows.')
|
| 14618 |
-
table = _ut().get_one(key, st=rt)
|
| 14619 |
-
if table is not None and 'rows' not in table:
|
| 14620 |
-
raise RuntimeError('The relation row reader returned only a definition.')
|
| 14621 |
-
return table
|
| 14622 |
-
|
| 14623 |
-
|
| 14624 |
def _resolve_table(rt, store, key, memo, want_rows=False):
|
| 14625 |
"""A database by key, from EITHER namespace — `(table_or_None, limit_or_None)`.
|
| 14626 |
|
|
@@ -14654,10 +14641,6 @@ def _resolve_table(rt, store, key, memo, want_rows=False):
|
|
| 14654 |
if cached is None:
|
| 14655 |
cached = memo[("mirror", key)] = _mirror_table(rt, key, hit)
|
| 14656 |
return cached
|
| 14657 |
-
if want_rows and 'rows' not in hit:
|
| 14658 |
-
if ('native', key) not in memo:
|
| 14659 |
-
memo[('native', key)] = (_relation_native_rows(rt, key), None)
|
| 14660 |
-
return memo[('native', key)]
|
| 14661 |
return hit, None
|
| 14662 |
return _module_table(rt, key, memo, want_rows=want_rows)
|
| 14663 |
|
|
@@ -14871,9 +14854,9 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14871 |
ifs = [f for f in fields if f.get('type') == 'if']
|
| 14872 |
if not links and not rollups and not ifs:
|
| 14873 |
return {}, limits
|
| 14874 |
-
rows = (t.get("rows") or {}
|
| 14875 |
-
if t is not None and (
|
| 14876 |
-
|
| 14877 |
# A registry source: NOW the pool is worth its cost, because this table really does
|
| 14878 |
# declare relations. One fetch, memoised, shared with every link and rollup below.
|
| 14879 |
# ⭐⭐ W53-T40 (d) — AND A READ-THROUGH SOURCE TAKES THE SAME SECOND FETCH, for a reason
|
|
@@ -15600,13 +15583,11 @@ class _RelationSnapshot(dict):
|
|
| 15600 |
if key not in self and key not in self.missing:
|
| 15601 |
if self.supplied is not None and key in self.supplied:
|
| 15602 |
table = copy.deepcopy(self.supplied[key])
|
| 15603 |
-
if table is not None and 'rows' not in table and _ut().materialises(key, defn=table):
|
| 15604 |
-
table = _relation_native_rows(self.runtime, key)
|
| 15605 |
else:
|
| 15606 |
definition = self.definitions.get(key)
|
| 15607 |
if definition is None:
|
| 15608 |
definition = _ut().get_definition(key, st=self.runtime)
|
| 15609 |
-
table = (
|
| 15610 |
if definition and _ut().materialises(key, defn=definition)
|
| 15611 |
else copy.deepcopy(definition))
|
| 15612 |
if table is None:
|
|
@@ -15646,7 +15627,7 @@ def refresh_relations(rt, log=print, tables=None, persist=True, *, target=None):
|
|
| 15646 |
return local
|
| 15647 |
patches = {}
|
| 15648 |
for key, old in before.items():
|
| 15649 |
-
if not isinstance(old, dict)
|
| 15650 |
continue
|
| 15651 |
fields = old.get('fields') or []
|
| 15652 |
derived = {str(f['key']) for f in fields if f.get('key')
|
|
|
|
| 2808 |
from core import account_rows, users
|
| 2809 |
resolve = getattr(rt, '_document_store', None)
|
| 2810 |
record = account_rows.get(username, st=resolve() if callable(resolve) else rt, fresh=True)
|
| 2811 |
+
if not record or record.get('active', True) is False or record.get('tenant') != rt.key:
|
|
|
|
|
|
|
|
|
|
| 2812 |
raise PermissionError('This alert owner is no longer active in this workspace.')
|
| 2813 |
+
return users._public(username, record)
|
| 2814 |
|
| 2815 |
|
| 2816 |
def alert_schema(rt, principal, database):
|
|
|
|
| 14608 |
return table, None
|
| 14609 |
|
| 14610 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14611 |
def _resolve_table(rt, store, key, memo, want_rows=False):
|
| 14612 |
"""A database by key, from EITHER namespace — `(table_or_None, limit_or_None)`.
|
| 14613 |
|
|
|
|
| 14641 |
if cached is None:
|
| 14642 |
cached = memo[("mirror", key)] = _mirror_table(rt, key, hit)
|
| 14643 |
return cached
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14644 |
return hit, None
|
| 14645 |
return _module_table(rt, key, memo, want_rows=want_rows)
|
| 14646 |
|
|
|
|
| 14854 |
ifs = [f for f in fields if f.get('type') == 'if']
|
| 14855 |
if not links and not rollups and not ifs:
|
| 14856 |
return {}, limits
|
| 14857 |
+
rows = (t or {}).get("rows") or {}
|
| 14858 |
+
if not rows and t is not None and (not t.get("_rows_built", True)
|
| 14859 |
+
or not _ut().materialises(table_key, defn=t)):
|
| 14860 |
# A registry source: NOW the pool is worth its cost, because this table really does
|
| 14861 |
# declare relations. One fetch, memoised, shared with every link and rollup below.
|
| 14862 |
# ⭐⭐ W53-T40 (d) — AND A READ-THROUGH SOURCE TAKES THE SAME SECOND FETCH, for a reason
|
|
|
|
| 15583 |
if key not in self and key not in self.missing:
|
| 15584 |
if self.supplied is not None and key in self.supplied:
|
| 15585 |
table = copy.deepcopy(self.supplied[key])
|
|
|
|
|
|
|
| 15586 |
else:
|
| 15587 |
definition = self.definitions.get(key)
|
| 15588 |
if definition is None:
|
| 15589 |
definition = _ut().get_definition(key, st=self.runtime)
|
| 15590 |
+
table = (_ut().get_one(key, st=self.runtime)
|
| 15591 |
if definition and _ut().materialises(key, defn=definition)
|
| 15592 |
else copy.deepcopy(definition))
|
| 15593 |
if table is None:
|
|
|
|
| 15627 |
return local
|
| 15628 |
patches = {}
|
| 15629 |
for key, old in before.items():
|
| 15630 |
+
if not isinstance(old, dict):
|
| 15631 |
continue
|
| 15632 |
fields = old.get('fields') or []
|
| 15633 |
derived = {str(f['key']) for f in fields if f.get('key')
|
api/deps.py
CHANGED
|
@@ -1809,62 +1809,17 @@ def _touch_active(uname):
|
|
| 1809 |
pass
|
| 1810 |
|
| 1811 |
|
| 1812 |
-
def refuse_view_as_write(request: Request):
|
| 1813 |
-
"""Refuse every view-as method except GET and HEAD before any write."""
|
| 1814 |
-
if "X-View-As" in request.headers and request.method not in {"GET", "HEAD"}:
|
| 1815 |
-
raise err(403, "view_as_read_only", "Viewing as a member is read-only.")
|
| 1816 |
-
|
| 1817 |
-
|
| 1818 |
-
def _view_as_refused():
|
| 1819 |
-
return err(403, "view_as_refused", "This member view is unavailable.")
|
| 1820 |
-
|
| 1821 |
-
|
| 1822 |
-
def _view_as_user(request, actor, tenant, rt):
|
| 1823 |
-
"""Resolve an active member of the actor's tenant and audit this request once."""
|
| 1824 |
-
if "X-View-As" not in request.headers:
|
| 1825 |
-
return actor
|
| 1826 |
-
if not perms.is_admin(actor):
|
| 1827 |
-
raise _view_as_refused()
|
| 1828 |
-
started_at = datetime.now(timezone.utc).isoformat()
|
| 1829 |
-
target = request.headers["X-View-As"].strip().lower()
|
| 1830 |
-
if not target:
|
| 1831 |
-
raise _view_as_refused()
|
| 1832 |
-
rec = copy.deepcopy(_session_registry(target).get(target))
|
| 1833 |
-
if (not rec or not rec.get("active", True) or rec.get("role", "user") != "user"
|
| 1834 |
-
or str(rec.get("tenant") or "").strip().lower() != tenant):
|
| 1835 |
-
raise _view_as_refused()
|
| 1836 |
-
member = users._public(target, rec)
|
| 1837 |
-
audit_key = (tenant, actor["username"], target)
|
| 1838 |
-
if getattr(request.state, "_view_as_audited", None) != audit_key:
|
| 1839 |
-
from core import activity_log
|
| 1840 |
-
activity_log.append(
|
| 1841 |
-
rt, kind="view_as", object={"app": "settings", "id": target},
|
| 1842 |
-
by=actor["username"], on_behalf_of=target,
|
| 1843 |
-
changes={"member": {"before": None, "after": target},
|
| 1844 |
-
"started_at": {"before": None, "after": started_at}})
|
| 1845 |
-
request.state._view_as_audited = audit_key
|
| 1846 |
-
return member
|
| 1847 |
-
|
| 1848 |
-
|
| 1849 |
def require_session(request: Request, response: Response) -> Session:
|
| 1850 |
"""Verify the cookie, resolve the tenant, and REISSUE the cookie (the 8h idle window is
|
| 1851 |
refreshed on use; the 30-day absolute expiry is carried over, never extended)."""
|
| 1852 |
-
refuse_view_as_write(request)
|
| 1853 |
-
viewing_as = "X-View-As" in request.headers
|
| 1854 |
raw = request.cookies.get(aios_session.COOKIE_NAME)
|
| 1855 |
if not raw:
|
| 1856 |
-
if viewing_as:
|
| 1857 |
-
raise _view_as_refused()
|
| 1858 |
raise err(401, "no_session", "sign in to continue")
|
| 1859 |
claims = aios_session.read(raw)
|
| 1860 |
if not claims:
|
| 1861 |
-
if viewing_as:
|
| 1862 |
-
raise _view_as_refused()
|
| 1863 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1864 |
user = _user_for(claims)
|
| 1865 |
if not user:
|
| 1866 |
-
if viewing_as:
|
| 1867 |
-
raise _view_as_refused()
|
| 1868 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1869 |
identity = {_TENANT_HEADER: str(claims["t"]),
|
| 1870 |
_PRINCIPAL_HEADER: _principal(claims["t"], user["username"])}
|
|
@@ -1880,12 +1835,8 @@ def require_session(request: Request, response: Response) -> Session:
|
|
| 1880 |
except KeyError:
|
| 1881 |
# An unknown tenant slug is a 401, not a 404: the cookie is not a valid identity for this
|
| 1882 |
# deployment, and confirming which slugs exist would answer a question we were not asked.
|
| 1883 |
-
if viewing_as:
|
| 1884 |
-
raise _view_as_refused()
|
| 1885 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1886 |
-
|
| 1887 |
-
user = _view_as_user(request, actor, claims["t"], rt)
|
| 1888 |
-
fresh, _ = aios_session.mint(claims["t"], actor["username"], int(actor.get("epoch") or 0),
|
| 1889 |
absolute_expiry=claims["x"])
|
| 1890 |
aios_session.set_cookie(response, request, fresh)
|
| 1891 |
# ⭐ STAMP THE TENANT THIS ANSWER WAS SERVED FOR (owner report 2026-08-09: "I can see
|
|
@@ -1906,13 +1857,11 @@ def require_session(request: Request, response: Response) -> Session:
|
|
| 1906 |
# `session.tenant` read back from something the request could influence.
|
| 1907 |
response.headers[_TENANT_HEADER] = str(claims["t"])
|
| 1908 |
response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"])
|
| 1909 |
-
|
| 1910 |
-
|
| 1911 |
-
# Ordinary reads only observe activity. View-as reads write their audit without
|
| 1912 |
-
# changing either person's activity stamps.
|
| 1913 |
# qa-runner has no global account record. Never let the normal activity telemetry create or
|
| 1914 |
# touch one through the default tenant-#0 users store.
|
| 1915 |
-
if (not
|
| 1916 |
and not (request.method == "GET" and request.url.path == "/api/v1/notifications")):
|
| 1917 |
_observe_active(claims["t"], user.get("username"))
|
| 1918 |
if request.method not in {"GET", "HEAD", "OPTIONS"}:
|
|
@@ -1928,8 +1877,6 @@ def require_changes_session(request: Request, response: Response) -> Session | N
|
|
| 1928 |
verified signed cookie is enough to return that no-data response; resolving the account record
|
| 1929 |
would itself turn those legacy requests into a permanent Neon keepalive.
|
| 1930 |
"""
|
| 1931 |
-
if "X-View-As" in request.headers:
|
| 1932 |
-
return require_session(request, response)
|
| 1933 |
if store.backend() != "pg":
|
| 1934 |
return require_session(request, response)
|
| 1935 |
if request.headers.get("x-aios-change-check") == "focus-v1":
|
|
@@ -1971,9 +1918,6 @@ def require_session_or_bearer(request: Request, response: Response) -> Session:
|
|
| 1971 |
|
| 1972 |
`require_session` stays cookie only. A bearer session never reissues the cookie.
|
| 1973 |
"""
|
| 1974 |
-
refuse_view_as_write(request)
|
| 1975 |
-
if "X-View-As" in request.headers:
|
| 1976 |
-
return _cookie_session(request, response)
|
| 1977 |
bearer = _bearer_header(request)
|
| 1978 |
if bearer is None or request.cookies.get(aios_session.COOKIE_NAME):
|
| 1979 |
try:
|
|
|
|
| 1809 |
pass
|
| 1810 |
|
| 1811 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1812 |
def require_session(request: Request, response: Response) -> Session:
|
| 1813 |
"""Verify the cookie, resolve the tenant, and REISSUE the cookie (the 8h idle window is
|
| 1814 |
refreshed on use; the 30-day absolute expiry is carried over, never extended)."""
|
|
|
|
|
|
|
| 1815 |
raw = request.cookies.get(aios_session.COOKIE_NAME)
|
| 1816 |
if not raw:
|
|
|
|
|
|
|
| 1817 |
raise err(401, "no_session", "sign in to continue")
|
| 1818 |
claims = aios_session.read(raw)
|
| 1819 |
if not claims:
|
|
|
|
|
|
|
| 1820 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1821 |
user = _user_for(claims)
|
| 1822 |
if not user:
|
|
|
|
|
|
|
| 1823 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1824 |
identity = {_TENANT_HEADER: str(claims["t"]),
|
| 1825 |
_PRINCIPAL_HEADER: _principal(claims["t"], user["username"])}
|
|
|
|
| 1835 |
except KeyError:
|
| 1836 |
# An unknown tenant slug is a 401, not a 404: the cookie is not a valid identity for this
|
| 1837 |
# deployment, and confirming which slugs exist would answer a question we were not asked.
|
|
|
|
|
|
|
| 1838 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1839 |
+
fresh, _ = aios_session.mint(claims["t"], user["username"], int(user.get("epoch") or 0),
|
|
|
|
|
|
|
| 1840 |
absolute_expiry=claims["x"])
|
| 1841 |
aios_session.set_cookie(response, request, fresh)
|
| 1842 |
# ⭐ STAMP THE TENANT THIS ANSWER WAS SERVED FOR (owner report 2026-08-09: "I can see
|
|
|
|
| 1857 |
# `session.tenant` read back from something the request could influence.
|
| 1858 |
response.headers[_TENANT_HEADER] = str(claims["t"])
|
| 1859 |
response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"])
|
| 1860 |
+
# Observe only AFTER authentication admission. Read methods stay store-read-only; only an
|
| 1861 |
+
# admitted mutation can launch the persisted stamp, without moving it outside request timing.
|
|
|
|
|
|
|
| 1862 |
# qa-runner has no global account record. Never let the normal activity telemetry create or
|
| 1863 |
# touch one through the default tenant-#0 users store.
|
| 1864 |
+
if (not user.get("qa_runner")
|
| 1865 |
and not (request.method == "GET" and request.url.path == "/api/v1/notifications")):
|
| 1866 |
_observe_active(claims["t"], user.get("username"))
|
| 1867 |
if request.method not in {"GET", "HEAD", "OPTIONS"}:
|
|
|
|
| 1877 |
verified signed cookie is enough to return that no-data response; resolving the account record
|
| 1878 |
would itself turn those legacy requests into a permanent Neon keepalive.
|
| 1879 |
"""
|
|
|
|
|
|
|
| 1880 |
if store.backend() != "pg":
|
| 1881 |
return require_session(request, response)
|
| 1882 |
if request.headers.get("x-aios-change-check") == "focus-v1":
|
|
|
|
| 1918 |
|
| 1919 |
`require_session` stays cookie only. A bearer session never reissues the cookie.
|
| 1920 |
"""
|
|
|
|
|
|
|
|
|
|
| 1921 |
bearer = _bearer_header(request)
|
| 1922 |
if bearer is None or request.cookies.get(aios_session.COOKIE_NAME):
|
| 1923 |
try:
|
api/loopy_catalog_tools.py
CHANGED
|
@@ -305,10 +305,8 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None):
|
|
| 305 |
import contextvars
|
| 306 |
import sys
|
| 307 |
import threading
|
| 308 |
-
import time
|
| 309 |
import catalog_store
|
| 310 |
import app_templates
|
| 311 |
-
from core import request_timing
|
| 312 |
|
| 313 |
render = render or sys.modules[__name__].catalog_pdf
|
| 314 |
budget = export_budget() if budget is None else budget
|
|
@@ -331,15 +329,8 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None):
|
|
| 331 |
import logging
|
| 332 |
logging.getLogger(__name__).warning("Catalog export result not saved (%s)", type(exc).__name__)
|
| 333 |
|
| 334 |
-
def run():
|
| 335 |
-
# HTTP completion cancels its timing state, so the export needs its own bounded scope.
|
| 336 |
-
with request_timing.request_scope() as measured:
|
| 337 |
-
measured.deadline = time.monotonic() + catalog_store.EXPORT_STALE_SECONDS
|
| 338 |
-
with request_timing.thread_scope():
|
| 339 |
-
work()
|
| 340 |
-
|
| 341 |
context = contextvars.copy_context()
|
| 342 |
-
worker = threading.Thread(target=context.run, args=(
|
| 343 |
try:
|
| 344 |
worker.start()
|
| 345 |
except Exception:
|
|
|
|
| 305 |
import contextvars
|
| 306 |
import sys
|
| 307 |
import threading
|
|
|
|
| 308 |
import catalog_store
|
| 309 |
import app_templates
|
|
|
|
| 310 |
|
| 311 |
render = render or sys.modules[__name__].catalog_pdf
|
| 312 |
budget = export_budget() if budget is None else budget
|
|
|
|
| 329 |
import logging
|
| 330 |
logging.getLogger(__name__).warning("Catalog export result not saved (%s)", type(exc).__name__)
|
| 331 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 332 |
context = contextvars.copy_context()
|
| 333 |
+
worker = threading.Thread(target=context.run, args=(work,), daemon=True, name="catalog-pdf")
|
| 334 |
try:
|
| 335 |
worker.start()
|
| 336 |
except Exception:
|
api/main.py
CHANGED
|
@@ -269,32 +269,20 @@ def _data_input_frozen(request: Request, exc: freeze_notice.DataInputFrozen):
|
|
| 269 |
|
| 270 |
@app.middleware("http")
|
| 271 |
async def _freeze_response(request: Request, call_next):
|
| 272 |
-
"""
|
| 273 |
-
import deps
|
| 274 |
-
try:
|
| 275 |
-
deps.refuse_view_as_write(request)
|
| 276 |
-
except StarletteHTTPException as exc:
|
| 277 |
-
return _error_shape(request, exc)
|
| 278 |
if request.method in {"GET", "HEAD", "OPTIONS"} or request.url.path == "/api/v1/query/chat/stream":
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
identity = getattr(request.state, "_view_as_audited", None)
|
| 292 |
-
if identity is not None:
|
| 293 |
-
tenant, _actor, member = identity
|
| 294 |
-
response.headers["X-Viewing-As"] = member
|
| 295 |
-
response.headers[deps._TENANT_HEADER] = str(tenant)
|
| 296 |
-
response.headers[deps._PRINCIPAL_HEADER] = deps._principal(tenant, member)
|
| 297 |
-
return response
|
| 298 |
|
| 299 |
|
| 300 |
@app.exception_handler(data_binding.StoreWriteRefused)
|
|
@@ -1674,17 +1662,16 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
|
|
| 1674 |
return sentence + suffix
|
| 1675 |
|
| 1676 |
|
| 1677 |
-
def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None
|
| 1678 |
-
"""Drain
|
| 1679 |
|
| 1680 |
-
HTTP handlers queue intents.
|
| 1681 |
-
A failed request cannot discard another tenant's queued work.
|
| 1682 |
`outcome`, when given, receives `{provider: "continuing" | "done" | "failed"}` so the loop can
|
| 1683 |
project only what a desk wake actually changed.
|
| 1684 |
"""
|
| 1685 |
from core import connector_sync
|
| 1686 |
-
|
| 1687 |
-
_drain_pending_amazon_revocations()
|
| 1688 |
|
| 1689 |
def progress_log(provider, tenant, receipt, rt):
|
| 1690 |
"""Publish only authored progress shapes, with one bounded detail per running receipt."""
|
|
@@ -1785,11 +1772,7 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
|
|
| 1785 |
# A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
|
| 1786 |
hold = during_odoo if hold_first_load is None else hold_first_load
|
| 1787 |
# Claim only work this turn executes. A later provider remains queued with an honest wait.
|
| 1788 |
-
|
| 1789 |
-
({"odoo"} if during_odoo else set()))
|
| 1790 |
-
if not amazon_only and globals().get("_AMAZON_DESK", {}).get("managed"):
|
| 1791 |
-
excluded.add("amazon_sp")
|
| 1792 |
-
pending = connector_sync.take_sync_requests(limit=1, exclude_providers=excluded,
|
| 1793 |
refresh_remote=True,
|
| 1794 |
exclude_requested_by=("first load",) if hold else ())
|
| 1795 |
while True:
|
|
@@ -2440,44 +2423,54 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 2440 |
None, str(db_failure.get("stage") or "materialization"))["message"],
|
| 2441 |
**({"cause": _raw_cause(key)} if _raw_cause(key) else {})}
|
| 2442 |
for key, db_failure in blocking.items() if isinstance(db_failure, dict)]
|
| 2443 |
-
|
| 2444 |
-
|
| 2445 |
try:
|
| 2446 |
prior = (_rt.get(_rel.CONFIG_KEY) or {}).get("failedDatabases") or []
|
| 2447 |
except Exception: # noqa: BLE001
|
| 2448 |
prior = []
|
| 2449 |
-
|
| 2450 |
-
|
| 2451 |
-
|
| 2452 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2453 |
key = item.get("key")
|
| 2454 |
-
if
|
| 2455 |
-
|
| 2456 |
-
|
| 2457 |
-
|
| 2458 |
-
|
| 2459 |
-
|
| 2460 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2461 |
try:
|
| 2462 |
-
|
| 2463 |
-
|
| 2464 |
-
continue
|
| 2465 |
-
if not any(_key == key and bucket in _rel.enabled_buckets(_rt)
|
| 2466 |
-
for bucket, _key, _label, _fields in _rel.TABLES):
|
| 2467 |
-
continue
|
| 2468 |
-
with _rt.mirror_handle().ro_cursor() as cur:
|
| 2469 |
-
nrows = routes_odoo_tables.population(key, cur=cur, rt=_rt)
|
| 2470 |
-
if isinstance(nrows, int) and nrows >= 0:
|
| 2471 |
-
recovered_database_keys.add(key)
|
| 2472 |
except Exception: # noqa: BLE001
|
| 2473 |
-
|
| 2474 |
-
|
| 2475 |
-
|
| 2476 |
-
|
| 2477 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2478 |
try:
|
| 2479 |
_rel.record_sync_outcome(_rt, ok=not safe_errors, at=refreshed_at,
|
| 2480 |
-
failed_databases=
|
| 2481 |
except Exception: # noqa: BLE001
|
| 2482 |
pass
|
| 2483 |
# Owner 2026-10-29: a database that fails or recovers is Telegrammed at once.
|
|
@@ -3277,77 +3270,15 @@ def _sheet_warm_after_pass(tenant, good, generation_before, rebuilt):
|
|
| 3277 |
|
| 3278 |
def _sync_idle_wait(event, seconds):
|
| 3279 |
"""The resync loop's wait. A `sync_worker.py --once` job exits here when it is idle (S82-T10)."""
|
| 3280 |
-
if globals().get("_AMAZON_DESK", {}).get("job") is not None:
|
| 3281 |
-
# An unclaimed or finishing desk turn still owns work, even between receipt states.
|
| 3282 |
-
return event.wait(seconds)
|
| 3283 |
import sync_runtime
|
| 3284 |
return sync_runtime.idle_wait(event, seconds)
|
| 3285 |
|
| 3286 |
|
| 3287 |
-
def _amazon_intraday_pass(tenant_key, previous_fingerprint):
|
| 3288 |
-
"""Run due Amazon tables and their projection without advancing the Odoo pass clock."""
|
| 3289 |
-
if globals().get("_AMAZON_DESK", {}).get("job") is not None:
|
| 3290 |
-
return previous_fingerprint
|
| 3291 |
-
try:
|
| 3292 |
-
from harness import amazon_store, runtime
|
| 3293 |
-
rt = runtime.get_runtime(tenant_key)
|
| 3294 |
-
if not amazon_store.credential(tenant_key, rt=rt) or amazon_store.first_load_job_live(rt):
|
| 3295 |
-
return previous_fingerprint
|
| 3296 |
-
plan = amazon_store.scheduled_sync_plan(tenant_key, rt=rt)
|
| 3297 |
-
if not plan["tables"]:
|
| 3298 |
-
return previous_fingerprint
|
| 3299 |
-
print("[aios-api] amazon intraday pass due: " + ", ".join(plan["tables"]), flush=True)
|
| 3300 |
-
RESYNC_STATE["slotPass"] = False
|
| 3301 |
-
_pull_amazon("intraday")
|
| 3302 |
-
state = _rebuild_amazon_relational("intraday", rt, previous_fingerprint)
|
| 3303 |
-
return state["fingerprint"] if state.get("known") else None
|
| 3304 |
-
except Exception as exc:
|
| 3305 |
-
print(f"[aios-api] amazon intraday pass failed ({type(exc).__name__}); "
|
| 3306 |
-
"the next cadence check retries it", flush=True)
|
| 3307 |
-
return None
|
| 3308 |
-
|
| 3309 |
-
|
| 3310 |
-
def _amazon_desk_pass(tenant, *, project=True):
|
| 3311 |
-
"""Run one Amazon desk claim off the slot clock and collect its projection after publication."""
|
| 3312 |
-
import process_role
|
| 3313 |
-
import time
|
| 3314 |
-
if not process_role.may_write():
|
| 3315 |
-
return False
|
| 3316 |
-
from core import connector_sync
|
| 3317 |
-
state = globals().setdefault("_AMAZON_DESK", {"managed": True, "job": None, "outcome": {}})
|
| 3318 |
-
job = state["job"]
|
| 3319 |
-
if job is not None:
|
| 3320 |
-
if job.is_alive() or not project:
|
| 3321 |
-
return True
|
| 3322 |
-
job.join()
|
| 3323 |
-
state["job"] = None
|
| 3324 |
-
_project_desk(state["outcome"], tenant)
|
| 3325 |
-
receipt = connector_sync.sync_request("amazon_sp", tenant)
|
| 3326 |
-
if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load"
|
| 3327 |
-
or max(receipt.get("notBefore") or 0,
|
| 3328 |
-
connector_sync.sync_not_before("amazon_sp", tenant) or 0) > time.time()):
|
| 3329 |
-
return False
|
| 3330 |
-
outcome = state["outcome"] = {}
|
| 3331 |
-
|
| 3332 |
-
def drain():
|
| 3333 |
-
try:
|
| 3334 |
-
_run_sync_requests(outcome, hold_first_load=True, amazon_only=True)
|
| 3335 |
-
except Exception as exc:
|
| 3336 |
-
print(f"[aios-api] amazon desk turn failed ({type(exc).__name__})", flush=True)
|
| 3337 |
-
finally:
|
| 3338 |
-
connector_sync.sync_wake_event().set()
|
| 3339 |
-
|
| 3340 |
-
# This turn inherits store-resync's heavy admission, including tenant-worker scope.
|
| 3341 |
-
state["job"] = process_role.start("store-resync", "heavy", drain)
|
| 3342 |
-
return state["job"] is not None
|
| 3343 |
-
|
| 3344 |
-
|
| 3345 |
def _store_resync_loop():
|
| 3346 |
"""Keep the analytical mirror current — the API-process stand-in for the re-sync the
|
| 3347 |
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
| 3348 |
refreshed pool re-reads the freshly synced store."""
|
| 3349 |
-
|
| 3350 |
-
if not _loop_role.may_write() or _freeze_holds("store resync"):
|
| 3351 |
return
|
| 3352 |
import time as _t
|
| 3353 |
from core import connector_sync
|
|
@@ -3383,7 +3314,6 @@ def _store_resync_loop():
|
|
| 3383 |
last_full_pass_at = _t.time()
|
| 3384 |
# The least time between two desk-only passes of an Amazon first-load continuation.
|
| 3385 |
desk_min_spacing_seconds = 30.0
|
| 3386 |
-
amazon_check_seconds = 60.0
|
| 3387 |
# H73: the start of the last daily-slot pass, and whether the boot restore already ran.
|
| 3388 |
last_slot_pass_at = _memory.get("slotPassAt")
|
| 3389 |
slot_restored = False
|
|
@@ -3474,7 +3404,7 @@ def _store_resync_loop():
|
|
| 3474 |
_slot = (globals().get("_resolve_resync_slot") or (lambda: None))()
|
| 3475 |
|
| 3476 |
def _near_slot():
|
| 3477 |
-
#
|
| 3478 |
if _slot is None:
|
| 3479 |
return False
|
| 3480 |
import odoo_relational as _rel_near
|
|
@@ -3489,7 +3419,6 @@ def _store_resync_loop():
|
|
| 3489 |
# END drifted later every day and past the 26-hour stale line).
|
| 3490 |
_woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
|
| 3491 |
max(0.0, _every - (_t.time() - last_full_pass_at)),
|
| 3492 |
-
amazon_check_seconds,
|
| 3493 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _t.time()))))
|
| 3494 |
connector_sync.sync_wake_event().clear()
|
| 3495 |
_slot_due = False
|
|
@@ -3503,7 +3432,6 @@ def _store_resync_loop():
|
|
| 3503 |
_woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
|
| 3504 |
0.0 if _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
|
| 3505 |
else max(0.0, _rel_slot.slot_after(_slot_now, _slot) + 1.0 - _slot_now),
|
| 3506 |
-
amazon_check_seconds,
|
| 3507 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _slot_now))))
|
| 3508 |
connector_sync.sync_wake_event().clear()
|
| 3509 |
_slot_now = _t.time()
|
|
@@ -3536,11 +3464,6 @@ def _store_resync_loop():
|
|
| 3536 |
f"{_slot['tz']} slot; cleared the slot_pass_interrupted failure")
|
| 3537 |
else:
|
| 3538 |
RESYNC_STATE.pop("nextPassAt", None)
|
| 3539 |
-
_cadence_due = not passes or _t.time() - last_full_pass_at >= _every
|
| 3540 |
-
_amazon_desk_busy = (_amazon_desk_pass(_cad["tenant"], project=not (_slot_due or _near_slot()))
|
| 3541 |
-
if _slot is not None or globals().get("_AMAZON_DESK") else False)
|
| 3542 |
-
if (not _slot_due if _slot is not None else not _cadence_due):
|
| 3543 |
-
last_amzn_fingerprint = _amazon_intraday_pass(_cad["tenant"], last_amzn_fingerprint)
|
| 3544 |
# ⛔⛔ HOTFIX 2026-09-29 — A RESTART INSIDE THE DAILY WINDOW IS NOT A SYNC. The owner
|
| 3545 |
# capped syncing at once a day, and a deploy is not a reason to sync again. So the first
|
| 3546 |
# pass after a boot that lands within one cadence of the last scheduled pass only catches
|
|
@@ -3583,16 +3506,13 @@ def _store_resync_loop():
|
|
| 3583 |
else:
|
| 3584 |
_project_desk(_desk, _cad["tenant"])
|
| 3585 |
if not slot_restored:
|
| 3586 |
-
|
| 3587 |
-
_pull_amazon("boot", False)
|
| 3588 |
# The restored mirror warms the workbooks once; Sheet warm reads only the mirror.
|
| 3589 |
_sheet_warm_after_pass(_cad["tenant"], True, None, True)
|
| 3590 |
slot_restored = True
|
| 3591 |
print(f"[aios-api] store resync: no Odoo pass outside the daily slot; the next one "
|
| 3592 |
f"runs at {RESYNC_STATE['nextPassAt']} ({_slot['at']} {_slot['tz']})")
|
| 3593 |
continue
|
| 3594 |
-
if (_slot is None and not _cadence_due and _odoo_requested is None and not _continuation):
|
| 3595 |
-
continue
|
| 3596 |
passes += 1
|
| 3597 |
backfill_next_at = None
|
| 3598 |
if not _continuation:
|
|
@@ -3839,10 +3759,9 @@ def _store_resync_loop():
|
|
| 3839 |
except Exception as e: # noqa: BLE001
|
| 3840 |
print(f"[aios-api] meta relational refresh failed (resync): "
|
| 3841 |
f"{type(e).__name__}: {e}")
|
| 3842 |
-
|
| 3843 |
-
if not _amazon_desk_busy and _resumed.get("amazon"):
|
| 3844 |
_pull_amazon("boot", False)
|
| 3845 |
-
|
| 3846 |
# The daily slot pass refreshes every Amazon table: a late extra pass yesterday must not defer it.
|
| 3847 |
RESYNC_STATE["slotPass"] = _slot_claimed
|
| 3848 |
try:
|
|
@@ -3856,8 +3775,7 @@ def _store_resync_loop():
|
|
| 3856 |
try:
|
| 3857 |
from harness import runtime as _runtime
|
| 3858 |
_rt_resync = _runtime.get_runtime(_cad["tenant"])
|
| 3859 |
-
_amzn_state = (
|
| 3860 |
-
_rebuild_amazon_relational("resync", _rt_resync, _amzn_previous))
|
| 3861 |
last_amzn_fingerprint = (_amzn_state["fingerprint"]
|
| 3862 |
if _amzn_state.get("known") else None)
|
| 3863 |
except Exception as e: # noqa: BLE001
|
|
|
|
| 269 |
|
| 270 |
@app.middleware("http")
|
| 271 |
async def _freeze_response(request: Request, call_next):
|
| 272 |
+
"""Keep a refused write readable even when a route catches its store error."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
if request.method in {"GET", "HEAD", "OPTIONS"} or request.url.path == "/api/v1/query/chat/stream":
|
| 274 |
+
return await call_next(request)
|
| 275 |
+
with freeze_notice.refusal_scope() as refused:
|
| 276 |
+
try:
|
| 277 |
+
response = await call_next(request)
|
| 278 |
+
except Exception:
|
| 279 |
+
if not refused[0]:
|
| 280 |
+
raise
|
| 281 |
+
return _data_input_frozen(request, freeze_notice.DataInputFrozen())
|
| 282 |
+
if refused[0]:
|
| 283 |
+
# Replace the response, including any cookie minted from unpersisted auth state.
|
| 284 |
+
return _data_input_frozen(request, freeze_notice.DataInputFrozen())
|
| 285 |
+
return response
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
|
| 287 |
|
| 288 |
@app.exception_handler(data_binding.StoreWriteRefused)
|
|
|
|
| 1662 |
return sentence + suffix
|
| 1663 |
|
| 1664 |
|
| 1665 |
+
def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None):
|
| 1666 |
+
"""Drain Sync now intents on the resident resync thread, the mirror's single writer.
|
| 1667 |
|
| 1668 |
+
D-29 / intake F6: HTTP handlers only queue intents. Provider calls stay here, before the
|
| 1669 |
+
resident projection pass. A failed request cannot discard another tenant's queued work.
|
| 1670 |
`outcome`, when given, receives `{provider: "continuing" | "done" | "failed"}` so the loop can
|
| 1671 |
project only what a desk wake actually changed.
|
| 1672 |
"""
|
| 1673 |
from core import connector_sync
|
| 1674 |
+
_drain_pending_amazon_revocations()
|
|
|
|
| 1675 |
|
| 1676 |
def progress_log(provider, tenant, receipt, rt):
|
| 1677 |
"""Publish only authored progress shapes, with one bounded detail per running receipt."""
|
|
|
|
| 1772 |
# A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
|
| 1773 |
hold = during_odoo if hold_first_load is None else hold_first_load
|
| 1774 |
# Claim only work this turn executes. A later provider remains queued with an honest wait.
|
| 1775 |
+
pending = connector_sync.take_sync_requests(limit=1, exclude_providers=("odoo",) if during_odoo else (),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1776 |
refresh_remote=True,
|
| 1777 |
exclude_requested_by=("first load",) if hold else ())
|
| 1778 |
while True:
|
|
|
|
| 2423 |
None, str(db_failure.get("stage") or "materialization"))["message"],
|
| 2424 |
**({"cause": _raw_cause(key)} if _raw_cause(key) else {})}
|
| 2425 |
for key, db_failure in blocking.items() if isinstance(db_failure, dict)]
|
| 2426 |
+
record_failed = failed_databases
|
| 2427 |
+
if table_keys is not None or safe_errors:
|
| 2428 |
try:
|
| 2429 |
prior = (_rt.get(_rel.CONFIG_KEY) or {}).get("failedDatabases") or []
|
| 2430 |
except Exception: # noqa: BLE001
|
| 2431 |
prior = []
|
| 2432 |
+
try:
|
| 2433 |
+
db_health = ((((_rt.get(routes_connectors._ODOO_HEALTH_STATUS) or {})
|
| 2434 |
+
.get("connections") or {}).get(cid) or {})
|
| 2435 |
+
.get("databases") or {})
|
| 2436 |
+
except Exception: # noqa: BLE001
|
| 2437 |
+
db_health = {}
|
| 2438 |
+
|
| 2439 |
+
def _recovered(item):
|
| 2440 |
+
# D-13512: a prior failure leaves once its database ran clean or reads rows again.
|
| 2441 |
key = item.get("key")
|
| 2442 |
+
if key in database_failures:
|
| 2443 |
+
return False
|
| 2444 |
+
if key in succeeded:
|
| 2445 |
+
return True
|
| 2446 |
+
stage = str(item.get("stage") or "materialization")
|
| 2447 |
+
row = ((db_health.get(key) or {}).get("stages") or {}).get(stage) or {}
|
| 2448 |
+
ran, failed_at = _rel._parse_iso(row.get("at")), _rel._parse_iso(item.get("at"))
|
| 2449 |
+
if row.get("state") == "ready" and ran and failed_at and ran > failed_at:
|
| 2450 |
+
return True
|
| 2451 |
+
# Only a missing binding is cured by a binding that reads: a count mismatch reads rows too.
|
| 2452 |
+
if (not _rel.is_read_through_key(key)
|
| 2453 |
+
or "no readable mirror binding" not in str(item.get("cause") or "")):
|
| 2454 |
+
return False
|
| 2455 |
try:
|
| 2456 |
+
from routes_odoo_tables import population as _population
|
| 2457 |
+
return (_population(key, rt=_rt) or 0) > 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2458 |
except Exception: # noqa: BLE001
|
| 2459 |
+
return False
|
| 2460 |
+
|
| 2461 |
+
kept = [item for item in prior if isinstance(item, dict) and not _recovered(item)]
|
| 2462 |
+
if failed_databases is not None and table_keys is not None:
|
| 2463 |
+
# D-2679: a changed-tables pass keeps the failures of databases it did not run.
|
| 2464 |
+
failed_databases = [
|
| 2465 |
+
item for item in kept
|
| 2466 |
+
if item.get("key") not in database_keys] + failed_databases
|
| 2467 |
+
record_failed = failed_databases
|
| 2468 |
+
elif len(kept) < len(prior):
|
| 2469 |
+
# D-13512: a pass with errors still drops the failures that recovered.
|
| 2470 |
+
record_failed = kept
|
| 2471 |
try:
|
| 2472 |
_rel.record_sync_outcome(_rt, ok=not safe_errors, at=refreshed_at,
|
| 2473 |
+
failed_databases=record_failed)
|
| 2474 |
except Exception: # noqa: BLE001
|
| 2475 |
pass
|
| 2476 |
# Owner 2026-10-29: a database that fails or recovers is Telegrammed at once.
|
|
|
|
| 3270 |
|
| 3271 |
def _sync_idle_wait(event, seconds):
|
| 3272 |
"""The resync loop's wait. A `sync_worker.py --once` job exits here when it is idle (S82-T10)."""
|
|
|
|
|
|
|
|
|
|
| 3273 |
import sync_runtime
|
| 3274 |
return sync_runtime.idle_wait(event, seconds)
|
| 3275 |
|
| 3276 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3277 |
def _store_resync_loop():
|
| 3278 |
"""Keep the analytical mirror current — the API-process stand-in for the re-sync the
|
| 3279 |
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
| 3280 |
refreshed pool re-reads the freshly synced store."""
|
| 3281 |
+
if _freeze_holds("store resync"):
|
|
|
|
| 3282 |
return
|
| 3283 |
import time as _t
|
| 3284 |
from core import connector_sync
|
|
|
|
| 3314 |
last_full_pass_at = _t.time()
|
| 3315 |
# The least time between two desk-only passes of an Amazon first-load continuation.
|
| 3316 |
desk_min_spacing_seconds = 30.0
|
|
|
|
| 3317 |
# H73: the start of the last daily-slot pass, and whether the boot restore already ran.
|
| 3318 |
last_slot_pass_at = _memory.get("slotPassAt")
|
| 3319 |
slot_restored = False
|
|
|
|
| 3404 |
_slot = (globals().get("_resolve_resync_slot") or (lambda: None))()
|
| 3405 |
|
| 3406 |
def _near_slot():
|
| 3407 |
+
# A first-load continuation does not start this close to the next Odoo slot.
|
| 3408 |
if _slot is None:
|
| 3409 |
return False
|
| 3410 |
import odoo_relational as _rel_near
|
|
|
|
| 3419 |
# END drifted later every day and past the 26-hour stale line).
|
| 3420 |
_woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
|
| 3421 |
max(0.0, _every - (_t.time() - last_full_pass_at)),
|
|
|
|
| 3422 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _t.time()))))
|
| 3423 |
connector_sync.sync_wake_event().clear()
|
| 3424 |
_slot_due = False
|
|
|
|
| 3432 |
_woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
|
| 3433 |
0.0 if _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
|
| 3434 |
else max(0.0, _rel_slot.slot_after(_slot_now, _slot) + 1.0 - _slot_now),
|
|
|
|
| 3435 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _slot_now))))
|
| 3436 |
connector_sync.sync_wake_event().clear()
|
| 3437 |
_slot_now = _t.time()
|
|
|
|
| 3464 |
f"{_slot['tz']} slot; cleared the slot_pass_interrupted failure")
|
| 3465 |
else:
|
| 3466 |
RESYNC_STATE.pop("nextPassAt", None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3467 |
# ⛔⛔ HOTFIX 2026-09-29 — A RESTART INSIDE THE DAILY WINDOW IS NOT A SYNC. The owner
|
| 3468 |
# capped syncing at once a day, and a deploy is not a reason to sync again. So the first
|
| 3469 |
# pass after a boot that lands within one cadence of the last scheduled pass only catches
|
|
|
|
| 3506 |
else:
|
| 3507 |
_project_desk(_desk, _cad["tenant"])
|
| 3508 |
if not slot_restored:
|
| 3509 |
+
_pull_amazon("boot", False)
|
|
|
|
| 3510 |
# The restored mirror warms the workbooks once; Sheet warm reads only the mirror.
|
| 3511 |
_sheet_warm_after_pass(_cad["tenant"], True, None, True)
|
| 3512 |
slot_restored = True
|
| 3513 |
print(f"[aios-api] store resync: no Odoo pass outside the daily slot; the next one "
|
| 3514 |
f"runs at {RESYNC_STATE['nextPassAt']} ({_slot['at']} {_slot['tz']})")
|
| 3515 |
continue
|
|
|
|
|
|
|
| 3516 |
passes += 1
|
| 3517 |
backfill_next_at = None
|
| 3518 |
if not _continuation:
|
|
|
|
| 3759 |
except Exception as e: # noqa: BLE001
|
| 3760 |
print(f"[aios-api] meta relational refresh failed (resync): "
|
| 3761 |
f"{type(e).__name__}: {e}")
|
| 3762 |
+
if _resumed.get("amazon"):
|
|
|
|
| 3763 |
_pull_amazon("boot", False)
|
| 3764 |
+
else:
|
| 3765 |
# The daily slot pass refreshes every Amazon table: a late extra pass yesterday must not defer it.
|
| 3766 |
RESYNC_STATE["slotPass"] = _slot_claimed
|
| 3767 |
try:
|
|
|
|
| 3775 |
try:
|
| 3776 |
from harness import runtime as _runtime
|
| 3777 |
_rt_resync = _runtime.get_runtime(_cad["tenant"])
|
| 3778 |
+
_amzn_state = _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous)
|
|
|
|
| 3779 |
last_amzn_fingerprint = (_amzn_state["fingerprint"]
|
| 3780 |
if _amzn_state.get("known") else None)
|
| 3781 |
except Exception as e: # noqa: BLE001
|
api/notify_destinations.py
CHANGED
|
@@ -1,14 +1,12 @@
|
|
| 1 |
"""Authenticated preference routes and destination-bound notification delivery."""
|
| 2 |
|
| 3 |
import re
|
| 4 |
-
from datetime import datetime, timezone
|
| 5 |
|
| 6 |
from fastapi import APIRouter, Body, Depends, Query
|
| 7 |
|
| 8 |
from core import notify_destinations as rows
|
| 9 |
from deps import Session, err, require_session
|
| 10 |
import email_send
|
| 11 |
-
import dash_norm
|
| 12 |
|
| 13 |
NotificationItem = rows.NotificationItem
|
| 14 |
NotificationPayload = rows.NotificationPayload
|
|
@@ -28,12 +26,6 @@ _REASONS = {
|
|
| 28 |
'notification_source_required': 'This notification needs source records so the Slack agent can check its read access.',
|
| 29 |
'transport_error': 'Slack delivery was not confirmed. Check Slack before retrying.',
|
| 30 |
'slack_readiness_unavailable': 'Slack readiness could not be checked. Check its connection and permissions.',
|
| 31 |
-
'in_app_unavailable': 'In-app notifications are unavailable for this workspace.',
|
| 32 |
-
'in_app_delivery_unconfirmed': 'The in-app notification could not be confirmed.',
|
| 33 |
-
'webhook_not_configured': 'Choose one of your saved alert webhooks in connector settings.',
|
| 34 |
-
'webhook_readiness_unavailable': 'Webhook readiness could not be checked. Check its saved configuration.',
|
| 35 |
-
'webhook_delivery_unconfirmed': 'Webhook delivery was not confirmed. Check the destination before retrying.',
|
| 36 |
-
'webhook_refused': 'The configured webhook refused this notification.',
|
| 37 |
}
|
| 38 |
|
| 39 |
|
|
@@ -51,56 +43,6 @@ def _own_session(session):
|
|
| 51 |
raise err(401, 'notification_user_required', 'Sign in to edit notification destinations.')
|
| 52 |
|
| 53 |
|
| 54 |
-
def _in_app_readiness(rt):
|
| 55 |
-
try:
|
| 56 |
-
import assistant_log
|
| 57 |
-
ready = (callable(getattr(assistant_log, 'append', None))
|
| 58 |
-
and callable(getattr(rt, 'get', None)) and callable(getattr(rt, 'update', None)))
|
| 59 |
-
available = getattr(rt, 'available', None)
|
| 60 |
-
if callable(available):
|
| 61 |
-
ready = ready and bool(available())
|
| 62 |
-
except Exception:
|
| 63 |
-
ready = False
|
| 64 |
-
reason = None if ready else 'in_app_unavailable'
|
| 65 |
-
return {'configured': ready, 'available': ready, 'reason': reason,
|
| 66 |
-
'message': '' if ready else _REASONS[reason]}
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
def _webhook_readiness(rt, user, identity):
|
| 70 |
-
try:
|
| 71 |
-
rows.webhook_fields(rt, user, identity)
|
| 72 |
-
import automation_engine
|
| 73 |
-
reason = None if callable(getattr(automation_engine, 'fetch_json', None)) else 'webhook_readiness_unavailable'
|
| 74 |
-
except ValueError:
|
| 75 |
-
reason = 'webhook_not_configured'
|
| 76 |
-
except Exception:
|
| 77 |
-
reason = 'webhook_readiness_unavailable'
|
| 78 |
-
return {'available': not reason, 'reason': reason,
|
| 79 |
-
'message': '' if reason is None else _REASONS[reason]}
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
def _configured_webhooks(rt, user):
|
| 83 |
-
"""Expose only safe metadata for this user's configured saved webhook references."""
|
| 84 |
-
from core import keychain
|
| 85 |
-
destinations = []
|
| 86 |
-
try:
|
| 87 |
-
if not callable(getattr(rt, 'get', None)):
|
| 88 |
-
return {'configured': False, 'destinations': [], 'reason': 'webhook_not_configured'}
|
| 89 |
-
for entry in keychain.list_entries(rt, 'generic'):
|
| 90 |
-
identity = entry.get('id')
|
| 91 |
-
if entry.get('createdBy') != user or not isinstance(identity, str) or not rows._WEBHOOK.fullmatch(identity):
|
| 92 |
-
continue
|
| 93 |
-
readiness = _webhook_readiness(rt, user, identity)
|
| 94 |
-
if readiness['available']:
|
| 95 |
-
label = entry.get('label')
|
| 96 |
-
destinations.append({'id': identity, 'label': dash_norm.plain(label[:200]) if isinstance(label, str) else identity,
|
| 97 |
-
'destination': 'webhook:' + identity, **readiness})
|
| 98 |
-
except Exception:
|
| 99 |
-
return {'configured': False, 'destinations': [], 'reason': 'webhook_readiness_unavailable'}
|
| 100 |
-
return {'configured': bool(destinations), 'destinations': destinations,
|
| 101 |
-
'reason': None if destinations else 'webhook_not_configured'}
|
| 102 |
-
|
| 103 |
-
|
| 104 |
@router.get('/destinations')
|
| 105 |
def get_destinations(session: Session = Depends(require_session)):
|
| 106 |
_own_session(session)
|
|
@@ -109,7 +51,6 @@ def get_destinations(session: Session = Depends(require_session)):
|
|
| 109 |
except Exception:
|
| 110 |
raise err(503, 'notification_store_unavailable', 'Notification destinations are unavailable for this workspace.')
|
| 111 |
mail_ready = email_send.availability(session.runtime)
|
| 112 |
-
in_app = _in_app_readiness(session.runtime)
|
| 113 |
projected = []
|
| 114 |
for preference in destinations:
|
| 115 |
readiness = {
|
|
@@ -119,14 +60,8 @@ def get_destinations(session: Session = Depends(require_session)):
|
|
| 119 |
}
|
| 120 |
if preference['email']:
|
| 121 |
readiness['email:' + preference['email']] = dict(mail_ready)
|
| 122 |
-
if preference.get('inApp'):
|
| 123 |
-
readiness['in-app:' + session.uname] = dict(in_app)
|
| 124 |
-
for identity in preference.get('webhooks', []):
|
| 125 |
-
readiness['webhook:' + identity] = _webhook_readiness(session.runtime, session.uname, identity)
|
| 126 |
projected.append({**preference, 'label': rows.KIND_LABELS[preference['kind']], 'readiness': readiness})
|
| 127 |
-
return {'destinations': projected, 'email': mail_ready
|
| 128 |
-
'inApp': {**in_app, 'destination': 'in-app:' + session.uname},
|
| 129 |
-
'webhooks': _configured_webhooks(session.runtime, session.uname)}
|
| 130 |
|
| 131 |
|
| 132 |
def _save(kind, body, session, *, replace):
|
|
@@ -135,13 +70,9 @@ def _save(kind, body, session, *, replace):
|
|
| 135 |
rows.kind_id(kind)
|
| 136 |
patch = rows.clean_patch(body, replace=replace)
|
| 137 |
except (ValueError, TypeError):
|
| 138 |
-
raise err(400, 'notification_destination_invalid', 'Choose a known notification kind and
|
| 139 |
-
if patch.get('inApp') and not _in_app_readiness(session.runtime)['available']:
|
| 140 |
-
raise err(400, 'notification_destination_invalid', _REASONS['in_app_unavailable'])
|
| 141 |
try:
|
| 142 |
return rows.edit(session.runtime, session.uname, kind, patch)
|
| 143 |
-
except ValueError:
|
| 144 |
-
raise err(400, 'notification_destination_invalid', 'Choose one of your configured webhook destinations.')
|
| 145 |
except Exception:
|
| 146 |
raise err(503, 'notification_store_unavailable', 'Notification destinations could not be saved for this workspace.')
|
| 147 |
|
|
@@ -253,36 +184,6 @@ def _dispatch_slack(rt, preference, channel, kind, items, *, dispatcher=None):
|
|
| 253 |
return {'state': 'sent', 'reason': None, 'message': ''}
|
| 254 |
|
| 255 |
|
| 256 |
-
def _dispatch_in_app(rt, preference, kind, items):
|
| 257 |
-
ready = _in_app_readiness(rt)
|
| 258 |
-
if not ready['available']:
|
| 259 |
-
return _refused(ready['reason'])
|
| 260 |
-
import assistant_log
|
| 261 |
-
items = [{**item, 'title': dash_norm.plain(item['title']), 'body': dash_norm.plain(item['body'])} for item in items]
|
| 262 |
-
body = '\n\n'.join(item['title'] + '\n' + item['body'] for item in items)
|
| 263 |
-
turn = {'role': 'system', 'door': 'wake', 'kind': 'notice', 'text': body,
|
| 264 |
-
'at': datetime.now(timezone.utc).isoformat(),
|
| 265 |
-
'payload': {'title': rows.KIND_LABELS[kind], 'body': body, 'items': items,
|
| 266 |
-
'notificationKind': kind, 'sender': 'Loopable'}}
|
| 267 |
-
receipt = assistant_log.append(rt, preference['user'], turn)
|
| 268 |
-
return ({'state': 'sent', 'reason': None, 'message': ''}
|
| 269 |
-
if isinstance(receipt, str) and receipt else _refused('in_app_delivery_unconfirmed'))
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
def _dispatch_webhook(rt, preference, identity, kind, items):
|
| 273 |
-
ready = _webhook_readiness(rt, preference['user'], identity)
|
| 274 |
-
if not ready['available']:
|
| 275 |
-
return _refused(ready['reason'])
|
| 276 |
-
fields = rows.webhook_fields(rt, preference['user'], identity)
|
| 277 |
-
import automation_engine
|
| 278 |
-
items = [{**item, 'title': dash_norm.plain(item['title']), 'body': dash_norm.plain(item['body'])} for item in items]
|
| 279 |
-
payload = {'version': rows.PAYLOAD_VERSION, 'tenant': rt.key, 'user': preference['user'],
|
| 280 |
-
'kind': kind, 'items': items}
|
| 281 |
-
status, _body = automation_engine.fetch_json(fields['url'], payload, timeout=20, max_kb=64)
|
| 282 |
-
return ({'state': 'sent', 'reason': None, 'message': ''}
|
| 283 |
-
if 200 <= status < 300 else _refused('webhook_refused'))
|
| 284 |
-
|
| 285 |
-
|
| 286 |
def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None):
|
| 287 |
"""Deliver trusted recipient-safe items to saved picks, optionally targeting one user."""
|
| 288 |
kind, items = rows.kind_id(kind), rows.clean_items(items)
|
|
@@ -294,9 +195,6 @@ def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None)
|
|
| 294 |
destinations = ['slack:' + channel for channel in preference['slackChannels']]
|
| 295 |
if preference['email']:
|
| 296 |
destinations.append('email:' + preference['email'])
|
| 297 |
-
if preference.get('inApp'):
|
| 298 |
-
destinations.append('in-app:' + preference['user'])
|
| 299 |
-
destinations.extend('webhook:' + identity for identity in preference.get('webhooks', []))
|
| 300 |
for destination in destinations:
|
| 301 |
current = rows.get(rt, preference['user'], kind)
|
| 302 |
if not rows.selected(current, destination):
|
|
@@ -304,18 +202,12 @@ def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None)
|
|
| 304 |
try:
|
| 305 |
if destination.startswith('slack:'):
|
| 306 |
outcome = _dispatch_slack(rt, current, destination[6:], kind, items, dispatcher=dispatcher)
|
| 307 |
-
elif destination.startswith('in-app:'):
|
| 308 |
-
outcome = _dispatch_in_app(rt, current, kind, items)
|
| 309 |
-
elif destination.startswith('webhook:'):
|
| 310 |
-
outcome = _dispatch_webhook(rt, current, destination[8:], kind, items)
|
| 311 |
else:
|
| 312 |
result = email_send.send_notification(rt, destination[6:], kind, items, transport=mail_transport)
|
| 313 |
outcome = ({'state': 'sent', 'reason': None, 'message': ''} if result.get('sent') is True
|
| 314 |
else _refused(result.get('reason'), message=result.get('message')))
|
| 315 |
except Exception:
|
| 316 |
-
|
| 317 |
-
'in-app': 'in_app_delivery_unconfirmed', 'webhook': 'webhook_delivery_unconfirmed'}
|
| 318 |
-
outcome = _refused(reason[destination.split(':', 1)[0]])
|
| 319 |
# Persistence failures propagate. An unrecorded refusal must never look successful.
|
| 320 |
recorded = rows.record_delivery(rt, preference['user'], kind, destination, outcome)
|
| 321 |
results.append({'user': preference['user'], 'destination': destination,
|
|
|
|
| 1 |
"""Authenticated preference routes and destination-bound notification delivery."""
|
| 2 |
|
| 3 |
import re
|
|
|
|
| 4 |
|
| 5 |
from fastapi import APIRouter, Body, Depends, Query
|
| 6 |
|
| 7 |
from core import notify_destinations as rows
|
| 8 |
from deps import Session, err, require_session
|
| 9 |
import email_send
|
|
|
|
| 10 |
|
| 11 |
NotificationItem = rows.NotificationItem
|
| 12 |
NotificationPayload = rows.NotificationPayload
|
|
|
|
| 26 |
'notification_source_required': 'This notification needs source records so the Slack agent can check its read access.',
|
| 27 |
'transport_error': 'Slack delivery was not confirmed. Check Slack before retrying.',
|
| 28 |
'slack_readiness_unavailable': 'Slack readiness could not be checked. Check its connection and permissions.',
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
}
|
| 30 |
|
| 31 |
|
|
|
|
| 43 |
raise err(401, 'notification_user_required', 'Sign in to edit notification destinations.')
|
| 44 |
|
| 45 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
@router.get('/destinations')
|
| 47 |
def get_destinations(session: Session = Depends(require_session)):
|
| 48 |
_own_session(session)
|
|
|
|
| 51 |
except Exception:
|
| 52 |
raise err(503, 'notification_store_unavailable', 'Notification destinations are unavailable for this workspace.')
|
| 53 |
mail_ready = email_send.availability(session.runtime)
|
|
|
|
| 54 |
projected = []
|
| 55 |
for preference in destinations:
|
| 56 |
readiness = {
|
|
|
|
| 60 |
}
|
| 61 |
if preference['email']:
|
| 62 |
readiness['email:' + preference['email']] = dict(mail_ready)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
projected.append({**preference, 'label': rows.KIND_LABELS[preference['kind']], 'readiness': readiness})
|
| 64 |
+
return {'destinations': projected, 'email': mail_ready}
|
|
|
|
|
|
|
| 65 |
|
| 66 |
|
| 67 |
def _save(kind, body, session, *, replace):
|
|
|
|
| 70 |
rows.kind_id(kind)
|
| 71 |
patch = rows.clean_patch(body, replace=replace)
|
| 72 |
except (ValueError, TypeError):
|
| 73 |
+
raise err(400, 'notification_destination_invalid', 'Choose a known notification kind, Slack channel ids and one email address.')
|
|
|
|
|
|
|
| 74 |
try:
|
| 75 |
return rows.edit(session.runtime, session.uname, kind, patch)
|
|
|
|
|
|
|
| 76 |
except Exception:
|
| 77 |
raise err(503, 'notification_store_unavailable', 'Notification destinations could not be saved for this workspace.')
|
| 78 |
|
|
|
|
| 184 |
return {'state': 'sent', 'reason': None, 'message': ''}
|
| 185 |
|
| 186 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None):
|
| 188 |
"""Deliver trusted recipient-safe items to saved picks, optionally targeting one user."""
|
| 189 |
kind, items = rows.kind_id(kind), rows.clean_items(items)
|
|
|
|
| 195 |
destinations = ['slack:' + channel for channel in preference['slackChannels']]
|
| 196 |
if preference['email']:
|
| 197 |
destinations.append('email:' + preference['email'])
|
|
|
|
|
|
|
|
|
|
| 198 |
for destination in destinations:
|
| 199 |
current = rows.get(rt, preference['user'], kind)
|
| 200 |
if not rows.selected(current, destination):
|
|
|
|
| 202 |
try:
|
| 203 |
if destination.startswith('slack:'):
|
| 204 |
outcome = _dispatch_slack(rt, current, destination[6:], kind, items, dispatcher=dispatcher)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
else:
|
| 206 |
result = email_send.send_notification(rt, destination[6:], kind, items, transport=mail_transport)
|
| 207 |
outcome = ({'state': 'sent', 'reason': None, 'message': ''} if result.get('sent') is True
|
| 208 |
else _refused(result.get('reason'), message=result.get('message')))
|
| 209 |
except Exception:
|
| 210 |
+
outcome = _refused('transport_error' if destination.startswith('slack:') else 'email_delivery_unconfirmed')
|
|
|
|
|
|
|
| 211 |
# Persistence failures propagate. An unrecorded refusal must never look successful.
|
| 212 |
recorded = rows.record_delivery(rt, preference['user'], kind, destination, outcome)
|
| 213 |
results.append({'user': preference['user'], 'destination': destination,
|
api/odoo_relational.py
CHANGED
|
@@ -103,7 +103,7 @@ LANDED_COST_LINES_KEY = "ut_odoo_landed_cost_lines"
|
|
| 103 |
from odoo_purchasing import (ORDERS_KEY as PURCHASE_ORDERS_KEY, LINES_KEY as PURCHASE_LINES_KEY,
|
| 104 |
PICKINGS_KEY, LANDED_PICKINGS_FIELD, landed_picking_field,
|
| 105 |
landed_picking_sql, require_landed_cost_mirror,
|
| 106 |
-
picking_fields
|
| 107 |
order_fields as purchase_order_fields,
|
| 108 |
line_fields as _purchase_line_fields,
|
| 109 |
source_specs as purchase_source_specs,
|
|
@@ -112,77 +112,6 @@ from odoo_purchasing import (ORDERS_KEY as PURCHASE_ORDERS_KEY, LINES_KEY as PUR
|
|
| 112 |
#: S78-T14: the databases whose row id is a `stock.picking` id. Their read wall admits a delivery photo.
|
| 113 |
DELIVERY_KEYS = (SHIPMENTS_KEY, PICKINGS_KEY)
|
| 114 |
|
| 115 |
-
DELIVERY_PHOTO_REFERENCE = "delivery_photo_attachments"
|
| 116 |
-
_PHOTO_PRODUCER_MISSING = "delivery_photo_metadata_unavailable"
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
def delivery_photo_reference_field():
|
| 120 |
-
"""Declare a metadata-only attachment-reference array using the supported JSON field type."""
|
| 121 |
-
return _preset({"key": DELIVERY_PHOTO_REFERENCE, "label": "Delivery photo attachments",
|
| 122 |
-
"type": "json", "source": "overlay", "default": False,
|
| 123 |
-
"description": "Attachment-reference array: id, res_model, res_id, name, mimetype, "
|
| 124 |
-
"file_size and create_date. Metadata only. No URLs or file contents. "
|
| 125 |
-
"An unavailable producer returns {error: {code, cause, fix}}.",
|
| 126 |
-
"attachmentReference": {"source": "odoo", "model": "ir.attachment", "multiple": True,
|
| 127 |
-
"fields": {"id": "int", "res_model": "text", "res_id": "int", "name": "text",
|
| 128 |
-
"mimetype": "text", "file_size": "int", "create_date": "datetime"}}})
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
def picking_fields():
|
| 132 |
-
"""Add delivery attachment references to the existing picking field contract."""
|
| 133 |
-
return _picking_fields() + [delivery_photo_reference_field()]
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
def _delivery_photo_references(value):
|
| 137 |
-
"""Decode references and distinguish a missing producer from an empty attachment list."""
|
| 138 |
-
if isinstance(value, str) and value != _PHOTO_PRODUCER_MISSING:
|
| 139 |
-
value = _json.loads(value)
|
| 140 |
-
if value is None or value == _PHOTO_PRODUCER_MISSING:
|
| 141 |
-
return {"error": {"code": "field_unavailable",
|
| 142 |
-
"cause": "Delivery photo attachments have no complete persisted metadata producer.",
|
| 143 |
-
"fix": "Run this tenant's Odoo source sync with delivery_photo_metadata, then publish "
|
| 144 |
-
"its complete observation snapshot."}}
|
| 145 |
-
return value
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
def _merge_delivery_photo_cols(sources):
|
| 149 |
-
"""Project tenant-local attachment metadata through the existing delivery row walls."""
|
| 150 |
-
out = dict(sources)
|
| 151 |
-
for key, spec in sources.items():
|
| 152 |
-
if base_table_key(key) not in DELIVERY_KEYS:
|
| 153 |
-
continue
|
| 154 |
-
owner = (spec["id"].rsplit(".", 1)[0] + "." if "." in spec["id"]
|
| 155 |
-
else spec["table"] + ".")
|
| 156 |
-
missing = ("'" + _json.dumps(_PHOTO_PRODUCER_MISSING) + "'", _delivery_photo_references)
|
| 157 |
-
marker = ("CASE (SELECT legacy FROM odoo_connections pc WHERE pc.connection_id = "
|
| 158 |
-
+ owner + "_connection_id) WHEN TRUE THEN 'delivery_photo_metadata' "
|
| 159 |
-
"WHEN FALSE THEN 'delivery_photo_metadata.connection.' || sha256("
|
| 160 |
-
+ owner + "_connection_id) END")
|
| 161 |
-
payload = ("SELECT to_json(list(struct_pack(id := coalesce(pa._source_id, pa.id), "
|
| 162 |
-
"res_model := pa.res_model, res_id := pa.res_id, name := pa.name, "
|
| 163 |
-
"mimetype := pa.mimetype, file_size := pa.file_size, create_date := pa.create_date) "
|
| 164 |
-
"ORDER BY coalesce(pa._source_id, pa.id))) FROM delivery_photo_metadata pa "
|
| 165 |
-
"WHERE pa.res_model = 'stock.picking' AND pa.res_id = coalesce("
|
| 166 |
-
+ owner + "_source_id, " + owner + "id) AND pa._connection_id "
|
| 167 |
-
"IS NOT DISTINCT FROM " + owner + "_connection_id")
|
| 168 |
-
sql = ("CASE WHEN EXISTS (SELECT 1 FROM _sync_state ps WHERE ps.entity = " + marker
|
| 169 |
-
+ " AND ps.phase = 'live') THEN coalesce((" + payload + "), '[]') ELSE "
|
| 170 |
-
+ missing[0] + " END")
|
| 171 |
-
cols = {**spec["cols"], DELIVERY_PHOTO_REFERENCE: (sql, _delivery_photo_references)}
|
| 172 |
-
optional = list(spec.get("optional") or ())
|
| 173 |
-
from harness import datastore
|
| 174 |
-
requirements = [("delivery_photo_metadata", field) for field in
|
| 175 |
-
(*datastore.DELIVERY_PHOTO_FIELDS, "_source_id", "_connection_id")]
|
| 176 |
-
requirements += [("odoo_connections", "connection_id"), ("odoo_connections", "legacy"),
|
| 177 |
-
("_sync_state", "entity"), ("_sync_state", "phase"),
|
| 178 |
-
("stock_picking", "_source_id"), ("stock_picking", "_connection_id")]
|
| 179 |
-
for requirement in requirements:
|
| 180 |
-
fallback = {"absent": (requirement,), "without": {"cols": {DELIVERY_PHOTO_REFERENCE: missing}}}
|
| 181 |
-
if fallback not in optional:
|
| 182 |
-
optional.append(fallback)
|
| 183 |
-
out[key] = {**spec, "cols": cols, "optional": tuple(optional)}
|
| 184 |
-
return out
|
| 185 |
-
|
| 186 |
#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
|
| 187 |
JOIN_KEY = "partner_id"
|
| 188 |
#: The product-grain equivalent.
|
|
@@ -780,7 +709,7 @@ def merge_audit_cols(sources):
|
|
| 780 |
if optional:
|
| 781 |
merged["optional"] = tuple(optional)
|
| 782 |
out[key] = merged
|
| 783 |
-
return
|
| 784 |
|
| 785 |
|
| 786 |
#: Materialised grids read their audited columns beside their reader's own SQL.
|
|
@@ -896,7 +825,7 @@ def shipment_fields():
|
|
| 896 |
"default": True, "description": "Elapsed days from the sale order date to the actual "
|
| 897 |
"shipment completion. Blank when either date is missing."},
|
| 898 |
_refreshed_field(),
|
| 899 |
-
)] + audit_fields(SHIPMENTS_KEY)
|
| 900 |
|
| 901 |
|
| 902 |
def agent_fields():
|
|
|
|
| 103 |
from odoo_purchasing import (ORDERS_KEY as PURCHASE_ORDERS_KEY, LINES_KEY as PURCHASE_LINES_KEY,
|
| 104 |
PICKINGS_KEY, LANDED_PICKINGS_FIELD, landed_picking_field,
|
| 105 |
landed_picking_sql, require_landed_cost_mirror,
|
| 106 |
+
picking_fields, picking_source_spec,
|
| 107 |
order_fields as purchase_order_fields,
|
| 108 |
line_fields as _purchase_line_fields,
|
| 109 |
source_specs as purchase_source_specs,
|
|
|
|
| 112 |
#: S78-T14: the databases whose row id is a `stock.picking` id. Their read wall admits a delivery photo.
|
| 113 |
DELIVERY_KEYS = (SHIPMENTS_KEY, PICKINGS_KEY)
|
| 114 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
|
| 116 |
JOIN_KEY = "partner_id"
|
| 117 |
#: The product-grain equivalent.
|
|
|
|
| 709 |
if optional:
|
| 710 |
merged["optional"] = tuple(optional)
|
| 711 |
out[key] = merged
|
| 712 |
+
return out
|
| 713 |
|
| 714 |
|
| 715 |
#: Materialised grids read their audited columns beside their reader's own SQL.
|
|
|
|
| 825 |
"default": True, "description": "Elapsed days from the sale order date to the actual "
|
| 826 |
"shipment completion. Blank when either date is missing."},
|
| 827 |
_refreshed_field(),
|
| 828 |
+
)] + audit_fields(SHIPMENTS_KEY)
|
| 829 |
|
| 830 |
|
| 831 |
def agent_fields():
|
api/routes_account_health.py
CHANGED
|
@@ -5,8 +5,7 @@ from urllib.parse import urlencode
|
|
| 5 |
from deps import Session, err, require_session
|
| 6 |
|
| 7 |
TEMPLATE_KIND = "account_health"
|
| 8 |
-
router = APIRouter(tags=["account-health"])
|
| 9 |
-
app_router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["account-health"])
|
| 10 |
SOURCE = "ut_amzn_account_health"
|
| 11 |
|
| 12 |
|
|
@@ -102,7 +101,7 @@ def owners(session):
|
|
| 102 |
return sorted((n for n in names if n), key=str.lower)
|
| 103 |
|
| 104 |
|
| 105 |
-
@
|
| 106 |
def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
|
| 107 |
offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 108 |
metric: str | None = Query(None), session: Session = Depends(_app_reader)):
|
|
@@ -116,7 +115,7 @@ def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
|
|
| 116 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 117 |
|
| 118 |
|
| 119 |
-
@
|
| 120 |
def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 121 |
limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
|
| 122 |
ah = _module()
|
|
@@ -134,7 +133,7 @@ def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Qu
|
|
| 134 |
return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
|
| 135 |
|
| 136 |
|
| 137 |
-
@
|
| 138 |
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 139 |
search: str = Query(""), offset: int = Query(0, ge=0), metric: str | None = Query(None),
|
| 140 |
cursor: str | None = Query(None), limit: int = Query(50, ge=1),
|
|
@@ -150,7 +149,7 @@ def list_items(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 150 |
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 151 |
|
| 152 |
|
| 153 |
-
@
|
| 154 |
def read_item(item_id: str, app: str = Query(..., min_length=1),
|
| 155 |
session: Session = Depends(_app_reader)):
|
| 156 |
ah = _module()
|
|
@@ -164,7 +163,7 @@ def read_item(item_id: str, app: str = Query(..., min_length=1),
|
|
| 164 |
"canEdit": _may_edit(session, _app(session, app))}
|
| 165 |
|
| 166 |
|
| 167 |
-
@
|
| 168 |
def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 169 |
due: str | None = Body(None), note: str | None = Body(None),
|
| 170 |
session: Session = Depends(_app_writer)):
|
|
@@ -177,13 +176,13 @@ def patch_item(item_id: str, owner: str | None = Body(None), status: str | None
|
|
| 177 |
due=due, note=note, st=session.runtime)
|
| 178 |
|
| 179 |
|
| 180 |
-
@
|
| 181 |
def read_settings(session: Session = Depends(_app_reader)):
|
| 182 |
ah = _module()
|
| 183 |
return _call(ah.settings, _tenant(session), st=session.runtime)
|
| 184 |
|
| 185 |
|
| 186 |
-
@
|
| 187 |
def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: str | None = Body(None),
|
| 188 |
riskOverdueDays: float | None = Body(None),
|
| 189 |
session: Session = Depends(_app_writer)):
|
|
@@ -192,7 +191,7 @@ def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: st
|
|
| 192 |
default_owner=defaultOwner, risk_overdue_days=riskOverdueDays, st=session.runtime)
|
| 193 |
|
| 194 |
|
| 195 |
-
@
|
| 196 |
def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 197 |
search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 198 |
session: Session = Depends(_app_reader)):
|
|
@@ -206,7 +205,7 @@ def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 206 |
return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 207 |
|
| 208 |
|
| 209 |
-
@
|
| 210 |
def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 211 |
caseId: str | None = Body(None), appealDate: str | None = Body(None),
|
| 212 |
outcome: str | None = Body(None), note: str | None = Body(None),
|
|
@@ -219,22 +218,6 @@ def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None
|
|
| 219 |
case_id=caseId, appeal_date=appealDate, outcome=outcome, note=note, st=session.runtime)
|
| 220 |
|
| 221 |
|
| 222 |
-
router.include_router(app_router)
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
@router.get("/api/v1/account-health/categories/{key}")
|
| 226 |
-
def read_category(key: str, session: Session = Depends(_app_reader)):
|
| 227 |
-
ah = _module()
|
| 228 |
-
context = _call(_narrowed, session)
|
| 229 |
-
|
| 230 |
-
def allowed(row, snapshot):
|
| 231 |
-
evidence = _call(ah.category_source, _tenant(session), session.runtime, row, snapshot)
|
| 232 |
-
return _source_allowed({"sourceEvidence": evidence}, context)
|
| 233 |
-
|
| 234 |
-
return _call(ah.category_detail, _tenant(session), key, user=session.user,
|
| 235 |
-
allowed=allowed if context is not None else None, st=session.runtime)
|
| 236 |
-
|
| 237 |
-
|
| 238 |
# --------------------------------------------------------------------------- R62 source narrowing
|
| 239 |
|
| 240 |
def _narrowed(session):
|
|
|
|
| 5 |
from deps import Session, err, require_session
|
| 6 |
|
| 7 |
TEMPLATE_KIND = "account_health"
|
| 8 |
+
router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["account-health"])
|
|
|
|
| 9 |
SOURCE = "ut_amzn_account_health"
|
| 10 |
|
| 11 |
|
|
|
|
| 101 |
return sorted((n for n in names if n), key=str.lower)
|
| 102 |
|
| 103 |
|
| 104 |
+
@router.get("/summary")
|
| 105 |
def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
|
| 106 |
offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 107 |
metric: str | None = Query(None), session: Session = Depends(_app_reader)):
|
|
|
|
| 115 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 116 |
|
| 117 |
|
| 118 |
+
@router.get("/history")
|
| 119 |
def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 120 |
limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
|
| 121 |
ah = _module()
|
|
|
|
| 133 |
return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
|
| 134 |
|
| 135 |
|
| 136 |
+
@router.get("/items")
|
| 137 |
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 138 |
search: str = Query(""), offset: int = Query(0, ge=0), metric: str | None = Query(None),
|
| 139 |
cursor: str | None = Query(None), limit: int = Query(50, ge=1),
|
|
|
|
| 149 |
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 150 |
|
| 151 |
|
| 152 |
+
@router.get("/items/{item_id}")
|
| 153 |
def read_item(item_id: str, app: str = Query(..., min_length=1),
|
| 154 |
session: Session = Depends(_app_reader)):
|
| 155 |
ah = _module()
|
|
|
|
| 163 |
"canEdit": _may_edit(session, _app(session, app))}
|
| 164 |
|
| 165 |
|
| 166 |
+
@router.patch("/items/{item_id}")
|
| 167 |
def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 168 |
due: str | None = Body(None), note: str | None = Body(None),
|
| 169 |
session: Session = Depends(_app_writer)):
|
|
|
|
| 176 |
due=due, note=note, st=session.runtime)
|
| 177 |
|
| 178 |
|
| 179 |
+
@router.get("/settings")
|
| 180 |
def read_settings(session: Session = Depends(_app_reader)):
|
| 181 |
ah = _module()
|
| 182 |
return _call(ah.settings, _tenant(session), st=session.runtime)
|
| 183 |
|
| 184 |
|
| 185 |
+
@router.put("/settings")
|
| 186 |
def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: str | None = Body(None),
|
| 187 |
riskOverdueDays: float | None = Body(None),
|
| 188 |
session: Session = Depends(_app_writer)):
|
|
|
|
| 191 |
default_owner=defaultOwner, risk_overdue_days=riskOverdueDays, st=session.runtime)
|
| 192 |
|
| 193 |
|
| 194 |
+
@router.get("/risks")
|
| 195 |
def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 196 |
search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 197 |
session: Session = Depends(_app_reader)):
|
|
|
|
| 205 |
return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 206 |
|
| 207 |
|
| 208 |
+
@router.patch("/risks/{item_id}")
|
| 209 |
def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 210 |
caseId: str | None = Body(None), appealDate: str | None = Body(None),
|
| 211 |
outcome: str | None = Body(None), note: str | None = Body(None),
|
|
|
|
| 218 |
case_id=caseId, appeal_date=appealDate, outcome=outcome, note=note, st=session.runtime)
|
| 219 |
|
| 220 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
# --------------------------------------------------------------------------- R62 source narrowing
|
| 222 |
|
| 223 |
def _narrowed(session):
|
api/routes_admin.py
CHANGED
|
@@ -143,7 +143,17 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None):
|
|
| 143 |
# editor will open, so the two cannot disagree.
|
| 144 |
"accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st,
|
| 145 |
surface_keys=surface_keys),
|
| 146 |
-
"perms_v": int(rec.get("perms_v") or 0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
|
| 148 |
|
| 149 |
def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None):
|
|
@@ -454,6 +464,9 @@ def update_user(username: str, body: dict = Body(default=None),
|
|
| 454 |
if result == "last_admin":
|
| 455 |
raise err(400, "last_admin",
|
| 456 |
"an active tenant must retain at least one administrator")
|
|
|
|
|
|
|
|
|
|
| 457 |
if result != "updated":
|
| 458 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 459 |
|
|
@@ -539,6 +552,164 @@ def expire_password(username: str, session: Session = Depends(admin_gate)):
|
|
| 539 |
return Response(status_code=204)
|
| 540 |
|
| 541 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 542 |
# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ──────────────────
|
| 543 |
#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema
|
| 544 |
#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list
|
|
@@ -2140,7 +2311,7 @@ def _leaf_count(nodes):
|
|
| 2140 |
return n
|
| 2141 |
|
| 2142 |
|
| 2143 |
-
def _fold_legacy_scope(rec, cleaned
|
| 2144 |
"""⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold.
|
| 2145 |
|
| 2146 |
Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent`
|
|
@@ -2189,8 +2360,7 @@ def _fold_legacy_scope(rec, cleaned, session=None):
|
|
| 2189 |
return cleaned # nothing to preserve: the account was unrestricted
|
| 2190 |
out = {}
|
| 2191 |
for key, e in cleaned.items():
|
| 2192 |
-
legacy_here = _prune_to_module(
|
| 2193 |
-
legacy, {f["key"] for f in _module_fields(key, session=session)})
|
| 2194 |
tree = e.get("filter")
|
| 2195 |
if not legacy_here:
|
| 2196 |
# This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than
|
|
@@ -2607,7 +2777,7 @@ def grant_database_access(session, username, database, role):
|
|
| 2607 |
is_table = key.startswith(user_tables.KEY_PREFIX)
|
| 2608 |
if role not in ('view', 'edit') or (not is_table and role != 'view'):
|
| 2609 |
raise err(409, 'unsupported_role', 'This database supports only viewing access.')
|
| 2610 |
-
tenant = str(session.tenant).strip().lower()
|
| 2611 |
if _tenant_of(session.user) != tenant:
|
| 2612 |
raise err(403, 'forbidden', 'Database access could not be granted.')
|
| 2613 |
# Read the account store strictly rather than the directory's fail-silent projection.
|
|
@@ -2632,7 +2802,7 @@ def grant_database_access(session, username, database, role):
|
|
| 2632 |
candidate = dict(prior or {}, access=True)
|
| 2633 |
cleaned = _clean_perms({key: candidate}, governed_keys=governed,
|
| 2634 |
session=session, stored=rec.get('perms'))
|
| 2635 |
-
folded = _fold_legacy_scope(rec, cleaned
|
| 2636 |
_guard_bu_widening(rec, folded, False)
|
| 2637 |
# Validation may prune retired fields. This click changes access, so retain stored walls.
|
| 2638 |
if folded[key].get('filter') != cleaned[key].get('filter'):
|
|
@@ -2740,7 +2910,7 @@ def set_database_block(session, username, database, entry, *, expect, confirm_wi
|
|
| 2740 |
raise err(400, "bad_perms", "The entry needs access set to true or false.")
|
| 2741 |
if not isinstance(expect, (dict, str)) or not expect:
|
| 2742 |
raise err(400, "missing_expect", "Read the account permissions first, then send its access state.")
|
| 2743 |
-
tenant = str(session.tenant).strip().lower()
|
| 2744 |
if _tenant_of(session.user) != tenant:
|
| 2745 |
raise err(403, "forbidden", "Database access could not be changed.")
|
| 2746 |
# Read the account store strictly rather than the directory's fail-silent projection.
|
|
@@ -2760,7 +2930,7 @@ def set_database_block(session, username, database, entry, *, expect, confirm_wi
|
|
| 2760 |
if field in entry}
|
| 2761 |
cleaned = _clean_perms({key: submitted}, governed_keys=governed, session=session,
|
| 2762 |
stored=rec.get("perms"))
|
| 2763 |
-
folded = _fold_legacy_scope(rec, cleaned
|
| 2764 |
_guard_bu_widening(rec, folded, bool(confirm_widen))
|
| 2765 |
candidate = {field: copy.deepcopy(folded[key][field]) for field in users.DATABASE_ENTRY_KEYS}
|
| 2766 |
prior = copy.deepcopy((rec.get("perms") or {}).get(key))
|
|
@@ -2841,8 +3011,7 @@ def put_perms(username: str, body: dict = Body(default=None),
|
|
| 2841 |
governed = {m["key"] for m in _mods}
|
| 2842 |
opaque = {key: entry for key, entry in cleaned.items() if key not in governed}
|
| 2843 |
known = _fold_legacy_scope(reg[uname],
|
| 2844 |
-
{key: entry for key, entry in cleaned.items() if key in governed}
|
| 2845 |
-
session=session)
|
| 2846 |
_guard_bu_widening(reg[uname], known, bool(body.get("confirm_widen")))
|
| 2847 |
cleaned = dict(known, **opaque)
|
| 2848 |
users.set_access(uname, perms=cleaned)
|
|
|
|
| 143 |
# editor will open, so the two cannot disagree.
|
| 144 |
"accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st,
|
| 145 |
surface_keys=surface_keys),
|
| 146 |
+
"perms_v": int(rec.get("perms_v") or 0),
|
| 147 |
+
# S79-T63: only a copied test account carries `test`, so a real account's keys stay exact.
|
| 148 |
+
**({"test": _test_marker(rec)} if _test_marker(rec) else {})}
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def _test_marker(rec):
|
| 152 |
+
mark = (rec or {}).get(users.TEST_ACCOUNT_KEY)
|
| 153 |
+
if not isinstance(mark, dict):
|
| 154 |
+
return None
|
| 155 |
+
return {"copiedFrom": mark.get("copied_from"), "createdBy": mark.get("created_by"),
|
| 156 |
+
"createdAt": mark.get("created_at")}
|
| 157 |
|
| 158 |
|
| 159 |
def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None):
|
|
|
|
| 464 |
if result == "last_admin":
|
| 465 |
raise err(400, "last_admin",
|
| 466 |
"an active tenant must retain at least one administrator")
|
| 467 |
+
if result in _TEST_ACCOUNT_REFUSALS:
|
| 468 |
+
# R42: a test account gets no email and no admin role, refused in the core writer.
|
| 469 |
+
raise err(400, result, _TEST_ACCOUNT_REFUSALS[result])
|
| 470 |
if result != "updated":
|
| 471 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 472 |
|
|
|
|
| 552 |
return Response(status_code=204)
|
| 553 |
|
| 554 |
|
| 555 |
+
# ── S79-T63 (R28, R42): test accounts that copy one member's permissions exactly ─────────────────
|
| 556 |
+
_TEST_ACCOUNT_REFUSALS = {
|
| 557 |
+
"test_account_no_email": "A test account cannot have an email address.",
|
| 558 |
+
"test_account_no_whatsapp": "A test account cannot connect WhatsApp.",
|
| 559 |
+
"test_account_not_admin": "A test account cannot be an administrator.",
|
| 560 |
+
}
|
| 561 |
+
|
| 562 |
+
|
| 563 |
+
def _copy_refusal(exc):
|
| 564 |
+
return err(exc.status, exc.code, str(exc))
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
#: The en and em dash, built from code points so this file carries no dash character (rule 2).
|
| 568 |
+
def _copy_failure(exc, verb):
|
| 569 |
+
# A store error can carry any text; rule 2 keeps dashes off the screen.
|
| 570 |
+
import dash_norm
|
| 571 |
+
cause = dash_norm.plain(exc.cause).strip().rstrip(".")
|
| 572 |
+
message = f"The test account was not {verb}. {cause}."
|
| 573 |
+
if exc.leftovers:
|
| 574 |
+
message += (" These parts could not be removed: " + ", ".join(exc.leftovers)
|
| 575 |
+
+ ". Remove the test account again to finish.")
|
| 576 |
+
elif verb == "created":
|
| 577 |
+
message += " Nothing was kept."
|
| 578 |
+
return err(503, "test_account_failed", message)
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
def _test_account_steps(session):
|
| 582 |
+
"""Copy steps for grant stores outside `core`: Connector grants, through their own route."""
|
| 583 |
+
import core.perm_scope as perm_scope
|
| 584 |
+
import routes_keychain as keychain
|
| 585 |
+
|
| 586 |
+
def held():
|
| 587 |
+
doc = session.runtime.get(perm_scope.CONNECTOR_GRANT_KEY, fresh=True)
|
| 588 |
+
if doc is not None and not isinstance(doc, dict):
|
| 589 |
+
raise RuntimeError("connector access could not be read")
|
| 590 |
+
return doc or {}
|
| 591 |
+
|
| 592 |
+
def copy_grants(username, source):
|
| 593 |
+
known = {kind["key"] for kind in keychain.connector_grant_kinds()}
|
| 594 |
+
kinds = sorted(k for k in (held().get(source) or ()) if k in known)
|
| 595 |
+
if kinds:
|
| 596 |
+
keychain.put_connector_grants(username, {"granted": kinds}, session)
|
| 597 |
+
|
| 598 |
+
def clear_grants(username):
|
| 599 |
+
if username in held():
|
| 600 |
+
keychain.put_connector_grants(username, {"granted": []}, session)
|
| 601 |
+
|
| 602 |
+
return [("connector access", copy_grants, clear_grants)]
|
| 603 |
+
|
| 604 |
+
|
| 605 |
+
@router.get("/admin/users/{username}/test-account/preview")
|
| 606 |
+
def preview_test_account(username: str, session: Session = Depends(admin_gate)):
|
| 607 |
+
"""What a test account copied from this member would carry, before anything is written."""
|
| 608 |
+
from core import test_accounts, user_tables
|
| 609 |
+
reg = _registry()
|
| 610 |
+
try:
|
| 611 |
+
plan = test_accounts.plan(username, reg, tenant=_tenant_of(session.user),
|
| 612 |
+
st=session.runtime)
|
| 613 |
+
lent = user_tables.lend_defs(session.runtime, schema_only=True)
|
| 614 |
+
except test_accounts.CopyRefused as exc:
|
| 615 |
+
raise _copy_refusal(exc)
|
| 616 |
+
except Exception as exc: # noqa: BLE001
|
| 617 |
+
raise err(503, "store_unavailable", "Access unavailable. Try again.") from exc
|
| 618 |
+
limits = _source_limits(plan["source"], reg.get(plan["source"]), session)
|
| 619 |
+
body = test_accounts.preview(plan, st=lent, limits=limits)
|
| 620 |
+
_label_grants(session, body["copy"]["grants"])
|
| 621 |
+
return body
|
| 622 |
+
|
| 623 |
+
|
| 624 |
+
def _source_limits(uname, rec, session):
|
| 625 |
+
"""The source's row limits, phrased for the administrator; None if they cannot be read.
|
| 626 |
+
|
| 627 |
+
Field walls are lifted for the phrasing only, so a condition on a hidden field is named
|
| 628 |
+
instead of reading "Limited by your access". The filters themselves are the source's.
|
| 629 |
+
"""
|
| 630 |
+
import core.perm_scope as perm_scope
|
| 631 |
+
try:
|
| 632 |
+
reader = deepcopy(users._public(uname, rec))
|
| 633 |
+
for entry in (reader.get("perms") or {}).values():
|
| 634 |
+
if isinstance(entry, dict):
|
| 635 |
+
entry.pop("hiddenFields", None)
|
| 636 |
+
return perm_scope.scope_limits(reader, st=session.runtime)
|
| 637 |
+
except Exception: # noqa: BLE001
|
| 638 |
+
return None
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
#: What a preview row says when a shared item's name cannot be read. Never a raw id.
|
| 642 |
+
_GRANT_FALLBACK = {"view": "A shared view", "folder": "A shared folder", "database": "A database",
|
| 643 |
+
"field": "A shared field", "cohort": "A locked view", "workbook": "A shared sheet"}
|
| 644 |
+
#: Naming stops after this many seconds; every grant is still listed, the rest by kind.
|
| 645 |
+
_LABEL_BUDGET_S = 3.0
|
| 646 |
+
|
| 647 |
+
|
| 648 |
+
def _label_grants(session, grants):
|
| 649 |
+
"""Name each grant through the share door's own resolver (`routes_shares._object_ref`)."""
|
| 650 |
+
import time
|
| 651 |
+
try:
|
| 652 |
+
from routes_shares import _object_ref
|
| 653 |
+
except Exception: # noqa: BLE001
|
| 654 |
+
_object_ref = None
|
| 655 |
+
deadline = time.monotonic() + _LABEL_BUDGET_S
|
| 656 |
+
for grant in grants:
|
| 657 |
+
if grant.get("label"):
|
| 658 |
+
continue
|
| 659 |
+
label = None
|
| 660 |
+
if time.monotonic() < deadline:
|
| 661 |
+
try:
|
| 662 |
+
if grant["kind"] == "workbook":
|
| 663 |
+
# `_object_ref` has no workbook branch; the workbook's own name is the label.
|
| 664 |
+
import core.sheets as sheets
|
| 665 |
+
label = str(sheets.get_workbook(session.runtime, grant["oid"]).get("name") or "")
|
| 666 |
+
elif _object_ref is not None:
|
| 667 |
+
label = str(_object_ref(session, grant["kind"], grant["oid"])[0] or "")
|
| 668 |
+
except Exception: # noqa: BLE001
|
| 669 |
+
label = None
|
| 670 |
+
label = (label or "").strip()
|
| 671 |
+
grant["label"] = label or _GRANT_FALLBACK.get(grant["kind"], "A shared item")
|
| 672 |
+
|
| 673 |
+
|
| 674 |
+
@router.post("/admin/test-accounts", status_code=201)
|
| 675 |
+
def create_test_account(body: dict = Body(default=None), session: Session = Depends(admin_gate)):
|
| 676 |
+
"""Create a test account whose permissions copy `source` exactly. The password shows once."""
|
| 677 |
+
from core import test_accounts
|
| 678 |
+
body = body if isinstance(body, dict) else {}
|
| 679 |
+
_require_store()
|
| 680 |
+
try:
|
| 681 |
+
made = test_accounts.create(body.get("source"), admin=session.uname,
|
| 682 |
+
tenant=_tenant_of(session.user), st=session.runtime,
|
| 683 |
+
steps=_test_account_steps(session))
|
| 684 |
+
except test_accounts.CopyRefused as exc:
|
| 685 |
+
raise _copy_refusal(exc)
|
| 686 |
+
except test_accounts.CopyFailed as exc:
|
| 687 |
+
raise _copy_failure(exc, "created")
|
| 688 |
+
except Exception as exc: # noqa: BLE001
|
| 689 |
+
raise err(503, "store_unavailable", "The test account was not created. Try again.") from exc
|
| 690 |
+
rec = _registry().get(made["username"])
|
| 691 |
+
if not isinstance(rec, dict):
|
| 692 |
+
raise err(503, "store_unavailable", "The test account was not saved. Try again.")
|
| 693 |
+
return {"user": _view(made["username"], rec), "password": made["password"]}
|
| 694 |
+
|
| 695 |
+
|
| 696 |
+
@router.delete("/admin/test-accounts/{username}", status_code=204)
|
| 697 |
+
def delete_test_account(username: str, session: Session = Depends(admin_gate)):
|
| 698 |
+
"""Remove a test account, its sign in and every grant naming it, in one call."""
|
| 699 |
+
from core import test_accounts
|
| 700 |
+
_require_store()
|
| 701 |
+
try:
|
| 702 |
+
test_accounts.remove(username, tenant=_tenant_of(session.user), st=session.runtime,
|
| 703 |
+
granter=session.uname, steps=_test_account_steps(session))
|
| 704 |
+
except test_accounts.CopyRefused as exc:
|
| 705 |
+
raise _copy_refusal(exc)
|
| 706 |
+
except test_accounts.CopyFailed as exc:
|
| 707 |
+
raise _copy_failure(exc, "fully removed")
|
| 708 |
+
except Exception as exc: # noqa: BLE001
|
| 709 |
+
raise err(503, "store_unavailable", "The test account was not removed. Try again.") from exc
|
| 710 |
+
return Response(status_code=204)
|
| 711 |
+
|
| 712 |
+
|
| 713 |
# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ──────────────────
|
| 714 |
#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema
|
| 715 |
#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list
|
|
|
|
| 2311 |
return n
|
| 2312 |
|
| 2313 |
|
| 2314 |
+
def _fold_legacy_scope(rec, cleaned):
|
| 2315 |
"""⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold.
|
| 2316 |
|
| 2317 |
Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent`
|
|
|
|
| 2360 |
return cleaned # nothing to preserve: the account was unrestricted
|
| 2361 |
out = {}
|
| 2362 |
for key, e in cleaned.items():
|
| 2363 |
+
legacy_here = _prune_to_module(legacy, {f["key"] for f in _module_fields(key)})
|
|
|
|
| 2364 |
tree = e.get("filter")
|
| 2365 |
if not legacy_here:
|
| 2366 |
# This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than
|
|
|
|
| 2777 |
is_table = key.startswith(user_tables.KEY_PREFIX)
|
| 2778 |
if role not in ('view', 'edit') or (not is_table and role != 'view'):
|
| 2779 |
raise err(409, 'unsupported_role', 'This database supports only viewing access.')
|
| 2780 |
+
tenant = str(session.runtime.tenant.slug).strip().lower()
|
| 2781 |
if _tenant_of(session.user) != tenant:
|
| 2782 |
raise err(403, 'forbidden', 'Database access could not be granted.')
|
| 2783 |
# Read the account store strictly rather than the directory's fail-silent projection.
|
|
|
|
| 2802 |
candidate = dict(prior or {}, access=True)
|
| 2803 |
cleaned = _clean_perms({key: candidate}, governed_keys=governed,
|
| 2804 |
session=session, stored=rec.get('perms'))
|
| 2805 |
+
folded = _fold_legacy_scope(rec, cleaned)
|
| 2806 |
_guard_bu_widening(rec, folded, False)
|
| 2807 |
# Validation may prune retired fields. This click changes access, so retain stored walls.
|
| 2808 |
if folded[key].get('filter') != cleaned[key].get('filter'):
|
|
|
|
| 2910 |
raise err(400, "bad_perms", "The entry needs access set to true or false.")
|
| 2911 |
if not isinstance(expect, (dict, str)) or not expect:
|
| 2912 |
raise err(400, "missing_expect", "Read the account permissions first, then send its access state.")
|
| 2913 |
+
tenant = str(session.runtime.tenant.slug).strip().lower()
|
| 2914 |
if _tenant_of(session.user) != tenant:
|
| 2915 |
raise err(403, "forbidden", "Database access could not be changed.")
|
| 2916 |
# Read the account store strictly rather than the directory's fail-silent projection.
|
|
|
|
| 2930 |
if field in entry}
|
| 2931 |
cleaned = _clean_perms({key: submitted}, governed_keys=governed, session=session,
|
| 2932 |
stored=rec.get("perms"))
|
| 2933 |
+
folded = _fold_legacy_scope(rec, cleaned)
|
| 2934 |
_guard_bu_widening(rec, folded, bool(confirm_widen))
|
| 2935 |
candidate = {field: copy.deepcopy(folded[key][field]) for field in users.DATABASE_ENTRY_KEYS}
|
| 2936 |
prior = copy.deepcopy((rec.get("perms") or {}).get(key))
|
|
|
|
| 3011 |
governed = {m["key"] for m in _mods}
|
| 3012 |
opaque = {key: entry for key, entry in cleaned.items() if key not in governed}
|
| 3013 |
known = _fold_legacy_scope(reg[uname],
|
| 3014 |
+
{key: entry for key, entry in cleaned.items() if key in governed})
|
|
|
|
| 3015 |
_guard_bu_widening(reg[uname], known, bool(body.get("confirm_widen")))
|
| 3016 |
cleaned = dict(known, **opaque)
|
| 3017 |
users.set_access(uname, perms=cleaned)
|
api/routes_amazon_projects.py
CHANGED
|
@@ -12,7 +12,7 @@ from pathlib import Path
|
|
| 12 |
import sys
|
| 13 |
from typing import Any
|
| 14 |
|
| 15 |
-
from fastapi import APIRouter, Body, Depends, HTTPException
|
| 16 |
|
| 17 |
from core import amazon_projects as projects
|
| 18 |
from core import amazon_project_results as project_results
|
|
@@ -219,7 +219,7 @@ def _distinct(found, pairs):
|
|
| 219 |
|
| 220 |
|
| 221 |
def cell_values(records):
|
| 222 |
-
"""
|
| 223 |
counted = projects.summary(records)
|
| 224 |
if not counted["total"]:
|
| 225 |
return {PROGRESS_KEY: None, LABEL_KEY: None}
|
|
@@ -343,34 +343,28 @@ def _result_sources(session, targets):
|
|
| 343 |
return available
|
| 344 |
|
| 345 |
|
| 346 |
-
class ProjectMirror(sku_action_results.Mirror):
|
| 347 |
-
"""Read a deduplicated project SKU union and retain source evidence for every target day."""
|
| 348 |
-
|
| 349 |
-
def __init__(self, con, skus, available):
|
| 350 |
-
super().__init__(con, {}, available)
|
| 351 |
-
self.skus, self.coverage = tuple(skus), []
|
| 352 |
-
|
| 353 |
-
def entity(self):
|
| 354 |
-
self.field(sku_action_results.ECONOMICS, "msku")
|
| 355 |
-
return '"msku" IN (' + ", ".join("?" for _ in self.skus) + ")", self.skus
|
| 356 |
-
|
| 357 |
-
def raw(self, table, days, values, **kwargs):
|
| 358 |
-
plan = super().raw(table, days, values, **kwargs)
|
| 359 |
-
sku = self.field(table, *kwargs["sku_fields"])
|
| 360 |
-
head, tail = plan.sql.split(" FROM ", 1)
|
| 361 |
-
coverage = sku_action_results.Plan(
|
| 362 |
-
head + ", " + sku_action_results._q(sku) + ' AS "__project_member" FROM ' + tail,
|
| 363 |
-
plan.args, plan.sources)
|
| 364 |
-
self.coverage.append((table, tuple(days), coverage))
|
| 365 |
-
return plan
|
| 366 |
-
|
| 367 |
-
|
| 368 |
def _result_figure(con, config, bounds, skus, available):
|
| 369 |
"""Reuse daily metric definitions and reconcile their union against separate source totals."""
|
| 370 |
if not skus:
|
| 371 |
return {"state": "unavailable", "reason": "Assign an Amazon target before measuring results."}
|
| 372 |
|
| 373 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 374 |
try:
|
| 375 |
mirror.select_scope(bounds)
|
| 376 |
except sku_action_results.SourceUnavailable as exc:
|
|
@@ -412,81 +406,6 @@ def _complete_result_membership(con, coverage, bounds, sources, skus, names=("cu
|
|
| 412 |
return True
|
| 413 |
|
| 414 |
|
| 415 |
-
_SERIES_UNAVAILABLE = {
|
| 416 |
-
"margin": "Margin requires daily net profit with verified Odoo or typed COGS. "
|
| 417 |
-
"amzn_economics alone cannot reconcile that cost.",
|
| 418 |
-
"returns": "Returns require FBA and MFN customer return events. "
|
| 419 |
-
"amzn_economics has no independent returned-unit aggregate.",
|
| 420 |
-
"returns_pct": "Returns % requires returned units over shipped units. "
|
| 421 |
-
"amzn_economics cannot independently reconcile those quantities.",
|
| 422 |
-
"refunds_pct": "Refunds % requires deduplicated refund units over shipped units. "
|
| 423 |
-
"amzn_economics cannot independently reconcile those quantities.",
|
| 424 |
-
"roas": "ROAS requires attributed ad sales over ad spend. "
|
| 425 |
-
"amzn_economics has no independent attributed ad-sales aggregate.",
|
| 426 |
-
}
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
def _series_figure(key, figure, bounds):
|
| 430 |
-
"""Adapt existing sales and units definitions without publishing partial source windows."""
|
| 431 |
-
if key in _SERIES_UNAVAILABLE:
|
| 432 |
-
return {"reason": _SERIES_UNAVAILABLE[key]}
|
| 433 |
-
if figure.get("reason"):
|
| 434 |
-
return {"reason": figure["reason"]}
|
| 435 |
-
current = figure.get("current")
|
| 436 |
-
if not current or current["status"] != "available":
|
| 437 |
-
return {"reason": (current or {}).get("reason") or "Some target source days are unavailable."}
|
| 438 |
-
if key != "sales_yoy":
|
| 439 |
-
return {"total": current["value"], "points": current["series"], "unit": figure["unit"],
|
| 440 |
-
"reason": "Today is incomplete and excluded." if bounds["to"] == bounds["current"][1].isoformat() else None}
|
| 441 |
-
prior = figure["comparison"]
|
| 442 |
-
if prior["status"] != "available" or prior["value"] is None or prior["value"] <= 0:
|
| 443 |
-
return {"reason": prior.get("reason") or "Sales YoY requires positive sales in the matching prior window."}
|
| 444 |
-
points = []
|
| 445 |
-
for now, before in zip(current["series"], prior["series"]):
|
| 446 |
-
if before["value"] is None or before["value"] <= 0:
|
| 447 |
-
return {"reason": "Sales YoY requires positive sales on every matching prior day."}
|
| 448 |
-
points.append({"date": now["date"], "value": 100 * (now["value"] - before["value"]) / before["value"]})
|
| 449 |
-
return {"total": 100 * (current["value"] - prior["value"]) / prior["value"], "points": points,
|
| 450 |
-
"reason": "Today is incomplete and excluded." if bounds["to"] == bounds["current"][1].isoformat() else None}
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
def _series_economics_total(con, key, bounds, skus, available):
|
| 454 |
-
"""Aggregate original economics rows without the display plan, daily rollup or display total."""
|
| 455 |
-
if key in _SERIES_UNAVAILABLE:
|
| 456 |
-
return {"value": None, "reason": _SERIES_UNAVAILABLE[key]}
|
| 457 |
-
if not skus or not bounds["days"]:
|
| 458 |
-
return {"value": None, "reason": "No completed target days are available in this range."}
|
| 459 |
-
mirror = ProjectMirror(con, skus, available)
|
| 460 |
-
try:
|
| 461 |
-
mirror.select_scope(bounds)
|
| 462 |
-
field = mirror.field(sku_action_results.ECONOMICS,
|
| 463 |
-
"sales_net_units_sold" if key == "units" else "sales_net_product_sales_amount")
|
| 464 |
-
number = f'TRY_CAST({sku_action_results._q(field)} AS DOUBLE)'
|
| 465 |
-
day = sku_action_results._day("start_date")
|
| 466 |
-
marks = ", ".join("?" for _ in skus)
|
| 467 |
-
|
| 468 |
-
def total(days):
|
| 469 |
-
row = con.execute(
|
| 470 |
-
f'SELECT SUM({number}), count(*), count({number}), count(DISTINCT (msku, {day})) '
|
| 471 |
-
f'FROM amzn_economics WHERE msku IN ({marks}) AND marketplace_id = ? '
|
| 472 |
-
f'AND sales_net_product_sales_currency_code = ? AND {day} >= ? AND {day} < ?',
|
| 473 |
-
(*skus, *mirror.scope, *days)).fetchone()
|
| 474 |
-
if (not row[1] or row[1] != row[2] or row[3] != len(skus) * bounds["days"]
|
| 475 |
-
or not project_results._finite(row[0])):
|
| 476 |
-
raise sku_action_results.SourceUnavailable("amzn_economics: some target values or days are unavailable.")
|
| 477 |
-
return float(row[0])
|
| 478 |
-
|
| 479 |
-
value = total(bounds["current"])
|
| 480 |
-
if key == "sales_yoy":
|
| 481 |
-
prior = total(bounds["comparison"])
|
| 482 |
-
if prior <= 0:
|
| 483 |
-
raise sku_action_results.SourceUnavailable("Sales YoY requires positive prior sales.")
|
| 484 |
-
value = 100 * (value - prior) / prior
|
| 485 |
-
return {"value": value, "reason": None}
|
| 486 |
-
except sku_action_results.SourceUnavailable as exc:
|
| 487 |
-
return {"value": None, "reason": str(exc)}
|
| 488 |
-
|
| 489 |
-
|
| 490 |
@router.get("")
|
| 491 |
def row_projects(grain: str | None = None, key: str | None = None,
|
| 492 |
session: Session = Depends(require_session)):
|
|
@@ -589,61 +508,6 @@ def project_result_figures(project_id: str, session: Session = Depends(require_s
|
|
| 589 |
con.close()
|
| 590 |
|
| 591 |
|
| 592 |
-
@router.get("/{project_id}/series")
|
| 593 |
-
def project_result_series(project_id: str, start: str | None = Query(None, alias="from"),
|
| 594 |
-
end: str | None = Query(None, alias="to"), scope: str = Query("all"),
|
| 595 |
-
session: Session = Depends(require_session)):
|
| 596 |
-
"""Serve the preset daily charts within the intersection of project targets and the modal filter."""
|
| 597 |
-
runtime = _reader(session)
|
| 598 |
-
record = _call(projects.get, runtime, project_id)
|
| 599 |
-
targets = _call(projects.targets, runtime, project_id)
|
| 600 |
-
con = None
|
| 601 |
-
try:
|
| 602 |
-
bounds = project_results.series_window(record, start, end, today=projects._today())
|
| 603 |
-
selected = project_results.series_scope(scope)
|
| 604 |
-
selected_targets = ([{"grain": "parent_asin" if selected["grain"] == "parent" else selected["grain"],
|
| 605 |
-
"key": key} for key in selected["keys"]] if selected else [])
|
| 606 |
-
available = _result_sources(session, targets + selected_targets)
|
| 607 |
-
con = routes_sku_actions._results_mirror(runtime)
|
| 608 |
-
reason = None
|
| 609 |
-
try:
|
| 610 |
-
skus = projects.resolve_target_skus(con, targets)
|
| 611 |
-
if selected:
|
| 612 |
-
selected_skus = set(projects.resolve_target_skus(con, selected_targets))
|
| 613 |
-
skus = [sku for sku in skus if sku in selected_skus]
|
| 614 |
-
if not skus:
|
| 615 |
-
reason = "No project targets match the selected scope."
|
| 616 |
-
elif not bounds["days"]:
|
| 617 |
-
reason = "Today is incomplete. No completed days are available in this range."
|
| 618 |
-
except projects.MirrorUnreadable as exc:
|
| 619 |
-
skus, reason = [], str(exc)
|
| 620 |
-
figures = {}
|
| 621 |
-
|
| 622 |
-
def read(key):
|
| 623 |
-
if reason:
|
| 624 |
-
return {"reason": reason}
|
| 625 |
-
metric = "sales" if key == "sales_yoy" else key
|
| 626 |
-
if metric not in _SERIES_UNAVAILABLE and metric not in figures:
|
| 627 |
-
figures[metric] = _result_figure(con, {"metric": metric}, bounds, skus, available)
|
| 628 |
-
return _series_figure(key, figures.get(metric, {}), bounds)
|
| 629 |
-
|
| 630 |
-
return project_results.series(record, bounds=bounds, read=read,
|
| 631 |
-
independent=lambda key: _series_economics_total(con, key, bounds, skus, available))
|
| 632 |
-
except HTTPException:
|
| 633 |
-
raise
|
| 634 |
-
except (project_results.ValidationFailed, sku_action_results.ValidationFailed):
|
| 635 |
-
raise err(409, "project_results_not_validated",
|
| 636 |
-
"A series total does not tie to its independent Amazon economics aggregate.") from None
|
| 637 |
-
except ValueError as exc:
|
| 638 |
-
raise err(400, "invalid_project_results", str(exc)) from None
|
| 639 |
-
except Exception as exc: # noqa: BLE001
|
| 640 |
-
print(f"[amazon-projects] series failed: {type(exc).__name__}: {exc}", file=sys.stderr)
|
| 641 |
-
raise err(503, "project_results_unavailable", "Project result sources are unavailable.") from None
|
| 642 |
-
finally:
|
| 643 |
-
if con is not None:
|
| 644 |
-
con.close()
|
| 645 |
-
|
| 646 |
-
|
| 647 |
@router.patch("/{project_id}")
|
| 648 |
def edit_project(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 649 |
"""Change the named fields. Targets named here are added."""
|
|
@@ -658,11 +522,8 @@ def edit_project(project_id: str, body: Any = Body(default=None), session: Sessi
|
|
| 658 |
record = _call(projects.update, runtime, project_id, values, by=_by(session))
|
| 659 |
if targets:
|
| 660 |
record = _call(projects.add_targets, runtime, project_id, targets, by=_by(session))["project"]
|
| 661 |
-
#
|
| 662 |
-
if {"status", "progress"} & set(values)
|
| 663 |
-
_repaint(runtime, _target_pairs(runtime, project_id))
|
| 664 |
-
elif targets:
|
| 665 |
-
_repaint(runtime, targets)
|
| 666 |
return {"project": _project(record, projects._today())}
|
| 667 |
|
| 668 |
|
|
|
|
| 12 |
import sys
|
| 13 |
from typing import Any
|
| 14 |
|
| 15 |
+
from fastapi import APIRouter, Body, Depends, HTTPException
|
| 16 |
|
| 17 |
from core import amazon_projects as projects
|
| 18 |
from core import amazon_project_results as project_results
|
|
|
|
| 219 |
|
| 220 |
|
| 221 |
def cell_values(records):
|
| 222 |
+
"""R19 and R67: the average progress as a stored pct and "N of M completed", blank with no counted project."""
|
| 223 |
counted = projects.summary(records)
|
| 224 |
if not counted["total"]:
|
| 225 |
return {PROGRESS_KEY: None, LABEL_KEY: None}
|
|
|
|
| 343 |
return available
|
| 344 |
|
| 345 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 346 |
def _result_figure(con, config, bounds, skus, available):
|
| 347 |
"""Reuse daily metric definitions and reconcile their union against separate source totals."""
|
| 348 |
if not skus:
|
| 349 |
return {"state": "unavailable", "reason": "Assign an Amazon target before measuring results."}
|
| 350 |
|
| 351 |
+
class ProjectMirror(sku_action_results.Mirror):
|
| 352 |
+
def entity(self):
|
| 353 |
+
self.field(sku_action_results.ECONOMICS, "msku")
|
| 354 |
+
return '"msku" IN (' + ", ".join("?" for _ in skus) + ")", tuple(skus)
|
| 355 |
+
|
| 356 |
+
def raw(self, table, days, values, **kwargs):
|
| 357 |
+
plan = super().raw(table, days, values, **kwargs)
|
| 358 |
+
sku = self.field(table, *kwargs["sku_fields"])
|
| 359 |
+
head, tail = plan.sql.split(" FROM ", 1)
|
| 360 |
+
coverage = sku_action_results.Plan(
|
| 361 |
+
head + ", " + sku_action_results._q(sku) + ' AS "__project_member" FROM ' + tail,
|
| 362 |
+
plan.args, plan.sources)
|
| 363 |
+
self.coverage.append((table, tuple(days), coverage))
|
| 364 |
+
return plan
|
| 365 |
+
|
| 366 |
+
mirror = ProjectMirror(con, {}, available)
|
| 367 |
+
mirror.coverage = []
|
| 368 |
try:
|
| 369 |
mirror.select_scope(bounds)
|
| 370 |
except sku_action_results.SourceUnavailable as exc:
|
|
|
|
| 406 |
return True
|
| 407 |
|
| 408 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
@router.get("")
|
| 410 |
def row_projects(grain: str | None = None, key: str | None = None,
|
| 411 |
session: Session = Depends(require_session)):
|
|
|
|
| 508 |
con.close()
|
| 509 |
|
| 510 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 511 |
@router.patch("/{project_id}")
|
| 512 |
def edit_project(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 513 |
"""Change the named fields. Targets named here are added."""
|
|
|
|
| 522 |
record = _call(projects.update, runtime, project_id, values, by=_by(session))
|
| 523 |
if targets:
|
| 524 |
record = _call(projects.add_targets, runtime, project_id, targets, by=_by(session))["project"]
|
| 525 |
+
# Only status and progress move a cell: a name or note edit repaints only the targets it adds.
|
| 526 |
+
_repaint(runtime, _target_pairs(runtime, project_id) if {"status", "progress"} & set(values) else targets)
|
|
|
|
|
|
|
|
|
|
| 527 |
return {"project": _project(record, projects._today())}
|
| 528 |
|
| 529 |
|
api/routes_assets.py
CHANGED
|
@@ -145,18 +145,6 @@ def code_access(session, application_id):
|
|
| 145 |
return True
|
| 146 |
|
| 147 |
|
| 148 |
-
def alerts_access(session, application_id):
|
| 149 |
-
"""Admit a source-free Alerts App through its registry and Automation permission walls."""
|
| 150 |
-
app = applications.load(session.runtime)["applications"].get(application_id)
|
| 151 |
-
if not isinstance(app, dict) or app.get("templateKind") != "alerts" or not applications._database_free(app):
|
| 152 |
-
return False
|
| 153 |
-
if (app.get("tenant") != session.tenant or session.tenant != session.runtime.key
|
| 154 |
-
or app.get("retired") is True or not applications.may_read(app, session.uname, session.admin)
|
| 155 |
-
or not _session_wall(session)("automation")):
|
| 156 |
-
raise applications.ApplicationError("application does not exist")
|
| 157 |
-
return True
|
| 158 |
-
|
| 159 |
-
|
| 160 |
# Tokens and the embedded Inter font come from the same stylesheet as the shell.
|
| 161 |
_CODE_KIT_COMPONENTS = """
|
| 162 |
:root {
|
|
@@ -440,8 +428,7 @@ def delete_application(application_id: str, body: dict = Body(default=None),
|
|
| 440 |
session: Session = Depends(require_session)):
|
| 441 |
try:
|
| 442 |
# A database-free App still needs its own read and edit permissions.
|
| 443 |
-
if
|
| 444 |
-
or alerts_access(session, application_id)):
|
| 445 |
allowed = set()
|
| 446 |
else:
|
| 447 |
current = _application_databases(session.runtime, application_id)
|
|
|
|
| 145 |
return True
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
# Tokens and the embedded Inter font come from the same stylesheet as the shell.
|
| 149 |
_CODE_KIT_COMPONENTS = """
|
| 150 |
:root {
|
|
|
|
| 428 |
session: Session = Depends(require_session)):
|
| 429 |
try:
|
| 430 |
# A database-free App still needs its own read and edit permissions.
|
| 431 |
+
if finance_access(session, application_id) or code_access(session, application_id):
|
|
|
|
| 432 |
allowed = set()
|
| 433 |
else:
|
| 434 |
current = _application_databases(session.runtime, application_id)
|
api/routes_connectors.py
CHANGED
|
@@ -65,7 +65,7 @@ from core import schema_overlay
|
|
| 65 |
#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
|
| 66 |
#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
|
| 67 |
#: second correct one here would leave the same two-copies shape that produced the defect.
|
| 68 |
-
from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available
|
| 69 |
#: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a
|
| 70 |
#: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any
|
| 71 |
#: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router`
|
|
@@ -1095,25 +1095,15 @@ def _contact_rows(rt, uname, is_admin, incomplete):
|
|
| 1095 |
def read(source, keys, fn):
|
| 1096 |
try:
|
| 1097 |
fn()
|
| 1098 |
-
except Exception
|
| 1099 |
-
logging.getLogger(__name__).warning("contact_check=%s exception=%s",
|
| 1100 |
-
source, type(exc).__name__)
|
| 1101 |
for key in keys:
|
| 1102 |
by_key[key]["state"] = "unavailable"
|
| 1103 |
-
|
| 1104 |
-
"loopable-api": "Loopable API"}[source]
|
| 1105 |
-
incomplete.append({"source": source, "cause": f"{label} connection status unavailable.",
|
| 1106 |
"recommendation": "Try again."})
|
| 1107 |
|
| 1108 |
def whatsapp():
|
| 1109 |
-
|
| 1110 |
-
|
| 1111 |
-
# Directory status reads the reader's canonical account, never registration challenges.
|
| 1112 |
-
record = account_rows.get(uname, st=users.store, fresh=True)
|
| 1113 |
-
if not isinstance(record, dict) or str(record.get("tenant") or "royal-imports") != rt.key:
|
| 1114 |
-
raise WhatsAppRefusal("This account is not available in this workspace.")
|
| 1115 |
-
connection = record.get("whatsapp")
|
| 1116 |
-
if isinstance(connection, dict) and connection.get("phone"):
|
| 1117 |
by_key["whatsapp"]["state"] = "connected"
|
| 1118 |
|
| 1119 |
def slack():
|
|
@@ -3066,88 +3056,6 @@ def _amazon_mirror_completion(rt):
|
|
| 3066 |
return None, False
|
| 3067 |
|
| 3068 |
|
| 3069 |
-
def _amazon_table_schedule(rt, connection_id=None, now=None):
|
| 3070 |
-
"""Read account cadence metadata without restoring a mirror or reading source rows."""
|
| 3071 |
-
import duckdb
|
| 3072 |
-
from core import keychain
|
| 3073 |
-
from harness import amazon_store as az, datastore
|
| 3074 |
-
|
| 3075 |
-
now = now or datetime.now(timezone.utc)
|
| 3076 |
-
result = {"tableSchedule": [], "scheduleState": "unavailable",
|
| 3077 |
-
"displayTimezone": stored_business_tz(rt) or "UTC"}
|
| 3078 |
-
|
| 3079 |
-
def read(cursor, entries):
|
| 3080 |
-
schedule = []
|
| 3081 |
-
for entry in entries:
|
| 3082 |
-
cid = entry["connection_id"]
|
| 3083 |
-
prefix = az._account_prefix(cid)
|
| 3084 |
-
keys = [prefix + az.entity_key(family + table)
|
| 3085 |
-
for table in az.TABLES for family in ("", "sync_tick_", az.WATERMARK_PREFIX)]
|
| 3086 |
-
rows = cursor.execute(
|
| 3087 |
-
"SELECT entity, phase, rows, updated_at, "
|
| 3088 |
-
"CASE WHEN phase = 'throttled' THEN cursor_wd ELSE NULL END FROM _sync_state "
|
| 3089 |
-
"WHERE entity IN (SELECT unnest(?))", [keys]).fetchall()
|
| 3090 |
-
markers = {entity: {"state": phase, "rows": count, "updated_at": _sync_stamp(at),
|
| 3091 |
-
"detail": _sync_stamp(detail)}
|
| 3092 |
-
for entity, phase, count, at, detail in rows}
|
| 3093 |
-
states = {table: state for table in az.TABLES
|
| 3094 |
-
if (state := markers.get(prefix + az.entity_key("sync_tick_" + table)))}
|
| 3095 |
-
if any(not state["updated_at"] for state in states.values()):
|
| 3096 |
-
raise ValueError("Amazon cadence metadata has an invalid timestamp")
|
| 3097 |
-
# C1 owns every due time, including throttle retries and calendar periods.
|
| 3098 |
-
for row in az.sync_schedule(states, now):
|
| 3099 |
-
table = row["table"]
|
| 3100 |
-
tick = states.get(table) or {}
|
| 3101 |
-
watermark = markers.get(prefix + az.entity_key(az.WATERMARK_PREFIX + table)) or {}
|
| 3102 |
-
landed = markers.get(prefix + az.entity_key(table)) or {}
|
| 3103 |
-
completed = (tick if tick.get("state") == "done"
|
| 3104 |
-
and landed.get("state") not in ("refused", "failed", "throttled") else watermark)
|
| 3105 |
-
schedule.append({**row, "connectionId": cid, "account": entry["label"],
|
| 3106 |
-
"lastSyncAt": (completed.get("updated_at")
|
| 3107 |
-
if completed.get("state") == "done" else None),
|
| 3108 |
-
"state": (landed.get("state")
|
| 3109 |
-
if landed.get("state") in ("refused", "failed", "throttled")
|
| 3110 |
-
else tick.get("state") or "never_run")})
|
| 3111 |
-
return schedule
|
| 3112 |
-
|
| 3113 |
-
try:
|
| 3114 |
-
path = az._tenant_mirror_path(rt.key, rt=rt)
|
| 3115 |
-
if not keychain.list_entries(rt, "amazon_sp"):
|
| 3116 |
-
return {**result, "scheduleState": "empty"}
|
| 3117 |
-
entries = az.account_entries(rt.key, rt)
|
| 3118 |
-
cid = str(connection_id or "").strip()
|
| 3119 |
-
if cid and cid != "default":
|
| 3120 |
-
entries = [entry for entry in entries if entry["connection_id"] == cid]
|
| 3121 |
-
if not entries:
|
| 3122 |
-
return result
|
| 3123 |
-
if not path.is_file():
|
| 3124 |
-
return result
|
| 3125 |
-
if path.resolve() == Path(datastore.DB_PATH).resolve():
|
| 3126 |
-
schedule = read(datastore.ro_cursor(), entries)
|
| 3127 |
-
else:
|
| 3128 |
-
with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor:
|
| 3129 |
-
schedule = read(cursor, entries)
|
| 3130 |
-
return {**result, "tableSchedule": schedule, "scheduleState": "ready"}
|
| 3131 |
-
except Exception as exc:
|
| 3132 |
-
raise_if_resource_error(exc)
|
| 3133 |
-
return result
|
| 3134 |
-
|
| 3135 |
-
|
| 3136 |
-
def _amazon_sync_staleness(completed, schedule, now=None):
|
| 3137 |
-
"""Use the fastest Amazon table's cadence and Odoo's existing grace period."""
|
| 3138 |
-
import odoo_relational as rel
|
| 3139 |
-
fastest = max(amazon_client.CADENCE[table]["perDay"]
|
| 3140 |
-
for table in amazon_client.GROUP_A)
|
| 3141 |
-
allowed = 86400 / fastest + rel.STALE_GRACE_SECONDS
|
| 3142 |
-
stamps = [row.get("lastSyncAt") for row in schedule if row["perDay"] == fastest]
|
| 3143 |
-
if not stamps:
|
| 3144 |
-
stamps = [completed]
|
| 3145 |
-
now = now or datetime.now(timezone.utc)
|
| 3146 |
-
stale = any((now - datetime.fromisoformat(stamp)).total_seconds() > allowed
|
| 3147 |
-
for value in stamps if (stamp := _sync_stamp(value)))
|
| 3148 |
-
return {"stale": stale, "staleAfterSeconds": allowed}
|
| 3149 |
-
|
| 3150 |
-
|
| 3151 |
def _daily_completed_sync(rt, connector_key):
|
| 3152 |
"""Read the stored last good sync; never create a mirror or contact a provider."""
|
| 3153 |
if connector_key == "amazon_sp":
|
|
@@ -3246,11 +3154,8 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
|
|
| 3246 |
if connector_key not in registered:
|
| 3247 |
raise err(404, "sync_provider_unknown",
|
| 3248 |
f"{connector_key} is not registered for connector sync")
|
| 3249 |
-
from core import keychain
|
| 3250 |
-
import odoo_relational as rel
|
| 3251 |
rt = (_odoo_status_reads(session.runtime) if connector_key == "odoo"
|
| 3252 |
-
else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY,
|
| 3253 |
-
connector_sync.STATUS_KEY, keychain.KEY, rel.CONFIG_KEY)
|
| 3254 |
if connector_key == "amazon_sp" else ()))
|
| 3255 |
resolved_connection = (_odoo_connection_slot(rt, connection_id)
|
| 3256 |
if connector_key == "odoo" else connection_id)
|
|
@@ -3274,15 +3179,9 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
|
|
| 3274 |
if connector_key == "amazon_sp" and completed is None and known and _amazon_restore_pending(rt):
|
| 3275 |
completed = _amazon_saved_completion(rt)
|
| 3276 |
answer["restoring"] = True
|
| 3277 |
-
answer.update(lastCompletedAt=completed, lastSyncAt=completed,
|
| 3278 |
-
lastCompletedKnown=known, canSync=bool(session.admin)
|
| 3279 |
-
|
| 3280 |
-
now = datetime.now(timezone.utc)
|
| 3281 |
-
schedule = _amazon_table_schedule(rt, resolved_connection, now)
|
| 3282 |
-
answer.update(schedule="per_table", **schedule,
|
| 3283 |
-
**_amazon_sync_staleness(completed, schedule["tableSchedule"], now))
|
| 3284 |
-
else:
|
| 3285 |
-
answer.update(schedule="daily", **_daily_sync_staleness(completed))
|
| 3286 |
return _safe_connector_status(connector_key, answer)
|
| 3287 |
|
| 3288 |
|
|
|
|
| 65 |
#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
|
| 66 |
#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
|
| 67 |
#: second correct one here would leave the same two-copies shape that produced the defect.
|
| 68 |
+
from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available
|
| 69 |
#: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a
|
| 70 |
#: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any
|
| 71 |
#: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router`
|
|
|
|
| 1095 |
def read(source, keys, fn):
|
| 1096 |
try:
|
| 1097 |
fn()
|
| 1098 |
+
except Exception: # noqa: BLE001
|
|
|
|
|
|
|
| 1099 |
for key in keys:
|
| 1100 |
by_key[key]["state"] = "unavailable"
|
| 1101 |
+
incomplete.append({"source": source, "cause": "Connection status unavailable.",
|
|
|
|
|
|
|
| 1102 |
"recommendation": "Try again."})
|
| 1103 |
|
| 1104 |
def whatsapp():
|
| 1105 |
+
import whatsapp_identity
|
| 1106 |
+
if whatsapp_identity.status(rt.key, uname).get("state") == "connected":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1107 |
by_key["whatsapp"]["state"] = "connected"
|
| 1108 |
|
| 1109 |
def slack():
|
|
|
|
| 3056 |
return None, False
|
| 3057 |
|
| 3058 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3059 |
def _daily_completed_sync(rt, connector_key):
|
| 3060 |
"""Read the stored last good sync; never create a mirror or contact a provider."""
|
| 3061 |
if connector_key == "amazon_sp":
|
|
|
|
| 3154 |
if connector_key not in registered:
|
| 3155 |
raise err(404, "sync_provider_unknown",
|
| 3156 |
f"{connector_key} is not registered for connector sync")
|
|
|
|
|
|
|
| 3157 |
rt = (_odoo_status_reads(session.runtime) if connector_key == "odoo"
|
| 3158 |
+
else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY, connector_sync.STATUS_KEY)
|
|
|
|
| 3159 |
if connector_key == "amazon_sp" else ()))
|
| 3160 |
resolved_connection = (_odoo_connection_slot(rt, connection_id)
|
| 3161 |
if connector_key == "odoo" else connection_id)
|
|
|
|
| 3179 |
if connector_key == "amazon_sp" and completed is None and known and _amazon_restore_pending(rt):
|
| 3180 |
completed = _amazon_saved_completion(rt)
|
| 3181 |
answer["restoring"] = True
|
| 3182 |
+
answer.update(schedule="daily", lastCompletedAt=completed, lastSyncAt=completed,
|
| 3183 |
+
lastCompletedKnown=known, canSync=bool(session.admin),
|
| 3184 |
+
**_daily_sync_staleness(completed))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3185 |
return _safe_connector_status(connector_key, answer)
|
| 3186 |
|
| 3187 |
|
api/routes_external.py
CHANGED
|
@@ -153,20 +153,6 @@ def _activity_store_view(real, insert):
|
|
| 153 |
return view
|
| 154 |
|
| 155 |
|
| 156 |
-
def _sheet_row_revision(tx, key):
|
| 157 |
-
"""Read the header's store revision on the same transaction as its sheet rows."""
|
| 158 |
-
from core import store_pg, view_audit
|
| 159 |
-
if isinstance(tx, sheets._PgSheetRows):
|
| 160 |
-
row = tx.con.execute(store_pg._q(
|
| 161 |
-
"SELECT rev, updated_at FROM {tbl} WHERE key = %s", "store_kv", tx.slug),
|
| 162 |
-
(tx.namespace + key,)).fetchone()
|
| 163 |
-
return store_pg._revision_value(row[0], row[1]) if row else store_pg._revision_value(None, None)
|
| 164 |
-
reader = getattr(type(tx), "revision", None)
|
| 165 |
-
if callable(reader):
|
| 166 |
-
return reader(tx, key)
|
| 167 |
-
raise view_audit.RevisionUnverifiable("The Sheet transaction cannot verify its header revision.")
|
| 168 |
-
|
| 169 |
-
|
| 170 |
class _PreviewRuntime:
|
| 171 |
"""Lazy copy-on-write view of one tenant runtime; no method may reach a real writer.
|
| 172 |
|
|
@@ -194,20 +180,11 @@ class _PreviewRuntime:
|
|
| 194 |
if key not in self._docs:
|
| 195 |
import application_registry
|
| 196 |
value = application_registry._read_strict(self._real, key)
|
| 197 |
-
revision = None
|
| 198 |
-
if key.startswith(sheets.WORKBOOK_KEY_PREFIX) and sheets.is_row_workbook(value):
|
| 199 |
-
# Hydrate through the reader, never delegate a native writer to this preview.
|
| 200 |
-
with sheets._sheet_rows_transaction(self._real) as tx:
|
| 201 |
-
if tx is None:
|
| 202 |
-
raise err(503, "workbook_unavailable", "The workbook snapshot is unavailable. Try again.")
|
| 203 |
-
head = tx.get(key)
|
| 204 |
-
value = sheets._verify_row_layout(tx, head) if sheets.is_row_workbook(head) else head
|
| 205 |
-
revision = _sheet_row_revision(tx, key)
|
| 206 |
value = copy.deepcopy(value)
|
| 207 |
self._initial[key] = copy.deepcopy(value)
|
| 208 |
self._docs[key] = value
|
| 209 |
try:
|
| 210 |
-
revision =
|
| 211 |
except Exception:
|
| 212 |
revision = None
|
| 213 |
self._base_revisions[key] = copy.deepcopy(revision)
|
|
@@ -351,70 +328,6 @@ class _ConfirmationRuntime:
|
|
| 351 |
def tenant(self):
|
| 352 |
return getattr(self._real, "tenant", None)
|
| 353 |
|
| 354 |
-
@contextmanager
|
| 355 |
-
def sheet_rows_transaction(self, *, write=False):
|
| 356 |
-
"""Guard native Sheet writes under the header lock with the preview's store revision."""
|
| 357 |
-
from core import view_audit
|
| 358 |
-
with sheets._sheet_rows_transaction(self._real, write=write) as tx:
|
| 359 |
-
if tx is None:
|
| 360 |
-
yield tx
|
| 361 |
-
return
|
| 362 |
-
owner = self
|
| 363 |
-
held = {}
|
| 364 |
-
|
| 365 |
-
class GuardedRows:
|
| 366 |
-
def get(self, key, *, lock=False, **selection):
|
| 367 |
-
if lock and not write:
|
| 368 |
-
raise view_audit.RevisionUnverifiable("A Sheet read cannot acquire write authority.")
|
| 369 |
-
value = tx.get(key, lock=lock, **selection)
|
| 370 |
-
if lock:
|
| 371 |
-
expected = owner._expected.get(key)
|
| 372 |
-
rev = expected.get("rev") if isinstance(expected, dict) else None
|
| 373 |
-
if (not key.startswith(sheets.WORKBOOK_KEY_PREFIX) or type(rev) is not int
|
| 374 |
-
or held):
|
| 375 |
-
raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed header.")
|
| 376 |
-
current = _sheet_row_revision(tx, key)
|
| 377 |
-
if current.get("rev") != rev or ("token" in expected and current.get("token") != expected["token"]):
|
| 378 |
-
raise view_audit.ViewRevisionStale(rev_current=current.get("rev"), rev_sent=rev)
|
| 379 |
-
before = (sheets._verify_row_layout(tx, value) if sheets.is_row_workbook(value)
|
| 380 |
-
else tx.get(key))
|
| 381 |
-
held.update(key=key, before=copy.deepcopy(before), head=value)
|
| 382 |
-
return value
|
| 383 |
-
|
| 384 |
-
def revision(self, key):
|
| 385 |
-
return _sheet_row_revision(tx, key)
|
| 386 |
-
|
| 387 |
-
def put(self, key, value):
|
| 388 |
-
header = held.get("key")
|
| 389 |
-
if not header or (key != header and key != sheets.sheet_row_key(
|
| 390 |
-
header[len(sheets.WORKBOOK_KEY_PREFIX):], value.get("id", ""))):
|
| 391 |
-
raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed workbook.")
|
| 392 |
-
if key == header and value and value.get("id") != header[len(sheets.WORKBOOK_KEY_PREFIX):]:
|
| 393 |
-
raise view_audit.RevisionUnverifiable("The Sheet header identity changed.")
|
| 394 |
-
tx.put(key, value)
|
| 395 |
-
|
| 396 |
-
def delete(self, key):
|
| 397 |
-
head = held.get("head") or {}
|
| 398 |
-
if key not in {sheets.sheet_row_key(head["id"], ref["id"])
|
| 399 |
-
for ref in head.get("rowSheets", [])}:
|
| 400 |
-
raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed sheet.")
|
| 401 |
-
tx.delete(key)
|
| 402 |
-
|
| 403 |
-
yield GuardedRows()
|
| 404 |
-
if held:
|
| 405 |
-
key = held["key"]
|
| 406 |
-
head = tx.get(key)
|
| 407 |
-
after = sheets._verify_row_layout(tx, head) if sheets.is_row_workbook(head) else head
|
| 408 |
-
produced = _sheet_row_revision(tx, key)
|
| 409 |
-
if held:
|
| 410 |
-
self._expected[key] = copy.deepcopy(produced)
|
| 411 |
-
if held["before"] != after:
|
| 412 |
-
previous = self._committed_documents.get(key)
|
| 413 |
-
self._committed_documents[key] = {
|
| 414 |
-
"before": previous["before"] if previous else held["before"],
|
| 415 |
-
"after": copy.deepcopy(after),
|
| 416 |
-
}
|
| 417 |
-
|
| 418 |
def update(self, name, callback, flush="sync", expect_rev=None, audit_row=None):
|
| 419 |
from core import view_audit
|
| 420 |
key = str(name)
|
|
@@ -1990,22 +1903,6 @@ def list_keys(session: Session = Depends(require_session)):
|
|
| 1990 |
return {"keys": keys, "can_mint": _can_use_analyst(session)}
|
| 1991 |
|
| 1992 |
|
| 1993 |
-
@router.post("/keys/verify")
|
| 1994 |
-
def verify_owned_key(response: Response, body=Body(default=None), session: Session = Depends(require_session)):
|
| 1995 |
-
"""Return a private diagnosis for this session owner's key without accepting bearer sign-in."""
|
| 1996 |
-
require_external_analyst(session)
|
| 1997 |
-
token = body.get("token") if isinstance(body, dict) else None
|
| 1998 |
-
if not isinstance(token, str) or len(token) > 1024:
|
| 1999 |
-
token = ""
|
| 2000 |
-
try:
|
| 2001 |
-
result = external_access.verify_for_owner(
|
| 2002 |
-
session.runtime, session.uname, session.tenant, token)
|
| 2003 |
-
except external_access.ExternalAccessError as exc:
|
| 2004 |
-
_management_refusal(exc)
|
| 2005 |
-
response.headers["Cache-Control"] = "no-store"
|
| 2006 |
-
return result
|
| 2007 |
-
|
| 2008 |
-
|
| 2009 |
@router.patch("/keys/{key_id}")
|
| 2010 |
def edit_key(key_id: str, body=Body(default=None), session: Session = Depends(require_session)):
|
| 2011 |
"""Edit metadata and narrowing scope without ever reading or replacing the secret."""
|
|
@@ -3327,8 +3224,11 @@ def _validate_personal_view_draft(session: Session, database: str, raw: dict) ->
|
|
| 3327 |
if any(key not in visible for key in _view_referenced_fields(config)):
|
| 3328 |
raise err(400, "assistant_unknown_field",
|
| 3329 |
"Every requested field must be readable in this database.")
|
| 3330 |
-
#
|
| 3331 |
-
|
|
|
|
|
|
|
|
|
|
| 3332 |
|
| 3333 |
|
| 3334 |
def _saved_view_filters(config):
|
|
@@ -3337,9 +3237,9 @@ def _saved_view_filters(config):
|
|
| 3337 |
_assert_query_filter_bounds([value] if isinstance(value, dict) else value)
|
| 3338 |
import aios_grid
|
| 3339 |
|
| 3340 |
-
def refuse(
|
| 3341 |
raise err(400, "view_filters_unsupported",
|
| 3342 |
-
|
| 3343 |
"Use colId, op and value leaves, with conj and children for nested groups.")
|
| 3344 |
|
| 3345 |
def node(raw):
|
|
@@ -3351,17 +3251,8 @@ def _saved_view_filters(config):
|
|
| 3351 |
or not isinstance(raw[key], list) or not raw[key]):
|
| 3352 |
refuse()
|
| 3353 |
return {"conj": raw.get("conj", "and"), "children": [node(child) for child in raw[key]]}
|
| 3354 |
-
if
|
| 3355 |
-
|
| 3356 |
-
"cannot be verified here. Use literal values or a readable rhs field instead.")
|
| 3357 |
-
op = raw.get("op")
|
| 3358 |
-
if not isinstance(op, str) or op not in aios_grid.FILTER_OPS:
|
| 3359 |
-
name = json.dumps(op[:120]) if isinstance(op, str) else "(missing or invalid)"
|
| 3360 |
-
refuse(f"Unsupported filter operator {name}. Use a supported operator: "
|
| 3361 |
-
+ ", ".join(sorted(aios_grid.FILTER_OPS)) + ".")
|
| 3362 |
-
if (set(raw) - {"colId", "op", "value", "value2", "values", "id", "locked",
|
| 3363 |
-
"dateWindow", "dateMode", "rhs"}
|
| 3364 |
-
or not isinstance(raw.get("colId"), str)):
|
| 3365 |
refuse()
|
| 3366 |
for key in ("value", "value2"):
|
| 3367 |
value = raw.get(key)
|
|
|
|
| 153 |
return view
|
| 154 |
|
| 155 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
class _PreviewRuntime:
|
| 157 |
"""Lazy copy-on-write view of one tenant runtime; no method may reach a real writer.
|
| 158 |
|
|
|
|
| 180 |
if key not in self._docs:
|
| 181 |
import application_registry
|
| 182 |
value = application_registry._read_strict(self._real, key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
value = copy.deepcopy(value)
|
| 184 |
self._initial[key] = copy.deepcopy(value)
|
| 185 |
self._docs[key] = value
|
| 186 |
try:
|
| 187 |
+
revision = self._real.revision(key)
|
| 188 |
except Exception:
|
| 189 |
revision = None
|
| 190 |
self._base_revisions[key] = copy.deepcopy(revision)
|
|
|
|
| 328 |
def tenant(self):
|
| 329 |
return getattr(self._real, "tenant", None)
|
| 330 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
def update(self, name, callback, flush="sync", expect_rev=None, audit_row=None):
|
| 332 |
from core import view_audit
|
| 333 |
key = str(name)
|
|
|
|
| 1903 |
return {"keys": keys, "can_mint": _can_use_analyst(session)}
|
| 1904 |
|
| 1905 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1906 |
@router.patch("/keys/{key_id}")
|
| 1907 |
def edit_key(key_id: str, body=Body(default=None), session: Session = Depends(require_session)):
|
| 1908 |
"""Edit metadata and narrowing scope without ever reading or replacing the secret."""
|
|
|
|
| 3224 |
if any(key not in visible for key in _view_referenced_fields(config)):
|
| 3225 |
raise err(400, "assistant_unknown_field",
|
| 3226 |
"Every requested field must be readable in this database.")
|
| 3227 |
+
# The ordinary Assistant validator owns filter syntax and operands. fields=[] keeps values
|
| 3228 |
+
# out of this validation result while still applying the live database/field/row walls.
|
| 3229 |
+
filters = config.get("filters")
|
| 3230 |
+
_external_read_scope(
|
| 3231 |
+
session, database, fields=[], filters=None if filters == [] else filters)
|
| 3232 |
|
| 3233 |
|
| 3234 |
def _saved_view_filters(config):
|
|
|
|
| 3237 |
_assert_query_filter_bounds([value] if isinstance(value, dict) else value)
|
| 3238 |
import aios_grid
|
| 3239 |
|
| 3240 |
+
def refuse():
|
| 3241 |
raise err(400, "view_filters_unsupported",
|
| 3242 |
+
"These filters cannot be saved without changing their meaning. "
|
| 3243 |
"Use colId, op and value leaves, with conj and children for nested groups.")
|
| 3244 |
|
| 3245 |
def node(raw):
|
|
|
|
| 3251 |
or not isinstance(raw[key], list) or not raw[key]):
|
| 3252 |
refuse()
|
| 3253 |
return {"conj": raw.get("conj", "and"), "children": [node(child) for child in raw[key]]}
|
| 3254 |
+
if (set(raw) - {"colId", "op", "value", "value2", "values", "id", "locked"}
|
| 3255 |
+
or not isinstance(raw.get("colId"), str) or raw.get("op") not in aios_grid.FILTER_OPS):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3256 |
refuse()
|
| 3257 |
for key in ("value", "value2"):
|
| 3258 |
value = raw.get(key)
|
api/routes_odoo_tables.py
CHANGED
|
@@ -1393,22 +1393,6 @@ def require_amazon_mirror(table_key, session, definition=None, st=None):
|
|
| 1393 |
cur.close()
|
| 1394 |
|
| 1395 |
|
| 1396 |
-
def require_dynamic_odoo_mirror(table_key, session, st=None):
|
| 1397 |
-
"""Admit a saved dynamic definition before resolving its publication or refusal."""
|
| 1398 |
-
if not str(table_key).startswith('ut_odoo_model_'):
|
| 1399 |
-
return None
|
| 1400 |
-
from routes_tables import _defn_or_refuse
|
| 1401 |
-
try:
|
| 1402 |
-
_defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 1403 |
-
except HTTPException as exc:
|
| 1404 |
-
if exc.status_code == 404:
|
| 1405 |
-
raise err(404, 'odoo_definition_not_published',
|
| 1406 |
-
'This Odoo database has no published definition. Enable it in connection '
|
| 1407 |
-
'settings and complete its first sync before reading rows.') from exc
|
| 1408 |
-
raise
|
| 1409 |
-
return not _materialised_table(table_key, st=st if st is not None else session.runtime)
|
| 1410 |
-
|
| 1411 |
-
|
| 1412 |
register_source_provider(_odoo_sources)
|
| 1413 |
register_source_provider(_amazon_sources)
|
| 1414 |
register_source_provider(_meta_sources)
|
|
@@ -1625,25 +1609,19 @@ def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_looku
|
|
| 1625 |
"""Resolve one admitted Amazon target and keep query failures distinct from stored tables."""
|
| 1626 |
import amazon_relational as amazon
|
| 1627 |
|
| 1628 |
-
if (definition or {}).get('oversize'):
|
| 1629 |
-
definition = {**definition, 'readThrough': True}
|
| 1630 |
try:
|
| 1631 |
if binding:
|
| 1632 |
columns = amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)
|
| 1633 |
spec = amazon.grid_source(cur, key, columns=columns, allow_lookups=allow_lookups)
|
| 1634 |
else:
|
| 1635 |
spec = amazon.page_source(cur, key, definition, allow_lookups=allow_lookups)
|
| 1636 |
-
if key == amazon.PREFIX + "inventory" and spec is not None and not spec.get("missing"):
|
| 1637 |
-
status = amazon.amazon_store.inventory_table_status(cur)
|
| 1638 |
-
spec = {**spec, 'sourceStatus': {
|
| 1639 |
-
name: status[name] for name in ('empty', 'code', 'reason') if name in status}}
|
| 1640 |
except Exception as exc: # noqa: BLE001
|
| 1641 |
from core.request_timing import raise_if_resource_error
|
| 1642 |
raise_if_resource_error(exc)
|
| 1643 |
raise err(503, "amazon_source_unavailable",
|
| 1644 |
"This Amazon database could not be resolved. Retry the request.") from exc
|
| 1645 |
if spec is not None and spec.get("missing"):
|
| 1646 |
-
raise err(
|
| 1647 |
if spec is not None and '_connection_id' in {
|
| 1648 |
name for name, _kind in amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)}:
|
| 1649 |
spec = {**spec, 'cols': {**spec['cols'], '_connection_id': (
|
|
@@ -1679,10 +1657,7 @@ def _source_for(cur, table_key, *, allow_lookups=True):
|
|
| 1679 |
if module_cache_source(key) is not None:
|
| 1680 |
return None
|
| 1681 |
if key.startswith("ut_odoo_model_"):
|
| 1682 |
-
|
| 1683 |
-
return _rel().dynamic_model_source(cur, key)
|
| 1684 |
-
except _rel().Refused as exc:
|
| 1685 |
-
raise err(409, 'odoo_mirror_binding_incomplete', str(exc)) from exc
|
| 1686 |
if key.startswith('ut_meta_'):
|
| 1687 |
return _meta_source(cur, key)
|
| 1688 |
if key.startswith("ut_amzn_"):
|
|
@@ -1698,7 +1673,7 @@ def _source_for(cur, table_key, *, allow_lookups=True):
|
|
| 1698 |
if not spec:
|
| 1699 |
return None
|
| 1700 |
if spec.get("missing"):
|
| 1701 |
-
raise err(
|
| 1702 |
from harness import datastore
|
| 1703 |
if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
|
| 1704 |
try:
|
|
@@ -1737,7 +1712,7 @@ def _source_for(cur, table_key, *, allow_lookups=True):
|
|
| 1737 |
if not cols:
|
| 1738 |
if key.startswith("ut_amzn_"):
|
| 1739 |
import amazon_relational
|
| 1740 |
-
raise err(
|
| 1741 |
amazon_relational.missing_source(key)["missing"])
|
| 1742 |
return None # the mirror has no such table on this deployment
|
| 1743 |
have_by_alias[alias] = cols
|
|
@@ -2667,33 +2642,16 @@ def _materialised_table(table_key, *, cur=None, st=None):
|
|
| 2667 |
if module_cache_source(table_key) is not None:
|
| 2668 |
return True
|
| 2669 |
if str(table_key).startswith("ut_odoo_model_"):
|
| 2670 |
-
|
| 2671 |
-
mandatory = bool((definition or {}).get('readThrough') or (definition or {}).get('oversize'))
|
| 2672 |
-
if st is not None and not mandatory:
|
| 2673 |
return True
|
| 2674 |
own_cursor = cur is None
|
| 2675 |
if own_cursor:
|
| 2676 |
if st is None:
|
| 2677 |
return True
|
| 2678 |
-
|
| 2679 |
-
cur = _table_cursor(st, table_key)
|
| 2680 |
-
except _rt().DatastoreMismatch as exc:
|
| 2681 |
-
raise err(409, 'cross_tenant_store', str(exc)) from exc
|
| 2682 |
-
except _rel().Refused as exc:
|
| 2683 |
-
raise err(409, 'odoo_mirror_binding_incomplete', str(exc)) from exc
|
| 2684 |
-
except RuntimeError as exc:
|
| 2685 |
-
raise err(409, 'odoo_mirror_not_loaded', str(exc)) from exc
|
| 2686 |
try:
|
| 2687 |
-
|
| 2688 |
-
|
| 2689 |
-
tenant_key=getattr(st, "key", None))
|
| 2690 |
-
except _rel().Refused as exc:
|
| 2691 |
-
raise err(409, 'odoo_mirror_binding_incomplete', str(exc)) from exc
|
| 2692 |
-
if mandatory and spec is None:
|
| 2693 |
-
raise err(409, 'odoo_mirror_not_loaded',
|
| 2694 |
-
'This Odoo database has no complete mirror publication. Complete its '
|
| 2695 |
-
'sync in connection settings before reading rows.')
|
| 2696 |
-
return spec is None
|
| 2697 |
finally:
|
| 2698 |
if own_cursor and cur is not None:
|
| 2699 |
cur.close()
|
|
@@ -2705,7 +2663,7 @@ def _materialised_table(table_key, *, cur=None, st=None):
|
|
| 2705 |
own_cursor = cur is None
|
| 2706 |
cur = datastore.ro_cursor() if own_cursor else cur
|
| 2707 |
try:
|
| 2708 |
-
definition = _ut().
|
| 2709 |
spec = _amazon_page_source(cur, table_key, definition, allow_lookups=False)
|
| 2710 |
return spec is None
|
| 2711 |
finally:
|
|
@@ -4051,8 +4009,6 @@ def resolved_fields(table_key, defn):
|
|
| 4051 |
if str(table_key).startswith("ut_amzn_"):
|
| 4052 |
import amazon_relational as amazon
|
| 4053 |
connected = table_key in amazon.SOURCE
|
| 4054 |
-
elif str(base).startswith('ut_odoo_model_'):
|
| 4055 |
-
connected = True
|
| 4056 |
else:
|
| 4057 |
connected = table_key in _sources()
|
| 4058 |
if not connected:
|
|
@@ -4152,7 +4108,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 4152 |
# try, not inside it.
|
| 4153 |
try:
|
| 4154 |
cur, mismatch = (_mirror_cur(session, table_key)
|
| 4155 |
-
if (table_key.startswith(("ut_amzn_", "ut_meta_"
|
| 4156 |
or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY
|
| 4157 |
or rel.base_table_key(table_key) in rel.stock_source_specs())
|
| 4158 |
else _mirror_cur(session))
|
|
@@ -4858,11 +4814,7 @@ def _window_principal(session):
|
|
| 4858 |
"""Bind pagination to identity and permissions without volatile activity stamps."""
|
| 4859 |
import registry_source
|
| 4860 |
policy = {k: v for k, v in session.user.items() if k not in registry_source.USER_NOISE}
|
| 4861 |
-
|
| 4862 |
-
if isinstance(claims, dict):
|
| 4863 |
-
# Authentication renews idle expiry on use without changing the caller's access.
|
| 4864 |
-
claims = {key: value for key, value in claims.items() if key != 'i'}
|
| 4865 |
-
return [session.tenant, session.uname, claims, policy]
|
| 4866 |
|
| 4867 |
|
| 4868 |
def _window_cursor(session, query, position=None, token=None):
|
|
@@ -5165,10 +5117,8 @@ def odoo_table_rows(table_key: str,
|
|
| 5165 |
ids = _record_ids(body)
|
| 5166 |
|
| 5167 |
# Admission resolves this Amazon database without publishing a process-wide binding.
|
| 5168 |
-
page = _page_defs(session) if str(table_key).startswith(
|
| 5169 |
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
| 5170 |
-
if str(table_key).startswith('ut_odoo_model_'):
|
| 5171 |
-
admitted = True if require_dynamic_odoo_mirror(table_key, session, st=page[0]) else None
|
| 5172 |
if (admitted is None if page is not None else _materialised_table(table_key, st=session.runtime)):
|
| 5173 |
return _audit_personal_response(table_key, _materialised_window(
|
| 5174 |
table_key, session, offset, limit, filters, filterConj,
|
|
@@ -5188,11 +5138,6 @@ def odoo_table_rows(table_key: str,
|
|
| 5188 |
read = _restrict_row_ids(read, ids)
|
| 5189 |
defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"]
|
| 5190 |
cols, limits, frm = read["cols"], read["limits"], read["frm"]
|
| 5191 |
-
source_status = spec.get('sourceStatus') or {}
|
| 5192 |
-
if source_status.get('empty') and source_status.get('reason'):
|
| 5193 |
-
limits = [*limits, {'subject': table_key, 'effect': 'source_empty',
|
| 5194 |
-
'code': source_status.get('code'), 'cause': source_status['reason'],
|
| 5195 |
-
'recommendation': ''}]
|
| 5196 |
base_where, wall_params = read["base_where"], read["wall_params"]
|
| 5197 |
where, params, sort_spec = read["where"], read["params"], read["sort_spec"]
|
| 5198 |
_gsql, _gparams = _group_pin(cols, groupBy, groupKey)
|
|
@@ -5337,7 +5282,6 @@ def odoo_table_rows(table_key: str,
|
|
| 5337 |
if str(row["pid"]) in notes:
|
| 5338 |
row[user_tables.CELL_NOTES_KEY] = notes[str(row["pid"])]
|
| 5339 |
return _audit_personal_response(table_key, {"ok": True, "fields": fields, "rows": rows,
|
| 5340 |
-
**source_status,
|
| 5341 |
# ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`.
|
| 5342 |
"total": win["total"], "totalUnfiltered": unfiltered,
|
| 5343 |
"offset": win["offset"], "limit": win["limit"], "limits": limits,
|
|
@@ -5802,10 +5746,8 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 5802 |
if group_limit is not None and (type(group_limit) is not int or not 1 <= group_limit <= MAX_GROUPS):
|
| 5803 |
raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
|
| 5804 |
|
| 5805 |
-
page = _page_defs(session) if str(table_key).startswith(
|
| 5806 |
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
| 5807 |
-
if str(table_key).startswith('ut_odoo_model_'):
|
| 5808 |
-
admitted = True if require_dynamic_odoo_mirror(table_key, session, st=page[0]) else None
|
| 5809 |
|
| 5810 |
if (admitted is None if page is not None else _materialised_table(table_key, st=session.runtime)):
|
| 5811 |
with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
|
|
|
|
| 1393 |
cur.close()
|
| 1394 |
|
| 1395 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1396 |
register_source_provider(_odoo_sources)
|
| 1397 |
register_source_provider(_amazon_sources)
|
| 1398 |
register_source_provider(_meta_sources)
|
|
|
|
| 1609 |
"""Resolve one admitted Amazon target and keep query failures distinct from stored tables."""
|
| 1610 |
import amazon_relational as amazon
|
| 1611 |
|
|
|
|
|
|
|
| 1612 |
try:
|
| 1613 |
if binding:
|
| 1614 |
columns = amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)
|
| 1615 |
spec = amazon.grid_source(cur, key, columns=columns, allow_lookups=allow_lookups)
|
| 1616 |
else:
|
| 1617 |
spec = amazon.page_source(cur, key, definition, allow_lookups=allow_lookups)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1618 |
except Exception as exc: # noqa: BLE001
|
| 1619 |
from core.request_timing import raise_if_resource_error
|
| 1620 |
raise_if_resource_error(exc)
|
| 1621 |
raise err(503, "amazon_source_unavailable",
|
| 1622 |
"This Amazon database could not be resolved. Retry the request.") from exc
|
| 1623 |
if spec is not None and spec.get("missing"):
|
| 1624 |
+
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
| 1625 |
if spec is not None and '_connection_id' in {
|
| 1626 |
name for name, _kind in amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)}:
|
| 1627 |
spec = {**spec, 'cols': {**spec['cols'], '_connection_id': (
|
|
|
|
| 1657 |
if module_cache_source(key) is not None:
|
| 1658 |
return None
|
| 1659 |
if key.startswith("ut_odoo_model_"):
|
| 1660 |
+
return _rel().dynamic_model_source(cur, key)
|
|
|
|
|
|
|
|
|
|
| 1661 |
if key.startswith('ut_meta_'):
|
| 1662 |
return _meta_source(cur, key)
|
| 1663 |
if key.startswith("ut_amzn_"):
|
|
|
|
| 1673 |
if not spec:
|
| 1674 |
return None
|
| 1675 |
if spec.get("missing"):
|
| 1676 |
+
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
| 1677 |
from harness import datastore
|
| 1678 |
if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
|
| 1679 |
try:
|
|
|
|
| 1712 |
if not cols:
|
| 1713 |
if key.startswith("ut_amzn_"):
|
| 1714 |
import amazon_relational
|
| 1715 |
+
raise err(503, "amazon_mirror_not_loaded",
|
| 1716 |
amazon_relational.missing_source(key)["missing"])
|
| 1717 |
return None # the mirror has no such table on this deployment
|
| 1718 |
have_by_alias[alias] = cols
|
|
|
|
| 2642 |
if module_cache_source(table_key) is not None:
|
| 2643 |
return True
|
| 2644 |
if str(table_key).startswith("ut_odoo_model_"):
|
| 2645 |
+
if st is not None and (_ut().get_definition(table_key, st=st) or {}).get("readThrough") is not True:
|
|
|
|
|
|
|
| 2646 |
return True
|
| 2647 |
own_cursor = cur is None
|
| 2648 |
if own_cursor:
|
| 2649 |
if st is None:
|
| 2650 |
return True
|
| 2651 |
+
cur = _table_cursor(st, table_key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2652 |
try:
|
| 2653 |
+
return _rel().dynamic_model_source(cur, table_key,
|
| 2654 |
+
tenant_key=getattr(st, "key", None)) is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2655 |
finally:
|
| 2656 |
if own_cursor and cur is not None:
|
| 2657 |
cur.close()
|
|
|
|
| 2663 |
own_cursor = cur is None
|
| 2664 |
cur = datastore.ro_cursor() if own_cursor else cur
|
| 2665 |
try:
|
| 2666 |
+
definition = _ut().get(table_key, st=st) if st is not None else None
|
| 2667 |
spec = _amazon_page_source(cur, table_key, definition, allow_lookups=False)
|
| 2668 |
return spec is None
|
| 2669 |
finally:
|
|
|
|
| 4009 |
if str(table_key).startswith("ut_amzn_"):
|
| 4010 |
import amazon_relational as amazon
|
| 4011 |
connected = table_key in amazon.SOURCE
|
|
|
|
|
|
|
| 4012 |
else:
|
| 4013 |
connected = table_key in _sources()
|
| 4014 |
if not connected:
|
|
|
|
| 4108 |
# try, not inside it.
|
| 4109 |
try:
|
| 4110 |
cur, mismatch = (_mirror_cur(session, table_key)
|
| 4111 |
+
if (table_key.startswith(("ut_amzn_", "ut_meta_"))
|
| 4112 |
or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY
|
| 4113 |
or rel.base_table_key(table_key) in rel.stock_source_specs())
|
| 4114 |
else _mirror_cur(session))
|
|
|
|
| 4814 |
"""Bind pagination to identity and permissions without volatile activity stamps."""
|
| 4815 |
import registry_source
|
| 4816 |
policy = {k: v for k, v in session.user.items() if k not in registry_source.USER_NOISE}
|
| 4817 |
+
return [session.tenant, session.uname, getattr(session, 'claims', {}), policy]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4818 |
|
| 4819 |
|
| 4820 |
def _window_cursor(session, query, position=None, token=None):
|
|
|
|
| 5117 |
ids = _record_ids(body)
|
| 5118 |
|
| 5119 |
# Admission resolves this Amazon database without publishing a process-wide binding.
|
| 5120 |
+
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 5121 |
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
|
|
|
|
|
|
| 5122 |
if (admitted is None if page is not None else _materialised_table(table_key, st=session.runtime)):
|
| 5123 |
return _audit_personal_response(table_key, _materialised_window(
|
| 5124 |
table_key, session, offset, limit, filters, filterConj,
|
|
|
|
| 5138 |
read = _restrict_row_ids(read, ids)
|
| 5139 |
defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"]
|
| 5140 |
cols, limits, frm = read["cols"], read["limits"], read["frm"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5141 |
base_where, wall_params = read["base_where"], read["wall_params"]
|
| 5142 |
where, params, sort_spec = read["where"], read["params"], read["sort_spec"]
|
| 5143 |
_gsql, _gparams = _group_pin(cols, groupBy, groupKey)
|
|
|
|
| 5282 |
if str(row["pid"]) in notes:
|
| 5283 |
row[user_tables.CELL_NOTES_KEY] = notes[str(row["pid"])]
|
| 5284 |
return _audit_personal_response(table_key, {"ok": True, "fields": fields, "rows": rows,
|
|
|
|
| 5285 |
# ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`.
|
| 5286 |
"total": win["total"], "totalUnfiltered": unfiltered,
|
| 5287 |
"offset": win["offset"], "limit": win["limit"], "limits": limits,
|
|
|
|
| 5746 |
if group_limit is not None and (type(group_limit) is not int or not 1 <= group_limit <= MAX_GROUPS):
|
| 5747 |
raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
|
| 5748 |
|
| 5749 |
+
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 5750 |
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
|
|
|
|
|
|
| 5751 |
|
| 5752 |
if (admitted is None if page is not None else _materialised_table(table_key, st=session.runtime)):
|
| 5753 |
with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
|
api/routes_tables.py
CHANGED
|
@@ -4476,8 +4476,6 @@ def delete_table_view(table_key: str, view_id: str, request: Request,
|
|
| 4476 |
"rev_current": stale.rev_current}})
|
| 4477 |
except view_audit.RevisionUnverifiable as unknown:
|
| 4478 |
raise err(503, "view_revision_unverifiable", str(unknown))
|
| 4479 |
-
from core import shares
|
| 4480 |
-
shares.drop_objects([('view', vid)], st=ops.st)
|
| 4481 |
return {"deleted": vid, "owner": owner}
|
| 4482 |
|
| 4483 |
|
|
|
|
| 4476 |
"rev_current": stale.rev_current}})
|
| 4477 |
except view_audit.RevisionUnverifiable as unknown:
|
| 4478 |
raise err(503, "view_revision_unverifiable", str(unknown))
|
|
|
|
|
|
|
| 4479 |
return {"deleted": vid, "owner": owner}
|
| 4480 |
|
| 4481 |
|
api/sync_worker.py
CHANGED
|
@@ -35,10 +35,9 @@ def configure(env=None):
|
|
| 35 |
worker_db = worker_dir / f"{safe}.duckdb"
|
| 36 |
env["AIOS_DUCKDB_STORE_DIR"] = str(worker_dir)
|
| 37 |
env["AIOS_DUCKDB_PATH"] = str(worker_db)
|
| 38 |
-
#
|
| 39 |
-
env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "
|
| 40 |
env["AIOS_DUCKDB_THREADS"] = env.get("AIOS_WORKER_DUCKDB_THREADS") or "1"
|
| 41 |
-
env["AIOS_DUCKDB_MAX_TEMP_SIZE"] = env.get("AIOS_WORKER_DUCKDB_MAX_TEMP_SIZE") or "16GB"
|
| 42 |
# The worker holds its own file read-write. Mirror versions are for the web workers only.
|
| 43 |
env["AIOS_MIRROR_VERSIONS"] = "0"
|
| 44 |
env["AIOS_UVICORN_WORKERS"] = "1"
|
|
@@ -130,7 +129,6 @@ def main(argv=None):
|
|
| 130 |
print(f"[sync-worker] pid={os.getpid()} tenant={tenant} "
|
| 131 |
f"scope={os.environ.get('AIOS_WORKER_SCOPE') or 'primary'} duckdb={worker_db} "
|
| 132 |
f"memory_limit={os.environ['AIOS_DUCKDB_MEMORY']} threads={os.environ['AIOS_DUCKDB_THREADS']} "
|
| 133 |
-
f"spill_limit={os.environ['AIOS_DUCKDB_MAX_TEMP_SIZE']} "
|
| 134 |
f"nice={niceness} job={mode or 'none'}",
|
| 135 |
flush=True)
|
| 136 |
print("[sync-worker] amazon_store.sync includes Brand Analytics reports in the worker role", flush=True)
|
|
|
|
| 35 |
worker_db = worker_dir / f"{safe}.duckdb"
|
| 36 |
env["AIOS_DUCKDB_STORE_DIR"] = str(worker_dir)
|
| 37 |
env["AIOS_DUCKDB_PATH"] = str(worker_db)
|
| 38 |
+
# The launcher (request_class.py --env) sizes these to the box. Unset: the 2 vCPU values.
|
| 39 |
+
env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "4GB"
|
| 40 |
env["AIOS_DUCKDB_THREADS"] = env.get("AIOS_WORKER_DUCKDB_THREADS") or "1"
|
|
|
|
| 41 |
# The worker holds its own file read-write. Mirror versions are for the web workers only.
|
| 42 |
env["AIOS_MIRROR_VERSIONS"] = "0"
|
| 43 |
env["AIOS_UVICORN_WORKERS"] = "1"
|
|
|
|
| 129 |
print(f"[sync-worker] pid={os.getpid()} tenant={tenant} "
|
| 130 |
f"scope={os.environ.get('AIOS_WORKER_SCOPE') or 'primary'} duckdb={worker_db} "
|
| 131 |
f"memory_limit={os.environ['AIOS_DUCKDB_MEMORY']} threads={os.environ['AIOS_DUCKDB_THREADS']} "
|
|
|
|
| 132 |
f"nice={niceness} job={mode or 'none'}",
|
| 133 |
flush=True)
|
| 134 |
print("[sync-worker] amazon_store.sync includes Brand Analytics reports in the worker role", flush=True)
|
api/tool_groups/apps.py
CHANGED
|
@@ -470,15 +470,7 @@ def apply_template(ctx, args):
|
|
| 470 |
allowed_databases=allowed, semantic_records=records,
|
| 471 |
field_contracts=_fields(ctx, allowed), **attribution)
|
| 472 |
return _result(result)
|
| 473 |
-
except applications.ApplicationError as exc:
|
| 474 |
-
# Keep typed refusals and inaccessible-App responses on their original path.
|
| 475 |
-
if type(exc) is applications.ApplicationError and "does not exist" not in str(exc):
|
| 476 |
-
kind = json.dumps(args["templateKind"], ensure_ascii=True)
|
| 477 |
-
exc = applications.ApplicationError(
|
| 478 |
-
f"Cannot apply templateKind {kind}. {str(exc).rstrip('.')}. "
|
| 479 |
-
"Check this template's inputs with apps.templates and retry.")
|
| 480 |
-
_refuse(exc)
|
| 481 |
-
except PermissionError as exc:
|
| 482 |
_refuse(exc)
|
| 483 |
|
| 484 |
|
|
|
|
| 470 |
allowed_databases=allowed, semantic_records=records,
|
| 471 |
field_contracts=_fields(ctx, allowed), **attribution)
|
| 472 |
return _result(result)
|
| 473 |
+
except (applications.ApplicationError, PermissionError) as exc:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 474 |
_refuse(exc)
|
| 475 |
|
| 476 |
|
api/whatsapp_identity.py
CHANGED
|
@@ -8,11 +8,9 @@ import re
|
|
| 8 |
import secrets
|
| 9 |
import time
|
| 10 |
from datetime import datetime, timezone
|
| 11 |
-
from types import SimpleNamespace
|
| 12 |
|
| 13 |
import core.store as store
|
| 14 |
import core.users as users
|
| 15 |
-
from core import account_rows
|
| 16 |
import telnyx_whatsapp
|
| 17 |
|
| 18 |
_store = store
|
|
@@ -50,16 +48,8 @@ def _ownership_read(source, key):
|
|
| 50 |
return reader(key) if callable(reader) else source.get(key, fresh=True)
|
| 51 |
|
| 52 |
|
| 53 |
-
def _account_source():
|
| 54 |
-
source = users.store
|
| 55 |
-
if account_rows._pg_source(source):
|
| 56 |
-
return source
|
| 57 |
-
# Compatibility stores must refuse stale fallback for canonical ownership reads too.
|
| 58 |
-
return SimpleNamespace(get=lambda key, fresh=False: _ownership_read(source, key))
|
| 59 |
-
|
| 60 |
-
|
| 61 |
def _account(tenant, username):
|
| 62 |
-
record =
|
| 63 |
if not isinstance(record, dict) or str(record.get("tenant") or "royal-imports") != tenant:
|
| 64 |
raise WhatsAppRefusal("This account is not available in this workspace.")
|
| 65 |
return record
|
|
@@ -68,7 +58,7 @@ def _account(tenant, username):
|
|
| 68 |
def _phone_conflict(phone, tenant, username, index=None):
|
| 69 |
index = (_ownership_read(_store, INDEX_KEY) or {}) if index is None else index
|
| 70 |
# The registry proves ownership. An interrupted index write proves no binding.
|
| 71 |
-
registry =
|
| 72 |
caller = registry.get(username)
|
| 73 |
if not isinstance(caller, dict) or str(caller.get("tenant") or "royal-imports") != tenant:
|
| 74 |
raise WhatsAppRefusal("This account is not available in this workspace.")
|
|
@@ -91,7 +81,7 @@ def lookup(phone_e164):
|
|
| 91 |
entry = (_ownership_read(_store, INDEX_KEY) or {}).get(phone_e164)
|
| 92 |
if not isinstance(entry, dict):
|
| 93 |
return None
|
| 94 |
-
record =
|
| 95 |
if not isinstance(record, dict):
|
| 96 |
return None
|
| 97 |
whatsapp = record.get("whatsapp")
|
|
|
|
| 8 |
import secrets
|
| 9 |
import time
|
| 10 |
from datetime import datetime, timezone
|
|
|
|
| 11 |
|
| 12 |
import core.store as store
|
| 13 |
import core.users as users
|
|
|
|
| 14 |
import telnyx_whatsapp
|
| 15 |
|
| 16 |
_store = store
|
|
|
|
| 48 |
return reader(key) if callable(reader) else source.get(key, fresh=True)
|
| 49 |
|
| 50 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
def _account(tenant, username):
|
| 52 |
+
record = (_ownership_read(users.store, "users") or {}).get(username)
|
| 53 |
if not isinstance(record, dict) or str(record.get("tenant") or "royal-imports") != tenant:
|
| 54 |
raise WhatsAppRefusal("This account is not available in this workspace.")
|
| 55 |
return record
|
|
|
|
| 58 |
def _phone_conflict(phone, tenant, username, index=None):
|
| 59 |
index = (_ownership_read(_store, INDEX_KEY) or {}) if index is None else index
|
| 60 |
# The registry proves ownership. An interrupted index write proves no binding.
|
| 61 |
+
registry = _ownership_read(users.store, "users") or {}
|
| 62 |
caller = registry.get(username)
|
| 63 |
if not isinstance(caller, dict) or str(caller.get("tenant") or "royal-imports") != tenant:
|
| 64 |
raise WhatsAppRefusal("This account is not available in this workspace.")
|
|
|
|
| 81 |
entry = (_ownership_read(_store, INDEX_KEY) or {}).get(phone_e164)
|
| 82 |
if not isinstance(entry, dict):
|
| 83 |
return None
|
| 84 |
+
record = (_ownership_read(users.store, "users") or {}).get(entry.get("username"))
|
| 85 |
if not isinstance(record, dict):
|
| 86 |
return None
|
| 87 |
whatsapp = record.get("whatsapp")
|
platform/core/activity_log.py
CHANGED
|
@@ -12,7 +12,7 @@ import re
|
|
| 12 |
|
| 13 |
DATABASE_KEY = 'ut_activity'
|
| 14 |
SYSTEM_KIND = 'activity'
|
| 15 |
-
KINDS = frozenset(('record', 'field', 'cell', 'file', 'app', 'view'
|
| 16 |
OBJECT_TYPES = frozenset(('database', 'workbook', 'file', 'app'))
|
| 17 |
CHANGES_MAX_BYTES = 4096
|
| 18 |
RETENTION_MONTHS = 13
|
|
@@ -242,7 +242,7 @@ def seal_copy_batch(rt, events, outbox, *, backup_path, definitions=None):
|
|
| 242 |
def append(rt, *, kind, object, by, via=None, on_behalf_of=None, changes=None, _delivery=None):
|
| 243 |
"""Append one C5 event and remove at most PRUNE_BATCH expired events in the same transaction."""
|
| 244 |
if kind not in KINDS:
|
| 245 |
-
raise ValueError('kind must be record, field, cell, file, app
|
| 246 |
object_type, object_key, object_id = _object(object)
|
| 247 |
by = _text(by, 'by')
|
| 248 |
via = _text(via, 'via', optional=True)
|
|
@@ -522,13 +522,12 @@ class PgActivityStore:
|
|
| 522 |
needed = {f'{self.name}_{suffix}' for suffix in (
|
| 523 |
'delivery', 'time', 'object_time', 'object_id_time', 'person_time')}
|
| 524 |
if (set(_COLUMNS) | {'delivery_key', 'delivery_version'} <= {r[0] for r in columns}
|
| 525 |
-
and needed <= {r[0] for r in indexes} and constraint
|
| 526 |
-
and "'view_as'" in constraint[0]):
|
| 527 |
return
|
| 528 |
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
|
| 529 |
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} (
|
| 530 |
id BIGSERIAL PRIMARY KEY, occurred_at TIMESTAMPTZ NOT NULL,
|
| 531 |
-
kind TEXT NOT NULL CHECK (kind IN ('record','field','cell','file','app','view'
|
| 532 |
object_type TEXT NOT NULL, object_key TEXT NOT NULL, object_id TEXT NOT NULL,
|
| 533 |
actor TEXT NOT NULL, via TEXT, on_behalf_of TEXT,
|
| 534 |
changes JSONB NOT NULL, changes_truncated BOOLEAN NOT NULL,
|
|
@@ -542,11 +541,11 @@ class PgActivityStore:
|
|
| 542 |
'SELECT pg_get_constraintdef(oid) FROM pg_constraint '
|
| 543 |
'WHERE conrelid = %s::regclass AND conname = %s',
|
| 544 |
(self.table, constraint)).fetchone()
|
| 545 |
-
if prior is None or "'
|
| 546 |
con.execute(f'ALTER TABLE {self.table} DROP CONSTRAINT IF EXISTS "{constraint}"')
|
| 547 |
# The former constraint already admitted every existing event kind.
|
| 548 |
con.execute(f'ALTER TABLE {self.table} ADD CONSTRAINT "{constraint}" '
|
| 549 |
-
"CHECK (kind IN ('record','field','cell','file','app','view'
|
| 550 |
for suffix, columns in (
|
| 551 |
('time', 'occurred_at DESC, id DESC'),
|
| 552 |
('object_time', 'object_type, object_key, occurred_at DESC, id DESC'),
|
|
|
|
| 12 |
|
| 13 |
DATABASE_KEY = 'ut_activity'
|
| 14 |
SYSTEM_KIND = 'activity'
|
| 15 |
+
KINDS = frozenset(('record', 'field', 'cell', 'file', 'app', 'view'))
|
| 16 |
OBJECT_TYPES = frozenset(('database', 'workbook', 'file', 'app'))
|
| 17 |
CHANGES_MAX_BYTES = 4096
|
| 18 |
RETENTION_MONTHS = 13
|
|
|
|
| 242 |
def append(rt, *, kind, object, by, via=None, on_behalf_of=None, changes=None, _delivery=None):
|
| 243 |
"""Append one C5 event and remove at most PRUNE_BATCH expired events in the same transaction."""
|
| 244 |
if kind not in KINDS:
|
| 245 |
+
raise ValueError('kind must be record, field, cell, file, app or view.')
|
| 246 |
object_type, object_key, object_id = _object(object)
|
| 247 |
by = _text(by, 'by')
|
| 248 |
via = _text(via, 'via', optional=True)
|
|
|
|
| 522 |
needed = {f'{self.name}_{suffix}' for suffix in (
|
| 523 |
'delivery', 'time', 'object_time', 'object_id_time', 'person_time')}
|
| 524 |
if (set(_COLUMNS) | {'delivery_key', 'delivery_version'} <= {r[0] for r in columns}
|
| 525 |
+
and needed <= {r[0] for r in indexes} and constraint and "'view'" in constraint[0]):
|
|
|
|
| 526 |
return
|
| 527 |
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
|
| 528 |
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} (
|
| 529 |
id BIGSERIAL PRIMARY KEY, occurred_at TIMESTAMPTZ NOT NULL,
|
| 530 |
+
kind TEXT NOT NULL CHECK (kind IN ('record','field','cell','file','app','view')),
|
| 531 |
object_type TEXT NOT NULL, object_key TEXT NOT NULL, object_id TEXT NOT NULL,
|
| 532 |
actor TEXT NOT NULL, via TEXT, on_behalf_of TEXT,
|
| 533 |
changes JSONB NOT NULL, changes_truncated BOOLEAN NOT NULL,
|
|
|
|
| 541 |
'SELECT pg_get_constraintdef(oid) FROM pg_constraint '
|
| 542 |
'WHERE conrelid = %s::regclass AND conname = %s',
|
| 543 |
(self.table, constraint)).fetchone()
|
| 544 |
+
if prior is None or "'view'" not in prior[0]:
|
| 545 |
con.execute(f'ALTER TABLE {self.table} DROP CONSTRAINT IF EXISTS "{constraint}"')
|
| 546 |
# The former constraint already admitted every existing event kind.
|
| 547 |
con.execute(f'ALTER TABLE {self.table} ADD CONSTRAINT "{constraint}" '
|
| 548 |
+
"CHECK (kind IN ('record','field','cell','file','app','view')) NOT VALID")
|
| 549 |
for suffix, columns in (
|
| 550 |
('time', 'occurred_at DESC, id DESC'),
|
| 551 |
('object_time', 'object_type, object_key, occurred_at DESC, id DESC'),
|
platform/core/amazon_project_results.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Typed project
|
| 2 |
|
| 3 |
from datetime import date, timedelta
|
| 4 |
import hashlib
|
|
@@ -13,121 +13,6 @@ METRICS = ('sales', 'units', 'returns', 'return_rate', 'refund_rate', 'roas')
|
|
| 13 |
GRAINS = ('sku', 'asin', 'parent_asin')
|
| 14 |
MAX_CONFIGS = 20
|
| 15 |
BOUNDARY = '[completedOn, endExclusive) in the tenant timezone. Today is excluded.'
|
| 16 |
-
SERIES_METRICS = (
|
| 17 |
-
('sales', 'Sales', 'currency'), ('units', 'Units', 'units'),
|
| 18 |
-
('margin', 'Margin', 'percent'), ('returns', 'Returns', 'units'),
|
| 19 |
-
('returns_pct', 'Returns %', 'percent'), ('refunds_pct', 'Refunds %', 'percent'),
|
| 20 |
-
('roas', 'ROAS', 'ratio'), ('sales_yoy', 'Sales YoY', 'percent'),
|
| 21 |
-
)
|
| 22 |
-
SERIES_BOUNDARY = 'Inclusive requested dates in the tenant timezone. Today is incomplete and excluded from totals.'
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
class ValidationFailed(RuntimeError):
|
| 26 |
-
"""A daily project series does not reconcile to its independent source aggregate."""
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
def series_scope(value='all'):
|
| 30 |
-
"""Parse the modal filter without allowing it to widen project membership."""
|
| 31 |
-
if value == 'all':
|
| 32 |
-
return None
|
| 33 |
-
if not isinstance(value, str) or ':' not in value:
|
| 34 |
-
raise ValueError('scope must be all, sku:<keys>, asin:<key> or parent:<key>.')
|
| 35 |
-
grain, text = value.split(':', 1)
|
| 36 |
-
keys = [key.strip() for key in text.split(',')]
|
| 37 |
-
if (grain not in ('sku', 'asin', 'parent') or not all(keys)
|
| 38 |
-
or (grain != 'sku' and len(keys) != 1)):
|
| 39 |
-
raise ValueError('scope must be all, sku:<keys>, asin:<key> or parent:<key>.')
|
| 40 |
-
return {'grain': grain, 'keys': list(dict.fromkeys(keys))}
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
def series_window(project, start=None, end=None, *, today=None):
|
| 44 |
-
"""Default to thirty days before start through today and aggregate only completed tenant days."""
|
| 45 |
-
today = today or periods.today()
|
| 46 |
-
|
| 47 |
-
def parsed(value):
|
| 48 |
-
if not isinstance(value, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}', value):
|
| 49 |
-
raise ValueError('Series dates must use YYYY-MM-DD.')
|
| 50 |
-
return date.fromisoformat(value)
|
| 51 |
-
|
| 52 |
-
first = (parsed(start) if start is not None else
|
| 53 |
-
parsed(project.get('start_on') or today.isoformat()) - timedelta(days=30))
|
| 54 |
-
last = parsed(end) if end is not None else today
|
| 55 |
-
if first > last or last > today:
|
| 56 |
-
raise ValueError('Series dates must be ordered and cannot end after today.')
|
| 57 |
-
stop = min(last + timedelta(days=1), today)
|
| 58 |
-
lo, hi, rule = periods.yoy_window(first.isoformat(), stop.isoformat(), 'day')
|
| 59 |
-
return {'from': first.isoformat(), 'to': last.isoformat(), 'days': (stop - first).days,
|
| 60 |
-
'current': (first, stop), 'comparison': (date.fromisoformat(lo), date.fromisoformat(hi)),
|
| 61 |
-
'compare_rule': rule}
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
def _finite(value):
|
| 65 |
-
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
def validate(answer, *, independent):
|
| 69 |
-
"""Requery independent totals and return failing checks for any changed or unproved series."""
|
| 70 |
-
checks = []
|
| 71 |
-
first, last = date.fromisoformat(answer['from']), date.fromisoformat(answer['to'])
|
| 72 |
-
dates = [(first + timedelta(days=i)).isoformat() for i in range((last - first).days + 1)]
|
| 73 |
-
expected_keys = [key for key, _label, _unit in SERIES_METRICS]
|
| 74 |
-
metrics = answer['metrics']
|
| 75 |
-
if [metric['key'] for metric in metrics] != expected_keys:
|
| 76 |
-
return [{'check': 'series total ties to economics', 'ok': False,
|
| 77 |
-
'reason': 'Series metrics do not follow the project result contract.'}]
|
| 78 |
-
for metric in metrics:
|
| 79 |
-
ours = metric.get('total')
|
| 80 |
-
source = independent(metric['key'])
|
| 81 |
-
expected, source_reason = source['value'], source.get('reason')
|
| 82 |
-
points = metric.get('points', [])
|
| 83 |
-
ok = [point.get('date') for point in points] == dates
|
| 84 |
-
if ours is None:
|
| 85 |
-
ok = ok and bool(metric.get('reason')) and all(point.get('value') is None for point in points)
|
| 86 |
-
else:
|
| 87 |
-
ok = (ok and _finite(ours) and _finite(expected)
|
| 88 |
-
and math.isclose(ours, expected, rel_tol=1e-9, abs_tol=1e-6)
|
| 89 |
-
and all(point.get('value') is None or _finite(point['value']) for point in points))
|
| 90 |
-
if metric['key'] in ('sales', 'units', 'returns'):
|
| 91 |
-
values = [point['value'] for point in points if point.get('value') is not None]
|
| 92 |
-
ok = ok and bool(values) and math.isclose(math.fsum(values), ours, rel_tol=1e-9, abs_tol=1e-6)
|
| 93 |
-
tie = metric.get('tie', {})
|
| 94 |
-
if tie.get('ok') or tie.get('ours') is not None or tie.get('independent') is not None:
|
| 95 |
-
ok = (ok and ours is not None and tie.get('ok') is True
|
| 96 |
-
and _finite(tie.get('ours')) and _finite(tie.get('independent'))
|
| 97 |
-
and math.isclose(tie['ours'], ours, rel_tol=1e-9, abs_tol=1e-6)
|
| 98 |
-
and _finite(expected)
|
| 99 |
-
and math.isclose(tie['independent'], expected, rel_tol=1e-9, abs_tol=1e-6))
|
| 100 |
-
checks.append({'check': 'series total ties to economics', 'metric': metric['key'], 'ok': bool(ok),
|
| 101 |
-
'ours': ours, 'independent': expected,
|
| 102 |
-
'reason': metric.get('reason') or source_reason})
|
| 103 |
-
return checks
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
def series(project, *, bounds, read, independent):
|
| 107 |
-
"""Publish the fixed chart order and project markers after independent economics reconciliation."""
|
| 108 |
-
first, last = date.fromisoformat(bounds['from']), date.fromisoformat(bounds['to'])
|
| 109 |
-
dates = [(first + timedelta(days=i)).isoformat() for i in range((last - first).days + 1)]
|
| 110 |
-
answer = {'projectId': project['id'], 'from': bounds['from'], 'to': bounds['to'],
|
| 111 |
-
'timezone': timezone_name(), 'boundary': SERIES_BOUNDARY,
|
| 112 |
-
'markers': {'start': project.get('start_on'), 'target': project.get('target_on'),
|
| 113 |
-
'completed': project.get('completed_on')}, 'metrics': []}
|
| 114 |
-
for key, label, unit in SERIES_METRICS:
|
| 115 |
-
measured = read(key)
|
| 116 |
-
total, reason = measured.get('total'), measured.get('reason')
|
| 117 |
-
if total is None and not reason:
|
| 118 |
-
reason = 'An independent daily source for this metric is unavailable.'
|
| 119 |
-
values = {point['date']: point['value'] for point in measured.get('points', [])} if total is not None else {}
|
| 120 |
-
answer['metrics'].append({'key': key, 'label': label, 'unit': measured.get('unit', unit),
|
| 121 |
-
'points': [{'date': day, 'value': values.get(day)} for day in dates],
|
| 122 |
-
'total': total, 'reason': reason,
|
| 123 |
-
'tie': {'ok': False, 'ours': None, 'independent': None}})
|
| 124 |
-
checks = validate(answer, independent=independent)
|
| 125 |
-
if any(not check['ok'] for check in checks):
|
| 126 |
-
raise ValidationFailed('A series total does not tie to its independent Amazon economics aggregate.')
|
| 127 |
-
for metric, check in zip(answer['metrics'], checks):
|
| 128 |
-
if metric['total'] is not None:
|
| 129 |
-
metric['tie'] = {'ok': True, 'ours': check['ours'], 'independent': check['independent']}
|
| 130 |
-
return answer
|
| 131 |
|
| 132 |
|
| 133 |
def configurations(values):
|
|
|
|
| 1 |
+
"""Typed project result configurations and tenant-day completion windows."""
|
| 2 |
|
| 3 |
from datetime import date, timedelta
|
| 4 |
import hashlib
|
|
|
|
| 13 |
GRAINS = ('sku', 'asin', 'parent_asin')
|
| 14 |
MAX_CONFIGS = 20
|
| 15 |
BOUNDARY = '[completedOn, endExclusive) in the tenant timezone. Today is excluded.'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
|
| 18 |
def configurations(values):
|
platform/core/amazon_projects.py
CHANGED
|
@@ -250,11 +250,10 @@ def health(project, today):
|
|
| 250 |
|
| 251 |
|
| 252 |
def summary(projects):
|
| 253 |
-
"""
|
| 254 |
counted = [p for p in projects if p.get('status') != CANCELED]
|
| 255 |
total = len(counted)
|
| 256 |
-
return {'average': round(sum(
|
| 257 |
-
for p in counted) / total) if total else None,
|
| 258 |
'completed': sum(1 for p in counted if p.get('status') == COMPLETED), 'total': total}
|
| 259 |
|
| 260 |
|
|
|
|
| 250 |
|
| 251 |
|
| 252 |
def summary(projects):
|
| 253 |
+
"""R19 and R67: average progress and "N of M completed", with Canceled left out of M."""
|
| 254 |
counted = [p for p in projects if p.get('status') != CANCELED]
|
| 255 |
total = len(counted)
|
| 256 |
+
return {'average': round(sum(p['progress'] for p in counted) / total) if total else None,
|
|
|
|
| 257 |
'completed': sum(1 for p in counted if p.get('status') == COMPLETED), 'total': total}
|
| 258 |
|
| 259 |
|
platform/core/connector_sync.py
CHANGED
|
@@ -162,23 +162,6 @@ def sync_not_before(provider, tenant):
|
|
| 162 |
return _SYNC_NOT_BEFORE.get((provider, tenant))
|
| 163 |
|
| 164 |
|
| 165 |
-
def _clear_sync_defer(provider, tenant):
|
| 166 |
-
"""Remove one obsolete provider hold under the desk lock, retaining explicit continuation waits."""
|
| 167 |
-
key = (provider, tenant)
|
| 168 |
-
held = _SYNC_NOT_BEFORE.pop(key, None)
|
| 169 |
-
receipt = _SYNC_REQUESTS.get(key)
|
| 170 |
-
if (held is not None and receipt and receipt["state"] == "queued"
|
| 171 |
-
and receipt.get("requestedBy") != "first load" and receipt.get("notBefore") == held):
|
| 172 |
-
receipt["notBefore"] = None
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
def clear_sync_defer(provider, tenant):
|
| 176 |
-
"""Release one provider hold after its producer adopts partition-specific waits."""
|
| 177 |
-
with _SYNC_REQUEST_LOCK:
|
| 178 |
-
_clear_sync_defer(provider, tenant)
|
| 179 |
-
_wake_sync_requests(time.time())
|
| 180 |
-
|
| 181 |
-
|
| 182 |
def register_request_listener(fn):
|
| 183 |
"""Call fn(receipt, connection_id) after each accepted ordinary request, outside the desk lock."""
|
| 184 |
if fn not in _REQUEST_LISTENERS:
|
|
@@ -238,13 +221,7 @@ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=Non
|
|
| 238 |
_wake_sync_requests(stamp)
|
| 239 |
return {**result, "accepted": accepted_id == identity, "duplicate": accepted_id != identity}
|
| 240 |
with _SYNC_REQUEST_LOCK:
|
| 241 |
-
if provider == "amazon_sp" and requested_by != "first load":
|
| 242 |
-
# Amazon's history floor belongs to owed tables, never a loaded-table refresh.
|
| 243 |
-
_clear_sync_defer(provider, tenant)
|
| 244 |
previous = _SYNC_REQUESTS.get((provider, tenant))
|
| 245 |
-
replaces_amazon_history = (provider == "amazon_sp" and previous
|
| 246 |
-
and previous.get("requestedBy") == "first load"
|
| 247 |
-
and requested_by != "first load")
|
| 248 |
if previous and previous["state"] in ("queued", "running"):
|
| 249 |
if not (previous["state"] == "queued" and previous.get("requestedBy") == "first load"
|
| 250 |
and requested_by != "first load"):
|
|
@@ -253,7 +230,7 @@ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=Non
|
|
| 253 |
"requestedAt": time.time() if now is None else now,
|
| 254 |
"notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0),
|
| 255 |
float(previous.get("notBefore") or 0) if previous
|
| 256 |
-
and previous["state"] == "queued"
|
| 257 |
"startedAt": None, "finishedAt": None, "detail": "",
|
| 258 |
"requestedBy": str(requested_by or ""), "accepted": True}
|
| 259 |
if requested_by == "first load" and previous and previous.get("firstLoad"):
|
|
@@ -494,9 +471,6 @@ def poll_remote_requests(rt, *, now=None):
|
|
| 494 |
if stamp - float(entry.get("finishedAt") or 0) > REMOTE_KEEP_SECONDS:
|
| 495 |
pruned.append(identity)
|
| 496 |
continue
|
| 497 |
-
manual_amazon = key[0] == "amazon_sp" and entry.get("requestedBy") != "first load"
|
| 498 |
-
if manual_amazon:
|
| 499 |
-
_clear_sync_defer(*key)
|
| 500 |
receipt = _SYNC_REQUESTS.get(key)
|
| 501 |
# Promote under the same lock as claims, so a continuation cannot win a manual hand-off race.
|
| 502 |
if (receipt and receipt["state"] == "queued" and receipt.get("requestedBy") == "first load"
|
|
@@ -504,7 +478,7 @@ def poll_remote_requests(rt, *, now=None):
|
|
| 504 |
receipt.update(id=identity, requestedBy=entry.get("requestedBy") or "",
|
| 505 |
requestedAt=entry["requestedAt"],
|
| 506 |
connectionId=entry.get("connectionId"),
|
| 507 |
-
notBefore=max(
|
| 508 |
float(entry.get("notBefore") or 0),
|
| 509 |
_SYNC_NOT_BEFORE.get(key, 0)) or None)
|
| 510 |
admitted.append(deepcopy(entry))
|
|
|
|
| 162 |
return _SYNC_NOT_BEFORE.get((provider, tenant))
|
| 163 |
|
| 164 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
def register_request_listener(fn):
|
| 166 |
"""Call fn(receipt, connection_id) after each accepted ordinary request, outside the desk lock."""
|
| 167 |
if fn not in _REQUEST_LISTENERS:
|
|
|
|
| 221 |
_wake_sync_requests(stamp)
|
| 222 |
return {**result, "accepted": accepted_id == identity, "duplicate": accepted_id != identity}
|
| 223 |
with _SYNC_REQUEST_LOCK:
|
|
|
|
|
|
|
|
|
|
| 224 |
previous = _SYNC_REQUESTS.get((provider, tenant))
|
|
|
|
|
|
|
|
|
|
| 225 |
if previous and previous["state"] in ("queued", "running"):
|
| 226 |
if not (previous["state"] == "queued" and previous.get("requestedBy") == "first load"
|
| 227 |
and requested_by != "first load"):
|
|
|
|
| 230 |
"requestedAt": time.time() if now is None else now,
|
| 231 |
"notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0),
|
| 232 |
float(previous.get("notBefore") or 0) if previous
|
| 233 |
+
and previous["state"] == "queued" else 0) or None,
|
| 234 |
"startedAt": None, "finishedAt": None, "detail": "",
|
| 235 |
"requestedBy": str(requested_by or ""), "accepted": True}
|
| 236 |
if requested_by == "first load" and previous and previous.get("firstLoad"):
|
|
|
|
| 471 |
if stamp - float(entry.get("finishedAt") or 0) > REMOTE_KEEP_SECONDS:
|
| 472 |
pruned.append(identity)
|
| 473 |
continue
|
|
|
|
|
|
|
|
|
|
| 474 |
receipt = _SYNC_REQUESTS.get(key)
|
| 475 |
# Promote under the same lock as claims, so a continuation cannot win a manual hand-off race.
|
| 476 |
if (receipt and receipt["state"] == "queued" and receipt.get("requestedBy") == "first load"
|
|
|
|
| 478 |
receipt.update(id=identity, requestedBy=entry.get("requestedBy") or "",
|
| 479 |
requestedAt=entry["requestedAt"],
|
| 480 |
connectionId=entry.get("connectionId"),
|
| 481 |
+
notBefore=max(float(receipt.get("notBefore") or 0),
|
| 482 |
float(entry.get("notBefore") or 0),
|
| 483 |
_SYNC_NOT_BEFORE.get(key, 0)) or None)
|
| 484 |
admitted.append(deepcopy(entry))
|
platform/core/external_access.py
CHANGED
|
@@ -11,8 +11,6 @@ from dataclasses import dataclass
|
|
| 11 |
from datetime import datetime, timezone
|
| 12 |
import hashlib
|
| 13 |
import hmac
|
| 14 |
-
import logging
|
| 15 |
-
import os
|
| 16 |
import secrets
|
| 17 |
import time
|
| 18 |
|
|
@@ -20,7 +18,6 @@ import core.users as users
|
|
| 20 |
from harness import runtime
|
| 21 |
|
| 22 |
|
| 23 |
-
_log = logging.getLogger(__name__)
|
| 24 |
BUCKET = "external_access"
|
| 25 |
ACCESS_READ = "read"
|
| 26 |
ACCESS_READWRITE = "readwrite"
|
|
@@ -60,10 +57,6 @@ class ExternalAccessError(RuntimeError):
|
|
| 60 |
class ExternalAuthError(ExternalAccessError):
|
| 61 |
"""A uniform authentication refusal that reveals no credential or tenant detail."""
|
| 62 |
|
| 63 |
-
def __init__(self, code: str, *, cause: str = "unknown_key"):
|
| 64 |
-
super().__init__(code)
|
| 65 |
-
self.cause = cause
|
| 66 |
-
|
| 67 |
|
| 68 |
def normalise_access(value, *, allow_refresh=False) -> str:
|
| 69 |
if value is None or value == "":
|
|
@@ -146,8 +139,6 @@ class VerifiedCredential:
|
|
| 146 |
access: str = ACCESS_READWRITE
|
| 147 |
databases: object = ALL_DATABASES
|
| 148 |
workbooks: tuple = ()
|
| 149 |
-
warning: str | None = None
|
| 150 |
-
origin_recorded: bool = False
|
| 151 |
|
| 152 |
|
| 153 |
def _now() -> str:
|
|
@@ -338,7 +329,6 @@ def mint(st, username: str, tenant: str, label: str, access=None, databases=None
|
|
| 338 |
"rotated_at": None,
|
| 339 |
"expires_at": expiry,
|
| 340 |
"test_connection": test_connection,
|
| 341 |
-
"minted_space": os.environ.get("SPACE_ID") or None,
|
| 342 |
}
|
| 343 |
book[key_id] = record
|
| 344 |
minted["record"] = record
|
|
@@ -434,7 +424,6 @@ def rotate(st, username: str, tenant: str, key_id: str) -> tuple[dict, str]:
|
|
| 434 |
record.update({
|
| 435 |
"salt": salt.hex(), "hash": _digest(salt, secret), "algo": ALGORITHM,
|
| 436 |
"rotated_at": _now(), "last_used": None,
|
| 437 |
-
"minted_space": os.environ.get("SPACE_ID") or None,
|
| 438 |
})
|
| 439 |
book[target] = record
|
| 440 |
rotated["record"] = record
|
|
@@ -525,63 +514,31 @@ def _comparison_material(record) -> tuple[bool, bytes, str]:
|
|
| 525 |
return False, _DUMMY_SALT, _DUMMY_HASH
|
| 526 |
|
| 527 |
|
| 528 |
-
def
|
| 529 |
-
"""Treat missing or malformed origin metadata as unproved."""
|
| 530 |
-
value = record.get("minted_space")
|
| 531 |
-
return value if isinstance(value, str) and value.strip() else None
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
def _credential_cause(record, tenant: str, key_id: str, secret: str, *, owner=None):
|
| 535 |
-
"""Inspect status only after the secret and optional session owner match the stored record."""
|
| 536 |
-
material_ok, salt, expected = _comparison_material(record)
|
| 537 |
-
matched = hmac.compare_digest(_digest(salt, secret), expected)
|
| 538 |
-
if (not material_ok or not matched or record.get("id") != key_id
|
| 539 |
-
or record.get("tenant") != tenant or not record.get("username")
|
| 540 |
-
or (owner is not None and record.get("username") != owner)):
|
| 541 |
-
return "unknown_key"
|
| 542 |
-
origin, current = _recorded_space(record), os.environ.get("SPACE_ID")
|
| 543 |
-
if origin and current and origin != current:
|
| 544 |
-
return "minted_elsewhere"
|
| 545 |
-
if record.get("revoked_at"):
|
| 546 |
-
return "revoked"
|
| 547 |
-
if record.get("expires_at"):
|
| 548 |
-
expiry = _epoch(record["expires_at"])
|
| 549 |
-
if expiry is None or expiry <= time.time():
|
| 550 |
-
return "expired"
|
| 551 |
-
return None
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
def verify_for_owner(st, username: str, tenant: str, token: str) -> dict:
|
| 555 |
-
"""Diagnose only a session owner's secret inside that session's tenant runtime."""
|
| 556 |
-
owner, tenant_key = _normalise_owner(username, tenant)
|
| 557 |
-
_assert_bound_runtime(st, tenant_key)
|
| 558 |
-
try:
|
| 559 |
-
verified = verify(token, owner_runtime=st, owner=owner)
|
| 560 |
-
except ExternalAuthError as exc:
|
| 561 |
-
return {"valid": False, "cause": exc.cause, "warning": None, "origin_recorded": None}
|
| 562 |
-
return {"valid": True, "cause": None, "warning": verified.warning,
|
| 563 |
-
"origin_recorded": verified.origin_recorded, "key_id": verified.key_id}
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
def verify(token: str, *, owner_runtime=None, owner=None) -> VerifiedCredential:
|
| 567 |
"""Verify a bearer token from fresh state and resolve its ordinary public-user principal."""
|
| 568 |
try:
|
| 569 |
tenant_key, key_id, secret = _parse_token(token)
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
|
| 574 |
-
else:
|
| 575 |
-
try:
|
| 576 |
-
st = runtime.get_runtime(tenant_key)
|
| 577 |
-
except KeyError as exc:
|
| 578 |
-
raise ExternalAuthError(AUTH_ERROR) from exc
|
| 579 |
|
| 580 |
book = _book(st)
|
| 581 |
record = book.get(key_id)
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 585 |
|
| 586 |
owner = str(record["username"]).strip().lower()
|
| 587 |
try:
|
|
@@ -592,8 +549,7 @@ def verify(token: str, *, owner_runtime=None, owner=None) -> VerifiedCredential:
|
|
| 592 |
access, databases = scope_of(record)
|
| 593 |
if not _stamp_is_due(record.get("last_used"), time.time()):
|
| 594 |
return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st,
|
| 595 |
-
access=access, databases=databases, workbooks=workbooks_of(record)
|
| 596 |
-
origin_recorded=bool(_recorded_space(record)))
|
| 597 |
|
| 598 |
used_at = _now()
|
| 599 |
|
|
@@ -601,32 +557,26 @@ def verify(token: str, *, owner_runtime=None, owner=None) -> VerifiedCredential:
|
|
| 601 |
if not isinstance(value, dict):
|
| 602 |
raise ExternalAuthError(AUTH_ERROR)
|
| 603 |
current = value.get(key_id)
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
|
|
|
|
|
|
| 607 |
book_now = dict(value)
|
| 608 |
stamped = dict(current)
|
| 609 |
stamped["last_used"] = used_at
|
| 610 |
book_now[key_id] = stamped
|
| 611 |
return book_now
|
| 612 |
|
| 613 |
-
warning = None
|
| 614 |
try:
|
| 615 |
st.update(BUCKET, _stamp, flush="sync")
|
| 616 |
except ExternalAuthError:
|
| 617 |
raise
|
| 618 |
except Exception as exc:
|
| 619 |
-
|
| 620 |
-
_log.warning("external_access last_used stamp failed: %s", type(exc).__name__)
|
| 621 |
-
warning = "store_write_refused"
|
| 622 |
return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st,
|
| 623 |
-
access=access, databases=databases, workbooks=workbooks_of(record)
|
| 624 |
-
warning=warning, origin_recorded=bool(_recorded_space(record)))
|
| 625 |
except ExternalAuthError:
|
| 626 |
raise
|
| 627 |
-
except ExternalAccessError as exc:
|
| 628 |
-
if owner_runtime is not None:
|
| 629 |
-
raise
|
| 630 |
-
raise ExternalAuthError(AUTH_ERROR) from exc
|
| 631 |
except Exception as exc:
|
| 632 |
raise ExternalAuthError(AUTH_ERROR) from exc
|
|
|
|
| 11 |
from datetime import datetime, timezone
|
| 12 |
import hashlib
|
| 13 |
import hmac
|
|
|
|
|
|
|
| 14 |
import secrets
|
| 15 |
import time
|
| 16 |
|
|
|
|
| 18 |
from harness import runtime
|
| 19 |
|
| 20 |
|
|
|
|
| 21 |
BUCKET = "external_access"
|
| 22 |
ACCESS_READ = "read"
|
| 23 |
ACCESS_READWRITE = "readwrite"
|
|
|
|
| 57 |
class ExternalAuthError(ExternalAccessError):
|
| 58 |
"""A uniform authentication refusal that reveals no credential or tenant detail."""
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
|
| 61 |
def normalise_access(value, *, allow_refresh=False) -> str:
|
| 62 |
if value is None or value == "":
|
|
|
|
| 139 |
access: str = ACCESS_READWRITE
|
| 140 |
databases: object = ALL_DATABASES
|
| 141 |
workbooks: tuple = ()
|
|
|
|
|
|
|
| 142 |
|
| 143 |
|
| 144 |
def _now() -> str:
|
|
|
|
| 329 |
"rotated_at": None,
|
| 330 |
"expires_at": expiry,
|
| 331 |
"test_connection": test_connection,
|
|
|
|
| 332 |
}
|
| 333 |
book[key_id] = record
|
| 334 |
minted["record"] = record
|
|
|
|
| 424 |
record.update({
|
| 425 |
"salt": salt.hex(), "hash": _digest(salt, secret), "algo": ALGORITHM,
|
| 426 |
"rotated_at": _now(), "last_used": None,
|
|
|
|
| 427 |
})
|
| 428 |
book[target] = record
|
| 429 |
rotated["record"] = record
|
|
|
|
| 514 |
return False, _DUMMY_SALT, _DUMMY_HASH
|
| 515 |
|
| 516 |
|
| 517 |
+
def verify(token: str) -> VerifiedCredential:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 518 |
"""Verify a bearer token from fresh state and resolve its ordinary public-user principal."""
|
| 519 |
try:
|
| 520 |
tenant_key, key_id, secret = _parse_token(token)
|
| 521 |
+
try:
|
| 522 |
+
st = runtime.get_runtime(tenant_key)
|
| 523 |
+
except KeyError as exc:
|
| 524 |
+
raise ExternalAuthError(AUTH_ERROR) from exc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 525 |
|
| 526 |
book = _book(st)
|
| 527 |
record = book.get(key_id)
|
| 528 |
+
material_ok, salt, expected = _comparison_material(record)
|
| 529 |
+
matched = hmac.compare_digest(_digest(salt, secret), expected)
|
| 530 |
+
record_ok = (
|
| 531 |
+
material_ok
|
| 532 |
+
and isinstance(record, dict)
|
| 533 |
+
and record.get("id") == key_id
|
| 534 |
+
and record.get("tenant") == tenant_key
|
| 535 |
+
and bool(record.get("username"))
|
| 536 |
+
and not record.get("revoked_at")
|
| 537 |
+
and (_epoch(record.get("expires_at")) is None
|
| 538 |
+
if not record.get("expires_at") else _epoch(record.get("expires_at")) > time.time())
|
| 539 |
+
)
|
| 540 |
+
if not record_ok or not matched:
|
| 541 |
+
raise ExternalAuthError(AUTH_ERROR)
|
| 542 |
|
| 543 |
owner = str(record["username"]).strip().lower()
|
| 544 |
try:
|
|
|
|
| 549 |
access, databases = scope_of(record)
|
| 550 |
if not _stamp_is_due(record.get("last_used"), time.time()):
|
| 551 |
return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st,
|
| 552 |
+
access=access, databases=databases, workbooks=workbooks_of(record))
|
|
|
|
| 553 |
|
| 554 |
used_at = _now()
|
| 555 |
|
|
|
|
| 557 |
if not isinstance(value, dict):
|
| 558 |
raise ExternalAuthError(AUTH_ERROR)
|
| 559 |
current = value.get(key_id)
|
| 560 |
+
if (not isinstance(current, dict) or current.get("revoked_at")
|
| 561 |
+
or current.get("username") != owner
|
| 562 |
+
or current.get("tenant") != tenant_key
|
| 563 |
+
or current.get("hash") != record.get("hash")):
|
| 564 |
+
raise ExternalAuthError(AUTH_ERROR)
|
| 565 |
book_now = dict(value)
|
| 566 |
stamped = dict(current)
|
| 567 |
stamped["last_used"] = used_at
|
| 568 |
book_now[key_id] = stamped
|
| 569 |
return book_now
|
| 570 |
|
|
|
|
| 571 |
try:
|
| 572 |
st.update(BUCKET, _stamp, flush="sync")
|
| 573 |
except ExternalAuthError:
|
| 574 |
raise
|
| 575 |
except Exception as exc:
|
| 576 |
+
raise ExternalAuthError(AUTH_ERROR) from exc
|
|
|
|
|
|
|
| 577 |
return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st,
|
| 578 |
+
access=access, databases=databases, workbooks=workbooks_of(record))
|
|
|
|
| 579 |
except ExternalAuthError:
|
| 580 |
raise
|
|
|
|
|
|
|
|
|
|
|
|
|
| 581 |
except Exception as exc:
|
| 582 |
raise ExternalAuthError(AUTH_ERROR) from exc
|
platform/core/notify_destinations.py
CHANGED
|
@@ -3,7 +3,6 @@
|
|
| 3 |
from contextlib import contextmanager
|
| 4 |
from datetime import datetime, timezone
|
| 5 |
import hashlib
|
| 6 |
-
import ipaddress
|
| 7 |
import json
|
| 8 |
import re
|
| 9 |
from typing import NotRequired, TypedDict
|
|
@@ -24,7 +23,6 @@ PAYLOAD_VERSION = 1
|
|
| 24 |
PAGE_SIZE = 100
|
| 25 |
_CHANNEL = re.compile(r'[CG][A-Z0-9]{1,79}\Z')
|
| 26 |
_EMAIL = re.compile(r'[^\s@,;<>]+@[^\s@,;<>]+\.[^\s@,;<>]+\Z')
|
| 27 |
-
_WEBHOOK = re.compile(r'k_[0-9a-f]{12}\Z')
|
| 28 |
|
| 29 |
|
| 30 |
class NotificationSource(TypedDict):
|
|
@@ -75,9 +73,9 @@ def email_address(value):
|
|
| 75 |
|
| 76 |
|
| 77 |
def clean_patch(patch, *, replace=False):
|
| 78 |
-
if not isinstance(patch, dict) or set(patch) - {'slackChannels', 'email'
|
| 79 |
-
raise ValueError('Only slackChannels
|
| 80 |
-
out = {'slackChannels': [], 'email': None
|
| 81 |
if 'slackChannels' in patch:
|
| 82 |
channels = patch['slackChannels']
|
| 83 |
if (not isinstance(channels, list)
|
|
@@ -86,49 +84,9 @@ def clean_patch(patch, *, replace=False):
|
|
| 86 |
out['slackChannels'] = list(dict.fromkeys(channels))
|
| 87 |
if 'email' in patch:
|
| 88 |
out['email'] = None if patch['email'] is None else email_address(patch['email'])
|
| 89 |
-
if 'inApp' in patch:
|
| 90 |
-
if type(patch['inApp']) is not bool:
|
| 91 |
-
raise ValueError('Use a boolean in-app preference.')
|
| 92 |
-
out['inApp'] = patch['inApp']
|
| 93 |
-
if 'webhooks' in patch:
|
| 94 |
-
hooks = patch['webhooks']
|
| 95 |
-
if (not isinstance(hooks, list)
|
| 96 |
-
or any(not isinstance(hook, str) or not _WEBHOOK.fullmatch(hook) for hook in hooks)):
|
| 97 |
-
raise ValueError('Use a list of saved webhook ids.')
|
| 98 |
-
out['webhooks'] = list(dict.fromkeys(hooks))
|
| 99 |
return out
|
| 100 |
|
| 101 |
|
| 102 |
-
def webhook_fields(rt, user, identity):
|
| 103 |
-
"""Resolve an owner-bound saved alert webhook without accepting inline credentials."""
|
| 104 |
-
from core import keychain
|
| 105 |
-
_backend(rt)
|
| 106 |
-
user = _text(user, 'User')
|
| 107 |
-
if not isinstance(identity, str) or not _WEBHOOK.fullmatch(identity):
|
| 108 |
-
raise ValueError('Choose a saved webhook id.')
|
| 109 |
-
entry = next((entry for entry in keychain.list_entries(rt, 'generic')
|
| 110 |
-
if entry['id'] == identity and entry.get('createdBy') == user), None)
|
| 111 |
-
if entry is None:
|
| 112 |
-
raise ValueError('Choose one of your configured webhooks.')
|
| 113 |
-
fields = keychain.read_fields(rt, identity)
|
| 114 |
-
if not isinstance(fields, dict) or fields.get('purpose') != 'alert_webhook':
|
| 115 |
-
raise ValueError('Choose a saved alert webhook.')
|
| 116 |
-
url = _text(fields.get('url'), 'Webhook URL', 2000)
|
| 117 |
-
parsed = urlsplit(url)
|
| 118 |
-
host = (parsed.hostname or '').rstrip('.').lower()
|
| 119 |
-
if (parsed.scheme != 'https' or not host or parsed.username or parsed.password
|
| 120 |
-
or parsed.fragment or parsed.port not in (None, 443)
|
| 121 |
-
or '.' not in host or host.endswith(('.localhost', '.local', '.internal'))):
|
| 122 |
-
raise ValueError('The webhook requires a public HTTPS URL.')
|
| 123 |
-
try:
|
| 124 |
-
address = ipaddress.ip_address(host)
|
| 125 |
-
except ValueError:
|
| 126 |
-
address = None
|
| 127 |
-
if address is not None and not address.is_global:
|
| 128 |
-
raise ValueError('The webhook requires a public HTTPS URL.')
|
| 129 |
-
return fields
|
| 130 |
-
|
| 131 |
-
|
| 132 |
def clean_items(items):
|
| 133 |
if not isinstance(items, list) or not items:
|
| 134 |
raise ValueError('Supply at least one notification item.')
|
|
@@ -191,8 +149,7 @@ def seed_definition(tenant, user, kind, channel):
|
|
| 191 |
|
| 192 |
def empty_row(tenant, user, kind):
|
| 193 |
return {'tenant': tenant, 'user': user, 'kind': kind, 'slackChannels': [],
|
| 194 |
-
'email': None, '
|
| 195 |
-
'revision': 0, 'delivery': {}, 'slackAutomations': {}}
|
| 196 |
|
| 197 |
|
| 198 |
def _backend(rt):
|
|
@@ -222,14 +179,7 @@ def preferences(rt, user):
|
|
| 222 |
|
| 223 |
|
| 224 |
def edit(rt, user, kind, patch, *, replace=False):
|
| 225 |
-
|
| 226 |
-
patch = clean_patch(patch, replace=replace)
|
| 227 |
-
prior = backend.read(user, kind) or empty_row(rt.key, user, kind)
|
| 228 |
-
for identity in patch.get('webhooks', []):
|
| 229 |
-
# Existing picks remain removable while their configured destination is unavailable.
|
| 230 |
-
if identity not in prior.get('webhooks', []):
|
| 231 |
-
webhook_fields(rt, user, identity)
|
| 232 |
-
return backend.edit(user, kind, patch)
|
| 233 |
|
| 234 |
|
| 235 |
def recipients(rt, kind, *, user=None):
|
|
@@ -249,14 +199,8 @@ def recipients(rt, kind, *, user=None):
|
|
| 249 |
|
| 250 |
|
| 251 |
def selected(row, destination):
|
| 252 |
-
if not isinstance(destination, str):
|
| 253 |
-
return False
|
| 254 |
if destination.startswith('slack:'):
|
| 255 |
return destination[6:] in row['slackChannels']
|
| 256 |
-
if destination.startswith('in-app:'):
|
| 257 |
-
return row.get('inApp') is True and destination[7:] == row['user']
|
| 258 |
-
if destination.startswith('webhook:'):
|
| 259 |
-
return destination[8:] in row.get('webhooks', [])
|
| 260 |
return destination.startswith('email:') and destination[6:] == row['email']
|
| 261 |
|
| 262 |
|
|
@@ -288,7 +232,6 @@ class PgRows:
|
|
| 288 |
raise ValueError('Invalid tenant schema.')
|
| 289 |
self.table = f'"{schema}"."notification_destinations"'
|
| 290 |
self.seeds = f'"{schema}"."notification_slack_automations"'
|
| 291 |
-
self.picks = f'"{schema}"."notification_channel_preferences"'
|
| 292 |
|
| 293 |
@contextmanager
|
| 294 |
def _connection(self, *, write=False):
|
|
@@ -310,7 +253,7 @@ class PgRows:
|
|
| 310 |
return con.execute('SELECT to_regclass(%s)', (self.table,)).fetchone()[0] is not None
|
| 311 |
|
| 312 |
def _ensure(self, con):
|
| 313 |
-
if self._exists(con)
|
| 314 |
return
|
| 315 |
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
|
| 316 |
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} (
|
|
@@ -323,42 +266,23 @@ class PgRows:
|
|
| 323 |
id TEXT PRIMARY KEY, user_id TEXT NOT NULL, kind TEXT NOT NULL, channel TEXT NOT NULL,
|
| 324 |
FOREIGN KEY (user_id, kind) REFERENCES {self.table} (user_id, kind) ON DELETE CASCADE,
|
| 325 |
UNIQUE (user_id, kind, channel))''')
|
| 326 |
-
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.picks} (
|
| 327 |
-
user_id TEXT NOT NULL, kind TEXT NOT NULL, in_app BOOLEAN NOT NULL DEFAULT FALSE,
|
| 328 |
-
webhooks JSONB NOT NULL DEFAULT '[]', PRIMARY KEY (user_id, kind),
|
| 329 |
-
FOREIGN KEY (user_id, kind) REFERENCES {self.table} (user_id, kind) ON DELETE CASCADE)''')
|
| 330 |
-
|
| 331 |
-
def _picks_exist(self, con):
|
| 332 |
-
return con.execute('SELECT to_regclass(%s)', (self.picks,)).fetchone()[0] is not None
|
| 333 |
-
|
| 334 |
-
def _projection(self, con):
|
| 335 |
-
base = f'user_id, kind, slack_channels, email, revision, delivery FROM {self.table}'
|
| 336 |
-
if not self._picks_exist(con):
|
| 337 |
-
return 'SELECT ' + base, ''
|
| 338 |
-
return (f'SELECT n.user_id, n.kind, n.slack_channels, n.email, n.revision, n.delivery, '
|
| 339 |
-
f'COALESCE(p.in_app, FALSE), COALESCE(p.webhooks, \'[]\') FROM {self.table} n '
|
| 340 |
-
f'LEFT JOIN {self.picks} p ON p.user_id = n.user_id AND p.kind = n.kind', 'n.')
|
| 341 |
|
| 342 |
def _row(self, values):
|
| 343 |
if values is None:
|
| 344 |
return None
|
| 345 |
-
user, kind, channels, email, revision, delivery = values
|
| 346 |
-
in_app, webhooks = values[6:] if len(values) == 8 else (False, [])
|
| 347 |
-
webhooks = json.loads(webhooks) if isinstance(webhooks, str) else webhooks
|
| 348 |
channels = json.loads(channels) if isinstance(channels, str) else channels
|
| 349 |
delivery = json.loads(delivery) if isinstance(delivery, str) else delivery
|
| 350 |
row = {**empty_row(self.tenant_slug, user, kind), 'slackChannels': channels,
|
| 351 |
-
'email': email, '
|
| 352 |
row['delivery'] = {key: state for key, state in delivery.items() if selected(row, key)}
|
| 353 |
row['slackAutomations'] = {c: automation_id(self.tenant_slug, user, kind, c) for c in channels}
|
| 354 |
return row
|
| 355 |
|
| 356 |
def _read(self, con, user, kind, *, lock=False):
|
| 357 |
-
projection, prefix = self._projection(con)
|
| 358 |
return self._row(con.execute(
|
| 359 |
-
f'
|
| 360 |
-
|
| 361 |
-
(user, kind)).fetchone())
|
| 362 |
|
| 363 |
def read(self, user, kind):
|
| 364 |
with self._connection() as con:
|
|
@@ -368,9 +292,9 @@ class PgRows:
|
|
| 368 |
with self._connection() as con:
|
| 369 |
if not self._exists(con):
|
| 370 |
return []
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
return [self._row(row) for row in rows]
|
| 375 |
|
| 376 |
def edit(self, user, kind, patch):
|
|
@@ -383,9 +307,6 @@ class PgRows:
|
|
| 383 |
con.execute(f'UPDATE {self.table} SET slack_channels = %s::jsonb, email = %s, '
|
| 384 |
'revision = revision + 1, delivery = %s::jsonb WHERE user_id = %s AND kind = %s',
|
| 385 |
(json.dumps(current['slackChannels']), current['email'], json.dumps(kept), user, kind))
|
| 386 |
-
con.execute(f'INSERT INTO {self.picks} (user_id, kind, in_app, webhooks) VALUES (%s, %s, %s, %s::jsonb) '
|
| 387 |
-
'ON CONFLICT (user_id, kind) DO UPDATE SET in_app = EXCLUDED.in_app, webhooks = EXCLUDED.webhooks',
|
| 388 |
-
(user, kind, current['inApp'], json.dumps(current['webhooks'])))
|
| 389 |
con.execute(f'DELETE FROM {self.seeds} WHERE user_id = %s AND kind = %s '
|
| 390 |
'AND NOT (channel = ANY(%s::text[]))', (user, kind, current['slackChannels']))
|
| 391 |
for channel in current['slackChannels']:
|
|
|
|
| 3 |
from contextlib import contextmanager
|
| 4 |
from datetime import datetime, timezone
|
| 5 |
import hashlib
|
|
|
|
| 6 |
import json
|
| 7 |
import re
|
| 8 |
from typing import NotRequired, TypedDict
|
|
|
|
| 23 |
PAGE_SIZE = 100
|
| 24 |
_CHANNEL = re.compile(r'[CG][A-Z0-9]{1,79}\Z')
|
| 25 |
_EMAIL = re.compile(r'[^\s@,;<>]+@[^\s@,;<>]+\.[^\s@,;<>]+\Z')
|
|
|
|
| 26 |
|
| 27 |
|
| 28 |
class NotificationSource(TypedDict):
|
|
|
|
| 73 |
|
| 74 |
|
| 75 |
def clean_patch(patch, *, replace=False):
|
| 76 |
+
if not isinstance(patch, dict) or set(patch) - {'slackChannels', 'email'}:
|
| 77 |
+
raise ValueError('Only slackChannels and email may be changed.')
|
| 78 |
+
out = {'slackChannels': [], 'email': None} if replace else {}
|
| 79 |
if 'slackChannels' in patch:
|
| 80 |
channels = patch['slackChannels']
|
| 81 |
if (not isinstance(channels, list)
|
|
|
|
| 84 |
out['slackChannels'] = list(dict.fromkeys(channels))
|
| 85 |
if 'email' in patch:
|
| 86 |
out['email'] = None if patch['email'] is None else email_address(patch['email'])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
return out
|
| 88 |
|
| 89 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
def clean_items(items):
|
| 91 |
if not isinstance(items, list) or not items:
|
| 92 |
raise ValueError('Supply at least one notification item.')
|
|
|
|
| 149 |
|
| 150 |
def empty_row(tenant, user, kind):
|
| 151 |
return {'tenant': tenant, 'user': user, 'kind': kind, 'slackChannels': [],
|
| 152 |
+
'email': None, 'revision': 0, 'delivery': {}, 'slackAutomations': {}}
|
|
|
|
| 153 |
|
| 154 |
|
| 155 |
def _backend(rt):
|
|
|
|
| 179 |
|
| 180 |
|
| 181 |
def edit(rt, user, kind, patch, *, replace=False):
|
| 182 |
+
return _backend(rt).edit(_text(user, 'User'), kind_id(kind), clean_patch(patch, replace=replace))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
|
| 184 |
|
| 185 |
def recipients(rt, kind, *, user=None):
|
|
|
|
| 199 |
|
| 200 |
|
| 201 |
def selected(row, destination):
|
|
|
|
|
|
|
| 202 |
if destination.startswith('slack:'):
|
| 203 |
return destination[6:] in row['slackChannels']
|
|
|
|
|
|
|
|
|
|
|
|
|
| 204 |
return destination.startswith('email:') and destination[6:] == row['email']
|
| 205 |
|
| 206 |
|
|
|
|
| 232 |
raise ValueError('Invalid tenant schema.')
|
| 233 |
self.table = f'"{schema}"."notification_destinations"'
|
| 234 |
self.seeds = f'"{schema}"."notification_slack_automations"'
|
|
|
|
| 235 |
|
| 236 |
@contextmanager
|
| 237 |
def _connection(self, *, write=False):
|
|
|
|
| 253 |
return con.execute('SELECT to_regclass(%s)', (self.table,)).fetchone()[0] is not None
|
| 254 |
|
| 255 |
def _ensure(self, con):
|
| 256 |
+
if self._exists(con):
|
| 257 |
return
|
| 258 |
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
|
| 259 |
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} (
|
|
|
|
| 266 |
id TEXT PRIMARY KEY, user_id TEXT NOT NULL, kind TEXT NOT NULL, channel TEXT NOT NULL,
|
| 267 |
FOREIGN KEY (user_id, kind) REFERENCES {self.table} (user_id, kind) ON DELETE CASCADE,
|
| 268 |
UNIQUE (user_id, kind, channel))''')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
|
| 270 |
def _row(self, values):
|
| 271 |
if values is None:
|
| 272 |
return None
|
| 273 |
+
user, kind, channels, email, revision, delivery = values
|
|
|
|
|
|
|
| 274 |
channels = json.loads(channels) if isinstance(channels, str) else channels
|
| 275 |
delivery = json.loads(delivery) if isinstance(delivery, str) else delivery
|
| 276 |
row = {**empty_row(self.tenant_slug, user, kind), 'slackChannels': channels,
|
| 277 |
+
'email': email, 'revision': revision}
|
| 278 |
row['delivery'] = {key: state for key, state in delivery.items() if selected(row, key)}
|
| 279 |
row['slackAutomations'] = {c: automation_id(self.tenant_slug, user, kind, c) for c in channels}
|
| 280 |
return row
|
| 281 |
|
| 282 |
def _read(self, con, user, kind, *, lock=False):
|
|
|
|
| 283 |
return self._row(con.execute(
|
| 284 |
+
f'SELECT user_id, kind, slack_channels, email, revision, delivery FROM {self.table} '
|
| 285 |
+
'WHERE user_id = %s AND kind = %s' + (' FOR UPDATE' if lock else ''), (user, kind)).fetchone())
|
|
|
|
| 286 |
|
| 287 |
def read(self, user, kind):
|
| 288 |
with self._connection() as con:
|
|
|
|
| 292 |
with self._connection() as con:
|
| 293 |
if not self._exists(con):
|
| 294 |
return []
|
| 295 |
+
rows = con.execute(f'SELECT user_id, kind, slack_channels, email, revision, delivery '
|
| 296 |
+
f'FROM {self.table} WHERE kind = %s AND user_id > %s '
|
| 297 |
+
'ORDER BY user_id LIMIT %s', (kind, after, limit)).fetchall()
|
| 298 |
return [self._row(row) for row in rows]
|
| 299 |
|
| 300 |
def edit(self, user, kind, patch):
|
|
|
|
| 307 |
con.execute(f'UPDATE {self.table} SET slack_channels = %s::jsonb, email = %s, '
|
| 308 |
'revision = revision + 1, delivery = %s::jsonb WHERE user_id = %s AND kind = %s',
|
| 309 |
(json.dumps(current['slackChannels']), current['email'], json.dumps(kept), user, kind))
|
|
|
|
|
|
|
|
|
|
| 310 |
con.execute(f'DELETE FROM {self.seeds} WHERE user_id = %s AND kind = %s '
|
| 311 |
'AND NOT (channel = ANY(%s::text[]))', (user, kind, current['slackChannels']))
|
| 312 |
for channel in current['slackChannels']:
|
platform/core/sheets.py
CHANGED
|
@@ -2477,10 +2477,9 @@ def _publish_cell_activity(st, events):
|
|
| 2477 |
for event in events:
|
| 2478 |
try:
|
| 2479 |
activity_log.append(st, **event)
|
| 2480 |
-
except Exception
|
| 2481 |
# A notification failure must not present a committed save as a failed edit.
|
| 2482 |
-
logging.getLogger(__name__).error("Sheet activity notification failed after persistence
|
| 2483 |
-
type(exc).__name__)
|
| 2484 |
|
| 2485 |
|
| 2486 |
def _read_workbook_doc(st, key):
|
|
|
|
| 2477 |
for event in events:
|
| 2478 |
try:
|
| 2479 |
activity_log.append(st, **event)
|
| 2480 |
+
except Exception:
|
| 2481 |
# A notification failure must not present a committed save as a failed edit.
|
| 2482 |
+
logging.getLogger(__name__).error("Sheet activity notification failed after persistence")
|
|
|
|
| 2483 |
|
| 2484 |
|
| 2485 |
def _read_workbook_doc(st, key):
|
platform/core/table_store.py
CHANGED
|
@@ -105,9 +105,10 @@ TRASH_RETENTION_SECONDS = TRASH_RETENTION_DAYS * 24 * 60 * 60
|
|
| 105 |
# browser re-reads only after some writes. So "stamp differs" alone refuses one tab's own second
|
| 106 |
# write, and the outbox then resends that refused write forever.
|
| 107 |
#
|
| 108 |
-
#
|
| 109 |
-
#
|
| 110 |
-
#
|
|
|
|
| 111 |
#
|
| 112 |
# ⛔ The record lives in the owner's own stratum beside `views`, so it rides the same update and
|
| 113 |
# the same settings row. `workspace()` never projects it.
|
|
@@ -120,7 +121,7 @@ TRASH_RETENTION_SECONDS = TRASH_RETENTION_DAYS * 24 * 60 * 60
|
|
| 120 |
VIEW_WRITES_KEY = 'viewWrites'
|
| 121 |
# Folder ids are surface-scoped and keep history separate from view ids.
|
| 122 |
FOLDER_WRITES_KEY = 'folderWrites'
|
| 123 |
-
#: Writes kept per view.
|
| 124 |
VIEW_WRITES_KEEP = 8
|
| 125 |
#: Views kept per stratum. The least recently written record goes first.
|
| 126 |
VIEW_WRITES_VIEWS = 400
|
|
@@ -149,7 +150,7 @@ def view_members(view):
|
|
| 149 |
return out
|
| 150 |
|
| 151 |
|
| 152 |
-
def _note_write(ws, view_id, at, kind, before, book_key
|
| 153 |
"""Append one write to the selected bounded history in this owner's stratum."""
|
| 154 |
if at is None or not view_id or not isinstance(ws, dict):
|
| 155 |
return
|
|
@@ -158,21 +159,11 @@ def _note_write(ws, view_id, at, kind, before, book_key, *, author=None):
|
|
| 158 |
book = ws[book_key] = {}
|
| 159 |
entry = book.get(view_id) if isinstance(book.get(view_id), dict) else {}
|
| 160 |
log = [row for row in entry.get('log') or () if isinstance(row, dict)]
|
| 161 |
-
|
| 162 |
-
log.append({'rev': int(at), 'kind': kind, 'before': before, 'author': author})
|
| 163 |
since = int(entry.get('since') or 0)
|
| 164 |
-
trimmed = entry.get('trimmed')
|
| 165 |
-
trimmed = dict(trimmed) if isinstance(trimmed, dict) else {}
|
| 166 |
while len(log) > VIEW_WRITES_KEEP:
|
| 167 |
-
|
| 168 |
-
removed_author = removed.get('author')
|
| 169 |
-
# Only a known, uninterrupted author run can excuse writes outside the ring.
|
| 170 |
-
if not removed_author or removed_author != trimmed.get('author'):
|
| 171 |
-
trimmed = {'author': removed_author, 'after': since}
|
| 172 |
-
since = max(since, int(removed.get('rev') or 0))
|
| 173 |
book[view_id] = {'since': since, 'log': log}
|
| 174 |
-
if trimmed:
|
| 175 |
-
book[view_id]['trimmed'] = trimmed
|
| 176 |
if len(book) > VIEW_WRITES_VIEWS:
|
| 177 |
# jsonb keeps no key order, so recency is read off each record's newest revision.
|
| 178 |
def newest(vid):
|
|
@@ -183,9 +174,9 @@ def _note_write(ws, view_id, at, kind, before, book_key, *, author=None):
|
|
| 183 |
book.pop(vid, None)
|
| 184 |
|
| 185 |
|
| 186 |
-
def note_view_write(ws, view_id, at, kind, before
|
| 187 |
"""Append one view, gone or place write to this stratum's record."""
|
| 188 |
-
_note_write(ws, view_id, at, kind, before, VIEW_WRITES_KEY
|
| 189 |
|
| 190 |
|
| 191 |
def _later_writes(ws, view_id, sent, kinds, *, book_key=VIEW_WRITES_KEY):
|
|
@@ -200,31 +191,25 @@ def _later_writes(ws, view_id, sent, kinds, *, book_key=VIEW_WRITES_KEY):
|
|
| 200 |
return later, int(sent) >= int(entry.get('since') or 0)
|
| 201 |
|
| 202 |
|
| 203 |
-
def stale_view_conflict(ws, view_id, mine, sent
|
| 204 |
"""Whether saving `mine` over `view_id` undoes a change made after revision `sent`."""
|
| 205 |
later, judgeable = _later_writes(ws, view_id, sent, ('view', 'gone'))
|
| 206 |
-
|
|
|
|
| 207 |
if not judgeable:
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
return
|
| 214 |
current = ((ws.get('views') or {}).get(view_id))
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
before = row.get('before') or {}
|
| 222 |
-
after = (later[index + 1].get('before') or {}) if index + 1 < len(later) else now
|
| 223 |
-
# The next before-image isolates this writer's changes from the requester's later edits.
|
| 224 |
-
if any(before.get(member) != after.get(member) and want.get(member) != now.get(member)
|
| 225 |
-
for member in set(before) | set(after)):
|
| 226 |
-
return True
|
| 227 |
-
return False
|
| 228 |
|
| 229 |
|
| 230 |
def stale_place_conflict(ws, view_id, folder, sent):
|
|
@@ -1406,15 +1391,14 @@ class TableStore:
|
|
| 1406 |
for folder_id in sorted(set(before_parents) | set(parents)):
|
| 1407 |
if before_parents.get(folder_id) != parents.get(folder_id):
|
| 1408 |
_note_write(ws, folder_id, landing.get('at'), 'parent',
|
| 1409 |
-
{'parent': before_parents.get(folder_id)}, FOLDER_WRITES_KEY
|
| 1410 |
-
author=actor or username)
|
| 1411 |
ws['folders'] = tree
|
| 1412 |
ws['itemFolders'] = placed
|
| 1413 |
now = placed.get('views') or {}
|
| 1414 |
for view_id in sorted(set(was) | {str(k) for k in now}):
|
| 1415 |
if was.get(view_id) != now.get(view_id):
|
| 1416 |
note_view_write(ws, view_id, landing.get('at'), 'place',
|
| 1417 |
-
{'folderId': was.get(view_id)}
|
| 1418 |
|
| 1419 |
def _audit(ws, record):
|
| 1420 |
now = (ws.get('itemFolders') or {}).get('views') or {}
|
|
@@ -2010,8 +1994,7 @@ class TableStore:
|
|
| 2010 |
prior = (ws.get('views') or {}).get(view_id)
|
| 2011 |
activity_before['value'] = view_audit.summary(prior)
|
| 2012 |
ws.setdefault('views', {})[view_id] = payload
|
| 2013 |
-
note_view_write(ws, view_id, landing.get('at'), 'view', view_members(prior)
|
| 2014 |
-
author=actor or username)
|
| 2015 |
audit(action=action or view_audit.derive_action(prior, payload),
|
| 2016 |
owner=username, view_id=view_id, actor=actor, origin=origin,
|
| 2017 |
before=view_audit.summary(prior), after=view_audit.summary(payload))
|
|
@@ -2027,8 +2010,7 @@ class TableStore:
|
|
| 2027 |
if rev is None or not rebase:
|
| 2028 |
raise
|
| 2029 |
self._land_stale(username, rev, stale,
|
| 2030 |
-
lambda ws, sent: stale_view_conflict(
|
| 2031 |
-
ws, view_id, requested, sent, requester=actor or username),
|
| 2032 |
_write)
|
| 2033 |
self.log_view_activity(view_id, 'saved', by=actor or username,
|
| 2034 |
before=activity_before.get('value'),
|
|
@@ -2073,8 +2055,7 @@ class TableStore:
|
|
| 2073 |
removed.clear()
|
| 2074 |
if isinstance(before, dict):
|
| 2075 |
removed['value'] = view_audit.summary(before)
|
| 2076 |
-
note_view_write(data.setdefault(owner, {}), vid, at, 'gone', view_members(before)
|
| 2077 |
-
author=who)
|
| 2078 |
views.pop(vid, None)
|
| 2079 |
audit(action='delete' if isinstance(before, dict) else None,
|
| 2080 |
owner=owner, view_id=vid, actor=actor, origin=origin,
|
|
|
|
| 105 |
# browser re-reads only after some writes. So "stamp differs" alone refuses one tab's own second
|
| 106 |
# write, and the outbox then resends that refused write forever.
|
| 107 |
#
|
| 108 |
+
# The record below tells the two apart. Each view write appends the view's member digests as
|
| 109 |
+
# they stood BEFORE it, tagged with the revision it landed as. For a stamp `R`, the earliest
|
| 110 |
+
# entry after `R` is the view as that sender saw it. A write is refused only when a member
|
| 111 |
+
# changed after `R` and this write would set it to something else.
|
| 112 |
#
|
| 113 |
# ⛔ The record lives in the owner's own stratum beside `views`, so it rides the same update and
|
| 114 |
# the same settings row. `workspace()` never projects it.
|
|
|
|
| 121 |
VIEW_WRITES_KEY = 'viewWrites'
|
| 122 |
# Folder ids are surface-scoped and keep history separate from view ids.
|
| 123 |
FOLDER_WRITES_KEY = 'folderWrites'
|
| 124 |
+
#: Writes kept per view. A sender older than the oldest kept write cannot be judged and is refused.
|
| 125 |
VIEW_WRITES_KEEP = 8
|
| 126 |
#: Views kept per stratum. The least recently written record goes first.
|
| 127 |
VIEW_WRITES_VIEWS = 400
|
|
|
|
| 150 |
return out
|
| 151 |
|
| 152 |
|
| 153 |
+
def _note_write(ws, view_id, at, kind, before, book_key):
|
| 154 |
"""Append one write to the selected bounded history in this owner's stratum."""
|
| 155 |
if at is None or not view_id or not isinstance(ws, dict):
|
| 156 |
return
|
|
|
|
| 159 |
book = ws[book_key] = {}
|
| 160 |
entry = book.get(view_id) if isinstance(book.get(view_id), dict) else {}
|
| 161 |
log = [row for row in entry.get('log') or () if isinstance(row, dict)]
|
| 162 |
+
log.append({'rev': int(at), 'kind': kind, 'before': before})
|
|
|
|
| 163 |
since = int(entry.get('since') or 0)
|
|
|
|
|
|
|
| 164 |
while len(log) > VIEW_WRITES_KEEP:
|
| 165 |
+
since = max(since, int(log.pop(0).get('rev') or 0))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
book[view_id] = {'since': since, 'log': log}
|
|
|
|
|
|
|
| 167 |
if len(book) > VIEW_WRITES_VIEWS:
|
| 168 |
# jsonb keeps no key order, so recency is read off each record's newest revision.
|
| 169 |
def newest(vid):
|
|
|
|
| 174 |
book.pop(vid, None)
|
| 175 |
|
| 176 |
|
| 177 |
+
def note_view_write(ws, view_id, at, kind, before):
|
| 178 |
"""Append one view, gone or place write to this stratum's record."""
|
| 179 |
+
_note_write(ws, view_id, at, kind, before, VIEW_WRITES_KEY)
|
| 180 |
|
| 181 |
|
| 182 |
def _later_writes(ws, view_id, sent, kinds, *, book_key=VIEW_WRITES_KEY):
|
|
|
|
| 191 |
return later, int(sent) >= int(entry.get('since') or 0)
|
| 192 |
|
| 193 |
|
| 194 |
+
def stale_view_conflict(ws, view_id, mine, sent):
|
| 195 |
"""Whether saving `mine` over `view_id` undoes a change made after revision `sent`."""
|
| 196 |
later, judgeable = _later_writes(ws, view_id, sent, ('view', 'gone'))
|
| 197 |
+
if not later:
|
| 198 |
+
return False
|
| 199 |
if not judgeable:
|
| 200 |
+
return True
|
| 201 |
+
if later[0].get('kind') == 'view' and later[0].get('before') is None:
|
| 202 |
+
# Created after `sent`, so the sender is the tab that created it. Judge only later writes.
|
| 203 |
+
later = later[1:]
|
| 204 |
+
if not later:
|
| 205 |
+
return False
|
| 206 |
current = ((ws.get('views') or {}).get(view_id))
|
| 207 |
+
if not isinstance(current, dict):
|
| 208 |
+
return True
|
| 209 |
+
base = later[0].get('before') or {}
|
| 210 |
+
now, want = view_members(current), view_members(mine) or {}
|
| 211 |
+
return any(now.get(member) != base.get(member) and want.get(member) != now.get(member)
|
| 212 |
+
for member in set(base) | set(now))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 213 |
|
| 214 |
|
| 215 |
def stale_place_conflict(ws, view_id, folder, sent):
|
|
|
|
| 1391 |
for folder_id in sorted(set(before_parents) | set(parents)):
|
| 1392 |
if before_parents.get(folder_id) != parents.get(folder_id):
|
| 1393 |
_note_write(ws, folder_id, landing.get('at'), 'parent',
|
| 1394 |
+
{'parent': before_parents.get(folder_id)}, FOLDER_WRITES_KEY)
|
|
|
|
| 1395 |
ws['folders'] = tree
|
| 1396 |
ws['itemFolders'] = placed
|
| 1397 |
now = placed.get('views') or {}
|
| 1398 |
for view_id in sorted(set(was) | {str(k) for k in now}):
|
| 1399 |
if was.get(view_id) != now.get(view_id):
|
| 1400 |
note_view_write(ws, view_id, landing.get('at'), 'place',
|
| 1401 |
+
{'folderId': was.get(view_id)})
|
| 1402 |
|
| 1403 |
def _audit(ws, record):
|
| 1404 |
now = (ws.get('itemFolders') or {}).get('views') or {}
|
|
|
|
| 1994 |
prior = (ws.get('views') or {}).get(view_id)
|
| 1995 |
activity_before['value'] = view_audit.summary(prior)
|
| 1996 |
ws.setdefault('views', {})[view_id] = payload
|
| 1997 |
+
note_view_write(ws, view_id, landing.get('at'), 'view', view_members(prior))
|
|
|
|
| 1998 |
audit(action=action or view_audit.derive_action(prior, payload),
|
| 1999 |
owner=username, view_id=view_id, actor=actor, origin=origin,
|
| 2000 |
before=view_audit.summary(prior), after=view_audit.summary(payload))
|
|
|
|
| 2010 |
if rev is None or not rebase:
|
| 2011 |
raise
|
| 2012 |
self._land_stale(username, rev, stale,
|
| 2013 |
+
lambda ws, sent: stale_view_conflict(ws, view_id, requested, sent),
|
|
|
|
| 2014 |
_write)
|
| 2015 |
self.log_view_activity(view_id, 'saved', by=actor or username,
|
| 2016 |
before=activity_before.get('value'),
|
|
|
|
| 2055 |
removed.clear()
|
| 2056 |
if isinstance(before, dict):
|
| 2057 |
removed['value'] = view_audit.summary(before)
|
| 2058 |
+
note_view_write(data.setdefault(owner, {}), vid, at, 'gone', view_members(before))
|
|
|
|
| 2059 |
views.pop(vid, None)
|
| 2060 |
audit(action='delete' if isinstance(before, dict) else None,
|
| 2061 |
owner=owner, view_id=vid, actor=actor, origin=origin,
|
platform/core/test_accounts.py
ADDED
|
@@ -0,0 +1,398 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Test accounts: an exact copy of one member's permissions that takes no seat and sends nothing.
|
| 2 |
+
|
| 3 |
+
S79-T63 (QI34, rulings R28 and R42). The copy carries the source's role, business units, modules,
|
| 4 |
+
agent, permission block (row filters and field walls stay locks, rule 3) and every share entry
|
| 5 |
+
that names the source. It never carries a password, MFA, email, WhatsApp, avatar or epoch.
|
| 6 |
+
Creation is all or nothing: a failure removes every part already written.
|
| 7 |
+
"""
|
| 8 |
+
import copy
|
| 9 |
+
import datetime as _dt
|
| 10 |
+
import re
|
| 11 |
+
|
| 12 |
+
import core.shares as shares
|
| 13 |
+
import core.store as store
|
| 14 |
+
import core.users as users
|
| 15 |
+
|
| 16 |
+
PREFIX = 'test-'
|
| 17 |
+
#: `routes_admin._UNAME_RE` admits 2 to 32 characters; a generated name stays inside it.
|
| 18 |
+
_NAME_MAX = 32
|
| 19 |
+
_RETRIES = 3
|
| 20 |
+
|
| 21 |
+
#: The record fields a copy takes verbatim from its source.
|
| 22 |
+
COPIED_FIELDS = ('role', 'bus', 'modules', 'agent', 'perms', 'perms_v')
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class CopyRefused(ValueError):
|
| 26 |
+
"""The request names a source or target this door does not copy or remove."""
|
| 27 |
+
|
| 28 |
+
def __init__(self, code, message, status=400):
|
| 29 |
+
self.code, self.status = code, status
|
| 30 |
+
super().__init__(message)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class CopyFailed(RuntimeError):
|
| 34 |
+
"""A copy or removal stopped part way. `leftovers` names what cleanup could not remove."""
|
| 35 |
+
|
| 36 |
+
def __init__(self, cause, leftovers=()):
|
| 37 |
+
self.cause, self.leftovers = cause, list(leftovers)
|
| 38 |
+
super().__init__(cause)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def is_test(record):
|
| 42 |
+
"""Whether this stored record is a copied test account."""
|
| 43 |
+
return users.is_test_account(record)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _tenant_of(record):
|
| 47 |
+
return str((record or {}).get('tenant') or 'royal-imports').strip().lower()
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def paid_seats(registry, tenant):
|
| 51 |
+
"""Active accounts in `tenant` that take a paid seat. A test account takes none (R42)."""
|
| 52 |
+
want = str(tenant or '').strip().lower()
|
| 53 |
+
return sum(1 for record in (registry or {}).values()
|
| 54 |
+
if isinstance(record, dict) and record.get('active', True)
|
| 55 |
+
and _tenant_of(record) == want and not is_test(record))
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def test_username(source, registry):
|
| 59 |
+
"""`test-<source>` cut to fit the username rule, with a numeric suffix on a collision."""
|
| 60 |
+
base = re.sub(r'[^a-z0-9._-]+', '-', str(source or '').strip().lower()).strip('.-_')
|
| 61 |
+
stem = (PREFIX + base)[:_NAME_MAX].rstrip('.-_')
|
| 62 |
+
taken = set(registry or {})
|
| 63 |
+
if stem not in taken:
|
| 64 |
+
return stem
|
| 65 |
+
for n in range(2, 1000):
|
| 66 |
+
suffix = f'-{n}'
|
| 67 |
+
candidate = stem[:_NAME_MAX - len(suffix)].rstrip('.-_') + suffix
|
| 68 |
+
if candidate not in taken:
|
| 69 |
+
return candidate
|
| 70 |
+
raise CopyRefused('no_free_username', 'No free name is left for another test account.', 409)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class _Doc:
|
| 74 |
+
"""A read-only store over one already-read shares document."""
|
| 75 |
+
|
| 76 |
+
def __init__(self, doc):
|
| 77 |
+
self._doc = doc if isinstance(doc, dict) else {}
|
| 78 |
+
|
| 79 |
+
def get(self, name, *args, **kwargs):
|
| 80 |
+
return self._doc if name == shares.SHARES_KEY else None
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class _Batch:
|
| 84 |
+
"""Stage `shares.set_grants` writes in memory, then replay them all in ONE store update.
|
| 85 |
+
|
| 86 |
+
Every staged closure keeps its `expect` check, so the replay refuses if any record changed.
|
| 87 |
+
"""
|
| 88 |
+
|
| 89 |
+
def __init__(self, st, doc):
|
| 90 |
+
self._st, self._doc, self._fns = st, copy.deepcopy(doc or {}), []
|
| 91 |
+
|
| 92 |
+
def get(self, name, *args, **kwargs):
|
| 93 |
+
if name != shares.SHARES_KEY:
|
| 94 |
+
raise RuntimeError(f'a staged grant write cannot read {name!r}')
|
| 95 |
+
return self._doc
|
| 96 |
+
|
| 97 |
+
def update(self, name, fn, flush='sync'):
|
| 98 |
+
if name != shares.SHARES_KEY:
|
| 99 |
+
raise RuntimeError(f'a staged grant write cannot write {name!r}')
|
| 100 |
+
self._doc = fn(self._doc)
|
| 101 |
+
self._fns.append(fn)
|
| 102 |
+
return self._doc
|
| 103 |
+
|
| 104 |
+
def commit(self):
|
| 105 |
+
if not self._fns:
|
| 106 |
+
return
|
| 107 |
+
fns = list(self._fns)
|
| 108 |
+
|
| 109 |
+
def replay(data):
|
| 110 |
+
data = data if isinstance(data, dict) else {}
|
| 111 |
+
for fn in fns:
|
| 112 |
+
data = fn(data)
|
| 113 |
+
return data
|
| 114 |
+
|
| 115 |
+
self._st.update(shares.SHARES_KEY, replay, flush='sync')
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _read_shares(st):
|
| 119 |
+
"""The tenant's shares document, read strictly: an unreadable bucket stops the copy."""
|
| 120 |
+
doc = st.get(shares.SHARES_KEY, fresh=True)
|
| 121 |
+
if doc is None:
|
| 122 |
+
return {}
|
| 123 |
+
if not isinstance(doc, dict):
|
| 124 |
+
raise RuntimeError('the sharing records could not be read')
|
| 125 |
+
return doc
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def grants_naming(username, doc):
|
| 129 |
+
"""`[{kind, oid, role}]` for every share entry that names `username`, from one document."""
|
| 130 |
+
username = str(username or '').strip().lower()
|
| 131 |
+
found = []
|
| 132 |
+
view = _Doc(doc)
|
| 133 |
+
for kind in shares.KINDS:
|
| 134 |
+
for oid, record in sorted(shares.grants_bulk(kind, st=view).items()):
|
| 135 |
+
for entry in record['entries']:
|
| 136 |
+
if entry['user'] == username:
|
| 137 |
+
found.append({'kind': kind, 'oid': oid, 'role': entry['role']})
|
| 138 |
+
return found
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def _check_source(source, registry, tenant):
|
| 142 |
+
uname = str(source or '').strip().lower()
|
| 143 |
+
record = (registry or {}).get(uname)
|
| 144 |
+
if not uname or not isinstance(record, dict) or _tenant_of(record) != tenant:
|
| 145 |
+
raise CopyRefused('no_such_user', f'No account named {uname or "(blank)"}.', 404)
|
| 146 |
+
if is_test(record):
|
| 147 |
+
raise CopyRefused('source_is_test', 'A test account cannot be copied. Copy a member.')
|
| 148 |
+
if str(record.get('role') or 'user').strip().lower() != 'user':
|
| 149 |
+
raise CopyRefused('source_is_admin',
|
| 150 |
+
'An administrator cannot be copied. Copy a member account.')
|
| 151 |
+
if not record.get('active', True):
|
| 152 |
+
raise CopyRefused('source_inactive', 'This account is turned off. Turn it on first.')
|
| 153 |
+
return uname, record
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def plan(source, registry, *, tenant, st):
|
| 157 |
+
"""What a copy of `source` would carry, read now: the access fields and every share entry."""
|
| 158 |
+
tenant = str(tenant or '').strip().lower()
|
| 159 |
+
uname, record = _check_source(source, registry, tenant)
|
| 160 |
+
access = {key: copy.deepcopy(record[key]) for key in COPIED_FIELDS if key in record}
|
| 161 |
+
return {'source': uname,
|
| 162 |
+
'sourceName': str(record.get('name') or uname),
|
| 163 |
+
'username': test_username(uname, registry),
|
| 164 |
+
'name': f'Test copy of {record.get("name") or uname}',
|
| 165 |
+
'access': access,
|
| 166 |
+
'grants': grants_naming(uname, _read_shares(st))}
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _password(username, name):
|
| 170 |
+
for _ in range(8):
|
| 171 |
+
candidate = users.generate_password()
|
| 172 |
+
if not users.password_failures(candidate, username=username, name=name):
|
| 173 |
+
return candidate
|
| 174 |
+
raise RuntimeError('could not generate a password for the test account')
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def copy_grants(username, grants, *, st, granter):
|
| 178 |
+
"""Add `username` beside each listed share entry, same role, in one store update."""
|
| 179 |
+
batch = _Batch(st, _read_shares(st))
|
| 180 |
+
for grant in grants:
|
| 181 |
+
current = shares.grants(grant['kind'], grant['oid'], st=batch)
|
| 182 |
+
if any(entry['user'] == username for entry in current['entries']):
|
| 183 |
+
continue
|
| 184 |
+
entries = current['entries'] + [{'user': username, 'role': grant['role']}]
|
| 185 |
+
shares.set_grants(grant['kind'], grant['oid'], entries, st=batch,
|
| 186 |
+
granter=granter, expect=current['entries'])
|
| 187 |
+
batch.commit()
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def remove_grants(username, *, st, granter):
|
| 191 |
+
"""Remove every share entry naming `username`, retrying if a record changes meanwhile."""
|
| 192 |
+
for attempt in range(_RETRIES):
|
| 193 |
+
doc = _read_shares(st)
|
| 194 |
+
batch = _Batch(st, doc)
|
| 195 |
+
for grant in grants_naming(username, doc):
|
| 196 |
+
current = shares.grants(grant['kind'], grant['oid'], st=batch)
|
| 197 |
+
entries = [entry for entry in current['entries'] if entry['user'] != username]
|
| 198 |
+
shares.set_grants(grant['kind'], grant['oid'], entries, st=batch,
|
| 199 |
+
granter=granter, expect=current['entries'])
|
| 200 |
+
try:
|
| 201 |
+
batch.commit()
|
| 202 |
+
return
|
| 203 |
+
except shares.GrantsChanged:
|
| 204 |
+
if attempt == _RETRIES - 1:
|
| 205 |
+
raise
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def _fresh_registry():
|
| 209 |
+
registry = store.get('users', fresh=True)
|
| 210 |
+
if not isinstance(registry, dict):
|
| 211 |
+
raise RuntimeError('the account registry could not be read')
|
| 212 |
+
return registry
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _confirm(username, plan_, marker, st):
|
| 216 |
+
record = _fresh_registry().get(username) or {}
|
| 217 |
+
if record.get(users.TEST_ACCOUNT_KEY) != marker:
|
| 218 |
+
raise RuntimeError('the account was not saved')
|
| 219 |
+
if any(record.get(key) != plan_['access'].get(key) for key in plan_['access']):
|
| 220 |
+
raise RuntimeError('the copied permissions were not saved')
|
| 221 |
+
held = {(g['kind'], g['oid'], g['role']) for g in grants_naming(username, _read_shares(st))}
|
| 222 |
+
missing = [g for g in plan_['grants'] if (g['kind'], g['oid'], g['role']) not in held]
|
| 223 |
+
if missing:
|
| 224 |
+
raise RuntimeError(f'{len(missing)} shared items were not saved')
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def _cleanup(username, tenant, marker, st, granter, steps):
|
| 228 |
+
"""Undo every part of a copy. Returns the names of the parts that could not be removed."""
|
| 229 |
+
leftovers = []
|
| 230 |
+
for name, _do, undo in reversed(list(steps)):
|
| 231 |
+
try:
|
| 232 |
+
undo(username)
|
| 233 |
+
except Exception: # noqa: BLE001
|
| 234 |
+
leftovers.append(name)
|
| 235 |
+
try:
|
| 236 |
+
remove_grants(username, st=st, granter=granter)
|
| 237 |
+
except Exception: # noqa: BLE001
|
| 238 |
+
leftovers.append('shared items')
|
| 239 |
+
try:
|
| 240 |
+
users.delete_test_account(username, tenant=tenant, marker=marker)
|
| 241 |
+
left = _fresh_registry().get(username)
|
| 242 |
+
if is_test(left) and left.get(users.TEST_ACCOUNT_KEY) == marker:
|
| 243 |
+
leftovers.append('sign in')
|
| 244 |
+
except Exception: # noqa: BLE001
|
| 245 |
+
leftovers.append('sign in')
|
| 246 |
+
return leftovers
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _cause(exc):
|
| 250 |
+
text = str(exc).strip() or type(exc).__name__
|
| 251 |
+
return text[:1].upper() + text[1:]
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def create(source, *, admin, tenant, st, steps=(), now=None):
|
| 255 |
+
"""Copy `source` into a new test account. Returns `{username, password, plan}`.
|
| 256 |
+
|
| 257 |
+
`steps` are `(name, do(username, source), undo(username))` for grant stores outside core.
|
| 258 |
+
Any failure removes what was written and raises `CopyFailed`.
|
| 259 |
+
"""
|
| 260 |
+
tenant = str(tenant or '').strip().lower()
|
| 261 |
+
admin = str(admin or '').strip().lower()
|
| 262 |
+
for attempt in range(_RETRIES):
|
| 263 |
+
plan_ = plan(source, _fresh_registry(), tenant=tenant, st=st)
|
| 264 |
+
username = plan_['username']
|
| 265 |
+
marker = {'copied_from': plan_['source'], 'created_by': admin,
|
| 266 |
+
'created_at': (now or _dt.datetime.now(_dt.timezone.utc)).isoformat(
|
| 267 |
+
timespec='seconds')}
|
| 268 |
+
password = _password(username, plan_['name'])
|
| 269 |
+
try:
|
| 270 |
+
users.create_test_account(username, password, name=plan_['name'],
|
| 271 |
+
access=plan_['access'], marker=marker, tenant=tenant)
|
| 272 |
+
except users.TestAccountExists:
|
| 273 |
+
if attempt == _RETRIES - 1:
|
| 274 |
+
raise CopyFailed('Another account took the test account name. Try again.')
|
| 275 |
+
continue
|
| 276 |
+
except Exception as exc: # noqa: BLE001
|
| 277 |
+
# The write may have landed before it reported failure: clean up as for any step.
|
| 278 |
+
leftovers = _cleanup(username, tenant, marker, st, admin, ())
|
| 279 |
+
raise CopyFailed(_cause(exc), leftovers) from exc
|
| 280 |
+
try:
|
| 281 |
+
copy_grants(username, plan_['grants'], st=st, granter=admin)
|
| 282 |
+
for _name, do, _undo in steps:
|
| 283 |
+
do(username, plan_['source'])
|
| 284 |
+
_confirm(username, plan_, marker, st)
|
| 285 |
+
except Exception as exc: # noqa: BLE001
|
| 286 |
+
leftovers = _cleanup(username, tenant, marker, st, admin, steps)
|
| 287 |
+
raise CopyFailed(_cause(exc), leftovers) from exc
|
| 288 |
+
return {'username': username, 'password': password, 'plan': plan_}
|
| 289 |
+
raise CopyFailed('The test account could not be created. Try again.')
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
def remove(username, *, tenant, st, granter, steps=()):
|
| 293 |
+
"""Delete a test account and every grant naming it. Refuses a real account."""
|
| 294 |
+
tenant = str(tenant or '').strip().lower()
|
| 295 |
+
uname = str(username or '').strip().lower()
|
| 296 |
+
record = _fresh_registry().get(uname)
|
| 297 |
+
if not isinstance(record, dict) or _tenant_of(record) != tenant:
|
| 298 |
+
raise CopyRefused('no_such_user', f'No account named {uname or "(blank)"}.', 404)
|
| 299 |
+
if not is_test(record):
|
| 300 |
+
raise CopyRefused('not_a_test_account',
|
| 301 |
+
'This is a real account. Only a test account can be removed here.')
|
| 302 |
+
leftovers = _cleanup(uname, tenant, record.get(users.TEST_ACCOUNT_KEY), st, granter, steps)
|
| 303 |
+
if leftovers:
|
| 304 |
+
raise CopyFailed('Some parts are still saved', leftovers)
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
# ------------------------------------------------------------------ preview
|
| 308 |
+
def _label(key, defs):
|
| 309 |
+
import core.perms as perms
|
| 310 |
+
import core.registry as registry
|
| 311 |
+
definition = (defs or {}).get(key) or {}
|
| 312 |
+
# The editor's surface rows (Assistant, Agents, AI apps (MCP)) are named by perms, not the registry.
|
| 313 |
+
surfaces = {m['key']: m['label'] for m in perms.governable_surface_modules()}
|
| 314 |
+
return str(definition.get('label') or (registry.BY_KEY.get(key) or {}).get('label')
|
| 315 |
+
or surfaces.get(key) or key)
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def _plural(n, one, many):
|
| 319 |
+
return f'{n} {one if n == 1 else many}'
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def preview(plan_, *, st=None, limits=None):
|
| 323 |
+
"""The contract body for GET .../test-account/preview, built from `plan`.
|
| 324 |
+
|
| 325 |
+
`st` is a definitions store (`user_tables.lend_defs`) for the database lock checks.
|
| 326 |
+
`limits` is `perm_scope.scope_limits` for the source; it names each row limit, business units included.
|
| 327 |
+
"""
|
| 328 |
+
from core import user_tables
|
| 329 |
+
access = plan_['access']
|
| 330 |
+
try:
|
| 331 |
+
defs = user_tables.all_defs(st=st, schema_only=True) or {}
|
| 332 |
+
except Exception: # noqa: BLE001
|
| 333 |
+
defs = {}
|
| 334 |
+
block = access.get('perms') if isinstance(access.get('perms'), dict) else {}
|
| 335 |
+
locks, reached = [], set()
|
| 336 |
+
said = {}
|
| 337 |
+
for limit in limits or ():
|
| 338 |
+
text = str(limit.get('text') or '').strip().rstrip('.')
|
| 339 |
+
if limit.get('database') and text:
|
| 340 |
+
said[str(limit['database'])] = text
|
| 341 |
+
for key in sorted(said):
|
| 342 |
+
locks.append({'kind': 'row_scope', 'label': f'{_label(key, defs)}: {said[key]}'})
|
| 343 |
+
for key, entry in sorted(block.items()):
|
| 344 |
+
if not isinstance(entry, dict) or not entry.get('access', True):
|
| 345 |
+
continue
|
| 346 |
+
reached.add(key)
|
| 347 |
+
if entry.get('filter') and key not in said:
|
| 348 |
+
locks.append({'kind': 'row_scope', 'label': f'Rows are limited in {_label(key, defs)}'})
|
| 349 |
+
hidden = [f for f in entry.get('hiddenFields') or () if f]
|
| 350 |
+
if hidden:
|
| 351 |
+
locks.append({'kind': 'field_wall',
|
| 352 |
+
'label': f'{_plural(len(hidden), "field is", "fields are")} hidden in '
|
| 353 |
+
f'{_label(key, defs)}'})
|
| 354 |
+
if not access.get('perms_v') and not said:
|
| 355 |
+
bus = access.get('bus', 'all')
|
| 356 |
+
if bus != 'all' and isinstance(bus, list):
|
| 357 |
+
names = ', '.join(users.BU_LABELS.get(b, str(b)) for b in bus)
|
| 358 |
+
locks.append({'kind': 'row_scope', 'label': f'Rows are limited to {names}'})
|
| 359 |
+
if access.get('agent'):
|
| 360 |
+
locks.append({'kind': 'row_scope', 'label': "Rows are limited to one salesperson's book"})
|
| 361 |
+
for grant in plan_['grants']:
|
| 362 |
+
if grant['kind'] == 'cohort':
|
| 363 |
+
scope, _cid = shares.split_cohort_oid(grant['oid'])
|
| 364 |
+
where = _label(scope, defs) if scope else 'a database'
|
| 365 |
+
locks.append({'kind': 'locked_view', 'label': f'A locked view in {where}'})
|
| 366 |
+
if grant['kind'] == 'database':
|
| 367 |
+
reached.add(grant['oid'])
|
| 368 |
+
for key in sorted(k for k in reached if str(k).startswith(user_tables.KEY_PREFIX)):
|
| 369 |
+
definition = defs.get(key)
|
| 370 |
+
if not isinstance(definition, dict):
|
| 371 |
+
continue
|
| 372 |
+
if not user_tables.records_mutable(key, st=st):
|
| 373 |
+
locks.append({'kind': 'locked_database',
|
| 374 |
+
'label': f'{_label(key, defs)} is locked for record changes'})
|
| 375 |
+
preset = [f for f in definition.get('fields') or ()
|
| 376 |
+
if isinstance(f, dict) and isinstance(f.get('automation'), dict)
|
| 377 |
+
and f['automation'].get('preset') is True
|
| 378 |
+
and not user_tables.preset_editable(f)]
|
| 379 |
+
if preset:
|
| 380 |
+
locks.append({'kind': 'preset_field',
|
| 381 |
+
'label': f'{_plural(len(preset), "pre-set field", "pre-set fields")} '
|
| 382 |
+
f'in {_label(key, defs)}'})
|
| 383 |
+
grants = []
|
| 384 |
+
for grant in plan_['grants']:
|
| 385 |
+
row = dict(grant)
|
| 386 |
+
if grant['kind'] == 'database':
|
| 387 |
+
row['label'] = _label(grant['oid'], defs)
|
| 388 |
+
grants.append(row)
|
| 389 |
+
database_access = [{'key': key, 'label': _label(key, defs),
|
| 390 |
+
'access': bool(entry.get('access', True))}
|
| 391 |
+
for key, entry in sorted(block.items()) if isinstance(entry, dict)]
|
| 392 |
+
return {'source': {'username': plan_['source'], 'name': plan_['sourceName']},
|
| 393 |
+
'testUsername': plan_['username'],
|
| 394 |
+
'copy': {'role': access.get('role', 'user'), 'bus': access.get('bus', 'all'),
|
| 395 |
+
'buLabels': users.allowed_bus_labels(access),
|
| 396 |
+
'modules': access.get('modules', 'all'), 'agent': access.get('agent'),
|
| 397 |
+
'grants': grants, 'databaseAccess': database_access, 'locks': locks},
|
| 398 |
+
'counts': {'grants': len(grants), 'locks': len(locks)}}
|
platform/core/user_tables.py
CHANGED
|
@@ -529,8 +529,6 @@ def oversize_of(table_key, st=None, defn=None):
|
|
| 529 |
defn = get_definition(str(table_key or '').strip(), st) or {}
|
| 530 |
except Exception: # noqa: BLE001
|
| 531 |
defn = {}
|
| 532 |
-
if not materialises(table_key, defn=defn):
|
| 533 |
-
return None
|
| 534 |
stamp = (defn or {}).get(OVERSIZE_FIELD)
|
| 535 |
return stamp if isinstance(stamp, dict) else None
|
| 536 |
|
|
|
|
| 529 |
defn = get_definition(str(table_key or '').strip(), st) or {}
|
| 530 |
except Exception: # noqa: BLE001
|
| 531 |
defn = {}
|
|
|
|
|
|
|
| 532 |
stamp = (defn or {}).get(OVERSIZE_FIELD)
|
| 533 |
return stamp if isinstance(stamp, dict) else None
|
| 534 |
|
platform/core/users.py
CHANGED
|
@@ -1176,6 +1176,8 @@ def set_whatsapp(username, value):
|
|
| 1176 |
u = reg.get(username)
|
| 1177 |
if not isinstance(u, dict):
|
| 1178 |
raise ValueError('This account is not available.')
|
|
|
|
|
|
|
| 1179 |
if value is None:
|
| 1180 |
u.pop('whatsapp', None)
|
| 1181 |
else:
|
|
@@ -1198,9 +1200,110 @@ def set_whatsapp_opt_out(username, on):
|
|
| 1198 |
return account_rows.update(username, _set, st=store)
|
| 1199 |
|
| 1200 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1201 |
def is_test_account(record):
|
| 1202 |
-
"""
|
| 1203 |
-
return isinstance(record, dict) and isinstance(record.get(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1204 |
|
| 1205 |
|
| 1206 |
#: The keys of one database entry that `set_database_access` writes.
|
|
@@ -1319,6 +1422,7 @@ def set_access(username, role=None, bus=None, modules=None, agent=None, email=No
|
|
| 1319 |
def _set(reg):
|
| 1320 |
u = reg.get(username)
|
| 1321 |
if u:
|
|
|
|
| 1322 |
if name is not None:
|
| 1323 |
u['name'] = name
|
| 1324 |
if role is not None:
|
|
@@ -1361,6 +1465,12 @@ def update_access_with_active_admin_guard(username, role=None, bus=None, modules
|
|
| 1361 |
if not isinstance(current, dict):
|
| 1362 |
outcome[0] = 'missing'
|
| 1363 |
return reg
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1364 |
# Build the whole candidate before touching reg. On a refused transition the original
|
| 1365 |
# record remains byte-for-byte intact, including its epoch.
|
| 1366 |
candidate = dict(current)
|
|
|
|
| 1176 |
u = reg.get(username)
|
| 1177 |
if not isinstance(u, dict):
|
| 1178 |
raise ValueError('This account is not available.')
|
| 1179 |
+
# R42: a test account has no outbound channel, so no writer may add one.
|
| 1180 |
+
refuse_test_account_change(u, whatsapp=value)
|
| 1181 |
if value is None:
|
| 1182 |
u.pop('whatsapp', None)
|
| 1183 |
else:
|
|
|
|
| 1200 |
return account_rows.update(username, _set, st=store)
|
| 1201 |
|
| 1202 |
|
| 1203 |
+
# ------------------------------------------------------------------ test accounts (S79-T63, R42)
|
| 1204 |
+
#: The record key that marks a copied test account: `{copied_from, created_by, created_at}`.
|
| 1205 |
+
TEST_ACCOUNT_KEY = 'test_account'
|
| 1206 |
+
|
| 1207 |
+
|
| 1208 |
+
class TestAccountRefused(ValueError):
|
| 1209 |
+
"""A writer refused to give a test account an outbound channel or the admin role."""
|
| 1210 |
+
|
| 1211 |
+
def __init__(self, code, message):
|
| 1212 |
+
self.code = code
|
| 1213 |
+
super().__init__(message)
|
| 1214 |
+
|
| 1215 |
+
|
| 1216 |
+
class TestAccountExists(ValueError):
|
| 1217 |
+
"""The chosen test username is already a key in the registry."""
|
| 1218 |
+
|
| 1219 |
+
|
| 1220 |
def is_test_account(record):
|
| 1221 |
+
"""Whether this stored record is a copied test account."""
|
| 1222 |
+
return isinstance(record, dict) and isinstance(record.get(TEST_ACCOUNT_KEY), dict)
|
| 1223 |
+
|
| 1224 |
+
|
| 1225 |
+
def refuse_test_account_change(record, *, email=None, whatsapp=None, role=None):
|
| 1226 |
+
"""Raise `TestAccountRefused` if a change would give a test account email, WhatsApp or admin."""
|
| 1227 |
+
if not is_test_account(record):
|
| 1228 |
+
return
|
| 1229 |
+
if email:
|
| 1230 |
+
raise TestAccountRefused('test_account_no_email',
|
| 1231 |
+
'A test account cannot have an email address.')
|
| 1232 |
+
if whatsapp:
|
| 1233 |
+
raise TestAccountRefused('test_account_no_whatsapp',
|
| 1234 |
+
'A test account cannot connect WhatsApp.')
|
| 1235 |
+
if role is not None and str(role).strip().lower() != 'user':
|
| 1236 |
+
raise TestAccountRefused('test_account_not_admin',
|
| 1237 |
+
'A test account cannot be an administrator.')
|
| 1238 |
+
|
| 1239 |
+
|
| 1240 |
+
def create_test_account(username, pw, *, name, access, marker, tenant):
|
| 1241 |
+
"""Write one test account in one update: `access` fields verbatim, no email, no WhatsApp.
|
| 1242 |
+
|
| 1243 |
+
Raises `TestAccountExists` when the key is taken by any record other than this exact write.
|
| 1244 |
+
"""
|
| 1245 |
+
import copy
|
| 1246 |
+
freeze_notice.check_write()
|
| 1247 |
+
original_username = (username or '').strip()
|
| 1248 |
+
username = original_username.lower()
|
| 1249 |
+
failures = password_failures(pw, username=username, name=name)
|
| 1250 |
+
if failures:
|
| 1251 |
+
raise PasswordPolicyError(failures)
|
| 1252 |
+
marker = dict(marker)
|
| 1253 |
+
salt = secrets.token_hex(16)
|
| 1254 |
+
digest = _hash(pw, salt)
|
| 1255 |
+
|
| 1256 |
+
def _add(reg):
|
| 1257 |
+
prior = reg.get(username)
|
| 1258 |
+
if prior is not None:
|
| 1259 |
+
# A replayed closure finds its own landed write; anything else is a real collision.
|
| 1260 |
+
if isinstance(prior, dict) and prior.get(TEST_ACCOUNT_KEY) == marker:
|
| 1261 |
+
return reg
|
| 1262 |
+
raise TestAccountExists(username)
|
| 1263 |
+
rec = {'salt': salt, 'hash': digest, 'name': name, 'role': 'user',
|
| 1264 |
+
'password_changed_at': _now_iso(),
|
| 1265 |
+
'bus': copy.deepcopy(access.get('bus', 'all')), 'active': True,
|
| 1266 |
+
'modules': copy.deepcopy(access.get('modules', 'all')),
|
| 1267 |
+
'agent': access.get('agent') or None, 'email': None,
|
| 1268 |
+
'tenant': str(tenant or 'royal-imports').strip().lower(),
|
| 1269 |
+
'username_original': original_username, TEST_ACCOUNT_KEY: dict(marker)}
|
| 1270 |
+
# The permission block and its marker travel together or not at all (see `_record`).
|
| 1271 |
+
for key in ('perms', 'perms_v'):
|
| 1272 |
+
if key in access:
|
| 1273 |
+
rec[key] = copy.deepcopy(access[key])
|
| 1274 |
+
reg[username] = rec
|
| 1275 |
+
return reg
|
| 1276 |
+
|
| 1277 |
+
account_rows.update(username, _add, st=store)
|
| 1278 |
+
|
| 1279 |
+
|
| 1280 |
+
def delete_test_account(username, *, tenant, marker=None):
|
| 1281 |
+
"""Delete one test account record. Returns deleted, missing or not_test; never a real account.
|
| 1282 |
+
|
| 1283 |
+
`marker`, when given, must equal the stored marker, so a rollback never removes a copy
|
| 1284 |
+
another administrator made under the same name.
|
| 1285 |
+
"""
|
| 1286 |
+
freeze_notice.check_write()
|
| 1287 |
+
username = (username or '').strip().lower()
|
| 1288 |
+
tenant = str(tenant or '').strip().lower()
|
| 1289 |
+
outcome = ['missing']
|
| 1290 |
+
|
| 1291 |
+
def _del(reg):
|
| 1292 |
+
outcome[0] = 'missing'
|
| 1293 |
+
rec = reg.get(username)
|
| 1294 |
+
if (not isinstance(rec, dict)
|
| 1295 |
+
or str(rec.get('tenant') or 'royal-imports').strip().lower() != tenant):
|
| 1296 |
+
return reg
|
| 1297 |
+
if not is_test_account(rec) or (marker is not None
|
| 1298 |
+
and rec.get(TEST_ACCOUNT_KEY) != dict(marker)):
|
| 1299 |
+
outcome[0] = 'not_test'
|
| 1300 |
+
return reg
|
| 1301 |
+
reg.pop(username, None)
|
| 1302 |
+
outcome[0] = 'deleted'
|
| 1303 |
+
return reg
|
| 1304 |
+
|
| 1305 |
+
account_rows.update(username, _del, st=store)
|
| 1306 |
+
return outcome[0]
|
| 1307 |
|
| 1308 |
|
| 1309 |
#: The keys of one database entry that `set_database_access` writes.
|
|
|
|
| 1422 |
def _set(reg):
|
| 1423 |
u = reg.get(username)
|
| 1424 |
if u:
|
| 1425 |
+
refuse_test_account_change(u, email=email, role=role)
|
| 1426 |
if name is not None:
|
| 1427 |
u['name'] = name
|
| 1428 |
if role is not None:
|
|
|
|
| 1465 |
if not isinstance(current, dict):
|
| 1466 |
outcome[0] = 'missing'
|
| 1467 |
return reg
|
| 1468 |
+
try:
|
| 1469 |
+
refuse_test_account_change(current, email=email, role=role)
|
| 1470 |
+
except TestAccountRefused as exc:
|
| 1471 |
+
outcome[0] = exc.code
|
| 1472 |
+
return reg
|
| 1473 |
+
|
| 1474 |
# Build the whole candidate before touching reg. On a refused transition the original
|
| 1475 |
# record remains byte-for-byte intact, including its epoch.
|
| 1476 |
candidate = dict(current)
|
platform/core/windows.py
CHANGED
|
@@ -286,10 +286,7 @@ def ending_bounds(kind, mode, value, today, step="month", offset=0, n=0, unit="m
|
|
| 286 |
"""Keep the kind's own start. An ending before it refuses instead of reversing bounds."""
|
| 287 |
if kind not in TO_DATE_KINDS:
|
| 288 |
return None
|
| 289 |
-
|
| 290 |
-
if kind == "ytd_last_year" and mode not in ("fixed", "cell"):
|
| 291 |
-
ending_today = resolve({"kind": kind}, today)[1]
|
| 292 |
-
end = resolve_ending(mode, value, ending_today, step, offset, n, unit)
|
| 293 |
if end is None:
|
| 294 |
return None
|
| 295 |
start = resolve({"kind": kind}, today)[0]
|
|
|
|
| 286 |
"""Keep the kind's own start. An ending before it refuses instead of reversing bounds."""
|
| 287 |
if kind not in TO_DATE_KINDS:
|
| 288 |
return None
|
| 289 |
+
end = resolve_ending(mode, value, today, step, offset, n, unit)
|
|
|
|
|
|
|
|
|
|
| 290 |
if end is None:
|
| 291 |
return None
|
| 292 |
start = resolve({"kind": kind}, today)[0]
|
platform/harness/amazon_ads_store.py
CHANGED
|
@@ -7,8 +7,9 @@ One table per ad type and report, `amzn_ads_<sp|sb|sd>_<report>`, spawned as `ut
|
|
| 7 |
|
| 8 |
⛔ HISTORY IS KEPT FROM THE CONNECT DAY. The first sync stamps the connect day once and reaches
|
| 9 |
back the longest window Amazon documents per table (`connector_spec.AMAZON_ADS_BACKFILL_DAYS`).
|
| 10 |
-
A later sync re-reads the
|
| 11 |
-
No sync deletes a day an earlier sync stored, so a day Amazon has aged
|
|
|
|
| 12 |
|
| 13 |
⛔ STANDING RULE 1. Every report is read whole and paged. The limits that cannot be removed (one
|
| 14 |
report at a time, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
|
|
@@ -34,7 +35,6 @@ if str(_HERE.parent) not in sys.path:
|
|
| 34 |
sys.path.insert(0, str(_HERE.parent))
|
| 35 |
|
| 36 |
from core import connector_spec # noqa: E402
|
| 37 |
-
from harness import amazon_client # noqa: E402
|
| 38 |
from harness import amazon_ads_client as client # noqa: E402
|
| 39 |
from harness import amazon_ads_schema as schema # noqa: E402
|
| 40 |
from harness import amazon_store # noqa: E402
|
|
@@ -45,8 +45,8 @@ from harness import datastore # noqa: E
|
|
| 45 |
ADS_AFTER_SYNC = ("modules.amazon_operator:on_ads_sync",)
|
| 46 |
|
| 47 |
TABLES = tuple(schema.TABLE_SPECS)
|
| 48 |
-
#:
|
| 49 |
-
RESTATEMENT_DAYS =
|
| 50 |
CONNECT_DAY_KEY = amazon_store.entity_key("ads_connect_day")
|
| 51 |
BUDGET_KEY = amazon_store.entity_key("ads_report_budget")
|
| 52 |
INSERT_BATCH_ROWS = 500
|
|
@@ -95,51 +95,11 @@ def _utc_today(now=None):
|
|
| 95 |
return now.date() if isinstance(now, datetime) else now
|
| 96 |
|
| 97 |
|
| 98 |
-
def _utc_now(now=None):
|
| 99 |
-
"""Normalize datetime and date callers to the same UTC cadence clock."""
|
| 100 |
-
now = now or datetime.now(timezone.utc)
|
| 101 |
-
if not isinstance(now, datetime):
|
| 102 |
-
now = datetime(now.year, now.month, now.day, tzinfo=timezone.utc)
|
| 103 |
-
return now.replace(tzinfo=timezone.utc) if now.tzinfo is None else now.astimezone(timezone.utc)
|
| 104 |
-
|
| 105 |
-
|
| 106 |
def _state_key(key, connection_id=None):
|
| 107 |
"""Keep the public table markers and give each connection its own continuation markers."""
|
| 108 |
return f"{key}::connection::{connection_id}" if connection_id else key
|
| 109 |
|
| 110 |
|
| 111 |
-
def _cadence_due(con, table, now, connection_id=None):
|
| 112 |
-
"""Only this connection's completed pass can delay its next cadence interval."""
|
| 113 |
-
cadence = amazon_client.CADENCE["ads_" + table]
|
| 114 |
-
tick = amazon_store._read_state(con, _state_key(
|
| 115 |
-
amazon_store.entity_key("sync_tick_" + _marker_table(table)), connection_id))
|
| 116 |
-
if not tick or tick.get("state") != "done":
|
| 117 |
-
return True
|
| 118 |
-
try:
|
| 119 |
-
stamp = amazon_store._utc(tick.get("detail") or tick["updated_at"])
|
| 120 |
-
except (ValueError, TypeError, AttributeError, OverflowError):
|
| 121 |
-
return True
|
| 122 |
-
return now >= stamp + timedelta(days=1) / cadence["perDay"]
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
def _intraday(con, table, today, connection_id=None):
|
| 126 |
-
"""Retry the daily lookback until every request for this connection succeeds."""
|
| 127 |
-
day = amazon_store._read_state(con, _state_key(
|
| 128 |
-
amazon_store.entity_key("lookback_day_" + _marker_table(table)), connection_id))
|
| 129 |
-
return bool(day and day.get("state") == "done" and day.get("detail") == today.isoformat())
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
def _record_cadence(con, table, now, connection_id=None):
|
| 133 |
-
"""Persist successful empty and nonempty passes with their daily lookback completion."""
|
| 134 |
-
tick = _state_key(amazon_store.entity_key("sync_tick_" + _marker_table(table)), connection_id)
|
| 135 |
-
stamp = now.isoformat()
|
| 136 |
-
amazon_store._record_state(con, tick, "done", detail=stamp)
|
| 137 |
-
con.execute("UPDATE _sync_state SET updated_at = ? WHERE entity = ?", [stamp, tick])
|
| 138 |
-
amazon_store._record_state(con, _state_key(
|
| 139 |
-
amazon_store.entity_key("lookback_day_" + _marker_table(table)), connection_id),
|
| 140 |
-
"done", detail=now.date().isoformat())
|
| 141 |
-
|
| 142 |
-
|
| 143 |
def _connect_day(con, today, connection_id=None, *, legacy=False):
|
| 144 |
"""The connect day, stamped once on the first sync and never moved."""
|
| 145 |
key = _state_key(CONNECT_DAY_KEY, connection_id)
|
|
@@ -170,10 +130,7 @@ def _day_marker(con, key):
|
|
| 170 |
|
| 171 |
def windows(con, table, connect_day, today, connection_id=None):
|
| 172 |
"""The report windows one sync asks for, oldest first, each at most 31 days."""
|
| 173 |
-
|
| 174 |
-
spec = schema.TABLE_SPECS[table]
|
| 175 |
-
cadence = amazon_client.CADENCE["ads_" + table]
|
| 176 |
-
lookback = cadence["intradayLookbackDays" if _intraday(con, table, today, connection_id) else "lookbackDays"]
|
| 177 |
last = today - timedelta(days=1)
|
| 178 |
target_floor = max(connect_day, today) - timedelta(days=spec["backfill_days"])
|
| 179 |
mark_name = _marker_table(table)
|
|
@@ -185,7 +142,7 @@ def windows(con, table, connect_day, today, connection_id=None):
|
|
| 185 |
else:
|
| 186 |
if oldest > target_floor:
|
| 187 |
spans.append((target_floor, oldest - timedelta(days=1)))
|
| 188 |
-
spans.append((max(target_floor, min(newest, last - timedelta(days=
|
| 189 |
out = []
|
| 190 |
for start, end in spans:
|
| 191 |
cursor = start
|
|
@@ -347,7 +304,7 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 347 |
profile_id=None):
|
| 348 |
"""Pull every offered Ads report into the tenant's Amazon mirror. -> a report dict.
|
| 349 |
|
| 350 |
-
When this call opens the connection and has a runtime, it saves
|
| 351 |
through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
|
| 352 |
"""
|
| 353 |
key = amazon_store._tenant_key(tenant_key)
|
|
@@ -366,10 +323,8 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 366 |
con, _path = amazon_store._connect_tenant_mirror(key, rt)
|
| 367 |
try:
|
| 368 |
datastore._ensure_sync_state(con)
|
| 369 |
-
now = _utc_now(now)
|
| 370 |
today = _utc_today(now)
|
| 371 |
synced = []
|
| 372 |
-
checkpointed = []
|
| 373 |
for cid, label, profile, legacy in connections:
|
| 374 |
connect_day = _connect_day(con, today, cid, legacy=legacy)
|
| 375 |
report.setdefault("connectDay", connect_day.isoformat())
|
|
@@ -384,8 +339,6 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 384 |
pulled, state, mark, reason = 0, "done", _marker_table(table), ""
|
| 385 |
if not spec["offered"]:
|
| 386 |
state, reason = "not_offered", spec["not_offered_reason"]
|
| 387 |
-
elif not _cadence_due(con, table, now, cid):
|
| 388 |
-
state = "skipped"
|
| 389 |
else:
|
| 390 |
for start, end in windows(con, table, connect_day, today, cid):
|
| 391 |
if used >= client.REPORTS_PER_DAY:
|
|
@@ -416,8 +369,6 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 416 |
_row_key=_row_key(cid, row["_row_key"]))
|
| 417 |
stored.append(row)
|
| 418 |
pulled += _upsert(con, table, stored)
|
| 419 |
-
if table_name(table) not in checkpointed:
|
| 420 |
-
checkpointed.append(table_name(table))
|
| 421 |
# Only this connection's landed window advances its continuation.
|
| 422 |
for prefix, day, later in ((amazon_store.WATERMARK_PREFIX, end, True),
|
| 423 |
(amazon_store.BACKFILL_FLOOR_PREFIX, start, False)):
|
|
@@ -426,8 +377,6 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 426 |
previous = _day_marker(con, marker)
|
| 427 |
if previous is None or (day > previous if later else day < previous):
|
| 428 |
amazon_store._record_state(con, marker, "done", _count(con, table), day.isoformat())
|
| 429 |
-
if state == "done":
|
| 430 |
-
_record_cadence(con, table, now, cid)
|
| 431 |
if reason:
|
| 432 |
_record_limit(con, table, reason, cid, state=state)
|
| 433 |
shown = f"{label}: {reason}" if cid else reason
|
|
@@ -436,20 +385,17 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 436 |
_clear_limit(con, table, cid)
|
| 437 |
total = _count(con, table)
|
| 438 |
entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
"done" if state == "done" else "partial", _count(con, table, cid), state)
|
| 442 |
previous = report["tables"].get(table, {})
|
| 443 |
-
severity = {"
|
| 444 |
-
aggregate = max((state, previous.get("state",
|
| 445 |
report["tables"][table] = {"state": aggregate, "pulled": previous.get("pulled", 0) + pulled,
|
| 446 |
"in_mirror": total}
|
| 447 |
if (pulled or origin_changed) and table_name(table) not in synced:
|
| 448 |
synced.append(table_name(table))
|
| 449 |
log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}")
|
| 450 |
for table, state in report["tables"].items():
|
| 451 |
-
if state["state"] == "skipped":
|
| 452 |
-
continue
|
| 453 |
amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
|
| 454 |
"done" if state["state"] == "done" else "partial",
|
| 455 |
state["in_mirror"], state["state"])
|
|
@@ -457,8 +403,8 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 457 |
_record_limit(con, table, report["limits"][table], state=state["state"])
|
| 458 |
else:
|
| 459 |
_clear_limit(con, table)
|
| 460 |
-
if own and rt is not None and
|
| 461 |
-
_persist(key, rt, con,
|
| 462 |
finally:
|
| 463 |
if own:
|
| 464 |
con.close()
|
|
|
|
| 7 |
|
| 8 |
⛔ HISTORY IS KEPT FROM THE CONNECT DAY. The first sync stamps the connect day once and reaches
|
| 9 |
back the longest window Amazon documents per table (`connector_spec.AMAZON_ADS_BACKFILL_DAYS`).
|
| 10 |
+
A later sync re-reads the last `RESTATEMENT_DAYS`, because Amazon restates attributed sales, and
|
| 11 |
+
upserts by row identity. No sync deletes a day an earlier sync stored, so a day Amazon has aged
|
| 12 |
+
out stays here.
|
| 13 |
|
| 14 |
⛔ STANDING RULE 1. Every report is read whole and paged. The limits that cannot be removed (one
|
| 15 |
report at a time, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
|
|
|
|
| 35 |
sys.path.insert(0, str(_HERE.parent))
|
| 36 |
|
| 37 |
from core import connector_spec # noqa: E402
|
|
|
|
| 38 |
from harness import amazon_ads_client as client # noqa: E402
|
| 39 |
from harness import amazon_ads_schema as schema # noqa: E402
|
| 40 |
from harness import amazon_store # noqa: E402
|
|
|
|
| 45 |
ADS_AFTER_SYNC = ("modules.amazon_operator:on_ads_sync",)
|
| 46 |
|
| 47 |
TABLES = tuple(schema.TABLE_SPECS)
|
| 48 |
+
#: Amazon restates attributed sales for about 14 days, so each sync re-reads them.
|
| 49 |
+
RESTATEMENT_DAYS = 14
|
| 50 |
CONNECT_DAY_KEY = amazon_store.entity_key("ads_connect_day")
|
| 51 |
BUDGET_KEY = amazon_store.entity_key("ads_report_budget")
|
| 52 |
INSERT_BATCH_ROWS = 500
|
|
|
|
| 95 |
return now.date() if isinstance(now, datetime) else now
|
| 96 |
|
| 97 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
def _state_key(key, connection_id=None):
|
| 99 |
"""Keep the public table markers and give each connection its own continuation markers."""
|
| 100 |
return f"{key}::connection::{connection_id}" if connection_id else key
|
| 101 |
|
| 102 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
def _connect_day(con, today, connection_id=None, *, legacy=False):
|
| 104 |
"""The connect day, stamped once on the first sync and never moved."""
|
| 105 |
key = _state_key(CONNECT_DAY_KEY, connection_id)
|
|
|
|
| 130 |
|
| 131 |
def windows(con, table, connect_day, today, connection_id=None):
|
| 132 |
"""The report windows one sync asks for, oldest first, each at most 31 days."""
|
| 133 |
+
spec = schema.TABLE_SPECS[schema.short_name(table)]
|
|
|
|
|
|
|
|
|
|
| 134 |
last = today - timedelta(days=1)
|
| 135 |
target_floor = max(connect_day, today) - timedelta(days=spec["backfill_days"])
|
| 136 |
mark_name = _marker_table(table)
|
|
|
|
| 142 |
else:
|
| 143 |
if oldest > target_floor:
|
| 144 |
spans.append((target_floor, oldest - timedelta(days=1)))
|
| 145 |
+
spans.append((max(target_floor, min(newest, last) - timedelta(days=RESTATEMENT_DAYS - 1)), last))
|
| 146 |
out = []
|
| 147 |
for start, end in spans:
|
| 148 |
cursor = start
|
|
|
|
| 304 |
profile_id=None):
|
| 305 |
"""Pull every offered Ads report into the tenant's Amazon mirror. -> a report dict.
|
| 306 |
|
| 307 |
+
When this call opens the connection and has a runtime, it saves the tables that landed rows
|
| 308 |
through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
|
| 309 |
"""
|
| 310 |
key = amazon_store._tenant_key(tenant_key)
|
|
|
|
| 323 |
con, _path = amazon_store._connect_tenant_mirror(key, rt)
|
| 324 |
try:
|
| 325 |
datastore._ensure_sync_state(con)
|
|
|
|
| 326 |
today = _utc_today(now)
|
| 327 |
synced = []
|
|
|
|
| 328 |
for cid, label, profile, legacy in connections:
|
| 329 |
connect_day = _connect_day(con, today, cid, legacy=legacy)
|
| 330 |
report.setdefault("connectDay", connect_day.isoformat())
|
|
|
|
| 339 |
pulled, state, mark, reason = 0, "done", _marker_table(table), ""
|
| 340 |
if not spec["offered"]:
|
| 341 |
state, reason = "not_offered", spec["not_offered_reason"]
|
|
|
|
|
|
|
| 342 |
else:
|
| 343 |
for start, end in windows(con, table, connect_day, today, cid):
|
| 344 |
if used >= client.REPORTS_PER_DAY:
|
|
|
|
| 369 |
_row_key=_row_key(cid, row["_row_key"]))
|
| 370 |
stored.append(row)
|
| 371 |
pulled += _upsert(con, table, stored)
|
|
|
|
|
|
|
| 372 |
# Only this connection's landed window advances its continuation.
|
| 373 |
for prefix, day, later in ((amazon_store.WATERMARK_PREFIX, end, True),
|
| 374 |
(amazon_store.BACKFILL_FLOOR_PREFIX, start, False)):
|
|
|
|
| 377 |
previous = _day_marker(con, marker)
|
| 378 |
if previous is None or (day > previous if later else day < previous):
|
| 379 |
amazon_store._record_state(con, marker, "done", _count(con, table), day.isoformat())
|
|
|
|
|
|
|
| 380 |
if reason:
|
| 381 |
_record_limit(con, table, reason, cid, state=state)
|
| 382 |
shown = f"{label}: {reason}" if cid else reason
|
|
|
|
| 385 |
_clear_limit(con, table, cid)
|
| 386 |
total = _count(con, table)
|
| 387 |
entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
|
| 388 |
+
amazon_store._record_state(con, _state_key(amazon_store.entity_key(mark), cid),
|
| 389 |
+
"done" if state == "done" else "partial", _count(con, table, cid), state)
|
|
|
|
| 390 |
previous = report["tables"].get(table, {})
|
| 391 |
+
severity = {"done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
|
| 392 |
+
aggregate = max((state, previous.get("state", "done")), key=severity.get)
|
| 393 |
report["tables"][table] = {"state": aggregate, "pulled": previous.get("pulled", 0) + pulled,
|
| 394 |
"in_mirror": total}
|
| 395 |
if (pulled or origin_changed) and table_name(table) not in synced:
|
| 396 |
synced.append(table_name(table))
|
| 397 |
log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}")
|
| 398 |
for table, state in report["tables"].items():
|
|
|
|
|
|
|
| 399 |
amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
|
| 400 |
"done" if state["state"] == "done" else "partial",
|
| 401 |
state["in_mirror"], state["state"])
|
|
|
|
| 403 |
_record_limit(con, table, report["limits"][table], state=state["state"])
|
| 404 |
else:
|
| 405 |
_clear_limit(con, table)
|
| 406 |
+
if own and rt is not None and synced:
|
| 407 |
+
_persist(key, rt, con, synced, report, log)
|
| 408 |
finally:
|
| 409 |
if own:
|
| 410 |
con.close()
|
platform/harness/amazon_client.py
CHANGED
|
@@ -416,7 +416,7 @@ REPORT_TYPES = {
|
|
| 416 |
},
|
| 417 |
"storage_fees": {
|
| 418 |
"report": "GET_FBA_STORAGE_FEE_CHARGES_DATA", "monthly": True,
|
| 419 |
-
"min_interval_s": 86400,
|
| 420 |
"validates_against": "Amazon document data-row count vs stored rows for its latest calendar month",
|
| 421 |
"role": "Amazon Fulfillment",
|
| 422 |
},
|
|
@@ -458,7 +458,6 @@ GROUP_A_TABLES = {
|
|
| 458 |
"report": "GET_FBA_REIMBURSEMENTS_DATA",
|
| 459 |
"role": "Amazon Fulfillment",
|
| 460 |
"min_interval_s": 14400,
|
| 461 |
-
"restatement_keys": ("reimbursement_id",),
|
| 462 |
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 463 |
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 464 |
},
|
|
@@ -467,7 +466,6 @@ GROUP_A_TABLES = {
|
|
| 467 |
"report": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
|
| 468 |
"role": "Amazon Fulfillment",
|
| 469 |
"min_interval_s": 14400,
|
| 470 |
-
"restatement_keys": ("order_id", "sku", "fnsku", "asin", "return_date"),
|
| 471 |
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 472 |
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 473 |
},
|
|
@@ -678,8 +676,8 @@ GROUP_A_TABLES = {
|
|
| 678 |
"label": "Amazon account health",
|
| 679 |
"report": "GET_V2_SELLER_PERFORMANCE_REPORT",
|
| 680 |
"role": "Selling Partner Insights",
|
| 681 |
-
"min_interval_s":
|
| 682 |
-
"cadence_note": "Current snapshot; refresh no sooner than
|
| 683 |
"snapshot": True,
|
| 684 |
"json_root": "performanceMetrics",
|
| 685 |
"json_schema": "accountHealthReport-2020-11-18.json",
|
|
@@ -736,7 +734,6 @@ GROUP_A_TABLES = {
|
|
| 736 |
"report": "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
|
| 737 |
"role": "Inventory and Order Tracking",
|
| 738 |
"min_interval_s": 14400,
|
| 739 |
-
"restatement_keys": ("order_id", "sku", "return_request_date"),
|
| 740 |
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 741 |
"max_window_days": 60,
|
| 742 |
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
|
@@ -1018,40 +1015,6 @@ GROUP_A = {
|
|
| 1018 |
GROUP_A.update({key: {"key": key, **spec} for key, spec in DECLARED_DATASETS.items()})
|
| 1019 |
|
| 1020 |
|
| 1021 |
-
# Source: AutoWiki/research/datadoe-sync-cadence.md. Unlisted tables keep the daily default.
|
| 1022 |
-
CADENCE = {
|
| 1023 |
-
table: {"perDay": 1, "lookbackDays": 1, "intradayLookbackDays": 1, "period": "day"}
|
| 1024 |
-
for table in GROUP_A
|
| 1025 |
-
}
|
| 1026 |
-
for _table, _spec in REPORT_TYPES.items():
|
| 1027 |
-
if _spec.get("monthly") or _spec.get("period") == "MONTH":
|
| 1028 |
-
CADENCE[_table].update(period="month", lookbackDays=60, intradayLookbackDays=60)
|
| 1029 |
-
elif _spec.get("weekly") or _spec.get("min_interval_s", 0) >= 7 * 86400:
|
| 1030 |
-
CADENCE[_table].update(period="week", lookbackDays=14, intradayLookbackDays=14)
|
| 1031 |
-
for _tables, _passes, _daily, _intraday in (
|
| 1032 |
-
(("orders", "order_items"), 6, 8, 1),
|
| 1033 |
-
(("settlements", "transactions"), 3, 8, 2),
|
| 1034 |
-
(("listings", "inventory_planning"), 4, 1, 1),
|
| 1035 |
-
(("account_health", "manage_inventory", "restock_recommendations"), 2, 1, 1),
|
| 1036 |
-
(("pricing", "fba_fee_estimates", "referral_fees"), 1, 4, 4),
|
| 1037 |
-
(("reimbursements", "long_term_storage_fees"), 1, 30, 30),
|
| 1038 |
-
(("fba_customer_returns", "mfn_returns"), 1, 60, 60),
|
| 1039 |
-
(("storage_fees",), 1, 1, 1),
|
| 1040 |
-
):
|
| 1041 |
-
for _table in _tables:
|
| 1042 |
-
CADENCE[_table].update(perDay=_passes, lookbackDays=_daily,
|
| 1043 |
-
intradayLookbackDays=_intraday, period="day")
|
| 1044 |
-
|
| 1045 |
-
# Ads uses namespaced mirror marker names, so its tables cannot collide with SP-API tables.
|
| 1046 |
-
from harness import amazon_ads_schema as _ads_schema
|
| 1047 |
-
|
| 1048 |
-
CADENCE.update({
|
| 1049 |
-
"ads_" + table: {"perDay": 4, "lookbackDays": 21, "intradayLookbackDays": 5, "period": "day"}
|
| 1050 |
-
for table in _ads_schema.TABLE_SPECS
|
| 1051 |
-
})
|
| 1052 |
-
del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
|
| 1053 |
-
|
| 1054 |
-
|
| 1055 |
# Inventory section 2.5: immutable refusal even if a future report roster admits one.
|
| 1056 |
EXCLUDED_REPORT_TYPES = frozenset({
|
| 1057 |
"GET_ORDER_REPORT_DATA_SHIPPING", "GET_ORDER_REPORT_DATA_TAX", "GET_ORDER_REPORT_DATA_ALL",
|
|
@@ -3418,8 +3381,7 @@ def report_document(fields, report_type, start=None, end=None, options=None,
|
|
| 3418 |
# a role-shaped refusal on screen and left the window to retry forever. An empty
|
| 3419 |
# document lets the window complete and the connector detail name the empty period.
|
| 3420 |
say(f" {report_type}: Amazon cancelled report {rid}; no data for the requested period")
|
| 3421 |
-
return "", {"reportId": rid, "reportDocumentId": None, "
|
| 3422 |
-
"cancelled": True,
|
| 3423 |
"dataStartTime": got.get("dataStartTime"),
|
| 3424 |
"dataEndTime": got.get("dataEndTime"),
|
| 3425 |
"createdTime": got.get("createdTime"), "compression": None,
|
|
@@ -3457,7 +3419,7 @@ def report_document(fields, report_type, start=None, end=None, options=None,
|
|
| 3457 |
raise AmazonError(f"the {report_type} report document carried no download URL")
|
| 3458 |
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""), spool=True)
|
| 3459 |
say(f" {report_type}: {len(text):,} chars")
|
| 3460 |
-
return text, {"reportId": rid, "reportDocumentId": doc_id,
|
| 3461 |
"dataStartTime": got.get("dataStartTime"),
|
| 3462 |
"dataEndTime": got.get("dataEndTime"), "createdTime": got.get("createdTime"),
|
| 3463 |
"compression": meta.get("compressionAlgorithm"), "marketplaceIds": got.get("marketplaceIds")}
|
|
|
|
| 416 |
},
|
| 417 |
"storage_fees": {
|
| 418 |
"report": "GET_FBA_STORAGE_FEE_CHARGES_DATA", "monthly": True,
|
| 419 |
+
"min_interval_s": 30 * 86400,
|
| 420 |
"validates_against": "Amazon document data-row count vs stored rows for its latest calendar month",
|
| 421 |
"role": "Amazon Fulfillment",
|
| 422 |
},
|
|
|
|
| 458 |
"report": "GET_FBA_REIMBURSEMENTS_DATA",
|
| 459 |
"role": "Amazon Fulfillment",
|
| 460 |
"min_interval_s": 14400,
|
|
|
|
| 461 |
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 462 |
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 463 |
},
|
|
|
|
| 466 |
"report": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
|
| 467 |
"role": "Amazon Fulfillment",
|
| 468 |
"min_interval_s": 14400,
|
|
|
|
| 469 |
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 470 |
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 471 |
},
|
|
|
|
| 676 |
"label": "Amazon account health",
|
| 677 |
"report": "GET_V2_SELLER_PERFORMANCE_REPORT",
|
| 678 |
"role": "Selling Partner Insights",
|
| 679 |
+
"min_interval_s": 86400,
|
| 680 |
+
"cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
|
| 681 |
"snapshot": True,
|
| 682 |
"json_root": "performanceMetrics",
|
| 683 |
"json_schema": "accountHealthReport-2020-11-18.json",
|
|
|
|
| 734 |
"report": "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
|
| 735 |
"role": "Inventory and Order Tracking",
|
| 736 |
"min_interval_s": 14400,
|
|
|
|
| 737 |
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 738 |
"max_window_days": 60,
|
| 739 |
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
|
|
|
| 1015 |
GROUP_A.update({key: {"key": key, **spec} for key, spec in DECLARED_DATASETS.items()})
|
| 1016 |
|
| 1017 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1018 |
# Inventory section 2.5: immutable refusal even if a future report roster admits one.
|
| 1019 |
EXCLUDED_REPORT_TYPES = frozenset({
|
| 1020 |
"GET_ORDER_REPORT_DATA_SHIPPING", "GET_ORDER_REPORT_DATA_TAX", "GET_ORDER_REPORT_DATA_ALL",
|
|
|
|
| 3381 |
# a role-shaped refusal on screen and left the window to retry forever. An empty
|
| 3382 |
# document lets the window complete and the connector detail name the empty period.
|
| 3383 |
say(f" {report_type}: Amazon cancelled report {rid}; no data for the requested period")
|
| 3384 |
+
return "", {"reportId": rid, "reportDocumentId": None, "cancelled": True,
|
|
|
|
| 3385 |
"dataStartTime": got.get("dataStartTime"),
|
| 3386 |
"dataEndTime": got.get("dataEndTime"),
|
| 3387 |
"createdTime": got.get("createdTime"), "compression": None,
|
|
|
|
| 3419 |
raise AmazonError(f"the {report_type} report document carried no download URL")
|
| 3420 |
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""), spool=True)
|
| 3421 |
say(f" {report_type}: {len(text):,} chars")
|
| 3422 |
+
return text, {"reportId": rid, "reportDocumentId": doc_id,
|
| 3423 |
"dataStartTime": got.get("dataStartTime"),
|
| 3424 |
"dataEndTime": got.get("dataEndTime"), "createdTime": got.get("createdTime"),
|
| 3425 |
"compression": meta.get("compressionAlgorithm"), "marketplaceIds": got.get("marketplaceIds")}
|
platform/harness/amazon_store.py
CHANGED
|
@@ -463,48 +463,27 @@ def read_database_enablement(rt):
|
|
| 463 |
return {"available": True, "databases": {key: stored.get(key, True) for key in keys}}
|
| 464 |
|
| 465 |
|
| 466 |
-
#
|
|
|
|
|
|
|
|
|
|
| 467 |
SCHEDULED_FLOOR_SECONDS = 86400
|
| 468 |
-
SCHEDULED_FLOOR_GRACE_SECONDS =
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
def _cadence_seconds(table, state=None):
|
| 472 |
-
"""Return the table's scheduled interval, using calendar months for monthly reports."""
|
| 473 |
-
cadence = _client().CADENCE[table]
|
| 474 |
-
if cadence["period"] == "week":
|
| 475 |
-
if state:
|
| 476 |
-
stamp = _utc(state["updated_at"])
|
| 477 |
-
return int((_week_start(stamp) + timedelta(days=7) - stamp).total_seconds())
|
| 478 |
-
return 7 * SCHEDULED_FLOOR_SECONDS
|
| 479 |
-
if cadence["period"] == "month":
|
| 480 |
-
stamp = _utc(state["updated_at"]) if state else None
|
| 481 |
-
if stamp is None:
|
| 482 |
-
return 30 * SCHEDULED_FLOOR_SECONDS
|
| 483 |
-
year, month = stamp.year + (stamp.month == 12), stamp.month % 12 + 1
|
| 484 |
-
due = stamp.replace(year=year, month=month, day=1,
|
| 485 |
-
hour=0, minute=0, second=0, microsecond=0)
|
| 486 |
-
return int((due - stamp).total_seconds())
|
| 487 |
-
return SCHEDULED_FLOOR_SECONDS // cadence["perDay"]
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
def sync_schedule(states, now=None):
|
| 491 |
-
"""Return every SP-API table's last sync, next sync and configured passes per day."""
|
| 492 |
-
now = _utc(now)
|
| 493 |
-
schedule = []
|
| 494 |
-
for table in TABLES:
|
| 495 |
-
state = states.get(table)
|
| 496 |
-
seconds = _cadence_seconds(table, state)
|
| 497 |
-
_wait, due, _seconds, _scheduled = _interval_wait(table, state, now, floor_s=seconds)
|
| 498 |
-
if due is None and state and state.get("state") == "done":
|
| 499 |
-
vendor = _client().REPORT_TYPES[table].get("min_interval_s", 0)
|
| 500 |
-
due = _utc(state["updated_at"]) + timedelta(seconds=max(vendor, seconds))
|
| 501 |
-
schedule.append({"table": table, "lastSyncAt": state.get("updated_at") if state else None,
|
| 502 |
-
"nextSyncAt": _iso(due or now), "perDay": _client().CADENCE[table]["perDay"]})
|
| 503 |
-
return schedule
|
| 504 |
|
| 505 |
|
| 506 |
def sync_plan(states, tables=None, *, now=None, slot_pass=False):
|
| 507 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 508 |
from harness.amazon_client import REPORT_STAGGER_SECONDS
|
| 509 |
now = _utc(now)
|
| 510 |
want = list(TABLES if tables is None else tables)
|
|
@@ -512,8 +491,10 @@ def sync_plan(states, tables=None, *, now=None, slot_pass=False):
|
|
| 512 |
if state and table != "settlements"}
|
| 513 |
deferred = {}
|
| 514 |
due = []
|
|
|
|
| 515 |
for table in want:
|
| 516 |
-
|
|
|
|
| 517 |
if (reason := _interval_reason(table, states.get(table), now, floor_s=floor)):
|
| 518 |
deferred[table] = reason
|
| 519 |
else:
|
|
@@ -527,37 +508,8 @@ def sync_plan(states, tables=None, *, now=None, slot_pass=False):
|
|
| 527 |
for table in due:
|
| 528 |
deferred[table] = ("table stagger: waiting for a later cadence tick; "
|
| 529 |
f"minimum spacing {REPORT_STAGGER_SECONDS:g}s")
|
| 530 |
-
return {"tables": selected
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
def scheduled_sync_plan(tenant_key, *, rt, now=None):
|
| 534 |
-
"""Combine enabled tables' due plans from each connection's own resident cadence ticks."""
|
| 535 |
-
path = _tenant_mirror_path(tenant_key, rt)
|
| 536 |
-
if path.resolve() != Path(datastore.DB_PATH).resolve():
|
| 537 |
-
raise RuntimeError("Amazon scheduling requires the tenant's resident mirror")
|
| 538 |
-
enabled = read_database_enablement(rt)
|
| 539 |
-
tables = [table for table in TABLES
|
| 540 |
-
if enabled["databases"].get(amazon_schema.DATABASE_PREFIX + table)]
|
| 541 |
-
if not tables:
|
| 542 |
-
return {"tables": [], "deferred": {}}
|
| 543 |
-
now = _utc(now)
|
| 544 |
-
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 545 |
-
try:
|
| 546 |
-
due, deferred = {}, {}
|
| 547 |
-
for entry in account_entries(tenant_key, rt):
|
| 548 |
-
prefix = _account_prefix(entry["connection_id"]) + entity_key("sync_tick_")
|
| 549 |
-
ticks = {entity[len(prefix):]: {"state": phase, "rows": rows, "updated_at": at, "detail": detail}
|
| 550 |
-
for entity, phase, rows, at, detail in con.execute(
|
| 551 |
-
"SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
|
| 552 |
-
"WHERE starts_with(entity, ?)", [prefix]).fetchall()}
|
| 553 |
-
plan = sync_plan(ticks, tables, now=now)
|
| 554 |
-
due.update(dict.fromkeys(plan["tables"]))
|
| 555 |
-
for table, reason in plan["deferred"].items():
|
| 556 |
-
deferred.setdefault(table, []).append(reason)
|
| 557 |
-
return {"tables": list(due), "deferred": {
|
| 558 |
-
table: "; ".join(dict.fromkeys(reasons)) for table, reasons in deferred.items() if table not in due}}
|
| 559 |
-
finally:
|
| 560 |
-
con.close()
|
| 561 |
|
| 562 |
|
| 563 |
def _readable_duration(seconds):
|
|
@@ -573,12 +525,10 @@ def _readable_duration(seconds):
|
|
| 573 |
|
| 574 |
|
| 575 |
def _interval_wait(table, state, now, floor_s=None):
|
| 576 |
-
"""
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
if wait > 0:
|
| 581 |
-
return wait, due, int(wait), False
|
| 582 |
vendor = (_client().REPORT_TYPES.get(table) or {}).get("min_interval_s") or 0
|
| 583 |
seconds = max(int(vendor), int(floor_s or 0))
|
| 584 |
if seconds and state and state["state"] == "done":
|
|
@@ -590,13 +540,11 @@ def _interval_wait(table, state, now, floor_s=None):
|
|
| 590 |
|
| 591 |
|
| 592 |
def _interval_reason(table, state, now, floor_s=None):
|
| 593 |
-
wait, due, seconds,
|
| 594 |
if wait <= 0:
|
| 595 |
return None
|
| 596 |
-
if
|
| 597 |
-
return f"Amazon
|
| 598 |
-
if scheduled:
|
| 599 |
-
return (f"This Amazon table refreshes every {_readable_duration(seconds)}; "
|
| 600 |
f"next due {_iso(due)}")
|
| 601 |
return (f"Amazon allows this report at most once every {_readable_duration(seconds)}; "
|
| 602 |
f"next due {_iso(due)}")
|
|
@@ -4016,7 +3964,7 @@ def _restored_state_rows(rows, tables, refused):
|
|
| 4016 |
return rows
|
| 4017 |
refused = set(refused)
|
| 4018 |
names = sorted(set(tables) | {table_name(t) for t in TABLES}, key=lambda name: (-len(name), name))
|
| 4019 |
-
families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "
|
| 4020 |
RESYNC_PREFIX)
|
| 4021 |
shared = {entity_key(key) for key in (FIRST_LOAD_END_KEY, FIRST_LOAD_STATUS_KEY, FIRST_LOAD_DUE_KEY)}
|
| 4022 |
|
|
@@ -4787,7 +4735,7 @@ def _record_state(con, key, phase, rows=0, detail=None):
|
|
| 4787 |
con.execute(
|
| 4788 |
"INSERT OR REPLACE INTO _sync_state "
|
| 4789 |
"(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)",
|
| 4790 |
-
[str(key), str(phase), None, (str(detail) if payload_key.startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_",
|
| 4791 |
"amzn_resync_", "amzn_waiting_"))
|
| 4792 |
else str(detail)[:300]) if detail else None, int(rows or 0),
|
| 4793 |
_iso(datetime.now(timezone.utc))])
|
|
@@ -4859,25 +4807,31 @@ def _first_load_due(con):
|
|
| 4859 |
|
| 4860 |
|
| 4861 |
def _restore_first_load_wait(tenant_key, con):
|
| 4862 |
-
"""Keep the history floor in the producer and release legacy connector-wide desk holds."""
|
| 4863 |
from core import connector_sync
|
| 4864 |
due = _first_load_due(con)
|
| 4865 |
-
|
|
|
|
|
|
|
| 4866 |
return due
|
| 4867 |
|
| 4868 |
|
| 4869 |
def _loaded_refresh_hold(con, table, now, *, slot_pass=False):
|
| 4870 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4871 |
now = _utc(now)
|
| 4872 |
tick = _read_state(con, entity_key("sync_tick_" + table))
|
| 4873 |
# A queued resync is a repair: only Amazon's own floor holds it.
|
| 4874 |
repair = queued_resync(con, table) is not None
|
| 4875 |
-
floor = 0 if repair else
|
| 4876 |
-
wait, due, _seconds,
|
| 4877 |
reason = None
|
| 4878 |
if wait > 0:
|
| 4879 |
# The card parses "next due <iso>" out of a reason, so every hold keeps that phrase.
|
| 4880 |
-
reason =
|
|
|
|
| 4881 |
mark = read_watermark(con, table)
|
| 4882 |
if _client().REPORT_TYPES.get(table, {}).get("weekly") and mark and not repair:
|
| 4883 |
week = _week_start(mark) + timedelta(days=7)
|
|
@@ -5175,8 +5129,6 @@ def first_load_status(tenant_key, rt=None):
|
|
| 5175 |
remaining = int(values["remaining"])
|
| 5176 |
# S82-T14: one pass sees only its own tables. The job's store record counts the whole load.
|
| 5177 |
job = first_load_job_summary(rt) if _store_documents(rt) else None
|
| 5178 |
-
if job:
|
| 5179 |
-
job["tables"]["inventory"] = inventory_table_status(con)
|
| 5180 |
if job and isinstance(job.get("remaining"), int) and state["state"] != "done":
|
| 5181 |
remaining = job["remaining"]
|
| 5182 |
out = {"state": state["state"], **first_load_payload(markers, remaining,
|
|
@@ -5461,7 +5413,7 @@ def _sync_writer(tenant_key, rt, log):
|
|
| 5461 |
def _sync_delegate(tenant_key, rt, log):
|
| 5462 |
"""Every table but the job's own takes one bounded first-load pass: settlements, period, Group A, SQP.
|
| 5463 |
|
| 5464 |
-
The same pass refreshes each loaded table whose
|
| 5465 |
"""
|
| 5466 |
def delegate(exclude, budget):
|
| 5467 |
tables = None if exclude is None else [t for t in TABLES if t not in set(exclude)]
|
|
@@ -5521,91 +5473,7 @@ def _status_stamp(status, connector):
|
|
| 5521 |
return max(stamps, key=lambda item: item[0])[1] if stamps else None
|
| 5522 |
|
| 5523 |
|
| 5524 |
-
def
|
| 5525 |
-
"""Read inventory counts and account-scoped source proof without changing the mirror or requesting reports."""
|
| 5526 |
-
rows = (int(con.execute("SELECT COUNT(*) FROM amzn_inventory").fetchone()[0])
|
| 5527 |
-
if _columns_of(con, "amzn_inventory") else 0)
|
| 5528 |
-
out = {"rows": rows, "empty": rows == 0}
|
| 5529 |
-
if rows:
|
| 5530 |
-
return out
|
| 5531 |
-
keys = ("amzn_inventory", "amzn_reconcile_inventory", "amzn_refused_inventory", "amzn_inventory_awd")
|
| 5532 |
-
clauses = " OR ".join("entity=? OR ends_with(entity, ?)" for _ in keys)
|
| 5533 |
-
saved = con.execute(
|
| 5534 |
-
"SELECT entity, phase, cursor_wd, rows, updated_at FROM _sync_state WHERE " + clauses
|
| 5535 |
-
+ " OR contains(entity, 'amzn_pending_report_')",
|
| 5536 |
-
[value for key in keys for value in (key, "__" + key)]).fetchall()
|
| 5537 |
-
accounts, pending, unknown_request = {}, [], False
|
| 5538 |
-
for entity, phase, detail, count, stamp in saved:
|
| 5539 |
-
prefix, key = (str(entity).split("__", 1) if str(entity).startswith("amzn_account_")
|
| 5540 |
-
and "__" in str(entity) else ("", str(entity)))
|
| 5541 |
-
if key.startswith("amzn_pending_report_"):
|
| 5542 |
-
try:
|
| 5543 |
-
request = json.loads(detail or "{}")
|
| 5544 |
-
except (ValueError, TypeError):
|
| 5545 |
-
request = {}
|
| 5546 |
-
if not isinstance(request, dict):
|
| 5547 |
-
request = {}
|
| 5548 |
-
if request.get("table") == "inventory":
|
| 5549 |
-
pending.append({"type": request.get("reportType"), "state": phase,
|
| 5550 |
-
"reportId": request.get("reportId"),
|
| 5551 |
-
"processingStatus": request.get("processingStatus"), "updatedAt": stamp})
|
| 5552 |
-
elif not request.get("table"):
|
| 5553 |
-
unknown_request = True
|
| 5554 |
-
continue
|
| 5555 |
-
accounts.setdefault(prefix, {})[key] = {"state": phase, "detail": detail,
|
| 5556 |
-
"rows": count, "updated_at": stamp}
|
| 5557 |
-
if pending:
|
| 5558 |
-
return {**out, "code": "first_load_pending", "reason":
|
| 5559 |
-
"Amazon is preparing the inventory report, so keep sync enabled and retry after it completes.",
|
| 5560 |
-
"reports": pending}
|
| 5561 |
-
# Account markers own their proof once account scoping exists.
|
| 5562 |
-
scoped = {key: state for key, state in accounts.items() if key}
|
| 5563 |
-
accounts = scoped or accounts
|
| 5564 |
-
reports = []
|
| 5565 |
-
for states in accounts.values():
|
| 5566 |
-
marker = states.get("amzn_inventory") or {}
|
| 5567 |
-
refused = refused_reports(states)
|
| 5568 |
-
if refused:
|
| 5569 |
-
return {**out, "code": "refused_by_amazon", "reason": refused[0]["reason"]}
|
| 5570 |
-
proof = states.get("amzn_reconcile_inventory") or {}
|
| 5571 |
-
try:
|
| 5572 |
-
detail = json.loads(proof.get("detail") or "{}")
|
| 5573 |
-
except (ValueError, TypeError):
|
| 5574 |
-
detail = {}
|
| 5575 |
-
if not isinstance(detail, dict):
|
| 5576 |
-
detail = {}
|
| 5577 |
-
dates = detail.get("dates")
|
| 5578 |
-
valid_dates = (isinstance(dates, dict) and bool(dates)
|
| 5579 |
-
and all(isinstance(day, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", day)
|
| 5580 |
-
and type(count) is int and count == 0 for day, count in dates.items()))
|
| 5581 |
-
awd = states.get("amzn_inventory_awd") or {}
|
| 5582 |
-
if (marker.get("state") != "done" or marker.get("rows") != 0 or proof.get("state") != "done"
|
| 5583 |
-
or type(detail.get("count")) is not int or detail["count"] != 0
|
| 5584 |
-
or not isinstance(detail.get("document"), str) or not detail["document"].strip()
|
| 5585 |
-
or detail.get("column") != "_snapshot_date" or not valid_dates
|
| 5586 |
-
or detail.get("start") != min(dates) or detail.get("end") != max(dates)
|
| 5587 |
-
or awd.get("state") != "done" or awd.get("rows") != 0):
|
| 5588 |
-
return {**out, "code": "first_load_pending", "reason":
|
| 5589 |
-
"Inventory source checks are incomplete, so retry Amazon sync to finish the FBA and AWD inventory checks."}
|
| 5590 |
-
reports.append({"type": _client().REPORT_TYPES["inventory"]["report"], "state": proof["state"],
|
| 5591 |
-
"reportId": detail.get("reportId"), "processingStatus": detail.get("processingStatus"),
|
| 5592 |
-
"documentId": detail["document"], "start": detail["start"], "end": detail["end"],
|
| 5593 |
-
"rows": detail["count"], "updatedAt": proof.get("updated_at")})
|
| 5594 |
-
if not reports:
|
| 5595 |
-
if unknown_request:
|
| 5596 |
-
return {**out, "code": "first_load_pending", "reason":
|
| 5597 |
-
"A saved Amazon request has no table identity, so retry sync to refresh the inventory report status."}
|
| 5598 |
-
return {**out, "code": "not_requested", "reason":
|
| 5599 |
-
"No inventory report request is saved, so start Amazon sync to request the FBA ledger."}
|
| 5600 |
-
periods = {(report["start"], report["end"]) for report in reports}
|
| 5601 |
-
start, end = next(iter(periods))
|
| 5602 |
-
period = (start if start == end else f"{start} to {end}") if len(periods) == 1 else "the loaded dates"
|
| 5603 |
-
return {**out, "code": "no_source_rows", "reason":
|
| 5604 |
-
f"Amazon returned no FBA ledger rows for {period} and no AWD inventory, so check seller stock and retry Amazon sync.",
|
| 5605 |
-
"reports": reports}
|
| 5606 |
-
|
| 5607 |
-
|
| 5608 |
-
def first_load_table_status(tasks, finished, failed, counts, *, table_status=None):
|
| 5609 |
"""D-2643, D-2642, D-2649: a row count or a named reason per table, and why orders are behind."""
|
| 5610 |
out = {}
|
| 5611 |
for table in dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES):
|
|
@@ -5619,25 +5487,17 @@ def first_load_table_status(tasks, finished, failed, counts, *, table_status=Non
|
|
| 5619 |
f"The first load is still loading this table. {len(left)} report windows remain."
|
| 5620 |
if left else "Amazon returned no rows for this seller in the loaded period."
|
| 5621 |
if own else "This table holds no rows and the first load owes it no windows.")
|
| 5622 |
-
if table == "inventory":
|
| 5623 |
-
row.update(empty=True, code="first_load_pending", reason=
|
| 5624 |
-
"Inventory source proof is not available in this job record, so retry Amazon sync to refresh its status.")
|
| 5625 |
if table in BEHIND_TABLES and left:
|
| 5626 |
done = [t for t in own if t["id"] in finished or t.get("landed")]
|
| 5627 |
oldest = min((t["start"] for t in done if t.get("start")), default=None)
|
| 5628 |
row["behind"] = (f"The first load is still loading this table. {len(left)} of {len(own)} report "
|
| 5629 |
"windows remain" + (f", and history before {oldest[:10]} is not loaded yet." if oldest
|
| 5630 |
else ", starting with the most recent month."))
|
| 5631 |
-
if table_status and table in table_status:
|
| 5632 |
-
row.update(table_status[table])
|
| 5633 |
-
if not row.get("empty"):
|
| 5634 |
-
row.pop("reason", None)
|
| 5635 |
-
row.pop("code", None)
|
| 5636 |
out[table] = row
|
| 5637 |
return out
|
| 5638 |
|
| 5639 |
|
| 5640 |
-
def first_load_job_summary(docs
|
| 5641 |
"""The latest generation record and the cursor's remaining count, read from the store."""
|
| 5642 |
try:
|
| 5643 |
record = _doc(docs, FIRST_LOAD_GENERATIONS_KEY)
|
|
@@ -5651,7 +5511,7 @@ def first_load_job_summary(docs, *, table_status=None):
|
|
| 5651 |
"remaining": record.get("remaining", last.get("remainingAfter")),
|
| 5652 |
"tablesPublished": list(last.get("tablesPublished") or []), "end": last.get("end"),
|
| 5653 |
"operation": last.get("operation"), "resume_at": last.get("resume_at"),
|
| 5654 |
-
"tables":
|
| 5655 |
|
| 5656 |
|
| 5657 |
def _record_generation(docs, row):
|
|
@@ -5927,15 +5787,7 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
|
|
| 5927 |
row_counts = counts(list(dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES)))
|
| 5928 |
except Exception: # noqa: BLE001
|
| 5929 |
row_counts = {}
|
| 5930 |
-
|
| 5931 |
-
if plan is None:
|
| 5932 |
-
con, _path = _connect_tenant_mirror(key, rt)
|
| 5933 |
-
try:
|
| 5934 |
-
table_status = {"inventory": inventory_table_status(con)}
|
| 5935 |
-
finally:
|
| 5936 |
-
con.close()
|
| 5937 |
-
record["tables"] = first_load_table_status(after, finished, failed, row_counts,
|
| 5938 |
-
table_status=table_status)
|
| 5939 |
record["state"] = "done" if not record["remainingAfter"] else "partial"
|
| 5940 |
status_after = read_connector_status(docs)
|
| 5941 |
record["odooLastGoodSyncAt"] = {"start": _status_stamp(status_before, "odoo"),
|
|
@@ -5960,7 +5812,7 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
|
|
| 5960 |
pass
|
| 5961 |
|
| 5962 |
|
| 5963 |
-
def sync_window(table, mark, *, now, since=None, backfill_days=None
|
| 5964 |
"""Pure request decision for one table: start, end, report, incremental and cause."""
|
| 5965 |
end = _utc(now)
|
| 5966 |
days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days
|
|
@@ -5978,11 +5830,6 @@ def sync_window(table, mark, *, now, since=None, backfill_days=None, intraday=Fa
|
|
| 5978 |
start, incremental = end - timedelta(days=days), False
|
| 5979 |
cause = f"first full load: {days} days of backfill"
|
| 5980 |
spec = _client().REPORT_TYPES.get(table) or {}
|
| 5981 |
-
if incremental and not spec.get("snapshot") and not _periodic(spec):
|
| 5982 |
-
cadence = _client().CADENCE[table]
|
| 5983 |
-
lookback = cadence["intradayLookbackDays" if intraday else "lookbackDays"]
|
| 5984 |
-
start = min(start, end - timedelta(days=lookback))
|
| 5985 |
-
cause += f"; re-read {lookback} days ({'intraday' if intraday else 'daily'})"
|
| 5986 |
rtype = spec.get("report_incremental") if incremental and spec.get("report_incremental") else spec.get("report")
|
| 5987 |
rtype = rtype or spec.get("api")
|
| 5988 |
if table in ("economics", "order_metrics"):
|
|
@@ -6011,22 +5858,7 @@ def sync_window(table, mark, *, now, since=None, backfill_days=None, intraday=Fa
|
|
| 6011 |
# The watermark filters issued reports by creation time; it never requests one.
|
| 6012 |
rtype = spec.get("report_discover")
|
| 6013 |
return {"start": start, "end": end, "report": rtype,
|
| 6014 |
-
"incremental": incremental, "
|
| 6015 |
-
|
| 6016 |
-
|
| 6017 |
-
def _lookback_intraday(con, table, now, *, slot_pass=False):
|
| 6018 |
-
"""Use the daily window until this account has completed it, including after a failed attempt."""
|
| 6019 |
-
day = _read_state(con, entity_key("lookback_day_" + table))
|
| 6020 |
-
return bool(not slot_pass and day and day.get("state") == "done"
|
| 6021 |
-
and day.get("detail") == now.date().isoformat())
|
| 6022 |
-
|
| 6023 |
-
|
| 6024 |
-
def _record_lookback_day(con, table, plan, now):
|
| 6025 |
-
"""Only a completed incremental daily request earns this account's intraday window."""
|
| 6026 |
-
if (plan["incremental"] and not plan.get("intraday")
|
| 6027 |
-
and plan["end"] >= now - timedelta(minutes=2)
|
| 6028 |
-
and not _lookback_intraday(con, table, now)):
|
| 6029 |
-
_record_state(con, entity_key("lookback_day_" + table), "done", detail=now.date().isoformat())
|
| 6030 |
|
| 6031 |
|
| 6032 |
def _sync_event(kind, tenant, table, detail, rt, report, log):
|
|
@@ -6123,21 +5955,10 @@ def _pending_report_key(table, report_type, **kwargs):
|
|
| 6123 |
return entity_key("pending_report_" + hashlib.sha256(identity.encode()).hexdigest())
|
| 6124 |
|
| 6125 |
|
| 6126 |
-
def _save_throttled_resume(con, table, exc, plan, completed, *, first_load, since):
|
| 6127 |
-
"""Keep the continuation beside the account's markers and leave the retry tick as an ISO deadline."""
|
| 6128 |
-
continuation = {**exc.as_resume(), "plan": {**plan, "start": _iso(plan["start"]), "end": _iso(plan["end"])},
|
| 6129 |
-
"completed": completed, "first_load": first_load,
|
| 6130 |
-
"since": _iso(_utc(since)) if since is not None else None}
|
| 6131 |
-
_record_state(con, entity_key("resume_" + table), "throttled", detail=json.dumps(continuation))
|
| 6132 |
-
due = datetime.now(timezone.utc) + timedelta(seconds=max(60.0, float(exc.retry_after or 60)))
|
| 6133 |
-
_record_state(con, entity_key("sync_tick_" + table), "throttled", detail=_iso(due))
|
| 6134 |
-
return max(0, (due - datetime.now(timezone.utc)).total_seconds())
|
| 6135 |
-
|
| 6136 |
-
|
| 6137 |
@contextmanager
|
| 6138 |
def _report_resume(con, table, report_type, **kwargs):
|
| 6139 |
"""Resume the same report/window for up to 24 hours instead of paying createReport again."""
|
| 6140 |
-
from harness.amazon_client import AmazonReportError, AmazonReportPending
|
| 6141 |
window = {key: kwargs.get(key) for key in ("start", "end", "options")}
|
| 6142 |
key = _pending_report_key(table, report_type, **kwargs)
|
| 6143 |
state = _read_state(con, key)
|
|
@@ -6152,15 +5973,7 @@ def _report_resume(con, table, report_type, **kwargs):
|
|
| 6152 |
yield kwargs
|
| 6153 |
except AmazonReportPending as exc:
|
| 6154 |
_record_state(con, key, "pending", detail=json.dumps({
|
| 6155 |
-
"reportId": exc.report_id, "createdAt": saved.get("createdAt", stamp), "window": window
|
| 6156 |
-
"table": table, "reportType": report_type, "processingStatus": exc.status}))
|
| 6157 |
-
raise
|
| 6158 |
-
except AmazonThrottled as exc:
|
| 6159 |
-
if exc.report_id is not None:
|
| 6160 |
-
_record_state(con, key, "throttled", detail=json.dumps({
|
| 6161 |
-
**exc.as_resume(), "reportId": str(exc.report_id),
|
| 6162 |
-
"createdAt": saved.get("createdAt", stamp), "window": window,
|
| 6163 |
-
"table": table, "reportType": report_type}))
|
| 6164 |
raise
|
| 6165 |
except AmazonReportError:
|
| 6166 |
con.execute("DELETE FROM _sync_state WHERE entity=?", [key])
|
|
@@ -6199,21 +6012,6 @@ def _record_reconciliation(con, table, marker):
|
|
| 6199 |
_record_state(con, key, "done", detail=json.dumps(marker, separators=(",", ":")))
|
| 6200 |
|
| 6201 |
|
| 6202 |
-
def _restatement_may_overlap(con, mirror, predicate, params, start, end, *, unsafe="TRUE"):
|
| 6203 |
-
"""Treat absent or invalid legacy period metadata as unknown when checking retained rows."""
|
| 6204 |
-
columns = set(_columns_of(con, mirror))
|
| 6205 |
-
def period(name):
|
| 6206 |
-
return f'TRY_CAST("{name}" AS TIMESTAMPTZ)' if name in columns else "NULL::TIMESTAMPTZ"
|
| 6207 |
-
old_start, old_end = period("_period_start"), period("_period_end")
|
| 6208 |
-
return bool(con.execute(
|
| 6209 |
-
f'SELECT 1 FROM {mirror} WHERE NOT COALESCE(({predicate}), FALSE) AND ({unsafe}) '
|
| 6210 |
-
f'AND ({old_start} IS NULL OR TRY_CAST(? AS TIMESTAMPTZ) IS NULL '
|
| 6211 |
-
f'OR {old_start} < TRY_CAST(? AS TIMESTAMPTZ)) '
|
| 6212 |
-
f'AND ({old_end} IS NULL OR {old_start} IS NULL OR {old_end} <= {old_start} '
|
| 6213 |
-
f'OR {old_end} > TRY_CAST(? AS TIMESTAMPTZ)) LIMIT 1',
|
| 6214 |
-
[*params, end, end, start]).fetchone())
|
| 6215 |
-
|
| 6216 |
-
|
| 6217 |
def _replace_period(con, table, rows, columns, column, start, end=None, marker=None, *, snapshot=False):
|
| 6218 |
"""Atomic range/date replacement and proof; an empty restatement removes old rows too.
|
| 6219 |
|
|
@@ -6243,44 +6041,10 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
|
|
| 6243 |
if column != "_snapshot_date" or end:
|
| 6244 |
raise ValueError("a snapshot replacement is keyed by one _snapshot_date")
|
| 6245 |
predicate, params = "TRUE", []
|
| 6246 |
-
source_keys = _client().REPORT_TYPES[table].get("restatement_keys", ())
|
| 6247 |
-
staged = None
|
| 6248 |
-
if source_keys:
|
| 6249 |
-
overlap_end = (marker or {}).get("end") or end
|
| 6250 |
-
missing = set(source_keys) - set(columns)
|
| 6251 |
-
if missing:
|
| 6252 |
-
# A same-window replay remains safe without source keys; a moving overlap must refuse.
|
| 6253 |
-
if _restatement_may_overlap(con, mirror, predicate, params, start, overlap_end):
|
| 6254 |
-
raise ValueError(f"{table}: overlapping restatement omitted source keys: "
|
| 6255 |
-
+ ", ".join(sorted(missing)) + "; request the complete source report")
|
| 6256 |
-
elif rows:
|
| 6257 |
-
staged = "amazon_restatement_" + uuid.uuid4().hex
|
| 6258 |
-
names = ", ".join(_quoted_identifier(c) for c in columns)
|
| 6259 |
-
con.execute(f"CREATE TEMP TABLE {staged} AS SELECT {names} FROM {mirror} WHERE FALSE")
|
| 6260 |
-
try:
|
| 6261 |
-
_replace(con, staged, columns, types, rows, source_table=mirror)
|
| 6262 |
-
required = [c for c in source_keys if c not in ("fnsku", "asin")]
|
| 6263 |
-
invalid = " OR ".join(f'NULLIF(TRIM(CAST({_quoted_identifier(c)} AS VARCHAR)), \'\') IS NULL'
|
| 6264 |
-
for c in required)
|
| 6265 |
-
if con.execute(f"SELECT 1 FROM {staged} WHERE {invalid} LIMIT 1").fetchone():
|
| 6266 |
-
raise ValueError(f"{table}: restatement omitted a source key value; request the complete source report")
|
| 6267 |
-
if _restatement_may_overlap(con, mirror, predicate, params, start, overlap_end, unsafe=invalid):
|
| 6268 |
-
raise ValueError(f"{table}: overlapping legacy rows omitted a source key value; "
|
| 6269 |
-
"restore the complete source report before restating")
|
| 6270 |
-
except BaseException:
|
| 6271 |
-
con.execute(f"DROP TABLE {staged}")
|
| 6272 |
-
raise
|
| 6273 |
con.execute("BEGIN TRANSACTION")
|
| 6274 |
try:
|
| 6275 |
con.execute(f'DELETE FROM {mirror} WHERE {predicate}', params)
|
| 6276 |
-
|
| 6277 |
-
matches = " AND ".join(f'{mirror}.{_quoted_identifier(c)} IS NOT DISTINCT FROM '
|
| 6278 |
-
f'k.{_quoted_identifier(c)}' for c in source_keys)
|
| 6279 |
-
con.execute(f'DELETE FROM {mirror} WHERE EXISTS (SELECT 1 FROM {staged} k WHERE {matches})')
|
| 6280 |
-
con.execute(f"INSERT INTO {mirror} ({names}) SELECT {names} FROM {staged}")
|
| 6281 |
-
count = con.execute(f"SELECT COUNT(*) FROM {staged}").fetchone()[0]
|
| 6282 |
-
else:
|
| 6283 |
-
count = _replace(con, mirror, columns, types, rows)
|
| 6284 |
if marker is not None:
|
| 6285 |
_record_reconciliation(con, table, marker)
|
| 6286 |
if table == "listing_items":
|
|
@@ -6290,9 +6054,6 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
|
|
| 6290 |
except Exception:
|
| 6291 |
con.execute("ROLLBACK")
|
| 6292 |
raise
|
| 6293 |
-
finally:
|
| 6294 |
-
if staged:
|
| 6295 |
-
con.execute(f"DROP TABLE {staged}")
|
| 6296 |
|
| 6297 |
|
| 6298 |
def document_row_count(text, roots=()):
|
|
@@ -6435,7 +6196,6 @@ def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count):
|
|
| 6435 |
dates = sorted(counts)
|
| 6436 |
marker = {"start": dates[0], "end": dates[-1], "column": "_snapshot_date",
|
| 6437 |
"count": source_count, "dates": counts, "document": meta.get("reportDocumentId")}
|
| 6438 |
-
marker.update({key: meta[key] for key in ("reportId", "processingStatus") if meta.get(key)})
|
| 6439 |
# Every earlier date and any undated pre-migration copy goes; AWD rows are another leg.
|
| 6440 |
predicate = "COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" else "TRUE"
|
| 6441 |
con.execute("BEGIN TRANSACTION")
|
|
@@ -7710,8 +7470,7 @@ def _sync_accounts_candidate(tenant_key="royal-imports", tables=None, since=None
|
|
| 7710 |
for entry in entries:
|
| 7711 |
cid = entry["connection_id"]
|
| 7712 |
try:
|
| 7713 |
-
with _account_workspace(con, tenant_key, rt, entry) as scope
|
| 7714 |
-
_client().rate_limit_scope(tenant_key, cid):
|
| 7715 |
_sync_credential(tenant_key, rt, connection_id=cid)
|
| 7716 |
result = _sync_candidate(tenant_key, tables=tables, since=since, log=log, rt=rt,
|
| 7717 |
max_wait=max_wait, scheduled=scheduled, first_load=first_load,
|
|
@@ -7882,7 +7641,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 7882 |
ticks.update({name: None for name in (table, pairs.get(table)) if name in want})
|
| 7883 |
for table in want:
|
| 7884 |
tick = ticks.get(table)
|
| 7885 |
-
# A queued resync is a repair: only Amazon's own floor and the stagger hold it.
|
| 7886 |
if (tick and tick["state"] == "done" and queued_resync(con, table) is not None
|
| 7887 |
and read_watermark(con, table) and not _interval_reason(table, tick, end)):
|
| 7888 |
ticks[table] = {**tick, "state": "resync"}
|
|
@@ -8016,15 +7775,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8016 |
else:
|
| 8017 |
for table in pending:
|
| 8018 |
mark = read_watermark(con, table)
|
| 8019 |
-
if table == "settlements" and since is None:
|
| 8020 |
-
# Empty discovery earns a request cutoff, never a stored source watermark.
|
| 8021 |
-
tick = _read_state(con, entity_key("sync_tick_" + table))
|
| 8022 |
-
if tick and tick.get("state") == "done":
|
| 8023 |
-
mark = _utc(tick["updated_at"])
|
| 8024 |
-
elif not mark:
|
| 8025 |
-
day = _read_state(con, entity_key("lookback_day_" + table))
|
| 8026 |
-
if day and day.get("state") == "done":
|
| 8027 |
-
mark = _utc(day["updated_at"])
|
| 8028 |
if not mark and since is None and table != "settlements" and table in completed:
|
| 8029 |
# A local fallback cannot reopen old daily history or invent a stored watermark.
|
| 8030 |
recent = end - timedelta(days=FIRST_LOAD_RECENT_DAYS)
|
|
@@ -8034,8 +7784,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8034 |
mark = recent
|
| 8035 |
if not recent <= mark <= end:
|
| 8036 |
mark = recent
|
| 8037 |
-
plan = plans[table] = sync_window(table, mark, now=end, since=since
|
| 8038 |
-
intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass))
|
| 8039 |
if table == SQP_TABLE:
|
| 8040 |
months = _month_windows(plan["start"], plan["end"])
|
| 8041 |
if not plan["incremental"]:
|
|
@@ -8049,16 +7798,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8049 |
else:
|
| 8050 |
work.append((table, None, None, None))
|
| 8051 |
|
| 8052 |
-
resumes = {}
|
| 8053 |
-
for table in pending:
|
| 8054 |
-
saved = _read_state(con, entity_key("resume_" + table))
|
| 8055 |
-
continuation = json.loads(saved["detail"]) if saved and saved.get("detail") else {}
|
| 8056 |
-
if (continuation and continuation.get("first_load") == first_load
|
| 8057 |
-
and continuation.get("since") == (_iso(_utc(since)) if since is not None else None)):
|
| 8058 |
-
resumes[table] = continuation
|
| 8059 |
-
plans[table] = {**continuation["plan"], "start": _utc(continuation["plan"]["start"]),
|
| 8060 |
-
"end": _utc(continuation["plan"]["end"])}
|
| 8061 |
-
|
| 8062 |
if sqp_limit is not None:
|
| 8063 |
def continuation_first(task):
|
| 8064 |
_table, w0, w1, batch = task
|
|
@@ -8095,14 +7834,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8095 |
and (task_end is not None or table == "settlements"
|
| 8096 |
or table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES)):
|
| 8097 |
break
|
| 8098 |
-
tick = _read_state(con, entity_key("sync_tick_" + table))
|
| 8099 |
-
if first_load and (tick or {}).get("state") == "throttled":
|
| 8100 |
-
reason = _interval_reason(table, tick, datetime.now(timezone.utc))
|
| 8101 |
-
if reason:
|
| 8102 |
-
report["tables"][table] = {"state": "deferred", "reason": reason}
|
| 8103 |
-
wait = _interval_wait(table, tick, datetime.now(timezone.utc))[0]
|
| 8104 |
-
report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait)
|
| 8105 |
-
continue
|
| 8106 |
if table == SQP_TABLE and task_start is not None and _sqp_period(task_start, task_end) == "WEEK":
|
| 8107 |
weekly_limit = {**sqp_limit, "refusedAsins": {}, "errors": []}
|
| 8108 |
causes = []
|
|
@@ -8125,16 +7856,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8125 |
report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900)
|
| 8126 |
_record_state(con, entity_key(table), "pending", 0, weekly_pending["reason"])
|
| 8127 |
except Exception as exc:
|
| 8128 |
-
if isinstance(exc, cx.AmazonThrottled):
|
| 8129 |
-
problem = _sync_problem(exc, table, exc.operation or "report_document", tenant_key, rt, report, log)
|
| 8130 |
-
report["tables"][table] = problem
|
| 8131 |
-
failed.add(table)
|
| 8132 |
-
pending_reports.add(table)
|
| 8133 |
-
wait = _save_throttled_resume(con, table, exc, plans[table], [],
|
| 8134 |
-
first_load=first_load, since=since)
|
| 8135 |
-
report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait)
|
| 8136 |
-
_record_state(con, entity_key(table), "throttled", detail=problem["reason"])
|
| 8137 |
-
continue
|
| 8138 |
causes.append(str(exc))
|
| 8139 |
if isinstance(exc, cx.AmazonError) and not isinstance(exc, (cx.AmazonReportPending, cx.AmazonThrottled)):
|
| 8140 |
marker = _sqp_month(con, task_start, task_end)
|
|
@@ -8151,6 +7872,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8151 |
continue
|
| 8152 |
if first_load and (table in GROUP_A_TABLES or (table in prior_refusals
|
| 8153 |
and cx.REPORT_TYPES[table].get("min_interval_s"))):
|
|
|
|
| 8154 |
reason = _interval_reason(table, tick, datetime.now(timezone.utc))
|
| 8155 |
if reason:
|
| 8156 |
report["tables"][table] = {"state": "deferred", "reason": reason}
|
|
@@ -8210,7 +7932,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8210 |
proof_key = entity_key("reconcile_" + table)
|
| 8211 |
previous_proof = _read_state(con, proof_key) if table in invalid_proofs else None
|
| 8212 |
resync, resync_count = [], 0
|
| 8213 |
-
completed_windows = list(resumes.get(table, {}).get("completed", []))
|
| 8214 |
try:
|
| 8215 |
if previous_proof:
|
| 8216 |
# The recorder accepts an empty object; only a parsed document can replace it with proof.
|
|
@@ -8220,9 +7941,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8220 |
_record_state(con, entity_key("sync_tick_" + table), "attempted")
|
| 8221 |
if sibling and not first_load:
|
| 8222 |
_record_state(con, entity_key("sync_tick_" + sibling), "attempted")
|
| 8223 |
-
sibling_written, sibling_cleared = 0,
|
| 8224 |
-
written, cols, cleared = 0, [],
|
| 8225 |
-
landed_window =
|
| 8226 |
pricing_asins = []
|
| 8227 |
operations = {}
|
| 8228 |
if first_load:
|
|
@@ -8272,17 +7993,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8272 |
"state": "done", "reports": len(discovered),
|
| 8273 |
"downloaded": downloaded, "skipped": skipped}
|
| 8274 |
if not discovered:
|
| 8275 |
-
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + table)])
|
| 8276 |
report["tables"][table] = {"rows": 0, "operations": operations,
|
| 8277 |
"state": "empty", "reason": "No issued settlement reports found."}
|
| 8278 |
landed.add((table, task_start, task_end, task_batch))
|
| 8279 |
if first_load:
|
| 8280 |
_record_state(con, key, "done", 0, "No issued settlement reports found.")
|
| 8281 |
-
#
|
| 8282 |
-
if scheduled and not first_load:
|
| 8283 |
-
_record_state(con, entity_key("sync_tick_" + table), "done", 0)
|
| 8284 |
-
if not first_load and since is None:
|
| 8285 |
-
_record_lookback_day(con, table, {**plan, "incremental": True}, end)
|
| 8286 |
continue
|
| 8287 |
if first_load and settlement_remaining:
|
| 8288 |
break
|
|
@@ -8331,8 +8047,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8331 |
plan["cause"] = (f"resync of {len(margin_windows)} stored windows "
|
| 8332 |
f"from {_iso(margin_windows[0][0])}")
|
| 8333 |
log(f"{table}: {plan['cause']}")
|
| 8334 |
-
elif
|
| 8335 |
-
and not spec.get("restatement_keys")):
|
| 8336 |
# One due window per table tick respects the vendor floor. The watermark
|
| 8337 |
# advances only to that window's end, so the following tick loses no history.
|
| 8338 |
remaining_windows = len(margin_windows) - 1
|
|
@@ -8348,9 +8063,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8348 |
if not reason.startswith("Amazon keeps this report back to "):
|
| 8349 |
_record_state(con, entity_key("limit_" + table), "done", detail="{}")
|
| 8350 |
for w0, w1 in margin_windows:
|
| 8351 |
-
window = [_iso(w0) if w0 is not None else None, _iso(w1) if w1 is not None else None]
|
| 8352 |
-
if window in completed_windows:
|
| 8353 |
-
continue
|
| 8354 |
# One detail window is one budget task, including its reconciliation query.
|
| 8355 |
probe = _DocumentProbe(table, spec.get("report") or operation)
|
| 8356 |
documents += 1
|
|
@@ -8366,7 +8078,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8366 |
spend_rows(count)
|
| 8367 |
written += count
|
| 8368 |
landed_window = True
|
| 8369 |
-
completed_windows.append(window)
|
| 8370 |
if table in LEGACY_SNAPSHOT_TABLES:
|
| 8371 |
# The dated replacement already landed; AWD must not clear that
|
| 8372 |
# document or historical FBA/AWD rows on an explicit first load.
|
|
@@ -8390,9 +8101,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8390 |
plan["end"] = windows[0][1]
|
| 8391 |
weekly = bool(table != "settlements" and cx.REPORT_TYPES[table].get("weekly"))
|
| 8392 |
for w0, w1 in (() if table == "settlements" else windows):
|
| 8393 |
-
window = [_iso(w0), _iso(w1)]
|
| 8394 |
-
if window in completed_windows:
|
| 8395 |
-
continue
|
| 8396 |
# W59-T43 / C2 has no amazon.access kind. Record each credential-backed
|
| 8397 |
# vendor attempt as a sync ACCESS phase, distinct from the completion event.
|
| 8398 |
# Lane F/combine own the vocabulary decision; no credential or row enters C2.
|
|
@@ -8471,7 +8179,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8471 |
con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1")
|
| 8472 |
cleared = True
|
| 8473 |
else:
|
| 8474 |
-
completed_windows.append(window)
|
| 8475 |
continue
|
| 8476 |
if sibling:
|
| 8477 |
sibling_mirror = table_name(sibling)
|
|
@@ -8512,7 +8219,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8512 |
_clear_week_wait(con, table, w0)
|
| 8513 |
probe.finish(len(rows) if cols else 0)
|
| 8514 |
spend_rows(len(rows) if cols else 0)
|
| 8515 |
-
completed_windows.append(window)
|
| 8516 |
if first_load:
|
| 8517 |
for reached in (table, sibling):
|
| 8518 |
if reached:
|
|
@@ -8562,8 +8268,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8562 |
log(f" pricing: getCompetitiveSummary {response_rows:,} ASIN responses, "
|
| 8563 |
f"{updated:,} mirror rows enriched")
|
| 8564 |
except Exception as e: # noqa: BLE001
|
| 8565 |
-
if isinstance(e, cx.AmazonThrottled):
|
| 8566 |
-
raise
|
| 8567 |
problem = _sync_problem(e, table, "getCompetitiveSummary", tenant_key,
|
| 8568 |
rt, report, log)
|
| 8569 |
_record_state(con, state_key, problem["state"], 0, problem["reason"])
|
|
@@ -8603,7 +8307,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8603 |
report["tables"][table]["nextWeek"] = _iso(period_repair_next)
|
| 8604 |
_record_state(con, key, "done", total_rows)
|
| 8605 |
_record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
|
| 8606 |
-
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + table)])
|
| 8607 |
if weekly and landed_window:
|
| 8608 |
# A landed week closes any earlier refusal of this table.
|
| 8609 |
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("refused_" + table)])
|
|
@@ -8619,10 +8322,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8619 |
or landed_window or (first_load and floors[table] is not None))
|
| 8620 |
if mark is not None and has_window:
|
| 8621 |
record_watermark(con, table, mark, total_rows)
|
| 8622 |
-
if not first_load and not resync:
|
| 8623 |
-
completed_plan = ({**plan, "incremental": True}
|
| 8624 |
-
if table == "settlements" and since is None else plan)
|
| 8625 |
-
_record_lookback_day(con, table, completed_plan, end)
|
| 8626 |
_sync_event("amazon.sync", tenant_key, table,
|
| 8627 |
{"state": "done", "rows": total_rows,
|
| 8628 |
"operations_ok": all(v["state"] == "done" for v in operations.values())},
|
|
@@ -8642,10 +8341,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8642 |
"incremental": sibling_plan["incremental"], "operations": {}}
|
| 8643 |
_record_state(con, entity_key(sibling), "done", sibling_total)
|
| 8644 |
_record_state(con, entity_key("sync_tick_" + sibling), "done", sibling_total)
|
| 8645 |
-
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + sibling)])
|
| 8646 |
record_watermark(con, sibling, sibling_plan["end"], sibling_total)
|
| 8647 |
-
if not first_load:
|
| 8648 |
-
_record_lookback_day(con, sibling, sibling_plan, end)
|
| 8649 |
_sync_event("amazon.sync", tenant_key, sibling,
|
| 8650 |
{"state": "done", "rows": sibling_total, "operations_ok": True},
|
| 8651 |
rt, report, log)
|
|
@@ -8667,17 +8363,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8667 |
# A failed pass retains its previous watermark so its window will be retried.
|
| 8668 |
# ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
|
| 8669 |
# ticket answering itself, and it must not cost the six tables that did work.
|
| 8670 |
-
problem = _sync_problem(e, table,
|
| 8671 |
failed.add(table)
|
| 8672 |
report["tables"][table] = {"rows": 0, **problem}
|
| 8673 |
-
if problem["state"] == "throttled":
|
| 8674 |
-
landed.discard((table, task_start, task_end, task_batch))
|
| 8675 |
-
pending_reports.add(table)
|
| 8676 |
-
if _columns_of(con, mirror):
|
| 8677 |
-
report["tables"][table]["rows"] = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]
|
| 8678 |
-
wait = _save_throttled_resume(con, table, e, plan, completed_windows,
|
| 8679 |
-
first_load=first_load, since=since)
|
| 8680 |
-
report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait)
|
| 8681 |
if problem["state"] == "pending":
|
| 8682 |
pending_reports.add(table)
|
| 8683 |
report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900)
|
|
@@ -8695,13 +8383,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8695 |
"reason": problem["reason"][:500], "amazonStatus": problem["amazonStatus"],
|
| 8696 |
"diagnostic": problem.get("diagnostic", "")[:500],
|
| 8697 |
"error": problem.get("error", ""), "attempts": attempts}))
|
| 8698 |
-
|
| 8699 |
-
|
| 8700 |
-
_save_throttled_resume(con, refused, e, sibling_plan, completed_windows,
|
| 8701 |
-
first_load=first_load, since=since)
|
| 8702 |
-
else:
|
| 8703 |
-
_record_state(con, entity_key("sync_tick_" + refused),
|
| 8704 |
-
"attempted" if problem["state"] == "pending" else "done")
|
| 8705 |
except Exception: # noqa: BLE001
|
| 8706 |
pass
|
| 8707 |
_sync_event("amazon.sync", tenant_key, table,
|
|
@@ -10929,8 +10612,8 @@ def group_a_selftest(log=print, check=None):
|
|
| 10929 |
clock[0] += timedelta(seconds=cx.REPORT_TYPES["long_term_storage_fees"]["min_interval_s"] + 60)
|
| 10930 |
idle = sync(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None,
|
| 10931 |
scheduled=True)["tables"]["long_term_storage_fees"]
|
| 10932 |
-
ck("NC: with the queue empty the
|
| 10933 |
-
idle.get("state") == "deferred" and "
|
| 10934 |
finally:
|
| 10935 |
con.close()
|
| 10936 |
|
|
|
|
| 463 |
return {"available": True, "databases": {key: stored.get(key, True) for key in keys}}
|
| 464 |
|
| 465 |
|
| 466 |
+
#: ⛔⛔ HOTFIX 2026-09-29 (owner): *"We should only sync daily at the fastest."* A scheduled pass
|
| 467 |
+
#: refreshes each table at most once a day. The two hours of grace keep a table stamped late in
|
| 468 |
+
#: yesterday's pass from sliding to 48 hours because today's pass reached it a little earlier.
|
| 469 |
+
#: An operator's Sync now keeps only Amazon's own report floors (owner ruling, same day).
|
| 470 |
SCHEDULED_FLOOR_SECONDS = 86400
|
| 471 |
+
SCHEDULED_FLOOR_GRACE_SECONDS = 2 * 3600
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 472 |
|
| 473 |
|
| 474 |
def sync_plan(states, tables=None, *, now=None, slot_pass=False):
|
| 475 |
+
"""Pure cadence decision: EVERY due table, oldest first, with no stagger sleep.
|
| 476 |
+
|
| 477 |
+
`slot_pass` is the tenant's daily slot pass: it IS the once-a-day sync, so only Amazon's own
|
| 478 |
+
report floors defer a table. Any other scheduled pass keeps the daily floor.
|
| 479 |
+
|
| 480 |
+
Attempt markers survive in the tenant's saved mirror. Failed tables rotate behind peers.
|
| 481 |
+
Settlement LIST discovery runs separately on every resync and consumes no report slot.
|
| 482 |
+
The client still owns pacing/retries within a table.
|
| 483 |
+
⛔ HOTFIX 2026-09-29: this picked ONE table per tick. At a once-a-day cadence that is one
|
| 484 |
+
table a day, about seven weeks between two refreshes of the same table, so a scheduled pass
|
| 485 |
+
now takes every table whose daily floor (and Amazon's own floor) has passed.
|
| 486 |
+
"""
|
| 487 |
from harness.amazon_client import REPORT_STAGGER_SECONDS
|
| 488 |
now = _utc(now)
|
| 489 |
want = list(TABLES if tables is None else tables)
|
|
|
|
| 491 |
if state and table != "settlements"}
|
| 492 |
deferred = {}
|
| 493 |
due = []
|
| 494 |
+
floor = 0 if slot_pass else SCHEDULED_FLOOR_SECONDS - SCHEDULED_FLOOR_GRACE_SECONDS
|
| 495 |
for table in want:
|
| 496 |
+
if table == "settlements":
|
| 497 |
+
continue
|
| 498 |
if (reason := _interval_reason(table, states.get(table), now, floor_s=floor)):
|
| 499 |
deferred[table] = reason
|
| 500 |
else:
|
|
|
|
| 508 |
for table in due:
|
| 509 |
deferred[table] = ("table stagger: waiting for a later cadence tick; "
|
| 510 |
f"minimum spacing {REPORT_STAGGER_SECONDS:g}s")
|
| 511 |
+
return {"tables": selected + (["settlements"] if "settlements" in want else []),
|
| 512 |
+
"deferred": deferred}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 513 |
|
| 514 |
|
| 515 |
def _readable_duration(seconds):
|
|
|
|
| 525 |
|
| 526 |
|
| 527 |
def _interval_wait(table, state, now, floor_s=None):
|
| 528 |
+
"""Seconds until `table` may be requested again (0 when due) and which floor decides it.
|
| 529 |
+
|
| 530 |
+
`floor_s` is the scheduled daily floor; Amazon's own `min_interval_s` always applies too.
|
| 531 |
+
"""
|
|
|
|
|
|
|
| 532 |
vendor = (_client().REPORT_TYPES.get(table) or {}).get("min_interval_s") or 0
|
| 533 |
seconds = max(int(vendor), int(floor_s or 0))
|
| 534 |
if seconds and state and state["state"] == "done":
|
|
|
|
| 540 |
|
| 541 |
|
| 542 |
def _interval_reason(table, state, now, floor_s=None):
|
| 543 |
+
wait, due, seconds, daily = _interval_wait(table, state, now, floor_s=floor_s)
|
| 544 |
if wait <= 0:
|
| 545 |
return None
|
| 546 |
+
if daily:
|
| 547 |
+
return (f"Scheduled syncs refresh each Amazon table at most once a day; "
|
|
|
|
|
|
|
| 548 |
f"next due {_iso(due)}")
|
| 549 |
return (f"Amazon allows this report at most once every {_readable_duration(seconds)}; "
|
| 550 |
f"next due {_iso(due)}")
|
|
|
|
| 3964 |
return rows
|
| 3965 |
refused = set(refused)
|
| 3966 |
names = sorted(set(tables) | {table_name(t) for t in TABLES}, key=lambda name: (-len(name), name))
|
| 3967 |
+
families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "reconcile_", "limit_", "refused_",
|
| 3968 |
RESYNC_PREFIX)
|
| 3969 |
shared = {entity_key(key) for key in (FIRST_LOAD_END_KEY, FIRST_LOAD_STATUS_KEY, FIRST_LOAD_DUE_KEY)}
|
| 3970 |
|
|
|
|
| 4735 |
con.execute(
|
| 4736 |
"INSERT OR REPLACE INTO _sync_state "
|
| 4737 |
"(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)",
|
| 4738 |
+
[str(key), str(phase), None, (str(detail) if payload_key.startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_",
|
| 4739 |
"amzn_resync_", "amzn_waiting_"))
|
| 4740 |
else str(detail)[:300]) if detail else None, int(rows or 0),
|
| 4741 |
_iso(datetime.now(timezone.utc))])
|
|
|
|
| 4807 |
|
| 4808 |
|
| 4809 |
def _restore_first_load_wait(tenant_key, con):
|
|
|
|
| 4810 |
from core import connector_sync
|
| 4811 |
due = _first_load_due(con)
|
| 4812 |
+
if _account_scope() is not None:
|
| 4813 |
+
return due
|
| 4814 |
+
connector_sync.defer_sync("amazon_sp", tenant_key, due.timestamp() if due else None)
|
| 4815 |
return due
|
| 4816 |
|
| 4817 |
|
| 4818 |
def _loaded_refresh_hold(con, table, now, *, slot_pass=False):
|
| 4819 |
+
"""`(next due, reason)` while a loaded table's own floor holds it during a first load, else None.
|
| 4820 |
+
|
| 4821 |
+
A weekly table is due when a new week has closed after its watermark (D-11328). Every other
|
| 4822 |
+
table, settlement discovery included, keeps the daily floor and Amazon's own floor.
|
| 4823 |
+
"""
|
| 4824 |
now = _utc(now)
|
| 4825 |
tick = _read_state(con, entity_key("sync_tick_" + table))
|
| 4826 |
# A queued resync is a repair: only Amazon's own floor holds it.
|
| 4827 |
repair = queued_resync(con, table) is not None
|
| 4828 |
+
floor = 0 if slot_pass or repair else SCHEDULED_FLOOR_SECONDS - SCHEDULED_FLOOR_GRACE_SECONDS
|
| 4829 |
+
wait, due, _seconds, daily = _interval_wait(table, tick, now, floor_s=floor)
|
| 4830 |
reason = None
|
| 4831 |
if wait > 0:
|
| 4832 |
# The card parses "next due <iso>" out of a reason, so every hold keeps that phrase.
|
| 4833 |
+
reason = (f"Each loaded Amazon table refreshes at most once a day while the first load continues. "
|
| 4834 |
+
f"It is next due {_iso(due)}" if daily else _interval_reason(table, tick, now, floor_s=floor))
|
| 4835 |
mark = read_watermark(con, table)
|
| 4836 |
if _client().REPORT_TYPES.get(table, {}).get("weekly") and mark and not repair:
|
| 4837 |
week = _week_start(mark) + timedelta(days=7)
|
|
|
|
| 5129 |
remaining = int(values["remaining"])
|
| 5130 |
# S82-T14: one pass sees only its own tables. The job's store record counts the whole load.
|
| 5131 |
job = first_load_job_summary(rt) if _store_documents(rt) else None
|
|
|
|
|
|
|
| 5132 |
if job and isinstance(job.get("remaining"), int) and state["state"] != "done":
|
| 5133 |
remaining = job["remaining"]
|
| 5134 |
out = {"state": state["state"], **first_load_payload(markers, remaining,
|
|
|
|
| 5413 |
def _sync_delegate(tenant_key, rt, log):
|
| 5414 |
"""Every table but the job's own takes one bounded first-load pass: settlements, period, Group A, SQP.
|
| 5415 |
|
| 5416 |
+
The same pass refreshes each loaded table whose daily floor is due, so loaded data keeps moving.
|
| 5417 |
"""
|
| 5418 |
def delegate(exclude, budget):
|
| 5419 |
tables = None if exclude is None else [t for t in TABLES if t not in set(exclude)]
|
|
|
|
| 5473 |
return max(stamps, key=lambda item: item[0])[1] if stamps else None
|
| 5474 |
|
| 5475 |
|
| 5476 |
+
def first_load_table_status(tasks, finished, failed, counts):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5477 |
"""D-2643, D-2642, D-2649: a row count or a named reason per table, and why orders are behind."""
|
| 5478 |
out = {}
|
| 5479 |
for table in dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES):
|
|
|
|
| 5487 |
f"The first load is still loading this table. {len(left)} report windows remain."
|
| 5488 |
if left else "Amazon returned no rows for this seller in the loaded period."
|
| 5489 |
if own else "This table holds no rows and the first load owes it no windows.")
|
|
|
|
|
|
|
|
|
|
| 5490 |
if table in BEHIND_TABLES and left:
|
| 5491 |
done = [t for t in own if t["id"] in finished or t.get("landed")]
|
| 5492 |
oldest = min((t["start"] for t in done if t.get("start")), default=None)
|
| 5493 |
row["behind"] = (f"The first load is still loading this table. {len(left)} of {len(own)} report "
|
| 5494 |
"windows remain" + (f", and history before {oldest[:10]} is not loaded yet." if oldest
|
| 5495 |
else ", starting with the most recent month."))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5496 |
out[table] = row
|
| 5497 |
return out
|
| 5498 |
|
| 5499 |
|
| 5500 |
+
def first_load_job_summary(docs):
|
| 5501 |
"""The latest generation record and the cursor's remaining count, read from the store."""
|
| 5502 |
try:
|
| 5503 |
record = _doc(docs, FIRST_LOAD_GENERATIONS_KEY)
|
|
|
|
| 5511 |
"remaining": record.get("remaining", last.get("remainingAfter")),
|
| 5512 |
"tablesPublished": list(last.get("tablesPublished") or []), "end": last.get("end"),
|
| 5513 |
"operation": last.get("operation"), "resume_at": last.get("resume_at"),
|
| 5514 |
+
"tables": last.get("tables") or {}}
|
| 5515 |
|
| 5516 |
|
| 5517 |
def _record_generation(docs, row):
|
|
|
|
| 5787 |
row_counts = counts(list(dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES)))
|
| 5788 |
except Exception: # noqa: BLE001
|
| 5789 |
row_counts = {}
|
| 5790 |
+
record["tables"] = first_load_table_status(after, finished, failed, row_counts)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5791 |
record["state"] = "done" if not record["remainingAfter"] else "partial"
|
| 5792 |
status_after = read_connector_status(docs)
|
| 5793 |
record["odooLastGoodSyncAt"] = {"start": _status_stamp(status_before, "odoo"),
|
|
|
|
| 5812 |
pass
|
| 5813 |
|
| 5814 |
|
| 5815 |
+
def sync_window(table, mark, *, now, since=None, backfill_days=None):
|
| 5816 |
"""Pure request decision for one table: start, end, report, incremental and cause."""
|
| 5817 |
end = _utc(now)
|
| 5818 |
days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days
|
|
|
|
| 5830 |
start, incremental = end - timedelta(days=days), False
|
| 5831 |
cause = f"first full load: {days} days of backfill"
|
| 5832 |
spec = _client().REPORT_TYPES.get(table) or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5833 |
rtype = spec.get("report_incremental") if incremental and spec.get("report_incremental") else spec.get("report")
|
| 5834 |
rtype = rtype or spec.get("api")
|
| 5835 |
if table in ("economics", "order_metrics"):
|
|
|
|
| 5858 |
# The watermark filters issued reports by creation time; it never requests one.
|
| 5859 |
rtype = spec.get("report_discover")
|
| 5860 |
return {"start": start, "end": end, "report": rtype,
|
| 5861 |
+
"incremental": incremental, "cause": cause}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5862 |
|
| 5863 |
|
| 5864 |
def _sync_event(kind, tenant, table, detail, rt, report, log):
|
|
|
|
| 5955 |
return entity_key("pending_report_" + hashlib.sha256(identity.encode()).hexdigest())
|
| 5956 |
|
| 5957 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5958 |
@contextmanager
|
| 5959 |
def _report_resume(con, table, report_type, **kwargs):
|
| 5960 |
"""Resume the same report/window for up to 24 hours instead of paying createReport again."""
|
| 5961 |
+
from harness.amazon_client import AmazonReportError, AmazonReportPending
|
| 5962 |
window = {key: kwargs.get(key) for key in ("start", "end", "options")}
|
| 5963 |
key = _pending_report_key(table, report_type, **kwargs)
|
| 5964 |
state = _read_state(con, key)
|
|
|
|
| 5973 |
yield kwargs
|
| 5974 |
except AmazonReportPending as exc:
|
| 5975 |
_record_state(con, key, "pending", detail=json.dumps({
|
| 5976 |
+
"reportId": exc.report_id, "createdAt": saved.get("createdAt", stamp), "window": window}))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5977 |
raise
|
| 5978 |
except AmazonReportError:
|
| 5979 |
con.execute("DELETE FROM _sync_state WHERE entity=?", [key])
|
|
|
|
| 6012 |
_record_state(con, key, "done", detail=json.dumps(marker, separators=(",", ":")))
|
| 6013 |
|
| 6014 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6015 |
def _replace_period(con, table, rows, columns, column, start, end=None, marker=None, *, snapshot=False):
|
| 6016 |
"""Atomic range/date replacement and proof; an empty restatement removes old rows too.
|
| 6017 |
|
|
|
|
| 6041 |
if column != "_snapshot_date" or end:
|
| 6042 |
raise ValueError("a snapshot replacement is keyed by one _snapshot_date")
|
| 6043 |
predicate, params = "TRUE", []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6044 |
con.execute("BEGIN TRANSACTION")
|
| 6045 |
try:
|
| 6046 |
con.execute(f'DELETE FROM {mirror} WHERE {predicate}', params)
|
| 6047 |
+
count = _replace(con, mirror, columns, types, rows)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6048 |
if marker is not None:
|
| 6049 |
_record_reconciliation(con, table, marker)
|
| 6050 |
if table == "listing_items":
|
|
|
|
| 6054 |
except Exception:
|
| 6055 |
con.execute("ROLLBACK")
|
| 6056 |
raise
|
|
|
|
|
|
|
|
|
|
| 6057 |
|
| 6058 |
|
| 6059 |
def document_row_count(text, roots=()):
|
|
|
|
| 6196 |
dates = sorted(counts)
|
| 6197 |
marker = {"start": dates[0], "end": dates[-1], "column": "_snapshot_date",
|
| 6198 |
"count": source_count, "dates": counts, "document": meta.get("reportDocumentId")}
|
|
|
|
| 6199 |
# Every earlier date and any undated pre-migration copy goes; AWD rows are another leg.
|
| 6200 |
predicate = "COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" else "TRUE"
|
| 6201 |
con.execute("BEGIN TRANSACTION")
|
|
|
|
| 7470 |
for entry in entries:
|
| 7471 |
cid = entry["connection_id"]
|
| 7472 |
try:
|
| 7473 |
+
with _account_workspace(con, tenant_key, rt, entry) as scope:
|
|
|
|
| 7474 |
_sync_credential(tenant_key, rt, connection_id=cid)
|
| 7475 |
result = _sync_candidate(tenant_key, tables=tables, since=since, log=log, rt=rt,
|
| 7476 |
max_wait=max_wait, scheduled=scheduled, first_load=first_load,
|
|
|
|
| 7641 |
ticks.update({name: None for name in (table, pairs.get(table)) if name in want})
|
| 7642 |
for table in want:
|
| 7643 |
tick = ticks.get(table)
|
| 7644 |
+
# A queued resync is a repair: only Amazon's own floor and the stagger hold it, never the daily floor.
|
| 7645 |
if (tick and tick["state"] == "done" and queued_resync(con, table) is not None
|
| 7646 |
and read_watermark(con, table) and not _interval_reason(table, tick, end)):
|
| 7647 |
ticks[table] = {**tick, "state": "resync"}
|
|
|
|
| 7775 |
else:
|
| 7776 |
for table in pending:
|
| 7777 |
mark = read_watermark(con, table)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7778 |
if not mark and since is None and table != "settlements" and table in completed:
|
| 7779 |
# A local fallback cannot reopen old daily history or invent a stored watermark.
|
| 7780 |
recent = end - timedelta(days=FIRST_LOAD_RECENT_DAYS)
|
|
|
|
| 7784 |
mark = recent
|
| 7785 |
if not recent <= mark <= end:
|
| 7786 |
mark = recent
|
| 7787 |
+
plan = plans[table] = sync_window(table, mark, now=end, since=since)
|
|
|
|
| 7788 |
if table == SQP_TABLE:
|
| 7789 |
months = _month_windows(plan["start"], plan["end"])
|
| 7790 |
if not plan["incremental"]:
|
|
|
|
| 7798 |
else:
|
| 7799 |
work.append((table, None, None, None))
|
| 7800 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7801 |
if sqp_limit is not None:
|
| 7802 |
def continuation_first(task):
|
| 7803 |
_table, w0, w1, batch = task
|
|
|
|
| 7834 |
and (task_end is not None or table == "settlements"
|
| 7835 |
or table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES)):
|
| 7836 |
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7837 |
if table == SQP_TABLE and task_start is not None and _sqp_period(task_start, task_end) == "WEEK":
|
| 7838 |
weekly_limit = {**sqp_limit, "refusedAsins": {}, "errors": []}
|
| 7839 |
causes = []
|
|
|
|
| 7856 |
report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900)
|
| 7857 |
_record_state(con, entity_key(table), "pending", 0, weekly_pending["reason"])
|
| 7858 |
except Exception as exc:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7859 |
causes.append(str(exc))
|
| 7860 |
if isinstance(exc, cx.AmazonError) and not isinstance(exc, (cx.AmazonReportPending, cx.AmazonThrottled)):
|
| 7861 |
marker = _sqp_month(con, task_start, task_end)
|
|
|
|
| 7872 |
continue
|
| 7873 |
if first_load and (table in GROUP_A_TABLES or (table in prior_refusals
|
| 7874 |
and cx.REPORT_TYPES[table].get("min_interval_s"))):
|
| 7875 |
+
tick = _read_state(con, entity_key("sync_tick_" + table))
|
| 7876 |
reason = _interval_reason(table, tick, datetime.now(timezone.utc))
|
| 7877 |
if reason:
|
| 7878 |
report["tables"][table] = {"state": "deferred", "reason": reason}
|
|
|
|
| 7932 |
proof_key = entity_key("reconcile_" + table)
|
| 7933 |
previous_proof = _read_state(con, proof_key) if table in invalid_proofs else None
|
| 7934 |
resync, resync_count = [], 0
|
|
|
|
| 7935 |
try:
|
| 7936 |
if previous_proof:
|
| 7937 |
# The recorder accepts an empty object; only a parsed document can replace it with proof.
|
|
|
|
| 7941 |
_record_state(con, entity_key("sync_tick_" + table), "attempted")
|
| 7942 |
if sibling and not first_load:
|
| 7943 |
_record_state(con, entity_key("sync_tick_" + sibling), "attempted")
|
| 7944 |
+
sibling_written, sibling_cleared = 0, False
|
| 7945 |
+
written, cols, cleared = 0, [], False
|
| 7946 |
+
landed_window = False
|
| 7947 |
pricing_asins = []
|
| 7948 |
operations = {}
|
| 7949 |
if first_load:
|
|
|
|
| 7993 |
"state": "done", "reports": len(discovered),
|
| 7994 |
"downloaded": downloaded, "skipped": skipped}
|
| 7995 |
if not discovered:
|
|
|
|
| 7996 |
report["tables"][table] = {"rows": 0, "operations": operations,
|
| 7997 |
"state": "empty", "reason": "No issued settlement reports found."}
|
| 7998 |
landed.add((table, task_start, task_end, task_batch))
|
| 7999 |
if first_load:
|
| 8000 |
_record_state(con, key, "done", 0, "No issued settlement reports found.")
|
| 8001 |
+
# Empty discovery earns no cursor, attempt stamp or document marker.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8002 |
continue
|
| 8003 |
if first_load and settlement_remaining:
|
| 8004 |
break
|
|
|
|
| 8047 |
plan["cause"] = (f"resync of {len(margin_windows)} stored windows "
|
| 8048 |
f"from {_iso(margin_windows[0][0])}")
|
| 8049 |
log(f"{table}: {plan['cause']}")
|
| 8050 |
+
elif table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1:
|
|
|
|
| 8051 |
# One due window per table tick respects the vendor floor. The watermark
|
| 8052 |
# advances only to that window's end, so the following tick loses no history.
|
| 8053 |
remaining_windows = len(margin_windows) - 1
|
|
|
|
| 8063 |
if not reason.startswith("Amazon keeps this report back to "):
|
| 8064 |
_record_state(con, entity_key("limit_" + table), "done", detail="{}")
|
| 8065 |
for w0, w1 in margin_windows:
|
|
|
|
|
|
|
|
|
|
| 8066 |
# One detail window is one budget task, including its reconciliation query.
|
| 8067 |
probe = _DocumentProbe(table, spec.get("report") or operation)
|
| 8068 |
documents += 1
|
|
|
|
| 8078 |
spend_rows(count)
|
| 8079 |
written += count
|
| 8080 |
landed_window = True
|
|
|
|
| 8081 |
if table in LEGACY_SNAPSHOT_TABLES:
|
| 8082 |
# The dated replacement already landed; AWD must not clear that
|
| 8083 |
# document or historical FBA/AWD rows on an explicit first load.
|
|
|
|
| 8101 |
plan["end"] = windows[0][1]
|
| 8102 |
weekly = bool(table != "settlements" and cx.REPORT_TYPES[table].get("weekly"))
|
| 8103 |
for w0, w1 in (() if table == "settlements" else windows):
|
|
|
|
|
|
|
|
|
|
| 8104 |
# W59-T43 / C2 has no amazon.access kind. Record each credential-backed
|
| 8105 |
# vendor attempt as a sync ACCESS phase, distinct from the completion event.
|
| 8106 |
# Lane F/combine own the vocabulary decision; no credential or row enters C2.
|
|
|
|
| 8179 |
con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1")
|
| 8180 |
cleared = True
|
| 8181 |
else:
|
|
|
|
| 8182 |
continue
|
| 8183 |
if sibling:
|
| 8184 |
sibling_mirror = table_name(sibling)
|
|
|
|
| 8219 |
_clear_week_wait(con, table, w0)
|
| 8220 |
probe.finish(len(rows) if cols else 0)
|
| 8221 |
spend_rows(len(rows) if cols else 0)
|
|
|
|
| 8222 |
if first_load:
|
| 8223 |
for reached in (table, sibling):
|
| 8224 |
if reached:
|
|
|
|
| 8268 |
log(f" pricing: getCompetitiveSummary {response_rows:,} ASIN responses, "
|
| 8269 |
f"{updated:,} mirror rows enriched")
|
| 8270 |
except Exception as e: # noqa: BLE001
|
|
|
|
|
|
|
| 8271 |
problem = _sync_problem(e, table, "getCompetitiveSummary", tenant_key,
|
| 8272 |
rt, report, log)
|
| 8273 |
_record_state(con, state_key, problem["state"], 0, problem["reason"])
|
|
|
|
| 8307 |
report["tables"][table]["nextWeek"] = _iso(period_repair_next)
|
| 8308 |
_record_state(con, key, "done", total_rows)
|
| 8309 |
_record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
|
|
|
|
| 8310 |
if weekly and landed_window:
|
| 8311 |
# A landed week closes any earlier refusal of this table.
|
| 8312 |
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("refused_" + table)])
|
|
|
|
| 8322 |
or landed_window or (first_load and floors[table] is not None))
|
| 8323 |
if mark is not None and has_window:
|
| 8324 |
record_watermark(con, table, mark, total_rows)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8325 |
_sync_event("amazon.sync", tenant_key, table,
|
| 8326 |
{"state": "done", "rows": total_rows,
|
| 8327 |
"operations_ok": all(v["state"] == "done" for v in operations.values())},
|
|
|
|
| 8341 |
"incremental": sibling_plan["incremental"], "operations": {}}
|
| 8342 |
_record_state(con, entity_key(sibling), "done", sibling_total)
|
| 8343 |
_record_state(con, entity_key("sync_tick_" + sibling), "done", sibling_total)
|
|
|
|
| 8344 |
record_watermark(con, sibling, sibling_plan["end"], sibling_total)
|
|
|
|
|
|
|
| 8345 |
_sync_event("amazon.sync", tenant_key, sibling,
|
| 8346 |
{"state": "done", "rows": sibling_total, "operations_ok": True},
|
| 8347 |
rt, report, log)
|
|
|
|
| 8363 |
# A failed pass retains its previous watermark so its window will be retried.
|
| 8364 |
# ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
|
| 8365 |
# ticket answering itself, and it must not cost the six tables that did work.
|
| 8366 |
+
problem = _sync_problem(e, table, operation, tenant_key, rt, report, log)
|
| 8367 |
failed.add(table)
|
| 8368 |
report["tables"][table] = {"rows": 0, **problem}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8369 |
if problem["state"] == "pending":
|
| 8370 |
pending_reports.add(table)
|
| 8371 |
report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900)
|
|
|
|
| 8383 |
"reason": problem["reason"][:500], "amazonStatus": problem["amazonStatus"],
|
| 8384 |
"diagnostic": problem.get("diagnostic", "")[:500],
|
| 8385 |
"error": problem.get("error", ""), "attempts": attempts}))
|
| 8386 |
+
_record_state(con, entity_key("sync_tick_" + refused),
|
| 8387 |
+
"attempted" if problem["state"] == "pending" else "done", 0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8388 |
except Exception: # noqa: BLE001
|
| 8389 |
pass
|
| 8390 |
_sync_event("amazon.sync", tenant_key, table,
|
|
|
|
| 10612 |
clock[0] += timedelta(seconds=cx.REPORT_TYPES["long_term_storage_fees"]["min_interval_s"] + 60)
|
| 10613 |
idle = sync(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None,
|
| 10614 |
scheduled=True)["tables"]["long_term_storage_fees"]
|
| 10615 |
+
ck("NC: with the queue empty the daily floor defers the table again",
|
| 10616 |
+
idle.get("state") == "deferred" and "once a day" in idle.get("reason", ""))
|
| 10617 |
finally:
|
| 10618 |
con.close()
|
| 10619 |
|
platform/harness/canonical_source.py
CHANGED
|
@@ -956,14 +956,14 @@ class MirrorSource(Source):
|
|
| 956 |
else:
|
| 957 |
ids = [int(i) for i in product_ids if i is not None]
|
| 958 |
out = {}
|
| 959 |
-
#
|
| 960 |
result = self.cur.execute(
|
| 961 |
"SELECT p.id, coalesce(f.qty, 0) FROM product_product p LEFT JOIN ("
|
| 962 |
"SELECT q.product_id, sum(coalesce(q.quantity, 0)) qty FROM stock_quant q "
|
| 963 |
"JOIN stock_location l ON l.id = q.location_id "
|
| 964 |
"WHERE l.usage = 'internal' AND starts_with(l.parent_path, $1) "
|
| 965 |
"GROUP BY 1) f ON f.product_id = p.id "
|
| 966 |
-
"WHERE p.id IN (SELECT unnest($2::BIGINT[]))",
|
| 967 |
[view[1], ids])
|
| 968 |
while True:
|
| 969 |
batch = result.fetchmany(BATCH)
|
|
|
|
| 956 |
else:
|
| 957 |
ids = [int(i) for i in product_ids if i is not None]
|
| 958 |
out = {}
|
| 959 |
+
# Odoo's search_read on product.product skips archived ids, so they stay absent here too.
|
| 960 |
result = self.cur.execute(
|
| 961 |
"SELECT p.id, coalesce(f.qty, 0) FROM product_product p LEFT JOIN ("
|
| 962 |
"SELECT q.product_id, sum(coalesce(q.quantity, 0)) qty FROM stock_quant q "
|
| 963 |
"JOIN stock_location l ON l.id = q.location_id "
|
| 964 |
"WHERE l.usage = 'internal' AND starts_with(l.parent_path, $1) "
|
| 965 |
"GROUP BY 1) f ON f.product_id = p.id "
|
| 966 |
+
"WHERE p.id IN (SELECT unnest($2::BIGINT[])) AND coalesce(p.active, false)",
|
| 967 |
[view[1], ids])
|
| 968 |
while True:
|
| 969 |
batch = result.fetchmany(BATCH)
|
platform/harness/datastore.py
CHANGED
|
@@ -513,8 +513,7 @@ def odoo_dynamic_model_identity(tenant_key, connection_id, model, table_key):
|
|
| 513 |
or key not in (base, base + "__" + cid) or len(key) > 60
|
| 514 |
or not re.fullmatch(r"[a-z0-9_]+", key)):
|
| 515 |
raise ValueError("Invalid dynamic Odoo database identity")
|
| 516 |
-
source = next((name for name, spec in ENTITIES.items()
|
| 517 |
-
if spec["model"] == native and not spec.get("metadata_only")), None)
|
| 518 |
if source is None:
|
| 519 |
token = hashlib.sha256(json.dumps([tenant, cid, native], separators=(",", ":")).encode()).hexdigest()
|
| 520 |
source = "odoo_dynamic_" + token
|
|
@@ -3669,20 +3668,6 @@ ENTITIES.update({
|
|
| 3669 |
},
|
| 3670 |
})
|
| 3671 |
|
| 3672 |
-
# Delivery references contain metadata only and share the tenant's publication boundary.
|
| 3673 |
-
DELIVERY_PHOTO_FIELDS = ("id", "res_model", "res_id", "name", "mimetype", "file_size", "create_date")
|
| 3674 |
-
DELIVERY_PHOTO_TYPES = ("image/jpeg", "image/png", "image/gif", "image/webp")
|
| 3675 |
-
# An ID cursor disables Odoo's implicit res_field filter, so the gallery scope must be explicit.
|
| 3676 |
-
DELIVERY_PHOTO_DOMAIN = [("res_model", "=", "stock.picking"),
|
| 3677 |
-
("type", "=", "binary"),
|
| 3678 |
-
("res_field", "=", False),
|
| 3679 |
-
("mimetype", "in", list(DELIVERY_PHOTO_TYPES))]
|
| 3680 |
-
ENTITIES["delivery_photo_metadata"] = {
|
| 3681 |
-
"model": "ir.attachment", "archivable": False, "optional": True,
|
| 3682 |
-
"fields": list(DELIVERY_PHOTO_FIELDS[1:]), "domain": DELIVERY_PHOTO_DOMAIN, "metadata_only": True,
|
| 3683 |
-
}
|
| 3684 |
-
INT_FIELDS += ("file_size",)
|
| 3685 |
-
|
| 3686 |
# The product pool's pricing and tags. Each table holds exactly Odoo's default (active_test) read,
|
| 3687 |
# so it carries no `active` column and a reader needs no active filter.
|
| 3688 |
ENTITIES.update({
|
|
@@ -3708,7 +3693,6 @@ NUMERIC_FIELDS += ("volume", "incoming_qty", "free_qty", "virtual_available", "f
|
|
| 3708 |
#: cannot see an Odoo delete, or a quant quantity that moved without a write_date change.
|
| 3709 |
#: Scope `internal` replaces the internal-location quants only. Any other scope is the whole table.
|
| 3710 |
REPLACE_EACH_PASS = {
|
| 3711 |
-
"delivery_photo_metadata": {"domain": DELIVERY_PHOTO_DOMAIN},
|
| 3712 |
"stock_quant": {"domain": [("location_id.usage", "=", "internal")], "scope": "internal"},
|
| 3713 |
"product_pricelist": {"domain": []},
|
| 3714 |
"product_pricelist_item": {"domain": []},
|
|
@@ -3998,7 +3982,7 @@ def _save_state(con, key, phase, last_id, cursor_wd):
|
|
| 3998 |
|
| 3999 |
|
| 4000 |
def _base_domain(spec):
|
| 4001 |
-
return
|
| 4002 |
|
| 4003 |
|
| 4004 |
def sync_entity(key, max_batches=200, batch=BATCH, log=print):
|
|
@@ -4271,9 +4255,6 @@ def _already(con, key, rec):
|
|
| 4271 |
def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch=BATCH, connection_id=None,
|
| 4272 |
client_factory=None):
|
| 4273 |
selected = list(ENTITIES) if entities is None else list(entities)
|
| 4274 |
-
if ("stock_picking" in selected and "delivery_photo_metadata" in ENTITIES
|
| 4275 |
-
and "delivery_photo_metadata" not in selected):
|
| 4276 |
-
selected.append("delivery_photo_metadata")
|
| 4277 |
if entities is not None and not selected:
|
| 4278 |
return {}
|
| 4279 |
# ⭐ D-10. `{}` rather than a dict of paused phases, and the caller decides why that matters:
|
|
|
|
| 513 |
or key not in (base, base + "__" + cid) or len(key) > 60
|
| 514 |
or not re.fullmatch(r"[a-z0-9_]+", key)):
|
| 515 |
raise ValueError("Invalid dynamic Odoo database identity")
|
| 516 |
+
source = next((name for name, spec in ENTITIES.items() if spec["model"] == native), None)
|
|
|
|
| 517 |
if source is None:
|
| 518 |
token = hashlib.sha256(json.dumps([tenant, cid, native], separators=(",", ":")).encode()).hexdigest()
|
| 519 |
source = "odoo_dynamic_" + token
|
|
|
|
| 3668 |
},
|
| 3669 |
})
|
| 3670 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3671 |
# The product pool's pricing and tags. Each table holds exactly Odoo's default (active_test) read,
|
| 3672 |
# so it carries no `active` column and a reader needs no active filter.
|
| 3673 |
ENTITIES.update({
|
|
|
|
| 3693 |
#: cannot see an Odoo delete, or a quant quantity that moved without a write_date change.
|
| 3694 |
#: Scope `internal` replaces the internal-location quants only. Any other scope is the whole table.
|
| 3695 |
REPLACE_EACH_PASS = {
|
|
|
|
| 3696 |
"stock_quant": {"domain": [("location_id.usage", "=", "internal")], "scope": "internal"},
|
| 3697 |
"product_pricelist": {"domain": []},
|
| 3698 |
"product_pricelist_item": {"domain": []},
|
|
|
|
| 3982 |
|
| 3983 |
|
| 3984 |
def _base_domain(spec):
|
| 3985 |
+
return [("active", "in", [True, False])] if spec["archivable"] else []
|
| 3986 |
|
| 3987 |
|
| 3988 |
def sync_entity(key, max_batches=200, batch=BATCH, log=print):
|
|
|
|
| 4255 |
def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch=BATCH, connection_id=None,
|
| 4256 |
client_factory=None):
|
| 4257 |
selected = list(ENTITIES) if entities is None else list(entities)
|
|
|
|
|
|
|
|
|
|
| 4258 |
if entities is not None and not selected:
|
| 4259 |
return {}
|
| 4260 |
# ⭐ D-10. `{}` rather than a dict of paused phases, and the caller decides why that matters:
|
platform/modules/account_health.py
CHANGED
|
@@ -24,7 +24,6 @@ import hashlib
|
|
| 24 |
import json
|
| 25 |
import math
|
| 26 |
from pathlib import Path
|
| 27 |
-
from urllib.parse import urlencode
|
| 28 |
|
| 29 |
APP_LABEL = "Account health"
|
| 30 |
SOURCE_KIND = "account_health"
|
|
@@ -56,11 +55,6 @@ SETTLEMENT_TABLE = "amzn_settlements"
|
|
| 56 |
COMPLAINT_METRICS = ("received_intellectual_property_complaints", "suspected_intellectual_property_violations",
|
| 57 |
"product_authenticity_customer_complaints")
|
| 58 |
SOURCE = "ut_amzn_account_health"
|
| 59 |
-
LISTING_SOURCE = "ut_amzn_listing_items"
|
| 60 |
-
CATEGORY_SOURCE_NOTE = (
|
| 61 |
-
"Amazon's account health report provides category totals, targets and reporting windows, but no violation records. "
|
| 62 |
-
"Related listings show listing issues. Amazon does not link them to this category."
|
| 63 |
-
)
|
| 64 |
NOTIFICATION_KIND = "account_health_new_items"
|
| 65 |
_sync_delivery = None
|
| 66 |
|
|
@@ -469,7 +463,7 @@ def _page_bounds(offset, limit, total):
|
|
| 469 |
return start, size
|
| 470 |
|
| 471 |
|
| 472 |
-
def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=None,
|
| 473 |
"""Read one metric page or one exact metric without loading the complete snapshot."""
|
| 474 |
st = _rt(tenant_key, st)
|
| 475 |
distance = settings(tenant_key, st=st)["alertDistancePct"]
|
|
@@ -501,8 +495,6 @@ def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=Non
|
|
| 501 |
"ELSE CASE WHEN CONTAINS(UPPER(condition), 'EQUAL') OR target = 0 THEN 'At most' ELSE 'Below' END END) "
|
| 502 |
"|| ' ' || CASE WHEN unit = 'percent' THEN printf('%.2f%%', target) ELSE CAST(target AS VARCHAR) END END, "
|
| 503 |
"condition, window_from, window_to)), ?)", [str(search or "").strip().lower()])
|
| 504 |
-
if category is not None:
|
| 505 |
-
where, filters = f"({where}) AND key = ?", [*filters, str(category)]
|
| 506 |
folded = ", ".join(f"'{s}'" for s in FOLDED)
|
| 507 |
total, folded_count = cur.execute(
|
| 508 |
sql + f" SELECT COUNT(*), COUNT(*) FILTER (WHERE status IN ({folded})) FROM metrics WHERE {where}",
|
|
@@ -538,110 +530,6 @@ def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=Non
|
|
| 538 |
"cursor": str(start + size) if start + size < out["metricTotal"] else None}
|
| 539 |
|
| 540 |
|
| 541 |
-
def _listing_array(value):
|
| 542 |
-
"""Decode a connector JSON array without inventing missing entries."""
|
| 543 |
-
if isinstance(value, str):
|
| 544 |
-
try:
|
| 545 |
-
value = json.loads(value)
|
| 546 |
-
except (TypeError, ValueError):
|
| 547 |
-
return None
|
| 548 |
-
return value if isinstance(value, list) else None
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
def _related_listings(tenant_key, st, user, market):
|
| 552 |
-
"""Read all permitted listing issue pages through the listing source's own wall."""
|
| 553 |
-
from core import perm_scope
|
| 554 |
-
st = _rt(tenant_key, st)
|
| 555 |
-
if user is None:
|
| 556 |
-
return [], "Related listings need access to the listing source."
|
| 557 |
-
related, cursor, incomplete = [], None, False
|
| 558 |
-
try:
|
| 559 |
-
while True:
|
| 560 |
-
page = perm_scope.scoped_table_page(user, LISTING_SOURCE, st=st, cursor=cursor, limit=1024)
|
| 561 |
-
if "issues" not in {field.get("key") for field in page["fields"]}:
|
| 562 |
-
return [], "Listing issues are unavailable with your current source access."
|
| 563 |
-
for row in page["rows"]:
|
| 564 |
-
if market and row.get("marketplace_id") and str(row["marketplace_id"]) != market:
|
| 565 |
-
continue
|
| 566 |
-
issues = _listing_array(row.get("issues"))
|
| 567 |
-
if issues is None:
|
| 568 |
-
incomplete = True
|
| 569 |
-
continue
|
| 570 |
-
summaries = _listing_array(row.get("summaries")) or []
|
| 571 |
-
summaries = [item for item in summaries if isinstance(item, dict)]
|
| 572 |
-
matching = [item for item in summaries if not market or item.get("marketplaceId") == market]
|
| 573 |
-
if market and summaries and not matching:
|
| 574 |
-
continue
|
| 575 |
-
asin = row.get("asin") or next((item.get("asin") for item in matching if item.get("asin")), None)
|
| 576 |
-
for issue in issues:
|
| 577 |
-
if not isinstance(issue, dict) or not (issue.get("message") or issue.get("code")):
|
| 578 |
-
incomplete = True
|
| 579 |
-
continue
|
| 580 |
-
related.append({"asin": asin, "sku": row.get("sku"),
|
| 581 |
-
"issue": issue.get("message") or issue["code"],
|
| 582 |
-
"severity": issue.get("severity")})
|
| 583 |
-
after = page["nextCursor"]
|
| 584 |
-
if after is None:
|
| 585 |
-
break
|
| 586 |
-
if cursor is not None and int(after) <= int(cursor):
|
| 587 |
-
raise RuntimeError("The listing source cursor did not advance.")
|
| 588 |
-
cursor = after
|
| 589 |
-
except perm_scope.Denied:
|
| 590 |
-
return [], "Related listings are unavailable with your current source access."
|
| 591 |
-
except (perm_scope.UnknownTable, perm_scope.Unresolvable):
|
| 592 |
-
return [], "Related listing data is unavailable."
|
| 593 |
-
return related, "Some listing issues are missing or unreadable." if incomplete else None
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
def category_detail(tenant_key, key, *, user=None, allowed=None, st=None):
|
| 597 |
-
"""Serve one permitted category and separate listing evidence without claiming violation records."""
|
| 598 |
-
st = _rt(tenant_key, st)
|
| 599 |
-
category, _sep, _market = str(key).partition("@")
|
| 600 |
-
labels = dict(COUNT_METRICS)
|
| 601 |
-
if category not in labels:
|
| 602 |
-
raise KeyError(key)
|
| 603 |
-
kept, offset, total = [], 0, 0
|
| 604 |
-
while True:
|
| 605 |
-
page = summary_page(tenant_key, category=category, metric=key if _sep else None,
|
| 606 |
-
offset=offset, limit=PAGE_SIZE, st=st)
|
| 607 |
-
total = page["metricTotal"]
|
| 608 |
-
kept.extend(row for row in page["metrics"] if allowed is None or allowed(row, page["snapshotDate"]))
|
| 609 |
-
if len(kept) > 1:
|
| 610 |
-
raise ValueError("Choose a marketplace with category@marketplace.")
|
| 611 |
-
if page["cursor"] is None:
|
| 612 |
-
break
|
| 613 |
-
offset = int(page["cursor"])
|
| 614 |
-
if not kept and (total or allowed is not None or _sep):
|
| 615 |
-
raise KeyError(key)
|
| 616 |
-
row = kept[0] if kept else {}
|
| 617 |
-
note = CATEGORY_SOURCE_NOTE
|
| 618 |
-
if not row:
|
| 619 |
-
note += " This category is missing from the saved report."
|
| 620 |
-
if row and row.get("value") is None:
|
| 621 |
-
note += " The category count is missing."
|
| 622 |
-
if row and row.get("target") is None:
|
| 623 |
-
note += " Amazon did not provide a target."
|
| 624 |
-
rule = str(row.get("targetCondition") or "").upper()
|
| 625 |
-
known_rule = "LESS" in rule or "GREATER" in rule
|
| 626 |
-
if row.get("target") is not None and not known_rule:
|
| 627 |
-
note += " The target rule is missing or unsupported."
|
| 628 |
-
if row and (row.get("windowFrom") is None or row.get("windowTo") is None):
|
| 629 |
-
note += " The reporting window is missing or incomplete."
|
| 630 |
-
related, listing_note = _related_listings(tenant_key, st, user, row.get("marketplaceId")) if row else ([], None)
|
| 631 |
-
if listing_note:
|
| 632 |
-
note += " " + listing_note
|
| 633 |
-
query = urlencode({"marketplaceID": row["marketplaceId"]}) if row.get("marketplaceId") else ""
|
| 634 |
-
status = row.get("status", "No data")
|
| 635 |
-
if row.get("value") is not None and row.get("target") is not None and not known_rule:
|
| 636 |
-
status = "No target"
|
| 637 |
-
return {"key": category, "label": labels[category], "count": row.get("value"),
|
| 638 |
-
"status": status, "target": row.get("condition") if known_rule else None,
|
| 639 |
-
"window": {"from": row.get("windowFrom"), "to": row.get("windowTo")},
|
| 640 |
-
"recordList": False, "sourceNote": note,
|
| 641 |
-
"sellerCentralUrl": "https://sellercentral.amazon.com/performance/dashboard" + ("?" + query if query else ""),
|
| 642 |
-
"related": related}
|
| 643 |
-
|
| 644 |
-
|
| 645 |
# --------------------------------------------------------------------------- settings and owner
|
| 646 |
|
| 647 |
#: Accounts that sign in for a process, never a person who can own an alert (same list as amazon_profit).
|
|
@@ -1031,7 +919,7 @@ def _title(row):
|
|
| 1031 |
return f"{row['label']} is {'at risk' if row['status'] == 'At risk' else 'near target'}"
|
| 1032 |
|
| 1033 |
|
| 1034 |
-
def _alert_source(tenant_key, st, row, snapshot_date
|
| 1035 |
"""Bind an alert to the actual mirror row supplying its metric value."""
|
| 1036 |
from harness.amazon_schema import IDENTITY_PID
|
| 1037 |
column = row["key"] + ("_defects_count" if row["key"] in dict(COUNT_METRICS) else "_rate")
|
|
@@ -1042,7 +930,6 @@ def _alert_source(tenant_key, st, row, snapshot_date, *, extra_fields=()):
|
|
| 1042 |
return {"source": None, "fields": fields}
|
| 1043 |
fields += [name for name in (row["key"] + "_target_value", row["key"] + "_target_condition")
|
| 1044 |
if name in cols]
|
| 1045 |
-
fields += [name for name in extra_fields if name in cols and name not in fields]
|
| 1046 |
order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else ""
|
| 1047 |
row_ids = []
|
| 1048 |
for field in (name for name in fields if name not in {"_snapshot_date", "marketplace_id"}):
|
|
@@ -1058,14 +945,6 @@ def _alert_source(tenant_key, st, row, snapshot_date, *, extra_fields=()):
|
|
| 1058 |
"fields": fields}
|
| 1059 |
|
| 1060 |
|
| 1061 |
-
def category_source(tenant_key, st, row, snapshot_date):
|
| 1062 |
-
"""Bind every category detail field to the report rows that supplied it."""
|
| 1063 |
-
key = row["key"]
|
| 1064 |
-
return _alert_source(tenant_key, st, row, snapshot_date, extra_fields=(
|
| 1065 |
-
key + "_status", key + "_reporting_date_range_reporting_date_from",
|
| 1066 |
-
key + "_reporting_date_range_reporting_date_to"))
|
| 1067 |
-
|
| 1068 |
-
|
| 1069 |
def raise_alerts(tenant_key, view=None, *, st=None, now=None):
|
| 1070 |
"""Raise one owned alert per metric that is near or past its target, never a second while one is open."""
|
| 1071 |
st = _rt(tenant_key, st)
|
|
@@ -1852,7 +1731,7 @@ def validate_dashboard(tenant_key, figures=None, *, start=None, end=None, st=Non
|
|
| 1852 |
return {"ok": bool(detail) and all(check["ok"] for check in detail), "detail": detail}
|
| 1853 |
|
| 1854 |
def validate(tenant_key, *, st=None, **period):
|
| 1855 |
-
"""Tie the rating
|
| 1856 |
st = _rt(tenant_key, st)
|
| 1857 |
view = summary(tenant_key, st=st)
|
| 1858 |
with _reader(tenant_key, st) as cur:
|
|
@@ -1863,33 +1742,17 @@ def validate(tenant_key, *, st=None, **period):
|
|
| 1863 |
date = cur.execute(f'SELECT MAX(CAST("_snapshot_date" AS VARCHAR)) FROM {AH_TABLE}').fetchone()[0] \
|
| 1864 |
if "_snapshot_date" in cols else None
|
| 1865 |
wanted = [("rating", "account_health_rating_ahr_score", 1.0)] + \
|
| 1866 |
-
[(p, f"{p}_rate", None) for p, _l, _d in RATE_METRICS]
|
| 1867 |
-
[(p, f"{p}_defects_count", 1.0) for p, _l in COUNT_METRICS] + [(WARNINGS[0], WARNINGS[2], 1.0)]
|
| 1868 |
for key, col, _s in wanted:
|
| 1869 |
if col not in cols:
|
| 1870 |
continue
|
| 1871 |
where = ' WHERE CAST("_snapshot_date" AS VARCHAR) = ?' if date is not None else ""
|
| 1872 |
order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else ""
|
| 1873 |
-
q = (f'SELECT CAST("marketplace_id" AS VARCHAR), '
|
| 1874 |
-
f'FIRST(TRY_CAST(REPLACE(TRIM(CAST("{col}" AS VARCHAR)), \',\', \'\') AS DOUBLE){order}) '
|
| 1875 |
f'FILTER (WHERE "{col}" IS NOT NULL) '
|
| 1876 |
f'FROM {AH_TABLE}{where} GROUP BY 1')
|
| 1877 |
for market, value in cur.execute(q, [date] if date is not None else []).fetchall():
|
| 1878 |
independent[(key, str(market or ""))] = value
|
| 1879 |
-
inbound_cols = _columns(cur, INBOUND_TABLE)
|
| 1880 |
-
if inbound_cols:
|
| 1881 |
-
column = next((name for name in ("issue_reported_date", "_period_start") if name in inbound_cols), None)
|
| 1882 |
-
where, args = "", []
|
| 1883 |
-
if column:
|
| 1884 |
-
try:
|
| 1885 |
-
end = dt.date.fromisoformat(str(date)[:10])
|
| 1886 |
-
except (TypeError, ValueError):
|
| 1887 |
-
end = _now().date()
|
| 1888 |
-
where = (f' WHERE TRY_CAST(SUBSTR(CAST("{column}" AS VARCHAR), 1, 10) AS DATE) '
|
| 1889 |
-
f'> ? AND TRY_CAST(SUBSTR(CAST("{column}" AS VARCHAR), 1, 10) AS DATE) <= ?')
|
| 1890 |
-
args = [end - dt.timedelta(days=INBOUND_DAYS), end]
|
| 1891 |
-
independent[(INBOUND[0], "")] = cur.execute(
|
| 1892 |
-
f"SELECT COUNT(*) FROM {INBOUND_TABLE}{where}", args).fetchone()[0]
|
| 1893 |
app = {}
|
| 1894 |
for m in view["marketplaces"]:
|
| 1895 |
app[("rating", m["marketplaceId"] or "")] = m["rating"]
|
|
@@ -1898,8 +1761,6 @@ def validate(tenant_key, *, st=None, **period):
|
|
| 1898 |
# The shown percent ties back through its scale; a no orders row ties its raw rate (Amazon's 0).
|
| 1899 |
raw = row["value"] / row.get("scale", 100.0) if row["value"] is not None else row.get("raw")
|
| 1900 |
app[(row["key"], row["marketplaceId"] or "")] = raw
|
| 1901 |
-
else:
|
| 1902 |
-
app[(row["key"], row["marketplaceId"] or "")] = row["value"]
|
| 1903 |
detail, app_total, ind_total = [], 0.0, 0.0
|
| 1904 |
for key in sorted(set(app) | set(independent)):
|
| 1905 |
a, b = app.get(key), independent.get(key)
|
|
@@ -1909,24 +1770,9 @@ def validate(tenant_key, *, st=None, **period):
|
|
| 1909 |
detail.append({"metric": key[0], "marketplaceId": key[1] or None, "app": a, "independent": b, "ok": tied})
|
| 1910 |
ok = bool(detail) and all(d["ok"] for d in detail) and str(view.get("snapshotDate")) == str(date)
|
| 1911 |
groups = _validate_groups(tenant_key, st, view)
|
| 1912 |
-
|
| 1913 |
-
for key, market in sorted(set(app) | set(independent)):
|
| 1914 |
-
if key not in dict(COUNT_METRICS):
|
| 1915 |
-
continue
|
| 1916 |
-
try:
|
| 1917 |
-
shown = category_detail(tenant_key, key + ("@" + market if market else ""), st=st)["count"]
|
| 1918 |
-
source = independent.get((key, market))
|
| 1919 |
-
tied = (shown is None and source is None) or (shown is not None and source is not None
|
| 1920 |
-
and abs(shown - source) <= 1e-7)
|
| 1921 |
-
except KeyError:
|
| 1922 |
-
shown, source = None, independent.get((key, market))
|
| 1923 |
-
tied = (key, market) not in app and source is None
|
| 1924 |
-
categories.append({"metric": key, "marketplaceId": market or None,
|
| 1925 |
-
"app": shown, "independent": source, "ok": tied})
|
| 1926 |
-
category_check = {"ok": all(row["ok"] for row in categories), "detail": categories}
|
| 1927 |
-
return {"ok": ok and reserves["ok"] and groups["ok"] and category_check["ok"], "app_total": round(app_total, 9),
|
| 1928 |
"independent_total": round(ind_total, 9), "detail": detail, "snapshotDate": date,
|
| 1929 |
-
"message": None if detail else EMPTY, "reserves": reserves, "groups": groups
|
| 1930 |
|
| 1931 |
|
| 1932 |
def _validate_groups(tenant_key, st, view):
|
|
|
|
| 24 |
import json
|
| 25 |
import math
|
| 26 |
from pathlib import Path
|
|
|
|
| 27 |
|
| 28 |
APP_LABEL = "Account health"
|
| 29 |
SOURCE_KIND = "account_health"
|
|
|
|
| 55 |
COMPLAINT_METRICS = ("received_intellectual_property_complaints", "suspected_intellectual_property_violations",
|
| 56 |
"product_authenticity_customer_complaints")
|
| 57 |
SOURCE = "ut_amzn_account_health"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
NOTIFICATION_KIND = "account_health_new_items"
|
| 59 |
_sync_delivery = None
|
| 60 |
|
|
|
|
| 463 |
return start, size
|
| 464 |
|
| 465 |
|
| 466 |
+
def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=None, st=None):
|
| 467 |
"""Read one metric page or one exact metric without loading the complete snapshot."""
|
| 468 |
st = _rt(tenant_key, st)
|
| 469 |
distance = settings(tenant_key, st=st)["alertDistancePct"]
|
|
|
|
| 495 |
"ELSE CASE WHEN CONTAINS(UPPER(condition), 'EQUAL') OR target = 0 THEN 'At most' ELSE 'Below' END END) "
|
| 496 |
"|| ' ' || CASE WHEN unit = 'percent' THEN printf('%.2f%%', target) ELSE CAST(target AS VARCHAR) END END, "
|
| 497 |
"condition, window_from, window_to)), ?)", [str(search or "").strip().lower()])
|
|
|
|
|
|
|
| 498 |
folded = ", ".join(f"'{s}'" for s in FOLDED)
|
| 499 |
total, folded_count = cur.execute(
|
| 500 |
sql + f" SELECT COUNT(*), COUNT(*) FILTER (WHERE status IN ({folded})) FROM metrics WHERE {where}",
|
|
|
|
| 530 |
"cursor": str(start + size) if start + size < out["metricTotal"] else None}
|
| 531 |
|
| 532 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 533 |
# --------------------------------------------------------------------------- settings and owner
|
| 534 |
|
| 535 |
#: Accounts that sign in for a process, never a person who can own an alert (same list as amazon_profit).
|
|
|
|
| 919 |
return f"{row['label']} is {'at risk' if row['status'] == 'At risk' else 'near target'}"
|
| 920 |
|
| 921 |
|
| 922 |
+
def _alert_source(tenant_key, st, row, snapshot_date):
|
| 923 |
"""Bind an alert to the actual mirror row supplying its metric value."""
|
| 924 |
from harness.amazon_schema import IDENTITY_PID
|
| 925 |
column = row["key"] + ("_defects_count" if row["key"] in dict(COUNT_METRICS) else "_rate")
|
|
|
|
| 930 |
return {"source": None, "fields": fields}
|
| 931 |
fields += [name for name in (row["key"] + "_target_value", row["key"] + "_target_condition")
|
| 932 |
if name in cols]
|
|
|
|
| 933 |
order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else ""
|
| 934 |
row_ids = []
|
| 935 |
for field in (name for name in fields if name not in {"_snapshot_date", "marketplace_id"}):
|
|
|
|
| 945 |
"fields": fields}
|
| 946 |
|
| 947 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 948 |
def raise_alerts(tenant_key, view=None, *, st=None, now=None):
|
| 949 |
"""Raise one owned alert per metric that is near or past its target, never a second while one is open."""
|
| 950 |
st = _rt(tenant_key, st)
|
|
|
|
| 1731 |
return {"ok": bool(detail) and all(check["ok"] for check in detail), "detail": detail}
|
| 1732 |
|
| 1733 |
def validate(tenant_key, *, st=None, **period):
|
| 1734 |
+
"""Tie the rating and each rate in `summary` to the raw `ut_amzn_account_health` snapshot."""
|
| 1735 |
st = _rt(tenant_key, st)
|
| 1736 |
view = summary(tenant_key, st=st)
|
| 1737 |
with _reader(tenant_key, st) as cur:
|
|
|
|
| 1742 |
date = cur.execute(f'SELECT MAX(CAST("_snapshot_date" AS VARCHAR)) FROM {AH_TABLE}').fetchone()[0] \
|
| 1743 |
if "_snapshot_date" in cols else None
|
| 1744 |
wanted = [("rating", "account_health_rating_ahr_score", 1.0)] + \
|
| 1745 |
+
[(p, f"{p}_rate", None) for p, _l, _d in RATE_METRICS]
|
|
|
|
| 1746 |
for key, col, _s in wanted:
|
| 1747 |
if col not in cols:
|
| 1748 |
continue
|
| 1749 |
where = ' WHERE CAST("_snapshot_date" AS VARCHAR) = ?' if date is not None else ""
|
| 1750 |
order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else ""
|
| 1751 |
+
q = (f'SELECT CAST("marketplace_id" AS VARCHAR), FIRST(TRY_CAST("{col}" AS DOUBLE){order}) '
|
|
|
|
| 1752 |
f'FILTER (WHERE "{col}" IS NOT NULL) '
|
| 1753 |
f'FROM {AH_TABLE}{where} GROUP BY 1')
|
| 1754 |
for market, value in cur.execute(q, [date] if date is not None else []).fetchall():
|
| 1755 |
independent[(key, str(market or ""))] = value
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1756 |
app = {}
|
| 1757 |
for m in view["marketplaces"]:
|
| 1758 |
app[("rating", m["marketplaceId"] or "")] = m["rating"]
|
|
|
|
| 1761 |
# The shown percent ties back through its scale; a no orders row ties its raw rate (Amazon's 0).
|
| 1762 |
raw = row["value"] / row.get("scale", 100.0) if row["value"] is not None else row.get("raw")
|
| 1763 |
app[(row["key"], row["marketplaceId"] or "")] = raw
|
|
|
|
|
|
|
| 1764 |
detail, app_total, ind_total = [], 0.0, 0.0
|
| 1765 |
for key in sorted(set(app) | set(independent)):
|
| 1766 |
a, b = app.get(key), independent.get(key)
|
|
|
|
| 1770 |
detail.append({"metric": key[0], "marketplaceId": key[1] or None, "app": a, "independent": b, "ok": tied})
|
| 1771 |
ok = bool(detail) and all(d["ok"] for d in detail) and str(view.get("snapshotDate")) == str(date)
|
| 1772 |
groups = _validate_groups(tenant_key, st, view)
|
| 1773 |
+
return {"ok": ok and reserves["ok"] and groups["ok"], "app_total": round(app_total, 9),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1774 |
"independent_total": round(ind_total, 9), "detail": detail, "snapshotDate": date,
|
| 1775 |
+
"message": None if detail else EMPTY, "reserves": reserves, "groups": groups}
|
| 1776 |
|
| 1777 |
|
| 1778 |
def _validate_groups(tenant_key, st, view):
|
platform/modules/amazon_profit.py
CHANGED
|
@@ -10,9 +10,9 @@ survive every period switch.
|
|
| 10 |
`ut_amazon_asin_profit` and `ut_amazon_parent_profit` roll the SKU rows up on every refresh.
|
| 11 |
Projects retain their own verified period totals and Impact in the tenant projects store.
|
| 12 |
|
| 13 |
-
`validate()` ties
|
| 14 |
-
|
| 15 |
-
A refresh runs it for the last complete month and writes no figures when it
|
| 16 |
fails. A passing refresh stores the verified range and the day the rest is due to settle.
|
| 17 |
|
| 18 |
Owned items (R10, R31): a refresh opens one item on a Losing or Thin SKU with no status, for the
|
|
@@ -615,16 +615,11 @@ def reported_net_proceeds(con, start, end):
|
|
| 615 |
def economics_currency(con):
|
| 616 |
if "currency_code" not in _columns(con, ECONOMICS):
|
| 617 |
return None
|
| 618 |
-
|
| 619 |
-
f"SELECT max(currency_code), count(DISTINCT currency_code), "
|
| 620 |
-
f"count(*) FILTER (WHERE NULLIF(trim(currency_code), '') IS NULL) FROM {ECONOMICS}").fetchone()
|
| 621 |
-
if count > 1 or missing:
|
| 622 |
-
raise SourceUnavailable("Amazon economics needs one known currency.")
|
| 623 |
-
return currency
|
| 624 |
|
| 625 |
|
| 626 |
def settlements_total(con, start, end, currency=None):
|
| 627 |
-
"""
|
| 628 |
|
| 629 |
`currency` keeps settlements whose header row names it: Amazon.ca and Amazon.com.mx settle
|
| 630 |
in their own currency, outside the economics marketplace. Their reports write DD.MM.YYYY.
|
|
@@ -634,188 +629,18 @@ def settlements_total(con, start, end, currency=None):
|
|
| 634 |
stream = ("JOIN (SELECT settlement_id FROM {t} WHERE currency IS NOT NULL GROUP BY settlement_id "
|
| 635 |
"HAVING any_value(currency) = ?) h USING (settlement_id)").format(t=SETTLEMENTS) if scoped else ""
|
| 636 |
total, rows, undated = con.execute(
|
| 637 |
-
"SELECT COALESCE(SUM(
|
| 638 |
-
"FILTER (WHERE d BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)), 0), "
|
| 639 |
"count(*) FILTER (WHERE d BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)), "
|
| 640 |
"count(*) FILTER (WHERE d IS NULL AND posted_date IS NOT NULL) "
|
| 641 |
"FROM (SELECT amount, posted_date, COALESCE(TRY_CAST(left(posted_date, 10) AS DATE), "
|
| 642 |
"CAST(try_strptime(left(posted_date, 10), '%d.%m.%Y') AS DATE)) AS d "
|
| 643 |
-
f"FROM {SETTLEMENTS} s {stream} WHERE COALESCE(transaction_type, '') <> ?
|
| 644 |
-
|
|
|
|
| 645 |
return {"total": float(total), "rows": int(rows), "undated": int(undated),
|
| 646 |
"currency": currency if scoped else None}
|
| 647 |
|
| 648 |
|
| 649 |
-
def settlement_components(con, start, end, currency=None):
|
| 650 |
-
"""Tie purchase-date order charges, service-month storage and daily ads to independent sources."""
|
| 651 |
-
month = dt.date.fromisoformat(start).strftime("%B %Y")
|
| 652 |
-
parts, problems = [], []
|
| 653 |
-
|
| 654 |
-
def problem(component, reason):
|
| 655 |
-
problems.append(f"Not verified: {component} in {month}: {reason}.")
|
| 656 |
-
|
| 657 |
-
def tie(component, ours, source, *, reason=None):
|
| 658 |
-
ours = float(ours)
|
| 659 |
-
source = None if source is None else float(source)
|
| 660 |
-
ok, gap = _tie(ours, source) if source is not None else (False, math.inf)
|
| 661 |
-
basis = {"product sales": "order purchase date", "order fees": "order purchase date",
|
| 662 |
-
"monthly storage": "storage service month", "ads": "daily ad report date"}
|
| 663 |
-
parts.append({"component": component, "month": start[:7], "ok": ok and not reason,
|
| 664 |
-
"a": ours, "b": source, "gap": gap, "tolerance": SETTLEMENT_TOLERANCE,
|
| 665 |
-
"dateBasis": basis.get(component, "order purchase date when present, otherwise posted date"),
|
| 666 |
-
"sourceTables": ([ECONOMICS] + [f"amzn_ads_{kind}_advertised_products" for kind in ("sp", "sb", "sd")]
|
| 667 |
-
if component == "ads" else [ECONOMICS, SETTLEMENTS, "amzn_orders"])})
|
| 668 |
-
if reason:
|
| 669 |
-
problem(component, reason)
|
| 670 |
-
elif not ok:
|
| 671 |
-
problem(component, f"independent Amazon amounts differ by {gap:.1%}")
|
| 672 |
-
|
| 673 |
-
sc = _columns(con, SETTLEMENTS)
|
| 674 |
-
ec = _columns(con, ECONOMICS)
|
| 675 |
-
required = set(_SETTLEMENT_REQUIRED) | {"settlement_id", "currency", "order_id", "amount_description"}
|
| 676 |
-
if not required <= sc:
|
| 677 |
-
problem("settlement components", "missing source columns " + ", ".join(sorted(required - sc)))
|
| 678 |
-
return {"total": None, "parts": parts, "reason": problems[0], "rows": 0}
|
| 679 |
-
if currency is None:
|
| 680 |
-
problem("settlement components", "economics currency unavailable")
|
| 681 |
-
return {"total": None, "parts": parts, "reason": problems[0], "rows": 0}
|
| 682 |
-
if not {"amazon_order_id", "purchase_date"} <= _columns(con, "amzn_orders"):
|
| 683 |
-
problem("order charges", "missing amzn_orders purchase dates")
|
| 684 |
-
return {"total": None, "parts": parts, "reason": problems[0], "rows": 0}
|
| 685 |
-
unscoped = con.execute(f"""SELECT count(*) FROM {SETTLEMENTS} s
|
| 686 |
-
LEFT JOIN (SELECT settlement_id FROM {SETTLEMENTS} GROUP BY settlement_id
|
| 687 |
-
HAVING count(DISTINCT currency) = 1) h USING (settlement_id)
|
| 688 |
-
WHERE h.settlement_id IS NULL AND (s.amount IS NOT NULL OR NULLIF(s.amount_type, '') IS NOT NULL)
|
| 689 |
-
AND (CAST({_moment_sql('posted_date')} AS DATE) BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)
|
| 690 |
-
OR {_moment_sql('posted_date')} IS NULL)""", [start, end]).fetchone()[0]
|
| 691 |
-
if unscoped:
|
| 692 |
-
problem("settlement currency", "missing or conflicting settlement header currency")
|
| 693 |
-
# Aggregate in SQL before reading: duplicate order lines must not multiply settlement amounts.
|
| 694 |
-
cte = f"""WITH orders AS (
|
| 695 |
-
SELECT amazon_order_id, min(CAST({_moment_sql('purchase_date')} AS DATE)) AS bought,
|
| 696 |
-
count(DISTINCT CAST({_moment_sql('purchase_date')} AS DATE)) AS dates,
|
| 697 |
-
count(*) FILTER (WHERE {_moment_sql('purchase_date')} IS NULL) AS bad_dates
|
| 698 |
-
FROM amzn_orders GROUP BY amazon_order_id),
|
| 699 |
-
streams AS (SELECT settlement_id FROM {SETTLEMENTS} GROUP BY settlement_id
|
| 700 |
-
HAVING count(DISTINCT currency) = 1 AND max(currency) = ?),
|
| 701 |
-
lines AS (SELECT s.*, TRY_CAST(amount AS DECIMAL(38, 2)) AS currency_amount,
|
| 702 |
-
CAST({_moment_sql('s.posted_date')} AS DATE) AS posted,
|
| 703 |
-
o.bought, o.dates, o.bad_dates,
|
| 704 |
-
lower(regexp_replace(COALESCE(amount_description, ''), '[^a-zA-Z0-9]', '', 'g')) AS description,
|
| 705 |
-
lower(regexp_replace(COALESCE(amount_type, ''), '[^a-zA-Z0-9]', '', 'g')) AS kind,
|
| 706 |
-
lower(regexp_replace(COALESCE(transaction_type, ''), '[^a-zA-Z0-9]', '', 'g')) AS transaction
|
| 707 |
-
FROM {SETTLEMENTS} s JOIN streams USING (settlement_id)
|
| 708 |
-
LEFT JOIN orders o ON trim(s.order_id) = o.amazon_order_id
|
| 709 |
-
WHERE (amount IS NOT NULL OR NULLIF(amount_type, '') IS NOT NULL)
|
| 710 |
-
AND COALESCE(transaction_type, '') <> ?),
|
| 711 |
-
classified AS (SELECT *, CASE
|
| 712 |
-
WHEN concat(kind, description, transaction) LIKE '%inbounddefect%' THEN 'Inbound Defect Fee'
|
| 713 |
-
WHEN concat(kind, description, transaction) LIKE '%mcf%' THEN 'MCF credits'
|
| 714 |
-
WHEN kind = 'itemwithheldtax' OR description IN ('tax', 'shippingtax', 'giftwraptax')
|
| 715 |
-
OR description LIKE 'marketplacefacilitatortax%' THEN 'tax'
|
| 716 |
-
WHEN (kind = 'itemprice' AND (description IN ('shipping', 'giftwrap')
|
| 717 |
-
OR description LIKE '%promotion%')) OR kind IN ('promotion', 'shipping', 'shippingfees')
|
| 718 |
-
THEN 'shipping and promotions'
|
| 719 |
-
WHEN 'costofadvertising' IN (kind, description, transaction)
|
| 720 |
-
OR description IN ('advertising', 'advertisingfee') THEN 'ads billing'
|
| 721 |
-
WHEN description IN ('storagefee', 'fbamonthlystoragefee', 'monthlystoragefee')
|
| 722 |
-
OR 'fbainventorystoragefee' IN (kind, description, transaction) THEN 'monthly storage'
|
| 723 |
-
WHEN concat(kind, description, transaction) LIKE '%reimbursement%' THEN 'reimbursements'
|
| 724 |
-
WHEN description IN ('subscription', 'subscriptionfee', 'fbaremovalfee', 'fbadisposalfee',
|
| 725 |
-
'fbainboundtransportationfee', 'couponredemptionfee', 'dealfee')
|
| 726 |
-
OR kind IN ('dealfee', 'lightningdealfee', 'dealparticipationfee', 'dealperformancefee',
|
| 727 |
-
'fbacustomerreturnfee', 'fbacustomerreturnsfee', 'fbacustomerreturnperunitfee',
|
| 728 |
-
'fbacustomerreturnprocessingfee', 'fbainboundplacementservicefee')
|
| 729 |
-
OR description IN ('lightningdealfee', 'dealparticipationfee', 'dealperformancefee',
|
| 730 |
-
'fbacustomerreturnfee', 'fbacustomerreturnsfee', 'fbacustomerreturnperunitfee',
|
| 731 |
-
'fbacustomerreturnprocessingfee', 'fbainboundplacementservicefee')
|
| 732 |
-
OR transaction IN ('dealfee', 'lightningdealfee', 'dealparticipationfee', 'dealperformancefee',
|
| 733 |
-
'fbacustomerreturnfee', 'fbacustomerreturnsfee', 'fbacustomerreturnperunitfee',
|
| 734 |
-
'fbacustomerreturnprocessingfee', 'fbainboundplacementservicefee') THEN 'account fees'
|
| 735 |
-
WHEN NULLIF(trim(order_id), '') IS NOT NULL AND kind = 'itemprice'
|
| 736 |
-
AND description = 'principal' THEN 'product sales'
|
| 737 |
-
WHEN NULLIF(trim(order_id), '') IS NOT NULL AND kind IN ('itemfee', 'itemfees') THEN 'order fees'
|
| 738 |
-
ELSE 'unmatched component' END AS component FROM lines),
|
| 739 |
-
dated AS (SELECT *, CASE
|
| 740 |
-
WHEN component = 'monthly storage' THEN CAST(date_trunc('month', posted) - INTERVAL '1 month' AS DATE)
|
| 741 |
-
WHEN NULLIF(trim(order_id), '') IS NOT NULL THEN bought
|
| 742 |
-
ELSE posted END AS basis FROM classified)
|
| 743 |
-
"""
|
| 744 |
-
groups = con.execute(cte + """SELECT component, COALESCE(SUM(currency_amount), 0), count(*),
|
| 745 |
-
count(*) FILTER (WHERE currency_amount IS NULL OR posted IS NULL),
|
| 746 |
-
count(*) FILTER (WHERE NULLIF(trim(order_id), '') IS NOT NULL
|
| 747 |
-
AND (bought IS NULL OR dates <> 1 OR bad_dates <> 0))
|
| 748 |
-
FROM dated WHERE basis BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)
|
| 749 |
-
OR ((basis IS NULL OR dates <> 1 OR bad_dates <> 0)
|
| 750 |
-
AND (posted BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) OR posted IS NULL))
|
| 751 |
-
GROUP BY component ORDER BY component""",
|
| 752 |
-
[currency, LOAN_REPAYMENT, start, end, start, end]).fetchall()
|
| 753 |
-
amounts = {name: total for name, total, *_ in groups}
|
| 754 |
-
if not groups:
|
| 755 |
-
problem("settlement components", "no settlement source rows on the purchase or service dates")
|
| 756 |
-
for name, _total, _n, bad, unmatched in groups:
|
| 757 |
-
if bad or unmatched:
|
| 758 |
-
problem(name, "unmatched purchase date, unreadable posting date or unavailable amount")
|
| 759 |
-
if name in ("unmatched component", "Inbound Defect Fee", "MCF credits"):
|
| 760 |
-
problem(name, "settlement category has no economics mapping")
|
| 761 |
-
fee_columns = sorted(c for c in ec if c.startswith("fee_") and c.endswith("_amount") and "_charge_" not in c)
|
| 762 |
-
storage_columns = [c for c in fee_columns if "storage" in c]
|
| 763 |
-
reimbursement_columns = [c for c in fee_columns if "reimbursement" in c]
|
| 764 |
-
account_columns = [c for c in fee_columns if any(name in c for name in
|
| 765 |
-
("subscription", "removal", "disposal", "inbound_transportation", "inbound_placement",
|
| 766 |
-
"coupon", "deal_", "customer_return", "return_processing", "returns_processing"))]
|
| 767 |
-
# Storage fee columns are accruals in economics; only monthly settlement charges use the prior month.
|
| 768 |
-
sums = ["sales_net_product_sales_amount", "fees_total_amount", *ads_columns(ec),
|
| 769 |
-
*storage_columns, *reimbursement_columns, *account_columns]
|
| 770 |
-
observed = con.execute("SELECT " + ", ".join(f'COALESCE(SUM(TRY_CAST("{c}" AS DECIMAL(38, 2))), 0)' for c in sums)
|
| 771 |
-
+ ", count(*) FILTER (WHERE "
|
| 772 |
-
+ " OR ".join(f'("{c}" IS NOT NULL AND TRY_CAST("{c}" AS DECIMAL(38, 2)) IS NULL)'
|
| 773 |
-
for c in sums) + ")"
|
| 774 |
-
+ f" FROM {ECONOMICS} WHERE left(start_date, 10) BETWEEN ? AND ?",
|
| 775 |
-
[start, end]).fetchone()
|
| 776 |
-
if observed[-1]:
|
| 777 |
-
problem("settlement components", "unavailable economics amount")
|
| 778 |
-
values = dict(zip(sums, observed[:-1]))
|
| 779 |
-
sales, fees = values[sums[0]], values[sums[1]]
|
| 780 |
-
storage = sum(values[c] for c in storage_columns)
|
| 781 |
-
reimbursements = sum(values[c] for c in reimbursement_columns)
|
| 782 |
-
account_fees = sum(values[c] for c in account_columns)
|
| 783 |
-
ads = sum(values[c] for c in ads_columns(ec))
|
| 784 |
-
tie("product sales", sales, amounts.get("product sales", 0.0))
|
| 785 |
-
tie("order fees", fees - storage - reimbursements - account_fees, -amounts.get("order fees", 0.0))
|
| 786 |
-
tie("monthly storage", storage, -amounts.get("monthly storage", 0.0))
|
| 787 |
-
tie("reimbursements", reimbursements, -amounts.get("reimbursements", 0.0))
|
| 788 |
-
tie("account fees", account_fees, -amounts.get("account fees", 0.0))
|
| 789 |
-
tie("sales tax", 0.0, amounts.get("tax", 0.0))
|
| 790 |
-
# Shipping and promotions lie outside net product sales; ad invoices use a separate billing period.
|
| 791 |
-
if ads or "ads billing" in amounts:
|
| 792 |
-
ad_total = 0
|
| 793 |
-
for kind in ("sp", "sb", "sd"):
|
| 794 |
-
table = f"amzn_ads_{kind}_advertised_products"
|
| 795 |
-
if not {"date", "cost"} <= _columns(con, table):
|
| 796 |
-
problem("ads", f"missing source {table}.cost")
|
| 797 |
-
ad_total = None
|
| 798 |
-
break
|
| 799 |
-
total, count, present = con.execute(
|
| 800 |
-
f"SELECT SUM(TRY_CAST(cost AS DECIMAL(38, 2))), count(*), "
|
| 801 |
-
f"count(TRY_CAST(cost AS DECIMAL(38, 2))) FROM {table} "
|
| 802 |
-
"WHERE left(CAST(date AS VARCHAR), 10) BETWEEN ? AND ?", [start, end]).fetchone()
|
| 803 |
-
if not count or count != present or total is None or not math.isfinite(float(total)):
|
| 804 |
-
problem("ads", f"unavailable daily amounts in {table}")
|
| 805 |
-
ad_total = None
|
| 806 |
-
break
|
| 807 |
-
ad_total += total
|
| 808 |
-
tie("ads", ads, ad_total)
|
| 809 |
-
else:
|
| 810 |
-
ad_total = 0
|
| 811 |
-
target = (float(sum(amounts.get(name, 0) for name in
|
| 812 |
-
("product sales", "order fees", "monthly storage", "reimbursements", "account fees")))
|
| 813 |
-
- float(ad_total)) if ad_total is not None else None
|
| 814 |
-
return {"total": target, "parts": parts, "reason": problems[0] if problems else None,
|
| 815 |
-
"rows": sum(row[2] for row in groups),
|
| 816 |
-
"excluded": {k: float(amounts[k]) for k in ("shipping and promotions", "ads billing") if k in amounts}}
|
| 817 |
-
|
| 818 |
-
|
| 819 |
_COVERAGE_REQUIRED = ("settlement_id", "total_amount", "settlement_start_date", "settlement_end_date")
|
| 820 |
|
| 821 |
|
|
@@ -2163,8 +1988,6 @@ def refusal(checks, month):
|
|
| 2163 |
)
|
| 2164 |
for index, check in enumerate(checks):
|
| 2165 |
if not check.get("ok"):
|
| 2166 |
-
if index == 0 and check.get("reason"):
|
| 2167 |
-
return check["reason"]
|
| 2168 |
if index < len(sentences):
|
| 2169 |
return sentences[index](check)
|
| 2170 |
return f"Not verified: the Amazon figures do not tie in {label}."
|
|
@@ -2862,9 +2685,7 @@ def _mirror_has_economics(con):
|
|
| 2862 |
return False
|
| 2863 |
|
| 2864 |
|
| 2865 |
-
def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS
|
| 2866 |
-
"amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products",
|
| 2867 |
-
"amzn_ads_sd_advertised_products")):
|
| 2868 |
"""An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back."""
|
| 2869 |
key = (tenant, tuple(tables))
|
| 2870 |
if key in _SNAPSHOT:
|
|
@@ -2999,7 +2820,7 @@ def typed_cogs_checks(con, computed, typed, usage=None, bounds=None):
|
|
| 2999 |
|
| 3000 |
def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None,
|
| 3001 |
typed_values=None, typed_period=None):
|
| 3002 |
-
"""
|
| 3003 |
Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values."""
|
| 3004 |
month = month or last_complete_month()
|
| 3005 |
bounds = period_bounds(month)
|
|
@@ -3008,35 +2829,25 @@ def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=No
|
|
| 3008 |
if con is None:
|
| 3009 |
con, source = validation_cursor()
|
| 3010 |
checks = []
|
| 3011 |
-
|
| 3012 |
-
|
| 3013 |
-
|
| 3014 |
-
span = settlement_coverage(con, start, end, currency)
|
| 3015 |
-
settled = settlements_total(con, start, end, currency)
|
| 3016 |
-
except SourceUnavailable as exc:
|
| 3017 |
-
label = dt.date.fromisoformat(start).strftime("%B %Y")
|
| 3018 |
-
raise SourceUnavailable(f"Not verified: settlement components in {label}: {exc}") from None
|
| 3019 |
lo, hi = span["start"] or start, span["end"] or end
|
| 3020 |
proceeds = net_proceeds_total(con, lo, hi)
|
| 3021 |
-
|
| 3022 |
if span["start"] is None:
|
| 3023 |
-
|
| 3024 |
-
|
| 3025 |
-
target = aligned["total"]
|
| 3026 |
-
ok, gap = _tie(proceeds, target) if target is not None else (False, math.inf)
|
| 3027 |
covered = {k: span[k] for k in ("start", "end", "label", "partial", "clear", "sentence")}
|
| 3028 |
checks.append({"check": f"amazon_profit {month}: net proceeds tie to settlements within 2% over "
|
| 3029 |
f"{span['label'] or 'no settled days'} ({gap:.2%} apart, {source})",
|
| 3030 |
-
"ok": ok
|
| 3031 |
-
"
|
| 3032 |
-
"components": aligned["parts"], "excluded": aligned.get("excluded", {}),
|
| 3033 |
-
"gap": round(gap, 4), "covered": covered, "tolerance": SETTLEMENT_TOLERANCE})
|
| 3034 |
dropped = net_proceeds_total(con, lo, hi, drop_fees=True)
|
| 3035 |
-
control_ok, control_gap = _tie(dropped,
|
| 3036 |
checks.append({"check": f"amazon_profit {month}: NC net proceeds without fees miss settlements "
|
| 3037 |
f"({control_gap:.2%} apart)",
|
| 3038 |
-
"ok": not control_ok, "a": round(dropped, 2),
|
| 3039 |
-
"b": None if target is None else round(target, 2),
|
| 3040 |
"gap": round(control_gap, 4)})
|
| 3041 |
checks.append({"check": f"amazon_profit {month}: every {settled['currency'] or 'settlement'} "
|
| 3042 |
"settlement row carries a readable date",
|
|
|
|
| 10 |
`ut_amazon_asin_profit` and `ut_amazon_parent_profit` roll the SKU rows up on every refresh.
|
| 11 |
Projects retain their own verified period totals and Impact in the tenant projects store.
|
| 12 |
|
| 13 |
+
`validate()` ties a month's net proceeds to Amazon settlements within 2% over the days its issued
|
| 14 |
+
settlements cover, excluding loan repayments and withheld tax, and proves a negative control that
|
| 15 |
+
drops fees goes red. A refresh runs it for the last complete month and writes no figures when it
|
| 16 |
fails. A passing refresh stores the verified range and the day the rest is due to settle.
|
| 17 |
|
| 18 |
Owned items (R10, R31): a refresh opens one item on a Losing or Thin SKU with no status, for the
|
|
|
|
| 615 |
def economics_currency(con):
|
| 616 |
if "currency_code" not in _columns(con, ECONOMICS):
|
| 617 |
return None
|
| 618 |
+
return con.execute(f"SELECT any_value(currency_code) FROM {ECONOMICS}").fetchone()[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 619 |
|
| 620 |
|
| 621 |
def settlements_total(con, start, end, currency=None):
|
| 622 |
+
"""Settlements posted in the window, excluding loan repayment and withheld tax.
|
| 623 |
|
| 624 |
`currency` keeps settlements whose header row names it: Amazon.ca and Amazon.com.mx settle
|
| 625 |
in their own currency, outside the economics marketplace. Their reports write DD.MM.YYYY.
|
|
|
|
| 629 |
stream = ("JOIN (SELECT settlement_id FROM {t} WHERE currency IS NOT NULL GROUP BY settlement_id "
|
| 630 |
"HAVING any_value(currency) = ?) h USING (settlement_id)").format(t=SETTLEMENTS) if scoped else ""
|
| 631 |
total, rows, undated = con.execute(
|
| 632 |
+
"SELECT COALESCE(SUM(amount) FILTER (WHERE d BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)), 0), "
|
|
|
|
| 633 |
"count(*) FILTER (WHERE d BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)), "
|
| 634 |
"count(*) FILTER (WHERE d IS NULL AND posted_date IS NOT NULL) "
|
| 635 |
"FROM (SELECT amount, posted_date, COALESCE(TRY_CAST(left(posted_date, 10) AS DATE), "
|
| 636 |
"CAST(try_strptime(left(posted_date, 10), '%d.%m.%Y') AS DATE)) AS d "
|
| 637 |
+
f"FROM {SETTLEMENTS} s {stream} WHERE COALESCE(transaction_type, '') <> ? "
|
| 638 |
+
"AND COALESCE(amount_type, '') <> ?)",
|
| 639 |
+
[start, end, start, end] + ([currency] if scoped else []) + [LOAN_REPAYMENT, WITHHELD_TAX]).fetchone()
|
| 640 |
return {"total": float(total), "rows": int(rows), "undated": int(undated),
|
| 641 |
"currency": currency if scoped else None}
|
| 642 |
|
| 643 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 644 |
_COVERAGE_REQUIRED = ("settlement_id", "total_amount", "settlement_start_date", "settlement_end_date")
|
| 645 |
|
| 646 |
|
|
|
|
| 1988 |
)
|
| 1989 |
for index, check in enumerate(checks):
|
| 1990 |
if not check.get("ok"):
|
|
|
|
|
|
|
| 1991 |
if index < len(sentences):
|
| 1992 |
return sentences[index](check)
|
| 1993 |
return f"Not verified: the Amazon figures do not tie in {label}."
|
|
|
|
| 2685 |
return False
|
| 2686 |
|
| 2687 |
|
| 2688 |
+
def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS)):
|
|
|
|
|
|
|
| 2689 |
"""An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back."""
|
| 2690 |
key = (tenant, tuple(tables))
|
| 2691 |
if key in _SNAPSHOT:
|
|
|
|
| 2820 |
|
| 2821 |
def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None,
|
| 2822 |
typed_values=None, typed_period=None):
|
| 2823 |
+
"""validate.py convention: net proceeds tie to settlements within 2% over the month's settled days.
|
| 2824 |
Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values."""
|
| 2825 |
month = month or last_complete_month()
|
| 2826 |
bounds = period_bounds(month)
|
|
|
|
| 2829 |
if con is None:
|
| 2830 |
con, source = validation_cursor()
|
| 2831 |
checks = []
|
| 2832 |
+
currency = economics_currency(con)
|
| 2833 |
+
span = settlement_coverage(con, start, end, currency)
|
| 2834 |
+
# Both sides tie the same covered days: economics by start_date, settlement lines by posted_date.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2835 |
lo, hi = span["start"] or start, span["end"] or end
|
| 2836 |
proceeds = net_proceeds_total(con, lo, hi)
|
| 2837 |
+
settled = settlements_total(con, lo, hi, currency)
|
| 2838 |
if span["start"] is None:
|
| 2839 |
+
settled = dict(settled, total=0.0, rows=0)
|
| 2840 |
+
ok, gap = _tie(proceeds, settled["total"])
|
|
|
|
|
|
|
| 2841 |
covered = {k: span[k] for k in ("start", "end", "label", "partial", "clear", "sentence")}
|
| 2842 |
checks.append({"check": f"amazon_profit {month}: net proceeds tie to settlements within 2% over "
|
| 2843 |
f"{span['label'] or 'no settled days'} ({gap:.2%} apart, {source})",
|
| 2844 |
+
"ok": ok, "a": round(proceeds, 2), "b": round(settled["total"], 2),
|
| 2845 |
+
"gap": round(gap, 4), "covered": covered})
|
|
|
|
|
|
|
| 2846 |
dropped = net_proceeds_total(con, lo, hi, drop_fees=True)
|
| 2847 |
+
control_ok, control_gap = _tie(dropped, settled["total"])
|
| 2848 |
checks.append({"check": f"amazon_profit {month}: NC net proceeds without fees miss settlements "
|
| 2849 |
f"({control_gap:.2%} apart)",
|
| 2850 |
+
"ok": not control_ok, "a": round(dropped, 2), "b": round(settled["total"], 2),
|
|
|
|
| 2851 |
"gap": round(control_gap, 4)})
|
| 2852 |
checks.append({"check": f"amazon_profit {month}: every {settled['currency'] or 'settlement'} "
|
| 2853 |
"settlement row carries a readable date",
|
platform/modules/amazon_profit_account.py
CHANGED
|
@@ -487,11 +487,9 @@ def validate(con, view):
|
|
| 487 |
span = profit.settlement_coverage(con, start, end, currency)
|
| 488 |
# The tie reads only the days issued settlements cover, on both sides.
|
| 489 |
lo, hi = span["start"] or start, span["end"] or end
|
| 490 |
-
settled = profit.
|
| 491 |
-
why = settled["reason"]
|
| 492 |
if span["start"] is None:
|
| 493 |
settled = dict(settled, rows=0)
|
| 494 |
-
why = f"Not verified: settlement components in {dt.date.fromisoformat(start):%B %Y}: no settled days."
|
| 495 |
elif (lo, hi) != (start, end) and bridge is not None:
|
| 496 |
proceeds = profit.net_proceeds_total(con, lo, hi)
|
| 497 |
except profit.SourceUnavailable as exc:
|
|
@@ -503,14 +501,12 @@ def validate(con, view):
|
|
| 503 |
gap = gap if gap is None or math.isfinite(gap) else None
|
| 504 |
covered = {k: span[k] for k in ("start", "end", "label", "partial", "clear", "sentence")} if span else None
|
| 505 |
checks.append({"check": "Net proceeds tie to settlements within 2%",
|
| 506 |
-
"ok": tied and not why,
|
| 507 |
"a": proceeds, "b": target, "gap": gap, "tolerance": profit.SETTLEMENT_TOLERANCE,
|
| 508 |
-
"reason": why or ("No settlement source rows for window" if target is None else
|
| 509 |
-
|
| 510 |
-
for part in settled["parts"]] if settled else [],
|
| 511 |
-
"excluded": settled.get("excluded", {}) if settled else {},
|
| 512 |
"covered": covered, "note": covered and covered["sentence"],
|
| 513 |
-
"sourceTables": [ECONOMICS, profit.SETTLEMENTS
|
| 514 |
values = [_cents(s["netProfit"]["value"]) for s in view["skus"]]
|
| 515 |
sku_net = sum(values) / 100 if values and all(c is not None for c in values) else None
|
| 516 |
check("Account net profit equals sum of SKU profit after COGS", metrics["netProfit"]["value"], sku_net,
|
|
|
|
| 487 |
span = profit.settlement_coverage(con, start, end, currency)
|
| 488 |
# The tie reads only the days issued settlements cover, on both sides.
|
| 489 |
lo, hi = span["start"] or start, span["end"] or end
|
| 490 |
+
settled = profit.settlements_total(con, lo, hi, currency)
|
|
|
|
| 491 |
if span["start"] is None:
|
| 492 |
settled = dict(settled, rows=0)
|
|
|
|
| 493 |
elif (lo, hi) != (start, end) and bridge is not None:
|
| 494 |
proceeds = profit.net_proceeds_total(con, lo, hi)
|
| 495 |
except profit.SourceUnavailable as exc:
|
|
|
|
| 501 |
gap = gap if gap is None or math.isfinite(gap) else None
|
| 502 |
covered = {k: span[k] for k in ("start", "end", "label", "partial", "clear", "sentence")} if span else None
|
| 503 |
checks.append({"check": "Net proceeds tie to settlements within 2%",
|
| 504 |
+
"ok": tied and not why and not settled["undated"],
|
| 505 |
"a": proceeds, "b": target, "gap": gap, "tolerance": profit.SETTLEMENT_TOLERANCE,
|
| 506 |
+
"reason": why or ("No settlement source rows for window" if target is None else
|
| 507 |
+
"Settlement dates unavailable" if settled["undated"] else None),
|
|
|
|
|
|
|
| 508 |
"covered": covered, "note": covered and covered["sentence"],
|
| 509 |
+
"sourceTables": [ECONOMICS, profit.SETTLEMENTS]})
|
| 510 |
values = [_cents(s["netProfit"]["value"]) for s in view["skus"]]
|
| 511 |
sku_net = sum(values) / 100 if values and all(c is not None for c in values) else None
|
| 512 |
check("Account net profit equals sum of SKU profit after COGS", metrics["netProfit"]["value"], sku_net,
|
platform/modules/procurement.py
CHANGED
|
@@ -20,7 +20,6 @@ READ-ONLY on Odoo; the team's Order Qty + master-data edits persist in the platf
|
|
| 20 |
"""
|
| 21 |
import sys
|
| 22 |
import json
|
| 23 |
-
import hashlib
|
| 24 |
import math
|
| 25 |
import calendar
|
| 26 |
import datetime as dt
|
|
@@ -154,22 +153,6 @@ def _preorders(cutoff_iso):
|
|
| 154 |
return qty, need
|
| 155 |
|
| 156 |
|
| 157 |
-
def _fba_refusal(product_id, quantity, report):
|
| 158 |
-
"""Classify unknown FBA stock without treating missing or invalid quantities as zero."""
|
| 159 |
-
if not product_id:
|
| 160 |
-
return {'cause_code': 'no_product_id', 'cause': 'no product id for this SKU'}
|
| 161 |
-
if report.get('available') is False:
|
| 162 |
-
cause_code = ('non_numeric_fba' if report.get('cause_code') == 'non_numeric_fba'
|
| 163 |
-
else 'report_unavailable')
|
| 164 |
-
return {'cause_code': cause_code,
|
| 165 |
-
'cause': report.get('cause') or 'FBA stock report unavailable'}
|
| 166 |
-
if (not isinstance(quantity, (int, float)) or isinstance(quantity, bool)
|
| 167 |
-
or not math.isfinite(quantity)):
|
| 168 |
-
return {'cause_code': 'non_numeric_fba',
|
| 169 |
-
'cause': 'FBA quantity is missing or is not a finite number for this product'}
|
| 170 |
-
return None
|
| 171 |
-
|
| 172 |
-
|
| 173 |
def recommendations(t=None):
|
| 174 |
"""Base per-SKU table (curated master data + live Odoo demand/stock/PO + alt-vendors). The UI
|
| 175 |
overlays the team's vendor/country/cost edits on top; reorder methods + CBM/container are here."""
|
|
@@ -208,13 +191,13 @@ def recommendations(t=None):
|
|
| 208 |
try:
|
| 209 |
import modules.warehouse as warehouse
|
| 210 |
fba_by_pid, fba_report = warehouse.fba_qty_by_product_id(
|
| 211 |
-
[p
|
| 212 |
except Exception as exc:
|
| 213 |
fba_by_pid, fba_report = None, {
|
| 214 |
'available': False, 'cause': 'the FBA warehouse read failed: ' + type(exc).__name__}
|
| 215 |
rows = []
|
| 216 |
for code, p in by_code.items():
|
| 217 |
-
pid = p
|
| 218 |
meta = sup.get(code, {})
|
| 219 |
lead = meta.get('lead') or 0
|
| 220 |
tmpl = O.m2o_id(p.get('product_tmpl_id'))
|
|
@@ -228,8 +211,7 @@ def recommendations(t=None):
|
|
| 228 |
on_hand = p.get('qty_available') or 0.0
|
| 229 |
po = inpo.get(pid, 0.0)
|
| 230 |
fba = (fba_by_pid or {}).get(pid)
|
| 231 |
-
|
| 232 |
-
fba_known = fba_refusal is None
|
| 233 |
avail = on_hand - fba + po if fba_known else None
|
| 234 |
days_cover = ((avail / avg_day) if avg_day > 0 else
|
| 235 |
(None if avail <= 0 else 9999.0)) if avail is not None else None
|
|
@@ -263,7 +245,8 @@ def recommendations(t=None):
|
|
| 263 |
})
|
| 264 |
if not fba_known:
|
| 265 |
rows[-1]['_coverage_report'] = {**fba_report, 'available': False,
|
| 266 |
-
|
|
|
|
| 267 |
return rows
|
| 268 |
|
| 269 |
|
|
@@ -666,13 +649,13 @@ def validate_coverage(rows, t=None, oracle=None, demand=None):
|
|
| 666 |
unavailable, wrong_days, wrong_buy = [], [], []
|
| 667 |
for r in rows:
|
| 668 |
pid = r.get('_product_id')
|
| 669 |
-
|
| 670 |
-
|
| 671 |
expected_days, expected_buy = None, None
|
| 672 |
-
if
|
| 673 |
unavailable.append({'code': r['code'],
|
| 674 |
-
'
|
| 675 |
-
|
| 676 |
else:
|
| 677 |
effective = stock['non_fba_by_pid'].get(pid, 0.0) + float(r.get('incoming_po') or 0.0)
|
| 678 |
daily = demand.get(pid, 0.0) / window_days
|
|
|
|
| 20 |
"""
|
| 21 |
import sys
|
| 22 |
import json
|
|
|
|
| 23 |
import math
|
| 24 |
import calendar
|
| 25 |
import datetime as dt
|
|
|
|
| 153 |
return qty, need
|
| 154 |
|
| 155 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
def recommendations(t=None):
|
| 157 |
"""Base per-SKU table (curated master data + live Odoo demand/stock/PO + alt-vendors). The UI
|
| 158 |
overlays the team's vendor/country/cost edits on top; reorder methods + CBM/container are here."""
|
|
|
|
| 191 |
try:
|
| 192 |
import modules.warehouse as warehouse
|
| 193 |
fba_by_pid, fba_report = warehouse.fba_qty_by_product_id(
|
| 194 |
+
[p['id'] for p in by_code.values()])
|
| 195 |
except Exception as exc:
|
| 196 |
fba_by_pid, fba_report = None, {
|
| 197 |
'available': False, 'cause': 'the FBA warehouse read failed: ' + type(exc).__name__}
|
| 198 |
rows = []
|
| 199 |
for code, p in by_code.items():
|
| 200 |
+
pid = p['id']
|
| 201 |
meta = sup.get(code, {})
|
| 202 |
lead = meta.get('lead') or 0
|
| 203 |
tmpl = O.m2o_id(p.get('product_tmpl_id'))
|
|
|
|
| 211 |
on_hand = p.get('qty_available') or 0.0
|
| 212 |
po = inpo.get(pid, 0.0)
|
| 213 |
fba = (fba_by_pid or {}).get(pid)
|
| 214 |
+
fba_known = isinstance(fba, (int, float)) and fba_report.get('available') is not False
|
|
|
|
| 215 |
avail = on_hand - fba + po if fba_known else None
|
| 216 |
days_cover = ((avail / avg_day) if avg_day > 0 else
|
| 217 |
(None if avail <= 0 else 9999.0)) if avail is not None else None
|
|
|
|
| 245 |
})
|
| 246 |
if not fba_known:
|
| 247 |
rows[-1]['_coverage_report'] = {**fba_report, 'available': False,
|
| 248 |
+
'cause': fba_report.get('cause')
|
| 249 |
+
or 'FBA stock unavailable for this product'}
|
| 250 |
return rows
|
| 251 |
|
| 252 |
|
|
|
|
| 649 |
unavailable, wrong_days, wrong_buy = [], [], []
|
| 650 |
for r in rows:
|
| 651 |
pid = r.get('_product_id')
|
| 652 |
+
known = (pid is not None and isinstance(r.get('_fba'), (int, float))
|
| 653 |
+
and r.get('_coverage_report', {}).get('available') is not False)
|
| 654 |
expected_days, expected_buy = None, None
|
| 655 |
+
if not known:
|
| 656 |
unavailable.append({'code': r['code'],
|
| 657 |
+
'cause': r.get('_coverage_report', {}).get('cause')
|
| 658 |
+
or 'FBA stock unavailable for this product'})
|
| 659 |
else:
|
| 660 |
effective = stock['non_fba_by_pid'].get(pid, 0.0) + float(r.get('incoming_po') or 0.0)
|
| 661 |
daily = demand.get(pid, 0.0) / window_days
|
platform/modules/returns_refunds.py
CHANGED
|
@@ -1987,33 +1987,6 @@ def _return_source(cur, table, date, quantity, sku="sku", reason="reason"):
|
|
| 1987 |
+ (f" FROM {table}" if cols else " WHERE false") + ") captured")
|
| 1988 |
|
| 1989 |
|
| 1990 |
-
def _refund_overlap_relations():
|
| 1991 |
-
"""Match wildcard identities through distinct projections with at most eight probes per identity."""
|
| 1992 |
-
fields = ('order_id', 'sku', 'settlement_id')
|
| 1993 |
-
mask = '+'.join(f'CASE WHEN {field} IS NULL THEN 0 ELSE {1 << i} END'
|
| 1994 |
-
for i, field in enumerate(fields))
|
| 1995 |
-
projection = ','.join(f"CASE WHEN (p.mask & {1 << i})!=0 THEN {field} ELSE '' END AS {field}"
|
| 1996 |
-
for i, field in enumerate(fields))
|
| 1997 |
-
matches = ' AND '.join(
|
| 1998 |
-
f"b.{field}=CASE WHEN (a.mask & m.mask & {1 << i})!=0 THEN a.{field} ELSE '' END"
|
| 1999 |
-
for i, field in enumerate(fields))
|
| 2000 |
-
identities = ' AND '.join(f'a.{field} IS NOT DISTINCT FROM o.{field}' for field in fields)
|
| 2001 |
-
return f"""refund_identities AS (
|
| 2002 |
-
SELECT DISTINCT source,account,order_id,sku,settlement_id,{mask} AS mask FROM refund_candidates),
|
| 2003 |
-
refund_overlap_keys AS (
|
| 2004 |
-
SELECT DISTINCT source,account,i.mask AS identity_mask,p.mask AS projection_mask,{projection}
|
| 2005 |
-
FROM refund_identities i CROSS JOIN range(8) p(mask) WHERE (i.mask & p.mask)=p.mask),
|
| 2006 |
-
refund_overlaps AS (
|
| 2007 |
-
SELECT a.source,a.account,a.order_id,a.sku,a.settlement_id,bool_or(b.source IS NOT NULL) AS overlap
|
| 2008 |
-
FROM refund_identities a CROSS JOIN range(8) m(mask)
|
| 2009 |
-
LEFT JOIN refund_overlap_keys b ON a.source!=b.source AND a.account=b.account
|
| 2010 |
-
AND b.identity_mask=m.mask AND b.projection_mask=(a.mask & m.mask) AND {matches}
|
| 2011 |
-
GROUP BY a.source,a.account,a.order_id,a.sku,a.settlement_id),
|
| 2012 |
-
refund_events AS (SELECT a.*,COALESCE(a.sku,a.asin,'{NO_SKU}') AS key,o.overlap
|
| 2013 |
-
FROM refund_candidates a JOIN refund_overlaps o ON a.source=o.source AND a.account=o.account
|
| 2014 |
-
AND {identities})"""
|
| 2015 |
-
|
| 2016 |
-
|
| 2017 |
def _refund_relations(cur):
|
| 2018 |
"""Keep source identities, capture deduplication and ambiguous overlap inside SQL."""
|
| 2019 |
sc, tc = (_columns(cur, table) for table in FINANCE_TABLES)
|
|
@@ -2073,20 +2046,24 @@ def _refund_relations(cur):
|
|
| 2073 |
SELECT 'finance',account,settlement_id,event_id,item_id,order_id,sku,asin,day,NULL,money,currency,
|
| 2074 |
conflict,event_id IS NULL OR position IS NULL OR NOT isfinite(position) OR position<0
|
| 2075 |
OR position!=trunc(position) OR sku IS NULL AND asin IS NULL, true FROM finance_events),
|
| 2076 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2077 |
|
| 2078 |
|
| 2079 |
-
def _refund_totals_sql(first, last, group=False
|
| 2080 |
"""Aggregate only event-grain refunds after source capture deduplication."""
|
| 2081 |
-
return ("SELECT " + ("
|
| 2082 |
"sum(money) AS refund_cost,min(currency) AS refund_currency,count(DISTINCT currency) AS currencies,"
|
| 2083 |
"count(*) FILTER (WHERE currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}') "
|
| 2084 |
"OR money IS NULL OR NOT isfinite(money) OR money<0) AS money_bad,"
|
| 2085 |
"count(*) FILTER (WHERE identity_bad OR conflict OR overlap OR day IS NULL) AS identity_bad,"
|
| 2086 |
"count(*) FILTER (WHERE unit_bad) AS unit_bad,count(*) FILTER (WHERE source='finance') AS finance_units,"
|
| 2087 |
"count(*) FILTER (WHERE overlap) AS overlaps FROM refund_events "
|
| 2088 |
-
|
| 2089 |
-
f"WHERE day BETWEEN DATE '{first}' AND DATE '{last}' OR day IS NULL" + (" GROUP BY key" if group else "")))
|
| 2090 |
|
| 2091 |
|
| 2092 |
def _rate_values(row, ready):
|
|
@@ -3896,51 +3873,25 @@ def _dashboard_windows(first, last):
|
|
| 3896 |
|
| 3897 |
|
| 3898 |
def _dashboard_rates(cur, first, last, independent):
|
| 3899 |
-
|
| 3900 |
-
|
| 3901 |
-
|
| 3902 |
-
def _dashboard_rate_windows(cur, windows, independent=False, *, ready=None):
|
| 3903 |
-
"""Fold total, comparison and monthly windows in one query over complete source captures."""
|
| 3904 |
-
bounds = ','.join(f"({i},DATE '{a.isoformat()}',DATE '{b.isoformat()}')" for i, (a, b) in enumerate(windows))
|
| 3905 |
merchant_sku = _first(_columns(cur, MFN_TABLE), 'merchant_sku', 'sku') or 'merchant_sku'
|
| 3906 |
physical = _return_source(cur, RETURNS_TABLE, 'return_date', 'quantity') + ' UNION ALL ' + _return_source(
|
| 3907 |
cur, MFN_TABLE, 'return_request_date', 'return_quantity', merchant_sku, 'return_reason')
|
| 3908 |
shipments = _return_source(cur, SHIPPED_TABLE, 'shipment_date', 'quantity')
|
| 3909 |
-
|
| 3910 |
-
|
| 3911 |
-
|
| 3912 |
-
|
| 3913 |
-
|
| 3914 |
-
FROM windows LEFT JOIN returns ON day>=first AND day<=last GROUP BY window_id),
|
| 3915 |
-
s AS (SELECT window_id,COALESCE(sum(units),0) AS shipped
|
| 3916 |
-
FROM windows LEFT JOIN shipments ON day>=first AND day<=last GROUP BY window_id),
|
| 3917 |
-
f AS (SELECT window_id,COALESCE(sum(units),0) AS refunded,sum(money) AS refund_cost,
|
| 3918 |
min(currency) AS refund_currency,count(DISTINCT currency) AS currencies,
|
| 3919 |
count(*) FILTER (WHERE money IS NULL OR money<0 OR NOT isfinite(money)
|
| 3920 |
-
OR currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}')) AS money_bad,
|
| 3921 |
bool_or(identity_bad OR conflict OR overlap OR day IS NULL) AS identity_bad,
|
| 3922 |
-
bool_or(unit_bad) AS unit_bad FROM refund_events
|
| 3923 |
-
|
| 3924 |
-
|
| 3925 |
-
LEFT JOIN f USING(window_id) ORDER BY window_id"""
|
| 3926 |
-
else:
|
| 3927 |
-
sql = prefix + """events AS (
|
| 3928 |
-
SELECT day,units AS returned,0.0 AS shipped FROM returns
|
| 3929 |
-
UNION ALL SELECT day,0.0,units FROM shipments),
|
| 3930 |
-
quantities AS (SELECT window_id,COALESCE(sum(returned),0) AS returned,
|
| 3931 |
-
COALESCE(sum(shipped),0) AS shipped FROM windows LEFT JOIN events
|
| 3932 |
-
ON day BETWEEN first AND last GROUP BY window_id),
|
| 3933 |
-
refunds AS (""" + _refund_totals_sql(None, None, windows=True) + """)
|
| 3934 |
-
SELECT quantities.*,refunds.* EXCLUDE(window_id) FROM quantities
|
| 3935 |
-
LEFT JOIN refunds USING(window_id) ORDER BY window_id"""
|
| 3936 |
-
ready = _rate_availability(cur) if ready is None else ready
|
| 3937 |
-
result = cur.execute(sql)
|
| 3938 |
-
names = [field[0] for field in result.description]
|
| 3939 |
-
rows = []
|
| 3940 |
-
while batch := result.fetchmany(MAX_PAGE_SIZE):
|
| 3941 |
-
rows.extend(dict(zip(names, row)) for row in batch)
|
| 3942 |
-
formatter = _independent_rate_values if independent else _rate_values
|
| 3943 |
-
return [formatter(row, ready) for row in rows]
|
| 3944 |
|
| 3945 |
|
| 3946 |
def _dashboard_figures(tenant_key, *, start, end, st=None, user=None, independent=False):
|
|
@@ -3950,12 +3901,12 @@ def _dashboard_figures(tenant_key, *, start, end, st=None, user=None, independen
|
|
| 3950 |
total, previous, points, reasons, reasons_value = {}, {}, [], [], None
|
| 3951 |
if allowed:
|
| 3952 |
with _screen_reader(tenant_key, st) as (cur, _present):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3953 |
ready = _rate_availability(cur)
|
| 3954 |
-
months = list(_dashboard_windows(first, last))
|
| 3955 |
-
windows = [(first, last), (first - (last - first) - dt.timedelta(days=1),
|
| 3956 |
-
first - dt.timedelta(days=1)), *months]
|
| 3957 |
-
total, previous, *monthly = _dashboard_rate_windows(cur, windows, independent, ready=ready)
|
| 3958 |
-
points = [(a.isoformat(), values) for (a, _), values in zip(months, monthly)]
|
| 3959 |
if ready['returns'] and ready['merchantReturns']:
|
| 3960 |
merchant_sku = _first(_columns(cur, MFN_TABLE), 'merchant_sku', 'sku') or 'merchant_sku'
|
| 3961 |
source = _return_source(cur, RETURNS_TABLE, 'return_date', 'quantity') + ' UNION ALL ' + _return_source(
|
|
@@ -4035,19 +3986,19 @@ def validate_figure(tenant_key, figure, *, start, end, st=None, user=None):
|
|
| 4035 |
series += [dict(label=reason_label(reason), value=units) for reason, units in batch]
|
| 4036 |
value, unit, prior = sum(row['value'] for row in series), 'returns', None
|
| 4037 |
else:
|
| 4038 |
-
|
| 4039 |
-
windows = [(first, last), (first - (last - first) - dt.timedelta(days=1),
|
| 4040 |
-
first - dt.timedelta(days=1)), *months]
|
| 4041 |
-
total, previous, *monthly = _dashboard_rate_windows(cur, windows, True)
|
| 4042 |
value = total[key]
|
| 4043 |
unit = total['currency'] if key == 'refundCost' else '%'
|
| 4044 |
if value is None or not unit:
|
| 4045 |
return failed
|
|
|
|
|
|
|
| 4046 |
prior = previous[key]
|
| 4047 |
if key == 'refundCost' and previous['currency'] != unit:
|
| 4048 |
prior = None
|
| 4049 |
series = []
|
| 4050 |
-
for
|
|
|
|
| 4051 |
if key == 'refundCost' and row[key] is not None and row['currency'] != unit:
|
| 4052 |
return failed
|
| 4053 |
series.append(dict(x=begin.isoformat(), y=row[key]))
|
|
|
|
| 1987 |
+ (f" FROM {table}" if cols else " WHERE false") + ") captured")
|
| 1988 |
|
| 1989 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1990 |
def _refund_relations(cur):
|
| 1991 |
"""Keep source identities, capture deduplication and ambiguous overlap inside SQL."""
|
| 1992 |
sc, tc = (_columns(cur, table) for table in FINANCE_TABLES)
|
|
|
|
| 2046 |
SELECT 'finance',account,settlement_id,event_id,item_id,order_id,sku,asin,day,NULL,money,currency,
|
| 2047 |
conflict,event_id IS NULL OR position IS NULL OR NOT isfinite(position) OR position<0
|
| 2048 |
OR position!=trunc(position) OR sku IS NULL AND asin IS NULL, true FROM finance_events),
|
| 2049 |
+
refund_events AS (SELECT a.*,COALESCE(a.sku,a.asin,'{NO_SKU}') AS key,
|
| 2050 |
+
EXISTS(SELECT 1 FROM refund_candidates b WHERE a.source!=b.source AND a.account=b.account
|
| 2051 |
+
AND (a.order_id=b.order_id OR a.order_id IS NULL OR b.order_id IS NULL)
|
| 2052 |
+
AND (a.sku=b.sku OR a.sku IS NULL OR b.sku IS NULL)
|
| 2053 |
+
AND (a.settlement_id=b.settlement_id OR a.settlement_id IS NULL OR b.settlement_id IS NULL)) AS overlap
|
| 2054 |
+
FROM refund_candidates a)"""
|
| 2055 |
|
| 2056 |
|
| 2057 |
+
def _refund_totals_sql(first, last, group=False):
|
| 2058 |
"""Aggregate only event-grain refunds after source capture deduplication."""
|
| 2059 |
+
return ("SELECT " + ("key," if group else "") + "COALESCE(sum(units),0) AS refunded,"
|
| 2060 |
"sum(money) AS refund_cost,min(currency) AS refund_currency,count(DISTINCT currency) AS currencies,"
|
| 2061 |
"count(*) FILTER (WHERE currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}') "
|
| 2062 |
"OR money IS NULL OR NOT isfinite(money) OR money<0) AS money_bad,"
|
| 2063 |
"count(*) FILTER (WHERE identity_bad OR conflict OR overlap OR day IS NULL) AS identity_bad,"
|
| 2064 |
"count(*) FILTER (WHERE unit_bad) AS unit_bad,count(*) FILTER (WHERE source='finance') AS finance_units,"
|
| 2065 |
"count(*) FILTER (WHERE overlap) AS overlaps FROM refund_events "
|
| 2066 |
+
f"WHERE day BETWEEN DATE '{first}' AND DATE '{last}' OR day IS NULL" + (" GROUP BY key" if group else ""))
|
|
|
|
| 2067 |
|
| 2068 |
|
| 2069 |
def _rate_values(row, ready):
|
|
|
|
| 3873 |
|
| 3874 |
|
| 3875 |
def _dashboard_rates(cur, first, last, independent):
|
| 3876 |
+
if not independent:
|
| 3877 |
+
return _aggregate(cur, _relations(cur, first, last, {'risingRatio': 1.5, 'risingMinPct': 5}))
|
| 3878 |
+
# Independent scalar folds avoid the serving membership and formatter projections.
|
|
|
|
|
|
|
|
|
|
| 3879 |
merchant_sku = _first(_columns(cur, MFN_TABLE), 'merchant_sku', 'sku') or 'merchant_sku'
|
| 3880 |
physical = _return_source(cur, RETURNS_TABLE, 'return_date', 'quantity') + ' UNION ALL ' + _return_source(
|
| 3881 |
cur, MFN_TABLE, 'return_request_date', 'return_quantity', merchant_sku, 'return_reason')
|
| 3882 |
shipments = _return_source(cur, SHIPPED_TABLE, 'shipment_date', 'quantity')
|
| 3883 |
+
operands = _mirror_one(cur, f"""WITH returns AS ({physical}),shipments AS ({shipments}),
|
| 3884 |
+
{_refund_relations(cur)} SELECT * FROM
|
| 3885 |
+
(SELECT COALESCE(sum(units),0) AS returned FROM returns WHERE day BETWEEN ? AND ?) r CROSS JOIN
|
| 3886 |
+
(SELECT COALESCE(sum(units),0) AS shipped FROM shipments WHERE day BETWEEN ? AND ?) s CROSS JOIN
|
| 3887 |
+
(SELECT COALESCE(sum(units),0) AS refunded,sum(money) AS refund_cost,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3888 |
min(currency) AS refund_currency,count(DISTINCT currency) AS currencies,
|
| 3889 |
count(*) FILTER (WHERE money IS NULL OR money<0 OR NOT isfinite(money)
|
| 3890 |
+
OR currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{{3}}')) AS money_bad,
|
| 3891 |
bool_or(identity_bad OR conflict OR overlap OR day IS NULL) AS identity_bad,
|
| 3892 |
+
bool_or(unit_bad) AS unit_bad FROM refund_events
|
| 3893 |
+
WHERE day>=? AND day<=? OR day IS NULL) f""", [first, last, first, last, first, last])
|
| 3894 |
+
return _independent_rate_values(operands, _rate_availability(cur))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3895 |
|
| 3896 |
|
| 3897 |
def _dashboard_figures(tenant_key, *, start, end, st=None, user=None, independent=False):
|
|
|
|
| 3901 |
total, previous, points, reasons, reasons_value = {}, {}, [], [], None
|
| 3902 |
if allowed:
|
| 3903 |
with _screen_reader(tenant_key, st) as (cur, _present):
|
| 3904 |
+
total = _dashboard_rates(cur, first, last, independent)
|
| 3905 |
+
previous = _dashboard_rates(cur, first - (last - first) - dt.timedelta(days=1),
|
| 3906 |
+
first - dt.timedelta(days=1), independent)
|
| 3907 |
+
points = [(a.isoformat(), _dashboard_rates(cur, a, b, independent))
|
| 3908 |
+
for a, b in _dashboard_windows(first, last)]
|
| 3909 |
ready = _rate_availability(cur)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3910 |
if ready['returns'] and ready['merchantReturns']:
|
| 3911 |
merchant_sku = _first(_columns(cur, MFN_TABLE), 'merchant_sku', 'sku') or 'merchant_sku'
|
| 3912 |
source = _return_source(cur, RETURNS_TABLE, 'return_date', 'quantity') + ' UNION ALL ' + _return_source(
|
|
|
|
| 3986 |
series += [dict(label=reason_label(reason), value=units) for reason, units in batch]
|
| 3987 |
value, unit, prior = sum(row['value'] for row in series), 'returns', None
|
| 3988 |
else:
|
| 3989 |
+
total = _dashboard_rates(cur, first, last, True)
|
|
|
|
|
|
|
|
|
|
| 3990 |
value = total[key]
|
| 3991 |
unit = total['currency'] if key == 'refundCost' else '%'
|
| 3992 |
if value is None or not unit:
|
| 3993 |
return failed
|
| 3994 |
+
previous = _dashboard_rates(cur, first - (last - first) - dt.timedelta(days=1),
|
| 3995 |
+
first - dt.timedelta(days=1), True)
|
| 3996 |
prior = previous[key]
|
| 3997 |
if key == 'refundCost' and previous['currency'] != unit:
|
| 3998 |
prior = None
|
| 3999 |
series = []
|
| 4000 |
+
for begin, stop in _dashboard_windows(first, last):
|
| 4001 |
+
row = _dashboard_rates(cur, begin, stop, True)
|
| 4002 |
if key == 'refundCost' and row[key] is not None and row['currency'] != unit:
|
| 4003 |
return failed
|
| 4004 |
series.append(dict(x=begin.isoformat(), y=row[key]))
|
platform/modules/warehouse.py
CHANGED
|
@@ -13,7 +13,6 @@ reconciles row-built aggregates to independent read_group/search_count paths.
|
|
| 13 |
"""
|
| 14 |
import sys
|
| 15 |
import datetime as dt
|
| 16 |
-
import math
|
| 17 |
from pathlib import Path
|
| 18 |
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 19 |
import core.odoo as O
|
|
@@ -793,14 +792,9 @@ def fba_qty_by_product_id(product_ids=None, wh=None):
|
|
| 793 |
for i in range(0, len(ids), _FBA_BATCH):
|
| 794 |
batches += 1
|
| 795 |
chunk = ids[i:i + _FBA_BATCH]
|
| 796 |
-
|
| 797 |
-
for r in o.execute_kw('product.product', 'search_read',
|
| 798 |
-
[[('id', 'in', chunk), ('active', 'in', [True, False])]],
|
| 799 |
{'fields': ['id', 'qty_available'], 'context': ctx}):
|
| 800 |
-
|
| 801 |
-
if (isinstance(quantity, (int, float)) and not isinstance(quantity, bool)
|
| 802 |
-
and math.isfinite(quantity)):
|
| 803 |
-
out[r['id']] = quantity
|
| 804 |
return out, fba_report(wh, len(ids), len(out), batches,
|
| 805 |
'stock.warehouse.code at runtime, never a pinned id')
|
| 806 |
|
|
|
|
| 13 |
"""
|
| 14 |
import sys
|
| 15 |
import datetime as dt
|
|
|
|
| 16 |
from pathlib import Path
|
| 17 |
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 18 |
import core.odoo as O
|
|
|
|
| 792 |
for i in range(0, len(ids), _FBA_BATCH):
|
| 793 |
batches += 1
|
| 794 |
chunk = ids[i:i + _FBA_BATCH]
|
| 795 |
+
for r in o.execute_kw('product.product', 'search_read', [[('id', 'in', chunk)]],
|
|
|
|
|
|
|
| 796 |
{'fields': ['id', 'qty_available'], 'context': ctx}):
|
| 797 |
+
out[r['id']] = r.get('qty_available') or 0.0
|
|
|
|
|
|
|
|
|
|
| 798 |
return out, fba_report(wh, len(ids), len(out), batches,
|
| 799 |
'stock.warehouse.code at runtime, never a pinned id')
|
| 800 |
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
07e8d9ecae1d7a64e32ba15267a878944ef225627eed362f47d4c656e681f706
|
web/src/apps/AppRail.tsx
CHANGED
|
@@ -13,7 +13,6 @@ interface RailApplication {
|
|
| 13 |
deepLink?: string;
|
| 14 |
templateKind?: string;
|
| 15 |
templateConnector?: string;
|
| 16 |
-
navPlacement?: { order?: number };
|
| 17 |
openTarget?: { database?: string };
|
| 18 |
compatibleDatabases?: string[];
|
| 19 |
capabilities?: Record<string, boolean>;
|
|
@@ -32,8 +31,7 @@ function railApplications(body: unknown): RailApplication[] {
|
|
| 32 |
|| row.capabilities?.read === false) return false;
|
| 33 |
seen.add(row.id);
|
| 34 |
return true;
|
| 35 |
-
}).sort((a, b) =>
|
| 36 |
-
|| a.name.localeCompare(b.name, undefined, { sensitivity: "base" }) || a.id.localeCompare(b.id));
|
| 37 |
}
|
| 38 |
|
| 39 |
/** An Amazon App's rail row draws its own mark, never the Amazon logo (QI37). */
|
|
|
|
| 13 |
deepLink?: string;
|
| 14 |
templateKind?: string;
|
| 15 |
templateConnector?: string;
|
|
|
|
| 16 |
openTarget?: { database?: string };
|
| 17 |
compatibleDatabases?: string[];
|
| 18 |
capabilities?: Record<string, boolean>;
|
|
|
|
| 31 |
|| row.capabilities?.read === false) return false;
|
| 32 |
seen.add(row.id);
|
| 33 |
return true;
|
| 34 |
+
}).sort((a, b) => a.name.localeCompare(b.name, undefined, { sensitivity: "base" }) || a.id.localeCompare(b.id));
|
|
|
|
| 35 |
}
|
| 36 |
|
| 37 |
/** An Amazon App's rail row draws its own mark, never the Amazon logo (QI37). */
|
web/src/apps/account-health/AppView.tsx
CHANGED
|
@@ -1,6 +1,5 @@
|
|
| 1 |
-
import { useCallback, useEffect,
|
| 2 |
import { API_V1, CREDENTIALS } from "../../apiContract";
|
| 3 |
-
import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface";
|
| 4 |
import { useEscapeClose } from "../../ui/useEscapeClose";
|
| 5 |
import { AppTabs } from "../AppTabs";
|
| 6 |
import { LineChart } from "../../viz/kit";
|
|
@@ -8,7 +7,7 @@ import Dashboard from "./Dashboard";
|
|
| 8 |
import "../../styles/sdlc79/account-health.css";
|
| 9 |
import "../../styles/sdlc81/account-health-risk.css";
|
| 10 |
|
| 11 |
-
// The dispatcher owns the App frame. This view owns its lists and detail
|
| 12 |
|
| 13 |
/** C2: the App as the dispatcher passes it. Structurally equal to lane A's AppViewProps application. */
|
| 14 |
interface AccountHealthApplication {
|
|
@@ -94,30 +93,16 @@ interface RiskPage {
|
|
| 94 |
|
| 95 |
interface Point { day: string; value: number | null; target: number | null; status: string }
|
| 96 |
interface HistoryPage { points: Point[]; cursor: string | null; label: string; unit: string }
|
| 97 |
-
interface Category {
|
| 98 |
-
key: string; label: string; count: number | null; status: string; target: string;
|
| 99 |
-
window: { from: string; to: string }; sourceNote: string; sellerCentralUrl: string;
|
| 100 |
-
related: { asin: string; sku: string; issue: string; severity: string }[];
|
| 101 |
-
}
|
| 102 |
-
type CategoryLoad = Load<Category> | { phase: "refused"; message: string };
|
| 103 |
|
| 104 |
type Load<T> = { phase: "loading"; value?: T } | { phase: "error"; message: string; value?: T }
|
| 105 |
| { phase: "ready"; value: T };
|
| 106 |
-
type Picked = { kind: "metric" | "
|
| 107 |
type Tab = "dashboard" | "metrics" | "alerts" | "risks";
|
| 108 |
|
| 109 |
const EMPTY = "No account health data";
|
| 110 |
const ALERT_STATUSES = ["Open", "Investigating", "Resolved", "Dismissed"];
|
| 111 |
const PAGE = 50;
|
| 112 |
const HISTORY_PAGE = 50;
|
| 113 |
-
const CATEGORY_KEYS = new Set([
|
| 114 |
-
"listing_policy_violations", "product_authenticity_customer_complaints", "product_condition_customer_complaints",
|
| 115 |
-
"product_safety_customer_complaints", "food_and_product_safety_issues", "received_intellectual_property_complaints",
|
| 116 |
-
"suspected_intellectual_property_violations", "restricted_product_policy_violations",
|
| 117 |
-
"customer_product_reviews_policy_violations", "other_policy_violations", "document_requests",
|
| 118 |
-
]);
|
| 119 |
-
const categoryKey = (row: Metric) => row.key || row.id.split("@")[0];
|
| 120 |
-
const isCategory = (row: Metric) => CATEGORY_KEYS.has(categoryKey(row));
|
| 121 |
|
| 122 |
const text = (value: unknown) => (typeof value === "string" ? value : "");
|
| 123 |
const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null);
|
|
@@ -248,21 +233,6 @@ function parseHistory(raw: unknown): HistoryPage {
|
|
| 248 |
};
|
| 249 |
}
|
| 250 |
|
| 251 |
-
function parseCategory(raw: unknown, key: string): Category {
|
| 252 |
-
const body = record(raw);
|
| 253 |
-
if (text(body.key) !== key || body.recordList !== false) throw new Error("This category could not be read.");
|
| 254 |
-
const window = record(body.window);
|
| 255 |
-
return {
|
| 256 |
-
key, label: text(body.label), count: num(body.count), status: text(body.status) || "No data",
|
| 257 |
-
target: text(body.target), window: { from: text(window.from), to: text(window.to) },
|
| 258 |
-
sourceNote: text(body.sourceNote), sellerCentralUrl: text(body.sellerCentralUrl),
|
| 259 |
-
related: list(body.related).map((raw) => {
|
| 260 |
-
const row = record(raw);
|
| 261 |
-
return { asin: text(row.asin), sku: text(row.sku), issue: text(row.issue), severity: text(row.severity) };
|
| 262 |
-
}),
|
| 263 |
-
};
|
| 264 |
-
}
|
| 265 |
-
|
| 266 |
/** "Oct 22" from an ISO day. A calendar day is read in UTC. */
|
| 267 |
function day(iso: string) {
|
| 268 |
if (!iso) return "";
|
|
@@ -324,135 +294,6 @@ function Failed({ message, onRetry }: { message: string; onRetry: () => void })
|
|
| 324 |
<button type="button" className="ah-link" onClick={onRetry}>Try again</button></p>;
|
| 325 |
}
|
| 326 |
|
| 327 |
-
function CategoryPopup({ metric, appId, onClose }: { metric: Metric; appId: string; onClose: () => void }) {
|
| 328 |
-
const panel = useRef<HTMLDivElement>(null);
|
| 329 |
-
const titleId = useId();
|
| 330 |
-
const noteId = useId();
|
| 331 |
-
const [load, setLoad] = useState<CategoryLoad>({ phase: "loading" });
|
| 332 |
-
const [attempt, setAttempt] = useState(0);
|
| 333 |
-
useOverlayLayer({ panelRef: panel, onDismiss: onClose, dismissOnOutside: false,
|
| 334 |
-
initialFocus: "[data-category-close]", restoreFocus: true, trapFocus: true });
|
| 335 |
-
|
| 336 |
-
useEffect(() => {
|
| 337 |
-
const controller = new AbortController();
|
| 338 |
-
setLoad({ phase: "loading" });
|
| 339 |
-
const query = new URLSearchParams({ app: appId });
|
| 340 |
-
void fetch(`${API_V1}/account-health/categories/${encodeURIComponent(metric.id)}?${query.toString()}`,
|
| 341 |
-
{ credentials: CREDENTIALS, signal: controller.signal })
|
| 342 |
-
.then(async (response) => {
|
| 343 |
-
if (response.status === 401 || response.status === 403) {
|
| 344 |
-
const message = await readJson(response, "You do not have access to this category.")
|
| 345 |
-
.then(() => "You do not have access to this category.")
|
| 346 |
-
.catch((reason: unknown) => failure(reason, "You do not have access to this category."));
|
| 347 |
-
if (!controller.signal.aborted) setLoad({ phase: "refused", message });
|
| 348 |
-
return;
|
| 349 |
-
}
|
| 350 |
-
const category = parseCategory(await readJson(response, "This category could not be read."), categoryKey(metric));
|
| 351 |
-
if (!controller.signal.aborted) setLoad({ phase: "ready", value: category });
|
| 352 |
-
})
|
| 353 |
-
.catch((reason: unknown) => {
|
| 354 |
-
if (!controller.signal.aborted) setLoad({ phase: "error", message: failure(reason, "This category could not be read.") });
|
| 355 |
-
});
|
| 356 |
-
return () => controller.abort();
|
| 357 |
-
}, [appId, metric, attempt]);
|
| 358 |
-
|
| 359 |
-
const category = load.phase === "ready" ? load.value : null;
|
| 360 |
-
const relatedIncomplete = Boolean(category
|
| 361 |
-
&& /unavailable|need access|missing or unreadable|missing from the saved report/i.test(category.sourceNote));
|
| 362 |
-
const count = category ? amount(category.count, "count") : "";
|
| 363 |
-
const dataState = load.phase !== "ready" ? load.phase === "refused" ? "no-permission" : load.phase
|
| 364 |
-
: category?.count === null ? "na" : category?.count === 0 ? "empty" : "ready";
|
| 365 |
-
|
| 366 |
-
return <BodyPortal>
|
| 367 |
-
<div className="lp-dialog-scrim">
|
| 368 |
-
<div ref={panel} className="lp-dialog ah-category-popup" role="dialog" aria-modal="true"
|
| 369 |
-
aria-labelledby={titleId} data-category={metric.id} data-state={dataState} tabIndex={-1}>
|
| 370 |
-
<header className="lp-dialog-head ah-category-head">
|
| 371 |
-
<span className="ah-muted">Account health</span>
|
| 372 |
-
<button type="button" className="ah-button is-quiet" data-category-close onClick={onClose}>Close</button>
|
| 373 |
-
</header>
|
| 374 |
-
<div className="lp-dialog-body ah-category-body" aria-busy={load.phase === "loading"}>
|
| 375 |
-
<section className="ah-category-card" aria-labelledby={titleId} data-category-card>
|
| 376 |
-
<header className="ah-category-card-head">
|
| 377 |
-
<div className="ah-category-title">
|
| 378 |
-
<h2 id={titleId} className="lp-dialog-title">{category?.label || metric.label}</h2>
|
| 379 |
-
<svg className="ah-category-info" viewBox="0 0 16 16" role="img" aria-label="Source information">
|
| 380 |
-
<title>Amazon account health</title>
|
| 381 |
-
<circle cx="8" cy="8" r="6.1" fill="none" stroke="currentColor" strokeWidth="1.25" />
|
| 382 |
-
<path d="M8 7.4v3.5" fill="none" stroke="currentColor" strokeWidth="1.4" strokeLinecap="round" />
|
| 383 |
-
<circle cx="8" cy="5.1" r="0.85" fill="currentColor" />
|
| 384 |
-
</svg>
|
| 385 |
-
</div>
|
| 386 |
-
{category && <span className="ah-category-count" data-category-count aria-label={`Count: ${count}`}>{count}</span>}
|
| 387 |
-
</header>
|
| 388 |
-
<p className="ah-note ah-category-subtitle" data-category-record-note>Amazon sends no record list for this category</p>
|
| 389 |
-
{load.phase === "loading" && <Skeleton rows={5} />}
|
| 390 |
-
{load.phase === "error" && <Failed message={load.message} onRetry={() => {
|
| 391 |
-
panel.current?.querySelector<HTMLButtonElement>("[data-category-close]")?.focus();
|
| 392 |
-
setAttempt((n) => n + 1);
|
| 393 |
-
}} />}
|
| 394 |
-
{load.phase === "refused" && <p className="ah-note" role="status">{load.message}</p>}
|
| 395 |
-
{category && <>
|
| 396 |
-
<div className="ah-category-empty" role="status" data-category-record-list
|
| 397 |
-
data-record-state={category.count === 0 ? "empty" : "unavailable"}>
|
| 398 |
-
<svg className={`ah-category-empty-mark${category.count === 0 ? " is-good" : ""}`}
|
| 399 |
-
viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="1.4" aria-hidden="true">
|
| 400 |
-
<circle cx="8" cy="8" r="6.1" />
|
| 401 |
-
<path d={category.count === 0 ? "m5 8 2 2 4-4" : "M5.5 8h5"} strokeLinecap="round" strokeLinejoin="round" />
|
| 402 |
-
</svg>
|
| 403 |
-
<p className="ah-empty">{category.count === 0 ? "No violations reported" : "Record list unavailable"}</p>
|
| 404 |
-
</div>
|
| 405 |
-
<dl className="ah-category-facts">
|
| 406 |
-
<div className="ah-category-fact"><dt>Status</dt><dd><Chip status={category.status} /></dd></div>
|
| 407 |
-
<div className="ah-category-fact"><dt>Amazon's target</dt><dd>{category.target || "No target"}</dd></div>
|
| 408 |
-
</dl>
|
| 409 |
-
<footer className="ah-category-footer">
|
| 410 |
-
<div className="ah-category-window"><span>Reporting window</span>
|
| 411 |
-
<span>{category.window.from && category.window.to
|
| 412 |
-
? <><time dateTime={category.window.from}>{day(category.window.from)}</time>{" to "}
|
| 413 |
-
<time dateTime={category.window.to}>{day(category.window.to)}</time></>
|
| 414 |
-
: "No data"}</span>
|
| 415 |
-
</div>
|
| 416 |
-
{category.sellerCentralUrl && <a className="ah-link" data-category-seller-central
|
| 417 |
-
href={category.sellerCentralUrl} target="_blank" rel="noopener noreferrer">Open in Seller Central</a>}
|
| 418 |
-
</footer>
|
| 419 |
-
</>}
|
| 420 |
-
</section>
|
| 421 |
-
{category &&
|
| 422 |
-
<section className="ah-section ah-category-related" aria-label="Flagged listings" aria-describedby={noteId}
|
| 423 |
-
data-related-state={relatedIncomplete ? "incomplete" : category.related.length ? "ready" : "empty"}>
|
| 424 |
-
<div className="ah-category-related-head">
|
| 425 |
-
<h3 className="ah-heading">Flagged listings</h3><span className="ah-muted">Related</span>
|
| 426 |
-
</div>
|
| 427 |
-
<p id={noteId} className="ah-note" data-category-source-note>{category.sourceNote}</p>
|
| 428 |
-
{category.related.length ? <ul className="ah-category-list">
|
| 429 |
-
{category.related.map((listing, index) => {
|
| 430 |
-
const severity = listing.severity.toLowerCase().replace(/_/g, " ");
|
| 431 |
-
const severityTone = /error|critical|fatal/.test(severity) ? "is-risk"
|
| 432 |
-
: /warn/.test(severity) ? "is-near" : "is-quiet";
|
| 433 |
-
return <li key={index} className="ah-category-listing" data-related-listing>
|
| 434 |
-
<div className="ah-category-listing-head">
|
| 435 |
-
<div className="ah-category-identifiers">
|
| 436 |
-
{listing.sku && <span className="ah-category-listing-name">Sku: {listing.sku}</span>}
|
| 437 |
-
{listing.asin && <span>Asin: {listing.asin}</span>}
|
| 438 |
-
{!listing.sku && !listing.asin && <span className="ah-category-listing-name">Listing</span>}
|
| 439 |
-
</div>
|
| 440 |
-
{severity && <span className={`ah-chip ${severityTone}`}>{severity[0].toUpperCase() + severity.slice(1)}</span>}
|
| 441 |
-
</div>
|
| 442 |
-
<p className="ah-note">{listing.issue || "No issue description"}</p>
|
| 443 |
-
</li>;
|
| 444 |
-
})}
|
| 445 |
-
</ul> : <div className="ah-category-related-empty" role="status">
|
| 446 |
-
{relatedIncomplete ? "Related listings unavailable" : "No flagged listings"}
|
| 447 |
-
</div>}
|
| 448 |
-
</section>
|
| 449 |
-
}
|
| 450 |
-
</div>
|
| 451 |
-
</div>
|
| 452 |
-
</div>
|
| 453 |
-
</BodyPortal>;
|
| 454 |
-
}
|
| 455 |
-
|
| 456 |
// Busy buttons stay focusable so Enter keeps its place without a focus restore.
|
| 457 |
function Pager({ page, previous, next, busy, onPrevious, onNext }: {
|
| 458 |
page: number; previous: boolean; next: boolean; busy: boolean; onPrevious: () => void; onNext: () => void;
|
|
@@ -797,11 +638,7 @@ function MetricCell({ column, row, markets, picked, groups, onPick }: {
|
|
| 797 |
switch (column) {
|
| 798 |
case "label": return <td className="ah-c-label">
|
| 799 |
<button type="button" className="ah-open" data-metric={row.id} aria-pressed={picked === row.id}
|
| 800 |
-
|
| 801 |
-
onClick={(event) => {
|
| 802 |
-
if (isCategory(row)) event.currentTarget.focus({ preventScroll: true });
|
| 803 |
-
onPick(row);
|
| 804 |
-
}}>{row.label}</button>
|
| 805 |
{markets > 1 && row.marketplaceId && <span className="ah-sub">{row.marketplaceId}</span>}
|
| 806 |
</td>;
|
| 807 |
case "status": return <td className="ah-c-status"><Chip status={row.status} /></td>;
|
|
@@ -1054,7 +891,7 @@ function AccountHealthBody({ application, canEdit }: AccountHealthViewProps) {
|
|
| 1054 |
}, [url, tab, riskStatusFilter, riskOffset, query, risksAttempt]);
|
| 1055 |
|
| 1056 |
useEffect(() => {
|
| 1057 |
-
if (!picked || picked.kind === "risk"
|
| 1058 |
const controller = new AbortController();
|
| 1059 |
setDetailAccess({ owners: [], canEdit: false });
|
| 1060 |
if (picked.kind === "alert" && picked.linked) {
|
|
@@ -1101,7 +938,7 @@ function AccountHealthBody({ application, canEdit }: AccountHealthViewProps) {
|
|
| 1101 |
const pick = (next: Picked) => {
|
| 1102 |
setDetailMetric(next.metric ?? null);
|
| 1103 |
setDetailAlert(next.alert ?? null);
|
| 1104 |
-
setDetailLoading(next.kind !== "risk"
|
| 1105 |
setDetailError("");
|
| 1106 |
opener.current = document.activeElement instanceof HTMLElement ? document.activeElement : null;
|
| 1107 |
setPicked(next);
|
|
@@ -1132,9 +969,9 @@ function AccountHealthBody({ application, canEdit }: AccountHealthViewProps) {
|
|
| 1132 |
const alert = detailAlert;
|
| 1133 |
const alerting = (view?.metrics ?? []).filter((row) => !FOLDED.includes(row.status));
|
| 1134 |
const folded = (view?.metrics ?? []).filter((row) => FOLDED.includes(row.status));
|
| 1135 |
-
const metricPick = (row: Metric) => pick({ kind:
|
| 1136 |
|
| 1137 |
-
const open = Boolean(picked
|
| 1138 |
const busy = tab === "risks" ? risks.phase !== "ready"
|
| 1139 |
: tab === "metrics" ? summary.phase !== "ready" : alerts.phase !== "ready";
|
| 1140 |
const shownOffset = (tab === "risks" ? riskPage?.offset : tab === "metrics" ? view?.offset : page?.offset) ?? 0;
|
|
@@ -1181,13 +1018,13 @@ function AccountHealthBody({ application, canEdit }: AccountHealthViewProps) {
|
|
| 1181 |
&& <p className="ah-empty" role="status">No matching metrics</p>}
|
| 1182 |
{summary.phase === "ready" && view?.state === "ready" && alerting.length > 0
|
| 1183 |
&& <MetricTable metrics={alerting} markets={view.markets} groups={view.groups}
|
| 1184 |
-
picked={picked?.kind === "metric"
|
| 1185 |
{summary.phase === "ready" && view?.state === "ready" && view.folded > 0 && <button type="button"
|
| 1186 |
className="ah-fold" aria-expanded={foldOpen} onClick={() => setFoldOpen((open) => !open)}>
|
| 1187 |
{foldText(view.folded)}</button>}
|
| 1188 |
{summary.phase === "ready" && view?.state === "ready" && foldOpen && folded.length > 0
|
| 1189 |
&& <MetricTable metrics={folded} markets={view.markets} groups={view.groups}
|
| 1190 |
-
picked={picked?.kind === "metric"
|
| 1191 |
</div>}
|
| 1192 |
|
| 1193 |
{tab === "alerts" && <div className="ah-body" role="tabpanel" aria-label="Alerts">
|
|
@@ -1235,8 +1072,6 @@ function AccountHealthBody({ application, canEdit }: AccountHealthViewProps) {
|
|
| 1235 |
onPrevious={() => movePage(Math.max(0, shownOffset - shownLimit))}
|
| 1236 |
onNext={() => movePage(shownOffset + shownLimit)} />}
|
| 1237 |
</div>
|
| 1238 |
-
{picked?.kind === "category" && picked.metric && <CategoryPopup key={picked.id} metric={picked.metric}
|
| 1239 |
-
appId={appId} onClose={closePanel} />}
|
| 1240 |
{open && <Panel key={`${picked?.kind}:${picked?.id}`} metric={metric} alert={alert} risk={picked?.risk ?? null}
|
| 1241 |
editable={picked?.kind === "risk" ? riskEditable : picked?.kind === "alert" && picked.linked
|
| 1242 |
? canEdit === true && detailAccess.canEdit && !detailLoading && !detailError : editable}
|
|
|
|
| 1 |
+
import { useCallback, useEffect, useRef, useState } from "react";
|
| 2 |
import { API_V1, CREDENTIALS } from "../../apiContract";
|
|
|
|
| 3 |
import { useEscapeClose } from "../../ui/useEscapeClose";
|
| 4 |
import { AppTabs } from "../AppTabs";
|
| 5 |
import { LineChart } from "../../viz/kit";
|
|
|
|
| 7 |
import "../../styles/sdlc79/account-health.css";
|
| 8 |
import "../../styles/sdlc81/account-health-risk.css";
|
| 9 |
|
| 10 |
+
// The dispatcher owns the App frame. This view owns its lists and right-side detail panel.
|
| 11 |
|
| 12 |
/** C2: the App as the dispatcher passes it. Structurally equal to lane A's AppViewProps application. */
|
| 13 |
interface AccountHealthApplication {
|
|
|
|
| 93 |
|
| 94 |
interface Point { day: string; value: number | null; target: number | null; status: string }
|
| 95 |
interface HistoryPage { points: Point[]; cursor: string | null; label: string; unit: string }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
|
| 97 |
type Load<T> = { phase: "loading"; value?: T } | { phase: "error"; message: string; value?: T }
|
| 98 |
| { phase: "ready"; value: T };
|
| 99 |
+
type Picked = { kind: "metric" | "alert" | "risk"; id: string; metric?: Metric; alert?: Alert; risk?: Risk; linked?: boolean };
|
| 100 |
type Tab = "dashboard" | "metrics" | "alerts" | "risks";
|
| 101 |
|
| 102 |
const EMPTY = "No account health data";
|
| 103 |
const ALERT_STATUSES = ["Open", "Investigating", "Resolved", "Dismissed"];
|
| 104 |
const PAGE = 50;
|
| 105 |
const HISTORY_PAGE = 50;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
|
| 107 |
const text = (value: unknown) => (typeof value === "string" ? value : "");
|
| 108 |
const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null);
|
|
|
|
| 233 |
};
|
| 234 |
}
|
| 235 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
/** "Oct 22" from an ISO day. A calendar day is read in UTC. */
|
| 237 |
function day(iso: string) {
|
| 238 |
if (!iso) return "";
|
|
|
|
| 294 |
<button type="button" className="ah-link" onClick={onRetry}>Try again</button></p>;
|
| 295 |
}
|
| 296 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 297 |
// Busy buttons stay focusable so Enter keeps its place without a focus restore.
|
| 298 |
function Pager({ page, previous, next, busy, onPrevious, onNext }: {
|
| 299 |
page: number; previous: boolean; next: boolean; busy: boolean; onPrevious: () => void; onNext: () => void;
|
|
|
|
| 638 |
switch (column) {
|
| 639 |
case "label": return <td className="ah-c-label">
|
| 640 |
<button type="button" className="ah-open" data-metric={row.id} aria-pressed={picked === row.id}
|
| 641 |
+
onClick={() => onPick(row)}>{row.label}</button>
|
|
|
|
|
|
|
|
|
|
|
|
|
| 642 |
{markets > 1 && row.marketplaceId && <span className="ah-sub">{row.marketplaceId}</span>}
|
| 643 |
</td>;
|
| 644 |
case "status": return <td className="ah-c-status"><Chip status={row.status} /></td>;
|
|
|
|
| 891 |
}, [url, tab, riskStatusFilter, riskOffset, query, risksAttempt]);
|
| 892 |
|
| 893 |
useEffect(() => {
|
| 894 |
+
if (!picked || picked.kind === "risk") return;
|
| 895 |
const controller = new AbortController();
|
| 896 |
setDetailAccess({ owners: [], canEdit: false });
|
| 897 |
if (picked.kind === "alert" && picked.linked) {
|
|
|
|
| 938 |
const pick = (next: Picked) => {
|
| 939 |
setDetailMetric(next.metric ?? null);
|
| 940 |
setDetailAlert(next.alert ?? null);
|
| 941 |
+
setDetailLoading(next.kind !== "risk");
|
| 942 |
setDetailError("");
|
| 943 |
opener.current = document.activeElement instanceof HTMLElement ? document.activeElement : null;
|
| 944 |
setPicked(next);
|
|
|
|
| 969 |
const alert = detailAlert;
|
| 970 |
const alerting = (view?.metrics ?? []).filter((row) => !FOLDED.includes(row.status));
|
| 971 |
const folded = (view?.metrics ?? []).filter((row) => FOLDED.includes(row.status));
|
| 972 |
+
const metricPick = (row: Metric) => pick({ kind: "metric", id: row.id, metric: row });
|
| 973 |
|
| 974 |
+
const open = Boolean(picked);
|
| 975 |
const busy = tab === "risks" ? risks.phase !== "ready"
|
| 976 |
: tab === "metrics" ? summary.phase !== "ready" : alerts.phase !== "ready";
|
| 977 |
const shownOffset = (tab === "risks" ? riskPage?.offset : tab === "metrics" ? view?.offset : page?.offset) ?? 0;
|
|
|
|
| 1018 |
&& <p className="ah-empty" role="status">No matching metrics</p>}
|
| 1019 |
{summary.phase === "ready" && view?.state === "ready" && alerting.length > 0
|
| 1020 |
&& <MetricTable metrics={alerting} markets={view.markets} groups={view.groups}
|
| 1021 |
+
picked={picked?.kind === "metric" ? picked.id : ""} onPick={metricPick} />}
|
| 1022 |
{summary.phase === "ready" && view?.state === "ready" && view.folded > 0 && <button type="button"
|
| 1023 |
className="ah-fold" aria-expanded={foldOpen} onClick={() => setFoldOpen((open) => !open)}>
|
| 1024 |
{foldText(view.folded)}</button>}
|
| 1025 |
{summary.phase === "ready" && view?.state === "ready" && foldOpen && folded.length > 0
|
| 1026 |
&& <MetricTable metrics={folded} markets={view.markets} groups={view.groups}
|
| 1027 |
+
picked={picked?.kind === "metric" ? picked.id : ""} onPick={metricPick} />}
|
| 1028 |
</div>}
|
| 1029 |
|
| 1030 |
{tab === "alerts" && <div className="ah-body" role="tabpanel" aria-label="Alerts">
|
|
|
|
| 1072 |
onPrevious={() => movePage(Math.max(0, shownOffset - shownLimit))}
|
| 1073 |
onNext={() => movePage(shownOffset + shownLimit)} />}
|
| 1074 |
</div>
|
|
|
|
|
|
|
| 1075 |
{open && <Panel key={`${picked?.kind}:${picked?.id}`} metric={metric} alert={alert} risk={picked?.risk ?? null}
|
| 1076 |
editable={picked?.kind === "risk" ? riskEditable : picked?.kind === "alert" && picked.linked
|
| 1077 |
? canEdit === true && detailAccess.canEdit && !detailLoading && !detailError : editable}
|
web/src/apps/amazon-profit/AppView.tsx
CHANGED
|
@@ -4,7 +4,6 @@ import { ACCESS_REFUSED_MESSAGE, clearTopicRowsCache, dropTableRowsCache } from
|
|
| 4 |
import { AMAZON_DATA_ENTRY_TABLE, AMAZON_PROFIT_TABLE, AmazonPeriodSelect, COMMAND_CENTER_TABLES,
|
| 5 |
CommandCenterTabStrip, CommandCenterTableContext } from "../../customer-grid/AmazonPeriodPicker";
|
| 6 |
import { OverlayProvider } from "../../customer-grid/OverlaySurface";
|
| 7 |
-
import { LP_BLUE_DEEP, LP_GREEN_DEEP } from "../../customer-grid/theme";
|
| 8 |
import type { AppViewProps } from "../../customer-grid/viewModes";
|
| 9 |
import { LogoLoader } from "../../ui/LogoLoader";
|
| 10 |
import FbaCapacityControl from "./FbaCapacityControl";
|
|
@@ -17,12 +16,8 @@ import "../../styles/sdlc81/command-center-app.css";
|
|
| 17 |
// Lazy, as every other host loads it: a static import pulls the grid into this App's chunk.
|
| 18 |
const CustomerGrid = lazy(() => import("../../customer-grid/CustomerGrid"));
|
| 19 |
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
/** Admission belongs to the selected tab's fresh probe. */
|
| 24 |
-
type TableProbe = { table: string; status: "ready"; tables: Record<string, boolean> }
|
| 25 |
-
| { table: string; status: "failed" } | null;
|
| 26 |
|
| 27 |
// The Command Center App body: the Table module's own Grid, hosted under the App frame.
|
| 28 |
// The tab strip switches `table` through the context, so the URL stays on the App.
|
|
@@ -33,17 +28,6 @@ export default function ProjectManagementView(_props: AppViewProps) {
|
|
| 33 |
const [projects, setProjects] = useState<ProjectsTarget | null>(null);
|
| 34 |
const [competitors, setCompetitors] = useState<ProjectsTarget | null>(null);
|
| 35 |
const [gridRevision, setGridRevision] = useState(0);
|
| 36 |
-
const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null);
|
| 37 |
-
const openTable = useCallback((next: string) => {
|
| 38 |
-
if (next === table) return;
|
| 39 |
-
// Clear synchronously: returning to a tab must not reuse its earlier admission.
|
| 40 |
-
setProbe(null);
|
| 41 |
-
setEntryEmpty(null);
|
| 42 |
-
setPicture(null);
|
| 43 |
-
setProjects(null);
|
| 44 |
-
setCompetitors(null);
|
| 45 |
-
setTable(next);
|
| 46 |
-
}, [table]);
|
| 47 |
const reloadGrid = useCallback(() => {
|
| 48 |
for (const key of [AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]) {
|
| 49 |
clearTopicRowsCache(`tables/${key}/rows`);
|
|
@@ -56,27 +40,23 @@ export default function ProjectManagementView(_props: AppViewProps) {
|
|
| 56 |
useEffect(() => {
|
| 57 |
const controller = new AbortController();
|
| 58 |
void fetch(`${API_V1}/amazon-profit/tables`, { credentials: CREDENTIALS, cache: "no-store",
|
| 59 |
-
signal:
|
| 60 |
.then(async (res) => {
|
| 61 |
if (!res.ok) throw new Error("probe refused");
|
| 62 |
const body = await res.json() as { tables?: Record<string, boolean> };
|
| 63 |
-
|
| 64 |
-
|| Object.values(body.tables).some(value => typeof value !== "boolean")) throw new Error("invalid probe");
|
| 65 |
-
if (!controller.signal.aborted) setProbe({ table, status: "ready", tables: body.tables });
|
| 66 |
})
|
| 67 |
-
.catch(() => { if (!controller.signal.aborted) setProbe(
|
| 68 |
return () => controller.abort();
|
| 69 |
}, [table]);
|
| 70 |
-
const
|
| 71 |
-
const
|
| 72 |
-
const probeFailed = currentProbe?.status === "failed";
|
| 73 |
-
const notBuilt = currentProbe?.status === "ready" && currentProbe.tables[table] === false;
|
| 74 |
-
const inaccessible = currentProbe?.status === "ready" && !Object.hasOwn(currentProbe.tables, table);
|
| 75 |
-
const admitted = currentProbe?.status === "ready" && currentProbe.tables[table] === true;
|
| 76 |
const dataEntry = table === AMAZON_DATA_ENTRY_TABLE;
|
| 77 |
// Only an empty, permitted row response proves every SKU has its data.
|
|
|
|
| 78 |
useEffect(() => {
|
| 79 |
-
if (!dataEntry
|
|
|
|
| 80 |
const controller = new AbortController();
|
| 81 |
void fetch(`${API_V1}/tables/${AMAZON_DATA_ENTRY_TABLE}/rows`, { credentials: CREDENTIALS,
|
| 82 |
cache: "no-store", signal: AbortSignal.any([controller.signal, AbortSignal.timeout(20000)]) })
|
|
@@ -87,32 +67,27 @@ export default function ProjectManagementView(_props: AppViewProps) {
|
|
| 87 |
})
|
| 88 |
.catch(() => { if (!controller.signal.aborted) setEntryEmpty({ table, empty: false }); });
|
| 89 |
return () => controller.abort();
|
| 90 |
-
}, [dataEntry,
|
| 91 |
const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table;
|
| 92 |
-
const allComplete = dataEntry &&
|
| 93 |
// A new key remounts the grid per table, as the Table module does on a database switch.
|
| 94 |
-
return <CommandCenterTableContext.Provider value={
|
| 95 |
<OverlayProvider>
|
| 96 |
-
{dataEntry &&
|
| 97 |
-
{
|
| 98 |
-
|
| 99 |
-
|
|
|
|
|
|
|
| 100 |
style={{ display: "flex", flexDirection: "column", flex: "1 1 0", minHeight: 0, padding: "4px 12px 0" }}>
|
| 101 |
-
<CommandCenterTabStrip database={table} onOpenTable={
|
| 102 |
-
|
| 103 |
-
:
|
| 104 |
-
: notBuilt ? "This table is not built yet. An admin Refresh builds it." : "Every SKU has its data."}</p>}
|
| 105 |
-
</div>
|
| 106 |
-
: table === "ut_amazon_projects" ? <div style={{ display: "flex", flexDirection: "column", flex: "1 1 0", minHeight: 0, padding: "4px 12px 0" }}>
|
| 107 |
-
<CommandCenterTabStrip database={table} onOpenTable={openTable} />
|
| 108 |
-
<ProjectsPage />
|
| 109 |
</div>
|
| 110 |
: !entryChecked ? <LogoLoader data-app-reason="loading" /> : <Suspense fallback={<LogoLoader data-app-reason="loading" />}>
|
| 111 |
{/* C3 App-only props (S82-T68). Projects keeps New record. */}
|
| 112 |
<CustomerGrid key={`${table}:${gridRevision}`} address={{ database: table }} hideViewRail
|
| 113 |
hideModeSwitch
|
| 114 |
-
appHistoryInToolbar
|
| 115 |
-
appProgressColors={PROJECT_PROGRESS_TABLES.has(table) ? PROJECT_PROGRESS_COLORS : undefined}
|
| 116 |
appCellActions={{
|
| 117 |
projects: row => setProjects(projectsTargetFromRow(table, row)),
|
| 118 |
competitors: row => setCompetitors(competitorsTargetFromRow(table, row)),
|
|
|
|
| 4 |
import { AMAZON_DATA_ENTRY_TABLE, AMAZON_PROFIT_TABLE, AmazonPeriodSelect, COMMAND_CENTER_TABLES,
|
| 5 |
CommandCenterTabStrip, CommandCenterTableContext } from "../../customer-grid/AmazonPeriodPicker";
|
| 6 |
import { OverlayProvider } from "../../customer-grid/OverlaySurface";
|
|
|
|
| 7 |
import type { AppViewProps } from "../../customer-grid/viewModes";
|
| 8 |
import { LogoLoader } from "../../ui/LogoLoader";
|
| 9 |
import FbaCapacityControl from "./FbaCapacityControl";
|
|
|
|
| 16 |
// Lazy, as every other host loads it: a static import pulls the grid into this App's chunk.
|
| 17 |
const CustomerGrid = lazy(() => import("../../customer-grid/CustomerGrid"));
|
| 18 |
|
| 19 |
+
/** Which Command Center tables exist, keyed by table; "failed" mounts the grid as before. */
|
| 20 |
+
type TableProbe = Record<string, boolean> | "failed" | null;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
// The Command Center App body: the Table module's own Grid, hosted under the App frame.
|
| 23 |
// The tab strip switches `table` through the context, so the URL stays on the App.
|
|
|
|
| 28 |
const [projects, setProjects] = useState<ProjectsTarget | null>(null);
|
| 29 |
const [competitors, setCompetitors] = useState<ProjectsTarget | null>(null);
|
| 30 |
const [gridRevision, setGridRevision] = useState(0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
const reloadGrid = useCallback(() => {
|
| 32 |
for (const key of [AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]) {
|
| 33 |
clearTopicRowsCache(`tables/${key}/rows`);
|
|
|
|
| 40 |
useEffect(() => {
|
| 41 |
const controller = new AbortController();
|
| 42 |
void fetch(`${API_V1}/amazon-profit/tables`, { credentials: CREDENTIALS, cache: "no-store",
|
| 43 |
+
signal: controller.signal })
|
| 44 |
.then(async (res) => {
|
| 45 |
if (!res.ok) throw new Error("probe refused");
|
| 46 |
const body = await res.json() as { tables?: Record<string, boolean> };
|
| 47 |
+
setProbe(body.tables ?? {});
|
|
|
|
|
|
|
| 48 |
})
|
| 49 |
+
.catch(() => { if (!controller.signal.aborted) setProbe((last) => last ?? "failed"); });
|
| 50 |
return () => controller.abort();
|
| 51 |
}, [table]);
|
| 52 |
+
const notBuilt = probe !== null && probe !== "failed" && probe[table] === false;
|
| 53 |
+
const inaccessible = probe !== null && probe !== "failed" && !Object.hasOwn(probe, table);
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
const dataEntry = table === AMAZON_DATA_ENTRY_TABLE;
|
| 55 |
// Only an empty, permitted row response proves every SKU has its data.
|
| 56 |
+
const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null);
|
| 57 |
useEffect(() => {
|
| 58 |
+
if (!dataEntry) { setEntryEmpty(null); return; }
|
| 59 |
+
if (probe === null || notBuilt || inaccessible) return;
|
| 60 |
const controller = new AbortController();
|
| 61 |
void fetch(`${API_V1}/tables/${AMAZON_DATA_ENTRY_TABLE}/rows`, { credentials: CREDENTIALS,
|
| 62 |
cache: "no-store", signal: AbortSignal.any([controller.signal, AbortSignal.timeout(20000)]) })
|
|
|
|
| 67 |
})
|
| 68 |
.catch(() => { if (!controller.signal.aborted) setEntryEmpty({ table, empty: false }); });
|
| 69 |
return () => controller.abort();
|
| 70 |
+
}, [dataEntry, probe, notBuilt, inaccessible, table]);
|
| 71 |
const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table;
|
| 72 |
+
const allComplete = dataEntry && !notBuilt && entryEmpty?.table === table && entryEmpty.empty;
|
| 73 |
// A new key remounts the grid per table, as the Table module does on a database switch.
|
| 74 |
+
return <CommandCenterTableContext.Provider value={setTable}>
|
| 75 |
<OverlayProvider>
|
| 76 |
+
{dataEntry && probe !== null && !inaccessible && <FbaCapacityControl />}
|
| 77 |
+
{table === "ut_amazon_projects" ? <div style={{ display: "flex", flexDirection: "column", flex: "1 1 0", minHeight: 0, padding: "4px 12px 0" }}>
|
| 78 |
+
<CommandCenterTabStrip database={table} onOpenTable={setTable} />
|
| 79 |
+
<ProjectsPage />
|
| 80 |
+
</div> : probe === null ? <LogoLoader data-app-reason="loading" /> : inaccessible || notBuilt || allComplete
|
| 81 |
+
? <div className="cc-not-built" data-testid={inaccessible ? "command-center-refused" : notBuilt ? "command-center-not-built" : "command-center-complete"}
|
| 82 |
style={{ display: "flex", flexDirection: "column", flex: "1 1 0", minHeight: 0, padding: "4px 12px 0" }}>
|
| 83 |
+
<CommandCenterTabStrip database={table} onOpenTable={setTable} />
|
| 84 |
+
<p className="ap-state" role="status">{inaccessible ? ACCESS_REFUSED_MESSAGE : notBuilt ? "This table is not built yet. An admin Refresh builds it."
|
| 85 |
+
: "Every SKU has its data."}</p>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
</div>
|
| 87 |
: !entryChecked ? <LogoLoader data-app-reason="loading" /> : <Suspense fallback={<LogoLoader data-app-reason="loading" />}>
|
| 88 |
{/* C3 App-only props (S82-T68). Projects keeps New record. */}
|
| 89 |
<CustomerGrid key={`${table}:${gridRevision}`} address={{ database: table }} hideViewRail
|
| 90 |
hideModeSwitch
|
|
|
|
|
|
|
| 91 |
appCellActions={{
|
| 92 |
projects: row => setProjects(projectsTargetFromRow(table, row)),
|
| 93 |
competitors: row => setCompetitors(competitorsTargetFromRow(table, row)),
|
web/src/apps/amazon-profit/ProjectResults.tsx
CHANGED
|
@@ -1,8 +1,5 @@
|
|
| 1 |
import { useEffect, useRef, useState } from "react";
|
| 2 |
-
import { API_V1, CREDENTIALS
|
| 3 |
-
import { refusalMessage } from "../../customer-grid/apiBridge";
|
| 4 |
-
import { LineChart, formatMeasure } from "../../viz/kit";
|
| 5 |
-
import "../../styles/sdlc84/project-results.css";
|
| 6 |
|
| 7 |
export type ResultMetric = "sales" | "units" | "returns" | "return_rate" | "refund_rate" | "roas";
|
| 8 |
export interface ResultConfig {
|
|
@@ -26,11 +23,73 @@ interface Result {
|
|
| 26 |
deltaPercent: number | null;
|
| 27 |
}
|
| 28 |
interface Answer { timezone: string; anchor: string | null; results: Result[] }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
const periodLabel = (period: WindowResult) => {
|
| 30 |
const last = new Date(`${period.end_exclusive}T00:00:00Z`);
|
| 31 |
last.setUTCDate(last.getUTCDate() - 1);
|
| 32 |
return `${period.start} to ${last.toISOString().slice(0, 10)}`;
|
| 33 |
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
export function ProjectResultPreview({ projectId, configs, revision, completedOn, onOpen }: {
|
| 35 |
projectId: string; configs: ResultConfig[]; revision: string; completedOn: string | null; onOpen: () => void;
|
| 36 |
}) {
|
|
@@ -80,135 +139,91 @@ export function ProjectResultPreview({ projectId, configs, revision, completedOn
|
|
| 80 |
</div>;
|
| 81 |
}
|
| 82 |
|
| 83 |
-
|
| 84 |
-
{ key: "sales", label: "Sales" },
|
| 85 |
-
{ key: "units", label: "Units" },
|
| 86 |
-
{ key: "margin", label: "Margin" },
|
| 87 |
-
{ key: "returns", label: "Returns" },
|
| 88 |
-
{ key: "returns_pct", label: "Returns %" },
|
| 89 |
-
{ key: "refunds_pct", label: "Refunds %" },
|
| 90 |
-
{ key: "roas", label: "ROAS" },
|
| 91 |
-
{ key: "sales_yoy", label: "Sales YoY" },
|
| 92 |
-
] as const;
|
| 93 |
-
type ChartKey = typeof CHARTS[number]["key"];
|
| 94 |
-
interface SeriesMetric {
|
| 95 |
-
key: ChartKey;
|
| 96 |
-
label: string;
|
| 97 |
-
unit: string;
|
| 98 |
-
points: Point[];
|
| 99 |
-
total: number | null;
|
| 100 |
-
tie: { ok: boolean; ours: number | null; independent: number | null };
|
| 101 |
-
reason: string | null;
|
| 102 |
-
}
|
| 103 |
-
interface SeriesAnswer {
|
| 104 |
-
projectId: string;
|
| 105 |
-
from: string;
|
| 106 |
-
to: string;
|
| 107 |
-
markers: { start: string | null; target: string | null; completed: string | null };
|
| 108 |
-
metrics: SeriesMetric[];
|
| 109 |
-
}
|
| 110 |
-
interface ChartVisibility { projectId: string; storageKey: string; hidden: ChartKey[] }
|
| 111 |
-
|
| 112 |
-
function readHiddenCharts(storageKey: string): ChartKey[] {
|
| 113 |
-
try {
|
| 114 |
-
const value: unknown = JSON.parse(localStorage.getItem(storageKey) ?? "[]");
|
| 115 |
-
return Array.isArray(value) ? CHARTS.filter(chart => value.includes(chart.key)).map(chart => chart.key) : [];
|
| 116 |
-
} catch { return []; }
|
| 117 |
-
}
|
| 118 |
-
|
| 119 |
-
export default function ProjectResults({ projectId, configs, completedOn, updatedAt, scope }: {
|
| 120 |
projectId: string; configs: ResultConfig[]; completedOn: string | null; updatedAt?: string;
|
| 121 |
canEdit: boolean; onSaved: () => void;
|
| 122 |
-
scope?: { grain: "sku" | "asin" | "parent"; keys: string[] };
|
| 123 |
}) {
|
| 124 |
-
const [
|
| 125 |
-
const [
|
| 126 |
-
const [
|
| 127 |
-
const [
|
| 128 |
-
const
|
| 129 |
-
const
|
| 130 |
-
const
|
|
|
|
|
|
|
|
|
|
| 131 |
useEffect(() => {
|
| 132 |
const controller = new AbortController();
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
void fetch(`${API_V1}/amazon-projects/${encodeURIComponent(projectId)}/
|
| 136 |
signal: AbortSignal.any([controller.signal, AbortSignal.timeout(20000)]) })
|
| 137 |
.then(async response => {
|
| 138 |
-
if (
|
| 139 |
-
const
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
setVisibility(previous => previous?.storageKey === storageKey ? previous
|
| 143 |
-
: { projectId, storageKey, hidden: readHiddenCharts(storageKey) });
|
| 144 |
-
const body = await response.json().catch(() => null);
|
| 145 |
-
if (!response.ok) throw new Error(refusalMessage(body, response.status, "Results could not be loaded"));
|
| 146 |
-
if (!body || body.projectId !== projectId || !Array.isArray(body.metrics)) throw new Error("Results could not be loaded");
|
| 147 |
-
if (!controller.signal.aborted) setLoaded({ request, answer: body as SeriesAnswer, error: "" });
|
| 148 |
-
}).catch(error => {
|
| 149 |
-
if (!controller.signal.aborted) setLoaded({ request, answer: null,
|
| 150 |
-
error: error instanceof Error ? error.message : "Results could not be loaded" });
|
| 151 |
-
});
|
| 152 |
return () => controller.abort();
|
| 153 |
-
}, [projectId,
|
| 154 |
-
const
|
| 155 |
-
const
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
const
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
setVisibility({ ...visibility, hidden: next });
|
| 163 |
try {
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
});
|
|
|
|
| 171 |
};
|
| 172 |
-
return <section
|
| 173 |
-
<
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
</div>
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
return <article key={chart.key} className="lp-proj-series-card" data-metric={chart.key} aria-label={`${label} chart`}>
|
| 191 |
-
<header className="lp-proj-series-head">
|
| 192 |
-
<div className="lp-proj-series-summary">
|
| 193 |
-
<h5>{label}</h5>
|
| 194 |
-
{loading ? <span className="lp-proj-series-value-skeleton" aria-hidden="true" />
|
| 195 |
-
: <span className="lp-proj-series-total">{available
|
| 196 |
-
? `${formatMeasure(metric.total, metric.unit)}${metric.unit === "ratio" ? "×" : ""}` : "Unavailable"}</span>}
|
| 197 |
-
{period ? <span className="lp-proj-series-period">{period}</span> : null}
|
| 198 |
-
</div>
|
| 199 |
-
<button type="button" className="lp-proj-series-action" data-chart-visibility aria-label={`Hide ${label} chart`}
|
| 200 |
-
disabled={visibility?.projectId !== projectId} onClick={() => changeHidden([...hidden, chart.key], index + (hidden.length > 0 ? 1 : 0))}>Hide</button>
|
| 201 |
-
</header>
|
| 202 |
-
{loading ? <div className="lp-proj-series-skeleton" role="status" aria-label={`Loading ${label} chart`} />
|
| 203 |
-
: <>
|
| 204 |
-
{available ? <LineChart label={label} unit={metric.unit}
|
| 205 |
-
series={[{ key: chart.key, label, points: metric.points }]} /> : null}
|
| 206 |
-
{reason || !available ? <p className="lp-proj-series-notice" role="status">
|
| 207 |
-
{reason || "This metric is unavailable"}
|
| 208 |
-
</p> : null}
|
| 209 |
-
</>}
|
| 210 |
-
</article>;
|
| 211 |
-
})}
|
| 212 |
</div> : null}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 213 |
</section>;
|
| 214 |
}
|
|
|
|
| 1 |
import { useEffect, useRef, useState } from "react";
|
| 2 |
+
import { API_V1, CREDENTIALS } from "../../apiContract";
|
|
|
|
|
|
|
|
|
|
| 3 |
|
| 4 |
export type ResultMetric = "sales" | "units" | "returns" | "return_rate" | "refund_rate" | "roas";
|
| 5 |
export interface ResultConfig {
|
|
|
|
| 23 |
deltaPercent: number | null;
|
| 24 |
}
|
| 25 |
interface Answer { timezone: string; anchor: string | null; results: Result[] }
|
| 26 |
+
const METRICS: Array<{ key: string; metric: ResultMetric; label: string; compare: "prior" | "yoy" }> = [
|
| 27 |
+
{ key: "sales", metric: "sales", label: "Sales", compare: "prior" },
|
| 28 |
+
{ key: "sales-yoy", metric: "sales", label: "Sales YoY", compare: "yoy" },
|
| 29 |
+
{ key: "return_rate", metric: "return_rate", label: "Returns %", compare: "prior" },
|
| 30 |
+
{ key: "units", metric: "units", label: "Units", compare: "prior" },
|
| 31 |
+
{ key: "returns", metric: "returns", label: "Returns", compare: "prior" },
|
| 32 |
+
{ key: "refund_rate", metric: "refund_rate", label: "Refunds %", compare: "prior" },
|
| 33 |
+
{ key: "roas", metric: "roas", label: "ROAS", compare: "prior" },
|
| 34 |
+
];
|
| 35 |
+
const configKey = (config: ResultConfig) => config.metric === "sales" && config.compare === "yoy" ? "sales-yoy" : config.metric;
|
| 36 |
const periodLabel = (period: WindowResult) => {
|
| 37 |
const last = new Date(`${period.end_exclusive}T00:00:00Z`);
|
| 38 |
last.setUTCDate(last.getUTCDate() - 1);
|
| 39 |
return `${period.start} to ${last.toISOString().slice(0, 10)}`;
|
| 40 |
};
|
| 41 |
+
function figure(value: number | null, unit: string) {
|
| 42 |
+
if (value === null || !Number.isFinite(value)) return "Unavailable";
|
| 43 |
+
if (/^[A-Z]{3}$/.test(unit)) return value.toLocaleString("en-US", { style: "currency", currency: unit, maximumFractionDigits: 2 });
|
| 44 |
+
return `${value.toLocaleString("en-US", { maximumFractionDigits: 2 })}${unit === "percent" ? "%" : unit === "ratio" ? "×" : ""}`;
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
function Trend({ result }: { result: Result }) {
|
| 48 |
+
const current = result.current, comparison = result.comparison;
|
| 49 |
+
if (!current || !comparison || result.state !== "ready") return <p className="lp-proj-muted" role="status">
|
| 50 |
+
{result.state === "waiting_for_completion" ? "Complete the project to view results"
|
| 51 |
+
: result.state === "waiting_for_window" || result.state === "waiting_for_completed_day"
|
| 52 |
+
? "Results appear when this period ends" : "Results unavailable"}
|
| 53 |
+
</p>;
|
| 54 |
+
const values = [...current.series, ...comparison.series].map(point => point.value)
|
| 55 |
+
.filter((value): value is number => value !== null && Number.isFinite(value));
|
| 56 |
+
if (!values.length) return <p className="lp-proj-muted" role="status">Results unavailable</p>;
|
| 57 |
+
const low = Math.min(0, ...values), high = Math.max(1, ...values);
|
| 58 |
+
const count = Math.max(current.series.length, comparison.series.length, 2);
|
| 59 |
+
const x = (index: number) => 70 + index * 700 / (count - 1);
|
| 60 |
+
const y = (value: number) => 180 - (value - low) * 150 / (high - low);
|
| 61 |
+
const path = (points: Point[]) => {
|
| 62 |
+
let joined = false;
|
| 63 |
+
return points.map((point, index) => {
|
| 64 |
+
if (point.value === null || !Number.isFinite(point.value)) { joined = false; return ""; }
|
| 65 |
+
const command = joined ? "L" : "M";
|
| 66 |
+
joined = true;
|
| 67 |
+
return `${command}${x(index)},${y(point.value)}`;
|
| 68 |
+
}).join(" ");
|
| 69 |
+
};
|
| 70 |
+
return <>
|
| 71 |
+
<dl className="lp-proj-result-values">
|
| 72 |
+
<div><dt>After completion</dt><dd>{figure(current.value, result.unit)}</dd></div>
|
| 73 |
+
<div><dt>{result.config.compare === "yoy" ? "Prior year" : "Before completion"}</dt><dd>{figure(comparison.value, result.unit)}</dd></div>
|
| 74 |
+
<div><dt>Change</dt><dd>{figure(result.delta, result.unit)}</dd></div>
|
| 75 |
+
</dl>
|
| 76 |
+
<svg className="lp-proj-trend" viewBox="0 0 800 220" role="img" aria-label={`${result.label} trend`}>
|
| 77 |
+
<title>{result.label}: {periodLabel(current)} compared with {periodLabel(comparison)}</title>
|
| 78 |
+
{[low, (high + low) / 2, high].map((value, index) => <g key={index}>
|
| 79 |
+
<line x1="70" y1={y(value)} x2="770" y2={y(value)} className="lp-proj-trend-grid" />
|
| 80 |
+
<text x="60" y={y(value) + 4} textAnchor="end">{figure(value, result.unit)}</text>
|
| 81 |
+
</g>)}
|
| 82 |
+
<path d={path(comparison.series)} className="lp-proj-trend-prior" />
|
| 83 |
+
<path d={path(current.series)} className="lp-proj-trend-current" />
|
| 84 |
+
<text x="70" y="207">Day 1</text><text x="770" y="207" textAnchor="end">Day {count}</text>
|
| 85 |
+
</svg>
|
| 86 |
+
<div className="lp-proj-trend-legend">
|
| 87 |
+
<span><i className="lp-proj-trend-current" />After completion · {periodLabel(current)}</span>
|
| 88 |
+
<span><i className="lp-proj-trend-prior" />{result.config.compare === "yoy" ? "Prior year" : "Before completion"} · {periodLabel(comparison)}</span>
|
| 89 |
+
</div>
|
| 90 |
+
</>;
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
export function ProjectResultPreview({ projectId, configs, revision, completedOn, onOpen }: {
|
| 94 |
projectId: string; configs: ResultConfig[]; revision: string; completedOn: string | null; onOpen: () => void;
|
| 95 |
}) {
|
|
|
|
| 139 |
</div>;
|
| 140 |
}
|
| 141 |
|
| 142 |
+
export default function ProjectResults({ projectId, configs, completedOn, updatedAt, canEdit, onSaved }: {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
projectId: string; configs: ResultConfig[]; completedOn: string | null; updatedAt?: string;
|
| 144 |
canEdit: boolean; onSaved: () => void;
|
|
|
|
| 145 |
}) {
|
| 146 |
+
const [draft, setDraft] = useState(configs);
|
| 147 |
+
const [answer, setAnswer] = useState<Answer | null>(null);
|
| 148 |
+
const [error, setError] = useState("");
|
| 149 |
+
const [busy, setBusy] = useState(false);
|
| 150 |
+
const [customPeriod, setCustomPeriod] = useState(false);
|
| 151 |
+
const [moreMetrics, setMoreMetrics] = useState(false);
|
| 152 |
+
const [days, setDays] = useState(configs[0]?.days ?? 7);
|
| 153 |
+
const [window, setWindow] = useState<ResultConfig["window"]>(configs[0]?.window ?? "fixed");
|
| 154 |
+
const configStamp = JSON.stringify(configs);
|
| 155 |
+
useEffect(() => { setDraft(JSON.parse(configStamp) as ResultConfig[]); }, [configStamp]);
|
| 156 |
useEffect(() => {
|
| 157 |
const controller = new AbortController();
|
| 158 |
+
setAnswer(null);
|
| 159 |
+
setError("");
|
| 160 |
+
void fetch(`${API_V1}/amazon-projects/${encodeURIComponent(projectId)}/results`, { credentials: CREDENTIALS,
|
| 161 |
signal: AbortSignal.any([controller.signal, AbortSignal.timeout(20000)]) })
|
| 162 |
.then(async response => {
|
| 163 |
+
if (!response.ok) throw new Error("Results could not be loaded.");
|
| 164 |
+
const body = await response.json() as Answer;
|
| 165 |
+
if (!controller.signal.aborted) setAnswer(body);
|
| 166 |
+
}).catch(() => { if (!controller.signal.aborted) setError("Results could not be loaded."); });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
return () => controller.abort();
|
| 168 |
+
}, [projectId, configStamp, completedOn, updatedAt]);
|
| 169 |
+
const dirty = JSON.stringify(draft) !== configStamp;
|
| 170 |
+
const period = (nextWindow: ResultConfig["window"], nextDays: number) => {
|
| 171 |
+
setWindow(nextWindow); setDays(nextDays);
|
| 172 |
+
setDraft(current => current.map(config => ({ ...config, window: nextWindow, days: nextDays })));
|
| 173 |
+
};
|
| 174 |
+
const save = async () => {
|
| 175 |
+
if (busy) return;
|
| 176 |
+
setBusy(true); setError("");
|
|
|
|
| 177 |
try {
|
| 178 |
+
const response = await fetch(`${API_V1}/amazon-projects/${encodeURIComponent(projectId)}`, {
|
| 179 |
+
method: "PATCH", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" },
|
| 180 |
+
signal: AbortSignal.timeout(20000), body: JSON.stringify({ resultConfigs: draft }),
|
| 181 |
+
});
|
| 182 |
+
if (!response.ok) throw new Error("The results were not saved.");
|
| 183 |
+
onSaved();
|
| 184 |
+
} catch { setError("The results were not saved."); }
|
| 185 |
+
finally { setBusy(false); }
|
| 186 |
};
|
| 187 |
+
return <section className="lp-proj-section lp-proj-results" aria-label="Results">
|
| 188 |
+
<h4 className="lp-proj-section-title">Results</h4>
|
| 189 |
+
<div className="lp-proj-result-controls">
|
| 190 |
+
<select className="lp-proj-input" aria-label="Result period" disabled={!canEdit || busy}
|
| 191 |
+
value={window === "completion_to_now" ? "now" : [7, 30, 90].includes(days) ? String(days) : "custom"}
|
| 192 |
+
onChange={event => {
|
| 193 |
+
if (event.target.value === "custom") setCustomPeriod(true);
|
| 194 |
+
period(event.target.value === "now" ? "completion_to_now" : "fixed",
|
| 195 |
+
["now", "custom"].includes(event.target.value) ? days : Number(event.target.value));
|
| 196 |
+
}}>
|
| 197 |
+
<option value="7">7 days before vs 7 days after completion</option>
|
| 198 |
+
<option value="30">30 days before vs 30 days after completion</option>
|
| 199 |
+
<option value="90">90 days before vs 90 days after completion</option>
|
| 200 |
+
<option value="now">Completion to now</option><option value="custom">Custom period</option>
|
| 201 |
+
</select>
|
| 202 |
+
<button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={!canEdit || busy}
|
| 203 |
+
aria-expanded={customPeriod} onClick={() => {
|
| 204 |
+
if (!customPeriod && window === "completion_to_now") period("fixed", days);
|
| 205 |
+
setCustomPeriod(value => !value);
|
| 206 |
+
}}>Custom period</button>
|
| 207 |
+
{customPeriod ? <label className="lp-proj-result-days">Days<input className="lp-proj-input" type="number" aria-label="Result days"
|
| 208 |
+
min={1} max={366} step={1} value={days} disabled={!canEdit || busy || window === "completion_to_now"}
|
| 209 |
+
onChange={event => { const value = event.target.valueAsNumber; if (Number.isInteger(value) && value >= 1 && value <= 366) period("fixed", value); }} /></label> : null}
|
| 210 |
</div>
|
| 211 |
+
<div className="lp-proj-result-metrics" role="group" aria-label="Result metrics">
|
| 212 |
+
{METRICS.filter(metric => moreMetrics || draft.some(config => configKey(config) === metric.key)).map(metric => <label key={metric.key}><input type="checkbox" disabled={!canEdit || busy}
|
| 213 |
+
checked={draft.some(config => configKey(config) === metric.key)} onChange={event => setDraft(current => event.target.checked
|
| 214 |
+
? [...current, { id: crypto.randomUUID(), metric: metric.metric, grain: current[0]?.grain ?? "sku", window, days, compare: metric.compare }]
|
| 215 |
+
: current.filter(config => configKey(config) !== metric.key))} />{metric.label}</label>)}
|
| 216 |
+
{canEdit ? <button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={busy} aria-expanded={moreMetrics}
|
| 217 |
+
onClick={() => setMoreMetrics(value => !value)}>{moreMetrics ? "Done" : "Add metric"}</button> : null}
|
| 218 |
+
</div>
|
| 219 |
+
{dirty && canEdit ? <div className="lp-proj-form-actions">
|
| 220 |
+
<button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={busy} onClick={() => setDraft(configs)}>Cancel</button>
|
| 221 |
+
<button type="button" className="lp-proj-btn lp-proj-btn-primary" disabled={busy} onClick={() => void save()}>{busy ? "Saving…" : "Save results"}</button>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 222 |
</div> : null}
|
| 223 |
+
{error ? <p className="lp-proj-error" role="status">{error}</p> : null}
|
| 224 |
+
{answer?.results.map(result => <div className="lp-proj-result" key={result.config.id}>
|
| 225 |
+
<h5 className="lp-proj-section-title">{result.label}{result.config.compare === "yoy" ? " YoY" : ""}</h5>
|
| 226 |
+
<Trend result={result} />
|
| 227 |
+
</div>)}
|
| 228 |
</section>;
|
| 229 |
}
|
web/src/apps/amazon-profit/Projects.tsx
CHANGED
|
@@ -1,10 +1,10 @@
|
|
| 1 |
import { createContext, useContext, useCallback, useEffect, useId, useMemo, useRef, useState, type FormEvent, type ReactNode } from "react";
|
| 2 |
import { createPortal } from "react-dom";
|
|
|
|
| 3 |
import { CellSignalHigh } from "@phosphor-icons/react/dist/csr/CellSignalHigh";
|
| 4 |
import { CellSignalMedium } from "@phosphor-icons/react/dist/csr/CellSignalMedium";
|
| 5 |
import { CellSignalLow } from "@phosphor-icons/react/dist/csr/CellSignalLow";
|
| 6 |
import { Check } from "@phosphor-icons/react/dist/csr/Check";
|
| 7 |
-
import { CaretDown } from "@phosphor-icons/react/dist/csr/CaretDown";
|
| 8 |
import { CircleDashed } from "@phosphor-icons/react/dist/csr/CircleDashed";
|
| 9 |
import { Cube } from "@phosphor-icons/react/dist/csr/Cube";
|
| 10 |
import { DotsThree } from "@phosphor-icons/react/dist/csr/DotsThree";
|
|
@@ -21,7 +21,7 @@ import { SidebarSimple } from "@phosphor-icons/react/dist/csr/SidebarSimple";
|
|
| 21 |
import { API_V1, CREDENTIALS, TOAST_EVENT, signal } from "../../apiContract";
|
| 22 |
import { refusalMessage } from "../../customer-grid/apiBridge";
|
| 23 |
import { assetUrl } from "../../customer-grid/catalogData";
|
| 24 |
-
import {
|
| 25 |
import "../../styles/sdlc82/amazon-projects.css";
|
| 26 |
import ProjectResults, { ProjectResultPreview } from "./ProjectResults";
|
| 27 |
import type { ResultConfig } from "./ProjectResults";
|
|
@@ -34,11 +34,6 @@ export const PROJECT_PRIORITIES = ["No priority", "Urgent", "High", "Medium", "L
|
|
| 34 |
type Status = (typeof PROJECT_STATUSES)[number];
|
| 35 |
type Priority = (typeof PROJECT_PRIORITIES)[number];
|
| 36 |
type Health = "On track" | "At risk" | "Off track" | null;
|
| 37 |
-
const STATUS_GROUPS: Array<{ label: string; statuses: readonly Status[] }> = [
|
| 38 |
-
{ label: "Not started", statuses: ["Backlog", "Planned"] },
|
| 39 |
-
{ label: "Active", statuses: ["In progress"] },
|
| 40 |
-
{ label: "Done", statuses: ["Completed", "Canceled"] },
|
| 41 |
-
];
|
| 42 |
|
| 43 |
export interface Project {
|
| 44 |
id: string;
|
|
@@ -108,7 +103,6 @@ interface ProjectInput {
|
|
| 108 |
ownerUsername?: string | null;
|
| 109 |
startOn?: string | null;
|
| 110 |
targetOn?: string | null;
|
| 111 |
-
completedOn?: string | null;
|
| 112 |
progress?: number;
|
| 113 |
notes?: string | null;
|
| 114 |
targets?: Array<{ grain: ProjectGrain; key: string }>;
|
|
@@ -117,9 +111,8 @@ interface ProjectInput {
|
|
| 117 |
type ProjectDraft = Required<Omit<ProjectInput, "targets">>;
|
| 118 |
|
| 119 |
function draftOf(project: Project): ProjectDraft {
|
| 120 |
-
const { name, status, priority, owner, startOn, targetOn,
|
| 121 |
-
return { name, status, priority, owner, ownerUsername: project.ownerUsername ?? null,
|
| 122 |
-
startOn, targetOn, completedOn, progress, notes };
|
| 123 |
}
|
| 124 |
|
| 125 |
const BASE = `${API_V1}/amazon-projects`;
|
|
@@ -136,40 +129,27 @@ const FIELD_LABEL: Record<string, string> = {
|
|
| 136 |
};
|
| 137 |
const enc = encodeURIComponent;
|
| 138 |
|
| 139 |
-
function OwnerSelect({ value, legacy, disabled, onChange
|
| 140 |
value: string | null; legacy?: string | null; disabled: boolean; onChange: (value: string | null) => void;
|
| 141 |
-
label?: string; emptyLabel?: string;
|
| 142 |
}) {
|
| 143 |
const [owners, setOwners] = useState<Array<{ username: string; name: string }>>([]);
|
| 144 |
const [failed, setFailed] = useState(false);
|
| 145 |
-
const [loading, setLoading] = useState(true);
|
| 146 |
-
const [attempt, setAttempt] = useState(0);
|
| 147 |
useEffect(() => {
|
| 148 |
const controller = new AbortController();
|
| 149 |
-
setLoading(true);
|
| 150 |
-
setFailed(false);
|
| 151 |
void call<{ owners: Array<{ username: string; name: string }> }>("/owners", { signal: controller.signal })
|
| 152 |
.then(answer => { if (!controller.signal.aborted) setOwners(answer.owners); })
|
| 153 |
-
.catch(() => { if (!controller.signal.aborted) setFailed(true); })
|
| 154 |
-
.finally(() => { if (!controller.signal.aborted) setLoading(false); });
|
| 155 |
return () => controller.abort();
|
| 156 |
-
}, [
|
| 157 |
const selected = value || (legacy ? "__legacy__" : "");
|
| 158 |
-
return <
|
| 159 |
-
aria-busy={loading}
|
| 160 |
onChange={event => onChange(event.target.value || null)}>
|
| 161 |
-
<option value="">
|
| 162 |
{legacy && !value ? <option value="__legacy__" disabled>{legacy}</option> : null}
|
| 163 |
{value && !owners.some(owner => owner.username === value)
|
| 164 |
? <option value={value} disabled>{legacy || "Previous owner"}</option> : null}
|
| 165 |
{owners.map(owner => <option key={owner.username} value={owner.username}>{owner.name}</option>)}
|
| 166 |
-
</select>
|
| 167 |
-
{failed ? <div className="lp-proj-setup-inline">
|
| 168 |
-
<span className="lp-proj-error" role="alert">Assignees did not load.</span>
|
| 169 |
-
<button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={disabled}
|
| 170 |
-
onClick={() => setAttempt(current => current + 1)}>Retry</button>
|
| 171 |
-
</div> : null}
|
| 172 |
-
</>;
|
| 173 |
}
|
| 174 |
|
| 175 |
class ProjectRequestError extends Error {
|
|
@@ -279,9 +259,9 @@ function StatusRing({ status, progress }: { status: Status; progress: number })
|
|
| 279 |
const share = clampPercent(progress) / 100;
|
| 280 |
if (status === "Completed" || status === "Canceled" || status === "Backlog") {
|
| 281 |
const Icon = status === "Completed" ? Check : status === "Canceled" ? Prohibit : CircleDashed;
|
| 282 |
-
return <Icon className="lp-proj-ring
|
| 283 |
}
|
| 284 |
-
return <svg className="lp-proj-ring
|
| 285 |
<circle cx="7" cy="7" r={radius} fill="none" stroke="currentColor" strokeWidth="1.5"
|
| 286 |
opacity={status === "Planned" ? 1 : 0.35} />
|
| 287 |
<circle cx="7" cy="7" r={radius} fill="none" stroke="currentColor" strokeWidth="1.5"
|
|
@@ -290,73 +270,12 @@ function StatusRing({ status, progress }: { status: Status; progress: number })
|
|
| 290 |
</svg>;
|
| 291 |
}
|
| 292 |
|
| 293 |
-
function StatusPill({ status, progress }: { status: Status; progress: number }) {
|
| 294 |
-
return <span className="lp-proj-status lp-proj-accent" data-status={status}>
|
| 295 |
-
<StatusRing status={status} progress={progress} />{status}
|
| 296 |
-
</span>;
|
| 297 |
-
}
|
| 298 |
-
|
| 299 |
-
function StatusPicker({ value, disabled, onChange }: {
|
| 300 |
-
value: Status; disabled: boolean; onChange: (status: Status) => void;
|
| 301 |
-
}) {
|
| 302 |
-
const [open, setOpen] = useState(false);
|
| 303 |
-
const trigger = useRef<HTMLButtonElement>(null);
|
| 304 |
-
const menuId = useId();
|
| 305 |
-
const expanded = open && !disabled;
|
| 306 |
-
return <>
|
| 307 |
-
<button ref={trigger} type="button" className="lp-proj-status lp-proj-status-trigger lp-proj-accent" data-status={value}
|
| 308 |
-
aria-label={`Status: ${value}`} aria-haspopup="menu" aria-expanded={expanded}
|
| 309 |
-
aria-controls={expanded ? menuId : undefined} disabled={disabled} onClick={() => setOpen(current => !current)}
|
| 310 |
-
onKeyDown={event => {
|
| 311 |
-
if (event.key === "ArrowDown" || event.key === "ArrowUp") {
|
| 312 |
-
event.preventDefault(); setOpen(true);
|
| 313 |
-
}
|
| 314 |
-
}}>
|
| 315 |
-
<span className="lp-proj-status-dot" aria-hidden="true" />{value}<CaretDown size={12} aria-hidden="true" />
|
| 316 |
-
</button>
|
| 317 |
-
{expanded ? <AnchoredOverlay anchor={trigger.current} className="lp-proj-status-menu" id={menuId} role="menu"
|
| 318 |
-
ariaLabel="Project status" initialFocus='[aria-checked="true"]' onDismiss={() => setOpen(false)}
|
| 319 |
-
onKeyDown={event => {
|
| 320 |
-
if (event.key === "Tab") { event.preventDefault(); setOpen(false); return; }
|
| 321 |
-
const items = Array.from(event.currentTarget.querySelectorAll<HTMLButtonElement>('[role="menuitemradio"]'));
|
| 322 |
-
const current = items.findIndex(item => item === document.activeElement);
|
| 323 |
-
let next: HTMLButtonElement | undefined;
|
| 324 |
-
if (event.key === "ArrowDown") next = items[(current + 1) % items.length];
|
| 325 |
-
else if (event.key === "ArrowUp") next = items[(current - 1 + items.length) % items.length];
|
| 326 |
-
else if (event.key === "Home") next = items[0];
|
| 327 |
-
else if (event.key === "End") next = items.at(-1);
|
| 328 |
-
else if (event.key.length === 1 && !event.ctrlKey && !event.metaKey && !event.altKey) {
|
| 329 |
-
next = [...items.slice(current + 1), ...items.slice(0, current + 1)]
|
| 330 |
-
.find(item => item.textContent?.toLowerCase().startsWith(event.key.toLowerCase()));
|
| 331 |
-
}
|
| 332 |
-
if (next) { event.preventDefault(); event.stopPropagation(); next.focus(); }
|
| 333 |
-
}}>
|
| 334 |
-
{STATUS_GROUPS.map((group, index) => <div key={group.label} className="lp-proj-status-group" role="group"
|
| 335 |
-
aria-labelledby={`${menuId}-${index}`}>
|
| 336 |
-
<span id={`${menuId}-${index}`} className="lp-proj-status-heading">{group.label}</span>
|
| 337 |
-
{group.statuses.map(status => <button type="button" key={status} role="menuitemradio" aria-checked={status === value}
|
| 338 |
-
tabIndex={status === value ? 0 : -1} className="lp-proj-status-option lp-proj-accent" data-status={status}
|
| 339 |
-
onClick={() => { onChange(status); setOpen(false); }}>
|
| 340 |
-
<span className="lp-proj-status-dot" aria-hidden="true" /><span>{status}</span>
|
| 341 |
-
{status === value ? <Check size={14} aria-hidden="true" /> : null}
|
| 342 |
-
</button>)}
|
| 343 |
-
</div>)}
|
| 344 |
-
</AnchoredOverlay> : null}
|
| 345 |
-
</>;
|
| 346 |
-
}
|
| 347 |
-
|
| 348 |
function HealthGlyph({ health }: { health: Health }) {
|
| 349 |
if (!health) return null;
|
| 350 |
const Icon = health === "On track" ? TrendUp : health === "At risk" ? Minus : TrendDown;
|
| 351 |
return <Icon className="lp-proj-health" data-health={health} size={14} aria-hidden="true" />;
|
| 352 |
}
|
| 353 |
|
| 354 |
-
function HealthChip({ health }: { health: Health }) {
|
| 355 |
-
return health ? <span className="lp-proj-health-chip" data-health={health}>
|
| 356 |
-
<HealthGlyph health={health} />{health}
|
| 357 |
-
</span> : null;
|
| 358 |
-
}
|
| 359 |
-
|
| 360 |
function PriorityGlyph({ priority }: { priority: Priority }) {
|
| 361 |
const Icon = priority === "Urgent" ? Warning : priority === "High" ? CellSignalHigh
|
| 362 |
: priority === "Medium" ? CellSignalMedium : priority === "Low" ? CellSignalLow : DotsThree;
|
|
@@ -489,8 +408,7 @@ function ProjectTable({ projects, grain, onOpen, empty, visible = DEFAULT_PROJEC
|
|
| 489 |
<tbody>
|
| 490 |
{projects.map((project) => {
|
| 491 |
const via = project.via && grain && (project.via.grain !== grain) ? project.via : null;
|
| 492 |
-
return <tr key={project.id} className="lp-proj-row
|
| 493 |
-
data-project-id={project.id} onClick={() => onOpen(project.id)}>
|
| 494 |
<td><div className="lp-proj-name-wrap">
|
| 495 |
<button type="button" className="lp-proj-name" onClick={(event) => { event.stopPropagation(); onOpen(project.id); }}>
|
| 496 |
<Cube className="lp-proj-name-glyph" size={14} aria-hidden="true" />
|
|
@@ -502,7 +420,8 @@ function ProjectTable({ projects, grain, onOpen, empty, visible = DEFAULT_PROJEC
|
|
| 502 |
let cell: ReactNode;
|
| 503 |
switch (key) {
|
| 504 |
case "health":
|
| 505 |
-
cell = <
|
|
|
|
| 506 |
break;
|
| 507 |
case "priority":
|
| 508 |
cell = <span className="lp-proj-cell"><PriorityGlyph priority={project.priority} />
|
|
@@ -518,7 +437,7 @@ function ProjectTable({ projects, grain, onOpen, empty, visible = DEFAULT_PROJEC
|
|
| 518 |
cell = formatDay(project[key]) || <span className="lp-proj-muted">No date</span>;
|
| 519 |
break;
|
| 520 |
case "status":
|
| 521 |
-
cell = <
|
| 522 |
break;
|
| 523 |
case "results":
|
| 524 |
cell = <ProjectResultPreview projectId={project.id} configs={project.resultConfigs ?? []}
|
|
@@ -664,7 +583,7 @@ function ProjectBrowser({ read, grain, onOpen }: {
|
|
| 664 |
<div id={`${id}-summary`} role="tabpanel" aria-labelledby={`${id}-${tab}`} title="Counts follow the other filters">
|
| 665 |
{tab === "Health" ? healthCounts.map(({ label, count }) => <button type="button" className="lp-proj-summary-row" key={label}
|
| 666 |
aria-pressed={health === label} onClick={() => setHealth(health === label ? "" : label)}>
|
| 667 |
-
<span>{label === "No health" ?
|
| 668 |
</button>) : ownerCounts.map(({ value, count }) => <button type="button" className="lp-proj-summary-row" key={ownerKey(value)}
|
| 669 |
aria-pressed={owner === ownerKey(value)} onClick={() => setOwner(owner === ownerKey(value) ? "" : ownerKey(value))}>
|
| 670 |
<Owner name={value} /><span>{count}</span>
|
|
@@ -676,46 +595,7 @@ function ProjectBrowser({ read, grain, onOpen }: {
|
|
| 676 |
</section>;
|
| 677 |
}
|
| 678 |
|
| 679 |
-
// ---- One project:
|
| 680 |
-
|
| 681 |
-
const PROJECT_PANES = [
|
| 682 |
-
{ key: "results", label: "Results" },
|
| 683 |
-
{ key: "setup", label: "Setup" },
|
| 684 |
-
{ key: "log", label: "Log" },
|
| 685 |
-
] as const;
|
| 686 |
-
type ProjectPane = (typeof PROJECT_PANES)[number]["key"];
|
| 687 |
-
|
| 688 |
-
function SetupRow({ field, label, error, children, readOnly = false }: {
|
| 689 |
-
field: string; label: string; error?: string; children: ReactNode; readOnly?: boolean;
|
| 690 |
-
}) {
|
| 691 |
-
const labelId = useId();
|
| 692 |
-
return <div className="lp-proj-setup-row" data-field={field} data-read-only={readOnly || undefined}
|
| 693 |
-
role="group" aria-labelledby={labelId}>
|
| 694 |
-
<span className="lp-proj-setup-label" id={labelId}>{label}</span>
|
| 695 |
-
<div className="lp-proj-setup-value">
|
| 696 |
-
{children}
|
| 697 |
-
{error ? <p className="lp-proj-error lp-proj-setup-error" role="alert">{error}</p> : null}
|
| 698 |
-
</div>
|
| 699 |
-
</div>;
|
| 700 |
-
}
|
| 701 |
-
|
| 702 |
-
function SetupDate({ label, value, canEdit, busy, onChange, max, hint }: {
|
| 703 |
-
label: string; value: string | null; canEdit: boolean; busy: boolean;
|
| 704 |
-
onChange: (value: string | null) => void; max?: string; hint?: string;
|
| 705 |
-
}) {
|
| 706 |
-
const [editing, setEditing] = useState(false);
|
| 707 |
-
const caption = value ? new Date(`${value}T00:00:00`).toLocaleDateString("en-US",
|
| 708 |
-
{ month: "short", day: "numeric", year: "numeric" }) : "Not set";
|
| 709 |
-
if (!canEdit) return value ? <time dateTime={value}>{caption}</time> : <span className="lp-proj-muted">Not set</span>;
|
| 710 |
-
return editing ? <input className="lp-proj-setup-input" type="date" aria-label={label}
|
| 711 |
-
value={value ?? ""} disabled={busy} max={max} autoFocus
|
| 712 |
-
onChange={event => onChange(event.target.value || null)} onBlur={() => setEditing(false)} />
|
| 713 |
-
: <button type="button" className="lp-proj-setup-edit" data-empty={!value || undefined}
|
| 714 |
-
aria-label={`${value ? "Edit" : "Add"} ${label.toLowerCase()}`} title={hint}
|
| 715 |
-
disabled={busy} onClick={() => setEditing(true)}>
|
| 716 |
-
{value ? caption : `Add ${label.toLowerCase()}`}
|
| 717 |
-
</button>;
|
| 718 |
-
}
|
| 719 |
|
| 720 |
function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }: {
|
| 721 |
id: string; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
|
|
@@ -727,19 +607,22 @@ function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }:
|
|
| 727 |
const [busy, setBusy] = useState(false);
|
| 728 |
const [adding, setAdding] = useState(false);
|
| 729 |
const [confirming, setConfirming] = useState(false);
|
| 730 |
-
const [
|
| 731 |
-
const
|
| 732 |
-
const [saved, setSaved] = useState(false);
|
| 733 |
-
const [pane, setPane] = useState<ProjectPane>("results");
|
| 734 |
const formId = useId();
|
| 735 |
-
const
|
| 736 |
const writing = useRef(false);
|
|
|
|
| 737 |
const loaded = detail !== null;
|
| 738 |
|
| 739 |
useEffect(() => {
|
| 740 |
-
if (loaded)
|
| 741 |
}, [loaded]);
|
| 742 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 743 |
useEffect(() => {
|
| 744 |
const controller = new AbortController();
|
| 745 |
call<Detail>(`/${enc(id)}`, { signal: controller.signal }, "This project did not load.")
|
|
@@ -753,8 +636,7 @@ function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }:
|
|
| 753 |
return () => controller.abort();
|
| 754 |
}, [id, tick]);
|
| 755 |
|
| 756 |
-
const write = useCallback(async (path: string, method: string, body: unknown, after?: () => void
|
| 757 |
-
fields?: Array<keyof ProjectDraft>) => {
|
| 758 |
if (writing.current || !mayEdit || !detail?.canEdit) return;
|
| 759 |
writing.current = true;
|
| 760 |
setBusy(true);
|
|
@@ -766,19 +648,10 @@ function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }:
|
|
| 766 |
if (method === "PATCH") setDraft(draftOf(project));
|
| 767 |
}
|
| 768 |
setError("");
|
| 769 |
-
setFieldErrors({});
|
| 770 |
-
if (method === "PATCH") setSaved(true);
|
| 771 |
if (after) after(); else setTick((value) => value + 1);
|
| 772 |
onChanged();
|
| 773 |
} catch (reason) {
|
| 774 |
-
|
| 775 |
-
if (fields?.length) {
|
| 776 |
-
const namedField = fields.find(field => message.toLowerCase().includes(
|
| 777 |
-
({ startOn: "start_on", targetOn: "target_on", completedOn: "completed_on",
|
| 778 |
-
ownerUsername: "owner" } as Partial<Record<keyof ProjectDraft, string>>)[field] ?? field.toLowerCase()));
|
| 779 |
-
setFieldErrors(Object.fromEntries((namedField ? [namedField] : fields).map(field => [field, message])));
|
| 780 |
-
setError("");
|
| 781 |
-
} else setError(message);
|
| 782 |
if (reason instanceof ProjectRequestError && reason.status === 403) {
|
| 783 |
setDetail((current) => current ? { ...current, canEdit: false } : current);
|
| 784 |
onChanged();
|
|
@@ -789,158 +662,105 @@ function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }:
|
|
| 789 |
}
|
| 790 |
}, [detail?.canEdit, mayEdit, onChanged]);
|
| 791 |
|
| 792 |
-
const patch = useCallback((change: ProjectInput
|
| 793 |
-
|
| 794 |
-
|
| 795 |
-
const
|
| 796 |
-
<div className="lp-proj-detail-rail-list" role="tablist" aria-label="Project sections" aria-orientation="vertical">
|
| 797 |
-
{PROJECT_PANES.map(({ key, label }, index) => <button key={key} type="button" role="tab"
|
| 798 |
-
className={`lp-proj-detail-rail-row${pane === key ? " is-on" : ""}`}
|
| 799 |
-
id={`${formId}-tab-${key}`} aria-controls={`${formId}-pane-${key}`}
|
| 800 |
-
aria-selected={pane === key} tabIndex={pane === key ? 0 : -1} onClick={() => setPane(key)}
|
| 801 |
-
onKeyDown={event => {
|
| 802 |
-
let next: number;
|
| 803 |
-
if (event.key === "ArrowDown" || event.key === "ArrowRight") next = (index + 1) % PROJECT_PANES.length;
|
| 804 |
-
else if (event.key === "ArrowUp" || event.key === "ArrowLeft") next = (index - 1 + PROJECT_PANES.length) % PROJECT_PANES.length;
|
| 805 |
-
else if (event.key === "Home") next = 0;
|
| 806 |
-
else if (event.key === "End") next = PROJECT_PANES.length - 1;
|
| 807 |
-
else return;
|
| 808 |
-
event.preventDefault();
|
| 809 |
-
event.stopPropagation();
|
| 810 |
-
setPane(PROJECT_PANES[next].key);
|
| 811 |
-
railRef.current?.querySelectorAll<HTMLButtonElement>('[role="tab"]')[next]?.focus();
|
| 812 |
-
}}><span className="lp-proj-detail-rail-label">{label}</span></button>)}
|
| 813 |
-
</div>
|
| 814 |
-
</nav>;
|
| 815 |
-
|
| 816 |
-
if (!detail || !draft) return <div className="lp-proj-detail-layout">
|
| 817 |
-
{rail}
|
| 818 |
-
<div className="lp-proj-panes">
|
| 819 |
-
{PROJECT_PANES.map(({ key }) => <section key={key} className="lp-proj-pane" role="tabpanel"
|
| 820 |
-
id={`${formId}-pane-${key}`} aria-labelledby={`${formId}-tab-${key}`} hidden={pane !== key}
|
| 821 |
-
tabIndex={0} aria-busy={!error}>
|
| 822 |
-
{error ? <div><p className="lp-proj-error" role="alert">{error}</p>
|
| 823 |
-
<button type="button" className="lp-proj-btn" onClick={() => { setError(""); setTick(value => value + 1); }}>Retry</button>
|
| 824 |
-
</div> : <LoadingRows />}
|
| 825 |
-
</section>)}
|
| 826 |
-
</div>
|
| 827 |
-
</div>;
|
| 828 |
-
const { project, targets } = detail;
|
| 829 |
-
const activity = [...detail.activity].sort((a, b) => Date.parse(b.at) - Date.parse(a.at));
|
| 830 |
const canEdit = mayEdit && detail.canEdit;
|
| 831 |
const display = canEdit ? draft : project;
|
| 832 |
const progress = Number.isFinite(display.progress) ? clampPercent(display.progress) : 0;
|
| 833 |
const dirty = JSON.stringify(draft) !== JSON.stringify(draftOf(project));
|
| 834 |
-
const set = <K extends keyof ProjectDraft>(key: K, value: ProjectDraft[K]) =>
|
| 835 |
-
|
| 836 |
-
setFieldErrors(current => ({ ...current, [key]: undefined }));
|
| 837 |
-
setDraft(current => current ? { ...current, [key]: value,
|
| 838 |
-
...(key === "status" && value !== "Completed" ? { completedOn: null } : {}),
|
| 839 |
-
...(key === "completedOn" && value ? { status: "Completed" as Status } : {}),
|
| 840 |
-
} : current);
|
| 841 |
-
};
|
| 842 |
const save = (event: FormEvent) => {
|
| 843 |
event.preventDefault();
|
| 844 |
if (!canEdit || !dirty) return;
|
| 845 |
const name = draft.name.trim();
|
| 846 |
-
if (!name) {
|
| 847 |
if (!Number.isInteger(draft.progress) || draft.progress < 0 || draft.progress > 100) {
|
| 848 |
-
|
| 849 |
-
}
|
| 850 |
-
const today = new Date().toISOString().slice(0, 10);
|
| 851 |
-
if (draft.completedOn && draft.completedOn > today) {
|
| 852 |
-
setFieldErrors({ completedOn: "Choose today or an earlier date." }); return;
|
| 853 |
-
}
|
| 854 |
-
if (project.completedOn && draft.status === "Completed" && !draft.completedOn) {
|
| 855 |
-
setFieldErrors({ completedOn: "Choose a completed date, or change Status to reopen the project." }); return;
|
| 856 |
}
|
| 857 |
-
const
|
| 858 |
-
|
| 859 |
-
|
| 860 |
-
if (changed.includes("owner")) fields.ownerUsername = draft.ownerUsername;
|
| 861 |
-
if (changed.includes("name")) fields.name = name;
|
| 862 |
-
if (changed.includes("notes")) fields.notes = draft.notes?.trim() || null;
|
| 863 |
-
setSaved(false);
|
| 864 |
-
setFieldErrors({});
|
| 865 |
-
void patch(fields, changed);
|
| 866 |
};
|
| 867 |
|
| 868 |
-
|
| 869 |
-
|
| 870 |
-
|
| 871 |
-
|
| 872 |
-
|
| 873 |
-
|
| 874 |
-
|
| 875 |
-
|
| 876 |
-
|
| 877 |
-
<
|
| 878 |
-
|
| 879 |
-
|
| 880 |
-
|
| 881 |
-
|
| 882 |
-
|
| 883 |
-
|
| 884 |
-
|
| 885 |
-
|
| 886 |
-
|
| 887 |
-
: <StatusPill status={project.status} progress={progress} />}
|
| 888 |
-
</SetupRow>
|
| 889 |
-
<SetupRow field="priority" label="Priority" error={fieldErrors.priority}>
|
| 890 |
-
<div className="lp-proj-setup-inline"><PriorityGlyph priority={display.priority} />
|
| 891 |
{canEdit
|
| 892 |
-
? <select
|
| 893 |
-
data-empty={draft.priority === "No priority" || undefined}
|
| 894 |
onChange={(event) => set("priority", event.target.value as Priority)}>
|
| 895 |
-
{PROJECT_PRIORITIES.map((value) => <option key={value} value={value}>{value
|
| 896 |
</select>
|
| 897 |
: <span>{project.priority}</span>}
|
| 898 |
-
</
|
| 899 |
-
|
| 900 |
-
|
| 901 |
-
|
| 902 |
-
|
| 903 |
-
|
| 904 |
-
</
|
| 905 |
-
|
| 906 |
-
|
| 907 |
-
|
| 908 |
-
|
| 909 |
-
|
| 910 |
-
|
| 911 |
-
|
| 912 |
-
|
| 913 |
-
|
| 914 |
-
|
| 915 |
-
|
| 916 |
-
|
| 917 |
-
|
| 918 |
-
|
| 919 |
-
|
| 920 |
-
|
| 921 |
-
|
| 922 |
-
|
| 923 |
-
|
| 924 |
-
|
| 925 |
-
{
|
| 926 |
-
|
| 927 |
-
placeholder="Add description" value={draft.notes ?? ""} rows={4} disabled={busy}
|
| 928 |
-
autoFocus={editingDescription} onChange={event => set("notes", event.target.value)}
|
| 929 |
-
onBlur={() => setEditingDescription(false)} />
|
| 930 |
-
: <button type="button" className="lp-proj-setup-edit" data-empty disabled={busy}
|
| 931 |
-
onClick={() => setEditingDescription(true)}>Add description</button>
|
| 932 |
-
: <p className="lp-proj-notes-text">{project.notes || <span className="lp-proj-muted">Not set</span>}</p>}
|
| 933 |
-
</SetupRow>
|
| 934 |
</div>
|
| 935 |
|
| 936 |
{error ? <p className="lp-proj-error" role="alert">{error}</p> : null}
|
| 937 |
{detail.cause ? <p className="lp-proj-muted" role="status">{detail.cause}</p> : null}
|
| 938 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 939 |
<section className="lp-proj-section" aria-label="SKUs">
|
| 940 |
<div className="lp-proj-section-head">
|
| 941 |
-
<span className="lp-proj-muted">{targets.length
|
| 942 |
{canEdit && !adding
|
| 943 |
-
? <button type="button" className="lp-proj-
|
| 944 |
: null}
|
| 945 |
</div>
|
| 946 |
{canEdit && adding
|
|
@@ -960,29 +780,40 @@ function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }:
|
|
| 960 |
: null}
|
| 961 |
</li>)}
|
| 962 |
</ul>
|
| 963 |
-
:
|
| 964 |
</section>
|
| 965 |
-
</SetupRow>
|
| 966 |
-
<div className="lp-proj-setup-footer" aria-label="Project timestamps">
|
| 967 |
-
{([ ["createdAt", "Created"], ["updatedAt", "Updated"] ] as const).map(([field, label]) =>
|
| 968 |
-
<SetupRow key={field} field={field} label={label} readOnly>
|
| 969 |
-
{project[field] ? <time dateTime={project[field]} title={project[field]}>
|
| 970 |
-
{new Date(project[field]).toLocaleDateString("en-US", { month: "short", day: "numeric", year: "numeric" })}
|
| 971 |
-
</time> : <span>Not available</span>}
|
| 972 |
-
</SetupRow>)}
|
| 973 |
-
</div>
|
| 974 |
-
{saved && !dirty ? <span className="lp-proj-muted" role="status">Changes saved</span> : null}
|
| 975 |
</div>
|
| 976 |
|
| 977 |
<aside className="lp-proj-side">
|
| 978 |
-
<
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 979 |
<button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={busy} onClick={onCancel}>Cancel</button>
|
| 980 |
{canEdit ? <button type="submit" form={formId} className="lp-proj-btn lp-proj-btn-primary" disabled={busy || !dirty}>
|
| 981 |
{busy ? "Saving…" : "Save changes"}
|
| 982 |
</button> : null}
|
| 983 |
{canEdit && dirty ? <button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={busy}
|
| 984 |
-
onClick={() => { setDraft(draftOf(project)); setError("");
|
| 985 |
-
</div>
|
| 986 |
{canEdit
|
| 987 |
? <div className="lp-proj-delete">
|
| 988 |
{confirming
|
|
@@ -996,35 +827,6 @@ function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }:
|
|
| 996 |
: null}
|
| 997 |
</aside>
|
| 998 |
</form>;
|
| 999 |
-
|
| 1000 |
-
return <div className="lp-proj-detail-layout">
|
| 1001 |
-
{rail}
|
| 1002 |
-
<div className="lp-proj-panes">
|
| 1003 |
-
<section className="lp-proj-pane lp-proj-pane-stack" role="tabpanel" id={`${formId}-pane-results`}
|
| 1004 |
-
aria-labelledby={`${formId}-tab-results`} hidden={pane !== "results"} tabIndex={0}>
|
| 1005 |
-
<h2 className="lp-proj-title">{project.name}</h2>
|
| 1006 |
-
<ProjectResults projectId={id} configs={project.resultConfigs ?? []} completedOn={project.completedOn}
|
| 1007 |
-
updatedAt={project.updatedAt} canEdit={canEdit} onSaved={() => { setTick(value => value + 1); onChanged(); }} />
|
| 1008 |
-
</section>
|
| 1009 |
-
<section className="lp-proj-pane" role="tabpanel" id={`${formId}-pane-setup`}
|
| 1010 |
-
aria-labelledby={`${formId}-tab-setup`} hidden={pane !== "setup"} tabIndex={0}>
|
| 1011 |
-
{setup}
|
| 1012 |
-
</section>
|
| 1013 |
-
<section className="lp-proj-pane lp-proj-pane-stack lp-proj-log" role="tabpanel" id={`${formId}-pane-log`}
|
| 1014 |
-
aria-labelledby={`${formId}-tab-log`} hidden={pane !== "log"} tabIndex={0}>
|
| 1015 |
-
<h2 className="lp-proj-title">{project.name}</h2>
|
| 1016 |
-
{activity.length
|
| 1017 |
-
? <ol className="lp-proj-activity" aria-label="Activity">
|
| 1018 |
-
{activity.map((entry, index) => <li key={`${entry.at}:${index}`}>
|
| 1019 |
-
<span className="lp-proj-activity-dot" aria-hidden="true" />
|
| 1020 |
-
<span>{activityLine(entry)} <time className="lp-proj-muted" dateTime={entry.at} title={entry.at}>
|
| 1021 |
-
{formatWhen(entry.at)}</time></span>
|
| 1022 |
-
</li>)}
|
| 1023 |
-
</ol>
|
| 1024 |
-
: <p className="lp-proj-muted">No activity yet</p>}
|
| 1025 |
-
</section>
|
| 1026 |
-
</div>
|
| 1027 |
-
</div>;
|
| 1028 |
}
|
| 1029 |
|
| 1030 |
function TargetLine({ target }: { target: Target }) {
|
|
@@ -1133,8 +935,7 @@ function ProjectForm({ targets, onCreated, onCancel, onForbidden }: {
|
|
| 1133 |
onCancel: () => void; onForbidden: () => void;
|
| 1134 |
}) {
|
| 1135 |
const [draft, setDraft] = useState<ProjectDraft>({
|
| 1136 |
-
name: "", status: "Planned", priority: "Medium", owner: "", ownerUsername: null,
|
| 1137 |
-
startOn: "", targetOn: "", completedOn: null, progress: 0, notes: "",
|
| 1138 |
});
|
| 1139 |
const formId = useId();
|
| 1140 |
const [busy, setBusy] = useState(false);
|
|
@@ -1172,8 +973,10 @@ function ProjectForm({ targets, onCreated, onCancel, onForbidden }: {
|
|
| 1172 |
<label className="lp-proj-field lp-proj-field-wide"><span>Project name</span>
|
| 1173 |
<input className="lp-proj-input" value={draft.name} onChange={(event) => set("name", event.target.value)}
|
| 1174 |
placeholder="Name your project" autoFocus required maxLength={300} disabled={busy} /></label>
|
| 1175 |
-
<
|
| 1176 |
-
<
|
|
|
|
|
|
|
| 1177 |
<label className="lp-proj-field"><span>Owner</span>
|
| 1178 |
<OwnerSelect value={draft.ownerUsername} disabled={busy} onChange={value => set("ownerUsername", value)} /></label>
|
| 1179 |
<label className="lp-proj-field"><span>Priority</span>
|
|
|
|
| 1 |
import { createContext, useContext, useCallback, useEffect, useId, useMemo, useRef, useState, type FormEvent, type ReactNode } from "react";
|
| 2 |
import { createPortal } from "react-dom";
|
| 3 |
+
import { ArrowRight } from "@phosphor-icons/react/dist/csr/ArrowRight";
|
| 4 |
import { CellSignalHigh } from "@phosphor-icons/react/dist/csr/CellSignalHigh";
|
| 5 |
import { CellSignalMedium } from "@phosphor-icons/react/dist/csr/CellSignalMedium";
|
| 6 |
import { CellSignalLow } from "@phosphor-icons/react/dist/csr/CellSignalLow";
|
| 7 |
import { Check } from "@phosphor-icons/react/dist/csr/Check";
|
|
|
|
| 8 |
import { CircleDashed } from "@phosphor-icons/react/dist/csr/CircleDashed";
|
| 9 |
import { Cube } from "@phosphor-icons/react/dist/csr/Cube";
|
| 10 |
import { DotsThree } from "@phosphor-icons/react/dist/csr/DotsThree";
|
|
|
|
| 21 |
import { API_V1, CREDENTIALS, TOAST_EVENT, signal } from "../../apiContract";
|
| 22 |
import { refusalMessage } from "../../customer-grid/apiBridge";
|
| 23 |
import { assetUrl } from "../../customer-grid/catalogData";
|
| 24 |
+
import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface";
|
| 25 |
import "../../styles/sdlc82/amazon-projects.css";
|
| 26 |
import ProjectResults, { ProjectResultPreview } from "./ProjectResults";
|
| 27 |
import type { ResultConfig } from "./ProjectResults";
|
|
|
|
| 34 |
type Status = (typeof PROJECT_STATUSES)[number];
|
| 35 |
type Priority = (typeof PROJECT_PRIORITIES)[number];
|
| 36 |
type Health = "On track" | "At risk" | "Off track" | null;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
export interface Project {
|
| 39 |
id: string;
|
|
|
|
| 103 |
ownerUsername?: string | null;
|
| 104 |
startOn?: string | null;
|
| 105 |
targetOn?: string | null;
|
|
|
|
| 106 |
progress?: number;
|
| 107 |
notes?: string | null;
|
| 108 |
targets?: Array<{ grain: ProjectGrain; key: string }>;
|
|
|
|
| 111 |
type ProjectDraft = Required<Omit<ProjectInput, "targets">>;
|
| 112 |
|
| 113 |
function draftOf(project: Project): ProjectDraft {
|
| 114 |
+
const { name, status, priority, owner, startOn, targetOn, progress, notes } = project;
|
| 115 |
+
return { name, status, priority, owner, ownerUsername: project.ownerUsername ?? null, startOn, targetOn, progress, notes };
|
|
|
|
| 116 |
}
|
| 117 |
|
| 118 |
const BASE = `${API_V1}/amazon-projects`;
|
|
|
|
| 129 |
};
|
| 130 |
const enc = encodeURIComponent;
|
| 131 |
|
| 132 |
+
function OwnerSelect({ value, legacy, disabled, onChange }: {
|
| 133 |
value: string | null; legacy?: string | null; disabled: boolean; onChange: (value: string | null) => void;
|
|
|
|
| 134 |
}) {
|
| 135 |
const [owners, setOwners] = useState<Array<{ username: string; name: string }>>([]);
|
| 136 |
const [failed, setFailed] = useState(false);
|
|
|
|
|
|
|
| 137 |
useEffect(() => {
|
| 138 |
const controller = new AbortController();
|
|
|
|
|
|
|
| 139 |
void call<{ owners: Array<{ username: string; name: string }> }>("/owners", { signal: controller.signal })
|
| 140 |
.then(answer => { if (!controller.signal.aborted) setOwners(answer.owners); })
|
| 141 |
+
.catch(() => { if (!controller.signal.aborted) setFailed(true); });
|
|
|
|
| 142 |
return () => controller.abort();
|
| 143 |
+
}, []);
|
| 144 |
const selected = value || (legacy ? "__legacy__" : "");
|
| 145 |
+
return <select className="lp-proj-input" aria-label="Owner" value={selected} disabled={disabled || failed}
|
|
|
|
| 146 |
onChange={event => onChange(event.target.value || null)}>
|
| 147 |
+
<option value="">Unassigned</option>
|
| 148 |
{legacy && !value ? <option value="__legacy__" disabled>{legacy}</option> : null}
|
| 149 |
{value && !owners.some(owner => owner.username === value)
|
| 150 |
? <option value={value} disabled>{legacy || "Previous owner"}</option> : null}
|
| 151 |
{owners.map(owner => <option key={owner.username} value={owner.username}>{owner.name}</option>)}
|
| 152 |
+
</select>;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
}
|
| 154 |
|
| 155 |
class ProjectRequestError extends Error {
|
|
|
|
| 259 |
const share = clampPercent(progress) / 100;
|
| 260 |
if (status === "Completed" || status === "Canceled" || status === "Backlog") {
|
| 261 |
const Icon = status === "Completed" ? Check : status === "Canceled" ? Prohibit : CircleDashed;
|
| 262 |
+
return <Icon className="lp-proj-ring" data-status={status} size={14} aria-hidden="true" />;
|
| 263 |
}
|
| 264 |
+
return <svg className="lp-proj-ring" data-status={status} width="14" height="14" viewBox="0 0 14 14" aria-hidden="true">
|
| 265 |
<circle cx="7" cy="7" r={radius} fill="none" stroke="currentColor" strokeWidth="1.5"
|
| 266 |
opacity={status === "Planned" ? 1 : 0.35} />
|
| 267 |
<circle cx="7" cy="7" r={radius} fill="none" stroke="currentColor" strokeWidth="1.5"
|
|
|
|
| 270 |
</svg>;
|
| 271 |
}
|
| 272 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
function HealthGlyph({ health }: { health: Health }) {
|
| 274 |
if (!health) return null;
|
| 275 |
const Icon = health === "On track" ? TrendUp : health === "At risk" ? Minus : TrendDown;
|
| 276 |
return <Icon className="lp-proj-health" data-health={health} size={14} aria-hidden="true" />;
|
| 277 |
}
|
| 278 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
function PriorityGlyph({ priority }: { priority: Priority }) {
|
| 280 |
const Icon = priority === "Urgent" ? Warning : priority === "High" ? CellSignalHigh
|
| 281 |
: priority === "Medium" ? CellSignalMedium : priority === "Low" ? CellSignalLow : DotsThree;
|
|
|
|
| 408 |
<tbody>
|
| 409 |
{projects.map((project) => {
|
| 410 |
const via = project.via && grain && (project.via.grain !== grain) ? project.via : null;
|
| 411 |
+
return <tr key={project.id} className="lp-proj-row" data-project-id={project.id} onClick={() => onOpen(project.id)}>
|
|
|
|
| 412 |
<td><div className="lp-proj-name-wrap">
|
| 413 |
<button type="button" className="lp-proj-name" onClick={(event) => { event.stopPropagation(); onOpen(project.id); }}>
|
| 414 |
<Cube className="lp-proj-name-glyph" size={14} aria-hidden="true" />
|
|
|
|
| 420 |
let cell: ReactNode;
|
| 421 |
switch (key) {
|
| 422 |
case "health":
|
| 423 |
+
cell = <span className="lp-proj-cell"><HealthGlyph health={project.health} />
|
| 424 |
+
{project.health ? <span>{project.health}</span> : null}</span>;
|
| 425 |
break;
|
| 426 |
case "priority":
|
| 427 |
cell = <span className="lp-proj-cell"><PriorityGlyph priority={project.priority} />
|
|
|
|
| 437 |
cell = formatDay(project[key]) || <span className="lp-proj-muted">No date</span>;
|
| 438 |
break;
|
| 439 |
case "status":
|
| 440 |
+
cell = <span className="lp-proj-cell"><StatusRing status={project.status} progress={project.progress} />{project.status}</span>;
|
| 441 |
break;
|
| 442 |
case "results":
|
| 443 |
cell = <ProjectResultPreview projectId={project.id} configs={project.resultConfigs ?? []}
|
|
|
|
| 583 |
<div id={`${id}-summary`} role="tabpanel" aria-labelledby={`${id}-${tab}`} title="Counts follow the other filters">
|
| 584 |
{tab === "Health" ? healthCounts.map(({ label, count }) => <button type="button" className="lp-proj-summary-row" key={label}
|
| 585 |
aria-pressed={health === label} onClick={() => setHealth(health === label ? "" : label)}>
|
| 586 |
+
<span><HealthGlyph health={label === "No health" ? null : label} />{label}</span><span>{count}</span>
|
| 587 |
</button>) : ownerCounts.map(({ value, count }) => <button type="button" className="lp-proj-summary-row" key={ownerKey(value)}
|
| 588 |
aria-pressed={owner === ownerKey(value)} onClick={() => setOwner(owner === ownerKey(value) ? "" : ownerKey(value))}>
|
| 589 |
<Owner name={value} /><span>{count}</span>
|
|
|
|
| 595 |
</section>;
|
| 596 |
}
|
| 597 |
|
| 598 |
+
// ---- One project: the detail reference's dates pair and progress, then notes, SKUs, activity ----
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 599 |
|
| 600 |
function ProjectDetail({ id, canEdit: mayEdit, onChanged, onDeleted, onCancel }: {
|
| 601 |
id: string; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
|
|
|
|
| 607 |
const [busy, setBusy] = useState(false);
|
| 608 |
const [adding, setAdding] = useState(false);
|
| 609 |
const [confirming, setConfirming] = useState(false);
|
| 610 |
+
const [editingDates, setEditingDates] = useState(false);
|
| 611 |
+
const dateEditorId = useId();
|
|
|
|
|
|
|
| 612 |
const formId = useId();
|
| 613 |
+
const startDateRef = useRef<HTMLInputElement>(null);
|
| 614 |
const writing = useRef(false);
|
| 615 |
+
const mainRef = useRef<HTMLDivElement>(null);
|
| 616 |
const loaded = detail !== null;
|
| 617 |
|
| 618 |
useEffect(() => {
|
| 619 |
+
if (loaded) mainRef.current?.querySelector<HTMLElement>("input, h3")?.focus();
|
| 620 |
}, [loaded]);
|
| 621 |
|
| 622 |
+
useEffect(() => {
|
| 623 |
+
if (editingDates) startDateRef.current?.focus();
|
| 624 |
+
}, [editingDates]);
|
| 625 |
+
|
| 626 |
useEffect(() => {
|
| 627 |
const controller = new AbortController();
|
| 628 |
call<Detail>(`/${enc(id)}`, { signal: controller.signal }, "This project did not load.")
|
|
|
|
| 636 |
return () => controller.abort();
|
| 637 |
}, [id, tick]);
|
| 638 |
|
| 639 |
+
const write = useCallback(async (path: string, method: string, body: unknown, after?: () => void) => {
|
|
|
|
| 640 |
if (writing.current || !mayEdit || !detail?.canEdit) return;
|
| 641 |
writing.current = true;
|
| 642 |
setBusy(true);
|
|
|
|
| 648 |
if (method === "PATCH") setDraft(draftOf(project));
|
| 649 |
}
|
| 650 |
setError("");
|
|
|
|
|
|
|
| 651 |
if (after) after(); else setTick((value) => value + 1);
|
| 652 |
onChanged();
|
| 653 |
} catch (reason) {
|
| 654 |
+
setError(messageOf(reason, "The change was not saved."));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 655 |
if (reason instanceof ProjectRequestError && reason.status === 403) {
|
| 656 |
setDetail((current) => current ? { ...current, canEdit: false } : current);
|
| 657 |
onChanged();
|
|
|
|
| 662 |
}
|
| 663 |
}, [detail?.canEdit, mayEdit, onChanged]);
|
| 664 |
|
| 665 |
+
const patch = useCallback((change: ProjectInput) => write(`/${enc(id)}`, "PATCH", change), [id, write]);
|
| 666 |
+
|
| 667 |
+
if (!detail || !draft) return error ? <p className="lp-proj-error" role="alert">{error}</p> : <LoadingRows />;
|
| 668 |
+
const { project, targets, activity } = detail;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 669 |
const canEdit = mayEdit && detail.canEdit;
|
| 670 |
const display = canEdit ? draft : project;
|
| 671 |
const progress = Number.isFinite(display.progress) ? clampPercent(display.progress) : 0;
|
| 672 |
const dirty = JSON.stringify(draft) !== JSON.stringify(draftOf(project));
|
| 673 |
+
const set = <K extends keyof ProjectDraft>(key: K, value: ProjectDraft[K]) =>
|
| 674 |
+
setDraft((current) => current ? { ...current, [key]: value } : current);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 675 |
const save = (event: FormEvent) => {
|
| 676 |
event.preventDefault();
|
| 677 |
if (!canEdit || !dirty) return;
|
| 678 |
const name = draft.name.trim();
|
| 679 |
+
if (!name) { setError("Name the project."); return; }
|
| 680 |
if (!Number.isInteger(draft.progress) || draft.progress < 0 || draft.progress > 100) {
|
| 681 |
+
setError("Enter a whole percent from 0 to 100."); return;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 682 |
}
|
| 683 |
+
const { owner, ownerUsername, ...fields } = draft;
|
| 684 |
+
void patch({ ...fields, name, notes: draft.notes?.trim() || null,
|
| 685 |
+
...(!ownerUsername && owner ? {} : { ownerUsername }) });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 686 |
};
|
| 687 |
|
| 688 |
+
return <form id={formId} className="lp-proj-detail" data-can-edit={canEdit ? "true" : "false"} onSubmit={save}>
|
| 689 |
+
<div className="lp-proj-main" ref={mainRef}>
|
| 690 |
+
{canEdit
|
| 691 |
+
? <input className="lp-proj-title-input" value={draft.name} aria-label="Project name" disabled={busy}
|
| 692 |
+
required maxLength={300} onChange={(event) => set("name", event.target.value)} />
|
| 693 |
+
: <h3 className="lp-proj-title" tabIndex={-1}>{project.name}</h3>}
|
| 694 |
+
|
| 695 |
+
<div className="lp-proj-props" role="group" aria-label="Properties">
|
| 696 |
+
<span className="lp-proj-props-label">Properties</span>
|
| 697 |
+
<div className="lp-proj-props-row">
|
| 698 |
+
<span className="lp-proj-chip"><StatusRing status={display.status} progress={progress} />
|
| 699 |
+
{canEdit
|
| 700 |
+
? <select aria-label="Status" value={draft.status} disabled={busy}
|
| 701 |
+
onChange={(event) => set("status", event.target.value as Status)}>
|
| 702 |
+
{PROJECT_STATUSES.map((value) => <option key={value} value={value}>{value}</option>)}
|
| 703 |
+
</select>
|
| 704 |
+
: <span>{project.status}</span>}
|
| 705 |
+
</span>
|
| 706 |
+
<span className="lp-proj-chip"><PriorityGlyph priority={display.priority} />
|
|
|
|
|
|
|
|
|
|
|
|
|
| 707 |
{canEdit
|
| 708 |
+
? <select aria-label="Priority" value={draft.priority} disabled={busy}
|
|
|
|
| 709 |
onChange={(event) => set("priority", event.target.value as Priority)}>
|
| 710 |
+
{PROJECT_PRIORITIES.map((value) => <option key={value} value={value}>{value}</option>)}
|
| 711 |
</select>
|
| 712 |
: <span>{project.priority}</span>}
|
| 713 |
+
</span>
|
| 714 |
+
<span className="lp-proj-chip">
|
| 715 |
+
{canEdit
|
| 716 |
+
? <OwnerSelect value={draft.ownerUsername} legacy={draft.owner} disabled={busy}
|
| 717 |
+
onChange={value => setDraft(current => current ? { ...current, ownerUsername: value, owner: null } : current)} />
|
| 718 |
+
: <Owner name={project.owner} />}
|
| 719 |
+
</span>
|
| 720 |
+
<span className="lp-proj-chip lp-proj-dates" data-dates="start-target">
|
| 721 |
+
{canEdit && editingDates
|
| 722 |
+
? <span className="lp-proj-date-editor" id={dateEditorId}>
|
| 723 |
+
<input ref={startDateRef} type="date" aria-label="Start date" value={draft.startOn ?? ""} disabled={busy}
|
| 724 |
+
onChange={(event) => set("startOn", event.target.value || null)} />
|
| 725 |
+
<ArrowRight size={14} aria-hidden="true" />
|
| 726 |
+
<input type="date" aria-label="Target date" value={draft.targetOn ?? ""} disabled={busy}
|
| 727 |
+
onChange={(event) => set("targetOn", event.target.value || null)} />
|
| 728 |
+
</span>
|
| 729 |
+
: <span className="lp-proj-date-labels">
|
| 730 |
+
<time dateTime={display.startOn || undefined} title={display.startOn || "No start date"}>
|
| 731 |
+
{formatDay(display.startOn) || "No start"}</time>
|
| 732 |
+
<ArrowRight size={14} aria-hidden="true" />
|
| 733 |
+
<time dateTime={display.targetOn || undefined} title={display.targetOn || "No target date"}>
|
| 734 |
+
{formatDay(display.targetOn) || "No target"}</time>
|
| 735 |
+
</span>}
|
| 736 |
+
{canEdit ? <button type="button" className="lp-proj-date-toggle" disabled={busy}
|
| 737 |
+
aria-expanded={editingDates} aria-controls={editingDates ? dateEditorId : undefined}
|
| 738 |
+
onClick={() => setEditingDates((current) => !current)}>{editingDates ? "Done" : "Edit dates"}</button> : null}
|
| 739 |
+
</span>
|
| 740 |
+
{project.health ? <span className="lp-proj-chip"><HealthGlyph health={project.health} />{project.health}</span> : null}
|
| 741 |
+
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 742 |
</div>
|
| 743 |
|
| 744 |
{error ? <p className="lp-proj-error" role="alert">{error}</p> : null}
|
| 745 |
{detail.cause ? <p className="lp-proj-muted" role="status">{detail.cause}</p> : null}
|
| 746 |
+
|
| 747 |
+
<ProjectResults projectId={id} configs={project.resultConfigs ?? []} completedOn={project.completedOn}
|
| 748 |
+
updatedAt={project.updatedAt} canEdit={canEdit} onSaved={() => { setTick(value => value + 1); onChanged(); }} />
|
| 749 |
+
|
| 750 |
+
<section className="lp-proj-section" aria-label="Notes">
|
| 751 |
+
<h4 className="lp-proj-section-title">Notes</h4>
|
| 752 |
+
{canEdit
|
| 753 |
+
? <textarea className="lp-proj-notes" aria-label="Notes" placeholder="Add notes" value={draft.notes ?? ""}
|
| 754 |
+
rows={4} disabled={busy}
|
| 755 |
+
onChange={(event) => set("notes", event.target.value)} />
|
| 756 |
+
: <p className="lp-proj-notes-text">{project.notes || <span className="lp-proj-muted">No notes</span>}</p>}
|
| 757 |
+
</section>
|
| 758 |
+
|
| 759 |
<section className="lp-proj-section" aria-label="SKUs">
|
| 760 |
<div className="lp-proj-section-head">
|
| 761 |
+
<h4 className="lp-proj-section-title">SKUs <span className="lp-proj-muted">{targets.length}</span></h4>
|
| 762 |
{canEdit && !adding
|
| 763 |
+
? <button type="button" className="lp-proj-btn" disabled={busy} onClick={() => setAdding(true)}>Add SKUs</button>
|
| 764 |
: null}
|
| 765 |
</div>
|
| 766 |
{canEdit && adding
|
|
|
|
| 780 |
: null}
|
| 781 |
</li>)}
|
| 782 |
</ul>
|
| 783 |
+
: <p className="lp-proj-muted">No SKUs yet</p>}
|
| 784 |
</section>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 785 |
</div>
|
| 786 |
|
| 787 |
<aside className="lp-proj-side">
|
| 788 |
+
<section className="lp-proj-card" aria-label="Dates">
|
| 789 |
+
<h4 className="lp-proj-section-title">Dates</h4>
|
| 790 |
+
<dl className="lp-proj-facts">
|
| 791 |
+
<div><dt>Start</dt><dd>{formatDay(display.startOn) || "Not set"}</dd></div>
|
| 792 |
+
<div><dt>Target</dt><dd>{formatDay(display.targetOn) || "Not set"}</dd></div>
|
| 793 |
+
{project.completedOn ? <div><dt>Completed</dt><dd>{formatDay(project.completedOn)}</dd></div> : null}
|
| 794 |
+
</dl>
|
| 795 |
+
</section>
|
| 796 |
+
|
| 797 |
+
<section className="lp-proj-card" aria-label="Activity">
|
| 798 |
+
<h4 className="lp-proj-section-title">Activity</h4>
|
| 799 |
+
{activity.length
|
| 800 |
+
? <ol className="lp-proj-activity">
|
| 801 |
+
{activity.map((entry, index) => <li key={`${entry.at}:${index}`}>
|
| 802 |
+
<span className="lp-proj-activity-dot" aria-hidden="true" />
|
| 803 |
+
<span>{activityLine(entry)} <span className="lp-proj-muted">{formatWhen(entry.at)}</span></span>
|
| 804 |
+
</li>)}
|
| 805 |
+
</ol>
|
| 806 |
+
: <p className="lp-proj-muted">No activity yet</p>}
|
| 807 |
+
</section>
|
| 808 |
+
|
| 809 |
+
<ProjectFooter><div className="lp-proj-save">
|
| 810 |
<button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={busy} onClick={onCancel}>Cancel</button>
|
| 811 |
{canEdit ? <button type="submit" form={formId} className="lp-proj-btn lp-proj-btn-primary" disabled={busy || !dirty}>
|
| 812 |
{busy ? "Saving…" : "Save changes"}
|
| 813 |
</button> : null}
|
| 814 |
{canEdit && dirty ? <button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={busy}
|
| 815 |
+
onClick={() => { setDraft(draftOf(project)); setError(""); }}>Discard changes</button> : null}
|
| 816 |
+
</div></ProjectFooter>
|
| 817 |
{canEdit
|
| 818 |
? <div className="lp-proj-delete">
|
| 819 |
{confirming
|
|
|
|
| 827 |
: null}
|
| 828 |
</aside>
|
| 829 |
</form>;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 830 |
}
|
| 831 |
|
| 832 |
function TargetLine({ target }: { target: Target }) {
|
|
|
|
| 935 |
onCancel: () => void; onForbidden: () => void;
|
| 936 |
}) {
|
| 937 |
const [draft, setDraft] = useState<ProjectDraft>({
|
| 938 |
+
name: "", status: "Planned", priority: "Medium", owner: "", ownerUsername: null, startOn: "", targetOn: "", progress: 0, notes: "",
|
|
|
|
| 939 |
});
|
| 940 |
const formId = useId();
|
| 941 |
const [busy, setBusy] = useState(false);
|
|
|
|
| 973 |
<label className="lp-proj-field lp-proj-field-wide"><span>Project name</span>
|
| 974 |
<input className="lp-proj-input" value={draft.name} onChange={(event) => set("name", event.target.value)}
|
| 975 |
placeholder="Name your project" autoFocus required maxLength={300} disabled={busy} /></label>
|
| 976 |
+
<label className="lp-proj-field"><span>Status</span>
|
| 977 |
+
<select className="lp-proj-input" value={draft.status} disabled={busy} onChange={(event) => set("status", event.target.value as Status)}>
|
| 978 |
+
{PROJECT_STATUSES.map((value) => <option key={value} value={value}>{value}</option>)}
|
| 979 |
+
</select></label>
|
| 980 |
<label className="lp-proj-field"><span>Owner</span>
|
| 981 |
<OwnerSelect value={draft.ownerUsername} disabled={busy} onChange={value => set("ownerUsername", value)} /></label>
|
| 982 |
<label className="lp-proj-field"><span>Priority</span>
|