Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- RELEASES.json +2 -51
- VERSION +1 -1
- api/amazon_relational.py +291 -40
- api/app_routes.py +16 -3
- api/app_templates.py +6 -6
- api/application_registry.py +317 -18
- api/assistant_mirror.py +30 -2
- api/automation_engine.py +42 -43
- api/catalog_store.py +5 -2
- api/deps.py +70 -10
- api/external_openapi.py +5 -3
- api/intelligence_store.py +11 -353
- api/main.py +326 -82
- api/meta_relational.py +20 -1
- api/odoo_relational.py +1 -0
- api/registry_source.py +23 -19
- api/request_timing.py +11 -5
- api/routes_account_health.py +9 -5
- api/routes_aged_stock.py +92 -22
- api/routes_amazon_deletion.py +15 -3
- api/routes_amazon_operator.py +54 -10
- api/routes_competitor_watch.py +22 -5
- api/routes_connectors.py +47 -13
- api/routes_customers.py +79 -16
- api/routes_external.py +197 -41
- api/routes_geo.py +1 -1
- api/routes_grid.py +76 -71
- api/routes_keychain.py +165 -37
- api/routes_nav.py +19 -42
- api/routes_odoo_tables.py +300 -15
- api/routes_payouts.py +53 -20
- api/routes_products.py +23 -7
- api/routes_ranking_diagnostic.py +7 -3
- api/routes_reimbursements.py +16 -7
- api/routes_restock_planner.py +87 -6
- api/routes_returns_repeat.py +7 -5
- api/routes_sheet.py +45 -4
- api/routes_tables.py +281 -33
- api/routes_templates.py +9 -8
- api/tool_groups/data.py +34 -5
- api/tool_groups/sheet.py +27 -0
- platform/ai_client_steps.json +32 -44
- platform/aios_grid.py +9 -0
- platform/aios_grid_fields.json +1 -1
- platform/core/activity_log.py +29 -5
- platform/core/connector_directory.py +1 -0
- platform/core/connector_spec.py +71 -63
- platform/core/connector_sync.py +98 -20
- platform/core/edit_receipts.py +216 -0
- platform/core/external_tools.py +6 -2
RELEASES.json
CHANGED
|
@@ -1,53 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
-
"releases": [
|
| 4 |
-
{
|
| 5 |
-
"version": "v84",
|
| 6 |
-
"sha": "7bb3a2e09",
|
| 7 |
-
"date": "2026-10-24",
|
| 8 |
-
"subject": "release(sdlc78): v84 QA records the staging build change and what v84 excludes"
|
| 9 |
-
},
|
| 10 |
-
{
|
| 11 |
-
"version": "v83",
|
| 12 |
-
"sha": "9b7471968",
|
| 13 |
-
"date": "2026-10-21",
|
| 14 |
-
"subject": "release v83"
|
| 15 |
-
},
|
| 16 |
-
{
|
| 17 |
-
"version": "v82",
|
| 18 |
-
"sha": "9c61b1ee9",
|
| 19 |
-
"date": "2026-10-17",
|
| 20 |
-
"subject": "release v82"
|
| 21 |
-
},
|
| 22 |
-
{
|
| 23 |
-
"version": "v81",
|
| 24 |
-
"sha": "e141ccc85",
|
| 25 |
-
"date": "2026-10-15",
|
| 26 |
-
"subject": "release v81"
|
| 27 |
-
},
|
| 28 |
-
{
|
| 29 |
-
"version": "v80",
|
| 30 |
-
"sha": "869b8fcc6",
|
| 31 |
-
"date": "2026-10-13",
|
| 32 |
-
"subject": "release v80"
|
| 33 |
-
},
|
| 34 |
-
{
|
| 35 |
-
"version": "v79",
|
| 36 |
-
"sha": "406417b8d",
|
| 37 |
-
"date": "2026-10-13",
|
| 38 |
-
"subject": "release v79"
|
| 39 |
-
},
|
| 40 |
-
{
|
| 41 |
-
"version": "v78",
|
| 42 |
-
"sha": "aad6561d6",
|
| 43 |
-
"date": "2026-10-09",
|
| 44 |
-
"subject": "release v78: SDLC 71 with the D-7536 and D-7537 repairs"
|
| 45 |
-
},
|
| 46 |
-
{
|
| 47 |
-
"version": "v77",
|
| 48 |
-
"sha": "e741e5ca6",
|
| 49 |
-
"date": "2026-10-05",
|
| 50 |
-
"subject": "release v77"
|
| 51 |
-
}
|
| 52 |
-
]
|
| 53 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "2990461bb",
|
| 3 |
+
"releases": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
2990461bb
|
api/amazon_relational.py
CHANGED
|
@@ -133,6 +133,113 @@ def source_attribution(key):
|
|
| 133 |
}
|
| 134 |
|
| 135 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
#: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
|
| 137 |
_DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
|
| 138 |
"settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
|
|
@@ -261,12 +368,14 @@ def mirror_columns(cur, table, *, strict=False):
|
|
| 261 |
"""[(name, type)] for a mirror table, or [] when it has never been synced."""
|
| 262 |
if strict:
|
| 263 |
# A successful empty metadata query proves absence on this cursor only.
|
| 264 |
-
return cur.execute(
|
| 265 |
"SELECT column_name, data_type FROM information_schema.columns "
|
| 266 |
"WHERE table_catalog = current_database() AND table_schema = current_schema() "
|
| 267 |
"AND table_name = ? ORDER BY ordinal_position", [table]).fetchall()
|
|
|
|
| 268 |
try:
|
| 269 |
-
return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()
|
|
|
|
| 270 |
except Exception: # noqa: BLE001
|
| 271 |
return []
|
| 272 |
|
|
@@ -322,6 +431,14 @@ def _latest_listings_sql(listing_names):
|
|
| 322 |
if not {"seller_sku", "asin1"} <= set(listing_names):
|
| 323 |
return None
|
| 324 |
table = SOURCE[LISTINGS]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
latest = (f' AND "_snapshot_date" = (SELECT MAX("_snapshot_date") FROM {table})'
|
| 326 |
if "_snapshot_date" in listing_names else "")
|
| 327 |
return ("SELECT DISTINCT sku, asin FROM (SELECT TRIM(CAST(seller_sku AS VARCHAR)) AS sku, "
|
|
@@ -369,6 +486,27 @@ def _parent_lookup_sql(cur):
|
|
| 369 |
JSON is a read-only SQL relation here, with SQL quotes escaped separately from JSON.
|
| 370 |
No temporary table or tenant mirror write is needed, and the complete mapping is retained.
|
| 371 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 372 |
identity = _parent_revision(cur)
|
| 373 |
if identity is None:
|
| 374 |
return amazon_store.asin_parents_sql(cur)
|
|
@@ -427,11 +565,21 @@ def lookups(cur, key, names=None, *, allowed=True):
|
|
| 427 |
"expr": "CAST(NULL AS VARCHAR)", "opens": {"via": via} if via else None,
|
| 428 |
"description": _LOOKUP_DESCRIPTIONS[name]}
|
| 429 |
for name, after, via in keys if name not in names]
|
| 430 |
-
listing_names =
|
| 431 |
parents = None if "parent_asin" in names else _parent_lookup_sql(cur)
|
| 432 |
latest = _latest_listings_sql(listing_names)
|
| 433 |
table = SOURCE[key]
|
| 434 |
out = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 435 |
|
| 436 |
def add(lookup_key, after, expr, via):
|
| 437 |
if lookup_key in names:
|
|
@@ -445,11 +593,11 @@ def lookups(cur, key, names=None, *, allowed=True):
|
|
| 445 |
via = asin_via if "asin1" in listing_names else None
|
| 446 |
if parents:
|
| 447 |
add(LOOKUP_PARENT, asin,
|
| 448 |
-
f"(SELECT _p.parent_asin FROM ({parents}) AS _p WHERE _p.asin = {ref})", via)
|
| 449 |
if latest and not sku:
|
| 450 |
add(LOOKUP_SKU, asin,
|
| 451 |
f"(SELECT string_agg(DISTINCT _l.sku, ', ' ORDER BY _l.sku) "
|
| 452 |
-
f"FROM ({latest}) AS _l WHERE _l.asin = {ref})", via)
|
| 453 |
elif sku and latest:
|
| 454 |
ref = f"TRIM(CAST({table}.{_quoted(sku)} AS VARCHAR))"
|
| 455 |
via = sku_via if "seller_sku" in listing_names else None
|
|
@@ -457,12 +605,14 @@ def lookups(cur, key, names=None, *, allowed=True):
|
|
| 457 |
if not names & _NATIVE_ASIN:
|
| 458 |
add(LOOKUP_ASIN, sku,
|
| 459 |
f"(SELECT string_agg(DISTINCT _l.asin, ', ' ORDER BY _l.asin) "
|
| 460 |
-
f"FROM ({latest}) AS _l WHERE _l.sku = {ref})", via)
|
| 461 |
if parents:
|
| 462 |
add(LOOKUP_PARENT, sku,
|
| 463 |
f"(SELECT string_agg(DISTINCT _p.parent_asin, ', ' ORDER BY _p.parent_asin) "
|
| 464 |
f"FROM ({latest}) AS _l JOIN ({parents}) AS _p ON _p.asin = _l.asin "
|
| 465 |
-
|
|
|
|
|
|
|
| 466 |
return out
|
| 467 |
|
| 468 |
|
|
@@ -494,7 +644,7 @@ def _fields_for(cur, key, *, read_through=False):
|
|
| 494 |
"label": declared.get(name, {}).get("label") or written.get(name) or _label(name),
|
| 495 |
"type": _ftype(name, dtype),
|
| 496 |
"source": "overlay",
|
| 497 |
-
"default": name == pinned or name in shown or name in product,
|
| 498 |
"pinned": name == pinned,
|
| 499 |
**({"format": fmt} if fmt else {}),
|
| 500 |
}, flow="amazon_relational"))
|
|
@@ -514,6 +664,13 @@ def _fields_for(cur, key, *, read_through=False):
|
|
| 514 |
live_columns[target] = {name for name, _dtype in columns}
|
| 515 |
out.extend(_odoo._preset(field, flow="amazon_relational")
|
| 516 |
for field in connector_spec.system_link_fields(key, live_columns).get(key, ()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 517 |
if not read_through and "refreshed" not in names:
|
| 518 |
out.append(_odoo._preset(_odoo._refreshed_field(), flow="amazon_relational"))
|
| 519 |
return out
|
|
@@ -636,17 +793,24 @@ def _integer(value):
|
|
| 636 |
def grid_source(cur, key, *, columns=None, allow_lookups=True):
|
| 637 |
"""One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
|
| 638 |
columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns
|
|
|
|
| 639 |
if not columns:
|
| 640 |
return missing_source(key)
|
| 641 |
names = {name for name, _dtype in columns}
|
| 642 |
table = key.removeprefix(PREFIX)
|
| 643 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 644 |
fallback = sorted(set((*identity, *_fallback_columns(table))))
|
| 645 |
source = _quoted(SOURCE[key])
|
| 646 |
|
| 647 |
def value(name, owner=source):
|
| 648 |
return (f"NULLIF(CAST({owner}.{_quoted(name)} AS VARCHAR), '')"
|
| 649 |
-
if name in
|
| 650 |
|
| 651 |
def digest(values):
|
| 652 |
return f"sha256(to_json(list_value({', '.join(values)})))"
|
|
@@ -655,13 +819,16 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True):
|
|
| 655 |
peer = '"_amazon_identity_peer"'
|
| 656 |
peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in identity)
|
| 657 |
partition = ", ".join(value(name, peer) for name in fallback)
|
| 658 |
-
#
|
| 659 |
occurrence = (
|
| 660 |
-
'(
|
| 661 |
f'(SELECT {peer}.rowid AS "_identity_rowid", '
|
| 662 |
f'row_number() OVER (PARTITION BY {partition} ORDER BY {peer}.rowid) - 1 '
|
| 663 |
f'AS "_identity_occurrence" FROM {source} AS {peer} WHERE NOT ({peer_complete})) '
|
| 664 |
-
|
|
|
|
|
|
|
|
|
|
| 665 |
fallback_digest = digest(["'__amazon_missing_identity__'",
|
| 666 |
*(value(name) for name in fallback), occurrence])
|
| 667 |
digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} "
|
|
@@ -672,6 +839,31 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True):
|
|
| 672 |
# The shared grid, relation and selection contracts all use numeric pids. Thirteen
|
| 673 |
# hex digits retain a stable 52-bit identity that JavaScript represents exactly.
|
| 674 |
ident = f"greatest(1, CAST(('0x' || substr(({digest_sql}), 1, 13)) AS BIGINT))"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 675 |
cols = {}
|
| 676 |
personal = set(amazon_store.personal_columns(table))
|
| 677 |
for name, dtype in columns:
|
|
@@ -767,9 +959,12 @@ def _read(cur, key):
|
|
| 767 |
# expressions the read-through binding uses, in this one SELECT (never per row).
|
| 768 |
derived = [(lookup["key"], lookup["expr"]) for lookup in lookups(cur, key)]
|
| 769 |
extra = "".join(f", {expr}" for _k, expr in derived)
|
|
|
|
|
|
|
|
|
|
| 770 |
rows = cur.execute(f'SELECT {spec["id"]}, {quoted}{extra} FROM {SOURCE[key]} '
|
| 771 |
f'ORDER BY {_quoted(SOURCE[key])}.rowid').fetchall()
|
| 772 |
-
names = cols + [k for k, _expr in derived]
|
| 773 |
out = []
|
| 774 |
for r in rows:
|
| 775 |
row = {}
|
|
@@ -792,6 +987,8 @@ def _read(cur, key):
|
|
| 792 |
clauses.append(f'CAST({_quoted(column)} AS VARCHAR) IN '
|
| 793 |
f'({",".join("?" for _ in values)})')
|
| 794 |
params.extend(values)
|
|
|
|
|
|
|
| 795 |
projection = ", ".join(_quoted(column) for column in columns)
|
| 796 |
found = cur.execute(
|
| 797 |
f'SELECT {target_spec["id"]}, {projection} FROM {SOURCE[target]} '
|
|
@@ -838,13 +1035,20 @@ def fill_link_cells(rows, fields, read_targets, *, max_bytes=None, capacity_refu
|
|
| 838 |
Neither path stores history rows or caps the matching target set.
|
| 839 |
"""
|
| 840 |
edges = link_edges(fields)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 841 |
requests, weights = {}, {}
|
| 842 |
for _field, target, frm, on in edges:
|
| 843 |
values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
|
| 844 |
requests.setdefault(target, {}).setdefault(on, set()).update(values)
|
| 845 |
for row in rows:
|
| 846 |
if row.get(frm) not in (None, ""):
|
| 847 |
-
slot = (target, on, str(row[frm]))
|
| 848 |
weights[slot] = weights.get(slot, 0) + 1
|
| 849 |
indexes = {}
|
| 850 |
remaining = max_bytes
|
|
@@ -858,17 +1062,20 @@ def fill_link_cells(rows, fields, read_targets, *, max_bytes=None, capacity_refu
|
|
| 858 |
value = str(row[column]) if row.get(column) not in (None, "") else None
|
| 859 |
if value not in wanted:
|
| 860 |
continue
|
| 861 |
-
|
|
|
|
|
|
|
|
|
|
| 862 |
if bucket is not None and pid in bucket:
|
| 863 |
continue
|
| 864 |
if remaining is not None:
|
| 865 |
# Charge every output occurrence before allocating its distinct ID.
|
| 866 |
-
cost = (len(str(pid)) + (1 if bucket else 2)) * weights
|
| 867 |
if cost > remaining:
|
| 868 |
raise capacity_refusal()
|
| 869 |
remaining -= cost
|
| 870 |
if bucket is None:
|
| 871 |
-
bucket = indexes[target][column][
|
| 872 |
bucket.add(pid)
|
| 873 |
finally:
|
| 874 |
close = getattr(iterator, "close", None)
|
|
@@ -877,7 +1084,7 @@ def fill_link_cells(rows, fields, read_targets, *, max_bytes=None, capacity_refu
|
|
| 877 |
for field, target, frm, on in edges:
|
| 878 |
for row in rows:
|
| 879 |
row[field["key"]] = ",".join(str(pid) for pid in sorted(
|
| 880 |
-
indexes[target][on].get(str(row.get(frm)), ())))
|
| 881 |
|
| 882 |
|
| 883 |
def available(tenant_key="royal-imports"):
|
|
@@ -994,9 +1201,12 @@ def _lead_with_identity(stored, report, pinned):
|
|
| 994 |
|
| 995 |
|
| 996 |
def _listing_identity_rows(cur):
|
| 997 |
-
"""Capture SKU targets and recoverable numeric IDs from
|
| 998 |
key = PREFIX + "listing_items"
|
| 999 |
names = {name for name, _dtype in mirror_columns(cur, SOURCE[key])}
|
|
|
|
|
|
|
|
|
|
| 1000 |
old = ["_snapshot_date", "sku", "_report_row_number"]
|
| 1001 |
if "_report_document_id" in names:
|
| 1002 |
old.append("_report_document_id")
|
|
@@ -1006,16 +1216,23 @@ def _listing_identity_rows(cur):
|
|
| 1006 |
digest = f"sha256(to_json(list_value({', '.join(values)})))"
|
| 1007 |
legacy = f"CASE WHEN {complete} THEN greatest(1, CAST(('0x' || substr({digest}, 1, 13)) AS BIGINT)) END"
|
| 1008 |
sku = _quoted("sku") if "sku" in names else "NULL::VARCHAR"
|
|
|
|
|
|
|
| 1009 |
spec = grid_source(cur, key)
|
| 1010 |
-
return cur.execute(f'SELECT {sku}, {spec["id"]}, {legacy}
|
|
|
|
| 1011 |
|
| 1012 |
|
| 1013 |
-
def _migrate_listing_identity(rt, current, identities, source_fields, read_through):
|
| 1014 |
-
"""Copy authored listing cells only when
|
| 1015 |
from core import shared_overlay
|
| 1016 |
|
| 1017 |
key = PREFIX + "listing_items"
|
| 1018 |
-
targets, aliases, target_ids = {}, {}, set()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1019 |
|
| 1020 |
def refuse():
|
| 1021 |
raise Refused("Amazon listing SKU migration is ambiguous. Existing cells were retained.")
|
|
@@ -1026,44 +1243,69 @@ def _migrate_listing_identity(rt, current, identities, source_fields, read_throu
|
|
| 1026 |
refuse()
|
| 1027 |
aliases[old] = target
|
| 1028 |
|
| 1029 |
-
for sku, pid, previous in identities:
|
| 1030 |
if sku in (None, ""):
|
| 1031 |
continue
|
| 1032 |
-
|
| 1033 |
-
|
|
|
|
| 1034 |
refuse()
|
| 1035 |
-
targets[
|
|
|
|
| 1036 |
target_ids.add(str(pid))
|
| 1037 |
alias(pid, pid)
|
| 1038 |
-
|
| 1039 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1040 |
|
| 1041 |
authored = {field["key"] for field in current.get("fields", ())
|
| 1042 |
if (field.get("automation") or {}).get("flowId") != "amazon_relational"
|
| 1043 |
or _odoo._ut().user_edited(field)}
|
| 1044 |
stored = current.get("rows") or {}
|
| 1045 |
-
source_names = {*source_fields, "refreshed"}
|
| 1046 |
moved, notes, seen = {}, {}, set()
|
| 1047 |
for old, row in stored.items():
|
| 1048 |
sku = str(row.get("sku") or "")
|
| 1049 |
values = {name: value for name, value in row.items()
|
| 1050 |
-
if name in authored or name not in source_names}
|
| 1051 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1052 |
if values:
|
| 1053 |
refuse()
|
| 1054 |
continue
|
| 1055 |
-
if
|
| 1056 |
refuse()
|
| 1057 |
-
seen.add(
|
| 1058 |
-
target = targets[sku]
|
| 1059 |
alias(old, target)
|
| 1060 |
moved[target] = dict(row)
|
| 1061 |
if read_through and values:
|
| 1062 |
notes[target] = values
|
| 1063 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1064 |
# Read strictly before declaring migration complete. A failed read is never an empty overlay.
|
| 1065 |
captured = dict((rt.get(shared_overlay.bucket(key)) or {}).get("cells") or {})
|
| 1066 |
-
if
|
| 1067 |
values and str(pid) not in aliases for pid, values in captured.items()):
|
| 1068 |
refuse()
|
| 1069 |
for old, target in aliases.items():
|
|
@@ -1093,7 +1335,7 @@ def _migrate_listing_identity(rt, current, identities, source_fields, read_throu
|
|
| 1093 |
if any(delta.values()):
|
| 1094 |
shared_overlay.put_rows(key, delta, st=Writer())
|
| 1095 |
current["rows"] = moved
|
| 1096 |
-
current["amazonListingIdentity"] =
|
| 1097 |
|
| 1098 |
|
| 1099 |
def apply_plan(rt, built, username="amazon", today=None):
|
|
@@ -1119,6 +1361,12 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1119 |
for key, _label, _fields, _rows in plans}
|
| 1120 |
listing_ids = (_listing_identity_rows(cur_mirror)
|
| 1121 |
if PREFIX + "listing_items" in built else ())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1122 |
# S79-T13: each Ads table's description states its history window and any recorded limit.
|
| 1123 |
ads_descriptions = {key: amazon_ads_store.table_description(SOURCE[key], cur_mirror)
|
| 1124 |
for key, *_rest in plans if key in _ADS_SPEC}
|
|
@@ -1187,7 +1435,10 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1187 |
if key == PREFIX + "listing_items":
|
| 1188 |
current = doc.get(key) or {}
|
| 1189 |
_migrate_listing_identity(rt, current, listing_ids, report_order[key],
|
| 1190 |
-
isinstance(rows, dict) and rows.get("readThrough") is True
|
|
|
|
|
|
|
|
|
|
| 1191 |
fields = [field for field in fields
|
| 1192 |
if not (field.get("type") == "link"
|
| 1193 |
and (field.get("automation") or {}).get("flowId") == "amazon_relational")]
|
|
@@ -1261,7 +1512,7 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1261 |
if ads_descriptions.get(key):
|
| 1262 |
doc[key]["description"] = ads_descriptions[key]
|
| 1263 |
if key == PREFIX + "listing_items":
|
| 1264 |
-
doc[key]["amazonListingIdentity"] =
|
| 1265 |
return doc
|
| 1266 |
|
| 1267 |
rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
|
|
|
|
| 133 |
}
|
| 134 |
|
| 135 |
|
| 136 |
+
def credential_table_owner(_key, _rt):
|
| 137 |
+
"""Combined Amazon tables have row owners and never belong to one credential."""
|
| 138 |
+
return None
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
CONNECTION_REVOCATIONS_KEY = "amazon_connection_revocations"
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def record_connection_revocation(rt, connection_id, *, entries_before):
|
| 145 |
+
"""Keep metadata for retrying scoped cleanup after the credential itself has been removed."""
|
| 146 |
+
cid = str(connection_id or "")
|
| 147 |
+
entries = [{key: e[key] for key in ("id", "connection_id", "type", "slot", "label") if key in e}
|
| 148 |
+
for e in entries_before if e.get("type") == "amazon_sp"]
|
| 149 |
+
if not cid or not any(e.get("connection_id") == cid for e in entries):
|
| 150 |
+
raise ValueError("Amazon revocation requires the original connection metadata")
|
| 151 |
+
def save(doc):
|
| 152 |
+
doc[cid] = {"state": "pending", "entries": entries}
|
| 153 |
+
return doc
|
| 154 |
+
rt.update(CONNECTION_REVOCATIONS_KEY, save, flush="sync")
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def pending_connection_revocation(rt, connection_id):
|
| 158 |
+
"""Return retry metadata only while this connection's cleanup remains unfinished."""
|
| 159 |
+
row = (rt.get(CONNECTION_REVOCATIONS_KEY) or {}).get(str(connection_id)) or {}
|
| 160 |
+
return row.get("entries") if row.get("state") == "pending" else None
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def connection_footprint(rt, connection_id, *, entries_before=None):
|
| 164 |
+
"""Disclose this connection's rows without claiming ownership of a combined table."""
|
| 165 |
+
from contextlib import closing
|
| 166 |
+
from core import keychain
|
| 167 |
+
entries = amazon_store._ordered_account_entries(
|
| 168 |
+
entries_before if entries_before is not None else keychain.list_entries(rt))
|
| 169 |
+
cid = str(connection_id or "")
|
| 170 |
+
if not cid or not any(e["connection_id"] == cid for e in entries):
|
| 171 |
+
raise ValueError("Amazon footprint requires an explicit stored connection")
|
| 172 |
+
path = amazon_store._tenant_mirror_path(rt.key, rt)
|
| 173 |
+
out = {"type": "amazon_sp", "connectionId": cid, "action": "purge", "purge": [], "keep": [],
|
| 174 |
+
"rows": 0, "rowsKnown": path.is_file(), "why": "Only this connection's rows are deleted. Shared tables remain."}
|
| 175 |
+
projection = getattr(rt, "get_projection", None)
|
| 176 |
+
definitions = projection("user_tables", drop=("rows",)) if callable(projection) else rt.get("user_tables")
|
| 177 |
+
if definitions is None:
|
| 178 |
+
definitions = {}
|
| 179 |
+
if not isinstance(definitions, dict):
|
| 180 |
+
raise RuntimeError("Amazon database definitions are unreadable")
|
| 181 |
+
if not path.is_file():
|
| 182 |
+
out["purge"] = [{"key": key, "label": defn.get("label") or key, "rows": 0, "rowsKnown": False}
|
| 183 |
+
for key, defn in definitions.items() if key in SOURCE and key not in _ADS_SPEC]
|
| 184 |
+
return out
|
| 185 |
+
import duckdb
|
| 186 |
+
cursor = datastore.ro_cursor() if path.resolve() == Path(datastore.DB_PATH).resolve() else datastore.bounded_connection(
|
| 187 |
+
duckdb.connect(str(path), read_only=True))
|
| 188 |
+
with closing(cursor) as cur:
|
| 189 |
+
bound = amazon_store._read_state(cur, "amzn_account_binding")
|
| 190 |
+
legacy_owner = bound["detail"] if bound else entries[0]["connection_id"]
|
| 191 |
+
for key in SOURCE.keys() - _ADS_SPEC.keys():
|
| 192 |
+
defn = definitions.get(key) or {}
|
| 193 |
+
columns = amazon_store._columns_of(cur, SOURCE[key])
|
| 194 |
+
if not columns and key not in definitions:
|
| 195 |
+
continue
|
| 196 |
+
if not columns:
|
| 197 |
+
count = 0
|
| 198 |
+
elif amazon_schema.CONNECTION_ID in columns:
|
| 199 |
+
count = cur.execute(f'SELECT count(*) FROM {SOURCE[key]} WHERE "_connection_id"=?', [cid]).fetchone()[0]
|
| 200 |
+
else:
|
| 201 |
+
count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0] if cid == legacy_owner else 0
|
| 202 |
+
out["purge"].append({"key": key, "label": defn.get("label") or LABELS.get(key) or key,
|
| 203 |
+
"rows": count, "rowsKnown": True})
|
| 204 |
+
out["rows"] += count
|
| 205 |
+
out["purge"].sort(key=lambda table: table["key"])
|
| 206 |
+
return out
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def purge_connection(rt, connection_id, *, entries_before=None):
|
| 210 |
+
"""Remove this account's stored snapshots and observations while retaining shared definitions."""
|
| 211 |
+
cid = str(connection_id or "")
|
| 212 |
+
if entries_before is None:
|
| 213 |
+
entries_before = pending_connection_revocation(rt, cid)
|
| 214 |
+
if not entries_before:
|
| 215 |
+
raise ValueError("Amazon scoped cleanup requires saved revocation metadata")
|
| 216 |
+
record_connection_revocation(rt, cid, entries_before=entries_before)
|
| 217 |
+
footprint = connection_footprint(rt, cid, entries_before=entries_before)
|
| 218 |
+
from contextlib import closing
|
| 219 |
+
con, _path = amazon_store._connect_tenant_mirror(rt.key, rt)
|
| 220 |
+
with closing(con):
|
| 221 |
+
binding = amazon_store._read_state(con, "amzn_account_binding")
|
| 222 |
+
entries = amazon_store._ordered_account_entries(entries_before)
|
| 223 |
+
legacy_owner = binding["detail"] if binding else entries[0]["connection_id"]
|
| 224 |
+
def sweep(doc):
|
| 225 |
+
for key, table in doc.items():
|
| 226 |
+
if key not in SOURCE or key in _ADS_SPEC or not isinstance(table, dict):
|
| 227 |
+
continue
|
| 228 |
+
rows = table.get("rows") or {}
|
| 229 |
+
for rid, row in list(rows.items()):
|
| 230 |
+
if row.get(amazon_schema.CONNECTION_ID, legacy_owner) == cid:
|
| 231 |
+
rows.pop(rid)
|
| 232 |
+
return doc
|
| 233 |
+
rt.update("user_tables", sweep, flush="sync")
|
| 234 |
+
removed = amazon_store.purge_connection(rt.key, cid, rt=rt, entries_before=entries_before)
|
| 235 |
+
def complete(doc):
|
| 236 |
+
doc[cid] = {"state": "done", "snapshotId": removed["snapshotId"]}
|
| 237 |
+
return doc
|
| 238 |
+
rt.update(CONNECTION_REVOCATIONS_KEY, complete, flush="sync")
|
| 239 |
+
return {"purged": [table["key"] for table in footprint["purge"]], "refused": [], "failed": [],
|
| 240 |
+
"rows": removed["rows"], "rowsKnown": True, "footprint": footprint}
|
| 241 |
+
|
| 242 |
+
|
| 243 |
#: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
|
| 244 |
_DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
|
| 245 |
"settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
|
|
|
|
| 368 |
"""[(name, type)] for a mirror table, or [] when it has never been synced."""
|
| 369 |
if strict:
|
| 370 |
# A successful empty metadata query proves absence on this cursor only.
|
| 371 |
+
return [(name, kind) for name, kind in cur.execute(
|
| 372 |
"SELECT column_name, data_type FROM information_schema.columns "
|
| 373 |
"WHERE table_catalog = current_database() AND table_schema = current_schema() "
|
| 374 |
"AND table_name = ? ORDER BY ordinal_position", [table]).fetchall()
|
| 375 |
+
if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS]
|
| 376 |
try:
|
| 377 |
+
return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()
|
| 378 |
+
if r[1] not in amazon_schema.MIRROR_INTERNAL_COLUMNS]
|
| 379 |
except Exception: # noqa: BLE001
|
| 380 |
return []
|
| 381 |
|
|
|
|
| 431 |
if not {"seller_sku", "asin1"} <= set(listing_names):
|
| 432 |
return None
|
| 433 |
table = SOURCE[LISTINGS]
|
| 434 |
+
if amazon_schema.CONNECTION_ID in listing_names:
|
| 435 |
+
latest = (', max("_snapshot_date") OVER (PARTITION BY "_connection_id") AS latest_date'
|
| 436 |
+
if "_snapshot_date" in listing_names else "")
|
| 437 |
+
date = ', "_snapshot_date"' if latest else ""
|
| 438 |
+
predicate = ' AND "_snapshot_date" IS NOT DISTINCT FROM latest_date' if latest else ""
|
| 439 |
+
return ('SELECT DISTINCT "_connection_id", sku, asin FROM (SELECT "_connection_id", '
|
| 440 |
+
'TRIM(CAST(seller_sku AS VARCHAR)) AS sku, TRIM(CAST(asin1 AS VARCHAR)) AS asin'
|
| 441 |
+
f'{date}{latest} FROM {table}) WHERE sku <> \'\' AND asin <> \'\'{predicate}')
|
| 442 |
latest = (f' AND "_snapshot_date" = (SELECT MAX("_snapshot_date") FROM {table})'
|
| 443 |
if "_snapshot_date" in listing_names else "")
|
| 444 |
return ("SELECT DISTINCT sku, asin FROM (SELECT TRIM(CAST(seller_sku AS VARCHAR)) AS sku, "
|
|
|
|
| 486 |
JSON is a read-only SQL relation here, with SQL quotes escaped separately from JSON.
|
| 487 |
No temporary table or tenant mirror write is needed, and the complete mapping is retained.
|
| 488 |
"""
|
| 489 |
+
sources = {"amzn_" + source: amazon_store._columns_of(cur, "amzn_" + source)
|
| 490 |
+
for source in _PARENT_SOURCES}
|
| 491 |
+
scoped = [table for table, names in sources.items() if amazon_schema.CONNECTION_ID in names]
|
| 492 |
+
if scoped:
|
| 493 |
+
base = amazon_store.asin_parents_sql(cur)
|
| 494 |
+
if base is None:
|
| 495 |
+
return None
|
| 496 |
+
ids = cur.execute(" UNION ".join(f'SELECT DISTINCT "_connection_id" FROM {table}'
|
| 497 |
+
for table in scoped)).fetchall()
|
| 498 |
+
parts = []
|
| 499 |
+
for (cid,) in ids:
|
| 500 |
+
literal = "NULL::VARCHAR" if cid is None else "'" + str(cid).replace("'", "''") + "'"
|
| 501 |
+
ctes = []
|
| 502 |
+
for table, names in sources.items():
|
| 503 |
+
if not names:
|
| 504 |
+
continue
|
| 505 |
+
where = (f'"_connection_id" IS NOT DISTINCT FROM {literal}'
|
| 506 |
+
if amazon_schema.CONNECTION_ID in names else "FALSE")
|
| 507 |
+
ctes.append(f'{table} AS (SELECT * FROM main.{table} WHERE {where})')
|
| 508 |
+
parts.append(f'(WITH {", ".join(ctes)} SELECT {literal} AS "_connection_id", p.* FROM ({base}) p)')
|
| 509 |
+
return " UNION ALL ".join(parts) if parts else None
|
| 510 |
identity = _parent_revision(cur)
|
| 511 |
if identity is None:
|
| 512 |
return amazon_store.asin_parents_sql(cur)
|
|
|
|
| 565 |
"expr": "CAST(NULL AS VARCHAR)", "opens": {"via": via} if via else None,
|
| 566 |
"description": _LOOKUP_DESCRIPTIONS[name]}
|
| 567 |
for name, after, via in keys if name not in names]
|
| 568 |
+
listing_names = amazon_store._columns_of(cur, SOURCE[LISTINGS])
|
| 569 |
parents = None if "parent_asin" in names else _parent_lookup_sql(cur)
|
| 570 |
latest = _latest_listings_sql(listing_names)
|
| 571 |
table = SOURCE[key]
|
| 572 |
out = []
|
| 573 |
+
scoped = amazon_schema.CONNECTION_ID in amazon_store._columns_of(cur, table)
|
| 574 |
+
parent_scoped = any(amazon_schema.CONNECTION_ID in amazon_store._columns_of(cur, "amzn_" + source)
|
| 575 |
+
for source in _PARENT_SOURCES)
|
| 576 |
+
def account_match(alias):
|
| 577 |
+
tagged = parent_scoped if alias == "_p" else amazon_schema.CONNECTION_ID in listing_names
|
| 578 |
+
if scoped and tagged:
|
| 579 |
+
return f' AND {alias}."_connection_id" IS NOT DISTINCT FROM {table}."_connection_id"'
|
| 580 |
+
if scoped:
|
| 581 |
+
return " AND FALSE"
|
| 582 |
+
return f' AND {alias}."_connection_id" IS NULL' if tagged else ""
|
| 583 |
|
| 584 |
def add(lookup_key, after, expr, via):
|
| 585 |
if lookup_key in names:
|
|
|
|
| 593 |
via = asin_via if "asin1" in listing_names else None
|
| 594 |
if parents:
|
| 595 |
add(LOOKUP_PARENT, asin,
|
| 596 |
+
f"(SELECT _p.parent_asin FROM ({parents}) AS _p WHERE _p.asin = {ref}{account_match('_p')})", via)
|
| 597 |
if latest and not sku:
|
| 598 |
add(LOOKUP_SKU, asin,
|
| 599 |
f"(SELECT string_agg(DISTINCT _l.sku, ', ' ORDER BY _l.sku) "
|
| 600 |
+
f"FROM ({latest}) AS _l WHERE _l.asin = {ref}{account_match('_l')})", via)
|
| 601 |
elif sku and latest:
|
| 602 |
ref = f"TRIM(CAST({table}.{_quoted(sku)} AS VARCHAR))"
|
| 603 |
via = sku_via if "seller_sku" in listing_names else None
|
|
|
|
| 605 |
if not names & _NATIVE_ASIN:
|
| 606 |
add(LOOKUP_ASIN, sku,
|
| 607 |
f"(SELECT string_agg(DISTINCT _l.asin, ', ' ORDER BY _l.asin) "
|
| 608 |
+
f"FROM ({latest}) AS _l WHERE _l.sku = {ref}{account_match('_l')})", via)
|
| 609 |
if parents:
|
| 610 |
add(LOOKUP_PARENT, sku,
|
| 611 |
f"(SELECT string_agg(DISTINCT _p.parent_asin, ', ' ORDER BY _p.parent_asin) "
|
| 612 |
f"FROM ({latest}) AS _l JOIN ({parents}) AS _p ON _p.asin = _l.asin "
|
| 613 |
+
+ ('AND _p."_connection_id" IS NOT DISTINCT FROM _l."_connection_id" '
|
| 614 |
+
if parent_scoped and amazon_schema.CONNECTION_ID in listing_names else "")
|
| 615 |
+
+ f"WHERE _l.sku = {ref}{account_match('_l')})", via)
|
| 616 |
return out
|
| 617 |
|
| 618 |
|
|
|
|
| 644 |
"label": declared.get(name, {}).get("label") or written.get(name) or _label(name),
|
| 645 |
"type": _ftype(name, dtype),
|
| 646 |
"source": "overlay",
|
| 647 |
+
"default": name == pinned or name in shown or name in product or name == amazon_schema.STORE,
|
| 648 |
"pinned": name == pinned,
|
| 649 |
**({"format": fmt} if fmt else {}),
|
| 650 |
}, flow="amazon_relational"))
|
|
|
|
| 664 |
live_columns[target] = {name for name, _dtype in columns}
|
| 665 |
out.extend(_odoo._preset(field, flow="amazon_relational")
|
| 666 |
for field in connector_spec.system_link_fields(key, live_columns).get(key, ()))
|
| 667 |
+
if amazon_schema.STORE in names:
|
| 668 |
+
excess = max(0, sum(bool(field.get("default")) for field in out) - 12)
|
| 669 |
+
for field in reversed(out):
|
| 670 |
+
if excess and field.get("default") and field["key"] not in {
|
| 671 |
+
pinned, amazon_schema.STORE, *product, *LOOKUP_KEYS}:
|
| 672 |
+
field["default"] = False
|
| 673 |
+
excess -= 1
|
| 674 |
if not read_through and "refreshed" not in names:
|
| 675 |
out.append(_odoo._preset(_odoo._refreshed_field(), flow="amazon_relational"))
|
| 676 |
return out
|
|
|
|
| 793 |
def grid_source(cur, key, *, columns=None, allow_lookups=True):
|
| 794 |
"""One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
|
| 795 |
columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns
|
| 796 |
+
columns = [(name, kind) for name, kind in columns if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS]
|
| 797 |
if not columns:
|
| 798 |
return missing_source(key)
|
| 799 |
names = {name for name, _dtype in columns}
|
| 800 |
table = key.removeprefix(PREFIX)
|
| 801 |
+
prepared = {row[0] for row in cur.execute(
|
| 802 |
+
"SELECT column_name FROM information_schema.columns WHERE table_catalog = current_database() "
|
| 803 |
+
"AND table_schema = current_schema() AND table_name = ? AND column_name IN (?, ?, ?)",
|
| 804 |
+
[SOURCE[key], amazon_schema.IDENTITY_OCCURRENCE, amazon_schema.IDENTITY_PID,
|
| 805 |
+
amazon_schema.CONNECTION_ID]).fetchall()}
|
| 806 |
+
identity_names = names | prepared
|
| 807 |
+
identity = _identity_columns(table, identity_names)
|
| 808 |
fallback = sorted(set((*identity, *_fallback_columns(table))))
|
| 809 |
source = _quoted(SOURCE[key])
|
| 810 |
|
| 811 |
def value(name, owner=source):
|
| 812 |
return (f"NULLIF(CAST({owner}.{_quoted(name)} AS VARCHAR), '')"
|
| 813 |
+
if name in identity_names else "NULL::VARCHAR")
|
| 814 |
|
| 815 |
def digest(values):
|
| 816 |
return f"sha256(to_json(list_value({', '.join(values)})))"
|
|
|
|
| 819 |
peer = '"_amazon_identity_peer"'
|
| 820 |
peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in identity)
|
| 821 |
partition = ", ".join(value(name, peer) for name in fallback)
|
| 822 |
+
# Unprepared mirrors rank once in a materialized relation, outside the row lookup.
|
| 823 |
occurrence = (
|
| 824 |
+
'(WITH "_amazon_missing_identity" AS MATERIALIZED '
|
| 825 |
f'(SELECT {peer}.rowid AS "_identity_rowid", '
|
| 826 |
f'row_number() OVER (PARTITION BY {partition} ORDER BY {peer}.rowid) - 1 '
|
| 827 |
f'AS "_identity_occurrence" FROM {source} AS {peer} WHERE NOT ({peer_complete})) '
|
| 828 |
+
'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_missing_identity" '
|
| 829 |
+
f'WHERE "_identity_rowid" = {source}.rowid)')
|
| 830 |
+
if amazon_schema.IDENTITY_OCCURRENCE in prepared:
|
| 831 |
+
occurrence = f'CAST({source}.{_quoted(amazon_schema.IDENTITY_OCCURRENCE)} AS VARCHAR)'
|
| 832 |
fallback_digest = digest(["'__amazon_missing_identity__'",
|
| 833 |
*(value(name) for name in fallback), occurrence])
|
| 834 |
digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} "
|
|
|
|
| 839 |
# The shared grid, relation and selection contracts all use numeric pids. Thirteen
|
| 840 |
# hex digits retain a stable 52-bit identity that JavaScript represents exactly.
|
| 841 |
ident = f"greatest(1, CAST(('0x' || substr(({digest_sql}), 1, 13)) AS BIGINT))"
|
| 842 |
+
if amazon_schema.IDENTITY_PID in prepared:
|
| 843 |
+
ident = f'{source}.{_quoted(amazon_schema.IDENTITY_PID)}'
|
| 844 |
+
elif amazon_schema.CONNECTION_ID in prepared:
|
| 845 |
+
legacy_owner = amazon_store.legacy_identity_owner(cur, table)
|
| 846 |
+
if legacy_owner:
|
| 847 |
+
literal = "'" + str(legacy_owner).replace("'", "''") + "'"
|
| 848 |
+
old_identity = _identity_columns(table, identity_names - {amazon_schema.CONNECTION_ID})
|
| 849 |
+
old_fallback = sorted(set((*old_identity, *_fallback_columns(table))))
|
| 850 |
+
old_complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in old_identity)
|
| 851 |
+
old_peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in old_identity)
|
| 852 |
+
old_partition = ", ".join(value(name, peer) for name in old_fallback)
|
| 853 |
+
old_occurrence = (
|
| 854 |
+
'(WITH "_amazon_legacy_identity" AS MATERIALIZED '
|
| 855 |
+
f'(SELECT {peer}.rowid AS "_identity_rowid", '
|
| 856 |
+
f'row_number() OVER (PARTITION BY {old_partition} ORDER BY {peer}.rowid) - 1 '
|
| 857 |
+
f'AS "_identity_occurrence" FROM {source} AS {peer} '
|
| 858 |
+
f'WHERE {peer}."_connection_id"={literal} AND NOT ({old_peer_complete})) '
|
| 859 |
+
'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_legacy_identity" '
|
| 860 |
+
f'WHERE "_identity_rowid" = {source}.rowid)')
|
| 861 |
+
old_missing = digest(["'__amazon_missing_identity__'", *(value(name) for name in old_fallback),
|
| 862 |
+
old_occurrence])
|
| 863 |
+
old_digest = (f"CASE WHEN {old_complete} THEN {digest([value(name) for name in old_identity])} "
|
| 864 |
+
f"ELSE {old_missing} END")
|
| 865 |
+
old_ident = f"greatest(1, CAST(('0x' || substr(({old_digest}), 1, 13)) AS BIGINT))"
|
| 866 |
+
ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
|
| 867 |
cols = {}
|
| 868 |
personal = set(amazon_store.personal_columns(table))
|
| 869 |
for name, dtype in columns:
|
|
|
|
| 959 |
# expressions the read-through binding uses, in this one SELECT (never per row).
|
| 960 |
derived = [(lookup["key"], lookup["expr"]) for lookup in lookups(cur, key)]
|
| 961 |
extra = "".join(f", {expr}" for _k, expr in derived)
|
| 962 |
+
owner_columns = ([amazon_schema.CONNECTION_ID]
|
| 963 |
+
if amazon_schema.CONNECTION_ID in amazon_store._columns_of(cur, SOURCE[key]) else [])
|
| 964 |
+
extra += "".join(f", {_quoted(name)}" for name in owner_columns)
|
| 965 |
rows = cur.execute(f'SELECT {spec["id"]}, {quoted}{extra} FROM {SOURCE[key]} '
|
| 966 |
f'ORDER BY {_quoted(SOURCE[key])}.rowid').fetchall()
|
| 967 |
+
names = cols + [k for k, _expr in derived] + owner_columns
|
| 968 |
out = []
|
| 969 |
for r in rows:
|
| 970 |
row = {}
|
|
|
|
| 987 |
clauses.append(f'CAST({_quoted(column)} AS VARCHAR) IN '
|
| 988 |
f'({",".join("?" for _ in values)})')
|
| 989 |
params.extend(values)
|
| 990 |
+
if amazon_schema.CONNECTION_ID in amazon_store._columns_of(cur, SOURCE[target]):
|
| 991 |
+
columns.append(amazon_schema.CONNECTION_ID)
|
| 992 |
projection = ", ".join(_quoted(column) for column in columns)
|
| 993 |
found = cur.execute(
|
| 994 |
f'SELECT {target_spec["id"]}, {projection} FROM {SOURCE[target]} '
|
|
|
|
| 1035 |
Neither path stores history rows or caps the matching target set.
|
| 1036 |
"""
|
| 1037 |
edges = link_edges(fields)
|
| 1038 |
+
scoped = {target for field, target, _frm, _on in edges
|
| 1039 |
+
if (field.get("link") or {}).get("connectionScope") == amazon_schema.CONNECTION_ID}
|
| 1040 |
+
def match_key(target, row, value):
|
| 1041 |
+
if target not in scoped:
|
| 1042 |
+
return value
|
| 1043 |
+
cid = row.get(amazon_schema.CONNECTION_ID)
|
| 1044 |
+
return (str(cid), value) if cid not in (None, "") else None
|
| 1045 |
requests, weights = {}, {}
|
| 1046 |
for _field, target, frm, on in edges:
|
| 1047 |
values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
|
| 1048 |
requests.setdefault(target, {}).setdefault(on, set()).update(values)
|
| 1049 |
for row in rows:
|
| 1050 |
if row.get(frm) not in (None, ""):
|
| 1051 |
+
slot = (target, on, match_key(target, row, str(row[frm])))
|
| 1052 |
weights[slot] = weights.get(slot, 0) + 1
|
| 1053 |
indexes = {}
|
| 1054 |
remaining = max_bytes
|
|
|
|
| 1062 |
value = str(row[column]) if row.get(column) not in (None, "") else None
|
| 1063 |
if value not in wanted:
|
| 1064 |
continue
|
| 1065 |
+
matched = match_key(target, row, value)
|
| 1066 |
+
if matched is None or not weights.get((target, column, matched)):
|
| 1067 |
+
continue
|
| 1068 |
+
bucket = indexes[target][column].get(matched)
|
| 1069 |
if bucket is not None and pid in bucket:
|
| 1070 |
continue
|
| 1071 |
if remaining is not None:
|
| 1072 |
# Charge every output occurrence before allocating its distinct ID.
|
| 1073 |
+
cost = (len(str(pid)) + (1 if bucket else 2)) * weights.get((target, column, matched), 0)
|
| 1074 |
if cost > remaining:
|
| 1075 |
raise capacity_refusal()
|
| 1076 |
remaining -= cost
|
| 1077 |
if bucket is None:
|
| 1078 |
+
bucket = indexes[target][column][matched] = set()
|
| 1079 |
bucket.add(pid)
|
| 1080 |
finally:
|
| 1081 |
close = getattr(iterator, "close", None)
|
|
|
|
| 1084 |
for field, target, frm, on in edges:
|
| 1085 |
for row in rows:
|
| 1086 |
row[field["key"]] = ",".join(str(pid) for pid in sorted(
|
| 1087 |
+
indexes[target][on].get(match_key(target, row, str(row.get(frm))), ())))
|
| 1088 |
|
| 1089 |
|
| 1090 |
def available(tenant_key="royal-imports"):
|
|
|
|
| 1201 |
|
| 1202 |
|
| 1203 |
def _listing_identity_rows(cur):
|
| 1204 |
+
"""Capture connection/SKU targets and recoverable numeric IDs from earlier listing declarations."""
|
| 1205 |
key = PREFIX + "listing_items"
|
| 1206 |
names = {name for name, _dtype in mirror_columns(cur, SOURCE[key])}
|
| 1207 |
+
owner = (_quoted(amazon_schema.CONNECTION_ID)
|
| 1208 |
+
if amazon_schema.CONNECTION_ID in amazon_store._columns_of(cur, SOURCE[key])
|
| 1209 |
+
else "NULL::VARCHAR")
|
| 1210 |
old = ["_snapshot_date", "sku", "_report_row_number"]
|
| 1211 |
if "_report_document_id" in names:
|
| 1212 |
old.append("_report_document_id")
|
|
|
|
| 1216 |
digest = f"sha256(to_json(list_value({', '.join(values)})))"
|
| 1217 |
legacy = f"CASE WHEN {complete} THEN greatest(1, CAST(('0x' || substr({digest}, 1, 13)) AS BIGINT)) END"
|
| 1218 |
sku = _quoted("sku") if "sku" in names else "NULL::VARCHAR"
|
| 1219 |
+
unscoped = (f"greatest(1, CAST(('0x' || substr(sha256(to_json(list_value(CAST({sku} AS VARCHAR)))), "
|
| 1220 |
+
"1, 13)) AS BIGINT))")
|
| 1221 |
spec = grid_source(cur, key)
|
| 1222 |
+
return cur.execute(f'SELECT {owner}, {sku}, {spec["id"]}, {legacy}, {unscoped} '
|
| 1223 |
+
f'FROM {SOURCE[key]}').fetchall()
|
| 1224 |
|
| 1225 |
|
| 1226 |
+
def _migrate_listing_identity(rt, current, identities, source_fields, read_through, legacy_owner=None):
|
| 1227 |
+
"""Copy authored listing cells only when connection/SKU or retained lineage proves their target."""
|
| 1228 |
from core import shared_overlay
|
| 1229 |
|
| 1230 |
key = PREFIX + "listing_items"
|
| 1231 |
+
targets, aliases, target_ids, by_sku, candidates = {}, {}, set(), {}, {}
|
| 1232 |
+
version = "connection_sku" if any(row[0] not in (None, "") for row in identities) else "sku"
|
| 1233 |
+
previous_version = current.get("amazonListingIdentity")
|
| 1234 |
+
migrating = previous_version != version
|
| 1235 |
+
legacy_owner = str(legacy_owner) if legacy_owner not in (None, "") else None
|
| 1236 |
|
| 1237 |
def refuse():
|
| 1238 |
raise Refused("Amazon listing SKU migration is ambiguous. Existing cells were retained.")
|
|
|
|
| 1243 |
refuse()
|
| 1244 |
aliases[old] = target
|
| 1245 |
|
| 1246 |
+
for owner, sku, pid, previous, unscoped in identities:
|
| 1247 |
if sku in (None, ""):
|
| 1248 |
continue
|
| 1249 |
+
owner = str(owner) if owner not in (None, "") else None
|
| 1250 |
+
identity = (owner, str(sku))
|
| 1251 |
+
if identity in targets or str(pid) in target_ids:
|
| 1252 |
refuse()
|
| 1253 |
+
targets[identity] = str(pid)
|
| 1254 |
+
by_sku.setdefault(str(sku), []).append(identity)
|
| 1255 |
target_ids.add(str(pid))
|
| 1256 |
alias(pid, pid)
|
| 1257 |
+
old_ids = []
|
| 1258 |
+
if migrating and previous_version not in ("sku", "connection_sku") and previous is not None:
|
| 1259 |
+
old_ids.append(previous)
|
| 1260 |
+
if migrating and owner is not None:
|
| 1261 |
+
old_ids.append(unscoped)
|
| 1262 |
+
for old in old_ids:
|
| 1263 |
+
candidates.setdefault(str(old), set()).add(identity)
|
| 1264 |
|
| 1265 |
authored = {field["key"] for field in current.get("fields", ())
|
| 1266 |
if (field.get("automation") or {}).get("flowId") != "amazon_relational"
|
| 1267 |
or _odoo._ut().user_edited(field)}
|
| 1268 |
stored = current.get("rows") or {}
|
| 1269 |
+
source_names = {*source_fields, "refreshed", amazon_schema.CONNECTION_ID}
|
| 1270 |
moved, notes, seen = {}, {}, set()
|
| 1271 |
for old, row in stored.items():
|
| 1272 |
sku = str(row.get("sku") or "")
|
| 1273 |
values = {name: value for name, value in row.items()
|
| 1274 |
+
if name != amazon_schema.CONNECTION_ID and (name in authored or name not in source_names)}
|
| 1275 |
+
owner = row.get(amazon_schema.CONNECTION_ID)
|
| 1276 |
+
if owner not in (None, ""):
|
| 1277 |
+
target = targets.get((str(owner), sku))
|
| 1278 |
+
elif str(old) in target_ids:
|
| 1279 |
+
target = str(old) if any(targets[identity] == str(old) for identity in by_sku.get(sku, ())) else None
|
| 1280 |
+
elif legacy_owner is not None and version == "connection_sku" and previous_version != "connection_sku":
|
| 1281 |
+
target = targets.get((legacy_owner, sku))
|
| 1282 |
+
else:
|
| 1283 |
+
matches = by_sku.get(sku, ())
|
| 1284 |
+
target = targets[matches[0]] if len(matches) == 1 else None
|
| 1285 |
+
if target is None:
|
| 1286 |
if values:
|
| 1287 |
refuse()
|
| 1288 |
continue
|
| 1289 |
+
if target in seen:
|
| 1290 |
refuse()
|
| 1291 |
+
seen.add(target)
|
|
|
|
| 1292 |
alias(old, target)
|
| 1293 |
moved[target] = dict(row)
|
| 1294 |
if read_through and values:
|
| 1295 |
notes[target] = values
|
| 1296 |
|
| 1297 |
+
# Stored ownership is stronger evidence than an old numeric alias shared by two accounts.
|
| 1298 |
+
for old, matches in candidates.items():
|
| 1299 |
+
if old in aliases:
|
| 1300 |
+
continue
|
| 1301 |
+
owned = {identity for identity in matches if identity[0] == legacy_owner} if legacy_owner else set()
|
| 1302 |
+
resolved = owned if legacy_owner is not None and version == "connection_sku" else matches
|
| 1303 |
+
if len(resolved) == 1:
|
| 1304 |
+
alias(old, targets[next(iter(resolved))])
|
| 1305 |
+
|
| 1306 |
# Read strictly before declaring migration complete. A failed read is never an empty overlay.
|
| 1307 |
captured = dict((rt.get(shared_overlay.bucket(key)) or {}).get("cells") or {})
|
| 1308 |
+
if migrating and any(
|
| 1309 |
values and str(pid) not in aliases for pid, values in captured.items()):
|
| 1310 |
refuse()
|
| 1311 |
for old, target in aliases.items():
|
|
|
|
| 1335 |
if any(delta.values()):
|
| 1336 |
shared_overlay.put_rows(key, delta, st=Writer())
|
| 1337 |
current["rows"] = moved
|
| 1338 |
+
current["amazonListingIdentity"] = version
|
| 1339 |
|
| 1340 |
|
| 1341 |
def apply_plan(rt, built, username="amazon", today=None):
|
|
|
|
| 1361 |
for key, _label, _fields, _rows in plans}
|
| 1362 |
listing_ids = (_listing_identity_rows(cur_mirror)
|
| 1363 |
if PREFIX + "listing_items" in built else ())
|
| 1364 |
+
listing_owner = None
|
| 1365 |
+
if PREFIX + "listing_items" in built:
|
| 1366 |
+
from core import keychain
|
| 1367 |
+
binding = amazon_store._read_state(cur_mirror, "amzn_account_binding")
|
| 1368 |
+
listing_owner = ((binding or {}).get("detail")
|
| 1369 |
+
or keychain.default_connection_id(rt, "amazon_sp"))
|
| 1370 |
# S79-T13: each Ads table's description states its history window and any recorded limit.
|
| 1371 |
ads_descriptions = {key: amazon_ads_store.table_description(SOURCE[key], cur_mirror)
|
| 1372 |
for key, *_rest in plans if key in _ADS_SPEC}
|
|
|
|
| 1435 |
if key == PREFIX + "listing_items":
|
| 1436 |
current = doc.get(key) or {}
|
| 1437 |
_migrate_listing_identity(rt, current, listing_ids, report_order[key],
|
| 1438 |
+
isinstance(rows, dict) and rows.get("readThrough") is True,
|
| 1439 |
+
legacy_owner=listing_owner)
|
| 1440 |
+
listing_version = ("connection_sku" if any(row[0] not in (None, "") for row in listing_ids)
|
| 1441 |
+
else "sku")
|
| 1442 |
fields = [field for field in fields
|
| 1443 |
if not (field.get("type") == "link"
|
| 1444 |
and (field.get("automation") or {}).get("flowId") == "amazon_relational")]
|
|
|
|
| 1512 |
if ads_descriptions.get(key):
|
| 1513 |
doc[key]["description"] = ads_descriptions[key]
|
| 1514 |
if key == PREFIX + "listing_items":
|
| 1515 |
+
doc[key]["amazonListingIdentity"] = listing_version
|
| 1516 |
return doc
|
| 1517 |
|
| 1518 |
rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
|
api/app_routes.py
CHANGED
|
@@ -1,14 +1,25 @@
|
|
| 1 |
"""C3: mount every C1 App router by name, recording a failed import instead of raising it."""
|
| 2 |
from __future__ import annotations
|
| 3 |
|
| 4 |
-
import sys
|
| 5 |
from importlib import import_module
|
| 6 |
|
| 7 |
import app_templates
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
|
| 10 |
def _failures_record():
|
| 11 |
-
main =
|
| 12 |
record = getattr(main, "ROUTER_MOUNT_FAILURES", None) if main is not None else None
|
| 13 |
return record if isinstance(record, dict) else app_templates.ROUTER_MOUNT_FAILURES
|
| 14 |
|
|
@@ -44,6 +55,8 @@ def held_dependencies(kind):
|
|
| 44 |
def mount_app_routers(app, failures=None):
|
| 45 |
"""Include each C1 `router` once, with main.py `_mount_router` semantics; returns those mounted."""
|
| 46 |
record = failures if isinstance(failures, dict) else _failures_record()
|
|
|
|
|
|
|
| 47 |
mounted = []
|
| 48 |
for row in app_templates.TEMPLATES:
|
| 49 |
name = row["router"]
|
|
@@ -58,7 +71,7 @@ def mount_app_routers(app, failures=None):
|
|
| 58 |
failure = {"error": type(exc).__name__,
|
| 59 |
"missing_module": exc.name if isinstance(exc, ModuleNotFoundError) else ""}
|
| 60 |
record.setdefault(name, []).append(failure)
|
| 61 |
-
if record is not
|
| 62 |
# The offer predicate reads main.py's record and this one, never a caller's own dict.
|
| 63 |
app_templates.ROUTER_MOUNT_FAILURES.setdefault(name, []).append(failure)
|
| 64 |
# Exception messages can contain configuration values; report the class/module only.
|
|
|
|
| 1 |
"""C3: mount every C1 App router by name, recording a failed import instead of raising it."""
|
| 2 |
from __future__ import annotations
|
| 3 |
|
|
|
|
| 4 |
from importlib import import_module
|
| 5 |
|
| 6 |
import app_templates
|
| 7 |
+
from fastapi import APIRouter, Depends
|
| 8 |
+
from deps import Session, require_session, running_main
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
availability_router = APIRouter(prefix="/api/v1/applications", tags=["applications"])
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@availability_router.get("/{app_id}/availability")
|
| 15 |
+
def app_availability(app_id: str, session: Session = Depends(require_session)):
|
| 16 |
+
"""Read one App's availability through the authenticated tenant and App walls."""
|
| 17 |
+
import application_registry
|
| 18 |
+
return application_registry.opening(session, app_id)
|
| 19 |
|
| 20 |
|
| 21 |
def _failures_record():
|
| 22 |
+
main = running_main()
|
| 23 |
record = getattr(main, "ROUTER_MOUNT_FAILURES", None) if main is not None else None
|
| 24 |
return record if isinstance(record, dict) else app_templates.ROUTER_MOUNT_FAILURES
|
| 25 |
|
|
|
|
| 55 |
def mount_app_routers(app, failures=None):
|
| 56 |
"""Include each C1 `router` once, with main.py `_mount_router` semantics; returns those mounted."""
|
| 57 |
record = failures if isinstance(failures, dict) else _failures_record()
|
| 58 |
+
if not _included(app, availability_router):
|
| 59 |
+
app.include_router(availability_router)
|
| 60 |
mounted = []
|
| 61 |
for row in app_templates.TEMPLATES:
|
| 62 |
name = row["router"]
|
|
|
|
| 71 |
failure = {"error": type(exc).__name__,
|
| 72 |
"missing_module": exc.name if isinstance(exc, ModuleNotFoundError) else ""}
|
| 73 |
record.setdefault(name, []).append(failure)
|
| 74 |
+
if record is not app_templates.ROUTER_MOUNT_FAILURES:
|
| 75 |
# The offer predicate reads main.py's record and this one, never a caller's own dict.
|
| 76 |
app_templates.ROUTER_MOUNT_FAILURES.setdefault(name, []).append(failure)
|
| 77 |
# Exception messages can contain configuration values; report the class/module only.
|
api/app_templates.py
CHANGED
|
@@ -18,6 +18,7 @@ CONNECTOR_ENTRY_TYPES = MappingProxyType({"odoo": "odoo", "amazon": "amazon_sp",
|
|
| 18 |
|
| 19 |
def _amazon(kind, label, purpose, process, sources, *, connector="amazon", proposes=True, tools=()):
|
| 20 |
return {"kind": kind, "label": label, "purpose": purpose, "connector": connector,
|
|
|
|
| 21 |
"sources": tuple(sources), "process": process, "mark": kind,
|
| 22 |
"module": "modules." + kind, "router": "routes_" + kind,
|
| 23 |
"view": kind.replace("_", "-"), "tools": tuple(tools), "proposes": proposes}
|
|
@@ -27,10 +28,10 @@ TEMPLATES = (
|
|
| 27 |
# Routing and Catalog render through Map mode; their routes are mounted by main.py.
|
| 28 |
# They read FIELD_BINDINGS over their source, so an App of either kind stores no field map.
|
| 29 |
{"kind": "route", "label": "Routing", "purpose": "Plan delivery routes for your customers.",
|
| 30 |
-
"connector": "odoo", "sources": ("customer_data",), "process": "None", "mark": "route",
|
| 31 |
"module": None, "router": None, "view": None, "tools": (), "proposes": False},
|
| 32 |
{"kind": "catalog", "label": "Catalog", "purpose": "Build and print product catalogs.",
|
| 33 |
-
"connector": "odoo", "sources": ("product_data",), "process": "None", "mark": "catalog",
|
| 34 |
"module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"),
|
| 35 |
"proposes": False},
|
| 36 |
_amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.",
|
|
@@ -64,13 +65,12 @@ TEMPLATES = (
|
|
| 64 |
BY_KIND = MappingProxyType({row["kind"]: row for row in TEMPLATES})
|
| 65 |
KINDS = tuple(BY_KIND)
|
| 66 |
|
| 67 |
-
#
|
| 68 |
-
|
| 69 |
-
HELD_KINDS = frozenset(row["kind"] for row in TEMPLATES if row["connector"] in ("amazon", "amazon_ads"))
|
| 70 |
|
| 71 |
|
| 72 |
def held(kind_or_app):
|
| 73 |
-
"""True when a template kind, or an App record's `templateKind`,
|
| 74 |
kind = kind_or_app.get("templateKind") if isinstance(kind_or_app, dict) else kind_or_app
|
| 75 |
return isinstance(kind, str) and kind in HELD_KINDS
|
| 76 |
|
|
|
|
| 18 |
|
| 19 |
def _amazon(kind, label, purpose, process, sources, *, connector="amazon", proposes=True, tools=()):
|
| 20 |
return {"kind": kind, "label": label, "purpose": purpose, "connector": connector,
|
| 21 |
+
"tags": ["Amazon Ads" if connector == "amazon_ads" else "Amazon"],
|
| 22 |
"sources": tuple(sources), "process": process, "mark": kind,
|
| 23 |
"module": "modules." + kind, "router": "routes_" + kind,
|
| 24 |
"view": kind.replace("_", "-"), "tools": tuple(tools), "proposes": proposes}
|
|
|
|
| 28 |
# Routing and Catalog render through Map mode; their routes are mounted by main.py.
|
| 29 |
# They read FIELD_BINDINGS over their source, so an App of either kind stores no field map.
|
| 30 |
{"kind": "route", "label": "Routing", "purpose": "Plan delivery routes for your customers.",
|
| 31 |
+
"connector": "odoo", "tags": ["Odoo"], "sources": ("customer_data",), "process": "None", "mark": "route",
|
| 32 |
"module": None, "router": None, "view": None, "tools": (), "proposes": False},
|
| 33 |
{"kind": "catalog", "label": "Catalog", "purpose": "Build and print product catalogs.",
|
| 34 |
+
"connector": "odoo", "tags": ["Odoo"], "sources": ("product_data",), "process": "None", "mark": "catalog",
|
| 35 |
"module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"),
|
| 36 |
"proposes": False},
|
| 37 |
_amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.",
|
|
|
|
| 65 |
BY_KIND = MappingProxyType({row["kind"]: row for row in TEMPLATES})
|
| 66 |
KINDS = tuple(BY_KIND)
|
| 67 |
|
| 68 |
+
# SDLC 80 ships the rebuilt Apps through their existing connector, grant and source walls.
|
| 69 |
+
HELD_KINDS = frozenset()
|
|
|
|
| 70 |
|
| 71 |
|
| 72 |
def held(kind_or_app):
|
| 73 |
+
"""True when a template kind, or an App record's `templateKind`, has a release hold."""
|
| 74 |
kind = kind_or_app.get("templateKind") if isinstance(kind_or_app, dict) else kind_or_app
|
| 75 |
return isinstance(kind, str) and kind in HELD_KINDS
|
| 76 |
|
api/application_registry.py
CHANGED
|
@@ -20,6 +20,7 @@ APPLICATION_TYPES = ("chart", "route", "map", "code")
|
|
| 20 |
# Finance descriptors remain available to the governed fact routes.
|
| 21 |
FINANCE_TEMPLATE_KIND = "finance"
|
| 22 |
_KEY = "applications_v1"
|
|
|
|
| 23 |
#: D-8392 (R58): the most live Apps one account may create. The cap counts per account, never per tenant.
|
| 24 |
_MAX = 500
|
| 25 |
_MAX_DISPLAY_BYTES = 512 * 1024
|
|
@@ -417,11 +418,11 @@ def _stored_doc(raw):
|
|
| 417 |
return doc
|
| 418 |
|
| 419 |
|
| 420 |
-
def _update_document(runtime, mutate):
|
| 421 |
"""Keep every registry write, including migrations, safe for JSONB storage."""
|
| 422 |
def apply(raw):
|
| 423 |
return _stored_doc(mutate(raw))
|
| 424 |
-
return runtime.update(
|
| 425 |
|
| 426 |
|
| 427 |
def _clean_doc(raw):
|
|
@@ -444,15 +445,197 @@ def _clean_doc(raw):
|
|
| 444 |
return doc
|
| 445 |
|
| 446 |
|
| 447 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 448 |
"""The read door: a Routing or Catalog App carries its C1 binding (`app_templates.bound`)."""
|
| 449 |
-
try:
|
| 450 |
-
|
| 451 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 452 |
for key, app in doc["applications"].items()}
|
| 453 |
return doc
|
| 454 |
|
| 455 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 456 |
def _server_origin(origin_door=None, via=None):
|
| 457 |
"""Stamp the authenticated server door without inferring it from a display label."""
|
| 458 |
door = "app" if origin_door is None and via is None else origin_door
|
|
@@ -925,7 +1108,7 @@ def validate_application(payload, tenant, *, semantic_records=None, field_contra
|
|
| 925 |
|
| 926 |
def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
|
| 927 |
is_default=False):
|
| 928 |
-
row=copy.deepcopy(
|
| 929 |
row.setdefault("origin", None)
|
| 930 |
if row.get("type") == "code":
|
| 931 |
# Only the authenticated code door returns the body.
|
|
@@ -944,6 +1127,12 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
|
|
| 944 |
# C2: the client mounts apps/<templateView>/AppView.tsx and fills the two marks.
|
| 945 |
row.update(templateView=catalogue["view"], templateMark=catalogue["mark"],
|
| 946 |
templateConnector=catalogue["connector"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 947 |
if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
|
| 948 |
# A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
|
| 949 |
row["capabilities"]["default"] = False
|
|
@@ -990,6 +1179,9 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 990 |
doc = load(runtime)
|
| 991 |
rows=[]
|
| 992 |
for app in doc["applications"].values():
|
|
|
|
|
|
|
|
|
|
| 993 |
if _app_templates.held(app):
|
| 994 |
continue
|
| 995 |
if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
|
|
@@ -1000,7 +1192,11 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 1000 |
if database not in (app.get("compatibleDatabases") or ()): continue
|
| 1001 |
elif not _listed(app, user): continue
|
| 1002 |
if (not _database_free(app)
|
| 1003 |
-
and not _allowed(app.get("compatibleDatabases"), allowed_databases)):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1004 |
if kind and app.get("type") != kind: continue
|
| 1005 |
if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
|
| 1006 |
default_key = f"{database}::{app.get('type')}" if database else ""
|
|
@@ -1077,17 +1273,22 @@ def _code_audit_config(app):
|
|
| 1077 |
"htmlSha256": hashlib.sha256(html).hexdigest()}}
|
| 1078 |
|
| 1079 |
|
| 1080 |
-
def _update_with_activity(runtime, app_id, user, mutate, *, via=None, on_behalf_of=None
|
|
|
|
| 1081 |
"""Log the final committed App change once, excluding previews and unbound legacy stores."""
|
| 1082 |
from core.user_tables import _activity_writer
|
| 1083 |
append = _activity_writer(runtime, user)
|
| 1084 |
if append is None:
|
| 1085 |
-
return _update_document(runtime, mutate)
|
| 1086 |
app_ids = (app_id,) if isinstance(app_id, str) else tuple(app_id)
|
| 1087 |
pending = {}
|
| 1088 |
|
| 1089 |
def snapshot(doc, identity):
|
| 1090 |
app = copy.deepcopy((doc.get("applications") or {}).get(identity) or {})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1091 |
if app:
|
| 1092 |
if app.get("type") == "code":
|
| 1093 |
app["config"] = _code_audit_config(app)
|
|
@@ -1114,7 +1315,7 @@ def _update_with_activity(runtime, app_id, user, mutate, *, via=None, on_behalf_
|
|
| 1114 |
pending[identity] = changes
|
| 1115 |
return result
|
| 1116 |
|
| 1117 |
-
result = _update_document(runtime, apply)
|
| 1118 |
if result is not False:
|
| 1119 |
for identity, changes in pending.items():
|
| 1120 |
append(kind="app", object={"app": identity, "id": identity}, by=user,
|
|
@@ -1122,8 +1323,59 @@ def _update_with_activity(runtime, app_id, user, mutate, *, via=None, on_behalf_
|
|
| 1122 |
return result
|
| 1123 |
|
| 1124 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1125 |
def create(runtime, tenant, user, payload, *, semantic_records=None, field_contracts=None,
|
| 1126 |
allowed_databases=None, via=None, on_behalf_of=None, origin_door=None):
|
|
|
|
|
|
|
| 1127 |
app = _build_application(
|
| 1128 |
tenant, user, payload, semantic_records=semantic_records, field_contracts=field_contracts,
|
| 1129 |
allowed_databases=allowed_databases, origin_door=origin_door, via=via)
|
|
@@ -1131,14 +1383,16 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, field_contr
|
|
| 1131 |
advice = _validate_chart_rules(
|
| 1132 |
app, semantic_records=semantic_records, field_contracts=field_contracts)
|
| 1133 |
_validate_catalog_rules(app)
|
| 1134 |
-
identity = template_identity(app)
|
| 1135 |
def apply(raw):
|
| 1136 |
doc=_clean_doc(raw)
|
| 1137 |
refusal = capacity_refusal(doc, user)
|
| 1138 |
if refusal: raise ApplicationError(refusal)
|
| 1139 |
# R216 and R247 allow every template to have several independent copies.
|
| 1140 |
doc["applications"][app_id]=app; return doc
|
| 1141 |
-
|
|
|
|
|
|
|
|
|
|
| 1142 |
result = _public(app,user,allowed_databases=allowed_databases)
|
| 1143 |
if app["type"] == "chart":
|
| 1144 |
result["advice"] = advice
|
|
@@ -1194,6 +1448,7 @@ def _ensure_route_seed(runtime, tenant):
|
|
| 1194 |
if not isinstance(routes, list):
|
| 1195 |
raise ApplicationError("route store routes must be a list")
|
| 1196 |
route_count = len(routes)
|
|
|
|
| 1197 |
except Exception as exc:
|
| 1198 |
return {"outcome": "skipped", "reason": f"route read failed: {type(exc).__name__}: {exc}"}
|
| 1199 |
|
|
@@ -1205,12 +1460,15 @@ def _ensure_route_seed(runtime, tenant):
|
|
| 1205 |
if "routeApplication" in migrations:
|
| 1206 |
result = {"outcome": "skipped", "reason": "route Application migration already applied"}
|
| 1207 |
return raw
|
| 1208 |
-
|
|
|
|
|
|
|
|
|
|
| 1209 |
if isinstance(app, dict) and app.get("templateKind") == "route"), None)
|
| 1210 |
app_id = existing.get("id") if existing is not None else None
|
| 1211 |
outcome = "existing" if existing is not None else "no_routes"
|
| 1212 |
if existing is None and route_count:
|
| 1213 |
-
refusal = capacity_refusal(
|
| 1214 |
if refusal:
|
| 1215 |
raise ApplicationError(refusal)
|
| 1216 |
app = _build_application(
|
|
@@ -1285,6 +1543,7 @@ def migrate_c1_bindings(runtime, tenant, *, app_ids=None):
|
|
| 1285 |
scope = None if app_ids is None else set(app_ids)
|
| 1286 |
try:
|
| 1287 |
doc = _clean_doc(_read_strict(runtime, _KEY))
|
|
|
|
| 1288 |
except Exception as exc:
|
| 1289 |
return {"outcome": "skipped", "reason": f"registry read failed: {type(exc).__name__}: {exc}",
|
| 1290 |
"rewritten": [], "kept": []}
|
|
@@ -1310,6 +1569,34 @@ def migrate_c1_bindings(runtime, tenant, *, app_ids=None):
|
|
| 1310 |
result.update(outcome="migrated", rewritten=sorted(planned))
|
| 1311 |
return current
|
| 1312 |
_update_document(runtime, apply)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1313 |
if result["rewritten"]:
|
| 1314 |
try:
|
| 1315 |
from catalog_store import migrate_snapshots
|
|
@@ -1590,7 +1877,8 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, fie
|
|
| 1590 |
app["revision"]=int(app.get("revision") or 0)+1
|
| 1591 |
app["updatedAt"]=write_execution.utcnow().isoformat(); landed.update(app)
|
| 1592 |
return doc
|
| 1593 |
-
_update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of
|
|
|
|
| 1594 |
result = _public(landed,user,is_admin,allowed_databases=allowed_databases,
|
| 1595 |
target_database=target_database)
|
| 1596 |
if landed["type"] == "chart" and not set(patch) <= {"starred"}:
|
|
@@ -1932,7 +2220,16 @@ def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None,
|
|
| 1932 |
for key in list(doc["defaults"]):
|
| 1933 |
if doc["defaults"][key]==app_id: doc["defaults"].pop(key)
|
| 1934 |
return doc
|
| 1935 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1936 |
result = {"deleted": app_id}
|
| 1937 |
# After the registry write: a failed clear is reported, never an undone delete.
|
| 1938 |
cleared, failed = clear_view_references(runtime, app_id, removed.get("compatibleDatabases"))
|
|
@@ -1965,7 +2262,9 @@ def set_default(runtime, database, kind, app_id, user, is_admin=False, *, allowe
|
|
| 1965 |
raise PermissionError("that database is not available")
|
| 1966 |
key=f"{database}::{kind}"
|
| 1967 |
def apply(raw):
|
| 1968 |
-
doc=_clean_doc(raw)
|
|
|
|
|
|
|
| 1969 |
if (not isinstance(app, dict) or not may_edit(app,user,is_admin)
|
| 1970 |
or database not in (app.get("compatibleDatabases") or ())
|
| 1971 |
or app.get("type")!=kind):
|
|
|
|
| 20 |
# Finance descriptors remain available to the governed fact routes.
|
| 21 |
FINANCE_TEMPLATE_KIND = "finance"
|
| 22 |
_KEY = "applications_v1"
|
| 23 |
+
_RECORD_PREFIX = "applications_v1_app/"
|
| 24 |
#: D-8392 (R58): the most live Apps one account may create. The cap counts per account, never per tenant.
|
| 25 |
_MAX = 500
|
| 26 |
_MAX_DISPLAY_BYTES = 512 * 1024
|
|
|
|
| 418 |
return doc
|
| 419 |
|
| 420 |
|
| 421 |
+
def _update_document(runtime, mutate, *, key=_KEY):
|
| 422 |
"""Keep every registry write, including migrations, safe for JSONB storage."""
|
| 423 |
def apply(raw):
|
| 424 |
return _stored_doc(mutate(raw))
|
| 425 |
+
return runtime.update(key, apply)
|
| 426 |
|
| 427 |
|
| 428 |
def _clean_doc(raw):
|
|
|
|
| 445 |
return doc
|
| 446 |
|
| 447 |
|
| 448 |
+
def _app_projection(app):
|
| 449 |
+
row = _app_templates.bound(app)
|
| 450 |
+
if isinstance(row, dict):
|
| 451 |
+
contract = _template_contract(str(row.get("templateKind") or ""))
|
| 452 |
+
if contract and contract.get("view") and not contract.get("fieldRoles"):
|
| 453 |
+
row = dict(row)
|
| 454 |
+
row.setdefault("fieldMap", {})
|
| 455 |
+
return row
|
| 456 |
+
|
| 457 |
+
|
| 458 |
+
def _record_documents(runtime):
|
| 459 |
+
"""Read independent App records through the tenant's existing document doors."""
|
| 460 |
+
listing = getattr(runtime, "list_keys", None)
|
| 461 |
+
if not callable(listing):
|
| 462 |
+
return {} # Legacy stores without document enumeration retain their original format.
|
| 463 |
+
keys = listing(_RECORD_PREFIX)
|
| 464 |
+
reader = getattr(runtime, "get_many", None)
|
| 465 |
+
records = reader(keys) if keys and callable(reader) else None
|
| 466 |
+
if records is None:
|
| 467 |
+
records = {key: _read_strict(runtime, key) for key in keys}
|
| 468 |
+
result = {}
|
| 469 |
+
for key in keys:
|
| 470 |
+
doc = _clean_doc(records.get(key))
|
| 471 |
+
if len(doc["applications"]) > 1:
|
| 472 |
+
raise ApplicationError("an App record must contain one App")
|
| 473 |
+
for identity, app in doc["applications"].items():
|
| 474 |
+
if (not isinstance(app, dict) or app.get("id") != identity
|
| 475 |
+
or app.get("tenant") != runtime.key):
|
| 476 |
+
raise ApplicationError("the App record does not match its tenant")
|
| 477 |
+
result[key] = doc
|
| 478 |
+
return result
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
def _application_key(runtime, app_id):
|
| 482 |
+
for key, doc in _record_documents(runtime).items():
|
| 483 |
+
if app_id in doc["applications"]:
|
| 484 |
+
return key
|
| 485 |
+
return _KEY
|
| 486 |
+
|
| 487 |
+
|
| 488 |
+
def load(runtime, *, strict=False):
|
| 489 |
"""The read door: a Routing or Catalog App carries its C1 binding (`app_templates.bound`)."""
|
| 490 |
+
try:
|
| 491 |
+
doc = _clean_doc(runtime.get(_KEY))
|
| 492 |
+
for record in _record_documents(runtime).values():
|
| 493 |
+
doc["applications"].update(record["applications"])
|
| 494 |
+
except Exception as exc:
|
| 495 |
+
if strict:
|
| 496 |
+
raise
|
| 497 |
+
raise_if_resource_error(exc)
|
| 498 |
+
return blank_doc()
|
| 499 |
+
doc["applications"] = {key: _app_projection(app)
|
| 500 |
for key, app in doc["applications"].items()}
|
| 501 |
return doc
|
| 502 |
|
| 503 |
|
| 504 |
+
_AVAILABILITY_MESSAGES = {
|
| 505 |
+
"router_unavailable": "This App cannot open because its routes are unavailable.",
|
| 506 |
+
"no_grant": "Your access limit prevents this App from reading its source.",
|
| 507 |
+
"not_connected": "Connect the App's connector to open this App.",
|
| 508 |
+
"no_rows": "This App has no source rows yet.",
|
| 509 |
+
"source_unavailable": "This App cannot read its source right now.",
|
| 510 |
+
"held": "This App is not available yet.",
|
| 511 |
+
}
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
def availability_reason(state, connector=None):
|
| 515 |
+
"""The public reason vocabulary contains no exception or configuration text."""
|
| 516 |
+
message = _AVAILABILITY_MESSAGES.get(state)
|
| 517 |
+
if state == "not_connected":
|
| 518 |
+
label = {"amazon": "Amazon", "amazon_ads": "Amazon Ads", "odoo": "Odoo"}.get(connector)
|
| 519 |
+
if label:
|
| 520 |
+
message = f"Connect {label} to open this App."
|
| 521 |
+
return {"state": state, "message": message,
|
| 522 |
+
"lock": "Access limit" if state == "no_grant" else None}
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
def _mount_reason(kind):
|
| 526 |
+
catalogue = _app_templates.BY_KIND.get(str(kind or ""))
|
| 527 |
+
if catalogue and catalogue["router"] in _app_templates.router_failures():
|
| 528 |
+
return availability_reason("router_unavailable")
|
| 529 |
+
return None
|
| 530 |
+
|
| 531 |
+
|
| 532 |
+
def _source_rows_exist(session, sources):
|
| 533 |
+
"""Probe at most one row per declared mirror source in the session's tenant."""
|
| 534 |
+
import contextlib
|
| 535 |
+
import stat
|
| 536 |
+
import duckdb
|
| 537 |
+
from pathlib import Path
|
| 538 |
+
from core import connector_spec
|
| 539 |
+
from harness import amazon_store, datastore
|
| 540 |
+
|
| 541 |
+
specs = [connector_spec.spec_for(source) for source in sources]
|
| 542 |
+
if any(spec is None for spec in specs):
|
| 543 |
+
raise RuntimeError("App source is not a declared mirror table")
|
| 544 |
+
path = amazon_store._tenant_mirror_path(session.tenant, rt=session.runtime)
|
| 545 |
+
try:
|
| 546 |
+
mode = path.stat().st_mode
|
| 547 |
+
except FileNotFoundError:
|
| 548 |
+
return False
|
| 549 |
+
if not stat.S_ISREG(mode):
|
| 550 |
+
raise RuntimeError("App mirror is not a database file")
|
| 551 |
+
shared = path.resolve() == Path(datastore.DB_PATH).resolve()
|
| 552 |
+
cursor = (datastore.ro_cursor() if shared else
|
| 553 |
+
datastore.bounded_connection(duckdb.connect(str(path), read_only=True)))
|
| 554 |
+
with contextlib.closing(cursor):
|
| 555 |
+
for spec in specs:
|
| 556 |
+
present = cursor.execute(
|
| 557 |
+
"SELECT 1 FROM information_schema.tables WHERE table_schema = 'main' "
|
| 558 |
+
"AND table_name = ? LIMIT 1", [spec.source]).fetchone()
|
| 559 |
+
if present:
|
| 560 |
+
table = '"' + spec.source.replace('"', '""') + '"'
|
| 561 |
+
if cursor.execute(f"SELECT 1 FROM {table} LIMIT 1").fetchone():
|
| 562 |
+
return True
|
| 563 |
+
return False
|
| 564 |
+
|
| 565 |
+
|
| 566 |
+
def _readable_template_app(session, app_id):
|
| 567 |
+
"""Resolve App identity before reading any source or exposing a refusal."""
|
| 568 |
+
from deps import err
|
| 569 |
+
if (session.tenant != getattr(session.runtime, "key", None)
|
| 570 |
+
or session.tenant != getattr(getattr(session.runtime, "tenant", None), "key", None)):
|
| 571 |
+
raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
|
| 572 |
+
try:
|
| 573 |
+
app = load(session.runtime, strict=True)["applications"].get(app_id)
|
| 574 |
+
except Exception:
|
| 575 |
+
raise err(503, "app_unavailable", "This App cannot be read right now.") from None
|
| 576 |
+
if (not isinstance(app, dict) or app.get("id") != app_id or app.get("tenant") != session.tenant
|
| 577 |
+
or app.get("retired") is True or not may_read(app, session.uname, session.admin)):
|
| 578 |
+
raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
|
| 579 |
+
kind = app.get("templateKind")
|
| 580 |
+
catalogue = _app_templates.BY_KIND.get(str(kind or ""))
|
| 581 |
+
if not catalogue or not catalogue["view"]:
|
| 582 |
+
raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
|
| 583 |
+
return app, catalogue
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def _availability_for_app(session, app, catalogue):
|
| 587 |
+
from core import perm_scope
|
| 588 |
+
from routes_connectors import _entry_types
|
| 589 |
+
|
| 590 |
+
kind = app["templateKind"]
|
| 591 |
+
if _app_templates.held(kind):
|
| 592 |
+
return availability_reason("held")
|
| 593 |
+
mount = _mount_reason(kind)
|
| 594 |
+
if mount:
|
| 595 |
+
return mount
|
| 596 |
+
if not _app_templates.seams_ready(kind):
|
| 597 |
+
return availability_reason("router_unavailable")
|
| 598 |
+
try:
|
| 599 |
+
sources = catalogue["sources"]
|
| 600 |
+
for source in sources:
|
| 601 |
+
if not perm_scope.may_read(session.user, source, st=session.runtime):
|
| 602 |
+
return availability_reason("no_grant")
|
| 603 |
+
# These App readers cannot apply a row or field wall, so refuse before probing.
|
| 604 |
+
if perm_scope.wall_declared(session.user, source, st=session.runtime):
|
| 605 |
+
return availability_reason("no_grant")
|
| 606 |
+
types, _scopes, failure = _entry_types(session.runtime, session.uname)
|
| 607 |
+
if failure is not None:
|
| 608 |
+
return availability_reason("source_unavailable")
|
| 609 |
+
connected = _app_templates.entry_type(catalogue["connector"]) in set(types or ())
|
| 610 |
+
if not connected and catalogue["connector"] == "odoo":
|
| 611 |
+
from routes_keychain import env_odoo_available
|
| 612 |
+
connected = env_odoo_available(session.runtime)
|
| 613 |
+
if not connected:
|
| 614 |
+
return availability_reason("not_connected", catalogue["connector"])
|
| 615 |
+
if sources and not _source_rows_exist(session, sources):
|
| 616 |
+
return availability_reason("no_rows")
|
| 617 |
+
except Exception:
|
| 618 |
+
return availability_reason("source_unavailable")
|
| 619 |
+
return availability_reason("ready")
|
| 620 |
+
|
| 621 |
+
|
| 622 |
+
def availability(session, app_id):
|
| 623 |
+
"""Resolve a named App without weakening its tenant, sharing or source permissions."""
|
| 624 |
+
app, catalogue = _readable_template_app(session, app_id)
|
| 625 |
+
return _availability_for_app(session, app, catalogue)
|
| 626 |
+
|
| 627 |
+
|
| 628 |
+
def opening(session, app_id):
|
| 629 |
+
"""Return only frame metadata and availability for a standalone template App."""
|
| 630 |
+
app, catalogue = _readable_template_app(session, app_id)
|
| 631 |
+
return {"application": {
|
| 632 |
+
"id": app["id"], "name": app["name"], "templateKind": catalogue["kind"],
|
| 633 |
+
"templateView": catalogue["view"], "templateMark": catalogue["mark"],
|
| 634 |
+
"templateConnector": catalogue["connector"], "fieldMap": {},
|
| 635 |
+
"capabilities": {"read": True, "edit": may_edit(app, session.uname, session.admin)},
|
| 636 |
+
}, "availability": _availability_for_app(session, app, catalogue)}
|
| 637 |
+
|
| 638 |
+
|
| 639 |
def _server_origin(origin_door=None, via=None):
|
| 640 |
"""Stamp the authenticated server door without inferring it from a display label."""
|
| 641 |
door = "app" if origin_door is None and via is None else origin_door
|
|
|
|
| 1108 |
|
| 1109 |
def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
|
| 1110 |
is_default=False):
|
| 1111 |
+
row=copy.deepcopy(_app_projection(app))
|
| 1112 |
row.setdefault("origin", None)
|
| 1113 |
if row.get("type") == "code":
|
| 1114 |
# Only the authenticated code door returns the body.
|
|
|
|
| 1127 |
# C2: the client mounts apps/<templateView>/AppView.tsx and fills the two marks.
|
| 1128 |
row.update(templateView=catalogue["view"], templateMark=catalogue["mark"],
|
| 1129 |
templateConnector=catalogue["connector"])
|
| 1130 |
+
row["deepLink"] = f"/app/template/{row['id']}"
|
| 1131 |
+
mount = _mount_reason(row.get("templateKind"))
|
| 1132 |
+
if mount:
|
| 1133 |
+
row["availability"] = mount
|
| 1134 |
+
elif not _database_free(app) and not compatible:
|
| 1135 |
+
row["availability"] = availability_reason("no_grant")
|
| 1136 |
if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
|
| 1137 |
# A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
|
| 1138 |
row["capabilities"]["default"] = False
|
|
|
|
| 1179 |
doc = load(runtime)
|
| 1180 |
rows=[]
|
| 1181 |
for app in doc["applications"].values():
|
| 1182 |
+
if (isinstance(app, dict) and getattr(runtime, "key", None)
|
| 1183 |
+
and app.get("tenant") != runtime.key):
|
| 1184 |
+
continue
|
| 1185 |
if _app_templates.held(app):
|
| 1186 |
continue
|
| 1187 |
if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
|
|
|
|
| 1192 |
if database not in (app.get("compatibleDatabases") or ()): continue
|
| 1193 |
elif not _listed(app, user): continue
|
| 1194 |
if (not _database_free(app)
|
| 1195 |
+
and not _allowed(app.get("compatibleDatabases"), allowed_databases)):
|
| 1196 |
+
catalogue = _app_templates.BY_KIND.get(str(app.get("templateKind") or ""))
|
| 1197 |
+
# An owned or shared template may explain a lost source grant without serving data.
|
| 1198 |
+
if database or not catalogue or not catalogue["view"]:
|
| 1199 |
+
continue
|
| 1200 |
if kind and app.get("type") != kind: continue
|
| 1201 |
if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
|
| 1202 |
default_key = f"{database}::{app.get('type')}" if database else ""
|
|
|
|
| 1273 |
"htmlSha256": hashlib.sha256(html).hexdigest()}}
|
| 1274 |
|
| 1275 |
|
| 1276 |
+
def _update_with_activity(runtime, app_id, user, mutate, *, via=None, on_behalf_of=None,
|
| 1277 |
+
document_key=_KEY):
|
| 1278 |
"""Log the final committed App change once, excluding previews and unbound legacy stores."""
|
| 1279 |
from core.user_tables import _activity_writer
|
| 1280 |
append = _activity_writer(runtime, user)
|
| 1281 |
if append is None:
|
| 1282 |
+
return _update_document(runtime, mutate, key=document_key)
|
| 1283 |
app_ids = (app_id,) if isinstance(app_id, str) else tuple(app_id)
|
| 1284 |
pending = {}
|
| 1285 |
|
| 1286 |
def snapshot(doc, identity):
|
| 1287 |
app = copy.deepcopy((doc.get("applications") or {}).get(identity) or {})
|
| 1288 |
+
if not app and document_key == _KEY:
|
| 1289 |
+
key = _application_key(runtime, identity)
|
| 1290 |
+
if key != _KEY:
|
| 1291 |
+
app = _clean_doc(_read_strict(runtime, key))["applications"].get(identity, {})
|
| 1292 |
if app:
|
| 1293 |
if app.get("type") == "code":
|
| 1294 |
app["config"] = _code_audit_config(app)
|
|
|
|
| 1315 |
pending[identity] = changes
|
| 1316 |
return result
|
| 1317 |
|
| 1318 |
+
result = _update_document(runtime, apply, key=document_key)
|
| 1319 |
if result is not False:
|
| 1320 |
for identity, changes in pending.items():
|
| 1321 |
append(kind="app", object={"app": identity, "id": identity}, by=user,
|
|
|
|
| 1323 |
return result
|
| 1324 |
|
| 1325 |
|
| 1326 |
+
class _AppSlotOccupied(Exception):
|
| 1327 |
+
pass
|
| 1328 |
+
|
| 1329 |
+
|
| 1330 |
+
def _create_record(runtime, app, user, *, via=None, on_behalf_of=None):
|
| 1331 |
+
"""Claim one independently locked App slot without a registry or index write."""
|
| 1332 |
+
legacy = _clean_doc(_read_strict(runtime, _KEY))
|
| 1333 |
+
records = _record_documents(runtime)
|
| 1334 |
+
combined = _clean_doc(legacy)
|
| 1335 |
+
for record in records.values():
|
| 1336 |
+
combined["applications"].update(record["applications"])
|
| 1337 |
+
refusal = capacity_refusal(combined, user)
|
| 1338 |
+
if refusal:
|
| 1339 |
+
raise ApplicationError(refusal)
|
| 1340 |
+
owner = hashlib.sha256(user.encode("utf-8")).hexdigest()
|
| 1341 |
+
prefix = f"{_RECORD_PREFIX}{owner}/"
|
| 1342 |
+
occupied = {int(key[len(prefix):]) for key, doc in records.items()
|
| 1343 |
+
if key.startswith(prefix) and key[len(prefix):].isdigit() and doc["applications"]}
|
| 1344 |
+
legacy_count = sum(isinstance(row, dict) and row.get("createdBy") == user
|
| 1345 |
+
and row.get("retired") is not True for row in legacy["applications"].values())
|
| 1346 |
+
# Existing slots above a lowered capacity still consume that account's allowance.
|
| 1347 |
+
limit = account_capacity() - legacy_count
|
| 1348 |
+
while True:
|
| 1349 |
+
reduced = account_capacity() - legacy_count - sum(slot >= limit for slot in occupied)
|
| 1350 |
+
if reduced == limit:
|
| 1351 |
+
break
|
| 1352 |
+
limit = reduced
|
| 1353 |
+
slots = [slot for slot in range(max(0, limit)) if slot not in occupied]
|
| 1354 |
+
# Different creates start at different records, including when their reads race.
|
| 1355 |
+
slots.sort(key=lambda slot: hashlib.sha256(f"{app['id']}:{slot}".encode()).digest())
|
| 1356 |
+
for slot in slots:
|
| 1357 |
+
def apply(raw):
|
| 1358 |
+
doc = _clean_doc(raw)
|
| 1359 |
+
if doc["applications"]:
|
| 1360 |
+
raise _AppSlotOccupied()
|
| 1361 |
+
doc["applications"][app["id"]] = app
|
| 1362 |
+
return doc
|
| 1363 |
+
try:
|
| 1364 |
+
saved = _update_with_activity(runtime, app["id"], user, apply, via=via,
|
| 1365 |
+
on_behalf_of=on_behalf_of, document_key=prefix + str(slot))
|
| 1366 |
+
except _AppSlotOccupied:
|
| 1367 |
+
continue
|
| 1368 |
+
if saved is False:
|
| 1369 |
+
raise ApplicationError("the App was not saved")
|
| 1370 |
+
return
|
| 1371 |
+
cap = account_capacity()
|
| 1372 |
+
raise ApplicationError(f"Your account can hold up to {cap} App{'' if cap == 1 else 's'}. Delete an App to add another.")
|
| 1373 |
+
|
| 1374 |
+
|
| 1375 |
def create(runtime, tenant, user, payload, *, semantic_records=None, field_contracts=None,
|
| 1376 |
allowed_databases=None, via=None, on_behalf_of=None, origin_door=None):
|
| 1377 |
+
if getattr(runtime, "key", tenant) != tenant or getattr(getattr(runtime, "tenant", None), "key", tenant) != tenant:
|
| 1378 |
+
raise PermissionError("an Application cannot be created for another tenant")
|
| 1379 |
app = _build_application(
|
| 1380 |
tenant, user, payload, semantic_records=semantic_records, field_contracts=field_contracts,
|
| 1381 |
allowed_databases=allowed_databases, origin_door=origin_door, via=via)
|
|
|
|
| 1383 |
advice = _validate_chart_rules(
|
| 1384 |
app, semantic_records=semantic_records, field_contracts=field_contracts)
|
| 1385 |
_validate_catalog_rules(app)
|
|
|
|
| 1386 |
def apply(raw):
|
| 1387 |
doc=_clean_doc(raw)
|
| 1388 |
refusal = capacity_refusal(doc, user)
|
| 1389 |
if refusal: raise ApplicationError(refusal)
|
| 1390 |
# R216 and R247 allow every template to have several independent copies.
|
| 1391 |
doc["applications"][app_id]=app; return doc
|
| 1392 |
+
if callable(getattr(runtime, "list_keys", None)):
|
| 1393 |
+
_create_record(runtime, app, user, via=via, on_behalf_of=on_behalf_of)
|
| 1394 |
+
else:
|
| 1395 |
+
_update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of)
|
| 1396 |
result = _public(app,user,allowed_databases=allowed_databases)
|
| 1397 |
if app["type"] == "chart":
|
| 1398 |
result["advice"] = advice
|
|
|
|
| 1448 |
if not isinstance(routes, list):
|
| 1449 |
raise ApplicationError("route store routes must be a list")
|
| 1450 |
route_count = len(routes)
|
| 1451 |
+
records = _record_documents(runtime)
|
| 1452 |
except Exception as exc:
|
| 1453 |
return {"outcome": "skipped", "reason": f"route read failed: {type(exc).__name__}: {exc}"}
|
| 1454 |
|
|
|
|
| 1460 |
if "routeApplication" in migrations:
|
| 1461 |
result = {"outcome": "skipped", "reason": "route Application migration already applied"}
|
| 1462 |
return raw
|
| 1463 |
+
applications = dict(doc["applications"])
|
| 1464 |
+
for record in records.values():
|
| 1465 |
+
applications.update(record["applications"])
|
| 1466 |
+
existing = next((app for app in applications.values()
|
| 1467 |
if isinstance(app, dict) and app.get("templateKind") == "route"), None)
|
| 1468 |
app_id = existing.get("id") if existing is not None else None
|
| 1469 |
outcome = "existing" if existing is not None else "no_routes"
|
| 1470 |
if existing is None and route_count:
|
| 1471 |
+
refusal = capacity_refusal({"applications": applications}, "loopable")
|
| 1472 |
if refusal:
|
| 1473 |
raise ApplicationError(refusal)
|
| 1474 |
app = _build_application(
|
|
|
|
| 1543 |
scope = None if app_ids is None else set(app_ids)
|
| 1544 |
try:
|
| 1545 |
doc = _clean_doc(_read_strict(runtime, _KEY))
|
| 1546 |
+
records = _record_documents(runtime)
|
| 1547 |
except Exception as exc:
|
| 1548 |
return {"outcome": "skipped", "reason": f"registry read failed: {type(exc).__name__}: {exc}",
|
| 1549 |
"rewritten": [], "kept": []}
|
|
|
|
| 1569 |
result.update(outcome="migrated", rewritten=sorted(planned))
|
| 1570 |
return current
|
| 1571 |
_update_document(runtime, apply)
|
| 1572 |
+
for key, record in records.items():
|
| 1573 |
+
planned, held = _c1_plan(record, scope)
|
| 1574 |
+
result["kept"].extend(held)
|
| 1575 |
+
if not planned:
|
| 1576 |
+
continue
|
| 1577 |
+
rewritten = []
|
| 1578 |
+
|
| 1579 |
+
def apply_record(raw):
|
| 1580 |
+
rewritten.clear()
|
| 1581 |
+
current = _clean_doc(raw)
|
| 1582 |
+
fresh, _held = _c1_plan(current, scope)
|
| 1583 |
+
if not fresh:
|
| 1584 |
+
return raw
|
| 1585 |
+
marker = current.setdefault("migrations", {}).setdefault(
|
| 1586 |
+
C1_BINDINGS_MARKER, {"rewritten": {}, "kept": []})
|
| 1587 |
+
for identity in fresh:
|
| 1588 |
+
before = current["applications"][identity]
|
| 1589 |
+
marker["rewritten"][identity] = {field: copy.deepcopy(before[field]) for field in
|
| 1590 |
+
("sourceDatabase", "fieldMap", "compatibleDatabases")
|
| 1591 |
+
if field in before}
|
| 1592 |
+
marker["at"] = write_execution.utcnow().isoformat()
|
| 1593 |
+
current["applications"].update(fresh)
|
| 1594 |
+
rewritten.extend(sorted(fresh))
|
| 1595 |
+
return current
|
| 1596 |
+
_update_document(runtime, apply_record, key=key)
|
| 1597 |
+
if rewritten:
|
| 1598 |
+
result["rewritten"].extend(rewritten)
|
| 1599 |
+
result["outcome"] = "migrated"
|
| 1600 |
if result["rewritten"]:
|
| 1601 |
try:
|
| 1602 |
from catalog_store import migrate_snapshots
|
|
|
|
| 1877 |
app["revision"]=int(app.get("revision") or 0)+1
|
| 1878 |
app["updatedAt"]=write_execution.utcnow().isoformat(); landed.update(app)
|
| 1879 |
return doc
|
| 1880 |
+
_update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of,
|
| 1881 |
+
document_key=_application_key(runtime, app_id))
|
| 1882 |
result = _public(landed,user,is_admin,allowed_databases=allowed_databases,
|
| 1883 |
target_database=target_database)
|
| 1884 |
if landed["type"] == "chart" and not set(patch) <= {"starred"}:
|
|
|
|
| 2220 |
for key in list(doc["defaults"]):
|
| 2221 |
if doc["defaults"][key]==app_id: doc["defaults"].pop(key)
|
| 2222 |
return doc
|
| 2223 |
+
document_key = _application_key(runtime, app_id)
|
| 2224 |
+
_update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of,
|
| 2225 |
+
document_key=document_key)
|
| 2226 |
+
if document_key != _KEY:
|
| 2227 |
+
def clear_defaults(raw):
|
| 2228 |
+
doc = _clean_doc(raw)
|
| 2229 |
+
doc["defaults"] = {key: value for key, value in doc["defaults"].items() if value != app_id}
|
| 2230 |
+
return doc
|
| 2231 |
+
if app_id in _clean_doc(_read_strict(runtime, _KEY))["defaults"].values():
|
| 2232 |
+
_update_document(runtime, clear_defaults)
|
| 2233 |
result = {"deleted": app_id}
|
| 2234 |
# After the registry write: a failed clear is reported, never an undone delete.
|
| 2235 |
cleared, failed = clear_view_references(runtime, app_id, removed.get("compatibleDatabases"))
|
|
|
|
| 2262 |
raise PermissionError("that database is not available")
|
| 2263 |
key=f"{database}::{kind}"
|
| 2264 |
def apply(raw):
|
| 2265 |
+
doc=_clean_doc(raw)
|
| 2266 |
+
record_key = _application_key(runtime, app_id)
|
| 2267 |
+
app = (doc if record_key == _KEY else _clean_doc(_read_strict(runtime, record_key)))["applications"].get(app_id)
|
| 2268 |
if (not isinstance(app, dict) or not may_edit(app,user,is_admin)
|
| 2269 |
or database not in (app.get("compatibleDatabases") or ())
|
| 2270 |
or app.get("type")!=kind):
|
api/assistant_mirror.py
CHANGED
|
@@ -441,7 +441,7 @@ def _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys, *, table_
|
|
| 441 |
|
| 442 |
|
| 443 |
def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None,
|
| 444 |
-
filter_keys=None, defn=None):
|
| 445 |
"""Every permitted row of a mirror-served grid, or a refusal that names its cause.
|
| 446 |
|
| 447 |
Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
|
|
@@ -516,6 +516,8 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t
|
|
| 516 |
stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
|
| 517 |
defn=defn if isinstance(defn, dict) else None)
|
| 518 |
rows = rot._pool_from_window(spec, win, stamp=stamp)
|
|
|
|
|
|
|
| 519 |
rows.sort(key=lambda row: row["pid"])
|
| 520 |
return rows, stamp, int(win["total"])
|
| 521 |
|
|
@@ -543,6 +545,30 @@ def _json_value(rot, value):
|
|
| 543 |
return value
|
| 544 |
|
| 545 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 546 |
def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None,
|
| 547 |
filter_tree=None, filter_keys=None):
|
| 548 |
"""Server-side aggregates over EVERY permitted row, in SQL. S67-T34 (D-616, D-2631).
|
|
@@ -606,7 +632,7 @@ def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None,
|
|
| 606 |
|
| 607 |
|
| 608 |
def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree=None,
|
| 609 |
-
filter_tree=None, filter_keys=None, defn=None, sorts=None):
|
| 610 |
"""Return one SQL-ordered permitted page, its source stamp and its full population count."""
|
| 611 |
from harness import datastore
|
| 612 |
rot = _rot()
|
|
@@ -639,4 +665,6 @@ def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree
|
|
| 639 |
stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
|
| 640 |
defn=defn if isinstance(defn, dict) else None)
|
| 641 |
rows = rot._pool_from_window(spec, win, stamp=stamp)
|
|
|
|
|
|
|
| 642 |
return rows, stamp, int(win["total"])
|
|
|
|
| 441 |
|
| 442 |
|
| 443 |
def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None,
|
| 444 |
+
filter_keys=None, defn=None, serialize=True):
|
| 445 |
"""Every permitted row of a mirror-served grid, or a refusal that names its cause.
|
| 446 |
|
| 447 |
Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
|
|
|
|
| 516 |
stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
|
| 517 |
defn=defn if isinstance(defn, dict) else None)
|
| 518 |
rows = rot._pool_from_window(spec, win, stamp=stamp)
|
| 519 |
+
if serialize:
|
| 520 |
+
rows = _typed_rows(rot, fields, rows)
|
| 521 |
rows.sort(key=lambda row: row["pid"])
|
| 522 |
return rows, stamp, int(win["total"])
|
| 523 |
|
|
|
|
| 545 |
return value
|
| 546 |
|
| 547 |
|
| 548 |
+
def field_value(rot, field, value):
|
| 549 |
+
"""Serialize a native checkbox as a Boolean while preserving other JSON values."""
|
| 550 |
+
if field.get("type") == "checkbox" and value is not None:
|
| 551 |
+
if isinstance(value, str):
|
| 552 |
+
return value.strip().lower() in {"1", "true", "t", "yes", "y", "on"}
|
| 553 |
+
return bool(value)
|
| 554 |
+
return _json_value(rot, value)
|
| 555 |
+
|
| 556 |
+
|
| 557 |
+
def _typed_rows(rot, fields, rows, *, preserve_blank=False):
|
| 558 |
+
"""Apply the declared field codecs without changing source records."""
|
| 559 |
+
flags = [field for field in fields if field.get("type") == "checkbox"]
|
| 560 |
+
if not flags:
|
| 561 |
+
return rows
|
| 562 |
+
|
| 563 |
+
def encode(field, value):
|
| 564 |
+
if preserve_blank and (value is None or isinstance(value, str) and value.strip() in {'', '(none)'}):
|
| 565 |
+
return value
|
| 566 |
+
return field_value(rot, field, value)
|
| 567 |
+
|
| 568 |
+
return [{**row, **{field["key"]: encode(field, row[field["key"]])
|
| 569 |
+
for field in flags if field["key"] in row}} for row in rows]
|
| 570 |
+
|
| 571 |
+
|
| 572 |
def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None,
|
| 573 |
filter_tree=None, filter_keys=None):
|
| 574 |
"""Server-side aggregates over EVERY permitted row, in SQL. S67-T34 (D-616, D-2631).
|
|
|
|
| 632 |
|
| 633 |
|
| 634 |
def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree=None,
|
| 635 |
+
filter_tree=None, filter_keys=None, defn=None, sorts=None, serialize=True):
|
| 636 |
"""Return one SQL-ordered permitted page, its source stamp and its full population count."""
|
| 637 |
from harness import datastore
|
| 638 |
rot = _rot()
|
|
|
|
| 665 |
stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
|
| 666 |
defn=defn if isinstance(defn, dict) else None)
|
| 667 |
rows = rot._pool_from_window(spec, win, stamp=stamp)
|
| 668 |
+
if serialize:
|
| 669 |
+
rows = _typed_rows(rot, fields, rows)
|
| 670 |
return rows, stamp, int(win["total"])
|
api/automation_engine.py
CHANGED
|
@@ -14958,55 +14958,54 @@ def _refresh_relations_inplace(tables, log=print, rt=None):
|
|
| 14958 |
|
| 14959 |
|
| 14960 |
def refresh_relations(rt, log=print, tables=None, persist=True):
|
| 14961 |
-
"""
|
| 14962 |
-
|
| 14963 |
-
|
| 14964 |
-
|
| 14965 |
-
|
| 14966 |
-
|
| 14967 |
-
|
| 14968 |
-
|
| 14969 |
-
|
| 14970 |
-
|
| 14971 |
-
|
| 14972 |
-
|
| 14973 |
-
|
| 14974 |
-
|
| 14975 |
-
|
| 14976 |
-
|
| 14977 |
-
|
| 14978 |
-
|
| 14979 |
-
|
| 14980 |
-
|
| 14981 |
-
|
| 14982 |
-
|
| 14983 |
-
|
| 14984 |
-
|
| 14985 |
-
for key, table in (ut_all(rt) or {}).items()
|
| 14986 |
-
}
|
| 14987 |
-
if not _refresh_relations_inplace(snapshot, log=log, rt=rt):
|
| 14988 |
return 0
|
| 14989 |
actual = [0]
|
| 14990 |
|
| 14991 |
def _up(cur):
|
| 14992 |
-
|
| 14993 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14994 |
return cur
|
| 14995 |
|
| 14996 |
-
# ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug).
|
| 14997 |
-
#
|
| 14998 |
-
# 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and
|
| 14999 |
-
# `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget.
|
| 15000 |
-
# A derived-cell recompute has no business forcing a commit on someone typing.
|
| 15001 |
-
# 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new
|
| 15002 |
-
# row lives only in the store cache for 2-20s, and this call — on the SAME key
|
| 15003 |
-
# (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the
|
| 15004 |
-
# result. `POST /rows` answered 201 and the row was gone.
|
| 15005 |
-
#
|
| 15006 |
-
# ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it
|
| 15007 |
-
# is what makes the other ~39 sync writers of this key safe. This line is not that fix and
|
| 15008 |
-
# must not be mistaken for it — it removes the trigger and the cost, nothing more. Both
|
| 15009 |
-
# landed together on purpose: one is correctness, one is the hot path.
|
| 15010 |
rt.update(UT_STORE_KEY, _up, flush="async")
|
| 15011 |
return actual[0]
|
| 15012 |
|
|
|
|
| 14958 |
|
| 14959 |
|
| 14960 |
def refresh_relations(rt, log=print, tables=None, persist=True):
|
| 14961 |
+
"""Compute relations outside the write lock and merge only unchanged derived cells."""
|
| 14962 |
+
snapshot = tables if tables is not None else ut_all(rt)
|
| 14963 |
+
# Keep authored inputs before computation, including nested cell values.
|
| 14964 |
+
before = copy.deepcopy(snapshot)
|
| 14965 |
+
local = _refresh_relations_inplace(snapshot, log=log, rt=rt)
|
| 14966 |
+
if not local or not persist:
|
| 14967 |
+
return local
|
| 14968 |
+
patches = {}
|
| 14969 |
+
for key, old in before.items():
|
| 14970 |
+
if not isinstance(old, dict):
|
| 14971 |
+
continue
|
| 14972 |
+
fields = old.get('fields') or []
|
| 14973 |
+
derived = {str(f['key']) for f in fields if f.get('key')
|
| 14974 |
+
and (f.get('type') in ('rollup', 'if') or _ut().is_derived_link(f))}
|
| 14975 |
+
new_rows = (snapshot.get(key) or {}).get('rows') or {}
|
| 14976 |
+
for rid, row in (old.get('rows') or {}).items():
|
| 14977 |
+
after = new_rows.get(rid)
|
| 14978 |
+
if not isinstance(row, dict) or not isinstance(after, dict):
|
| 14979 |
+
continue
|
| 14980 |
+
changed = {fk: copy.deepcopy(after[fk]) for fk in derived
|
| 14981 |
+
if fk in after and (fk not in row or row[fk] != after[fk])}
|
| 14982 |
+
if changed:
|
| 14983 |
+
patches.setdefault(str(key), {})[str(rid)] = (row, changed)
|
| 14984 |
+
if not patches:
|
|
|
|
|
|
|
|
|
|
| 14985 |
return 0
|
| 14986 |
actual = [0]
|
| 14987 |
|
| 14988 |
def _up(cur):
|
| 14989 |
+
actual[0] = 0
|
| 14990 |
+
lock = getattr(cur, 'lock_records', None)
|
| 14991 |
+
if callable(lock):
|
| 14992 |
+
lock({key: list(rows) for key, rows in patches.items()})
|
| 14993 |
+
for key, rows in patches.items():
|
| 14994 |
+
table = (cur or {}).get(key)
|
| 14995 |
+
if not isinstance(table, dict) or table.get('fields') != before[key].get('fields'):
|
| 14996 |
+
continue
|
| 14997 |
+
current_rows = table.get('rows')
|
| 14998 |
+
if current_rows is None:
|
| 14999 |
+
continue
|
| 15000 |
+
for rid, (old_row, changes) in rows.items():
|
| 15001 |
+
current = current_rows.get(rid)
|
| 15002 |
+
# A changed authored input or a deleted row waits for the next refresh.
|
| 15003 |
+
if not isinstance(current, dict) or current != old_row:
|
| 15004 |
+
continue
|
| 15005 |
+
current.update(changes)
|
| 15006 |
+
actual[0] += 1
|
| 15007 |
return cur
|
| 15008 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15009 |
rt.update(UT_STORE_KEY, _up, flush="async")
|
| 15010 |
return actual[0]
|
| 15011 |
|
api/catalog_store.py
CHANGED
|
@@ -212,7 +212,7 @@ def require_app(session, app_id, *, write=False, allow_orphan=False):
|
|
| 212 |
raise err(403, "catalog_tenant_forbidden", "This catalog belongs to another workspace. Open the correct workspace.")
|
| 213 |
if not runtime.available():
|
| 214 |
raise err(503, "store_unavailable", "Catalog storage is unavailable. Try again when the store is ready.")
|
| 215 |
-
app = applications.
|
| 216 |
if app is None and allow_orphan and session.admin and not write:
|
| 217 |
app = _load(runtime)["appSnapshots"].get(app_id)
|
| 218 |
if (not isinstance(app, dict) or app.get("tenant") != session.tenant
|
|
@@ -231,6 +231,9 @@ def require_app(session, app_id, *, write=False, allow_orphan=False):
|
|
| 231 |
_assert_write_scope(session, "catalogs")
|
| 232 |
contracts = applications.chart_field_contracts(session, [source])
|
| 233 |
fields = applications._contract_fields(source, field_contracts=contracts)
|
|
|
|
|
|
|
|
|
|
| 234 |
try:
|
| 235 |
app = applications.bound_application(app, fields)
|
| 236 |
except applications.ApplicationError:
|
|
@@ -260,7 +263,7 @@ def _binding_wall(record, app, fields):
|
|
| 260 |
def _record(session, catalog_id, *, write=False):
|
| 261 |
record = _load(session.runtime)["catalogs"].get(catalog_id)
|
| 262 |
if record is None:
|
| 263 |
-
apps = applications.
|
| 264 |
for app in apps.values():
|
| 265 |
if not isinstance(app, dict):
|
| 266 |
continue
|
|
|
|
| 212 |
raise err(403, "catalog_tenant_forbidden", "This catalog belongs to another workspace. Open the correct workspace.")
|
| 213 |
if not runtime.available():
|
| 214 |
raise err(503, "store_unavailable", "Catalog storage is unavailable. Try again when the store is ready.")
|
| 215 |
+
app = applications.load(runtime, strict=True)["applications"].get(app_id)
|
| 216 |
if app is None and allow_orphan and session.admin and not write:
|
| 217 |
app = _load(runtime)["appSnapshots"].get(app_id)
|
| 218 |
if (not isinstance(app, dict) or app.get("tenant") != session.tenant
|
|
|
|
| 231 |
_assert_write_scope(session, "catalogs")
|
| 232 |
contracts = applications.chart_field_contracts(session, [source])
|
| 233 |
fields = applications._contract_fields(source, field_contracts=contracts)
|
| 234 |
+
# Projected C1 fields retain their optional roles; authored legacy maps stay strict.
|
| 235 |
+
if not app_templates.own_binding(app):
|
| 236 |
+
app = {key: value for key, value in app.items() if key not in {"sourceDatabase", "fieldMap"}}
|
| 237 |
try:
|
| 238 |
app = applications.bound_application(app, fields)
|
| 239 |
except applications.ApplicationError:
|
|
|
|
| 263 |
def _record(session, catalog_id, *, write=False):
|
| 264 |
record = _load(session.runtime)["catalogs"].get(catalog_id)
|
| 265 |
if record is None:
|
| 266 |
+
apps = applications.load(session.runtime, strict=True)["applications"]
|
| 267 |
for app in apps.values():
|
| 268 |
if not isinstance(app, dict):
|
| 269 |
continue
|
api/deps.py
CHANGED
|
@@ -642,14 +642,14 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 642 |
rows, stamp, total = assistant_mirror.read_page(
|
| 643 |
session, database, cur, spec, fields, offset=mode[1], limit=mode[2],
|
| 644 |
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
|
| 645 |
-
defn=definition, sorts=mode[3] if len(mode) > 3 else None)
|
| 646 |
if isinstance(extra, dict):
|
| 647 |
extra["total"] = total
|
| 648 |
else:
|
| 649 |
rows, stamp, total = assistant_mirror.read_rows(
|
| 650 |
session, database, cur, spec, fields,
|
| 651 |
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
|
| 652 |
-
defn=definition)
|
| 653 |
except assistant_mirror.MirrorRefusal as exc:
|
| 654 |
_assistant_refuse(exc.status, exc.code, exc.message)
|
| 655 |
raise # unreachable; see above
|
|
@@ -717,7 +717,8 @@ def _assistant_mirror_groups(session, database, cur, spec, fields, specs, option
|
|
| 717 |
f'"a{i}"' for i in range(len(specs))), params=tuple(params),
|
| 718 |
order_by=", ".join(f'"g{i}"' for i in range(len(group_exprs))),
|
| 719 |
offset=options["offset"], limit=options["limit"], cur=cur)
|
| 720 |
-
|
|
|
|
| 721 |
for i, group in enumerate(options["groupBy"])},
|
| 722 |
"aggregates": [{**item, "value": mirror._json_value(
|
| 723 |
rot, row[len(group_exprs) + i])} for i, item in enumerate(specs)]}
|
|
@@ -808,8 +809,34 @@ def _assistant_activity_table(session, definition, fields, filters, mode, member
|
|
| 808 |
return rows, canonical, {"read": "field contract"}, definition.get("label") or "Activity"
|
| 809 |
|
| 810 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 811 |
def assistant_read_scope(session: Session, database, fields=None, filters=None, *, mode=None,
|
| 812 |
-
also_as=()):
|
| 813 |
"""Return a detached, permissioned snapshot for one Assistant tool call.
|
| 814 |
|
| 815 |
Signature (the C8 contract for E):
|
|
@@ -836,11 +863,17 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 836 |
`core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and
|
| 837 |
unsupported operands before a row is read. The returned dictionaries are deep copies, never
|
| 838 |
a live cache/store object, cursor, connector, route, provider, or credential.
|
|
|
|
|
|
|
| 839 |
"""
|
| 840 |
import core.perm_scope as perm_scope
|
| 841 |
from harness import filter_eval
|
| 842 |
|
| 843 |
key = str(database or "").strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 844 |
members = tuple(also_as)
|
| 845 |
if members:
|
| 846 |
# C15 members belong to this session's tenant. A record from another tenant
|
|
@@ -951,6 +984,15 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 951 |
else:
|
| 952 |
_assistant_refuse(404, "assistant_unknown_database", "that database is not available")
|
| 953 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 954 |
# Permanent permissions run against the full source contract. A permission filter may name
|
| 955 |
# a field that is intentionally hidden from the returned schema; resolving it after the field
|
| 956 |
# projection would make that filter unanswerable and (depending on the evaluator) could widen.
|
|
@@ -1003,7 +1045,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 1003 |
# C16: with no grant there is no row wall, but a declared recipient list still binds
|
| 1004 |
# every reader, administrators included (S68-T57).
|
| 1005 |
permitted_rows = (perm_scope.assistant_apply_row_scope(
|
| 1006 |
-
rows, session.user, key,
|
| 1007 |
if grant is not None else perm_scope.apply_audience_scope(
|
| 1008 |
rows, session.user, key, canonical_fields, st=getattr(session, "runtime", None),
|
| 1009 |
defn=_audience_defn))
|
|
@@ -1016,12 +1058,12 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 1016 |
if members:
|
| 1017 |
hidden = set(access["hidden"])
|
| 1018 |
member_rows = builtin_scope.get("rows", permitted_rows)
|
| 1019 |
-
member_fields = builtin_scope.get("fields",
|
| 1020 |
for member in members:
|
| 1021 |
hidden.update(perm_scope.hidden_keys(member, key, member_fields, st=session.runtime))
|
| 1022 |
if mode not in ("schema", "query_schema"):
|
| 1023 |
member_rows = perm_scope.apply_row_scope(
|
| 1024 |
-
member_rows, member, key, builtin_scope.get("row_fields",
|
| 1025 |
st=session.runtime)
|
| 1026 |
if builtin_scope and mode not in ("schema", "query_schema"):
|
| 1027 |
allowed_pids = {row.get("pid") for row in member_rows}
|
|
@@ -1039,15 +1081,24 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 1039 |
except ValueError as exc:
|
| 1040 |
_assistant_refuse(400, "assistant_bad_filter", str(exc))
|
| 1041 |
if requested_filter is not None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1042 |
permitted_rows = [row for row in permitted_rows
|
| 1043 |
-
if
|
| 1044 |
|
| 1045 |
records = []
|
|
|
|
|
|
|
| 1046 |
for row in permitted_rows:
|
| 1047 |
if not isinstance(row, dict) or row.get("pid") is None:
|
| 1048 |
continue
|
| 1049 |
records.append({"pid": copy.deepcopy(row["pid"]),
|
| 1050 |
-
**{field: copy.deepcopy(
|
|
|
|
|
|
|
| 1051 |
|
| 1052 |
out = {
|
| 1053 |
"database": key,
|
|
@@ -1062,7 +1113,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 1062 |
set(hidden) | ({field.get("key") for field in canonical_fields}
|
| 1063 |
- {field.get("key") for field in visible})),
|
| 1064 |
definition=_audience_defn, members=members),
|
| 1065 |
-
"filters": copy.deepcopy(requested_filter),
|
| 1066 |
"records": records,
|
| 1067 |
}
|
| 1068 |
if "aggregates" in extra:
|
|
@@ -1597,6 +1648,15 @@ def require_session(request: Request, response: Response) -> Session:
|
|
| 1597 |
user = _user_for(claims)
|
| 1598 |
if not user:
|
| 1599 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1600 |
try:
|
| 1601 |
rt = runtime.get_runtime(claims["t"])
|
| 1602 |
except KeyError:
|
|
|
|
| 642 |
rows, stamp, total = assistant_mirror.read_page(
|
| 643 |
session, database, cur, spec, fields, offset=mode[1], limit=mode[2],
|
| 644 |
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
|
| 645 |
+
defn=definition, sorts=mode[3] if len(mode) > 3 else None, serialize=False)
|
| 646 |
if isinstance(extra, dict):
|
| 647 |
extra["total"] = total
|
| 648 |
else:
|
| 649 |
rows, stamp, total = assistant_mirror.read_rows(
|
| 650 |
session, database, cur, spec, fields,
|
| 651 |
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
|
| 652 |
+
defn=definition, serialize=False)
|
| 653 |
except assistant_mirror.MirrorRefusal as exc:
|
| 654 |
_assistant_refuse(exc.status, exc.code, exc.message)
|
| 655 |
raise # unreachable; see above
|
|
|
|
| 717 |
f'"a{i}"' for i in range(len(specs))), params=tuple(params),
|
| 718 |
order_by=", ".join(f'"g{i}"' for i in range(len(group_exprs))),
|
| 719 |
offset=options["offset"], limit=options["limit"], cur=cur)
|
| 720 |
+
by_key = {field["key"]: field for field in fields}
|
| 721 |
+
groups = [{"keys": {group["field"]: mirror.field_value(rot, by_key[group["field"]], row[i])
|
| 722 |
for i, group in enumerate(options["groupBy"])},
|
| 723 |
"aggregates": [{**item, "value": mirror._json_value(
|
| 724 |
rot, row[len(group_exprs) + i])} for i, item in enumerate(specs)]}
|
|
|
|
| 809 |
return rows, canonical, {"read": "field contract"}, definition.get("label") or "Activity"
|
| 810 |
|
| 811 |
|
| 812 |
+
def _assistant_identity_request(database, filters, identity_pid, *, trusted_filters=False):
|
| 813 |
+
"""Separate one opaque identity equality from visible field operands."""
|
| 814 |
+
if database not in _ASSISTANT_BUILT_INS:
|
| 815 |
+
if identity_pid is not None:
|
| 816 |
+
_assistant_refuse(400, "assistant_bad_filter", "Opaque identity reads require a built-in database.")
|
| 817 |
+
return filters, identity_pid
|
| 818 |
+
if (trusted_filters and isinstance(filters, dict) and set(filters) == {"conj", "nodes"}
|
| 819 |
+
and filters.get("conj") == "and" and isinstance(filters.get("nodes"), list)
|
| 820 |
+
and len(filters["nodes"]) == 1):
|
| 821 |
+
leaf = filters["nodes"][0]
|
| 822 |
+
if (isinstance(leaf, dict) and leaf.get("colId") == "pid"
|
| 823 |
+
and leaf.get("op") == "eq" and "value" in leaf
|
| 824 |
+
and not set(leaf) - {"id", "colId", "op", "value"}):
|
| 825 |
+
value = leaf["value"]
|
| 826 |
+
if value is None:
|
| 827 |
+
_assistant_refuse(400, "assistant_bad_filter", "Use one non-null opaque record identity.")
|
| 828 |
+
if identity_pid is not None and (type(value) is not type(identity_pid) or value != identity_pid):
|
| 829 |
+
_assistant_refuse(400, "assistant_bad_filter", "Opaque identity pins must agree.")
|
| 830 |
+
identity_pid, filters = value, None
|
| 831 |
+
if identity_pid is not None and (type(identity_pid) not in (int, str)
|
| 832 |
+
or isinstance(identity_pid, str)
|
| 833 |
+
and (not identity_pid.strip() or len(identity_pid) > 200)):
|
| 834 |
+
_assistant_refuse(400, "assistant_bad_filter", "Use one opaque integer or string record identity.")
|
| 835 |
+
return filters, identity_pid
|
| 836 |
+
|
| 837 |
+
|
| 838 |
def assistant_read_scope(session: Session, database, fields=None, filters=None, *, mode=None,
|
| 839 |
+
also_as=(), identity_pid=None):
|
| 840 |
"""Return a detached, permissioned snapshot for one Assistant tool call.
|
| 841 |
|
| 842 |
Signature (the C8 contract for E):
|
|
|
|
| 863 |
`core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and
|
| 864 |
unsupported operands before a row is read. The returned dictionaries are deep copies, never
|
| 865 |
a live cache/store object, cursor, connector, route, provider, or credential.
|
| 866 |
+
`identity_pid` intersects built-in records by opaque identity after every permission wall.
|
| 867 |
+
Only a verified browser session may express that pin through a single pid equality filter.
|
| 868 |
"""
|
| 869 |
import core.perm_scope as perm_scope
|
| 870 |
from harness import filter_eval
|
| 871 |
|
| 872 |
key = str(database or "").strip()
|
| 873 |
+
original_filters = filters
|
| 874 |
+
filters, identity_pid = _assistant_identity_request(
|
| 875 |
+
key, filters, identity_pid,
|
| 876 |
+
trusted_filters=(session.claims or {}).get("external_via") == "session")
|
| 877 |
members = tuple(also_as)
|
| 878 |
if members:
|
| 879 |
# C15 members belong to this session's tenant. A record from another tenant
|
|
|
|
| 984 |
else:
|
| 985 |
_assistant_refuse(404, "assistant_unknown_database", "that database is not available")
|
| 986 |
|
| 987 |
+
evaluation_fields = canonical_fields
|
| 988 |
+
if source_kind == "mirror_user_table":
|
| 989 |
+
import assistant_mirror
|
| 990 |
+
# Preserve raw blanks through both row predicates, then encode the response once.
|
| 991 |
+
rows = assistant_mirror._typed_rows(None, canonical_fields, rows, preserve_blank=True)
|
| 992 |
+
# Numeric flag comparisons accept Boolean operands without changing raw blank checks.
|
| 993 |
+
evaluation_fields = [dict(field, type="int") if field.get("type") == "checkbox" else field
|
| 994 |
+
for field in canonical_fields]
|
| 995 |
+
|
| 996 |
# Permanent permissions run against the full source contract. A permission filter may name
|
| 997 |
# a field that is intentionally hidden from the returned schema; resolving it after the field
|
| 998 |
# projection would make that filter unanswerable and (depending on the evaluator) could widen.
|
|
|
|
| 1045 |
# C16: with no grant there is no row wall, but a declared recipient list still binds
|
| 1046 |
# every reader, administrators included (S68-T57).
|
| 1047 |
permitted_rows = (perm_scope.assistant_apply_row_scope(
|
| 1048 |
+
rows, session.user, key, evaluation_fields, st=getattr(session, "runtime", None))
|
| 1049 |
if grant is not None else perm_scope.apply_audience_scope(
|
| 1050 |
rows, session.user, key, canonical_fields, st=getattr(session, "runtime", None),
|
| 1051 |
defn=_audience_defn))
|
|
|
|
| 1058 |
if members:
|
| 1059 |
hidden = set(access["hidden"])
|
| 1060 |
member_rows = builtin_scope.get("rows", permitted_rows)
|
| 1061 |
+
member_fields = builtin_scope.get("fields", evaluation_fields)
|
| 1062 |
for member in members:
|
| 1063 |
hidden.update(perm_scope.hidden_keys(member, key, member_fields, st=session.runtime))
|
| 1064 |
if mode not in ("schema", "query_schema"):
|
| 1065 |
member_rows = perm_scope.apply_row_scope(
|
| 1066 |
+
member_rows, member, key, builtin_scope.get("row_fields", evaluation_fields),
|
| 1067 |
st=session.runtime)
|
| 1068 |
if builtin_scope and mode not in ("schema", "query_schema"):
|
| 1069 |
allowed_pids = {row.get("pid") for row in member_rows}
|
|
|
|
| 1081 |
except ValueError as exc:
|
| 1082 |
_assistant_refuse(400, "assistant_bad_filter", str(exc))
|
| 1083 |
if requested_filter is not None:
|
| 1084 |
+
visible_keys = {field["key"] for field in visible}
|
| 1085 |
+
evaluation_visible = [field for field in evaluation_fields if field["key"] in visible_keys]
|
| 1086 |
+
permitted_rows = [row for row in permitted_rows
|
| 1087 |
+
if filter_eval.permits(requested_filter, row, evaluation_visible)]
|
| 1088 |
+
if identity_pid is not None:
|
| 1089 |
permitted_rows = [row for row in permitted_rows
|
| 1090 |
+
if row.get("pid") is not None and str(row["pid"]) == str(identity_pid)]
|
| 1091 |
|
| 1092 |
records = []
|
| 1093 |
+
checkbox_fields = ({field["key"]: field for field in visible if field.get("type") == "checkbox"}
|
| 1094 |
+
if source_kind == "mirror_user_table" else {})
|
| 1095 |
for row in permitted_rows:
|
| 1096 |
if not isinstance(row, dict) or row.get("pid") is None:
|
| 1097 |
continue
|
| 1098 |
records.append({"pid": copy.deepcopy(row["pid"]),
|
| 1099 |
+
**{field: copy.deepcopy(assistant_mirror.field_value(
|
| 1100 |
+
None, checkbox_fields[field], row.get(field)) if field in checkbox_fields
|
| 1101 |
+
else row.get(field)) for field in selected_keys}})
|
| 1102 |
|
| 1103 |
out = {
|
| 1104 |
"database": key,
|
|
|
|
| 1113 |
set(hidden) | ({field.get("key") for field in canonical_fields}
|
| 1114 |
- {field.get("key") for field in visible})),
|
| 1115 |
definition=_audience_defn, members=members),
|
| 1116 |
+
"filters": copy.deepcopy(original_filters if identity_pid is not None else requested_filter),
|
| 1117 |
"records": records,
|
| 1118 |
}
|
| 1119 |
if "aggregates" in extra:
|
|
|
|
| 1648 |
user = _user_for(claims)
|
| 1649 |
if not user:
|
| 1650 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1651 |
+
identity = {_TENANT_HEADER: str(claims["t"]),
|
| 1652 |
+
_PRINCIPAL_HEADER: _principal(claims["t"], user["username"])}
|
| 1653 |
+
if request.method not in {"GET", "HEAD", "OPTIONS"} and any(
|
| 1654 |
+
header in request.headers and request.headers[header] != value
|
| 1655 |
+
for header, value in identity.items()):
|
| 1656 |
+
refused = err(409, "mutation_identity_changed",
|
| 1657 |
+
"Your account or workspace changed. Reload before retrying this edit.")
|
| 1658 |
+
refused.headers = identity
|
| 1659 |
+
raise refused
|
| 1660 |
try:
|
| 1661 |
rt = runtime.get_runtime(claims["t"])
|
| 1662 |
except KeyError:
|
api/external_openapi.py
CHANGED
|
@@ -6,7 +6,7 @@ import re
|
|
| 6 |
from copy import copy, deepcopy
|
| 7 |
|
| 8 |
|
| 9 |
-
#
|
| 10 |
_JSON_TYPE = {
|
| 11 |
"text": "string", "status": "string", "select": "string", "user": "string",
|
| 12 |
"multiselect": "string", "phone": "string", "email": "string", "url": "string",
|
|
@@ -40,7 +40,8 @@ def _record_name(database):
|
|
| 40 |
def _field_schema(field, source_kind):
|
| 41 |
if field.get("type") == "multiselect":
|
| 42 |
return {"type": ["string", "array", "null"], "items": {"type": "string"}}
|
| 43 |
-
kind =
|
|
|
|
| 44 |
types = [kind]
|
| 45 |
# Materialised table writes retain numeric cells as strings; reads preserve those bytes.
|
| 46 |
if kind in {"number", "integer"}:
|
|
@@ -119,7 +120,8 @@ def _schemas(databases, records):
|
|
| 119 |
"QueryRequest": _object({
|
| 120 |
"fields": _array({"type": "string"}, maxItems=200),
|
| 121 |
"filters": _ref("Filter"), "sorts": _array(_ref("Sort"), maxItems=8),
|
| 122 |
-
"limit": {"type": "integer", "minimum": 1, "
|
|
|
|
| 123 |
"cursor": {"type": ["string", "null"], "maxLength": 768},
|
| 124 |
"aggregates": _array(_ref("AggregateRequest"), maxItems=16),
|
| 125 |
"groupBy": _array(_ref("GroupBy"), minItems=1, maxItems=4),
|
|
|
|
| 6 |
from copy import copy, deepcopy
|
| 7 |
|
| 8 |
|
| 9 |
+
# Authored checkboxes and structured cells retain the grid's string representation.
|
| 10 |
_JSON_TYPE = {
|
| 11 |
"text": "string", "status": "string", "select": "string", "user": "string",
|
| 12 |
"multiselect": "string", "phone": "string", "email": "string", "url": "string",
|
|
|
|
| 40 |
def _field_schema(field, source_kind):
|
| 41 |
if field.get("type") == "multiselect":
|
| 42 |
return {"type": ["string", "array", "null"], "items": {"type": "string"}}
|
| 43 |
+
kind = ("boolean" if field.get("type") == "checkbox" and source_kind == "mirror_user_table"
|
| 44 |
+
else _JSON_TYPE.get(field.get("type"), "string"))
|
| 45 |
types = [kind]
|
| 46 |
# Materialised table writes retain numeric cells as strings; reads preserve those bytes.
|
| 47 |
if kind in {"number", "integer"}:
|
|
|
|
| 120 |
"QueryRequest": _object({
|
| 121 |
"fields": _array({"type": "string"}, maxItems=200),
|
| 122 |
"filters": _ref("Filter"), "sorts": _array(_ref("Sort"), maxItems=8),
|
| 123 |
+
"limit": {"type": "integer", "minimum": 1, "default": 100,
|
| 124 |
+
"description": "Each response returns at most 500 rows. Follow next_cursor to read all requested rows."},
|
| 125 |
"cursor": {"type": ["string", "null"], "maxLength": 768},
|
| 126 |
"aggregates": _array(_ref("AggregateRequest"), maxItems=16),
|
| 127 |
"groupBy": _array(_ref("GroupBy"), minItems=1, maxItems=4),
|
api/intelligence_store.py
CHANGED
|
@@ -86,6 +86,8 @@ def normalize_path(value, *, folder=False):
|
|
| 86 |
|
| 87 |
|
| 88 |
def _assert_authored_path(path):
|
|
|
|
|
|
|
| 89 |
import wiki_log
|
| 90 |
if wiki_log.reserved(path):
|
| 91 |
raise IntelligenceError("wiki_generated", "The system maintains wiki/log/ and wiki/index/.", 403)
|
|
@@ -275,8 +277,7 @@ def _files(session, *, include_generated=False, wiki_only=False, raw=None):
|
|
| 275 |
_check_file_capacity(session, raw)
|
| 276 |
import wiki_log
|
| 277 |
files = _project_templates(session, raw)
|
| 278 |
-
|
| 279 |
-
files = _project_database_notes(session, files)
|
| 280 |
files = {path: record for path, record in files.items() if not wiki_log.reserved(path)}
|
| 281 |
files = {path: record for path, record in files.items()
|
| 282 |
if not wiki_log.purpose_file(path) or wiki_log.purpose_record(session, path, files) is not None}
|
|
@@ -294,140 +295,6 @@ def _database_note_path(path):
|
|
| 294 |
return path.casefold().startswith(DATABASE_NOTES_PREFIX)
|
| 295 |
|
| 296 |
|
| 297 |
-
def _database_note_definition(session, key):
|
| 298 |
-
"""Resolve a note's title and permission key from current database definitions."""
|
| 299 |
-
from core import registry, user_tables
|
| 300 |
-
|
| 301 |
-
if key.startswith("ut_"):
|
| 302 |
-
definition = user_tables.all_defs(st=session.runtime).get(key)
|
| 303 |
-
if definition:
|
| 304 |
-
return {"key": key, "label": definition["label"], "owner": definition.get("createdBy") or "role:admin"}
|
| 305 |
-
for topic, definition in registry.governable_modules().items():
|
| 306 |
-
# Registry grids store notes under their scope, such as customer, rather than their module key.
|
| 307 |
-
if key in (topic, definition["key"]):
|
| 308 |
-
return {**definition, "owner": "role:admin"}
|
| 309 |
-
return None
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
def _database_note_can_edit(session, key):
|
| 313 |
-
"""Use the grid's Database-note edit authority after current reader admission."""
|
| 314 |
-
from core import external_access, user_tables
|
| 315 |
-
|
| 316 |
-
definition = _database_note_definition(session, key)
|
| 317 |
-
if not definition or not can_read(session, {"reads": [definition["key"]]}):
|
| 318 |
-
return False
|
| 319 |
-
if not external_access.may_write(credential_scope(session)[0]):
|
| 320 |
-
return False
|
| 321 |
-
return (user_tables.may_edit_database(key, session.uname, is_admin=session.admin, st=session.runtime)
|
| 322 |
-
if key.startswith("ut_") else bool(session.admin))
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
def _database_note_record(session, key, *, stored=None, with_body=False):
|
| 326 |
-
"""Project canonical note metadata without storing a Brain version or a body copy."""
|
| 327 |
-
from core import user_tables
|
| 328 |
-
|
| 329 |
-
path = DATABASE_NOTES_PREFIX + key + ".md"
|
| 330 |
-
if normalize_path(path) != path or "/" in key or not key:
|
| 331 |
-
raise IntelligenceError("no_file", "This workspace has no file at that path.", 404)
|
| 332 |
-
definition = _database_note_definition(session, key)
|
| 333 |
-
if not definition or not can_read(session, {"reads": [definition["key"]]}):
|
| 334 |
-
raise IntelligenceError("no_file", "This workspace has no file at that path.", 404)
|
| 335 |
-
note = user_tables.shared_note("database", key, st=session.runtime)
|
| 336 |
-
if not note or note.get("deleted"):
|
| 337 |
-
raise IntelligenceError("no_file", "This workspace has no file at that path.", 404)
|
| 338 |
-
meta = {"title": note.get("title") or definition["label"], "section": "database-notes", "owner": definition["owner"],
|
| 339 |
-
"description": f"Shared note for {definition['label']}.", "reads": [definition["key"]],
|
| 340 |
-
"approvers": [], "approvals_needed": 1, "tools": [], "databaseKey": key}
|
| 341 |
-
text = note.get("body") or ""
|
| 342 |
-
entry = {"metadata": meta, "version": int(note.get("version") or 0),
|
| 343 |
-
"bytes": len(text.encode("utf-8")), "author": note.get("author") or "",
|
| 344 |
-
"authorKind": "user", "created": note.get("updated")}
|
| 345 |
-
if with_body:
|
| 346 |
-
entry["body"] = text
|
| 347 |
-
revisions = note.get("revisions") or []
|
| 348 |
-
return {"path": path, "databaseKey": key, "canEdit": _database_note_can_edit(session, key),
|
| 349 |
-
"revisionHistory": "database-note", "versions": [entry],
|
| 350 |
-
"created": revisions[0].get("at") if revisions else entry["created"],
|
| 351 |
-
"proposals": (stored or {}).get("proposals") or {},
|
| 352 |
-
**({"note": note} if with_body else {})}
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
def _project_database_notes(session, raw):
|
| 356 |
-
"""Reserve the note namespace and expose only current, readable Database rows."""
|
| 357 |
-
from core import user_tables
|
| 358 |
-
|
| 359 |
-
files = {path: record for path, record in raw.items() if not _database_note_path(path)}
|
| 360 |
-
notes = _document(session.runtime.get(user_tables.SHARED_NOTES_KEY))
|
| 361 |
-
for subject in notes:
|
| 362 |
-
if not isinstance(subject, str) or not subject.startswith("database:"):
|
| 363 |
-
continue
|
| 364 |
-
key = subject[len("database:"):]
|
| 365 |
-
path = DATABASE_NOTES_PREFIX + key + ".md"
|
| 366 |
-
try:
|
| 367 |
-
files[path] = _database_note_record(session, key, stored=raw.get(path))
|
| 368 |
-
except IntelligenceError as exc:
|
| 369 |
-
if exc.code not in ("no_file", "bad_path"):
|
| 370 |
-
raise
|
| 371 |
-
return files
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
def _database_note_file(session, path, *, version=0):
|
| 375 |
-
if not path.startswith(DATABASE_NOTES_PREFIX) or not path.endswith(".md"):
|
| 376 |
-
raise IntelligenceError("no_file", "This workspace has no file at that path.", 404)
|
| 377 |
-
record = _database_note_record(session, path[len(DATABASE_NOTES_PREFIX):-3], with_body=True)
|
| 378 |
-
note = record["note"]
|
| 379 |
-
current = _head(record)
|
| 380 |
-
entry = current
|
| 381 |
-
if version and version != current["version"]:
|
| 382 |
-
revision = next((row for row in note.get("revisions") or [] if row.get("version") == version), None)
|
| 383 |
-
if revision is None:
|
| 384 |
-
raise IntelligenceError("no_version", "This file has no such version.", 404)
|
| 385 |
-
entry = {**current, "body": revision["body"], "version": version,
|
| 386 |
-
"author": revision["author"], "created": revision["at"],
|
| 387 |
-
"bytes": len(revision["body"].encode("utf-8"))}
|
| 388 |
-
history = [{"version": row["version"], "author": row["author"], "authorKind": "user",
|
| 389 |
-
"created": row["at"], "bytes": len(row["body"].encode("utf-8"))}
|
| 390 |
-
for row in reversed(note.get("revisions") or [])]
|
| 391 |
-
return {**_row(record, entry, session=session), "body": entry["body"], "atVersion": entry["version"],
|
| 392 |
-
"latestVersion": current["version"], "history": history, "versions": len(history), "limits": limits()}
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
def _put_database_note(session, key, body, actor, base_version, base_updated, *, title=None, deleted=False):
|
| 396 |
-
"""Let the canonical writer enforce admission and its exact base on every atomic attempt."""
|
| 397 |
-
from core import user_tables
|
| 398 |
-
|
| 399 |
-
class CheckedRuntime:
|
| 400 |
-
def __getattr__(self, name):
|
| 401 |
-
return getattr(session.runtime, name)
|
| 402 |
-
|
| 403 |
-
def update(self, bucket, callback, **kwargs):
|
| 404 |
-
if bucket != user_tables.SHARED_NOTES_KEY:
|
| 405 |
-
raise IntelligenceError("database_note_reserved", "Database notes use their shared store.", 403)
|
| 406 |
-
|
| 407 |
-
def checked(raw):
|
| 408 |
-
if not _database_note_can_edit(session, key):
|
| 409 |
-
raise IntelligenceError("note_read_only", "Your Database role does not allow note edits.", 403)
|
| 410 |
-
prior = (_document(raw).get("database:" + key) or {})
|
| 411 |
-
if int(prior.get("version") or 0) != base_version or prior.get("updated") != base_updated:
|
| 412 |
-
raise IntelligenceError("version_conflict", "The note changed. Reload it before saving.", 409)
|
| 413 |
-
updated = callback(raw)
|
| 414 |
-
note = updated["database:" + key]
|
| 415 |
-
if title is not None or prior.get("title"):
|
| 416 |
-
note["title"] = title if title is not None else prior["title"]
|
| 417 |
-
if deleted:
|
| 418 |
-
note["deleted"] = True
|
| 419 |
-
return updated
|
| 420 |
-
|
| 421 |
-
return session.runtime.update(bucket, checked, **kwargs)
|
| 422 |
-
|
| 423 |
-
try:
|
| 424 |
-
return user_tables.put_shared_note("database", key, body, actor, st=CheckedRuntime())
|
| 425 |
-
except IntelligenceError:
|
| 426 |
-
raise
|
| 427 |
-
except Exception:
|
| 428 |
-
raise IntelligenceError("store_unavailable", "The note change could not be confirmed. Reload and try again.", 503) from None
|
| 429 |
-
|
| 430 |
-
|
| 431 |
@lru_cache(maxsize=1)
|
| 432 |
def _template_catalog():
|
| 433 |
"""Cache only shipped text, never tenant connections, grants, schemas or overrides."""
|
|
@@ -753,7 +620,7 @@ class _FolderReadRuntime:
|
|
| 753 |
if name not in self._documents:
|
| 754 |
if name == STORE_KEY:
|
| 755 |
reader = getattr(self._runtime, "get_many", None)
|
| 756 |
-
values = reader([STORE_KEY, TEMPLATE_USAGE_KEY, "keychain"
|
| 757 |
if values is not None:
|
| 758 |
self._documents.update(values)
|
| 759 |
if name not in self._documents:
|
|
@@ -1012,12 +879,7 @@ def linked_files(ctx, database, field=None):
|
|
| 1012 |
entry = _head(record)
|
| 1013 |
if not entry or not can_read(session, entry["metadata"]):
|
| 1014 |
continue
|
| 1015 |
-
|
| 1016 |
-
keys = [record["databaseKey"], *entry["metadata"]["reads"]]
|
| 1017 |
-
if field is not None:
|
| 1018 |
-
keys += _LINK_TOKENS.findall(_database_note_file(session, path)["body"])
|
| 1019 |
-
else:
|
| 1020 |
-
keys = _link_index(record)["keys"]
|
| 1021 |
if database in keys and (field is None or field in keys):
|
| 1022 |
meta = entry["metadata"]
|
| 1023 |
rows.append({"path": path, "title": meta["title"], "section": meta["section"],
|
|
@@ -1064,7 +926,7 @@ def read_file(session, path, *, version=0, record_use=True):
|
|
| 1064 |
if type(version) is not int or version < 0:
|
| 1065 |
raise IntelligenceError("bad_version", "Use a positive version number, or zero for the latest version.")
|
| 1066 |
if _database_note_path(path):
|
| 1067 |
-
|
| 1068 |
|
| 1069 |
def readable(raw):
|
| 1070 |
record = _project_templates(session, _document(raw)).get(path)
|
|
@@ -1202,12 +1064,7 @@ def _proposal_readable(session, record, proposal):
|
|
| 1202 |
or not wiki_log.purpose_object(session, proposal["metadata"])):
|
| 1203 |
return False
|
| 1204 |
if _database_note_path(record.get("path", "")):
|
| 1205 |
-
|
| 1206 |
-
_database_note_file(session, record["path"])
|
| 1207 |
-
except IntelligenceError as exc:
|
| 1208 |
-
if exc.code == "no_file":
|
| 1209 |
-
return False
|
| 1210 |
-
raise
|
| 1211 |
template = _template_catalog().get(record.get("path"))
|
| 1212 |
if template and record["path"] not in _project_templates(session, {record["path"]: record}):
|
| 1213 |
return False
|
|
@@ -1244,13 +1101,6 @@ def _recipients(session, policy, *, record, proposal):
|
|
| 1244 |
if wiki_log.purpose_file(record.get("path", "")):
|
| 1245 |
obj = wiki_log.purpose_object(recipient, proposal["metadata"])
|
| 1246 |
matches = bool(obj and obj["owner"] == name)
|
| 1247 |
-
if _database_note_path(record.get("path", "")):
|
| 1248 |
-
try:
|
| 1249 |
-
matches = _database_note_can_edit(recipient, record["path"][len(DATABASE_NOTES_PREFIX):-3])
|
| 1250 |
-
except HTTPException as exc:
|
| 1251 |
-
if exc.status_code != 403:
|
| 1252 |
-
raise
|
| 1253 |
-
matches = False
|
| 1254 |
if matches:
|
| 1255 |
try:
|
| 1256 |
if not _proposal_readable(recipient, record, proposal):
|
|
@@ -1284,10 +1134,7 @@ def _public_proposal(session, proposal, record):
|
|
| 1284 |
row["approvedBy"] = [name for name in proposal["approvedBy"] if name in eligible]
|
| 1285 |
row["recipients"] = eligible
|
| 1286 |
row["canDecide"] = (not trusted_identity(session)[1].get("agent") and session.uname in eligible
|
| 1287 |
-
and
|
| 1288 |
-
or (_database_note_path(record.get("path", "")) and proposal["status"] == "approving"))
|
| 1289 |
-
and (session.uname not in row["approvedBy"]
|
| 1290 |
-
or (_database_note_path(record.get("path", "")) and proposal["status"] == "approving")))
|
| 1291 |
return row
|
| 1292 |
|
| 1293 |
|
|
@@ -1329,7 +1176,7 @@ def _notify_proposal(session, proposal):
|
|
| 1329 |
send = getattr(module, "send_request", None)
|
| 1330 |
title = f"Review {proposal['title']}"
|
| 1331 |
body = f"{proposal['path']}\nProposal {proposal['id']}\n\n{proposal['diff']}"
|
| 1332 |
-
if "personal_scope" in proposal["metadata"]
|
| 1333 |
# Inbox copies cannot keep private content after its source access is revoked.
|
| 1334 |
if "personal_scope" in proposal["metadata"]:
|
| 1335 |
title = "Review a personal change"
|
|
@@ -1397,11 +1244,7 @@ def file_history(session, path, *, version=0):
|
|
| 1397 |
if type(version) is not int or version < 0:
|
| 1398 |
raise IntelligenceError("bad_version", "Choose a version number.")
|
| 1399 |
if _database_note_path(path):
|
| 1400 |
-
|
| 1401 |
-
return {"path": path, "history": file["history"], "trimmed": 0,
|
| 1402 |
-
"latestVersion": file["latestVersion"], "selectedVersion": file["atVersion"],
|
| 1403 |
-
"previousVersion": None, "diff": None, "canRestore": False,
|
| 1404 |
-
"databaseKey": file["databaseKey"], "revisionHistory": "database-note", "limits": limits()}
|
| 1405 |
documents = _files(session, include_generated=wiki_log.reserved(path), wiki_only=path.startswith(WIKI_PREFIX))
|
| 1406 |
if wiki_log.purpose_file(path):
|
| 1407 |
purpose = wiki_log.purpose_record(session, path, _project_templates(session, _document(session.runtime.get(STORE_KEY))))
|
|
@@ -1437,9 +1280,6 @@ def restore_file(session, path, version, *, expected_version):
|
|
| 1437 |
_authorize(session)
|
| 1438 |
path = normalize_path(path)
|
| 1439 |
_assert_authored_path(path)
|
| 1440 |
-
if _database_note_path(path):
|
| 1441 |
-
_database_note_file(session, path)
|
| 1442 |
-
raise IntelligenceError("database_note_reserved", "Edit the Database note through its shared note history.", 403)
|
| 1443 |
if type(version) is not int or version < 1 or type(expected_version) is not int or expected_version < 1:
|
| 1444 |
raise IntelligenceError("bad_version", "Choose a saved version and the current file version.")
|
| 1445 |
identity = trusted_identity(session)[1]
|
|
@@ -1548,158 +1388,6 @@ def insights(session, *, now=None):
|
|
| 1548 |
{"key": "sheets", "label": "Sheets"}, {"key": "apps", "label": "Apps"}]}
|
| 1549 |
|
| 1550 |
|
| 1551 |
-
def _write_database_note(session, identity, path, body, *, expected_version=None):
|
| 1552 |
-
"""Save a person's exact note text or store an agent proposal without a Brain head."""
|
| 1553 |
-
from core import external_access
|
| 1554 |
-
|
| 1555 |
-
current = _database_note_file(session, path)
|
| 1556 |
-
key = current["databaseKey"]
|
| 1557 |
-
if not isinstance(body, str):
|
| 1558 |
-
raise IntelligenceError("no_body", "A Database note needs text.")
|
| 1559 |
-
if len(body) > 100000 or len(body.encode("utf-8")) > MAX_BODY_BYTES:
|
| 1560 |
-
raise IntelligenceError("body_too_long", "A Database note can be at most 100,000 characters and 128 KB.", 413)
|
| 1561 |
-
if expected_version is not None and expected_version != current["version"]:
|
| 1562 |
-
raise IntelligenceError("version_conflict", "The note changed. Reload it before saving.", 409)
|
| 1563 |
-
if not identity.get("agent"):
|
| 1564 |
-
note = _put_database_note(session, key, body, identity["author"], current["version"], current["updated"])
|
| 1565 |
-
_log_write(session, path, identity, current["version"], note["version"])
|
| 1566 |
-
return _database_note_file(session, path)
|
| 1567 |
-
if not external_access.may_write(credential_scope(session)[0]):
|
| 1568 |
-
raise IntelligenceError("key_read_only", "This credential is read only.", 403)
|
| 1569 |
-
from routes_agent_harness import MAX_PENDING_PROPOSALS, MAX_PROPOSALS_PER_AGENT
|
| 1570 |
-
|
| 1571 |
-
proposal_id = "ip_" + write_execution.new_id()[:24]
|
| 1572 |
-
saved = {}
|
| 1573 |
-
|
| 1574 |
-
def propose(raw):
|
| 1575 |
-
saved.clear()
|
| 1576 |
-
documents = dict(_document(raw))
|
| 1577 |
-
old = documents.get(path) or {}
|
| 1578 |
-
record = _database_note_record(session, key, stored=old, with_body=True)
|
| 1579 |
-
head = _head(record)
|
| 1580 |
-
if expected_version is not None and head["version"] != expected_version:
|
| 1581 |
-
raise IntelligenceError("version_conflict", "The note changed. Reload it before saving.", 409)
|
| 1582 |
-
proposals = dict(old.get("proposals") or {})
|
| 1583 |
-
if sum(row["status"] in ("pending", "approving") for row in proposals.values()) >= MAX_PENDING_PROPOSALS:
|
| 1584 |
-
raise IntelligenceError("too_many_proposals", "Review the pending changes before adding another.", 409)
|
| 1585 |
-
if len(proposals) >= MAX_PROPOSALS_PER_AGENT:
|
| 1586 |
-
terminal = next((name for name, row in proposals.items() if row["status"] in ("approved", "rejected", "stale")), None)
|
| 1587 |
-
if terminal:
|
| 1588 |
-
del proposals[terminal]
|
| 1589 |
-
proposal = _new_proposal(session, record, body, head["metadata"], identity, proposal_id)
|
| 1590 |
-
proposals[proposal_id] = proposal
|
| 1591 |
-
# This envelope holds proposals only. Canonical text and revisions stay in shared_notes.
|
| 1592 |
-
documents[path] = {"path": path, "databaseKey": key, "versions": [], "proposals": proposals}
|
| 1593 |
-
saved.update(proposal)
|
| 1594 |
-
return documents
|
| 1595 |
-
|
| 1596 |
-
_update(session, propose)
|
| 1597 |
-
try:
|
| 1598 |
-
delivery = _notify_proposal(session, saved)
|
| 1599 |
-
except Exception:
|
| 1600 |
-
delivery = "failed"
|
| 1601 |
-
record = _files(session).get(path, {})
|
| 1602 |
-
if not record or not _proposal_readable(session, record, saved):
|
| 1603 |
-
raise IntelligenceError("no_proposal", "This proposal is unavailable.", 404)
|
| 1604 |
-
return {"proposal": _public_proposal(session, saved, record), "delivery": delivery}
|
| 1605 |
-
|
| 1606 |
-
|
| 1607 |
-
def _decide_database_note(session, path, proposal_id, decision):
|
| 1608 |
-
"""Claim a decision, write through the canonical note's atomic guard, then retain its receipt."""
|
| 1609 |
-
key = path[len(DATABASE_NOTES_PREFIX):-3]
|
| 1610 |
-
saved = {}
|
| 1611 |
-
|
| 1612 |
-
def claim(raw):
|
| 1613 |
-
saved.clear()
|
| 1614 |
-
documents = dict(_document(raw))
|
| 1615 |
-
old = documents.get(path) or {}
|
| 1616 |
-
proposal = deepcopy((old.get("proposals") or {}).get(proposal_id))
|
| 1617 |
-
if not proposal or not _proposal_readable(session, old, proposal):
|
| 1618 |
-
raise IntelligenceError("no_proposal", "This proposal is unavailable.", 404)
|
| 1619 |
-
eligible = _recipients(session, proposal["policy"], record=old, proposal=proposal)
|
| 1620 |
-
if session.uname not in eligible:
|
| 1621 |
-
raise IntelligenceError("not_approver", "Your Database role does not allow note approval.", 403)
|
| 1622 |
-
record = _database_note_record(session, key, with_body=True)
|
| 1623 |
-
head = _head(record)
|
| 1624 |
-
if proposal["status"] == "approving" and head["version"] == proposal["baseVersion"] + 1 \
|
| 1625 |
-
and head["body"] == proposal["body"] and head["author"] == proposal["author"][:120]:
|
| 1626 |
-
# Recover a canonical write whose decision receipt was interrupted.
|
| 1627 |
-
proposal.update(status="approved", approvedVersion=head["version"], approvedAt=head["created"])
|
| 1628 |
-
if proposal["status"] == "approved" and decision == "approve":
|
| 1629 |
-
pass
|
| 1630 |
-
elif proposal["status"] == "rejected" and decision == "reject":
|
| 1631 |
-
pass
|
| 1632 |
-
elif proposal["status"] in ("pending", "approving") and _stale(record, proposal):
|
| 1633 |
-
proposal["status"] = "stale"
|
| 1634 |
-
saved["stale"] = True
|
| 1635 |
-
elif proposal["status"] == "stale":
|
| 1636 |
-
saved["stale"] = True
|
| 1637 |
-
elif proposal["status"] not in ("pending", "approving"):
|
| 1638 |
-
raise IntelligenceError("proposal_not_pending", "This proposal already has a decision.", 409)
|
| 1639 |
-
elif decision == "reject":
|
| 1640 |
-
proposal.update(status="rejected", rejectedBy=session.uname)
|
| 1641 |
-
else:
|
| 1642 |
-
proposal["approvedBy"] = list(dict.fromkeys([
|
| 1643 |
-
*[name for name in proposal["approvedBy"] if name in eligible], session.uname]))
|
| 1644 |
-
proposal["status"] = "approving"
|
| 1645 |
-
saved["write"] = True
|
| 1646 |
-
documents[path] = {"path": path, "databaseKey": key, "versions": [],
|
| 1647 |
-
"proposals": {**(old.get("proposals") or {}), proposal_id: proposal}}
|
| 1648 |
-
saved["proposal"] = proposal
|
| 1649 |
-
return documents
|
| 1650 |
-
|
| 1651 |
-
_update(session, claim)
|
| 1652 |
-
if saved.get("stale"):
|
| 1653 |
-
raise IntelligenceError("proposal_stale", "The note changed. Ask the agent for a new proposal.", 409)
|
| 1654 |
-
proposal = saved["proposal"]
|
| 1655 |
-
if saved.get("write"):
|
| 1656 |
-
wrote = True
|
| 1657 |
-
try:
|
| 1658 |
-
note = _put_database_note(session, key, proposal["body"], proposal["author"],
|
| 1659 |
-
proposal["baseVersion"], proposal["baseCreated"])
|
| 1660 |
-
except IntelligenceError as exc:
|
| 1661 |
-
if exc.code != "version_conflict":
|
| 1662 |
-
raise
|
| 1663 |
-
accepted = ((_document(session.runtime.get(STORE_KEY)).get(path) or {}).get("proposals") or {}).get(proposal_id) or {}
|
| 1664 |
-
current = _database_note_record(session, key, with_body=True)
|
| 1665 |
-
head = _head(current)
|
| 1666 |
-
if accepted.get("status") == "approved":
|
| 1667 |
-
return {"proposal": _public_proposal(session, accepted, current), "file": _database_note_file(session, path)}
|
| 1668 |
-
if (head["version"] == proposal["baseVersion"] + 1 and head["body"] == proposal["body"]
|
| 1669 |
-
and head["author"] == proposal["author"][:120]):
|
| 1670 |
-
note = current["note"]
|
| 1671 |
-
wrote = False
|
| 1672 |
-
else:
|
| 1673 |
-
|
| 1674 |
-
def stale(raw):
|
| 1675 |
-
documents = deepcopy(_document(raw))
|
| 1676 |
-
documents[path]["proposals"][proposal_id]["status"] = "stale"
|
| 1677 |
-
return documents
|
| 1678 |
-
|
| 1679 |
-
_update(session, stale)
|
| 1680 |
-
raise IntelligenceError("proposal_stale", "The note changed. Ask the agent for a new proposal.", 409) from None
|
| 1681 |
-
|
| 1682 |
-
def receipt(raw):
|
| 1683 |
-
documents = deepcopy(_document(raw))
|
| 1684 |
-
proposals = documents[path]["proposals"]
|
| 1685 |
-
proposals[proposal_id].update(status="approved", approvedVersion=note["version"], approvedAt=note["updated"])
|
| 1686 |
-
for name, other in proposals.items():
|
| 1687 |
-
if name != proposal_id and other["status"] in ("pending", "approving"):
|
| 1688 |
-
other["status"] = "stale"
|
| 1689 |
-
return documents
|
| 1690 |
-
|
| 1691 |
-
_update(session, receipt)
|
| 1692 |
-
identity = {name: proposal[name] for name in ("author", "authorKind", "agent", "onBehalfOf")}
|
| 1693 |
-
if wrote:
|
| 1694 |
-
_log_write(session, path, identity, proposal["baseVersion"], note["version"])
|
| 1695 |
-
record = _files(session).get(path, {})
|
| 1696 |
-
proposal = (record.get("proposals") or {}).get(proposal_id)
|
| 1697 |
-
if not proposal or not _proposal_readable(session, record, proposal):
|
| 1698 |
-
raise IntelligenceError("no_proposal", "This proposal is unavailable.", 404)
|
| 1699 |
-
return {"proposal": _public_proposal(session, proposal, record),
|
| 1700 |
-
**({"file": _database_note_file(session, path)} if proposal["status"] == "approved" else {})}
|
| 1701 |
-
|
| 1702 |
-
|
| 1703 |
def _purpose_metadata(session, path, metadata, documents=None):
|
| 1704 |
"""Keep purpose ownership and authorization bound to the current full object identity."""
|
| 1705 |
import wiki_log
|
|
@@ -1808,8 +1496,6 @@ def write_file(caller, path, body, *, author_kind="user", expected_version=None,
|
|
| 1808 |
raise IntelligenceError("bad_author_kind", "authorKind must be user or agent.")
|
| 1809 |
if expected_version is not None and (type(expected_version) is not int or expected_version < 0):
|
| 1810 |
raise IntelligenceError("bad_version", "expectedVersion must be a whole number of zero or more.")
|
| 1811 |
-
if _database_note_path(path):
|
| 1812 |
-
return _write_database_note(session, identity, path, body, expected_version=expected_version)
|
| 1813 |
text, metadata = prepare_markdown(body, session.uname)
|
| 1814 |
import wiki_log
|
| 1815 |
if wiki_log.purpose_file(path):
|
|
@@ -1942,10 +1628,6 @@ def decide_proposal(session, proposal_id, decision):
|
|
| 1942 |
raise IntelligenceError("person_required", "A person must review this change.", 403)
|
| 1943 |
if decision not in ("approve", "reject"):
|
| 1944 |
raise IntelligenceError("bad_decision", "Choose Approve or Decline.")
|
| 1945 |
-
for path, record in _document(session.runtime.get(STORE_KEY)).items():
|
| 1946 |
-
if _database_note_path(path) and proposal_id in (record.get("proposals") or {}):
|
| 1947 |
-
return _decide_database_note(session, path, proposal_id, decision)
|
| 1948 |
-
saved = {}
|
| 1949 |
|
| 1950 |
def decide(raw):
|
| 1951 |
saved.clear()
|
|
@@ -2024,18 +1706,6 @@ def rename_file(caller, path, title, *, expected_version):
|
|
| 2024 |
raise IntelligenceError("bad_title", "Use a title of 1 to 200 characters.")
|
| 2025 |
if type(expected_version) is not int or expected_version < 1:
|
| 2026 |
raise IntelligenceError("bad_version", "Reload the file before trying again.")
|
| 2027 |
-
if _database_note_path(path):
|
| 2028 |
-
if identity.get("agent"):
|
| 2029 |
-
raise IntelligenceError("approval_required", "An agent cannot rename a shared note.", 403)
|
| 2030 |
-
record = _database_note_record(session, path[len(DATABASE_NOTES_PREFIX):-3], with_body=True)
|
| 2031 |
-
if record["path"] != path:
|
| 2032 |
-
raise IntelligenceError("no_file", "This workspace has no file at that path.", 404)
|
| 2033 |
-
entry = _head(record)
|
| 2034 |
-
_put_database_note(session, record["databaseKey"], entry["body"], session.uname,
|
| 2035 |
-
expected_version, entry["created"], title=title.strip())
|
| 2036 |
-
saved = _database_note_file(session, path)
|
| 2037 |
-
_log_write(session, path, identity, expected_version, saved["version"])
|
| 2038 |
-
return saved
|
| 2039 |
saved = {}
|
| 2040 |
|
| 2041 |
def rename(raw):
|
|
@@ -2065,25 +1735,13 @@ def rename_file(caller, path, title, *, expected_version):
|
|
| 2065 |
|
| 2066 |
|
| 2067 |
def delete_file(caller, path, *, expected_version=None):
|
| 2068 |
-
"""Delete an editable workspace file
|
| 2069 |
session, identity = trusted_identity(caller)
|
| 2070 |
_authorize(session)
|
| 2071 |
path = normalize_path(path)
|
| 2072 |
_assert_authored_path(path)
|
| 2073 |
if expected_version is not None and (type(expected_version) is not int or expected_version < 1):
|
| 2074 |
raise IntelligenceError("bad_version", "Reload the file before trying again.")
|
| 2075 |
-
if _database_note_path(path):
|
| 2076 |
-
if identity.get("agent"):
|
| 2077 |
-
raise IntelligenceError("approval_required", "An agent cannot delete a shared note.", 403)
|
| 2078 |
-
record = _database_note_record(session, path[len(DATABASE_NOTES_PREFIX):-3], with_body=True)
|
| 2079 |
-
if record["path"] != path:
|
| 2080 |
-
raise IntelligenceError("no_file", "This workspace has no file at that path.", 404)
|
| 2081 |
-
entry = _head(record)
|
| 2082 |
-
_put_database_note(session, record["databaseKey"], "", session.uname,
|
| 2083 |
-
expected_version if expected_version is not None else entry["version"],
|
| 2084 |
-
entry["created"], deleted=True)
|
| 2085 |
-
_log_write(session, path, identity, entry["version"], None)
|
| 2086 |
-
return {"deleted": path}
|
| 2087 |
removed = {}
|
| 2088 |
|
| 2089 |
def remove(raw):
|
|
|
|
| 86 |
|
| 87 |
|
| 88 |
def _assert_authored_path(path):
|
| 89 |
+
if _database_note_path(path):
|
| 90 |
+
raise IntelligenceError("no_file", "This workspace has no file at that path.", 404)
|
| 91 |
import wiki_log
|
| 92 |
if wiki_log.reserved(path):
|
| 93 |
raise IntelligenceError("wiki_generated", "The system maintains wiki/log/ and wiki/index/.", 403)
|
|
|
|
| 277 |
_check_file_capacity(session, raw)
|
| 278 |
import wiki_log
|
| 279 |
files = _project_templates(session, raw)
|
| 280 |
+
files = {path: record for path, record in files.items() if not _database_note_path(path)}
|
|
|
|
| 281 |
files = {path: record for path, record in files.items() if not wiki_log.reserved(path)}
|
| 282 |
files = {path: record for path, record in files.items()
|
| 283 |
if not wiki_log.purpose_file(path) or wiki_log.purpose_record(session, path, files) is not None}
|
|
|
|
| 295 |
return path.casefold().startswith(DATABASE_NOTES_PREFIX)
|
| 296 |
|
| 297 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
@lru_cache(maxsize=1)
|
| 299 |
def _template_catalog():
|
| 300 |
"""Cache only shipped text, never tenant connections, grants, schemas or overrides."""
|
|
|
|
| 620 |
if name not in self._documents:
|
| 621 |
if name == STORE_KEY:
|
| 622 |
reader = getattr(self._runtime, "get_many", None)
|
| 623 |
+
values = reader([STORE_KEY, TEMPLATE_USAGE_KEY, "keychain"]) if callable(reader) else None
|
| 624 |
if values is not None:
|
| 625 |
self._documents.update(values)
|
| 626 |
if name not in self._documents:
|
|
|
|
| 879 |
entry = _head(record)
|
| 880 |
if not entry or not can_read(session, entry["metadata"]):
|
| 881 |
continue
|
| 882 |
+
keys = _link_index(record)["keys"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 883 |
if database in keys and (field is None or field in keys):
|
| 884 |
meta = entry["metadata"]
|
| 885 |
rows.append({"path": path, "title": meta["title"], "section": meta["section"],
|
|
|
|
| 926 |
if type(version) is not int or version < 0:
|
| 927 |
raise IntelligenceError("bad_version", "Use a positive version number, or zero for the latest version.")
|
| 928 |
if _database_note_path(path):
|
| 929 |
+
raise IntelligenceError("no_file", "This workspace has no file at that path.", 404)
|
| 930 |
|
| 931 |
def readable(raw):
|
| 932 |
record = _project_templates(session, _document(raw)).get(path)
|
|
|
|
| 1064 |
or not wiki_log.purpose_object(session, proposal["metadata"])):
|
| 1065 |
return False
|
| 1066 |
if _database_note_path(record.get("path", "")):
|
| 1067 |
+
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1068 |
template = _template_catalog().get(record.get("path"))
|
| 1069 |
if template and record["path"] not in _project_templates(session, {record["path"]: record}):
|
| 1070 |
return False
|
|
|
|
| 1101 |
if wiki_log.purpose_file(record.get("path", "")):
|
| 1102 |
obj = wiki_log.purpose_object(recipient, proposal["metadata"])
|
| 1103 |
matches = bool(obj and obj["owner"] == name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1104 |
if matches:
|
| 1105 |
try:
|
| 1106 |
if not _proposal_readable(recipient, record, proposal):
|
|
|
|
| 1134 |
row["approvedBy"] = [name for name in proposal["approvedBy"] if name in eligible]
|
| 1135 |
row["recipients"] = eligible
|
| 1136 |
row["canDecide"] = (not trusted_identity(session)[1].get("agent") and session.uname in eligible
|
| 1137 |
+
and proposal["status"] == "pending" and session.uname not in row["approvedBy"])
|
|
|
|
|
|
|
|
|
|
| 1138 |
return row
|
| 1139 |
|
| 1140 |
|
|
|
|
| 1176 |
send = getattr(module, "send_request", None)
|
| 1177 |
title = f"Review {proposal['title']}"
|
| 1178 |
body = f"{proposal['path']}\nProposal {proposal['id']}\n\n{proposal['diff']}"
|
| 1179 |
+
if "personal_scope" in proposal["metadata"]:
|
| 1180 |
# Inbox copies cannot keep private content after its source access is revoked.
|
| 1181 |
if "personal_scope" in proposal["metadata"]:
|
| 1182 |
title = "Review a personal change"
|
|
|
|
| 1244 |
if type(version) is not int or version < 0:
|
| 1245 |
raise IntelligenceError("bad_version", "Choose a version number.")
|
| 1246 |
if _database_note_path(path):
|
| 1247 |
+
raise IntelligenceError("no_file", "This workspace has no file at that path.", 404)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1248 |
documents = _files(session, include_generated=wiki_log.reserved(path), wiki_only=path.startswith(WIKI_PREFIX))
|
| 1249 |
if wiki_log.purpose_file(path):
|
| 1250 |
purpose = wiki_log.purpose_record(session, path, _project_templates(session, _document(session.runtime.get(STORE_KEY))))
|
|
|
|
| 1280 |
_authorize(session)
|
| 1281 |
path = normalize_path(path)
|
| 1282 |
_assert_authored_path(path)
|
|
|
|
|
|
|
|
|
|
| 1283 |
if type(version) is not int or version < 1 or type(expected_version) is not int or expected_version < 1:
|
| 1284 |
raise IntelligenceError("bad_version", "Choose a saved version and the current file version.")
|
| 1285 |
identity = trusted_identity(session)[1]
|
|
|
|
| 1388 |
{"key": "sheets", "label": "Sheets"}, {"key": "apps", "label": "Apps"}]}
|
| 1389 |
|
| 1390 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1391 |
def _purpose_metadata(session, path, metadata, documents=None):
|
| 1392 |
"""Keep purpose ownership and authorization bound to the current full object identity."""
|
| 1393 |
import wiki_log
|
|
|
|
| 1496 |
raise IntelligenceError("bad_author_kind", "authorKind must be user or agent.")
|
| 1497 |
if expected_version is not None and (type(expected_version) is not int or expected_version < 0):
|
| 1498 |
raise IntelligenceError("bad_version", "expectedVersion must be a whole number of zero or more.")
|
|
|
|
|
|
|
| 1499 |
text, metadata = prepare_markdown(body, session.uname)
|
| 1500 |
import wiki_log
|
| 1501 |
if wiki_log.purpose_file(path):
|
|
|
|
| 1628 |
raise IntelligenceError("person_required", "A person must review this change.", 403)
|
| 1629 |
if decision not in ("approve", "reject"):
|
| 1630 |
raise IntelligenceError("bad_decision", "Choose Approve or Decline.")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1631 |
|
| 1632 |
def decide(raw):
|
| 1633 |
saved.clear()
|
|
|
|
| 1706 |
raise IntelligenceError("bad_title", "Use a title of 1 to 200 characters.")
|
| 1707 |
if type(expected_version) is not int or expected_version < 1:
|
| 1708 |
raise IntelligenceError("bad_version", "Reload the file before trying again.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1709 |
saved = {}
|
| 1710 |
|
| 1711 |
def rename(raw):
|
|
|
|
| 1735 |
|
| 1736 |
|
| 1737 |
def delete_file(caller, path, *, expected_version=None):
|
| 1738 |
+
"""Delete an editable workspace file."""
|
| 1739 |
session, identity = trusted_identity(caller)
|
| 1740 |
_authorize(session)
|
| 1741 |
path = normalize_path(path)
|
| 1742 |
_assert_authored_path(path)
|
| 1743 |
if expected_version is not None and (type(expected_version) is not int or expected_version < 1):
|
| 1744 |
raise IntelligenceError("bad_version", "Reload the file before trying again.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1745 |
removed = {}
|
| 1746 |
|
| 1747 |
def remove(raw):
|
api/main.py
CHANGED
|
@@ -38,7 +38,7 @@ from dotenv import load_dotenv # noqa: E402
|
|
| 38 |
load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
|
| 39 |
import process_role # noqa: E402 (before any DuckDB connect: the web role sets its memory cap)
|
| 40 |
|
| 41 |
-
from fastapi import Body, Depends, FastAPI, Request # noqa: E402
|
| 42 |
from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
|
| 43 |
from fastapi.responses import JSONResponse # noqa: E402
|
| 44 |
from fastapi.staticfiles import StaticFiles # noqa: E402
|
|
@@ -46,13 +46,11 @@ from starlette.exceptions import HTTPException as StarletteHTTPException # noqa
|
|
| 46 |
|
| 47 |
import aios_session # noqa: E402
|
| 48 |
import routes_admin # noqa: E402
|
| 49 |
-
import routes_competitor_watch # noqa: E402 (S78-T16 / S78-T17 competitor watch)
|
| 50 |
import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox)
|
| 51 |
import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery)
|
| 52 |
import routes_auth # noqa: E402
|
| 53 |
import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A)
|
| 54 |
import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A)
|
| 55 |
-
import routes_notes # noqa: E402 (wave 47 instr 19 / W47-T16 — shared-note read state, lane C's router)
|
| 56 |
import routes_charts # noqa: E402 (wave 47 instr 18 / W47-T30 / R7 — saved charts as records, lane H's router)
|
| 57 |
import routes_sheet # noqa: E402 (wave 49 instr 1 / W49-T02 / C1 — the Sheet workbook doors, lane A's router)
|
| 58 |
import routes_customers # noqa: E402
|
|
@@ -238,13 +236,13 @@ async def _unexpected_error(request: Request, exc: Exception):
|
|
| 238 |
status_code=500)
|
| 239 |
|
| 240 |
|
| 241 |
-
from core.odoo import OdooPoolTimeout, OdooRequestTimeout
|
| 242 |
from core.store_pg import SharedReadTimeout, PostgresOperationTimeout
|
| 243 |
from core.request_timing import RequestDeadlineExceeded
|
| 244 |
from request_timing import DeadlineCapacityExceeded
|
| 245 |
from duckdb import OutOfMemoryException
|
| 246 |
# Each busy cause needs its own handler: the catch-all `Exception` one answers, then re-raises to uvicorn.
|
| 247 |
-
for _resource_error in (OdooPoolTimeout, OdooRequestTimeout, SharedReadTimeout, PostgresOperationTimeout,
|
| 248 |
RequestDeadlineExceeded, DeadlineCapacityExceeded, OutOfMemoryException):
|
| 249 |
app.add_exception_handler(_resource_error, _resource_unavailable)
|
| 250 |
try:
|
|
@@ -410,24 +408,59 @@ def _mirror_answers():
|
|
| 410 |
return False
|
| 411 |
|
| 412 |
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 416 |
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 424 |
if report["ready"]:
|
| 425 |
-
|
|
|
|
| 426 |
body["failing"] = report["failing"]
|
| 427 |
body["error"] = {"code": "not_ready",
|
| 428 |
"message": "Not ready: " + ", ".join(report["failing"]) + ". "
|
| 429 |
+ " ".join(report["detail"].values())}
|
| 430 |
-
|
|
|
|
| 431 |
|
| 432 |
|
| 433 |
# W59-T43: boot stays available while a cross-lane module is absent, but never silently.
|
|
@@ -554,9 +587,7 @@ app.include_router(routes_web_agent.router) # R10 / C5 — E's router, A's lin
|
|
| 554 |
# `main.py` can sit in exactly ONE lane's fence, so a router built in any other lane has nowhere to
|
| 555 |
# be mounted from, and the line gets deferred to a session that no longer exists. Mounting empty
|
| 556 |
# routers at plan time removes the failure mode instead of warning about it a sixth time.
|
| 557 |
-
#
|
| 558 |
-
# (W47-T30 / R7, saved charts as their own records). Neither lane may edit this file.
|
| 559 |
-
app.include_router(routes_notes.router)
|
| 560 |
app.include_router(routes_charts.router)
|
| 561 |
# ⭐ WAVE 49 (instruction 1 / C1, W49-T02) — THE SHEET WORKBOOK DOORS. Lane A owns both this file
|
| 562 |
# and that router, so this is the one mount in this block that crosses no fence. It is still
|
|
@@ -655,12 +686,10 @@ app.include_router(routes_tickets.router)
|
|
| 655 |
app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers)
|
| 656 |
app.include_router(routes_external.router)
|
| 657 |
import app_routes
|
| 658 |
-
app.include_router(routes_competitor_watch.router, # S78-T16 / S78-T17 competitor watch; held by R11
|
| 659 |
-
dependencies=app_routes.held_dependencies("competitor_watch"))
|
| 660 |
_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
|
| 661 |
_mount_router("routes_amazon_profit") # S78-T03: the Amazon SKU profit refresh and settings doors.
|
| 662 |
_mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
|
| 663 |
-
app_routes.mount_app_routers(app)
|
| 664 |
|
| 665 |
|
| 666 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
|
@@ -950,7 +979,7 @@ def _pull_meta(why, tenant_key=None):
|
|
| 950 |
outcomes[key] = False
|
| 951 |
try:
|
| 952 |
rt = runtime.get_runtime(key)
|
| 953 |
-
if not _meta.
|
| 954 |
continue
|
| 955 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 956 |
log=lambda *_a: None, insights_days=7)
|
|
@@ -964,6 +993,65 @@ def _pull_meta(why, tenant_key=None):
|
|
| 964 |
return outcomes.get(tenant_key or runtime.tenant_for_store_path(), False)
|
| 965 |
|
| 966 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 967 |
def _pull_amazon(why, vendor=True):
|
| 968 |
"""Produce Amazon rows in this container before projecting them, on the caller's thread.
|
| 969 |
|
|
@@ -983,6 +1071,7 @@ def _pull_amazon(why, vendor=True):
|
|
| 983 |
rt.datastore_path.resolve() != _ds.DB_PATH.resolve() or
|
| 984 |
_ds.path_for(tenant_key).resolve() != _ds.DB_PATH.resolve()):
|
| 985 |
raise RuntimeError("Amazon sync requires the requesting tenant's resident mirror.")
|
|
|
|
| 986 |
if not _amazon.credential(tenant_key, rt=rt):
|
| 987 |
print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential "
|
| 988 |
"for royal-imports; the connector is idle, not broken")
|
|
@@ -1075,6 +1164,7 @@ def _restore_amazon_mirror_once(why, tenant_key):
|
|
| 1075 |
if (rt.key != tenant_key or
|
| 1076 |
rt.datastore_path.resolve() != _ds.path_for(tenant_key).resolve()):
|
| 1077 |
raise RuntimeError("Amazon restore requires the requesting tenant's runtime.")
|
|
|
|
| 1078 |
if rt.download_bytes(_amazon.MIRROR_PREFIX + "manifest.json") is None:
|
| 1079 |
RESYNC_STATE.setdefault("amazonRestoreErrors", {}).pop(tenant_key, None)
|
| 1080 |
return True
|
|
@@ -1172,6 +1262,20 @@ class _SyncNowRefused(Exception):
|
|
| 1172 |
def _sync_now_cause(cause):
|
| 1173 |
"""Classify actual provider failures without returning vendor text or exception reprs."""
|
| 1174 |
import re
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1175 |
raw = str(cause)
|
| 1176 |
text = raw.lower()
|
| 1177 |
causes = (
|
|
@@ -1213,7 +1317,15 @@ def _sync_now_cause(cause):
|
|
| 1213 |
if isinstance(cause, TypeError):
|
| 1214 |
return "bad_arguments", "The connector failed because its provider call used incompatible arguments."
|
| 1215 |
if isinstance(cause, (KeyError, ValueError)):
|
| 1216 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1217 |
if isinstance(cause, str):
|
| 1218 |
# amazon_store._sync_problem appends the application-authored fix LAST. The
|
| 1219 |
# exception message may itself contain delimiters; none of it is display text.
|
|
@@ -1244,6 +1356,27 @@ def _sync_now_failure(cause):
|
|
| 1244 |
return _sync_now_cause(cause)[1]
|
| 1245 |
|
| 1246 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1247 |
def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
|
| 1248 |
"""Record one classified failure; leave successful receipt text untouched."""
|
| 1249 |
if ok:
|
|
@@ -1255,6 +1388,8 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
|
|
| 1255 |
locations = " > ".join(f"{os.path.basename(frame.filename)}:{frame.lineno}:{frame.name}"
|
| 1256 |
for frame in traceback.extract_tb(cause.__traceback__))
|
| 1257 |
print(f"[aios-api] sync failure location {type(cause).__name__}: {locations}")
|
|
|
|
|
|
|
| 1258 |
suffix = f" (cause code: {classifier})"
|
| 1259 |
# finish_sync_request stores only 300 characters. Reserve the suffix before it clips.
|
| 1260 |
budget = 300 - len(suffix)
|
|
@@ -1272,9 +1407,82 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
|
|
| 1272 |
project only what a desk wake actually changed.
|
| 1273 |
"""
|
| 1274 |
from core import connector_sync
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1275 |
odoo_requested = False
|
| 1276 |
-
|
| 1277 |
-
pending
|
|
|
|
| 1278 |
while True:
|
| 1279 |
receipt = next((row for row in pending
|
| 1280 |
if not during_odoo or row["provider"] != "odoo"), None)
|
|
@@ -1287,6 +1495,8 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
|
|
| 1287 |
ok, detail = False, ""
|
| 1288 |
first_load_progress, continuing = None, False
|
| 1289 |
floor_wait = None
|
|
|
|
|
|
|
| 1290 |
try:
|
| 1291 |
from harness import runtime, datastore, mirror_store
|
| 1292 |
rt = runtime.get_runtime(tenant)
|
|
@@ -1298,61 +1508,37 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
|
|
| 1298 |
datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()):
|
| 1299 |
raise _SyncNowRefused(
|
| 1300 |
"This tenant's mirror is not resident on this server. Sync requires its tenant worker.")
|
|
|
|
| 1301 |
if provider == "odoo":
|
| 1302 |
import core.odoo as odoo
|
| 1303 |
import odoo_relational as rel
|
| 1304 |
import routes_connectors as _sync_routes
|
| 1305 |
if rt.odoo_paused() or rel.frozen(rt):
|
| 1306 |
raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.")
|
| 1307 |
-
requested_cid = (
|
|
|
|
| 1308 |
or _sync_routes._odoo_connection_slot(rt, None))
|
| 1309 |
-
|
| 1310 |
-
|
| 1311 |
-
|
| 1312 |
-
|
| 1313 |
-
|
| 1314 |
-
|
| 1315 |
-
|
| 1316 |
-
|
| 1317 |
-
|
| 1318 |
-
|
| 1319 |
-
|
| 1320 |
-
|
| 1321 |
-
|
| 1322 |
-
|
| 1323 |
-
|
| 1324 |
-
|
| 1325 |
-
|
| 1326 |
-
|
| 1327 |
-
previous_client = getattr(odoo._tlocal, "client", None)
|
| 1328 |
-
previous_context = getattr(odoo._tlocal, "tenant_ctx", None)
|
| 1329 |
-
try:
|
| 1330 |
-
# Raw mirror pulls use the resolved tenant credential and a dedicated
|
| 1331 |
-
# transport. Do not borrow a request's cached XML-RPC client.
|
| 1332 |
-
odoo._tlocal.client = _DeskAwareOdooClient(
|
| 1333 |
-
lambda: odoo.OdooClient(creds=source._creds))
|
| 1334 |
-
odoo._tlocal.tenant_ctx = None
|
| 1335 |
-
from connector_onboarding import selected_mirror_entities
|
| 1336 |
-
selected_entities = selected_mirror_entities(rt)
|
| 1337 |
-
# Sync now counts each entity's source beside its copy, as the full pass does.
|
| 1338 |
-
# Looked up by name: extracted test fixtures of this function omit the counter.
|
| 1339 |
-
_counter = globals().get("_entity_source_counter")
|
| 1340 |
-
_after_entity = _counter(datastore) if _counter else None
|
| 1341 |
-
report = mirror_store.tracked_sync("odoo", rt, datastore.sync_all, entities=selected_entities,
|
| 1342 |
-
log=lambda *a, **k: None,
|
| 1343 |
-
**({"after_entity": _after_entity} if _after_entity else {}))
|
| 1344 |
-
finally:
|
| 1345 |
-
odoo._tlocal.client = previous_client
|
| 1346 |
-
odoo._tlocal.tenant_ctx = previous_context
|
| 1347 |
-
if not report and selected_entities != []:
|
| 1348 |
-
raise _SyncNowRefused(
|
| 1349 |
-
"Odoo did not run a sync. Check whether the connection is paused.")
|
| 1350 |
-
if any(row.get("phase") != "live" for row in report.values()):
|
| 1351 |
-
raise _SyncNowRefused(
|
| 1352 |
-
"Odoo backfill is incomplete. Run Sync now again to continue it.")
|
| 1353 |
-
problems = []
|
| 1354 |
elif provider == "amazon_sp":
|
| 1355 |
from harness import amazon_store as amazon
|
|
|
|
| 1356 |
if not amazon.credential(tenant, rt=rt):
|
| 1357 |
raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
|
| 1358 |
from connectors_amazon import notification_sync_options
|
|
@@ -1361,10 +1547,11 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
|
|
| 1361 |
first_load = False if targeted else amazon.needs_first_load(tenant, rt=rt)
|
| 1362 |
options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS,
|
| 1363 |
"refresh_loaded": True} if first_load else {}
|
|
|
|
| 1364 |
report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt,
|
| 1365 |
scheduled=False if targeted else receipt.get("requestedBy") == "first load",
|
| 1366 |
**target_options, **options,
|
| 1367 |
-
log=
|
| 1368 |
print("[aios-api] amazon sync done (sync desk): " + ", ".join(
|
| 1369 |
f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
|
| 1370 |
+ (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left"
|
|
@@ -1378,13 +1565,16 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
|
|
| 1378 |
floor_wait = float(report["nextDueSeconds"])
|
| 1379 |
problems = list(report.get("problems") or []) + list(report.get("audit_problems") or [])
|
| 1380 |
tables = report.get("tables") or {}
|
|
|
|
|
|
|
|
|
|
| 1381 |
if not problems and not tables and not continuing:
|
| 1382 |
raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.")
|
| 1383 |
if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()):
|
| 1384 |
raise _SyncNowRefused("All Amazon databases are disabled. Enable a database before syncing.")
|
| 1385 |
elif provider == "meta_ads":
|
| 1386 |
from harness import meta_store as meta
|
| 1387 |
-
if not meta.
|
| 1388 |
raise _SyncNowRefused("Meta Ads has no access token. Connect Meta Ads before syncing.")
|
| 1389 |
report = mirror_store.tracked_sync("meta", rt, meta.sync, tenant,
|
| 1390 |
log=lambda *a, **k: None, insights_days=7)
|
|
@@ -1400,15 +1590,27 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
|
|
| 1400 |
detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; "
|
| 1401 |
f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.")
|
| 1402 |
if problems:
|
| 1403 |
-
detail
|
|
|
|
|
|
|
| 1404 |
elif problems:
|
| 1405 |
-
detail = _sync_now_detail(provider, tenant, ok=False,
|
|
|
|
| 1406 |
else:
|
| 1407 |
ok, detail = True, "Sync finished."
|
|
|
|
|
|
|
| 1408 |
except _SyncNowRefused as exc:
|
| 1409 |
detail = str(exc)
|
| 1410 |
except Exception as exc: # noqa: BLE001
|
| 1411 |
detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1412 |
if first_load_progress is not None:
|
| 1413 |
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail,
|
| 1414 |
first_load=first_load_progress, request_id=receipt.get("id"))
|
|
@@ -2273,6 +2475,8 @@ def _resync_mirror_pass(manual, passes, entities=None):
|
|
| 2273 |
try:
|
| 2274 |
first = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all,
|
| 2275 |
entities=absent, log=_resync_log,
|
|
|
|
|
|
|
| 2276 |
_catch_up_complete=False)
|
| 2277 |
_log_entities("absent", first)
|
| 2278 |
print(f"[aios-api] store resync: synced {', '.join(absent)} first, "
|
|
@@ -2284,11 +2488,18 @@ def _resync_mirror_pass(manual, passes, entities=None):
|
|
| 2284 |
_after_entity = _entity_source_counter(_ds)
|
| 2285 |
sync_report = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all,
|
| 2286 |
entities=entities, log=_resync_log,
|
|
|
|
|
|
|
| 2287 |
**({"after_entity": _after_entity} if _after_entity else {}))
|
| 2288 |
_log_entities("full", sync_report)
|
| 2289 |
for key, row in first.items():
|
| 2290 |
if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict):
|
| 2291 |
sync_report[key]["pulled"] = sync_report[key].get("pulled", 0) + row.get("pulled", 0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2292 |
# Wave 21: purge hard-deleted rows the cursor sync cannot see (the boot pass's comment has
|
| 2293 |
# the measured case); without it deleted Odoo lines inflate every sum on the mirror FOREVER.
|
| 2294 |
# ⚠ INSIDE the non-manual branch: a reconcile is a pass over the mirror this loop was
|
|
@@ -2330,6 +2541,22 @@ def _absent_mirror_entities(ds, entities=None):
|
|
| 2330 |
return []
|
| 2331 |
|
| 2332 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2333 |
def _smallest_first(ds, keys):
|
| 2334 |
"""`keys` ordered by their Odoo row count, so a small table answers before a large backfill."""
|
| 2335 |
def size(key):
|
|
@@ -2499,17 +2726,26 @@ def _entity_source_counter(ds):
|
|
| 2499 |
if not tenant:
|
| 2500 |
return None
|
| 2501 |
counted = _PASS_SOURCE_COUNTS[tenant] = {}
|
|
|
|
| 2502 |
|
| 2503 |
def after_entity(key):
|
| 2504 |
# A volatile table (stock_quant) drifts between its copy and a count at the end of the pass.
|
| 2505 |
try:
|
| 2506 |
import odoo_relational as _rel_counts
|
| 2507 |
model = (ds.ENTITIES.get(key) or {}).get("model")
|
| 2508 |
-
|
|
|
|
|
|
|
|
|
|
| 2509 |
if not buckets:
|
| 2510 |
return
|
| 2511 |
-
got = _rel_counts.capture_source_counts(rt, buckets=buckets
|
| 2512 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2513 |
for bucket, (count, _mode) in sorted(got.items()):
|
| 2514 |
print(f"[aios-api] store resync: counted the Odoo source for {bucket} "
|
| 2515 |
f"beside the {key} copy: {count:,} rows", flush=True)
|
|
@@ -2772,13 +3008,17 @@ def _store_resync_loop():
|
|
| 2772 |
sync_report, delete_report, _err = None, None, _classify_resync_error(exc)
|
| 2773 |
if _err is None and sync_report:
|
| 2774 |
RESYNC_STATE["passes"] = RESYNC_STATE.get("passes", 0) + 1
|
| 2775 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2776 |
RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
|
| 2777 |
RESYNC_STATE["lastPassOk"] = _err is None
|
| 2778 |
RESYNC_STATE["lastError"] = _err
|
| 2779 |
if _err is None:
|
| 2780 |
RESYNC_STATE["consecutiveFailures"] = 0
|
| 2781 |
-
if not _manual:
|
| 2782 |
RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
|
| 2783 |
# A published pull is the Odoo source stage's success, so the receipt's source stamp follows it.
|
| 2784 |
try:
|
|
@@ -2796,6 +3036,10 @@ def _store_resync_loop():
|
|
| 2796 |
print(f"[aios-api] store resync failed ({_err['kind']}, "
|
| 2797 |
f"{RESYNC_STATE['consecutiveFailures']} in a row): "
|
| 2798 |
f"{_err['type']}: {_err['message']}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2799 |
if _boot_catch_up and not _odoo_requested:
|
| 2800 |
# Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
|
| 2801 |
if not catch_up_retries:
|
|
|
|
| 38 |
load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
|
| 39 |
import process_role # noqa: E402 (before any DuckDB connect: the web role sets its memory cap)
|
| 40 |
|
| 41 |
+
from fastapi import Body, Depends, FastAPI, Request, Response # noqa: E402
|
| 42 |
from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
|
| 43 |
from fastapi.responses import JSONResponse # noqa: E402
|
| 44 |
from fastapi.staticfiles import StaticFiles # noqa: E402
|
|
|
|
| 46 |
|
| 47 |
import aios_session # noqa: E402
|
| 48 |
import routes_admin # noqa: E402
|
|
|
|
| 49 |
import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox)
|
| 50 |
import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery)
|
| 51 |
import routes_auth # noqa: E402
|
| 52 |
import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A)
|
| 53 |
import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A)
|
|
|
|
| 54 |
import routes_charts # noqa: E402 (wave 47 instr 18 / W47-T30 / R7 — saved charts as records, lane H's router)
|
| 55 |
import routes_sheet # noqa: E402 (wave 49 instr 1 / W49-T02 / C1 — the Sheet workbook doors, lane A's router)
|
| 56 |
import routes_customers # noqa: E402
|
|
|
|
| 236 |
status_code=500)
|
| 237 |
|
| 238 |
|
| 239 |
+
from core.odoo import OdooPoolTimeout, OdooRequestTimeout, OdooBreakerOpen
|
| 240 |
from core.store_pg import SharedReadTimeout, PostgresOperationTimeout
|
| 241 |
from core.request_timing import RequestDeadlineExceeded
|
| 242 |
from request_timing import DeadlineCapacityExceeded
|
| 243 |
from duckdb import OutOfMemoryException
|
| 244 |
# Each busy cause needs its own handler: the catch-all `Exception` one answers, then re-raises to uvicorn.
|
| 245 |
+
for _resource_error in (OdooPoolTimeout, OdooRequestTimeout, OdooBreakerOpen, SharedReadTimeout, PostgresOperationTimeout,
|
| 246 |
RequestDeadlineExceeded, DeadlineCapacityExceeded, OutOfMemoryException):
|
| 247 |
app.add_exception_handler(_resource_error, _resource_unavailable)
|
| 248 |
try:
|
|
|
|
| 408 |
return False
|
| 409 |
|
| 410 |
|
| 411 |
+
def _tenant_readiness(rt):
|
| 412 |
+
"""Check this tenant's reads without requiring an Odoo mirror for tenants without Odoo."""
|
| 413 |
+
checks = {"mirror": False, "pools": False, "first_reads": False}
|
| 414 |
+
detail = {"mirror": "The tenant mirror check did not answer.",
|
| 415 |
+
"pools": "The tenant store did not answer.",
|
| 416 |
+
"first_reads": "The tenant's first read did not answer."}
|
| 417 |
+
stage = "mirror"
|
| 418 |
+
try:
|
| 419 |
+
if rt.odoo_source() is not None:
|
| 420 |
+
rt.assert_datastore_matches()
|
| 421 |
+
return process_role.readiness(mirror_live=_mirror_answers())
|
| 422 |
+
checks["mirror"] = True
|
| 423 |
+
stage = "pools"
|
| 424 |
+
if not rt.available():
|
| 425 |
+
raise RuntimeError("store unavailable")
|
| 426 |
+
checks["pools"] = True
|
| 427 |
+
stage = "first_reads"
|
| 428 |
+
# HF display reads hide outages, so readiness uses the tenant's strict reader.
|
| 429 |
+
rt._store()._read_strict(rt.store_key("sync_settings"))
|
| 430 |
+
checks["first_reads"] = True
|
| 431 |
+
except Exception as exc: # noqa: BLE001
|
| 432 |
+
detail[stage] += f" ({type(exc).__name__})"
|
| 433 |
+
failing = [name for name in checks if not checks[name]]
|
| 434 |
+
return {"ready": not failing, "checks": checks, "failing": failing,
|
| 435 |
+
"detail": {name: detail[name] for name in failing}}
|
| 436 |
|
| 437 |
+
|
| 438 |
+
@app.get("/api/v1/ready")
|
| 439 |
+
def readiness_check(request: Request, response: Response):
|
| 440 |
+
"""Report tenant readiness publicly and reveal the version only after session verification."""
|
| 441 |
+
session = (deps_require_session(request, response)
|
| 442 |
+
if request.cookies.get(aios_session.COOKIE_NAME) else None)
|
| 443 |
+
try:
|
| 444 |
+
from harness import runtime as _ready_runtime
|
| 445 |
+
rt = (session.runtime if session else
|
| 446 |
+
_ready_runtime.get_runtime(_ready_runtime.tenant_for_store_path()))
|
| 447 |
+
report = _tenant_readiness(rt)
|
| 448 |
+
except Exception as exc: # noqa: BLE001
|
| 449 |
+
report = {"ready": False, "checks": {"tenant": False}, "failing": ["tenant"],
|
| 450 |
+
"detail": {"tenant": f"The tenant could not be resolved ({type(exc).__name__})."}}
|
| 451 |
+
body = {"ready": report["ready"], "checks": report["checks"]}
|
| 452 |
+
if session is not None:
|
| 453 |
+
body["version"] = os.environ.get("AIOS_VERSION") or "unknown"
|
| 454 |
+
response.headers["Cache-Control"] = "no-store"
|
| 455 |
if report["ready"]:
|
| 456 |
+
response.status_code = 200
|
| 457 |
+
return body
|
| 458 |
body["failing"] = report["failing"]
|
| 459 |
body["error"] = {"code": "not_ready",
|
| 460 |
"message": "Not ready: " + ", ".join(report["failing"]) + ". "
|
| 461 |
+ " ".join(report["detail"].values())}
|
| 462 |
+
response.status_code = 503
|
| 463 |
+
return body
|
| 464 |
|
| 465 |
|
| 466 |
# W59-T43: boot stays available while a cross-lane module is absent, but never silently.
|
|
|
|
| 587 |
# `main.py` can sit in exactly ONE lane's fence, so a router built in any other lane has nowhere to
|
| 588 |
# be mounted from, and the line gets deferred to a session that no longer exists. Mounting empty
|
| 589 |
# routers at plan time removes the failure mode instead of warning about it a sixth time.
|
| 590 |
+
# Saved charts mount before the static application.
|
|
|
|
|
|
|
| 591 |
app.include_router(routes_charts.router)
|
| 592 |
# ⭐ WAVE 49 (instruction 1 / C1, W49-T02) — THE SHEET WORKBOOK DOORS. Lane A owns both this file
|
| 593 |
# and that router, so this is the one mount in this block that crosses no fence. It is still
|
|
|
|
| 686 |
app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers)
|
| 687 |
app.include_router(routes_external.router)
|
| 688 |
import app_routes
|
|
|
|
|
|
|
| 689 |
_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
|
| 690 |
_mount_router("routes_amazon_profit") # S78-T03: the Amazon SKU profit refresh and settings doors.
|
| 691 |
_mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
|
| 692 |
+
app_routes.mount_app_routers(app, ROUTER_MOUNT_FAILURES)
|
| 693 |
|
| 694 |
|
| 695 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
|
|
|
| 979 |
outcomes[key] = False
|
| 980 |
try:
|
| 981 |
rt = runtime.get_runtime(key)
|
| 982 |
+
if not _meta._connection_rows(rt, key, {"problems": []}):
|
| 983 |
continue
|
| 984 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 985 |
log=lambda *_a: None, insights_days=7)
|
|
|
|
| 993 |
return outcomes.get(tenant_key or runtime.tenant_for_store_path(), False)
|
| 994 |
|
| 995 |
|
| 996 |
+
def _drain_amazon_connection_revocations(rt):
|
| 997 |
+
"""Finish scoped deletions on the resident writer before any Amazon restore or pull."""
|
| 998 |
+
try:
|
| 999 |
+
pending = rt.get("amazon_connection_revocations", fresh=True) or {}
|
| 1000 |
+
if not isinstance(pending, dict):
|
| 1001 |
+
raise ValueError("Unreadable Amazon revocation metadata")
|
| 1002 |
+
pending = {cid: row for cid, row in pending.items()
|
| 1003 |
+
if isinstance(row, dict) and row.get("state") == "pending"}
|
| 1004 |
+
if pending:
|
| 1005 |
+
import process_role as _role
|
| 1006 |
+
from harness import datastore as _ds
|
| 1007 |
+
if (not _role.may_write() or rt.datastore_path.resolve() != _ds.DB_PATH.resolve()
|
| 1008 |
+
or _ds.path_for(rt.key).resolve() != _ds.DB_PATH.resolve()):
|
| 1009 |
+
raise RuntimeError("Amazon cleanup requires the tenant's resident writer")
|
| 1010 |
+
import amazon_relational as _ar
|
| 1011 |
+
failures = []
|
| 1012 |
+
for cid, row in pending.items():
|
| 1013 |
+
try:
|
| 1014 |
+
result = _ar.purge_connection(rt, cid, entries_before=row.get("entries"))
|
| 1015 |
+
if not isinstance(result, dict) or result.get("failed") or result.get("refused"):
|
| 1016 |
+
raise RuntimeError("Amazon connection cleanup did not finish")
|
| 1017 |
+
except Exception as exc:
|
| 1018 |
+
failures.append(type(exc).__name__)
|
| 1019 |
+
if failures:
|
| 1020 |
+
raise RuntimeError("Amazon connection cleanup did not finish")
|
| 1021 |
+
RESYNC_STATE.setdefault("amazonRevocationErrors", {}).pop(rt.key, None)
|
| 1022 |
+
except Exception as exc:
|
| 1023 |
+
message = "Amazon connection cleanup is pending. Retry on the tenant's resident sync worker."
|
| 1024 |
+
RESYNC_STATE.setdefault("amazonRevocationErrors", {})[rt.key] = {
|
| 1025 |
+
"type": type(exc).__name__, "message": message}
|
| 1026 |
+
print(f"[aios-api] amazon connection cleanup pending ({rt.key}): {type(exc).__name__}")
|
| 1027 |
+
raise _SyncNowRefused(message) from None
|
| 1028 |
+
|
| 1029 |
+
|
| 1030 |
+
def _drain_pending_amazon_revocations():
|
| 1031 |
+
"""Handle cleanup-only wakes without requiring a credential, definition or sync receipt."""
|
| 1032 |
+
import process_role as _role
|
| 1033 |
+
if not _role.may_write():
|
| 1034 |
+
return
|
| 1035 |
+
from harness import runtime, datastore
|
| 1036 |
+
try:
|
| 1037 |
+
tenants = runtime.known_tenants()
|
| 1038 |
+
RESYNC_STATE.pop("amazonRevocationDiscoveryError", None)
|
| 1039 |
+
except Exception as exc:
|
| 1040 |
+
RESYNC_STATE["amazonRevocationDiscoveryError"] = {
|
| 1041 |
+
"type": type(exc).__name__, "message": "Amazon cleanup tenants could not be read. Retry the sync worker."}
|
| 1042 |
+
return
|
| 1043 |
+
for key in tenants:
|
| 1044 |
+
try:
|
| 1045 |
+
rt = runtime.get_runtime(key)
|
| 1046 |
+
if rt.key == key and rt.datastore_path.resolve() == datastore.DB_PATH.resolve():
|
| 1047 |
+
_drain_amazon_connection_revocations(rt)
|
| 1048 |
+
except KeyError:
|
| 1049 |
+
continue
|
| 1050 |
+
except Exception as exc:
|
| 1051 |
+
print(f"[aios-api] amazon cleanup wake unavailable ({key}): {type(exc).__name__}")
|
| 1052 |
+
continue
|
| 1053 |
+
|
| 1054 |
+
|
| 1055 |
def _pull_amazon(why, vendor=True):
|
| 1056 |
"""Produce Amazon rows in this container before projecting them, on the caller's thread.
|
| 1057 |
|
|
|
|
| 1071 |
rt.datastore_path.resolve() != _ds.DB_PATH.resolve() or
|
| 1072 |
_ds.path_for(tenant_key).resolve() != _ds.DB_PATH.resolve()):
|
| 1073 |
raise RuntimeError("Amazon sync requires the requesting tenant's resident mirror.")
|
| 1074 |
+
_drain_amazon_connection_revocations(rt)
|
| 1075 |
if not _amazon.credential(tenant_key, rt=rt):
|
| 1076 |
print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential "
|
| 1077 |
"for royal-imports; the connector is idle, not broken")
|
|
|
|
| 1164 |
if (rt.key != tenant_key or
|
| 1165 |
rt.datastore_path.resolve() != _ds.path_for(tenant_key).resolve()):
|
| 1166 |
raise RuntimeError("Amazon restore requires the requesting tenant's runtime.")
|
| 1167 |
+
_drain_amazon_connection_revocations(rt)
|
| 1168 |
if rt.download_bytes(_amazon.MIRROR_PREFIX + "manifest.json") is None:
|
| 1169 |
RESYNC_STATE.setdefault("amazonRestoreErrors", {}).pop(tenant_key, None)
|
| 1170 |
return True
|
|
|
|
| 1262 |
def _sync_now_cause(cause):
|
| 1263 |
"""Classify actual provider failures without returning vendor text or exception reprs."""
|
| 1264 |
import re
|
| 1265 |
+
if isinstance(cause, dict):
|
| 1266 |
+
sentence = "The connector could not read an expected value in the sync response."
|
| 1267 |
+
if cause.get("classifier") == "unreadable_value":
|
| 1268 |
+
table = cause.get("table")
|
| 1269 |
+
if isinstance(table, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", table):
|
| 1270 |
+
sentence = f"Amazon {table}: unreadable value."
|
| 1271 |
+
filename, line, function = (cause.get(key) for key in ("file", "line", "function"))
|
| 1272 |
+
if (isinstance(filename, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}\.py", filename)
|
| 1273 |
+
and type(line) is int and 0 < line < 10000000
|
| 1274 |
+
and isinstance(function, str) and re.fullmatch(
|
| 1275 |
+
r"[A-Za-z_][A-Za-z0-9_]{0,79}|<(?:lambda|listcomp|dictcomp|setcomp|genexpr|module)>", function)):
|
| 1276 |
+
sentence += f" Raised at {filename}:{line}:{function}."
|
| 1277 |
+
return "unreadable_value", sentence
|
| 1278 |
+
return "unclassified", "The connector reported a failure. Inspect its sync report and retry."
|
| 1279 |
raw = str(cause)
|
| 1280 |
text = raw.lower()
|
| 1281 |
causes = (
|
|
|
|
| 1317 |
if isinstance(cause, TypeError):
|
| 1318 |
return "bad_arguments", "The connector failed because its provider call used incompatible arguments."
|
| 1319 |
if isinstance(cause, (KeyError, ValueError)):
|
| 1320 |
+
sentence = "The connector could not read an expected value in the sync response."
|
| 1321 |
+
import traceback
|
| 1322 |
+
frames = traceback.extract_tb(cause.__traceback__)
|
| 1323 |
+
if frames:
|
| 1324 |
+
frame = frames[-1]
|
| 1325 |
+
filename = frame.filename.replace("\\", "/").rsplit("/", 1)[-1]
|
| 1326 |
+
if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}\.py", filename):
|
| 1327 |
+
sentence += f" Raised at {filename}:{frame.lineno}."
|
| 1328 |
+
return "unreadable_value", sentence
|
| 1329 |
if isinstance(cause, str):
|
| 1330 |
# amazon_store._sync_problem appends the application-authored fix LAST. The
|
| 1331 |
# exception message may itself contain delimiters; none of it is display text.
|
|
|
|
| 1356 |
return _sync_now_cause(cause)[1]
|
| 1357 |
|
| 1358 |
|
| 1359 |
+
def _amazon_sync_receipt(report):
|
| 1360 |
+
"""Project table windows and resync counts without vendor diagnostics or exception values."""
|
| 1361 |
+
import re
|
| 1362 |
+
details = []
|
| 1363 |
+
for table, row in (report.get("tables") or {}).items():
|
| 1364 |
+
if not isinstance(row, dict) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", str(table)):
|
| 1365 |
+
continue
|
| 1366 |
+
parts = []
|
| 1367 |
+
progress = row.get("resync")
|
| 1368 |
+
if isinstance(progress, dict) and all(type(progress.get(k)) is int and progress[k] >= 0
|
| 1369 |
+
for k in ("landed", "left")):
|
| 1370 |
+
parts.append(f"{progress['landed']} windows landed, {progress['left']} left")
|
| 1371 |
+
window = row.get("window")
|
| 1372 |
+
if isinstance(window, str):
|
| 1373 |
+
# Window descriptions are authored by the loader, never copied from its diagnostics.
|
| 1374 |
+
parts.append(" ".join(window.split()))
|
| 1375 |
+
if parts:
|
| 1376 |
+
details.append(f"{table}: " + "; ".join(parts))
|
| 1377 |
+
return " | ".join(details)
|
| 1378 |
+
|
| 1379 |
+
|
| 1380 |
def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
|
| 1381 |
"""Record one classified failure; leave successful receipt text untouched."""
|
| 1382 |
if ok:
|
|
|
|
| 1388 |
locations = " > ".join(f"{os.path.basename(frame.filename)}:{frame.lineno}:{frame.name}"
|
| 1389 |
for frame in traceback.extract_tb(cause.__traceback__))
|
| 1390 |
print(f"[aios-api] sync failure location {type(cause).__name__}: {locations}")
|
| 1391 |
+
if detail:
|
| 1392 |
+
sentence += " " + detail
|
| 1393 |
suffix = f" (cause code: {classifier})"
|
| 1394 |
# finish_sync_request stores only 300 characters. Reserve the suffix before it clips.
|
| 1395 |
budget = 300 - len(suffix)
|
|
|
|
| 1407 |
project only what a desk wake actually changed.
|
| 1408 |
"""
|
| 1409 |
from core import connector_sync
|
| 1410 |
+
_drain_pending_amazon_revocations()
|
| 1411 |
+
|
| 1412 |
+
def progress_log(provider, tenant, receipt, rt):
|
| 1413 |
+
"""Publish only authored progress shapes, with one bounded detail per running receipt."""
|
| 1414 |
+
import re
|
| 1415 |
+
from collections import deque
|
| 1416 |
+
lines = deque(maxlen=2)
|
| 1417 |
+
|
| 1418 |
+
def publish_progress(*parts, **_kwargs):
|
| 1419 |
+
text = " ".join(str(part) for part in parts).strip()
|
| 1420 |
+
if provider == "odoo":
|
| 1421 |
+
match = re.fullmatch(
|
| 1422 |
+
r"([a-z][a-z0-9_]*): (column backfill|column refresh|m2m) "
|
| 1423 |
+
r"([a-z][a-z0-9_]*(?:\+[a-z][a-z0-9_]*)*) "
|
| 1424 |
+
r"(starting at id|resuming at id|-> id) ([0-9,]+) "
|
| 1425 |
+
r"\(([0-9,]+) rows(?: already done)?\)", text)
|
| 1426 |
+
if match:
|
| 1427 |
+
entity, kind, fields, step, last_id, count = match.groups()
|
| 1428 |
+
safe = f"{entity}: {kind} {fields} {step} {last_id} ({count} rows)"
|
| 1429 |
+
elif re.fullmatch(r"[a-z][a-z0-9_]*: backfill …[0-9,]+ rows \(id≤[0-9,]+\)", text):
|
| 1430 |
+
safe = text
|
| 1431 |
+
elif re.fullmatch(r"[a-z][a-z0-9_]*: backfill starting at id [0-9,]+ \([0-9,]+ rows\)", text):
|
| 1432 |
+
safe = text
|
| 1433 |
+
else:
|
| 1434 |
+
return
|
| 1435 |
+
else:
|
| 1436 |
+
match = re.fullmatch(
|
| 1437 |
+
r"([a-z][a-z0-9_]*): resync of ([0-9]+) stored windows(?: from [0-9TZ:+. -]+)?", text)
|
| 1438 |
+
summary = re.fullmatch(
|
| 1439 |
+
r"Amazon resync ([a-z][a-z0-9_]*): ([0-9]+) windows landed, ([0-9]+) left", text)
|
| 1440 |
+
if match:
|
| 1441 |
+
safe = f"Amazon {match[1]}: resync of {match[2]} stored windows"
|
| 1442 |
+
elif summary:
|
| 1443 |
+
safe = f"Amazon {summary[1]}: {summary[2]} windows landed, {summary[3]} left"
|
| 1444 |
+
else:
|
| 1445 |
+
return
|
| 1446 |
+
lines.append(safe[:220])
|
| 1447 |
+
detail = " | ".join(lines)[-300:]
|
| 1448 |
+
print(f"[aios-api] {provider} sync desk: {safe[:220]}", flush=True)
|
| 1449 |
+
# The desk owns lifecycle state. Progress must not finish or replace another claim.
|
| 1450 |
+
with connector_sync._SYNC_REQUEST_LOCK:
|
| 1451 |
+
current = next((row for row in connector_sync._SYNC_REQUESTS.values()
|
| 1452 |
+
if row["provider"] == provider and row["tenant"] == tenant
|
| 1453 |
+
and row["state"] == "running"
|
| 1454 |
+
and row.get("id") == receipt.get("id")
|
| 1455 |
+
and row.get("startedAt") == receipt.get("startedAt")), None)
|
| 1456 |
+
if current is None:
|
| 1457 |
+
return
|
| 1458 |
+
current["detail"] = detail
|
| 1459 |
+
if current.get("tables") is not None:
|
| 1460 |
+
identity, token = current["id"], current["claimToken"]
|
| 1461 |
+
|
| 1462 |
+
def publish(document):
|
| 1463 |
+
doc = connector_sync._sync_request_document(document)
|
| 1464 |
+
row = doc["requests"].get(identity)
|
| 1465 |
+
if (row and row.get("tenant") == tenant and row.get("state") == "running"
|
| 1466 |
+
and row.get("claimToken") == token):
|
| 1467 |
+
row["detail"] = detail
|
| 1468 |
+
return doc
|
| 1469 |
+
|
| 1470 |
+
rt.update(connector_sync.SYNC_REQUESTS_KEY, publish, flush="sync")
|
| 1471 |
+
# Ordinary requests cross processes through the existing tenant-bound desk publisher.
|
| 1472 |
+
connector_sync.poll_remote_requests(rt)
|
| 1473 |
+
|
| 1474 |
+
def log(*parts, **kwargs):
|
| 1475 |
+
try:
|
| 1476 |
+
publish_progress(*parts, **kwargs)
|
| 1477 |
+
except Exception as exc:
|
| 1478 |
+
print(f"[aios-api] sync progress unavailable ({type(exc).__name__})", flush=True)
|
| 1479 |
+
|
| 1480 |
+
return log, lines
|
| 1481 |
+
|
| 1482 |
odoo_requested = False
|
| 1483 |
+
# Claim only work this turn executes. A later provider remains queued with an honest wait.
|
| 1484 |
+
pending = connector_sync.take_sync_requests(limit=1, exclude_providers=("odoo",) if during_odoo else (),
|
| 1485 |
+
refresh_remote=True)
|
| 1486 |
while True:
|
| 1487 |
receipt = next((row for row in pending
|
| 1488 |
if not during_odoo or row["provider"] != "odoo"), None)
|
|
|
|
| 1495 |
ok, detail = False, ""
|
| 1496 |
first_load_progress, continuing = None, False
|
| 1497 |
floor_wait = None
|
| 1498 |
+
progress = ()
|
| 1499 |
+
amazon_detail, amazon_failure = "", None
|
| 1500 |
try:
|
| 1501 |
from harness import runtime, datastore, mirror_store
|
| 1502 |
rt = runtime.get_runtime(tenant)
|
|
|
|
| 1508 |
datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()):
|
| 1509 |
raise _SyncNowRefused(
|
| 1510 |
"This tenant's mirror is not resident on this server. Sync requires its tenant worker.")
|
| 1511 |
+
connector_sync.poll_remote_requests(rt)
|
| 1512 |
if provider == "odoo":
|
| 1513 |
import core.odoo as odoo
|
| 1514 |
import odoo_relational as rel
|
| 1515 |
import routes_connectors as _sync_routes
|
| 1516 |
if rt.odoo_paused() or rel.frozen(rt):
|
| 1517 |
raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.")
|
| 1518 |
+
requested_cid = ((receipt.get("connectionId") if "connectionId" in receipt
|
| 1519 |
+
else _sync_routes.sync_request_connection(provider, tenant))
|
| 1520 |
or _sync_routes._odoo_connection_slot(rt, None))
|
| 1521 |
+
# Named and default requests share one all-connection publication cohort.
|
| 1522 |
+
odoo_requested = True
|
| 1523 |
+
from connector_onboarding import selected_mirror_entities
|
| 1524 |
+
selected_entities = selected_mirror_entities(rt)
|
| 1525 |
+
_counter = globals().get("_entity_source_counter")
|
| 1526 |
+
_after_entity = _counter(datastore) if _counter else None
|
| 1527 |
+
_log, progress = progress_log(provider, tenant, receipt, rt)
|
| 1528 |
+
report = mirror_store.tracked_sync("odoo", rt, datastore.sync_all, entities=selected_entities,
|
| 1529 |
+
connection_id=requested_cid, log=_log,
|
| 1530 |
+
client_factory=lambda creds: _DeskAwareOdooClient(lambda: odoo.OdooClient(creds=creds)),
|
| 1531 |
+
**({"after_entity": _after_entity} if _after_entity else {}))
|
| 1532 |
+
if not report and selected_entities != []:
|
| 1533 |
+
raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
|
| 1534 |
+
if any(row.get("phase") != "live" for row in report.values()):
|
| 1535 |
+
raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
|
| 1536 |
+
if mirror_store.pending_sync_entities("odoo", tenant, rt) is not None:
|
| 1537 |
+
raise _SyncNowRefused("Odoo publication is held. The previous snapshot remains available.")
|
| 1538 |
+
problems = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1539 |
elif provider == "amazon_sp":
|
| 1540 |
from harness import amazon_store as amazon
|
| 1541 |
+
_drain_amazon_connection_revocations(rt)
|
| 1542 |
if not amazon.credential(tenant, rt=rt):
|
| 1543 |
raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
|
| 1544 |
from connectors_amazon import notification_sync_options
|
|
|
|
| 1547 |
first_load = False if targeted else amazon.needs_first_load(tenant, rt=rt)
|
| 1548 |
options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS,
|
| 1549 |
"refresh_loaded": True} if first_load else {}
|
| 1550 |
+
_log, progress = progress_log(provider, tenant, receipt, rt)
|
| 1551 |
report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt,
|
| 1552 |
scheduled=False if targeted else receipt.get("requestedBy") == "first load",
|
| 1553 |
**target_options, **options,
|
| 1554 |
+
log=_log)
|
| 1555 |
print("[aios-api] amazon sync done (sync desk): " + ", ".join(
|
| 1556 |
f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
|
| 1557 |
+ (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left"
|
|
|
|
| 1565 |
floor_wait = float(report["nextDueSeconds"])
|
| 1566 |
problems = list(report.get("problems") or []) + list(report.get("audit_problems") or [])
|
| 1567 |
tables = report.get("tables") or {}
|
| 1568 |
+
amazon_detail = _amazon_sync_receipt(report)
|
| 1569 |
+
amazon_failure = next((row["failure"] for row in tables.values()
|
| 1570 |
+
if isinstance(row, dict) and isinstance(row.get("failure"), dict)), None)
|
| 1571 |
if not problems and not tables and not continuing:
|
| 1572 |
raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.")
|
| 1573 |
if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()):
|
| 1574 |
raise _SyncNowRefused("All Amazon databases are disabled. Enable a database before syncing.")
|
| 1575 |
elif provider == "meta_ads":
|
| 1576 |
from harness import meta_store as meta
|
| 1577 |
+
if not meta._connection_rows(rt, tenant, {"problems": []}):
|
| 1578 |
raise _SyncNowRefused("Meta Ads has no access token. Connect Meta Ads before syncing.")
|
| 1579 |
report = mirror_store.tracked_sync("meta", rt, meta.sync, tenant,
|
| 1580 |
log=lambda *a, **k: None, insights_days=7)
|
|
|
|
| 1590 |
detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; "
|
| 1591 |
f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.")
|
| 1592 |
if problems:
|
| 1593 |
+
detail = _sync_now_detail(provider, tenant, ok=False,
|
| 1594 |
+
detail=amazon_detail or detail,
|
| 1595 |
+
cause=amazon_failure or problems[0])
|
| 1596 |
elif problems:
|
| 1597 |
+
detail = _sync_now_detail(provider, tenant, ok=False, detail=amazon_detail,
|
| 1598 |
+
cause=amazon_failure or problems[0])
|
| 1599 |
else:
|
| 1600 |
ok, detail = True, "Sync finished."
|
| 1601 |
+
if amazon_detail and not problems:
|
| 1602 |
+
detail += " " + amazon_detail
|
| 1603 |
except _SyncNowRefused as exc:
|
| 1604 |
detail = str(exc)
|
| 1605 |
except Exception as exc: # noqa: BLE001
|
| 1606 |
detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
|
| 1607 |
+
if progress and not amazon_detail:
|
| 1608 |
+
import re
|
| 1609 |
+
suffix = re.search(r" \(cause code: [a-z_]+\)$", detail)
|
| 1610 |
+
cause_code = suffix[0] if suffix else ""
|
| 1611 |
+
sentence = detail[:suffix.start()] if suffix else detail
|
| 1612 |
+
latest = progress[-1][:140]
|
| 1613 |
+
detail = sentence[:299 - len(latest) - len(cause_code)] + " " + latest + cause_code
|
| 1614 |
if first_load_progress is not None:
|
| 1615 |
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail,
|
| 1616 |
first_load=first_load_progress, request_id=receipt.get("id"))
|
|
|
|
| 2475 |
try:
|
| 2476 |
first = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all,
|
| 2477 |
entities=absent, log=_resync_log,
|
| 2478 |
+
client_factory=lambda creds: _DeskAwareOdooClient(
|
| 2479 |
+
lambda: _odoo.OdooClient(creds=creds)),
|
| 2480 |
_catch_up_complete=False)
|
| 2481 |
_log_entities("absent", first)
|
| 2482 |
print(f"[aios-api] store resync: synced {', '.join(absent)} first, "
|
|
|
|
| 2488 |
_after_entity = _entity_source_counter(_ds)
|
| 2489 |
sync_report = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all,
|
| 2490 |
entities=entities, log=_resync_log,
|
| 2491 |
+
client_factory=lambda creds: _DeskAwareOdooClient(
|
| 2492 |
+
lambda: _odoo.OdooClient(creds=creds)),
|
| 2493 |
**({"after_entity": _after_entity} if _after_entity else {}))
|
| 2494 |
_log_entities("full", sync_report)
|
| 2495 |
for key, row in first.items():
|
| 2496 |
if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict):
|
| 2497 |
sync_report[key]["pulled"] = sync_report[key].get("pulled", 0) + row.get("pulled", 0)
|
| 2498 |
+
_held_check = globals().get("_resync_publication_held")
|
| 2499 |
+
if (_held_check(sync_report) if _held_check else
|
| 2500 |
+
any(row.get("phase") == "backfill" for row in (sync_report or {}).values())):
|
| 2501 |
+
_resync_log("publication held; the next admitted pass resumes only its unfinished cohort")
|
| 2502 |
+
return sync_report, None, None
|
| 2503 |
# Wave 21: purge hard-deleted rows the cursor sync cannot see (the boot pass's comment has
|
| 2504 |
# the measured case); without it deleted Odoo lines inflate every sum on the mirror FOREVER.
|
| 2505 |
# ⚠ INSIDE the non-manual branch: a reconcile is a pass over the mirror this loop was
|
|
|
|
| 2541 |
return []
|
| 2542 |
|
| 2543 |
|
| 2544 |
+
def _resync_publication_held(sync_report, tenant=None):
|
| 2545 |
+
"""Keep projections behind the durable source boundary, including validation and transport failures."""
|
| 2546 |
+
if any(isinstance(row, dict) and row.get("phase") == "backfill"
|
| 2547 |
+
for row in (sync_report or {}).values()):
|
| 2548 |
+
return True
|
| 2549 |
+
try:
|
| 2550 |
+
from harness import datastore, mirror_store, runtime
|
| 2551 |
+
if not datastore.DB_PATH.with_suffix(".odoo.candidate.duckdb").exists():
|
| 2552 |
+
return False
|
| 2553 |
+
rt = runtime.get_runtime(tenant) if tenant else datastore._resident_odoo_runtime()
|
| 2554 |
+
return rt is not None and mirror_store.pending_sync_entities("odoo", rt.key, rt) is not None
|
| 2555 |
+
except Exception as exc:
|
| 2556 |
+
print(f"[aios-api] Odoo publication status unavailable ({type(exc).__name__}); projections held")
|
| 2557 |
+
return True
|
| 2558 |
+
|
| 2559 |
+
|
| 2560 |
def _smallest_first(ds, keys):
|
| 2561 |
"""`keys` ordered by their Odoo row count, so a small table answers before a large backfill."""
|
| 2562 |
def size(key):
|
|
|
|
| 2726 |
if not tenant:
|
| 2727 |
return None
|
| 2728 |
counted = _PASS_SOURCE_COUNTS[tenant] = {}
|
| 2729 |
+
per_connection = {}
|
| 2730 |
|
| 2731 |
def after_entity(key):
|
| 2732 |
# A volatile table (stock_quant) drifts between its copy and a count at the end of the pass.
|
| 2733 |
try:
|
| 2734 |
import odoo_relational as _rel_counts
|
| 2735 |
model = (ds.ENTITIES.get(key) or {}).get("model")
|
| 2736 |
+
binding = getattr(getattr(ds, "_SYNC_TLS", None), "connection", None)
|
| 2737 |
+
cid = binding["id"] if binding else None
|
| 2738 |
+
seen = per_connection.setdefault(cid, {})
|
| 2739 |
+
buckets = _rel_counts.source_count_buckets(model) - set(seen)
|
| 2740 |
if not buckets:
|
| 2741 |
return
|
| 2742 |
+
got = _rel_counts.capture_source_counts(rt, buckets=buckets,
|
| 2743 |
+
**({"connection_id": cid} if cid and cid != "legacy-env" else {}))
|
| 2744 |
+
seen.update(got)
|
| 2745 |
+
for bucket, (count, mode) in got.items():
|
| 2746 |
+
members = [values[bucket] for values in per_connection.values() if bucket in values]
|
| 2747 |
+
if all(other_mode == mode for _, other_mode in members):
|
| 2748 |
+
counted[bucket] = (sum(value for value, _ in members), mode)
|
| 2749 |
for bucket, (count, _mode) in sorted(got.items()):
|
| 2750 |
print(f"[aios-api] store resync: counted the Odoo source for {bucket} "
|
| 2751 |
f"beside the {key} copy: {count:,} rows", flush=True)
|
|
|
|
| 3008 |
sync_report, delete_report, _err = None, None, _classify_resync_error(exc)
|
| 3009 |
if _err is None and sync_report:
|
| 3010 |
RESYNC_STATE["passes"] = RESYNC_STATE.get("passes", 0) + 1
|
| 3011 |
+
_held_check = globals().get("_resync_publication_held")
|
| 3012 |
+
_publication_held = (_held_check(sync_report, _cad["tenant"]) if _held_check else
|
| 3013 |
+
any(row.get("phase") == "backfill" for row in (sync_report or {}).values()))
|
| 3014 |
+
RESYNC_STATE["publicationHeld"] = _publication_held
|
| 3015 |
+
source_fingerprints = {} if _publication_held else _resync_source_fingerprints(sync_report)
|
| 3016 |
RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
|
| 3017 |
RESYNC_STATE["lastPassOk"] = _err is None
|
| 3018 |
RESYNC_STATE["lastError"] = _err
|
| 3019 |
if _err is None:
|
| 3020 |
RESYNC_STATE["consecutiveFailures"] = 0
|
| 3021 |
+
if not _manual and not _publication_held:
|
| 3022 |
RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
|
| 3023 |
# A published pull is the Odoo source stage's success, so the receipt's source stamp follows it.
|
| 3024 |
try:
|
|
|
|
| 3036 |
print(f"[aios-api] store resync failed ({_err['kind']}, "
|
| 3037 |
f"{RESYNC_STATE['consecutiveFailures']} in a row): "
|
| 3038 |
f"{_err['type']}: {_err['message']}")
|
| 3039 |
+
if _publication_held:
|
| 3040 |
+
print("[aios-api] store resync: publication held; no vendor pulls or relational rebuild; "
|
| 3041 |
+
"the next admitted pass resumes the unfinished Odoo cohort")
|
| 3042 |
+
continue
|
| 3043 |
if _boot_catch_up and not _odoo_requested:
|
| 3044 |
# Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
|
| 3045 |
if not catch_up_retries:
|
api/meta_relational.py
CHANGED
|
@@ -146,7 +146,7 @@ def _fields_for(cur, key):
|
|
| 146 |
for name, dtype in cols:
|
| 147 |
f = {"key": name, "label": _label(name), "type": _ftype(name, dtype),
|
| 148 |
"source": "overlay", "default": name in (pinned, "id", "name", "status", "spend",
|
| 149 |
-
"impressions", "clicks", "date_start"),
|
| 150 |
"pinned": name == pinned}
|
| 151 |
out.append(_odoo._preset(f, flow="meta_relational"))
|
| 152 |
# Links require the stored and planned schemas held by apply_plan's atomic updater.
|
|
@@ -163,6 +163,14 @@ def _machine_preset(field):
|
|
| 163 |
|
| 164 |
def _row_id(key, row):
|
| 165 |
"""Use the declared source identity, preserving existing Graph object IDs."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
identity = IDENTITY[key]
|
| 167 |
values = [str(row.get(name) if row.get(name) is not None else '').strip()
|
| 168 |
for name in identity]
|
|
@@ -310,6 +318,17 @@ def apply_plan(rt, built, username="meta", today=None):
|
|
| 310 |
live_columns.update({key: {field['key'] for field in fields}
|
| 311 |
for key, (fields, _rows) in plans.items()})
|
| 312 |
generated = connector_spec.system_link_fields('meta', live_columns)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 313 |
table = doc.get(INSIGHTS_KEY)
|
| 314 |
if rekeys and isinstance(table, dict):
|
| 315 |
rows = table.get('rows') or {}
|
|
|
|
| 146 |
for name, dtype in cols:
|
| 147 |
f = {"key": name, "label": _label(name), "type": _ftype(name, dtype),
|
| 148 |
"source": "overlay", "default": name in (pinned, "id", "name", "status", "spend",
|
| 149 |
+
"impressions", "clicks", "date_start", "store"),
|
| 150 |
"pinned": name == pinned}
|
| 151 |
out.append(_odoo._preset(f, flow="meta_relational"))
|
| 152 |
# Links require the stored and planned schemas held by apply_plan's atomic updater.
|
|
|
|
| 163 |
|
| 164 |
def _row_id(key, row):
|
| 165 |
"""Use the declared source identity, preserving existing Graph object IDs."""
|
| 166 |
+
if "_row_key" in row:
|
| 167 |
+
value = str(row.get("_row_key") or "").strip()
|
| 168 |
+
if not value:
|
| 169 |
+
raise Refused(f"{key}: missing connection-aware row identity. Nothing was written.")
|
| 170 |
+
payload = json.dumps([key, [value]], ensure_ascii=True, separators=(',', ':'))
|
| 171 |
+
return str(int(hashlib.sha256(payload.encode('utf-8')).hexdigest()[:13], 16))
|
| 172 |
+
if row.get("_connection_id"):
|
| 173 |
+
raise Refused(f"{key}: missing connection-aware row identity. Nothing was written.")
|
| 174 |
identity = IDENTITY[key]
|
| 175 |
values = [str(row.get(name) if row.get(name) is not None else '').strip()
|
| 176 |
for name in identity]
|
|
|
|
| 318 |
live_columns.update({key: {field['key'] for field in fields}
|
| 319 |
for key, (fields, _rows) in plans.items()})
|
| 320 |
generated = connector_spec.system_link_fields('meta', live_columns)
|
| 321 |
+
scoped = {"account_id": "_account_key", "campaign_id": "_campaign_key",
|
| 322 |
+
"adset_id": "_adset_key", "ad_id": "_ad_key", "id": "_row_key"}
|
| 323 |
+
for source, links in generated.items():
|
| 324 |
+
for field in links:
|
| 325 |
+
link = field["link"]
|
| 326 |
+
target = link["table"]
|
| 327 |
+
if "_connection_id" in live_columns[source] or "_connection_id" in live_columns[target]:
|
| 328 |
+
left, right = scoped.get(link["from"]), scoped.get(link["on"])
|
| 329 |
+
if left not in live_columns[source] or right not in live_columns[target]:
|
| 330 |
+
raise Refused("Meta links need connection-aware endpoints. Nothing was written.")
|
| 331 |
+
link.update({"from": left, "on": right})
|
| 332 |
table = doc.get(INSIGHTS_KEY)
|
| 333 |
if rekeys and isinstance(table, dict):
|
| 334 |
rows = table.get('rows') or {}
|
api/odoo_relational.py
CHANGED
|
@@ -246,6 +246,7 @@ _AUDIT_AGENT_SQL = {
|
|
| 246 |
}
|
| 247 |
|
| 248 |
_INVOICE_AUDIT = (
|
|
|
|
| 249 |
_af("amount_total", "Total", "currency", "account.move.amount_total", "am.amount_total",
|
| 250 |
"num", "sum"),
|
| 251 |
_af("amount_total_signed", "Total signed", "currency", "account.move.amount_total_signed",
|
|
|
|
| 246 |
}
|
| 247 |
|
| 248 |
_INVOICE_AUDIT = (
|
| 249 |
+
_af("state", "Posting state", "text", "account.move.state", "am.state"),
|
| 250 |
_af("amount_total", "Total", "currency", "account.move.amount_total", "am.amount_total",
|
| 251 |
"num", "sum"),
|
| 252 |
_af("amount_total_signed", "Total signed", "currency", "account.move.amount_total_signed",
|
api/registry_source.py
CHANGED
|
@@ -354,7 +354,7 @@ class BatchStore:
|
|
| 354 |
raise_if_resource_error(exc)
|
| 355 |
raise err(503, 'canonical_source_unavailable', 'Saved view membership changed or is unavailable.') from exc
|
| 356 |
cur = getattr(self, '_canonical_source_cursor', None)
|
| 357 |
-
if cur is not None and selected:
|
| 358 |
from routes_odoo_tables import _membership_relation
|
| 359 |
cache_key = ('view_members', name, owner, vid)
|
| 360 |
if cache_key not in self.metadata:
|
|
@@ -366,7 +366,7 @@ class BatchStore:
|
|
| 366 |
kept = [int(row[0]) for row in cur.execute(
|
| 367 |
f'SELECT DISTINCT * FROM ({operand.sql}) members(pid) '
|
| 368 |
f'WHERE pid IN ({",".join("?" for _ in selected)}) ORDER BY pid',
|
| 369 |
-
[*operand.params, *selected]).fetchall()]
|
| 370 |
else:
|
| 371 |
# Metadata consumes no row memberships; retain only the curated witness.
|
| 372 |
witness_key = ('view_witness', name, owner, vid)
|
|
@@ -408,11 +408,12 @@ class BatchStore:
|
|
| 408 |
if name.endswith('_table_workspace'):
|
| 409 |
value = self._workspace(name)
|
| 410 |
elif name.endswith('__shared'):
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
|
|
|
| 416 |
elif name in {self.table_key + '_docs', 'customer_docs', 'product_docs'}:
|
| 417 |
values = (paths(self._runtime, name, [[pid] for pid in self.row_ids])
|
| 418 |
if self._canonical_needed is None else [None] * len(self.row_ids))
|
|
@@ -516,12 +517,12 @@ class CanonicalStore(BatchStore):
|
|
| 516 |
return super().get_entry(name, entry)
|
| 517 |
|
| 518 |
|
| 519 |
-
def canonical_metadata(session, table_key, *, metadata=None):
|
| 520 |
"""Read the canonical permitted definitions without constructing source rows."""
|
| 521 |
import routes_customers as customers
|
| 522 |
import routes_products as products
|
| 523 |
session.require(table_key)
|
| 524 |
-
lent = CanonicalStore(session, table_key, [], cur=
|
| 525 |
metadata={} if metadata is None else metadata, needed=frozenset())
|
| 526 |
actor = with_runtime(session, customers._AssemblyStore(lent, schema_only=True))
|
| 527 |
build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly
|
|
@@ -558,7 +559,7 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
|
|
| 558 |
team, agent = perm_scope.derive_pool_scope(session.user, table_key)
|
| 559 |
before = generation(session, table_key)
|
| 560 |
metadata = {'_scan': ids is None}
|
| 561 |
-
definition = canonical_metadata(session, table_key, metadata=metadata)
|
| 562 |
# Every permanent wall operand is populated before apply_row_scope judges the batch.
|
| 563 |
tree = perm_scope.entry(session.user, table_key) or {}
|
| 564 |
wall = tree.get('filter') or {}
|
|
@@ -632,31 +633,34 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
|
|
| 632 |
batches = frozen()
|
| 633 |
else:
|
| 634 |
raise err(503, 'connector_paused', 'This source is paused and its scoped snapshot is unavailable.')
|
| 635 |
-
|
| 636 |
from harness import canonical_source
|
| 637 |
def mirrored():
|
| 638 |
# Stock, sales and product reads come from the Odoo mirror, never live Odoo. A lent
|
| 639 |
# cursor that cannot see it (Sheet's in-memory one) gets a child cursor on the mirror.
|
| 640 |
source_cur, owned = cur, None
|
| 641 |
-
if
|
|
|
|
| 642 |
import routes_odoo_tables as rot
|
| 643 |
base, mismatch = rot._mirror_cur(session)
|
| 644 |
if mismatch:
|
| 645 |
raise err(409, 'cross_tenant_store', mismatch)
|
| 646 |
source_cur = owned = base.cursor()
|
| 647 |
try:
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 651 |
yield from source
|
| 652 |
finally:
|
| 653 |
if owned is not None:
|
| 654 |
owned.close()
|
| 655 |
batches = mirrored()
|
| 656 |
-
else:
|
| 657 |
-
batches = customer_data.source_batches(cur, st=runtime, agent_name=agent, team_id=team,
|
| 658 |
-
ids=ids, needed=required, ledger_batches=items(session.runtime, customer_data.LEDGER_KEY,
|
| 659 |
-
selected=ids) if not agent and team is None and session.runtime.exists(customer_data.LEDGER_KEY) else ())
|
| 660 |
with closing(batches):
|
| 661 |
for rows in batches:
|
| 662 |
runtime = CanonicalStore(session, table_key, rows, cur=cur, metadata=metadata, needed=required)
|
|
|
|
| 354 |
raise_if_resource_error(exc)
|
| 355 |
raise err(503, 'canonical_source_unavailable', 'Saved view membership changed or is unavailable.') from exc
|
| 356 |
cur = getattr(self, '_canonical_source_cursor', None)
|
| 357 |
+
if cur is not None and (selected or self.metadata.get('_scan')):
|
| 358 |
from routes_odoo_tables import _membership_relation
|
| 359 |
cache_key = ('view_members', name, owner, vid)
|
| 360 |
if cache_key not in self.metadata:
|
|
|
|
| 366 |
kept = [int(row[0]) for row in cur.execute(
|
| 367 |
f'SELECT DISTINCT * FROM ({operand.sql}) members(pid) '
|
| 368 |
f'WHERE pid IN ({",".join("?" for _ in selected)}) ORDER BY pid',
|
| 369 |
+
[*operand.params, *selected]).fetchall()] if selected else []
|
| 370 |
else:
|
| 371 |
# Metadata consumes no row memberships; retain only the curated witness.
|
| 372 |
witness_key = ('view_witness', name, owner, vid)
|
|
|
|
| 408 |
if name.endswith('_table_workspace'):
|
| 409 |
value = self._workspace(name)
|
| 410 |
elif name.endswith('__shared'):
|
| 411 |
+
key = ('shared_fields', name)
|
| 412 |
+
if key not in self.metadata:
|
| 413 |
+
self.metadata[key] = paths(self._runtime, name, [['fields']])[0] or {}
|
| 414 |
+
values = self._selected_json(name, ['cells'])
|
| 415 |
+
value = {'fields': self.metadata[key], 'cells': {
|
| 416 |
+
pid: cell for pid, cell in values.items() if isinstance(cell, dict)}}
|
| 417 |
elif name in {self.table_key + '_docs', 'customer_docs', 'product_docs'}:
|
| 418 |
values = (paths(self._runtime, name, [[pid] for pid in self.row_ids])
|
| 419 |
if self._canonical_needed is None else [None] * len(self.row_ids))
|
|
|
|
| 517 |
return super().get_entry(name, entry)
|
| 518 |
|
| 519 |
|
| 520 |
+
def canonical_metadata(session, table_key, *, metadata=None, cur=None):
|
| 521 |
"""Read the canonical permitted definitions without constructing source rows."""
|
| 522 |
import routes_customers as customers
|
| 523 |
import routes_products as products
|
| 524 |
session.require(table_key)
|
| 525 |
+
lent = CanonicalStore(session, table_key, [], cur=cur,
|
| 526 |
metadata={} if metadata is None else metadata, needed=frozenset())
|
| 527 |
actor = with_runtime(session, customers._AssemblyStore(lent, schema_only=True))
|
| 528 |
build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly
|
|
|
|
| 559 |
team, agent = perm_scope.derive_pool_scope(session.user, table_key)
|
| 560 |
before = generation(session, table_key)
|
| 561 |
metadata = {'_scan': ids is None}
|
| 562 |
+
definition = canonical_metadata(session, table_key, metadata=metadata, cur=cur)
|
| 563 |
# Every permanent wall operand is populated before apply_row_scope judges the batch.
|
| 564 |
tree = perm_scope.entry(session.user, table_key) or {}
|
| 565 |
wall = tree.get('filter') or {}
|
|
|
|
| 633 |
batches = frozen()
|
| 634 |
else:
|
| 635 |
raise err(503, 'connector_paused', 'This source is paused and its scoped snapshot is unavailable.')
|
| 636 |
+
else:
|
| 637 |
from harness import canonical_source
|
| 638 |
def mirrored():
|
| 639 |
# Stock, sales and product reads come from the Odoo mirror, never live Odoo. A lent
|
| 640 |
# cursor that cannot see it (Sheet's in-memory one) gets a child cursor on the mirror.
|
| 641 |
source_cur, owned = cur, None
|
| 642 |
+
kind = 'product' if table_key == 'product_data' else 'customer'
|
| 643 |
+
if canonical_source.mirror_missing(cur, kind):
|
| 644 |
import routes_odoo_tables as rot
|
| 645 |
base, mismatch = rot._mirror_cur(session)
|
| 646 |
if mismatch:
|
| 647 |
raise err(409, 'cross_tenant_store', mismatch)
|
| 648 |
source_cur = owned = base.cursor()
|
| 649 |
try:
|
| 650 |
+
if table_key == 'product_data':
|
| 651 |
+
source = product_data.source_batches(source_cur, st=runtime, team_id=team, ids=ids,
|
| 652 |
+
needed=required, raw_fields=[field['rawName'] for field in raw])
|
| 653 |
+
else:
|
| 654 |
+
source = customer_data.source_batches(source_cur, st=runtime, agent_name=agent,
|
| 655 |
+
team_id=team, ids=ids, needed=required,
|
| 656 |
+
ledger_batches=items(session.runtime, customer_data.LEDGER_KEY, selected=ids)
|
| 657 |
+
if not agent and team is None and session.runtime.exists(customer_data.LEDGER_KEY) else ())
|
| 658 |
+
with canonical_source.mirrored(source_cur, runtime, kind=kind), closing(source):
|
| 659 |
yield from source
|
| 660 |
finally:
|
| 661 |
if owned is not None:
|
| 662 |
owned.close()
|
| 663 |
batches = mirrored()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 664 |
with closing(batches):
|
| 665 |
for rows in batches:
|
| 666 |
runtime = CanonicalStore(session, table_key, rows, cur=cur, metadata=metadata, needed=required)
|
api/request_timing.py
CHANGED
|
@@ -20,12 +20,18 @@ _STATS = OrderedDict()
|
|
| 20 |
_ROUTE_LIMIT = 512
|
| 21 |
|
| 22 |
|
| 23 |
-
def request_timeout_s():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
try:
|
| 25 |
-
value = float(os.environ.get("AIOS_REQUEST_TIMEOUT_S",
|
| 26 |
-
return max(.01, value) if math.isfinite(value) else
|
| 27 |
except ValueError:
|
| 28 |
-
return
|
| 29 |
|
| 30 |
|
| 31 |
_BUSY_CAUSES = {
|
|
@@ -375,7 +381,7 @@ class RequestTiming:
|
|
| 375 |
if scope.get("type") != "http":
|
| 376 |
return await self.app(scope, receive, send)
|
| 377 |
with timing.request_scope() as acc:
|
| 378 |
-
acc.deadline = time.monotonic() + request_timeout_s()
|
| 379 |
status, failed, disconnected, complete = None, False, False, False
|
| 380 |
threshold = slow_threshold_ms()
|
| 381 |
_resolve_route(scope)
|
|
|
|
| 20 |
_ROUTE_LIMIT = 512
|
| 21 |
|
| 22 |
|
| 23 |
+
def request_timeout_s(scope=None):
|
| 24 |
+
"""Give cold pool reads a bounded budget unless the operator sets an explicit timeout."""
|
| 25 |
+
pool_read = bool(scope and (scope.get("method"), scope.get("path")) in {
|
| 26 |
+
("GET", "/api/v1/products"), ("GET", "/api/v1/customers"),
|
| 27 |
+
("POST", "/api/v1/products/aggregates"), ("POST", "/api/v1/customers/aggregates"),
|
| 28 |
+
})
|
| 29 |
+
default = 120.0 if pool_read else 45.0
|
| 30 |
try:
|
| 31 |
+
value = float(os.environ.get("AIOS_REQUEST_TIMEOUT_S", default))
|
| 32 |
+
return max(.01, value) if math.isfinite(value) else default
|
| 33 |
except ValueError:
|
| 34 |
+
return default
|
| 35 |
|
| 36 |
|
| 37 |
_BUSY_CAUSES = {
|
|
|
|
| 381 |
if scope.get("type") != "http":
|
| 382 |
return await self.app(scope, receive, send)
|
| 383 |
with timing.request_scope() as acc:
|
| 384 |
+
acc.deadline = time.monotonic() + request_timeout_s(scope)
|
| 385 |
status, failed, disconnected, complete = None, False, False, False
|
| 386 |
threshold = slow_threshold_ms()
|
| 387 |
_resolve_route(scope)
|
api/routes_account_health.py
CHANGED
|
@@ -96,9 +96,12 @@ def owners(session):
|
|
| 96 |
|
| 97 |
|
| 98 |
@router.get("/summary")
|
| 99 |
-
def read_summary(app: str = Query(..., min_length=1),
|
|
|
|
|
|
|
| 100 |
ah = _module()
|
| 101 |
-
out = _call(ah.
|
|
|
|
| 102 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 103 |
|
| 104 |
|
|
@@ -111,11 +114,12 @@ def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Qu
|
|
| 111 |
|
| 112 |
@router.get("/items")
|
| 113 |
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 114 |
-
|
|
|
|
| 115 |
session: Session = Depends(_app_reader)):
|
| 116 |
ah = _module()
|
| 117 |
-
listed = _call(ah.items, _tenant(session), session.user, status=status,
|
| 118 |
-
st=session.runtime)
|
| 119 |
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 120 |
|
| 121 |
|
|
|
|
| 96 |
|
| 97 |
|
| 98 |
@router.get("/summary")
|
| 99 |
+
def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
|
| 100 |
+
offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 101 |
+
metric: str | None = Query(None), session: Session = Depends(_app_reader)):
|
| 102 |
ah = _module()
|
| 103 |
+
out = _call(ah.summary_page, _tenant(session), search=search, offset=offset, limit=limit,
|
| 104 |
+
metric=metric, st=session.runtime)
|
| 105 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 106 |
|
| 107 |
|
|
|
|
| 114 |
|
| 115 |
@router.get("/items")
|
| 116 |
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 117 |
+
search: str = Query(""), offset: int = Query(0, ge=0), metric: str | None = Query(None),
|
| 118 |
+
cursor: str | None = Query(None), limit: int = Query(50, ge=1),
|
| 119 |
session: Session = Depends(_app_reader)):
|
| 120 |
ah = _module()
|
| 121 |
+
listed = _call(ah.items, _tenant(session), session.user, status=status, search=search, offset=offset,
|
| 122 |
+
cursor=cursor, limit=limit, metric=metric, st=session.runtime)
|
| 123 |
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 124 |
|
| 125 |
|
api/routes_aged_stock.py
CHANGED
|
@@ -1,5 +1,9 @@
|
|
| 1 |
"""Aged stock App routes (C3): summary, items, one SKU, settings, refresh and the rule 8 tie."""
|
|
|
|
|
|
|
|
|
|
| 2 |
from fastapi import APIRouter, Body, Depends, Query
|
|
|
|
| 3 |
|
| 4 |
from deps import Session, err, require_session
|
| 5 |
|
|
@@ -95,40 +99,33 @@ def owners(session):
|
|
| 95 |
return sorted((n for n in names if n), key=str.lower)
|
| 96 |
|
| 97 |
|
| 98 |
-
#: The item fields a Next tier row carries, so the screen lists it with no second read.
|
| 99 |
-
ROW_FIELDS = ("action", "recommended_action", "owner", "status", "due", "units")
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
def _summary_rows(tenant_key, st):
|
| 103 |
-
"""The module summary with each Next tier row joined to its item's action, owner, status and due."""
|
| 104 |
-
mod = _module()
|
| 105 |
-
out = mod.summary(tenant_key, st=st)
|
| 106 |
-
stored = mod._doc(st, mod.KEY_ITEMS).get("items") or {}
|
| 107 |
-
for row in out.get("nextTier") or ():
|
| 108 |
-
item = stored.get(row.get("id")) or {}
|
| 109 |
-
row["item"] = {key: item.get(key) for key in ROW_FIELDS}
|
| 110 |
-
return out
|
| 111 |
-
|
| 112 |
-
|
| 113 |
@router.get("/summary")
|
| 114 |
-
def read_summary(app: str = Query(..., min_length=1),
|
| 115 |
-
|
|
|
|
|
|
|
|
|
|
| 116 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 117 |
|
| 118 |
|
| 119 |
@router.get("/items")
|
| 120 |
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 121 |
action: str | None = Query(None), cursor: str | None = Query(None),
|
| 122 |
-
limit: int = Query(200, ge=1
|
|
|
|
|
|
|
| 123 |
session: Session = Depends(_app_reader)):
|
| 124 |
listed = _call(_module().items, _tenant(session), session.uname, status=status, action=action,
|
| 125 |
-
cursor=cursor, limit=limit, q=q, st=session.runtime)
|
| 126 |
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 127 |
|
| 128 |
|
| 129 |
@router.get("/items/{item_id}")
|
| 130 |
-
def read_item(item_id: str, app: str = Query(..., min_length=1),
|
| 131 |
-
|
|
|
|
|
|
|
|
|
|
| 132 |
return {**detail, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 133 |
|
| 134 |
|
|
@@ -158,7 +155,80 @@ def write_settings(adsFromDays: int | None = Body(None), priceDownFromDays: int
|
|
| 158 |
|
| 159 |
@router.post("/refresh")
|
| 160 |
def refresh(session: Session = Depends(_app_writer)):
|
| 161 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 162 |
|
| 163 |
|
| 164 |
@router.get("/validate")
|
|
|
|
| 1 |
"""Aged stock App routes (C3): summary, items, one SKU, settings, refresh and the rule 8 tie."""
|
| 2 |
+
import time
|
| 3 |
+
import uuid
|
| 4 |
+
|
| 5 |
from fastapi import APIRouter, Body, Depends, Query
|
| 6 |
+
from fastapi.responses import JSONResponse
|
| 7 |
|
| 8 |
from deps import Session, err, require_session
|
| 9 |
|
|
|
|
| 99 |
return sorted((n for n in names if n), key=str.lower)
|
| 100 |
|
| 101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
@router.get("/summary")
|
| 103 |
+
def read_summary(app: str = Query(..., min_length=1), offset: int = Query(0, ge=0),
|
| 104 |
+
limit: int = Query(24, ge=1, description="Requests above 200 use 200 months. monthLimit reports the effective size."),
|
| 105 |
+
session: Session = Depends(_app_reader)):
|
| 106 |
+
out = _call(_module().summary, _tenant(session), offset=offset,
|
| 107 |
+
limit=min(limit, _module().MAX_PAGE_SIZE), st=session.runtime)
|
| 108 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 109 |
|
| 110 |
|
| 111 |
@router.get("/items")
|
| 112 |
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 113 |
action: str | None = Query(None), cursor: str | None = Query(None),
|
| 114 |
+
limit: int = Query(200, ge=1, description="Requests above 200 use 200 items. limit reports the effective size."),
|
| 115 |
+
q: str | None = Query(None),
|
| 116 |
+
nextTier: bool = Query(False),
|
| 117 |
session: Session = Depends(_app_reader)):
|
| 118 |
listed = _call(_module().items, _tenant(session), session.uname, status=status, action=action,
|
| 119 |
+
cursor=cursor, limit=min(limit, _module().MAX_PAGE_SIZE), q=q, next_tier=nextTier, st=session.runtime)
|
| 120 |
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 121 |
|
| 122 |
|
| 123 |
@router.get("/items/{item_id}")
|
| 124 |
+
def read_item(item_id: str, app: str = Query(..., min_length=1), offset: int = Query(0, ge=0),
|
| 125 |
+
limit: int = Query(50, ge=1, description="Requests above 200 use 200 entries per history. detailLimit reports the effective size."),
|
| 126 |
+
session: Session = Depends(_app_reader)):
|
| 127 |
+
detail = _call(_module().item_detail, _tenant(session), item_id, offset=offset,
|
| 128 |
+
limit=min(limit, _module().MAX_PAGE_SIZE), st=session.runtime)
|
| 129 |
return {**detail, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 130 |
|
| 131 |
|
|
|
|
| 155 |
|
| 156 |
@router.post("/refresh")
|
| 157 |
def refresh(session: Session = Depends(_app_writer)):
|
| 158 |
+
identity, stamp = uuid.uuid4().hex, time.time()
|
| 159 |
+
|
| 160 |
+
def enqueue(doc):
|
| 161 |
+
doc = doc if isinstance(doc, dict) else {}
|
| 162 |
+
if doc.get("state") in ("queued", "running") and stamp - float(doc.get("requestedAt") or 0) < 600:
|
| 163 |
+
return doc
|
| 164 |
+
return {"state": "queued", "requestId": identity, "requestedAt": stamp}
|
| 165 |
+
|
| 166 |
+
queued = _call(session.runtime.update, _module().KEY_REFRESH, enqueue)
|
| 167 |
+
return JSONResponse(status_code=202, content=queued)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
@router.get("/refresh")
|
| 171 |
+
def refresh_status(session: Session = Depends(_app_reader)):
|
| 172 |
+
return _call(session.runtime.get, _module().KEY_REFRESH) or {"state": "idle"}
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def _drain_refresh(rt, *, now=None):
|
| 176 |
+
"""Claim one durable refresh intent in the sync worker and retain its terminal result."""
|
| 177 |
+
mod = _module()
|
| 178 |
+
stamp = time.time() if now is None else now
|
| 179 |
+
pending = rt.get(mod.KEY_REFRESH) or {}
|
| 180 |
+
if pending.get("state") != "queued" and not (
|
| 181 |
+
pending.get("state") == "running" and stamp - float(pending.get("startedAt") or 0) >= 600):
|
| 182 |
+
return
|
| 183 |
+
token, claimed = uuid.uuid4().hex, {}
|
| 184 |
+
|
| 185 |
+
def claim(doc):
|
| 186 |
+
if (doc or {}).get("requestId") == pending.get("requestId") and (
|
| 187 |
+
doc.get("state") == "queued" or (doc.get("state") == "running"
|
| 188 |
+
and stamp - float(doc.get("startedAt") or 0) >= 600)):
|
| 189 |
+
doc.update(state="running", startedAt=stamp, claim=token)
|
| 190 |
+
claimed.update(doc)
|
| 191 |
+
return doc
|
| 192 |
+
|
| 193 |
+
rt.update(mod.KEY_REFRESH, claim)
|
| 194 |
+
if not claimed:
|
| 195 |
+
return
|
| 196 |
+
try:
|
| 197 |
+
result = mod.refresh(rt.key, st=rt)
|
| 198 |
+
completion = {"state": "done", "result": result}
|
| 199 |
+
except Exception: # noqa: BLE001
|
| 200 |
+
completion = {"state": "failed", "message": "The aged stock view could not be refreshed. Try again shortly."}
|
| 201 |
+
|
| 202 |
+
def finish(doc):
|
| 203 |
+
if (doc or {}).get("claim") == token:
|
| 204 |
+
doc.update(completion, finishedAt=time.time())
|
| 205 |
+
return doc
|
| 206 |
+
rt.update(mod.KEY_REFRESH, finish)
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def _refresh_worker():
|
| 210 |
+
import threading
|
| 211 |
+
import process_role
|
| 212 |
+
from harness import runtime
|
| 213 |
+
stop = threading.Event()
|
| 214 |
+
while not stop.wait(1):
|
| 215 |
+
if not process_role.may_write():
|
| 216 |
+
continue
|
| 217 |
+
for tenant in runtime.known_tenants():
|
| 218 |
+
try:
|
| 219 |
+
_drain_refresh(runtime.get_runtime(tenant))
|
| 220 |
+
except Exception: # noqa: BLE001
|
| 221 |
+
continue
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def _start_refresh_worker():
|
| 225 |
+
import process_role
|
| 226 |
+
from core import data_binding
|
| 227 |
+
if not data_binding.gate_process():
|
| 228 |
+
process_role.start("aged-stock-refresh", "heavy", _refresh_worker)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
_start_refresh_worker()
|
| 232 |
|
| 233 |
|
| 234 |
@router.get("/validate")
|
api/routes_amazon_deletion.py
CHANGED
|
@@ -13,15 +13,27 @@ ATTENTION_MESSAGE = "Amazon data deletion needs attention from an administrator.
|
|
| 13 |
@router.get("/connectors/amazon_sp/deletion")
|
| 14 |
def deletion_status(session: Session = Depends(admin_gate)):
|
| 15 |
"""Read metadata and lifecycle evidence without decrypting credentials or running purge."""
|
|
|
|
| 16 |
from core import keychain
|
| 17 |
from harness import amazon_store
|
| 18 |
|
| 19 |
rt = session.runtime
|
| 20 |
try:
|
| 21 |
-
|
| 22 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
return {"state": "none"}
|
| 24 |
-
plan = amazon_store.scheduled_purge(rt.key, rt)
|
| 25 |
ledger = amazon_store._ledger(rt.key, rt)
|
| 26 |
now = amazon_store._utc()
|
| 27 |
checks = [row for row in ledger.get("checks", [])
|
|
|
|
| 13 |
@router.get("/connectors/amazon_sp/deletion")
|
| 14 |
def deletion_status(session: Session = Depends(admin_gate)):
|
| 15 |
"""Read metadata and lifecycle evidence without decrypting credentials or running purge."""
|
| 16 |
+
from amazon_relational import CONNECTION_REVOCATIONS_KEY
|
| 17 |
from core import keychain
|
| 18 |
from harness import amazon_store
|
| 19 |
|
| 20 |
rt = session.runtime
|
| 21 |
try:
|
| 22 |
+
revocations = rt.get(CONNECTION_REVOCATIONS_KEY)
|
| 23 |
+
if revocations is None:
|
| 24 |
+
revocations = {}
|
| 25 |
+
if any(not isinstance(row, dict) or row.get("state") != "done" for row in revocations.values()):
|
| 26 |
+
return {"state": "failed", "message": ATTENTION_MESSAGE}
|
| 27 |
+
connected = keychain.first_entry_of_type(rt, "amazon_sp") is not None
|
| 28 |
+
if connected:
|
| 29 |
+
# Reconnection cannot cancel an independent notice or termination obligation.
|
| 30 |
+
ledger = rt.get(amazon_store.LIFECYCLE_KEY) or {}
|
| 31 |
+
if not any(row["kind"] in ("notice", "termination") for row in
|
| 32 |
+
amazon_store._active_deletion_triggers(rt.key, ledger, amazon_store._utc())):
|
| 33 |
+
return {"state": "none"}
|
| 34 |
+
plan = amazon_store.scheduled_purge(rt.key, rt, ignore_credential_loss=connected)
|
| 35 |
+
if connected and plan is None:
|
| 36 |
return {"state": "none"}
|
|
|
|
| 37 |
ledger = amazon_store._ledger(rt.key, rt)
|
| 38 |
now = amazon_store._utc()
|
| 39 |
checks = [row for row in ledger.get("checks", [])
|
api/routes_amazon_operator.py
CHANGED
|
@@ -78,24 +78,21 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
|
|
| 78 |
|
| 79 |
|
| 80 |
@router.get("/today")
|
| 81 |
-
def read_today(app: str = Query(..., min_length=1),
|
|
|
|
|
|
|
| 82 |
op = _module()
|
| 83 |
-
out = _call(op.
|
|
|
|
| 84 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 85 |
|
| 86 |
|
| 87 |
@router.get("/history")
|
| 88 |
def read_history(status: str | None = Query(None), cursor: str | None = Query(None),
|
| 89 |
-
limit: int = Query(
|
| 90 |
op = _module()
|
| 91 |
-
|
| 92 |
st=session.runtime)
|
| 93 |
-
if cursor:
|
| 94 |
-
return page
|
| 95 |
-
# The first page carries the full count for the History tab. Uncapped: every row is counted (rule 1).
|
| 96 |
-
every = _call(op.items, _tenant(session), session.user, status=status, cursor=None, limit=None,
|
| 97 |
-
st=session.runtime)
|
| 98 |
-
return {**page, "total": len(every["items"])}
|
| 99 |
|
| 100 |
|
| 101 |
@router.get("/settings")
|
|
@@ -131,3 +128,50 @@ def dismiss(item_ids: list[str] = Body(...), reason: str = Body(..., min_length=
|
|
| 131 |
op = _module()
|
| 132 |
return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason,
|
| 133 |
st=session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
|
| 79 |
|
| 80 |
@router.get("/today")
|
| 81 |
+
def read_today(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 82 |
+
app_cursor: str | None = Query(None), held_cursor: str | None = Query(None),
|
| 83 |
+
limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 84 |
op = _module()
|
| 85 |
+
out = _call(op.screen_today, _tenant(session), st=session.runtime, cursor=cursor,
|
| 86 |
+
app_cursor=app_cursor, held_cursor=held_cursor, limit=limit, q=q)
|
| 87 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 88 |
|
| 89 |
|
| 90 |
@router.get("/history")
|
| 91 |
def read_history(status: str | None = Query(None), cursor: str | None = Query(None),
|
| 92 |
+
limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 93 |
op = _module()
|
| 94 |
+
return _call(op.screen_history, _tenant(session), status=status, cursor=cursor, limit=limit, q=q,
|
| 95 |
st=session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
|
| 97 |
|
| 98 |
@router.get("/settings")
|
|
|
|
| 128 |
op = _module()
|
| 129 |
return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason,
|
| 130 |
st=session.runtime)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _refresh_inboxes_once():
|
| 134 |
+
"""Refresh installed Operator inboxes independently of Ads data and screen requests."""
|
| 135 |
+
import application_registry
|
| 136 |
+
from harness import runtime
|
| 137 |
+
for tenant in runtime.known_tenants():
|
| 138 |
+
try:
|
| 139 |
+
st = runtime.get_runtime(tenant)
|
| 140 |
+
apps = application_registry.load(st)["applications"].values()
|
| 141 |
+
if any(a.get("templateKind") == TEMPLATE_KIND and not a.get("retired") for a in apps):
|
| 142 |
+
_module().pull_app_proposals(tenant, st=st)
|
| 143 |
+
except Exception:
|
| 144 |
+
import logging
|
| 145 |
+
logging.getLogger(__name__).exception("Operator inbox refresh failed for %s", tenant)
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
_inbox_thread = None
|
| 149 |
+
_inbox_stop = None
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
@router.on_event("startup")
|
| 153 |
+
def _start_inbox_refresh():
|
| 154 |
+
global _inbox_thread, _inbox_stop
|
| 155 |
+
import process_role
|
| 156 |
+
import threading
|
| 157 |
+
if _inbox_thread is not None and _inbox_thread.is_alive():
|
| 158 |
+
return
|
| 159 |
+
_inbox_stop = threading.Event()
|
| 160 |
+
|
| 161 |
+
def run():
|
| 162 |
+
while not _inbox_stop.is_set():
|
| 163 |
+
_refresh_inboxes_once()
|
| 164 |
+
_inbox_stop.wait(30)
|
| 165 |
+
_inbox_thread = process_role.start("amazon-operator-inbox", "heavy", run)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
@router.on_event("shutdown")
|
| 169 |
+
def _stop_inbox_refresh():
|
| 170 |
+
if _inbox_stop is not None:
|
| 171 |
+
_inbox_stop.set()
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
# The sync worker imports routers without invoking FastAPI startup hooks.
|
| 175 |
+
import process_role as _process_role
|
| 176 |
+
if _process_role.role() == "worker":
|
| 177 |
+
_start_inbox_refresh()
|
api/routes_competitor_watch.py
CHANGED
|
@@ -38,6 +38,19 @@ def _call(fn, *args, **kwargs):
|
|
| 38 |
raise err(400, "competitor_watch_invalid", str(exc)) from exc
|
| 39 |
except KeyError as exc:
|
| 40 |
raise err(404, "competitor_watch_alert_not_found", "That alert does not exist.") from exc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
except Exception as exc: # noqa: BLE001
|
| 42 |
raise err(503, "competitor_watch_unavailable", "The competitor watch could not be read. Try again shortly.") from exc
|
| 43 |
|
|
@@ -80,9 +93,10 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
|
|
| 80 |
|
| 81 |
|
| 82 |
@router.get("/terms")
|
| 83 |
-
def list_terms(
|
|
|
|
| 84 |
cw = _module()
|
| 85 |
-
return _call(cw.
|
| 86 |
|
| 87 |
|
| 88 |
@router.get("/terms/detail")
|
|
@@ -94,13 +108,13 @@ def term_detail(term: str = Query(..., min_length=1), session: Session = Depends
|
|
| 94 |
@router.post("/terms")
|
| 95 |
def add_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 96 |
cw = _module()
|
| 97 |
-
return {"terms": _call(cw.add_terms, _tenant(session), terms, st=session.runtime)}
|
| 98 |
|
| 99 |
|
| 100 |
@router.post("/terms/remove")
|
| 101 |
def remove_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 102 |
cw = _module()
|
| 103 |
-
return {"terms": _call(cw.remove_terms, _tenant(session), terms, st=session.runtime)}
|
| 104 |
|
| 105 |
|
| 106 |
@router.get("/settings")
|
|
@@ -119,9 +133,11 @@ def write_settings(undercutPct: float | None = Body(None), rankSurgePct: float |
|
|
| 119 |
|
| 120 |
@router.get("/alerts")
|
| 121 |
def list_alerts(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
|
|
|
| 122 |
session: Session = Depends(_app_reader)):
|
| 123 |
cw = _module()
|
| 124 |
-
listed = _call(cw.alerts, _tenant(session), status=status, st=session.runtime
|
|
|
|
| 125 |
return {**listed, "canEdit": _may_edit(session, _app(session, app))}
|
| 126 |
|
| 127 |
|
|
@@ -196,6 +212,7 @@ def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
|
|
| 196 |
comparison = detail.get("comparison") or {}
|
| 197 |
ready = comparison.get("ready") is True
|
| 198 |
return {"canEdit": bool(can_edit), "terms": list(terms), "term": term,
|
|
|
|
| 199 |
"week": {"start": periods[0], "end": _week_end(periods[0])} if periods else None,
|
| 200 |
"previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
|
| 201 |
"comparisonReady": ready,
|
|
|
|
| 38 |
raise err(400, "competitor_watch_invalid", str(exc)) from exc
|
| 39 |
except KeyError as exc:
|
| 40 |
raise err(404, "competitor_watch_alert_not_found", "That alert does not exist.") from exc
|
| 41 |
+
except _module().SourceNotReady as exc:
|
| 42 |
+
cw = _module()
|
| 43 |
+
cause = cw.READ_FAILED.format(cause=exc)
|
| 44 |
+
page = exc.page or {"termsReady": False}
|
| 45 |
+
page = {**page, "problems": list(dict.fromkeys([*(page.get("problems") or []), cause]))}
|
| 46 |
+
screen = screen_payload(cw, {}, page.get("terms", []), page.get("term"), False, page=page)
|
| 47 |
+
raise err(503, "competitor_watch_source_not_ready", cause, {"screen": screen}) from exc
|
| 48 |
+
except _module().TermsNotReady as exc:
|
| 49 |
+
screen = screen_payload(_module(), {}, [], None, False,
|
| 50 |
+
page={"problems": exc.problems, "termsReady": False})
|
| 51 |
+
raise err(503, "competitor_watch_terms_not_ready", str(exc), {"screen": screen}) from exc
|
| 52 |
+
except _module().AlertsNotReady as exc:
|
| 53 |
+
raise err(503, "competitor_watch_alerts_not_ready", str(exc)) from exc
|
| 54 |
except Exception as exc: # noqa: BLE001
|
| 55 |
raise err(503, "competitor_watch_unavailable", "The competitor watch could not be read. Try again shortly.") from exc
|
| 56 |
|
|
|
|
| 93 |
|
| 94 |
|
| 95 |
@router.get("/terms")
|
| 96 |
+
def list_terms(q: str = "", offset: int = 0, limit: int = 50,
|
| 97 |
+
session: Session = Depends(_app_reader)):
|
| 98 |
cw = _module()
|
| 99 |
+
return _call(cw.term_page, _tenant(session), st=session.runtime, search=q, offset=offset, limit=limit)
|
| 100 |
|
| 101 |
|
| 102 |
@router.get("/terms/detail")
|
|
|
|
| 108 |
@router.post("/terms")
|
| 109 |
def add_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 110 |
cw = _module()
|
| 111 |
+
return {"terms": _call(cw.add_terms, _tenant(session), terms, st=session.runtime, page_only=True)}
|
| 112 |
|
| 113 |
|
| 114 |
@router.post("/terms/remove")
|
| 115 |
def remove_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 116 |
cw = _module()
|
| 117 |
+
return {"terms": _call(cw.remove_terms, _tenant(session), terms, st=session.runtime, page_only=True)}
|
| 118 |
|
| 119 |
|
| 120 |
@router.get("/settings")
|
|
|
|
| 133 |
|
| 134 |
@router.get("/alerts")
|
| 135 |
def list_alerts(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 136 |
+
q: str = "", offset: int = 0, limit: int = 50, check_offset: int = 0, check_limit: int = 50,
|
| 137 |
session: Session = Depends(_app_reader)):
|
| 138 |
cw = _module()
|
| 139 |
+
listed = _call(cw.alerts, _tenant(session), status=status, st=session.runtime, search=q,
|
| 140 |
+
offset=offset, limit=limit, check_offset=check_offset, check_limit=check_limit)
|
| 141 |
return {**listed, "canEdit": _may_edit(session, _app(session, app))}
|
| 142 |
|
| 143 |
|
|
|
|
| 212 |
comparison = detail.get("comparison") or {}
|
| 213 |
ready = comparison.get("ready") is True
|
| 214 |
return {"canEdit": bool(can_edit), "terms": list(terms), "term": term,
|
| 215 |
+
"termsReady": (page or {}).get("termsReady", True),
|
| 216 |
"week": {"start": periods[0], "end": _week_end(periods[0])} if periods else None,
|
| 217 |
"previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
|
| 218 |
"comparisonReady": ready,
|
api/routes_connectors.py
CHANGED
|
@@ -131,6 +131,7 @@ class AiClientStepAction(BaseModel):
|
|
| 131 |
|
| 132 |
|
| 133 |
class AiClientStep(BaseModel):
|
|
|
|
| 134 |
text: str = Field(min_length=1)
|
| 135 |
block: str | None = None
|
| 136 |
action: AiClientStepAction | None = None
|
|
@@ -150,19 +151,31 @@ class AiClientGuide(BaseModel):
|
|
| 150 |
record: str = Field(min_length=1)
|
| 151 |
provenance: str = Field(min_length=1)
|
| 152 |
need: str = Field(min_length=1, strict=True)
|
| 153 |
-
steps: list[AiClientStep] = Field(min_length=
|
| 154 |
keySteps: list[AiClientStep] = Field(default_factory=list)
|
| 155 |
caveat: str = Field(min_length=1)
|
| 156 |
failureAction: str = Field(min_length=1, strict=True)
|
| 157 |
-
testNote: str = Field(
|
| 158 |
|
| 159 |
-
@field_validator("need", "failureAction"
|
| 160 |
@classmethod
|
| 161 |
def guide_text(cls, value: str) -> str:
|
| 162 |
if not value.strip():
|
| 163 |
raise ValueError("A guide needs all four fields.")
|
| 164 |
return value
|
| 165 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
|
| 167 |
class AiClientVerification(BaseModel):
|
| 168 |
mcp: str = Field(min_length=1)
|
|
@@ -1590,7 +1603,10 @@ def connectors_directory(session: Session = Depends(require_session)):
|
|
| 1590 |
# number of cards behind it by construction, and never a sum of endpoint counts (the ticket's
|
| 1591 |
# trap; `chip_counts` carries the argument).
|
| 1592 |
served = connector_directory.to_client_rows(rows)
|
|
|
|
| 1593 |
for row in served:
|
|
|
|
|
|
|
| 1594 |
if str(row.get("key") or "") == "scraper":
|
| 1595 |
row["spawns_tables"] = True
|
| 1596 |
_safe_connector_status(row.get("key"), row)
|
|
@@ -3679,17 +3695,17 @@ _CONNECT_TUTORIALS = {
|
|
| 3679 |
"odoo": {
|
| 3680 |
"need": "An Odoo URL, database, user and API key.",
|
| 3681 |
"steps": ["Copy your Odoo URL and ask your administrator for the database name.",
|
| 3682 |
-
"Sign in as the API user. Open My profile,
|
| 3683 |
-
"Create
|
| 3684 |
"failureAction": "Open the server URL in a browser and confirm it is reachable. "
|
| 3685 |
"Then check the database name and that the API key belongs to the API user.",
|
| 3686 |
"testNote": "Checks server reachability only. It does not verify the user or API key.",
|
| 3687 |
},
|
| 3688 |
"amazon_sp": {
|
| 3689 |
"need": "A Production app client and access to your Amazon seller account.",
|
| 3690 |
-
"steps": ["
|
| 3691 |
-
"
|
| 3692 |
-
"Authorize
|
| 3693 |
"In Seller Central, open Settings, Account info and copy your selling partner ID. "
|
| 3694 |
"Choose the marketplace that account sells in."],
|
| 3695 |
"failureAction": "In Solution Provider Portal, confirm the client ID and secret belong to "
|
|
@@ -3699,9 +3715,9 @@ _CONNECT_TUTORIALS = {
|
|
| 3699 |
},
|
| 3700 |
"amazon_ads": {
|
| 3701 |
"need": "An Amazon Ads API app and access to your advertising account.",
|
| 3702 |
-
"steps": ["Open your Amazon Ads API app's Login with Amazon security profile."
|
| 3703 |
"Copy its client ID and client secret.",
|
| 3704 |
-
"Authorize the app for your advertising account and copy the refresh token."
|
| 3705 |
"Copy the profile ID of the marketplace you advertise in."],
|
| 3706 |
"failureAction": "Authorize the app again for your advertising account, then check "
|
| 3707 |
"that the profile ID belongs to it.",
|
|
@@ -3716,7 +3732,7 @@ _CONNECT_TUTORIALS = {
|
|
| 3716 |
},
|
| 3717 |
"shopify": {
|
| 3718 |
"need": "Permission to create a custom app in your Shopify store.",
|
| 3719 |
-
"steps": ["In your store admin, open Settings, Apps and sales channels, then Develop apps."
|
| 3720 |
"Create a custom app, configure its read permissions and install it.",
|
| 3721 |
"Open API credentials and copy its Admin API access token."],
|
| 3722 |
"failureAction": "Open the installed custom app's API credentials and copy its Admin API token.",
|
|
@@ -3724,7 +3740,7 @@ _CONNECT_TUTORIALS = {
|
|
| 3724 |
},
|
| 3725 |
"meta_ads": {
|
| 3726 |
"need": "A Meta app and access to the ad account you want to read.",
|
| 3727 |
-
"steps": ["Open Meta for Developers and select your app."
|
| 3728 |
"Use Graph API Explorer to get a Marketing API token with ads_read.",
|
| 3729 |
"Confirm the token's user can access your ad account, then copy the token."],
|
| 3730 |
"failureAction": "Generate a current Marketing API token with ads_read and confirm its "
|
|
@@ -4048,9 +4064,17 @@ def _detail_keychain(rt, uname, is_admin, row):
|
|
| 4048 |
continue
|
| 4049 |
entries.append({
|
| 4050 |
"id": str(entry.get("id") or ""),
|
|
|
|
|
|
|
|
|
|
| 4051 |
"label": str(entry.get("label") or entry.get("id") or ""),
|
| 4052 |
"masked": str(entry.get("preview") or ""),
|
| 4053 |
"scope": str(entry.get("scope") or rk.DEFAULT_SCOPE),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4054 |
# `keychain.test_entry` answers for every stored entry: a registered prober for
|
| 4055 |
# the type when the connector layer filled that slot, and a read-only shape or
|
| 4056 |
# xmlrpc check otherwise. The flag says whether a Test control would have
|
|
@@ -4067,6 +4091,7 @@ def _detail_keychain(rt, uname, is_admin, row):
|
|
| 4067 |
})
|
| 4068 |
except Exception: # noqa: BLE001
|
| 4069 |
entries = []
|
|
|
|
| 4070 |
# ⭐ THE SECOND SOURCE. An OAuth card has no keychain entry type at all, so `entries` above is
|
| 4071 |
# empty for it by construction and this is the only reading that can fill the tab.
|
| 4072 |
if not entry_type and manage == "oauth":
|
|
@@ -4077,12 +4102,21 @@ def _detail_keychain(rt, uname, is_admin, row):
|
|
| 4077 |
"canConnect": bool(row.get("startUrl")) and connect_available,
|
| 4078 |
"canDisconnect": bool(entries),
|
| 4079 |
"connect": None}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4080 |
return {"entries": entries,
|
| 4081 |
"canConnect": can_connect,
|
| 4082 |
"canDisconnect": bool(can_connect and entries),
|
|
|
|
| 4083 |
# What the Connect form asks for. `None` when this card has no credential door at all,
|
| 4084 |
# which is what stops the client inventing one (Webhooks, the Scraper, a planned row).
|
| 4085 |
-
"connect":
|
| 4086 |
|
| 4087 |
|
| 4088 |
def _odoo_catalog_by_model(rt, connection_id=None, *, models=None):
|
|
|
|
| 131 |
|
| 132 |
|
| 133 |
class AiClientStep(BaseModel):
|
| 134 |
+
title: str | None = Field(default=None, min_length=1, max_length=48, strict=True)
|
| 135 |
text: str = Field(min_length=1)
|
| 136 |
block: str | None = None
|
| 137 |
action: AiClientStepAction | None = None
|
|
|
|
| 151 |
record: str = Field(min_length=1)
|
| 152 |
provenance: str = Field(min_length=1)
|
| 153 |
need: str = Field(min_length=1, strict=True)
|
| 154 |
+
steps: list[AiClientStep] = Field(min_length=2, max_length=3)
|
| 155 |
keySteps: list[AiClientStep] = Field(default_factory=list)
|
| 156 |
caveat: str = Field(min_length=1)
|
| 157 |
failureAction: str = Field(min_length=1, strict=True)
|
| 158 |
+
testNote: str = Field(strict=True)
|
| 159 |
|
| 160 |
+
@field_validator("need", "failureAction")
|
| 161 |
@classmethod
|
| 162 |
def guide_text(cls, value: str) -> str:
|
| 163 |
if not value.strip():
|
| 164 |
raise ValueError("A guide needs all four fields.")
|
| 165 |
return value
|
| 166 |
|
| 167 |
+
@field_validator("steps", "keySteps")
|
| 168 |
+
@classmethod
|
| 169 |
+
def short_steps(cls, steps: list[AiClientStep]) -> list[AiClientStep]:
|
| 170 |
+
if steps and not 2 <= len(steps) <= 3:
|
| 171 |
+
raise ValueError("An AI guide needs two or three steps.")
|
| 172 |
+
for step in steps:
|
| 173 |
+
if not step.title or not step.title.strip() or not step.block or not step.block.strip():
|
| 174 |
+
raise ValueError("An AI step needs a short title and a copy box.")
|
| 175 |
+
if re.search(r"\bTest\s*:", " ".join((step.title, step.text, step.block)), re.I):
|
| 176 |
+
raise ValueError("An AI guide has no Test line.")
|
| 177 |
+
return steps
|
| 178 |
+
|
| 179 |
|
| 180 |
class AiClientVerification(BaseModel):
|
| 181 |
mcp: str = Field(min_length=1)
|
|
|
|
| 1603 |
# number of cards behind it by construction, and never a sum of endpoint counts (the ticket's
|
| 1604 |
# trap; `chip_counts` carries the argument).
|
| 1605 |
served = connector_directory.to_client_rows(rows)
|
| 1606 |
+
sync_providers = connector_sync.providers()
|
| 1607 |
for row in served:
|
| 1608 |
+
# Registry support lets the modal start its fresh status read without another round trip.
|
| 1609 |
+
row["syncSupported"] = row.get("key") in sync_providers
|
| 1610 |
if str(row.get("key") or "") == "scraper":
|
| 1611 |
row["spawns_tables"] = True
|
| 1612 |
_safe_connector_status(row.get("key"), row)
|
|
|
|
| 3695 |
"odoo": {
|
| 3696 |
"need": "An Odoo URL, database, user and API key.",
|
| 3697 |
"steps": ["Copy your Odoo URL and ask your administrator for the database name.",
|
| 3698 |
+
"Sign in as the API user. Open My profile, Account security and API keys. "
|
| 3699 |
+
"Create and copy a key now. Keep this user's login with the key."],
|
| 3700 |
"failureAction": "Open the server URL in a browser and confirm it is reachable. "
|
| 3701 |
"Then check the database name and that the API key belongs to the API user.",
|
| 3702 |
"testNote": "Checks server reachability only. It does not verify the user or API key.",
|
| 3703 |
},
|
| 3704 |
"amazon_sp": {
|
| 3705 |
"need": "A Production app client and access to your Amazon seller account.",
|
| 3706 |
+
"steps": ["In Solution Provider Portal, open Apps and create or open your Production app. "
|
| 3707 |
+
"Copy its Login with Amazon client ID and client secret.",
|
| 3708 |
+
"Authorize that app for your own seller account and copy the refresh token. "
|
| 3709 |
"In Seller Central, open Settings, Account info and copy your selling partner ID. "
|
| 3710 |
"Choose the marketplace that account sells in."],
|
| 3711 |
"failureAction": "In Solution Provider Portal, confirm the client ID and secret belong to "
|
|
|
|
| 3715 |
},
|
| 3716 |
"amazon_ads": {
|
| 3717 |
"need": "An Amazon Ads API app and access to your advertising account.",
|
| 3718 |
+
"steps": ["Open your Amazon Ads API app's Login with Amazon security profile. "
|
| 3719 |
"Copy its client ID and client secret.",
|
| 3720 |
+
"Authorize the app for your advertising account and copy the refresh token. "
|
| 3721 |
"Copy the profile ID of the marketplace you advertise in."],
|
| 3722 |
"failureAction": "Authorize the app again for your advertising account, then check "
|
| 3723 |
"that the profile ID belongs to it.",
|
|
|
|
| 3732 |
},
|
| 3733 |
"shopify": {
|
| 3734 |
"need": "Permission to create a custom app in your Shopify store.",
|
| 3735 |
+
"steps": ["In your store admin, open Settings, Apps and sales channels, then Develop apps. "
|
| 3736 |
"Create a custom app, configure its read permissions and install it.",
|
| 3737 |
"Open API credentials and copy its Admin API access token."],
|
| 3738 |
"failureAction": "Open the installed custom app's API credentials and copy its Admin API token.",
|
|
|
|
| 3740 |
},
|
| 3741 |
"meta_ads": {
|
| 3742 |
"need": "A Meta app and access to the ad account you want to read.",
|
| 3743 |
+
"steps": ["Open Meta for Developers and select your app. "
|
| 3744 |
"Use Graph API Explorer to get a Marketing API token with ads_read.",
|
| 3745 |
"Confirm the token's user can access your ad account, then copy the token."],
|
| 3746 |
"failureAction": "Generate a current Marketing API token with ads_read and confirm its "
|
|
|
|
| 4064 |
continue
|
| 4065 |
entries.append({
|
| 4066 |
"id": str(entry.get("id") or ""),
|
| 4067 |
+
"type": entry_type,
|
| 4068 |
+
"connectionId": str(entry.get("connectionId") or entry.get("id") or ""),
|
| 4069 |
+
"slot": int(entry.get("slot") or 1),
|
| 4070 |
"label": str(entry.get("label") or entry.get("id") or ""),
|
| 4071 |
"masked": str(entry.get("preview") or ""),
|
| 4072 |
"scope": str(entry.get("scope") or rk.DEFAULT_SCOPE),
|
| 4073 |
+
"status": ("Paused" if entry_type == "odoo" and
|
| 4074 |
+
((rt.get(rk._CONNECTOR_FLAGS_KEY) or {}).get(
|
| 4075 |
+
entry.get("connectionId") or entry.get("id")) or {}).get("paused")
|
| 4076 |
+
else "Connected" if entry_type in ("amazon_sp", "amazon_ads")
|
| 4077 |
+
and rk._kc().entry_verified(rt, entry.get("id")) else "Needs a test"),
|
| 4078 |
# `keychain.test_entry` answers for every stored entry: a registered prober for
|
| 4079 |
# the type when the connector layer filled that slot, and a read-only shape or
|
| 4080 |
# xmlrpc check otherwise. The flag says whether a Test control would have
|
|
|
|
| 4091 |
})
|
| 4092 |
except Exception: # noqa: BLE001
|
| 4093 |
entries = []
|
| 4094 |
+
entries.sort(key=lambda entry: (entry["slot"], entry["id"]))
|
| 4095 |
# ⭐ THE SECOND SOURCE. An OAuth card has no keychain entry type at all, so `entries` above is
|
| 4096 |
# empty for it by construction and this is the only reading that can fill the tab.
|
| 4097 |
if not entry_type and manage == "oauth":
|
|
|
|
| 4102 |
"canConnect": bool(row.get("startUrl")) and connect_available,
|
| 4103 |
"canDisconnect": bool(entries),
|
| 4104 |
"connect": None}
|
| 4105 |
+
connect = _detail_connect(entry_type, bool(is_admin)) if can_connect else None
|
| 4106 |
+
if connect:
|
| 4107 |
+
from core import keychain as kc
|
| 4108 |
+
connect["defaultLabel"] = f"{kc.kind_label(entry_type)} {kc.next_slot(rt, entry_type)}"
|
| 4109 |
+
migration_pending = False
|
| 4110 |
+
if entry_type == "odoo":
|
| 4111 |
+
import routes_keychain as rk
|
| 4112 |
+
migration_pending = rk.env_odoo_migration_pending(rt)
|
| 4113 |
return {"entries": entries,
|
| 4114 |
"canConnect": can_connect,
|
| 4115 |
"canDisconnect": bool(can_connect and entries),
|
| 4116 |
+
"migrationPending": migration_pending,
|
| 4117 |
# What the Connect form asks for. `None` when this card has no credential door at all,
|
| 4118 |
# which is what stops the client inventing one (Webhooks, the Scraper, a planned row).
|
| 4119 |
+
"connect": connect}
|
| 4120 |
|
| 4121 |
|
| 4122 |
def _odoo_catalog_by_model(rt, connection_id=None, *, models=None):
|
api/routes_customers.py
CHANGED
|
@@ -29,10 +29,12 @@ import os
|
|
| 29 |
import threading
|
| 30 |
import time
|
| 31 |
|
| 32 |
-
from fastapi import APIRouter, Body, Depends
|
| 33 |
|
| 34 |
import scope_cache
|
| 35 |
-
from deps import Session, err, module_gate
|
|
|
|
|
|
|
| 36 |
|
| 37 |
router = APIRouter(prefix="/api/v1")
|
| 38 |
|
|
@@ -40,6 +42,50 @@ router = APIRouter(prefix="/api/v1")
|
|
| 40 |
#: 403 rather than an empty table that looks like "you have no customers".
|
| 41 |
MODULE = "customer_data"
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
|
| 44 |
|
| 45 |
# Waiters share one cold build (about 4 min on 2 vCPU); a private rebuild per waiter multiplies it.
|
|
@@ -664,10 +710,15 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 664 |
def own_copy(result):
|
| 665 |
# Both aliases lend immutable source rows. Copy the request-shaped metadata together.
|
| 666 |
borrowed = {id(result['rows_src']): result['rows_src']}
|
|
|
|
|
|
|
|
|
|
| 667 |
wall_rows = (result.get('_permission_scope') or {}).get('rows')
|
| 668 |
if wall_rows is not None:
|
| 669 |
borrowed[id(wall_rows)] = wall_rows
|
| 670 |
out = deepcopy(result, borrowed)
|
|
|
|
|
|
|
| 671 |
out["workspace"]["storageKey"] = storage_key
|
| 672 |
return out
|
| 673 |
|
|
@@ -840,6 +891,7 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke
|
|
| 840 |
rt = session.runtime
|
| 841 |
team_id, agent = _team_agent(session)
|
| 842 |
rows_src = _pool_for(rt, team_id, agent) if with_rows else []
|
|
|
|
| 843 |
# ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that
|
| 844 |
# frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so
|
| 845 |
# scoping here means a row this account may not see never enters ANY of them, rather than
|
|
@@ -863,7 +915,7 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke
|
|
| 863 |
# WRITE wall to this one's READ wall; lending it here alone would make a user-generated rule
|
| 864 |
# narrow what an account SEES while leaving what it may PATCH untouched.
|
| 865 |
if with_rows:
|
| 866 |
-
rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS,
|
| 867 |
st=rt)
|
| 868 |
pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
|
| 869 |
# ⭐ W38-T20 — THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because
|
|
@@ -895,7 +947,7 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke
|
|
| 895 |
_ov[_pid] = {**(_ov.get(_pid) or {}), **_cells}
|
| 896 |
ws["overlays"] = _ov
|
| 897 |
import odoo_relational
|
| 898 |
-
fields_base =
|
| 899 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 900 |
ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key,
|
| 901 |
fields_base=fields_base, st=rt)
|
|
@@ -1145,6 +1197,7 @@ def _payload(session: Session, related=None):
|
|
| 1145 |
from routes_tables import scope_linked_metrics, linked_metric_limits
|
| 1146 |
rows, g['fields'], metric_refusals = scope_linked_metrics(
|
| 1147 |
session, MODULE, rows, g['fields'], g['measures'], related)
|
|
|
|
| 1148 |
if related:
|
| 1149 |
g["pids"] = frozenset(row["pid"] for row in rows)
|
| 1150 |
# ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door
|
|
@@ -1213,7 +1266,7 @@ def _hidden_for(session: Session, defs=None):
|
|
| 1213 |
if defs is None:
|
| 1214 |
defs = shared_fields(st=session.runtime)
|
| 1215 |
return perm_scope.hidden_keys(
|
| 1216 |
-
session.user, MODULE, _merge_shared_fields(
|
| 1217 |
st=session.runtime)
|
| 1218 |
|
| 1219 |
def allowed_pids(session: Session):
|
|
@@ -1239,8 +1292,8 @@ def allowed_pids(session: Session):
|
|
| 1239 |
import core.perm_scope as perm_scope
|
| 1240 |
import aios_grid
|
| 1241 |
|
| 1242 |
-
rows = perm_scope.apply_row_scope(_pool_rows(session), session.user, MODULE,
|
| 1243 |
-
aios_grid.FIELDS, st=session.runtime)
|
| 1244 |
return frozenset(r["pid"] for r in rows if r.get("pid") is not None)
|
| 1245 |
|
| 1246 |
|
|
@@ -1276,6 +1329,7 @@ def assigned_view_rows(view_id: str, session: Session = Depends(module_gate(MODU
|
|
| 1276 |
rows = aios_grid.rows_from_pool(
|
| 1277 |
[row for row in g["rows_src"] if row.get("pid") in pids],
|
| 1278 |
g["fields"], g["ws"].get("overlays"), derived=g["derived"])
|
|
|
|
| 1279 |
return {"fields": g["fields"], "rows": rows, "viewId": view_id,
|
| 1280 |
"locked": True, "today": g["today"]}
|
| 1281 |
|
|
@@ -1388,7 +1442,7 @@ def migrate_routing_app_routes(runtime, tenant):
|
|
| 1388 |
"""Run R20 explicitly during integration, using the tenant's complete registry snapshot."""
|
| 1389 |
import application_registry as registry
|
| 1390 |
from core import route_system
|
| 1391 |
-
doc = registry.
|
| 1392 |
if not isinstance(doc, dict) or not isinstance(doc.get("applications", {}), dict):
|
| 1393 |
raise ValueError("the Application registry is unreadable")
|
| 1394 |
apps = [app for app in doc.get("applications", {}).values()
|
|
@@ -1420,7 +1474,7 @@ def _route_application(session, application, *, write=False):
|
|
| 1420 |
import application_registry as registry
|
| 1421 |
from core import route_system
|
| 1422 |
try:
|
| 1423 |
-
doc = registry.
|
| 1424 |
if not isinstance(doc, dict) or not isinstance(doc.get("applications", {}), dict):
|
| 1425 |
raise ValueError("invalid Application registry")
|
| 1426 |
except Exception:
|
|
@@ -1480,7 +1534,7 @@ def _home_route_application(session):
|
|
| 1480 |
if not me:
|
| 1481 |
return None
|
| 1482 |
try:
|
| 1483 |
-
doc = registry.
|
| 1484 |
apps = doc.get("applications", {}) if isinstance(doc, dict) else None
|
| 1485 |
if not isinstance(apps, dict):
|
| 1486 |
return None
|
|
@@ -2301,8 +2355,10 @@ def remove_zone(zone_id: str, session: Session = Depends(module_gate(MODULE)), a
|
|
| 2301 |
|
| 2302 |
|
| 2303 |
@router.patch("/customers/{pid}")
|
|
|
|
|
|
|
| 2304 |
def patch_customer(pid: int, body: dict = Body(default=None),
|
| 2305 |
-
session: Session = Depends(
|
| 2306 |
"""Write the EDITABLE overlay stratum only — Odoo stays read-only, forever.
|
| 2307 |
|
| 2308 |
Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing
|
|
@@ -2311,9 +2367,10 @@ def patch_customer(pid: int, body: dict = Body(default=None),
|
|
| 2311 |
them to keep in step. The response reports what was ACCEPTED, which is not always what was
|
| 2312 |
asked for.
|
| 2313 |
"""
|
| 2314 |
-
from core import grid_events
|
| 2315 |
|
| 2316 |
updates = dict(body or {})
|
|
|
|
| 2317 |
if not updates:
|
| 2318 |
raise err(400, "empty_patch", "no fields to update")
|
| 2319 |
from core import route_system
|
|
@@ -2325,18 +2382,23 @@ def patch_customer(pid: int, body: dict = Body(default=None),
|
|
| 2325 |
if retired.intersection(updates):
|
| 2326 |
raise err(403, "legacy_route_hidden",
|
| 2327 |
"this legacy Destination field was migrated; edit the named Route instead")
|
| 2328 |
-
|
|
|
|
| 2329 |
if pid not in pool:
|
| 2330 |
# 403, not 404: the pid may well exist — it is simply not in this session's book, and
|
| 2331 |
# saying "no such customer" would confirm the opposite to anyone who guessed right.
|
| 2332 |
raise err(403, "out_of_scope", "that customer is not in your book")
|
| 2333 |
-
payload = _payload(session)
|
| 2334 |
ctx = grid_events.EventCtx(
|
| 2335 |
uname=session.uname, allowed_pids=pool, fields=payload["fields"],
|
| 2336 |
admin=session.admin, fallback_ws=None, seen_ids={},
|
| 2337 |
-
hidden_keys=_hidden_for(session), table=_customer_table(session),
|
| 2338 |
st=session.runtime)
|
| 2339 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2340 |
# ⭐⭐ OWNER ITEM 7 (2026-08-23) — A ROUTE RANK IS EDITED HERE, NOT THROUGH `overlay_patch`.
|
| 2341 |
#
|
| 2342 |
# ⚠ AND **NOT** BECAUSE THE ORDINARY PATH WRITES THE WRONG STRATUM. It does not:
|
|
@@ -2359,7 +2421,8 @@ def patch_customer(pid: int, body: dict = Body(default=None),
|
|
| 2359 |
if updates:
|
| 2360 |
try:
|
| 2361 |
grid_events.handle_one(
|
| 2362 |
-
{"id": f"patch:{pid}:{time.time_ns()}",
|
|
|
|
| 2363 |
"pid": pid, "updates": updates}, ctx)
|
| 2364 |
except grid_events.StoreUnavailable:
|
| 2365 |
raise err(503, "store_unavailable",
|
|
|
|
| 29 |
import threading
|
| 30 |
import time
|
| 31 |
|
| 32 |
+
from fastapi import APIRouter, Body, Depends, Response
|
| 33 |
|
| 34 |
import scope_cache
|
| 35 |
+
from deps import Session, err, module_gate, require_session
|
| 36 |
+
from routes_grid import row_patch_receipt
|
| 37 |
+
from routes_tables import _row_patch_identity, _note_delta, _patch_cell_notes, _project_cell_notes, _builtin_note_context
|
| 38 |
|
| 39 |
router = APIRouter(prefix="/api/v1")
|
| 40 |
|
|
|
|
| 42 |
#: 403 rather than an empty table that looks like "you have no customers".
|
| 43 |
MODULE = "customer_data"
|
| 44 |
|
| 45 |
+
|
| 46 |
+
def _customer_fields(fields):
|
| 47 |
+
"""Expose the mirror's Store label without adding an upstream Odoo field."""
|
| 48 |
+
from routes_odoo_tables import mirror_store_field
|
| 49 |
+
declared = {field['key']: field for field in fields}
|
| 50 |
+
declared['Store'] = mirror_store_field()
|
| 51 |
+
declared['odoo_id'] = {'key': 'odoo_id', 'label': 'Odoo ID', 'type': 'int', 'source': 'odoo',
|
| 52 |
+
'default': False, 'readOnly': True, 'mirrorOnly': True}
|
| 53 |
+
return list(declared.values())
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _customer_store_rows(runtime, rows):
|
| 57 |
+
"""Project Store onto admitted customer identities with bounded mirror queries."""
|
| 58 |
+
if not rows:
|
| 59 |
+
return rows
|
| 60 |
+
from harness import datastore
|
| 61 |
+
cur = getattr(runtime, '_canonical_source_cursor', None)
|
| 62 |
+
admitted = callable(getattr(runtime, 'assert_datastore_matches', None))
|
| 63 |
+
if cur is None and not admitted:
|
| 64 |
+
return rows
|
| 65 |
+
from routes_odoo_tables import _table_cursor
|
| 66 |
+
cursor = cur.cursor() if cur is not None else _table_cursor(runtime)
|
| 67 |
+
try:
|
| 68 |
+
columns = datastore.columns_of('res_partner', cur=cursor)
|
| 69 |
+
if not columns and admitted and cur is not None:
|
| 70 |
+
cursor.close()
|
| 71 |
+
cursor = _table_cursor(runtime)
|
| 72 |
+
columns = datastore.columns_of('res_partner', cur=cursor)
|
| 73 |
+
if 'store' not in columns:
|
| 74 |
+
return rows
|
| 75 |
+
identities = list(dict.fromkeys(int(row['pid']) for row in rows))
|
| 76 |
+
stores = {}
|
| 77 |
+
source_ids = {}
|
| 78 |
+
for offset in range(0, len(identities), 256):
|
| 79 |
+
batch = identities[offset:offset + 256]
|
| 80 |
+
source_id = '_source_id' if '_source_id' in columns else 'id'
|
| 81 |
+
cursor.execute('SELECT id, Store, ' + source_id + ' FROM res_partner WHERE id IN (' + ','.join('?' for _ in batch) + ')', batch)
|
| 82 |
+
for pid, label, upstream in cursor.fetchmany(256):
|
| 83 |
+
stores[int(pid)] = label or ''
|
| 84 |
+
source_ids[int(pid)] = upstream if upstream is not None else ''
|
| 85 |
+
return [dict(row, Store=stores.get(int(row['pid']), ''), odoo_id=source_ids.get(int(row['pid']), '')) for row in rows]
|
| 86 |
+
finally:
|
| 87 |
+
cursor.close()
|
| 88 |
+
|
| 89 |
_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
|
| 90 |
|
| 91 |
# Waiters share one cold build (about 4 min on 2 vCPU); a private rebuild per waiter multiplies it.
|
|
|
|
| 710 |
def own_copy(result):
|
| 711 |
# Both aliases lend immutable source rows. Copy the request-shaped metadata together.
|
| 712 |
borrowed = {id(result['rows_src']): result['rows_src']}
|
| 713 |
+
note_runtime = result.get('_note_runtime')
|
| 714 |
+
if note_runtime is not None:
|
| 715 |
+
borrowed[id(note_runtime)] = note_runtime
|
| 716 |
wall_rows = (result.get('_permission_scope') or {}).get('rows')
|
| 717 |
if wall_rows is not None:
|
| 718 |
borrowed[id(wall_rows)] = wall_rows
|
| 719 |
out = deepcopy(result, borrowed)
|
| 720 |
+
if note_runtime is not None:
|
| 721 |
+
out['_note_runtime'] = session.runtime
|
| 722 |
out["workspace"]["storageKey"] = storage_key
|
| 723 |
return out
|
| 724 |
|
|
|
|
| 891 |
rt = session.runtime
|
| 892 |
team_id, agent = _team_agent(session)
|
| 893 |
rows_src = _pool_for(rt, team_id, agent) if with_rows else []
|
| 894 |
+
rows_src = _customer_store_rows(rt, rows_src)
|
| 895 |
# ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that
|
| 896 |
# frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so
|
| 897 |
# scoping here means a row this account may not see never enters ANY of them, rather than
|
|
|
|
| 915 |
# WRITE wall to this one's READ wall; lending it here alone would make a user-generated rule
|
| 916 |
# narrow what an account SEES while leaving what it may PATCH untouched.
|
| 917 |
if with_rows:
|
| 918 |
+
rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, _customer_fields(aios_grid.FIELDS),
|
| 919 |
st=rt)
|
| 920 |
pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
|
| 921 |
# ⭐ W38-T20 — THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because
|
|
|
|
| 947 |
_ov[_pid] = {**(_ov.get(_pid) or {}), **_cells}
|
| 948 |
ws["overlays"] = _ov
|
| 949 |
import odoo_relational
|
| 950 |
+
fields_base = _customer_fields(aios_grid.FIELDS) + odoo_relational.module_link_fields(MODULE)
|
| 951 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 952 |
ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key,
|
| 953 |
fields_base=fields_base, st=rt)
|
|
|
|
| 1197 |
from routes_tables import scope_linked_metrics, linked_metric_limits
|
| 1198 |
rows, g['fields'], metric_refusals = scope_linked_metrics(
|
| 1199 |
session, MODULE, rows, g['fields'], g['measures'], related)
|
| 1200 |
+
rows = _project_cell_notes(MODULE, rows, g['fields'], session.runtime)
|
| 1201 |
if related:
|
| 1202 |
g["pids"] = frozenset(row["pid"] for row in rows)
|
| 1203 |
# ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door
|
|
|
|
| 1266 |
if defs is None:
|
| 1267 |
defs = shared_fields(st=session.runtime)
|
| 1268 |
return perm_scope.hidden_keys(
|
| 1269 |
+
session.user, MODULE, _merge_shared_fields(_customer_fields(aios_grid.FIELDS), defs),
|
| 1270 |
st=session.runtime)
|
| 1271 |
|
| 1272 |
def allowed_pids(session: Session):
|
|
|
|
| 1292 |
import core.perm_scope as perm_scope
|
| 1293 |
import aios_grid
|
| 1294 |
|
| 1295 |
+
rows = perm_scope.apply_row_scope(_customer_store_rows(session.runtime, _pool_rows(session)), session.user, MODULE,
|
| 1296 |
+
_customer_fields(aios_grid.FIELDS), st=session.runtime)
|
| 1297 |
return frozenset(r["pid"] for r in rows if r.get("pid") is not None)
|
| 1298 |
|
| 1299 |
|
|
|
|
| 1329 |
rows = aios_grid.rows_from_pool(
|
| 1330 |
[row for row in g["rows_src"] if row.get("pid") in pids],
|
| 1331 |
g["fields"], g["ws"].get("overlays"), derived=g["derived"])
|
| 1332 |
+
rows = _project_cell_notes(MODULE, rows, g['fields'], session.runtime)
|
| 1333 |
return {"fields": g["fields"], "rows": rows, "viewId": view_id,
|
| 1334 |
"locked": True, "today": g["today"]}
|
| 1335 |
|
|
|
|
| 1442 |
"""Run R20 explicitly during integration, using the tenant's complete registry snapshot."""
|
| 1443 |
import application_registry as registry
|
| 1444 |
from core import route_system
|
| 1445 |
+
doc = registry.load(runtime, strict=True)
|
| 1446 |
if not isinstance(doc, dict) or not isinstance(doc.get("applications", {}), dict):
|
| 1447 |
raise ValueError("the Application registry is unreadable")
|
| 1448 |
apps = [app for app in doc.get("applications", {}).values()
|
|
|
|
| 1474 |
import application_registry as registry
|
| 1475 |
from core import route_system
|
| 1476 |
try:
|
| 1477 |
+
doc = registry.load(session.runtime, strict=True)
|
| 1478 |
if not isinstance(doc, dict) or not isinstance(doc.get("applications", {}), dict):
|
| 1479 |
raise ValueError("invalid Application registry")
|
| 1480 |
except Exception:
|
|
|
|
| 1534 |
if not me:
|
| 1535 |
return None
|
| 1536 |
try:
|
| 1537 |
+
doc = registry.load(session.runtime, strict=True)
|
| 1538 |
apps = doc.get("applications", {}) if isinstance(doc, dict) else None
|
| 1539 |
if not isinstance(apps, dict):
|
| 1540 |
return None
|
|
|
|
| 2355 |
|
| 2356 |
|
| 2357 |
@router.patch("/customers/{pid}")
|
| 2358 |
+
@_row_patch_identity(module=MODULE)
|
| 2359 |
+
@row_patch_receipt('customer')
|
| 2360 |
def patch_customer(pid: int, body: dict = Body(default=None),
|
| 2361 |
+
session: Session = Depends(require_session), response: Response = None):
|
| 2362 |
"""Write the EDITABLE overlay stratum only — Odoo stays read-only, forever.
|
| 2363 |
|
| 2364 |
Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing
|
|
|
|
| 2367 |
them to keep in step. The response reports what was ACCEPTED, which is not always what was
|
| 2368 |
asked for.
|
| 2369 |
"""
|
| 2370 |
+
from core import grid_events, edit_receipts
|
| 2371 |
|
| 2372 |
updates = dict(body or {})
|
| 2373 |
+
notes = _note_delta(updates)
|
| 2374 |
if not updates:
|
| 2375 |
raise err(400, "empty_patch", "no fields to update")
|
| 2376 |
from core import route_system
|
|
|
|
| 2382 |
if retired.intersection(updates):
|
| 2383 |
raise err(403, "legacy_route_hidden",
|
| 2384 |
"this legacy Destination field was migrated; edit the named Route instead")
|
| 2385 |
+
note_page = _builtin_note_context(session, MODULE, pid) if notes is not None else None
|
| 2386 |
+
pool = note_page['pids'] if note_page is not None else allowed_pids(session)
|
| 2387 |
if pid not in pool:
|
| 2388 |
# 403, not 404: the pid may well exist — it is simply not in this session's book, and
|
| 2389 |
# saying "no such customer" would confirm the opposite to anyone who guessed right.
|
| 2390 |
raise err(403, "out_of_scope", "that customer is not in your book")
|
| 2391 |
+
payload = note_page if note_page is not None else _payload(session)
|
| 2392 |
ctx = grid_events.EventCtx(
|
| 2393 |
uname=session.uname, allowed_pids=pool, fields=payload["fields"],
|
| 2394 |
admin=session.admin, fallback_ws=None, seen_ids={},
|
| 2395 |
+
hidden_keys=note_page['hidden'] if note_page is not None else _hidden_for(session), table=_customer_table(session),
|
| 2396 |
st=session.runtime)
|
| 2397 |
|
| 2398 |
+
if notes is not None:
|
| 2399 |
+
return _patch_cell_notes(MODULE, pid, notes, ctx,
|
| 2400 |
+
source_row_exists=lambda key, rid, admitted: key == MODULE and int(rid) in pool)
|
| 2401 |
+
|
| 2402 |
# ⭐⭐ OWNER ITEM 7 (2026-08-23) — A ROUTE RANK IS EDITED HERE, NOT THROUGH `overlay_patch`.
|
| 2403 |
#
|
| 2404 |
# ⚠ AND **NOT** BECAUSE THE ORDINARY PATH WRITES THE WRONG STRATUM. It does not:
|
|
|
|
| 2421 |
if updates:
|
| 2422 |
try:
|
| 2423 |
grid_events.handle_one(
|
| 2424 |
+
{"id": edit_receipts.current_op_id() or f"patch:{pid}:{time.time_ns()}",
|
| 2425 |
+
"type": "overlay_patch",
|
| 2426 |
"pid": pid, "updates": updates}, ctx)
|
| 2427 |
except grid_events.StoreUnavailable:
|
| 2428 |
raise err(503, "store_unavailable",
|
api/routes_external.py
CHANGED
|
@@ -27,7 +27,8 @@ import aios_session
|
|
| 27 |
from deps import Session, err, require_session
|
| 28 |
from core import audit, data_binding, external_access, external_oauth, external_tools, perm_scope, sheets
|
| 29 |
from harness import runtime
|
| 30 |
-
from tool_groups.data import (
|
|
|
|
| 31 |
query_syntax_section, with_query_syntax)
|
| 32 |
from tool_groups.sheet import WORKBOOK_MISSING_FIX
|
| 33 |
|
|
@@ -38,6 +39,7 @@ router = APIRouter(prefix=EXTERNAL_PREFIX, tags=["external"])
|
|
| 38 |
_CREDENTIAL_PREFIXES = ("lpb1.", "lpc1.", "lpo1.", "lpr1.")
|
| 39 |
_QUERY_DEFAULT_LIMIT = 100
|
| 40 |
_QUERY_MAX_LIMIT = 500
|
|
|
|
| 41 |
_QUERY_MAX_SORTS = 8
|
| 42 |
_QUERY_MAX_AGGREGATES = 16
|
| 43 |
_QUERY_MAX_CURSOR = 768
|
|
@@ -45,6 +47,19 @@ _QUERY_MAX_FILTER_DEPTH = 3
|
|
| 45 |
_QUERY_MAX_FILTER_NODES = 100
|
| 46 |
_QUERY_MAX_FILTER_SIBLINGS = 50
|
| 47 |
_QUERY_KEYS = frozenset({"fields", "filters", "sorts", "limit", "cursor", "aggregates", "groupBy"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
_MCP_FIXES = {
|
| 49 |
"query_filter_limit": "split it into separate queries",
|
| 50 |
"query_cursor_stale": "run the same query again without a cursor",
|
|
@@ -197,6 +212,13 @@ class _PreviewRuntime:
|
|
| 197 |
def get_many(self, names):
|
| 198 |
return {str(name): self.get(name) for name in names or ()}
|
| 199 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
def get_entry(self, name, entry):
|
| 201 |
return copy.deepcopy((self.get(name) or {}).get(entry))
|
| 202 |
|
|
@@ -2663,6 +2685,26 @@ def _external_image_urls(session, database, rows, records):
|
|
| 2663 |
return records
|
| 2664 |
|
| 2665 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2666 |
def _query_database_page(session: Session, database: str, payload: dict):
|
| 2667 |
if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS:
|
| 2668 |
raise err(400, "query_arguments_invalid",
|
|
@@ -2672,15 +2714,16 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 2672 |
cursor = (_read_query_cursor(session, payload["cursor"])
|
| 2673 |
if payload.get("cursor") is not None else None)
|
| 2674 |
limit = payload.get("limit", _QUERY_DEFAULT_LIMIT)
|
| 2675 |
-
if type(limit) is not int or
|
| 2676 |
requested = limit
|
| 2677 |
try:
|
| 2678 |
json.dumps(requested, allow_nan=False)
|
| 2679 |
except (TypeError, ValueError, OverflowError, RecursionError):
|
| 2680 |
requested = None
|
| 2681 |
raise err(400, "query_limit_invalid",
|
| 2682 |
-
|
| 2683 |
{"limit": _QUERY_MAX_LIMIT, "requested": requested})
|
|
|
|
| 2684 |
raw_sorts = payload.get("sorts") or []
|
| 2685 |
if not isinstance(raw_sorts, list) or len(raw_sorts) > _QUERY_MAX_SORTS:
|
| 2686 |
raise err(400, "query_sorts_invalid",
|
|
@@ -2691,8 +2734,15 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 2691 |
# database, field and row walls on every call before this adapter sorts, caps, or aggregates.
|
| 2692 |
# ⭐ S67-T34: a MIRROR-SERVED database is first resolved schema-only (every wall, no row), so
|
| 2693 |
# its aggregates run in SQL over every permitted row and its rows are read only as needed.
|
|
|
|
|
|
|
| 2694 |
scoped = _external_read_scope(
|
| 2695 |
-
session, database, fields=None, filters=
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2696 |
mirror = scoped.get("source_kind") == "mirror_user_table"
|
| 2697 |
visible_fields = list(scoped.get("fields") or [])
|
| 2698 |
visible_keys = {
|
|
@@ -2865,7 +2915,11 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 2865 |
|
| 2866 |
if not mirror:
|
| 2867 |
scoped = _external_read_scope(
|
| 2868 |
-
session, database, fields=None, filters=
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2869 |
rows = list(scoped.get("records") or [])
|
| 2870 |
inspected_keys = (set(selected_keys)
|
| 2871 |
| {rule["field"] for rule in sorts}
|
|
@@ -3469,6 +3523,54 @@ def _database_value_plan(session, database, record_id, values, fields, *, apply_
|
|
| 3469 |
return planned
|
| 3470 |
|
| 3471 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3472 |
def _plan_database_write(session, args):
|
| 3473 |
import routes_tables
|
| 3474 |
from core import user_tables
|
|
@@ -3476,10 +3578,12 @@ def _plan_database_write(session, args):
|
|
| 3476 |
action = args.get("action")
|
| 3477 |
database = str(args.get("database") or "").strip()
|
| 3478 |
if action not in {"record_create", "record_update", "record_delete",
|
| 3479 |
-
"field_create", "field_update", "field_delete"} or not database:
|
| 3480 |
-
raise err(400, "tool_arguments_invalid", "Name a database and an action that changes records or
|
| 3481 |
_assert_database_scope(session, database)
|
| 3482 |
_assert_write_scope(session, "database_object")
|
|
|
|
|
|
|
| 3483 |
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 3484 |
bucket_before = session.runtime.get(user_tables.STORE_KEY)
|
| 3485 |
if not isinstance(bucket_before, dict):
|
|
@@ -3496,7 +3600,13 @@ def _plan_database_write(session, args):
|
|
| 3496 |
bucket_after = copy.deepcopy(bucket_before)
|
| 3497 |
|
| 3498 |
if action.startswith("record_"):
|
| 3499 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3500 |
g = routes_tables.ut_write_ctx(session, database)
|
| 3501 |
rows = table.get("rows") or {}
|
| 3502 |
if action == "record_create":
|
|
@@ -3649,6 +3759,19 @@ def _execute_database_object(session, args, *, plan=None):
|
|
| 3649 |
if action == "query":
|
| 3650 |
return _query_database(session, database, args.get("query") or {})
|
| 3651 |
plan = plan or _plan_database_write(session, args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3652 |
expected_rev = plan.get("revision", {}).get("rev")
|
| 3653 |
expected_digest = plan.get("bucket_before_digest")
|
| 3654 |
replacement = copy.deepcopy(plan.get("bucket_after"))
|
|
@@ -4206,12 +4329,11 @@ def _build_about(value):
|
|
| 4206 |
else "Send about as an object with title, description and tags." if not isinstance(value, dict)
|
| 4207 |
else "Add " + ", ".join(missing) + " to about." if missing
|
| 4208 |
else "Fix " + ", ".join(invalid) + ".")
|
| 4209 |
-
|
| 4210 |
-
|
| 4211 |
-
|
| 4212 |
-
|
| 4213 |
-
|
| 4214 |
-
" without em or en dashes.")
|
| 4215 |
raise err(400, "build_about_required" if missing else "build_about_invalid", message, details) from None
|
| 4216 |
return cleaned
|
| 4217 |
|
|
@@ -4561,7 +4683,10 @@ def _operation_access(session, item, *, confirming=False):
|
|
| 4561 |
_external_read_scope(session, database, fields=[], mode="schema")
|
| 4562 |
else:
|
| 4563 |
_external_read_scope(session, database, fields=[])
|
| 4564 |
-
if name == "database_object":
|
|
|
|
|
|
|
|
|
|
| 4565 |
import routes_tables
|
| 4566 |
g = routes_tables.ut_write_ctx(session, args["database"])
|
| 4567 |
action = args["action"]
|
|
@@ -4768,8 +4893,9 @@ def _committed_build_evidence(session, writer, arguments, *, build_kind=None):
|
|
| 4768 |
for identity in before.keys() | after.keys():
|
| 4769 |
if before.get(identity) != after.get(identity):
|
| 4770 |
add("app", (after.get(identity) or before.get(identity) or {}).get("appId"))
|
| 4771 |
-
elif key in (application_registry._KEY, automation_engine.STORE_KEY)
|
| 4772 |
-
|
|
|
|
| 4773 |
if kind == "app":
|
| 4774 |
before, after = before.get("applications") or {}, after.get("applications") or {}
|
| 4775 |
for identity in before.keys() | after.keys():
|
|
@@ -4947,9 +5073,10 @@ def _confirm_change(session, args):
|
|
| 4947 |
before_revision = copy.deepcopy(recomputed.get("current_revision") or {})
|
| 4948 |
if tool == "database_object":
|
| 4949 |
from core import user_tables
|
|
|
|
|
|
|
| 4950 |
after_revision = _bucket_revision(
|
| 4951 |
-
fresh_session.runtime,
|
| 4952 |
-
fresh_session.runtime.get(user_tables.STORE_KEY))
|
| 4953 |
produced_rev = before_revision["rev"] + 1
|
| 4954 |
if after_revision.get("rev") != produced_rev:
|
| 4955 |
after_revision = {"rev": produced_rev}
|
|
@@ -5004,7 +5131,7 @@ _MCP_CODE_GROUPS = (
|
|
| 5004 |
("The arguments do not match this action. Read its guide and correct them.", """
|
| 5005 |
tool_arguments_invalid query_arguments_invalid query_aggregates_invalid query_aggregate_type
|
| 5006 |
assistant_bad_filter query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid
|
| 5007 |
-
bad_application bad_entries bad_value bad_values bad_request bad_recurrence invalid_field_contract
|
| 5008 |
external_edit_empty external_edit_invalid oauth_edit_empty oauth_edit_invalid
|
| 5009 |
"""),
|
| 5010 |
("You do not have permission to do this with this connection.", """
|
|
@@ -5144,6 +5271,7 @@ MCP_RESULT_CODES.update({
|
|
| 5144 |
"request_method_unknown": "The requested method is not available.",
|
| 5145 |
"request_rate_limited": "This AI has reached its hourly request cap. Try again later.",
|
| 5146 |
})
|
|
|
|
| 5147 |
_MCP_INTERNAL_PROSE = re.compile(
|
| 5148 |
r"\b(?:DuckDB|Postgres(?:ql)?|SQLite|Polars|Pandas|SQLAlchemy|PyArrow|HyperFormula)\b|"
|
| 5149 |
r"\b(?:api|web|platform)_[a-z_]+\b|\b(?:verify_\w+|\w+Error)\b|"
|
|
@@ -5444,7 +5572,7 @@ MCP_INSTRUCTIONS = "\n".join([
|
|
| 5444 |
"A build that breaks a required rule is refused with its fix.",
|
| 5445 |
"2. Discover before you query: use table.list_databases, then table.describe_database for the database you "
|
| 5446 |
"need. Use only field keys that describe_database returns. Never guess a field name. "
|
| 5447 |
-
"State the age of data older than one day before giving a number.",
|
| 5448 |
"3. Read with table.query_database. Filter and sort on the server, ask only for the fields you need, "
|
| 5449 |
"and follow the cursor for more pages instead of reading everything. Report totals from the "
|
| 5450 |
"counts the tool returns, not from the rows you happened to read. "
|
|
@@ -5592,28 +5720,41 @@ def _mcp_database_actions(session, rows):
|
|
| 5592 |
"""Resolve database grants through read-only permission predicates."""
|
| 5593 |
from core import grid_events, user_tables
|
| 5594 |
import routes_grid
|
|
|
|
| 5595 |
import routes_tables
|
| 5596 |
session = Session(session.tenant, session.user, session.claims, user_tables.lend_defs(session.runtime))
|
| 5597 |
allowed = {action: False for action in external_tools._DATABASE_ACTIONS if action not in {"list", "describe", "query"}}
|
| 5598 |
for row in rows:
|
| 5599 |
database = row["database"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5600 |
try:
|
| 5601 |
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 5602 |
if not user_tables.is_user_table(database, st=session.runtime):
|
| 5603 |
continue
|
| 5604 |
fields = row.get("fields", [])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5605 |
try:
|
| 5606 |
-
|
| 5607 |
-
|
| 5608 |
-
|
| 5609 |
-
not user_tables.is_computed_cell(field) and field.get("type") != "automation"
|
| 5610 |
-
and grid_events._may_edit_field_value(ctx, field["key"], field)
|
| 5611 |
-
for field in fields)
|
| 5612 |
-
allowed["record_create"] |= editable
|
| 5613 |
-
allowed["record_update"] |= editable
|
| 5614 |
allowed["record_delete"] = True
|
| 5615 |
except HTTPException:
|
| 5616 |
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5617 |
try:
|
| 5618 |
routes_tables._field_or_refuse(session, database)
|
| 5619 |
allowed["field_create"] = True
|
|
@@ -5754,6 +5895,7 @@ def _mcp_capabilities(session, groups, rows):
|
|
| 5754 |
"record_update": "edit record fields", "record_delete": "delete records",
|
| 5755 |
"field_create": "add fields as the database owner or an Administrator",
|
| 5756 |
"field_update": "edit a visible field", "field_delete": "delete a visible field",
|
|
|
|
| 5757 |
}[operation] + " in a writable database")
|
| 5758 |
granted = not required
|
| 5759 |
row.update(state=("not supported here" if not supported else
|
|
@@ -5993,8 +6135,11 @@ def _mcp_tool_specs() -> list[dict]:
|
|
| 5993 |
spec["inputSchema"]["properties"]["about"] = copy.deepcopy(external_tools.ABOUT_SCHEMA)
|
| 5994 |
spec["inputSchema"]["properties"]["action"]["enum"] = list(groups[spec["name"]]["actions"])
|
| 5995 |
if spec["name"] == "sheet":
|
| 5996 |
-
from tool_groups.sheet import WORKBOOK_RESOURCE_URI
|
|
|
|
| 5997 |
spec["_meta"] = {"ui": {"resourceUri": WORKBOOK_RESOURCE_URI}}
|
|
|
|
|
|
|
| 5998 |
spec["description"] = _mcp_plain(spec["description"], "Read this group's guide for actions.")
|
| 5999 |
spec["inputSchema"] = _mcp_schema_copy(spec["inputSchema"])
|
| 6000 |
if "title" in spec.get("annotations", {}):
|
|
@@ -6233,22 +6378,32 @@ def _mcp_hoist_call_metadata(name, arguments):
|
|
| 6233 |
def _mcp_intent_fix(name):
|
| 6234 |
"""Name the intent field and where this tool takes it."""
|
| 6235 |
import tool_groups
|
|
|
|
| 6236 |
if name == "confirm_change":
|
| 6237 |
return ("Send intent beside confirmation_token: 1 to 500 characters on why you are confirming this change. "
|
| 6238 |
-
"A client that sends only action and arguments may put both inside arguments.")
|
| 6239 |
if tool_groups.canonical_call(name, {})[0] in tool_groups.groups():
|
| 6240 |
-
|
| 6241 |
-
|
| 6242 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6243 |
|
| 6244 |
|
| 6245 |
def _mcp_about_fix(name):
|
| 6246 |
"""Name where a group call takes about, or nothing for a flat tool."""
|
| 6247 |
import tool_groups
|
|
|
|
| 6248 |
if isinstance(name, str) and tool_groups.canonical_call(name, {})[0] in tool_groups.groups():
|
| 6249 |
-
|
| 6250 |
-
|
| 6251 |
-
|
|
|
|
|
|
|
|
|
|
| 6252 |
|
| 6253 |
|
| 6254 |
def _mcp_requires_intent(name, arguments):
|
|
@@ -6286,12 +6441,13 @@ def _dispatch_mcp(req, session: Session, base_url: str = ""):
|
|
| 6286 |
return reply
|
| 6287 |
|
| 6288 |
|
| 6289 |
-
#:
|
| 6290 |
-
_MCP_NEXT_CALL = {"workbook_missing": WORKBOOK_MISSING_FIX, "assistant_unknown_database": UNKNOWN_DATABASE_FIX
|
|
|
|
| 6291 |
|
| 6292 |
|
| 6293 |
def _mcp_next_call(code, message):
|
| 6294 |
-
"""Append the
|
| 6295 |
text = str(message or "").rstrip()
|
| 6296 |
fix = _MCP_NEXT_CALL.get(code, "")
|
| 6297 |
if code == "assistant_unknown_database":
|
|
@@ -6507,7 +6663,7 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 6507 |
if info.get("code") == "workbook_name_ambiguous" and isinstance(details, dict):
|
| 6508 |
retry["candidates"] = details.get("candidates", [])
|
| 6509 |
message = info.get("message", "The app refused this operation.")
|
| 6510 |
-
if info.get("code")
|
| 6511 |
message += _mcp_about_fix(called)
|
| 6512 |
if info.get("code") in _MCP_NEXT_CALL:
|
| 6513 |
message = _mcp_next_call(info["code"], message)
|
|
|
|
| 27 |
from deps import Session, err, require_session
|
| 28 |
from core import audit, data_binding, external_access, external_oauth, external_tools, perm_scope, sheets
|
| 29 |
from harness import runtime
|
| 30 |
+
from tool_groups.data import (DISCOVERY_HELP, QUERY_PAGING_HELP, READ_REFUSAL_FIXES,
|
| 31 |
+
QUERY_SYNTAX_CODES, UNKNOWN_DATABASE_FIX, bad_filter_message,
|
| 32 |
query_syntax_section, with_query_syntax)
|
| 33 |
from tool_groups.sheet import WORKBOOK_MISSING_FIX
|
| 34 |
|
|
|
|
| 39 |
_CREDENTIAL_PREFIXES = ("lpb1.", "lpc1.", "lpo1.", "lpr1.")
|
| 40 |
_QUERY_DEFAULT_LIMIT = 100
|
| 41 |
_QUERY_MAX_LIMIT = 500
|
| 42 |
+
_QUERY_PAGING_HELP = QUERY_PAGING_HELP
|
| 43 |
_QUERY_MAX_SORTS = 8
|
| 44 |
_QUERY_MAX_AGGREGATES = 16
|
| 45 |
_QUERY_MAX_CURSOR = 768
|
|
|
|
| 47 |
_QUERY_MAX_FILTER_NODES = 100
|
| 48 |
_QUERY_MAX_FILTER_SIBLINGS = 50
|
| 49 |
_QUERY_KEYS = frozenset({"fields", "filters", "sorts", "limit", "cursor", "aggregates", "groupBy"})
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _register_query_paging_contract():
|
| 53 |
+
"""Keep the legacy and grouped query schemas aligned with the API's bounded page reader."""
|
| 54 |
+
from tool_groups.data import GROUP
|
| 55 |
+
for schema in (external_tools.TOOLS["query_database"]["inputSchema"],
|
| 56 |
+
GROUP["actions"]["query_database"]["schema"]):
|
| 57 |
+
schema["properties"]["limit"] = {
|
| 58 |
+
"type": "integer", "minimum": 1, "description": _QUERY_PAGING_HELP}
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
_register_query_paging_contract()
|
| 62 |
+
|
| 63 |
_MCP_FIXES = {
|
| 64 |
"query_filter_limit": "split it into separate queries",
|
| 65 |
"query_cursor_stale": "run the same query again without a cursor",
|
|
|
|
| 212 |
def get_many(self, names):
|
| 213 |
return {str(name): self.get(name) for name in names or ()}
|
| 214 |
|
| 215 |
+
def list_keys(self, prefix=""):
|
| 216 |
+
prefix = str(prefix)
|
| 217 |
+
listing = getattr(self._real, "list_keys", None)
|
| 218 |
+
keys = set(listing(prefix)) if callable(listing) else set()
|
| 219 |
+
keys.update(self._docs)
|
| 220 |
+
return sorted(key for key in keys if key.startswith(prefix))
|
| 221 |
+
|
| 222 |
def get_entry(self, name, entry):
|
| 223 |
return copy.deepcopy((self.get(name) or {}).get(entry))
|
| 224 |
|
|
|
|
| 2685 |
return records
|
| 2686 |
|
| 2687 |
|
| 2688 |
+
def _typed_selected_stop_identity(session, database, filters):
|
| 2689 |
+
"""Recognize one typed built-in identity equality without declaring a readable column."""
|
| 2690 |
+
if ((session.claims or {}).get("external_via") != "session"
|
| 2691 |
+
or database not in {"customer_data", "product_data"}
|
| 2692 |
+
or not isinstance(filters, dict) or set(filters) != {"conj", "nodes"}
|
| 2693 |
+
or filters.get("conj") != "and"):
|
| 2694 |
+
return None
|
| 2695 |
+
nodes = filters.get("nodes")
|
| 2696 |
+
if not isinstance(nodes, list) or len(nodes) != 1:
|
| 2697 |
+
return None
|
| 2698 |
+
leaf = nodes[0]
|
| 2699 |
+
if (not isinstance(leaf, dict) or set(leaf) - {"id", "colId", "op", "value"}
|
| 2700 |
+
or leaf.get("colId") != "pid" or leaf.get("op") != "eq"):
|
| 2701 |
+
return None
|
| 2702 |
+
identity = leaf.get("value")
|
| 2703 |
+
if type(identity) not in (int, str) or not str(identity).strip() or len(str(identity)) > 200:
|
| 2704 |
+
return None
|
| 2705 |
+
return identity
|
| 2706 |
+
|
| 2707 |
+
|
| 2708 |
def _query_database_page(session: Session, database: str, payload: dict):
|
| 2709 |
if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS:
|
| 2710 |
raise err(400, "query_arguments_invalid",
|
|
|
|
| 2714 |
cursor = (_read_query_cursor(session, payload["cursor"])
|
| 2715 |
if payload.get("cursor") is not None else None)
|
| 2716 |
limit = payload.get("limit", _QUERY_DEFAULT_LIMIT)
|
| 2717 |
+
if type(limit) is not int or limit < 1:
|
| 2718 |
requested = limit
|
| 2719 |
try:
|
| 2720 |
json.dumps(requested, allow_nan=False)
|
| 2721 |
except (TypeError, ValueError, OverflowError, RecursionError):
|
| 2722 |
requested = None
|
| 2723 |
raise err(400, "query_limit_invalid",
|
| 2724 |
+
_QUERY_PAGING_HELP,
|
| 2725 |
{"limit": _QUERY_MAX_LIMIT, "requested": requested})
|
| 2726 |
+
limit = min(limit, _QUERY_MAX_LIMIT)
|
| 2727 |
raw_sorts = payload.get("sorts") or []
|
| 2728 |
if not isinstance(raw_sorts, list) or len(raw_sorts) > _QUERY_MAX_SORTS:
|
| 2729 |
raise err(400, "query_sorts_invalid",
|
|
|
|
| 2734 |
# database, field and row walls on every call before this adapter sorts, caps, or aggregates.
|
| 2735 |
# ⭐ S67-T34: a MIRROR-SERVED database is first resolved schema-only (every wall, no row), so
|
| 2736 |
# its aggregates run in SQL over every permitted row and its rows are read only as needed.
|
| 2737 |
+
identity = _typed_selected_stop_identity(session, database, payload.get("filters"))
|
| 2738 |
+
scope_filters = None if identity is not None else payload.get("filters")
|
| 2739 |
scoped = _external_read_scope(
|
| 2740 |
+
session, database, fields=None, filters=scope_filters, mode="query_schema")
|
| 2741 |
+
if identity is not None and (
|
| 2742 |
+
scoped.get("source_kind") != "runtime_cache"
|
| 2743 |
+
or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters):
|
| 2744 |
+
raise err(409, "assistant_snapshot_unavailable",
|
| 2745 |
+
"Selected record reads are unavailable in this deployment. Retry after an update.")
|
| 2746 |
mirror = scoped.get("source_kind") == "mirror_user_table"
|
| 2747 |
visible_fields = list(scoped.get("fields") or [])
|
| 2748 |
visible_keys = {
|
|
|
|
| 2915 |
|
| 2916 |
if not mirror:
|
| 2917 |
scoped = _external_read_scope(
|
| 2918 |
+
session, database, fields=None, filters=scope_filters,
|
| 2919 |
+
**({"identity_pid": identity} if identity is not None else {}))
|
| 2920 |
+
if identity is not None:
|
| 2921 |
+
# Keep the identity predicate in the response and signed cursor's query digest.
|
| 2922 |
+
scoped["filters"] = copy.deepcopy(payload.get("filters"))
|
| 2923 |
rows = list(scoped.get("records") or [])
|
| 2924 |
inspected_keys = (set(selected_keys)
|
| 2925 |
| {rule["field"] for rule in sorts}
|
|
|
|
| 3523 |
return planned
|
| 3524 |
|
| 3525 |
|
| 3526 |
+
def _database_nav_body(args):
|
| 3527 |
+
"""Preserve omitted metadata fields when addressing the app's navigation writer."""
|
| 3528 |
+
return {"key": str(args.get("database") or "").strip(),
|
| 3529 |
+
**{key: copy.deepcopy(args[key]) for key in ("name", "icon") if key in args}}
|
| 3530 |
+
|
| 3531 |
+
|
| 3532 |
+
def _plan_database_rename(session, args):
|
| 3533 |
+
"""Run the app's metadata writer on a private copy and bind its navigation revision."""
|
| 3534 |
+
import routes_nav
|
| 3535 |
+
from core import user_tables
|
| 3536 |
+
|
| 3537 |
+
preview = _PreviewRuntime(session.runtime)
|
| 3538 |
+
# Metadata validation needs definitions, so do not load record rows into a fresh preview.
|
| 3539 |
+
source = session.runtime
|
| 3540 |
+
while isinstance(source, _PreviewRuntime) and user_tables.STORE_KEY not in source._docs:
|
| 3541 |
+
source = source._real
|
| 3542 |
+
definitions = user_tables.all_defs(st=source, schema_only=True)
|
| 3543 |
+
preview._initial[user_tables.STORE_KEY] = copy.deepcopy(definitions)
|
| 3544 |
+
preview._docs[user_tables.STORE_KEY] = copy.deepcopy(definitions)
|
| 3545 |
+
before = preview.get(routes_nav._NAV_META_KEY)
|
| 3546 |
+
revision = preview.revision(routes_nav._NAV_META_KEY)
|
| 3547 |
+
if type(revision.get("rev")) is not int:
|
| 3548 |
+
raise err(503, "confirmation_revision_unavailable",
|
| 3549 |
+
"The database metadata version could not be checked. Nothing was saved.")
|
| 3550 |
+
preview_session = Session(session.tenant, session.user, session.claims, preview)
|
| 3551 |
+
result = routes_nav.save_nav_meta(_database_nav_body(args), preview_session)
|
| 3552 |
+
diff, truncated = _diff_documents({routes_nav._NAV_META_KEY: {
|
| 3553 |
+
"before": before, "after": preview.get(routes_nav._NAV_META_KEY)}})
|
| 3554 |
+
database = result["key"]
|
| 3555 |
+
return {"diff": diff, "affected": {"ids": [database], "count": 1},
|
| 3556 |
+
"warnings": ["The preview diff reached its display cap; split this change."] if truncated else [],
|
| 3557 |
+
"revision": revision,
|
| 3558 |
+
"planned": {"database": database, "action": "rename", "meta": result["meta"]}}
|
| 3559 |
+
|
| 3560 |
+
|
| 3561 |
+
def _database_record_write_guard(session, database, *, field_keys=None):
|
| 3562 |
+
"""Pass update fields to the shared locked-record wall without unlocking row creation or deletion."""
|
| 3563 |
+
import routes_tables
|
| 3564 |
+
from core import user_tables
|
| 3565 |
+
|
| 3566 |
+
guard = routes_tables._records_or_refuse
|
| 3567 |
+
if field_keys is not None and not user_tables.records_mutable(database, st=session.runtime):
|
| 3568 |
+
# A deployment without the shared field-aware wall keeps its existing refusal.
|
| 3569 |
+
if "field_keys" in inspect.signature(guard).parameters:
|
| 3570 |
+
return guard(session, database, field_keys=field_keys)
|
| 3571 |
+
return guard(session, database)
|
| 3572 |
+
|
| 3573 |
+
|
| 3574 |
def _plan_database_write(session, args):
|
| 3575 |
import routes_tables
|
| 3576 |
from core import user_tables
|
|
|
|
| 3578 |
action = args.get("action")
|
| 3579 |
database = str(args.get("database") or "").strip()
|
| 3580 |
if action not in {"record_create", "record_update", "record_delete",
|
| 3581 |
+
"field_create", "field_update", "field_delete", "rename"} or not database:
|
| 3582 |
+
raise err(400, "tool_arguments_invalid", "Name a database and an action that changes records, fields or metadata.")
|
| 3583 |
_assert_database_scope(session, database)
|
| 3584 |
_assert_write_scope(session, "database_object")
|
| 3585 |
+
if action == "rename":
|
| 3586 |
+
return _plan_database_rename(session, args)
|
| 3587 |
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 3588 |
bucket_before = session.runtime.get(user_tables.STORE_KEY)
|
| 3589 |
if not isinstance(bucket_before, dict):
|
|
|
|
| 3600 |
bucket_after = copy.deepcopy(bucket_before)
|
| 3601 |
|
| 3602 |
if action.startswith("record_"):
|
| 3603 |
+
field_keys = None
|
| 3604 |
+
if action == "record_update":
|
| 3605 |
+
raw_values = args.get("values")
|
| 3606 |
+
if not isinstance(raw_values, dict) or not raw_values:
|
| 3607 |
+
raise err(400, "tool_arguments_invalid", "Record values must be a non-empty object.")
|
| 3608 |
+
field_keys = tuple(str(key) for key in raw_values)
|
| 3609 |
+
_database_record_write_guard(session, database, field_keys=field_keys)
|
| 3610 |
g = routes_tables.ut_write_ctx(session, database)
|
| 3611 |
rows = table.get("rows") or {}
|
| 3612 |
if action == "record_create":
|
|
|
|
| 3759 |
if action == "query":
|
| 3760 |
return _query_database(session, database, args.get("query") or {})
|
| 3761 |
plan = plan or _plan_database_write(session, args)
|
| 3762 |
+
if action == "rename":
|
| 3763 |
+
import routes_nav
|
| 3764 |
+
|
| 3765 |
+
guarded_session = Session(session.tenant, session.user, session.claims,
|
| 3766 |
+
_ConfirmationRuntime(session.runtime, {
|
| 3767 |
+
routes_nav._NAV_META_KEY: plan.get("revision")}))
|
| 3768 |
+
try:
|
| 3769 |
+
result = routes_nav.save_nav_meta(_database_nav_body(args), guarded_session)
|
| 3770 |
+
except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
|
| 3771 |
+
raise err(409, "confirmation_stale",
|
| 3772 |
+
"The database metadata changed after the preview, " + _STALE_FIX) from exc
|
| 3773 |
+
return {"database": database, "action": action, "meta": result["meta"],
|
| 3774 |
+
"affected": copy.deepcopy(plan.get("affected") or {})}
|
| 3775 |
expected_rev = plan.get("revision", {}).get("rev")
|
| 3776 |
expected_digest = plan.get("bucket_before_digest")
|
| 3777 |
replacement = copy.deepcopy(plan.get("bucket_after"))
|
|
|
|
| 4329 |
else "Send about as an object with title, description and tags." if not isinstance(value, dict)
|
| 4330 |
else "Add " + ", ".join(missing) + " to about." if missing
|
| 4331 |
else "Fix " + ", ".join(invalid) + ".")
|
| 4332 |
+
details["allowed"] = tags
|
| 4333 |
+
message += (" Use only title (1 to 240 characters), description (20 to 240 characters), and tags. "
|
| 4334 |
+
"Use no control characters or long dashes in title or description. "
|
| 4335 |
+
"Use one or more tags from this list, each once: " + ", ".join(tags) + ". "
|
| 4336 |
+
"Valid metadata: " + json.dumps(example, separators=(", ", ": ")) + ".")
|
|
|
|
| 4337 |
raise err(400, "build_about_required" if missing else "build_about_invalid", message, details) from None
|
| 4338 |
return cleaned
|
| 4339 |
|
|
|
|
| 4683 |
_external_read_scope(session, database, fields=[], mode="schema")
|
| 4684 |
else:
|
| 4685 |
_external_read_scope(session, database, fields=[])
|
| 4686 |
+
if name == "database_object" and args.get("action") == "rename":
|
| 4687 |
+
import routes_nav
|
| 4688 |
+
routes_nav._nav_meta_write_patch(_database_nav_body(args), session)
|
| 4689 |
+
elif name == "database_object":
|
| 4690 |
import routes_tables
|
| 4691 |
g = routes_tables.ut_write_ctx(session, args["database"])
|
| 4692 |
action = args["action"]
|
|
|
|
| 4893 |
for identity in before.keys() | after.keys():
|
| 4894 |
if before.get(identity) != after.get(identity):
|
| 4895 |
add("app", (after.get(identity) or before.get(identity) or {}).get("appId"))
|
| 4896 |
+
elif (key in (application_registry._KEY, automation_engine.STORE_KEY)
|
| 4897 |
+
or key.startswith("applications_v1_app/")):
|
| 4898 |
+
kind = "automation" if key == automation_engine.STORE_KEY else "app"
|
| 4899 |
if kind == "app":
|
| 4900 |
before, after = before.get("applications") or {}, after.get("applications") or {}
|
| 4901 |
for identity in before.keys() | after.keys():
|
|
|
|
| 5073 |
before_revision = copy.deepcopy(recomputed.get("current_revision") or {})
|
| 5074 |
if tool == "database_object":
|
| 5075 |
from core import user_tables
|
| 5076 |
+
import routes_nav
|
| 5077 |
+
bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY
|
| 5078 |
after_revision = _bucket_revision(
|
| 5079 |
+
fresh_session.runtime, bucket, fresh_session.runtime.get(bucket))
|
|
|
|
| 5080 |
produced_rev = before_revision["rev"] + 1
|
| 5081 |
if after_revision.get("rev") != produced_rev:
|
| 5082 |
after_revision = {"rev": produced_rev}
|
|
|
|
| 5131 |
("The arguments do not match this action. Read its guide and correct them.", """
|
| 5132 |
tool_arguments_invalid query_arguments_invalid query_aggregates_invalid query_aggregate_type
|
| 5133 |
assistant_bad_filter query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid
|
| 5134 |
+
bad_application bad_entries bad_value bad_values bad_request bad_recurrence bad_icon name_not_allowed invalid_field_contract
|
| 5135 |
external_edit_empty external_edit_invalid oauth_edit_empty oauth_edit_invalid
|
| 5136 |
"""),
|
| 5137 |
("You do not have permission to do this with this connection.", """
|
|
|
|
| 5271 |
"request_method_unknown": "The requested method is not available.",
|
| 5272 |
"request_rate_limited": "This AI has reached its hourly request cap. Try again later.",
|
| 5273 |
})
|
| 5274 |
+
MCP_RESULT_CODES.update(READ_REFUSAL_FIXES)
|
| 5275 |
_MCP_INTERNAL_PROSE = re.compile(
|
| 5276 |
r"\b(?:DuckDB|Postgres(?:ql)?|SQLite|Polars|Pandas|SQLAlchemy|PyArrow|HyperFormula)\b|"
|
| 5277 |
r"\b(?:api|web|platform)_[a-z_]+\b|\b(?:verify_\w+|\w+Error)\b|"
|
|
|
|
| 5572 |
"A build that breaks a required rule is refused with its fix.",
|
| 5573 |
"2. Discover before you query: use table.list_databases, then table.describe_database for the database you "
|
| 5574 |
"need. Use only field keys that describe_database returns. Never guess a field name. "
|
| 5575 |
+
"State the age of data older than one day before giving a number. " + DISCOVERY_HELP,
|
| 5576 |
"3. Read with table.query_database. Filter and sort on the server, ask only for the fields you need, "
|
| 5577 |
"and follow the cursor for more pages instead of reading everything. Report totals from the "
|
| 5578 |
"counts the tool returns, not from the rows you happened to read. "
|
|
|
|
| 5720 |
"""Resolve database grants through read-only permission predicates."""
|
| 5721 |
from core import grid_events, user_tables
|
| 5722 |
import routes_grid
|
| 5723 |
+
import routes_nav
|
| 5724 |
import routes_tables
|
| 5725 |
session = Session(session.tenant, session.user, session.claims, user_tables.lend_defs(session.runtime))
|
| 5726 |
allowed = {action: False for action in external_tools._DATABASE_ACTIONS if action not in {"list", "describe", "query"}}
|
| 5727 |
for row in rows:
|
| 5728 |
database = row["database"]
|
| 5729 |
+
try:
|
| 5730 |
+
routes_nav._nav_meta_write_patch({"key": database, "icon": None}, session)
|
| 5731 |
+
allowed["rename"] = True
|
| 5732 |
+
except HTTPException:
|
| 5733 |
+
pass
|
| 5734 |
try:
|
| 5735 |
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 5736 |
if not user_tables.is_user_table(database, st=session.runtime):
|
| 5737 |
continue
|
| 5738 |
fields = row.get("fields", [])
|
| 5739 |
+
ctx = routes_grid._ctx(session, fields, frozenset(), scope_key=database)
|
| 5740 |
+
editable = [field["key"] for field in fields
|
| 5741 |
+
if not user_tables.is_computed_cell(field) and field.get("type") != "automation"
|
| 5742 |
+
and grid_events._may_edit_field_value(ctx, field["key"], field)]
|
| 5743 |
try:
|
| 5744 |
+
_database_record_write_guard(session, database)
|
| 5745 |
+
allowed["record_create"] |= bool(editable)
|
| 5746 |
+
allowed["record_update"] |= bool(editable)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5747 |
allowed["record_delete"] = True
|
| 5748 |
except HTTPException:
|
| 5749 |
pass
|
| 5750 |
+
if not allowed["record_update"]:
|
| 5751 |
+
for key in editable:
|
| 5752 |
+
try:
|
| 5753 |
+
_database_record_write_guard(session, database, field_keys=(key,))
|
| 5754 |
+
allowed["record_update"] = True
|
| 5755 |
+
break
|
| 5756 |
+
except HTTPException:
|
| 5757 |
+
pass
|
| 5758 |
try:
|
| 5759 |
routes_tables._field_or_refuse(session, database)
|
| 5760 |
allowed["field_create"] = True
|
|
|
|
| 5895 |
"record_update": "edit record fields", "record_delete": "delete records",
|
| 5896 |
"field_create": "add fields as the database owner or an Administrator",
|
| 5897 |
"field_update": "edit a visible field", "field_delete": "delete a visible field",
|
| 5898 |
+
"rename": "change database names or icons as an Administrator",
|
| 5899 |
}[operation] + " in a writable database")
|
| 5900 |
granted = not required
|
| 5901 |
row.update(state=("not supported here" if not supported else
|
|
|
|
| 6135 |
spec["inputSchema"]["properties"]["about"] = copy.deepcopy(external_tools.ABOUT_SCHEMA)
|
| 6136 |
spec["inputSchema"]["properties"]["action"]["enum"] = list(groups[spec["name"]]["actions"])
|
| 6137 |
if spec["name"] == "sheet":
|
| 6138 |
+
from tool_groups.sheet import WORKBOOK_RESOURCE_URI, write_guide
|
| 6139 |
+
spec["description"] += " " + write_guide()
|
| 6140 |
spec["_meta"] = {"ui": {"resourceUri": WORKBOOK_RESOURCE_URI}}
|
| 6141 |
+
if spec["name"] == "table":
|
| 6142 |
+
spec["description"] += " " + _QUERY_PAGING_HELP
|
| 6143 |
spec["description"] = _mcp_plain(spec["description"], "Read this group's guide for actions.")
|
| 6144 |
spec["inputSchema"] = _mcp_schema_copy(spec["inputSchema"])
|
| 6145 |
if "title" in spec.get("annotations", {}):
|
|
|
|
| 6378 |
def _mcp_intent_fix(name):
|
| 6379 |
"""Name the intent field and where this tool takes it."""
|
| 6380 |
import tool_groups
|
| 6381 |
+
example = ' Example: {"intent": "Prepare the weekly sales review"}.'
|
| 6382 |
if name == "confirm_change":
|
| 6383 |
return ("Send intent beside confirmation_token: 1 to 500 characters on why you are confirming this change. "
|
| 6384 |
+
"A client that sends only action and arguments may put both inside arguments." + example)
|
| 6385 |
if tool_groups.canonical_call(name, {})[0] in tool_groups.groups():
|
| 6386 |
+
fix = ("Send intent beside action and arguments: 1 to 500 characters on why you are making this write. "
|
| 6387 |
+
"If your client sends only action and arguments, put intent inside arguments." + example)
|
| 6388 |
+
else:
|
| 6389 |
+
fix = "Send intent beside this tool's other arguments: 1 to 500 characters on why you are making this write." + example
|
| 6390 |
+
if name == "sheet" or isinstance(name, str) and name.startswith("sheet_"):
|
| 6391 |
+
from tool_groups.sheet import write_guide
|
| 6392 |
+
fix += " " + write_guide()
|
| 6393 |
+
return fix
|
| 6394 |
|
| 6395 |
|
| 6396 |
def _mcp_about_fix(name):
|
| 6397 |
"""Name where a group call takes about, or nothing for a flat tool."""
|
| 6398 |
import tool_groups
|
| 6399 |
+
fix = ""
|
| 6400 |
if isinstance(name, str) and tool_groups.canonical_call(name, {})[0] in tool_groups.groups():
|
| 6401 |
+
fix = (" Put about beside action and arguments. "
|
| 6402 |
+
"If your client sends only action and arguments, put about inside arguments.")
|
| 6403 |
+
if name == "sheet" or isinstance(name, str) and name.startswith("sheet_"):
|
| 6404 |
+
from tool_groups.sheet import write_guide
|
| 6405 |
+
fix += " " + write_guide()
|
| 6406 |
+
return fix
|
| 6407 |
|
| 6408 |
|
| 6409 |
def _mcp_requires_intent(name, arguments):
|
|
|
|
| 6441 |
return reply
|
| 6442 |
|
| 6443 |
|
| 6444 |
+
#: Known refusal codes whose MCP answer carries a repair.
|
| 6445 |
+
_MCP_NEXT_CALL = {"workbook_missing": WORKBOOK_MISSING_FIX, "assistant_unknown_database": UNKNOWN_DATABASE_FIX,
|
| 6446 |
+
**READ_REFUSAL_FIXES}
|
| 6447 |
|
| 6448 |
|
| 6449 |
def _mcp_next_call(code, message):
|
| 6450 |
+
"""Append the supported repair once for a known read or missing-item refusal."""
|
| 6451 |
text = str(message or "").rstrip()
|
| 6452 |
fix = _MCP_NEXT_CALL.get(code, "")
|
| 6453 |
if code == "assistant_unknown_database":
|
|
|
|
| 6663 |
if info.get("code") == "workbook_name_ambiguous" and isinstance(details, dict):
|
| 6664 |
retry["candidates"] = details.get("candidates", [])
|
| 6665 |
message = info.get("message", "The app refused this operation.")
|
| 6666 |
+
if info.get("code") in {"build_about_required", "build_about_invalid"} and isinstance(message, str):
|
| 6667 |
message += _mcp_about_fix(called)
|
| 6668 |
if info.get("code") in _MCP_NEXT_CALL:
|
| 6669 |
message = _mcp_next_call(info["code"], message)
|
api/routes_geo.py
CHANGED
|
@@ -338,7 +338,7 @@ def _zone_application(session, table_key, application, *, write=False):
|
|
| 338 |
import application_registry as registry
|
| 339 |
from core import route_system
|
| 340 |
try:
|
| 341 |
-
doc = registry.
|
| 342 |
if not isinstance(doc, dict) or not isinstance(doc.get("applications", {}), dict):
|
| 343 |
raise ValueError("invalid Application registry")
|
| 344 |
except Exception:
|
|
|
|
| 338 |
import application_registry as registry
|
| 339 |
from core import route_system
|
| 340 |
try:
|
| 341 |
+
doc = registry.load(session.runtime, strict=True)
|
| 342 |
if not isinstance(doc, dict) or not isinstance(doc.get("applications", {}), dict):
|
| 343 |
raise ValueError("invalid Application registry")
|
| 344 |
except Exception:
|
api/routes_grid.py
CHANGED
|
@@ -6,16 +6,7 @@ same `core.grid_events.handle_events` the Streamlit adapter runs. That is the wh
|
|
| 6 |
EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated
|
| 7 |
anything here would be a second wall to keep in step, and the two would drift on the first change.
|
| 8 |
|
| 9 |
-
|
| 10 |
-
`seen_ids` lives in `st.session_state` — genuinely per user session — because the client resends
|
| 11 |
-
its recent 24-event window on every emit inside one page session. A stateless API has no such
|
| 12 |
-
dict, and inventing a per-user server-side one would be exactly the resident per-tenant state
|
| 13 |
-
EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole
|
| 14 |
-
purpose — a batch that repeats an id processes it once), and a genuinely replayed request is
|
| 15 |
-
handled by the operations being idempotent. The one event where a replay is observable is
|
| 16 |
-
`add_to_list` (it would add the same pids to the same cohort twice — a set union, so the
|
| 17 |
-
membership is unchanged, but the toast count repeats). Noted rather than papered over; a
|
| 18 |
-
server-side idempotency key belongs with D2's session mirror (C-2).
|
| 19 |
|
| 20 |
STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises
|
| 21 |
`StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read
|
|
@@ -23,6 +14,8 @@ back. A 200 over a write that evaporated is the failure this rule exists to prev
|
|
| 23 |
"""
|
| 24 |
import datetime as dt
|
| 25 |
import time
|
|
|
|
|
|
|
| 26 |
|
| 27 |
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
|
| 28 |
|
|
@@ -37,6 +30,70 @@ MODULE = "customer_data"
|
|
| 37 |
_MAX_EVENTS = 24
|
| 38 |
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
def _ctx(session: Session, fields, pids, *, permission_fields=None, **kw):
|
| 41 |
from core import grid_events
|
| 42 |
import aios_grid
|
|
@@ -611,59 +668,6 @@ def workspace(scope: str = "customer",
|
|
| 611 |
return {"workspace": workspace}
|
| 612 |
|
| 613 |
|
| 614 |
-
def _shared_note_subject(session, scope, view_id):
|
| 615 |
-
"""Resolve a note subject only after the ordinary workspace wall admits it."""
|
| 616 |
-
scope = _scope_or_400(scope)
|
| 617 |
-
payload = workspace(scope=scope, session=session)["workspace"]
|
| 618 |
-
if view_id:
|
| 619 |
-
view = next((item for item in (payload.get("views") or [])
|
| 620 |
-
if str(item.get("id") or "") == str(view_id)), None)
|
| 621 |
-
if view is None:
|
| 622 |
-
raise err(404, "view_not_found", "that View is not visible in this database")
|
| 623 |
-
return "database", _database_key_for_scope(scope), view
|
| 624 |
-
return "database", _database_key_for_scope(scope), None
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
@router.get("/grid/shared-note")
|
| 628 |
-
def get_shared_note(scope: str = "customer", viewId: str = "",
|
| 629 |
-
session: Session = Depends(require_session)):
|
| 630 |
-
"""Return the database note from any visible View."""
|
| 631 |
-
from core import user_tables
|
| 632 |
-
|
| 633 |
-
kind, subject_id, _view = _shared_note_subject(session, scope, viewId)
|
| 634 |
-
return {"note": user_tables.shared_note(kind, subject_id, st=session.runtime)}
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
@router.put("/grid/shared-note")
|
| 638 |
-
def update_shared_note(scope: str = "customer", viewId: str = "",
|
| 639 |
-
body: dict = Body(default=None),
|
| 640 |
-
session: Session = Depends(require_session)):
|
| 641 |
-
"""Edit a shared note through the same database/View permission wall as the grid."""
|
| 642 |
-
from core import shares, user_tables
|
| 643 |
-
|
| 644 |
-
kind, subject_id, view = _shared_note_subject(session, scope, viewId)
|
| 645 |
-
may_edit = bool(session.admin)
|
| 646 |
-
if subject_id.startswith("ut_"):
|
| 647 |
-
may_edit = user_tables.may_edit_database(
|
| 648 |
-
subject_id, session.uname, is_admin=session.admin, st=session.runtime)
|
| 649 |
-
if view is not None and not may_edit:
|
| 650 |
-
owner = str(view.get("createdBy") or view.get("owner") or "").strip().lower()
|
| 651 |
-
may_edit = bool(
|
| 652 |
-
owner == str(session.uname or "").strip().lower()
|
| 653 |
-
or shares.may_edit("view", str(view.get("id") or viewId), session.uname,
|
| 654 |
-
is_admin=session.admin, st=session.runtime)
|
| 655 |
-
)
|
| 656 |
-
if not may_edit:
|
| 657 |
-
raise err(403, "note_read_only", "your role does not allow note edits")
|
| 658 |
-
try:
|
| 659 |
-
note = user_tables.put_shared_note(
|
| 660 |
-
kind, subject_id, (body or {}).get("body"), session.uname, st=session.runtime)
|
| 661 |
-
except ValueError as exc:
|
| 662 |
-
raise err(400, "note_too_long", str(exc),
|
| 663 |
-
{"limit": user_tables.SHARED_NOTE_MAX_CHARS, "unit": "characters"}) from exc
|
| 664 |
-
return {"note": note}
|
| 665 |
-
|
| 666 |
-
|
| 667 |
_RECURRENCE_CONDITION_OPS = frozenset({"eq", "neq", "isNotEmpty", "between"})
|
| 668 |
_RECURRENCE_ORDERED_TYPES = frozenset({
|
| 669 |
"date", "datetime", "int", "integer", "number", "currency", "pct", "rating",
|
|
@@ -2120,6 +2124,7 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2120 |
# simply had not been given them — the standalone shell could read measures it could
|
| 2121 |
# never write.
|
| 2122 |
ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, via=via,
|
|
|
|
| 2123 |
permission_fields=(g.get('_permission_scope') or {}).get('fields'),
|
| 2124 |
measure_keys=frozenset(m["key"] for m in g["measures"]),
|
| 2125 |
resolved_ids=frozenset(g["measure_sets"]),
|
|
@@ -2134,7 +2139,7 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2134 |
results = []
|
| 2135 |
try:
|
| 2136 |
for one in events:
|
| 2137 |
-
eid = str(one.get("id") or "") if isinstance(one, dict) else ""
|
| 2138 |
# ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler
|
| 2139 |
# appends to a shared list, so the refusals belonging to THIS event are exactly the
|
| 2140 |
# ones that appeared across THIS call. Reading the list once after the loop would
|
|
@@ -2142,20 +2147,20 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2142 |
# the same class of unfalsifiable answer the channel exists to end.
|
| 2143 |
_before = len(ctx.out.refusals)
|
| 2144 |
rerender = grid_events.handle_one(one, ctx)
|
| 2145 |
-
row = {**ctx.out.writes.get(eid, {}), "id": eid, "
|
|
|
|
| 2146 |
mine = ctx.out.refusals[_before:]
|
| 2147 |
if mine:
|
| 2148 |
# `refused` is the SHAPE a caller branches on; the first reason is the one that
|
| 2149 |
# stopped this write (a handler returns at its first refusal).
|
| 2150 |
row["refused"] = mine[0]
|
| 2151 |
results.append(row)
|
| 2152 |
-
except shared_overlay.WriteUnavailable as exc:
|
| 2153 |
-
refusal = err(503, "store_unavailable",
|
| 2154 |
-
|
|
|
|
|
|
|
| 2155 |
raise refusal from exc
|
| 2156 |
-
except grid_events.StoreUnavailable:
|
| 2157 |
-
raise err(503, "store_unavailable",
|
| 2158 |
-
"the tenant store is unavailable, so none of your changes were saved")
|
| 2159 |
|
| 2160 |
out = {"results": results, "rerender": any(r["rerender"] for r in results)}
|
| 2161 |
if ctx.out.refusals:
|
|
|
|
| 6 |
EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated
|
| 7 |
anything here would be a second wall to keep in step, and the two would drift on the first change.
|
| 8 |
|
| 9 |
+
Edit receipts retain each event's first result in the tenant store across requests and restarts.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises
|
| 12 |
`StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read
|
|
|
|
| 14 |
"""
|
| 15 |
import datetime as dt
|
| 16 |
import time
|
| 17 |
+
from functools import wraps
|
| 18 |
+
from inspect import signature
|
| 19 |
|
| 20 |
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
|
| 21 |
|
|
|
|
| 30 |
_MAX_EVENTS = 24
|
| 31 |
|
| 32 |
|
| 33 |
+
def row_patch_receipt(scope):
|
| 34 |
+
"""Retain the complete row PATCH response and skip its writes and hooks on replay."""
|
| 35 |
+
def decorate(fn):
|
| 36 |
+
sig = signature(fn)
|
| 37 |
+
|
| 38 |
+
@wraps(fn)
|
| 39 |
+
def wrapped(*args, **kwargs):
|
| 40 |
+
from core import edit_receipts
|
| 41 |
+
bound = sig.bind(*args, **kwargs)
|
| 42 |
+
body = bound.arguments.get('body')
|
| 43 |
+
if not isinstance(body, dict) or 'opId' not in body:
|
| 44 |
+
return fn(*args, **kwargs)
|
| 45 |
+
try:
|
| 46 |
+
identity = edit_receipts.op_id(body['opId'])
|
| 47 |
+
except ValueError as exc:
|
| 48 |
+
raise err(400, 'bad_op_id', str(exc)) from exc
|
| 49 |
+
session = bound.arguments['session']
|
| 50 |
+
scope_key = scope if isinstance(scope, str) else scope(bound.arguments)
|
| 51 |
+
bound.arguments['body'] = {key: value for key, value in body.items() if key != 'opId'}
|
| 52 |
+
|
| 53 |
+
def apply():
|
| 54 |
+
try:
|
| 55 |
+
answer = dict(fn(*bound.args, **bound.kwargs), opId=identity)
|
| 56 |
+
return {'answer': answer, 'event': edit_receipts.current_event()}
|
| 57 |
+
except HTTPException as exc:
|
| 58 |
+
if exc.status_code >= 500:
|
| 59 |
+
raise
|
| 60 |
+
detail = exc.detail
|
| 61 |
+
if isinstance(detail, dict):
|
| 62 |
+
detail = dict(detail, opId=identity)
|
| 63 |
+
error = (detail.get('error') or {}) if isinstance(detail, dict) else {}
|
| 64 |
+
refusal = {'event': 'overlay_patch', 'code': error.get('code') or 'write_refused',
|
| 65 |
+
'message': error.get('message') or str(detail)}
|
| 66 |
+
event = edit_receipts.current_event() or {
|
| 67 |
+
'rerender': True, 'write': {'ok': False, 'value': None, 'refused': True,
|
| 68 |
+
'reason': refusal['code'], 'opId': identity},
|
| 69 |
+
'refusals': [refusal], 'doc': None, 'toast': None}
|
| 70 |
+
return {'status': exc.status_code, 'detail': detail,
|
| 71 |
+
'headers': exc.headers, 'event': event}
|
| 72 |
+
|
| 73 |
+
try:
|
| 74 |
+
result = edit_receipts.run_once(
|
| 75 |
+
session.runtime, session.uname,
|
| 76 |
+
edit_receipts.operation(scope_key, 'overlay_patch', bound.arguments['pid']),
|
| 77 |
+
identity, apply)
|
| 78 |
+
except edit_receipts.ReceiptUnavailable as exc:
|
| 79 |
+
raise err(503, 'store_unavailable', str(exc)) from exc
|
| 80 |
+
if 'status' in result:
|
| 81 |
+
raise HTTPException(result['status'], result['detail'], headers=result['headers'])
|
| 82 |
+
if 'answer' in result:
|
| 83 |
+
return result['answer']
|
| 84 |
+
# A grid event may have committed this operation before its row PATCH replay.
|
| 85 |
+
write = result['write']
|
| 86 |
+
answer = {'pid': bound.arguments['pid'], 'updates': write.get('value') or {},
|
| 87 |
+
'opId': identity}
|
| 88 |
+
if write.get('refused'):
|
| 89 |
+
answer['refused'] = list(bound.arguments['body'])
|
| 90 |
+
answer['reason'] = write.get('reason') or 'write_refused'
|
| 91 |
+
answer['refusals'] = result.get('refusals') or []
|
| 92 |
+
return answer
|
| 93 |
+
return wrapped
|
| 94 |
+
return decorate
|
| 95 |
+
|
| 96 |
+
|
| 97 |
def _ctx(session: Session, fields, pids, *, permission_fields=None, **kw):
|
| 98 |
from core import grid_events
|
| 99 |
import aios_grid
|
|
|
|
| 668 |
return {"workspace": workspace}
|
| 669 |
|
| 670 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 671 |
_RECURRENCE_CONDITION_OPS = frozenset({"eq", "neq", "isNotEmpty", "between"})
|
| 672 |
_RECURRENCE_ORDERED_TYPES = frozenset({
|
| 673 |
"date", "datetime", "int", "integer", "number", "currency", "pct", "rating",
|
|
|
|
| 2124 |
# simply had not been given them — the standalone shell could read measures it could
|
| 2125 |
# never write.
|
| 2126 |
ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, via=via,
|
| 2127 |
+
durable_receipts=True,
|
| 2128 |
permission_fields=(g.get('_permission_scope') or {}).get('fields'),
|
| 2129 |
measure_keys=frozenset(m["key"] for m in g["measures"]),
|
| 2130 |
resolved_ids=frozenset(g["measure_sets"]),
|
|
|
|
| 2139 |
results = []
|
| 2140 |
try:
|
| 2141 |
for one in events:
|
| 2142 |
+
eid = str(one.get("opId", one.get("id")) or "") if isinstance(one, dict) else ""
|
| 2143 |
# ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler
|
| 2144 |
# appends to a shared list, so the refusals belonging to THIS event are exactly the
|
| 2145 |
# ones that appeared across THIS call. Reading the list once after the loop would
|
|
|
|
| 2147 |
# the same class of unfalsifiable answer the channel exists to end.
|
| 2148 |
_before = len(ctx.out.refusals)
|
| 2149 |
rerender = grid_events.handle_one(one, ctx)
|
| 2150 |
+
row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid,
|
| 2151 |
+
"rerender": bool(rerender)}
|
| 2152 |
mine = ctx.out.refusals[_before:]
|
| 2153 |
if mine:
|
| 2154 |
# `refused` is the SHAPE a caller branches on; the first reason is the one that
|
| 2155 |
# stopped this write (a handler returns at its first refusal).
|
| 2156 |
row["refused"] = mine[0]
|
| 2157 |
results.append(row)
|
| 2158 |
+
except (shared_overlay.WriteUnavailable, grid_events.StoreUnavailable) as exc:
|
| 2159 |
+
refusal = err(503, "store_unavailable",
|
| 2160 |
+
"This edit could not be confirmed as saved. Retry with the same op id. "
|
| 2161 |
+
"Earlier operations in this batch may have been saved.")
|
| 2162 |
+
refusal.detail.update(ok=False, value=None, refused=True, reason="store_unavailable", opId=eid)
|
| 2163 |
raise refusal from exc
|
|
|
|
|
|
|
|
|
|
| 2164 |
|
| 2165 |
out = {"results": results, "rerender": any(r["rerender"] for r in results)}
|
| 2166 |
if ctx.out.refusals:
|
api/routes_keychain.py
CHANGED
|
@@ -592,11 +592,12 @@ def migrate_env_odoo(rt):
|
|
| 592 |
⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a
|
| 593 |
third-party failure must not take the boot down.
|
| 594 |
"""
|
| 595 |
-
out = {"done": False, "entry": "", "carried_pause": False, "why": ""}
|
| 596 |
if not env_odoo_available(rt):
|
| 597 |
# Not tenant #0, or this deployment has no env Odoo at all. Both are normal states.
|
| 598 |
out["why"] = "this tenant has no environment Odoo credential to migrate"
|
| 599 |
return out
|
|
|
|
| 600 |
fields, why = _env_odoo_fields()
|
| 601 |
if not fields:
|
| 602 |
out["why"] = why
|
|
@@ -612,9 +613,11 @@ def migrate_env_odoo(rt):
|
|
| 612 |
row, why = _kc().ensure_entry_of_type(
|
| 613 |
rt, "odoo", "Odoo (migrated from this deployment)", fields, "system")
|
| 614 |
if row is None:
|
| 615 |
-
out["why"] =
|
|
|
|
| 616 |
return out
|
| 617 |
out["done"], out["entry"] = True, row["id"]
|
|
|
|
| 618 |
if was_paused:
|
| 619 |
# The freeze followed the credential. Without this the connector silently RESUMES pulling
|
| 620 |
# live Odoo at the first boot after the migration.
|
|
@@ -629,16 +632,36 @@ def migrate_env_odoo(rt):
|
|
| 629 |
return cur
|
| 630 |
|
| 631 |
try:
|
| 632 |
-
rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync")
|
|
|
|
|
|
|
|
|
|
| 633 |
out["carried_pause"] = True
|
| 634 |
-
except Exception
|
| 635 |
-
#
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 639 |
return out
|
| 640 |
|
| 641 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 642 |
def _own_row(session, entry_id):
|
| 643 |
"""The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the
|
| 644 |
caller cannot see: telling a member that somebody else's personal credential EXISTS is the
|
|
@@ -735,9 +758,11 @@ def _register_credential_tables():
|
|
| 735 |
try:
|
| 736 |
from core import user_tables as ut
|
| 737 |
import routes_connectors as rc
|
|
|
|
| 738 |
# Instruction 30 / C13 names the Amazon Data Protection Policy, not every connector.
|
| 739 |
ut.register_credential_tables(
|
| 740 |
"amazon_sp", rc.CONNECTED_PREFIXES["amazon_sp"], on_revoke=ut.REVOKE_PURGE,
|
|
|
|
| 741 |
why="The Amazon Data Protection Policy requires deleting the seller data "
|
| 742 |
"authorised by a revoked credential.")
|
| 743 |
# W32 R10's second sentence and D-259 require these grids to survive a reconnect.
|
|
@@ -763,9 +788,13 @@ def _register_credential_tables():
|
|
| 763 |
def _credential_revocation(row, rt):
|
| 764 |
"""A failed census is UNKNOWN, never evidence of an empty seller footprint."""
|
| 765 |
registered = _register_credential_tables()
|
| 766 |
-
etype
|
|
|
|
| 767 |
if registered:
|
| 768 |
try:
|
|
|
|
|
|
|
|
|
|
| 769 |
from core import user_tables as ut
|
| 770 |
return registered, ut.credential_footprint(etype, cid, rt=rt, st=rt)
|
| 771 |
except Exception as exc:
|
|
@@ -787,7 +816,7 @@ def _revocation_residue(entry_type):
|
|
| 787 |
return []
|
| 788 |
return [{
|
| 789 |
"where": "The tenant DuckDB mirror",
|
| 790 |
-
"why": "Connection removal
|
| 791 |
"fix": "If automatic deletion could not be scheduled, after every Amazon credential is revoked an operator must run "
|
| 792 |
"platform/harness/amazon_store.py --purge --tenant <tenant> "
|
| 793 |
"--confirm-tenant <tenant> and retain the residue-free receipt.",
|
|
@@ -822,18 +851,22 @@ def _amazon_lifecycle_warning(session, action):
|
|
| 822 |
return {"warning": warning}
|
| 823 |
|
| 824 |
|
| 825 |
-
def _amazon_deletion_receipt(session
|
| 826 |
try:
|
| 827 |
-
if not product_purge_succeeded:
|
| 828 |
-
raise RuntimeError("product cleanup is incomplete")
|
| 829 |
-
from harness import amazon_store as az
|
| 830 |
rt = session.runtime
|
| 831 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 832 |
plan = az.scheduled_purge(rt.key, rt)
|
| 833 |
receipt = {"purgeScheduledAt": plan["purgeAt"], "deadlineAt": plan["deadlineAt"],
|
| 834 |
"residue": [{"where": "Amazon data in this workspace",
|
| 835 |
"why": az.deletion_schedule_message(plan),
|
| 836 |
-
"fix": "Add the Amazon credential again before the scheduled date to cancel deletion."
|
|
|
|
|
|
|
| 837 |
deadline = az._utc(plan["deadlineAt"]).date().isoformat()
|
| 838 |
except Exception:
|
| 839 |
return {**_amazon_lifecycle_warning(session, "revocation"),
|
|
@@ -855,6 +888,22 @@ def _amazon_deletion_receipt(session, *, product_purge_succeeded=True):
|
|
| 855 |
return receipt
|
| 856 |
|
| 857 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 858 |
def _write_revocation_warning(rt, entry_id, warning=None):
|
| 859 |
"""Prune expired consent on every write; failed persistence requires explicit confirmation."""
|
| 860 |
now = int(time.time())
|
|
@@ -965,6 +1014,7 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 965 |
raise err(400, "bad_initial_selection", str(exc))
|
| 966 |
except Exception:
|
| 967 |
raise err(503, "store_unavailable", "The database choices were not saved. Try again.")
|
|
|
|
| 968 |
try:
|
| 969 |
row = kc.add_entry(session.runtime, body.get("label"), body.get("type"),
|
| 970 |
body.get("fields"), session.uname)
|
|
@@ -986,10 +1036,8 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 986 |
# of a side bucket is to publish a credential, not to hide one. So the second write is not
|
| 987 |
# best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was
|
| 988 |
# stored. `business` needs no row at all, so this branch is the only one that can be partial.
|
| 989 |
-
#
|
| 990 |
-
|
| 991 |
-
# find a second switch. The tables were never dropped, so "resume" is one flag.
|
| 992 |
-
if etype == "odoo":
|
| 993 |
try:
|
| 994 |
_rel_reconnect(session.runtime)
|
| 995 |
except Exception: # noqa: BLE001
|
|
@@ -1108,6 +1156,18 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
|
|
| 1108 |
# another person's personal entry's 404, but ids have 12 random hex characters and are not
|
| 1109 |
# enumerable. A 404 on a lost response's retry would break this policy path's idempotence.
|
| 1110 |
if not any(entry["id"] == str(entry_id) for entry in entries):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1111 |
return {"ok": True, "alreadyGone": True,
|
| 1112 |
"purged": {"tables": [], "rows": 0, "rowsKnown": True,
|
| 1113 |
"refused": [], "failed": []}}
|
|
@@ -1145,7 +1205,9 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
|
|
| 1145 |
if stored else "The warning could not be saved. "
|
| 1146 |
"Pass confirm=purge to confirm explicitly.")
|
| 1147 |
raise err(409, "confirm_purge",
|
| 1148 |
-
f"Removing this connection also deletes {len(tables):,} {kind} {noun}"
|
|
|
|
|
|
|
| 1149 |
+ count + "That data cannot be recovered. " + next_step)
|
| 1150 |
|
| 1151 |
class _Ownership:
|
|
@@ -1158,26 +1220,44 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
|
|
| 1158 |
return {"entries": {entry["id"]: entry for entry in entries}}
|
| 1159 |
return rt.get(name)
|
| 1160 |
|
| 1161 |
-
#
|
| 1162 |
-
|
| 1163 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1164 |
try:
|
| 1165 |
kc.delete_entry(rt, entry_id)
|
| 1166 |
-
|
|
|
|
| 1167 |
except Exception:
|
| 1168 |
raise err(503, "store_unavailable", "The delete did not land. Try again.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1169 |
if not purges:
|
| 1170 |
_write_revocation_warning(rt, entry_id)
|
| 1171 |
-
lifecycle = (_amazon_deletion_receipt(session, product_purge_succeeded=footprint["rowsKnown"])
|
| 1172 |
-
if row["type"] == "amazon_sp" else {})
|
| 1173 |
if footprint["rowsKnown"]:
|
| 1174 |
-
return {"ok": True, **
|
| 1175 |
# Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes
|
| 1176 |
# that incomplete cleanup explicit instead of returning an invented "nothing to purge".
|
| 1177 |
return {"ok": True, "registry": registered, "footprint": footprint,
|
| 1178 |
"purged": {"tables": [], "rows": 0, "rowsKnown": False,
|
| 1179 |
"refused": [], "failed": []},
|
| 1180 |
-
"residue": _revocation_residue(row["type"]), **
|
| 1181 |
from core import user_tables as ut
|
| 1182 |
try:
|
| 1183 |
result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt)
|
|
@@ -1186,14 +1266,11 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
|
|
| 1186 |
result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [],
|
| 1187 |
"failed": [{"key": key, "error": type(exc).__name__} for key in tables]}
|
| 1188 |
_write_revocation_warning(rt, entry_id)
|
| 1189 |
-
lifecycle = (_amazon_deletion_receipt(
|
| 1190 |
-
session, product_purge_succeeded=not result["failed"] and not result["refused"])
|
| 1191 |
-
if row["type"] == "amazon_sp" else {})
|
| 1192 |
return {"ok": True,
|
| 1193 |
"purged": {"tables": result["purged"], "rows": result["rows"],
|
| 1194 |
"rowsKnown": result["rowsKnown"], "refused": result["refused"],
|
| 1195 |
"failed": result["failed"]},
|
| 1196 |
-
"residue": _revocation_residue(row["type"])
|
| 1197 |
|
| 1198 |
|
| 1199 |
@router.post("/admin/keychain/{entry_id}/test")
|
|
@@ -1625,8 +1702,47 @@ def _rewrite_server_db(session, server_db):
|
|
| 1625 |
return f"the connection now points at the {server_db} database"
|
| 1626 |
|
| 1627 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1628 |
@router.post("/admin/connectors/odoo/disconnect")
|
| 1629 |
-
def odoo_disconnect(session: Session = Depends(require_session)):
|
| 1630 |
"""R10 — remove the credential and FREEZE the grids as static data.
|
| 1631 |
|
| 1632 |
⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the
|
|
@@ -1640,6 +1756,11 @@ def odoo_disconnect(session: Session = Depends(require_session)):
|
|
| 1640 |
careful.
|
| 1641 |
"""
|
| 1642 |
_odoo_admin(session)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1643 |
rel = _rel()
|
| 1644 |
import datetime as _dt
|
| 1645 |
entry = _odoo_entry(session)
|
|
@@ -1695,14 +1816,21 @@ def odoo_disconnect(session: Session = Depends(require_session)):
|
|
| 1695 |
|
| 1696 |
paused_mirror = False
|
| 1697 |
if flag_key:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1698 |
def _pause(cur):
|
| 1699 |
cur = cur if isinstance(cur, dict) else {}
|
| 1700 |
-
|
|
|
|
| 1701 |
return cur
|
| 1702 |
try:
|
| 1703 |
flags = session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync")
|
| 1704 |
-
|
| 1705 |
-
|
|
|
|
| 1706 |
raise RuntimeError("the Odoo pause was not confirmed")
|
| 1707 |
paused_mirror = True
|
| 1708 |
except Exception: # noqa: BLE001
|
|
|
|
| 592 |
⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a
|
| 593 |
third-party failure must not take the boot down.
|
| 594 |
"""
|
| 595 |
+
out = {"done": False, "entry": "", "carried_pause": False, "why": "", "pending": False}
|
| 596 |
if not env_odoo_available(rt):
|
| 597 |
# Not tenant #0, or this deployment has no env Odoo at all. Both are normal states.
|
| 598 |
out["why"] = "this tenant has no environment Odoo credential to migrate"
|
| 599 |
return out
|
| 600 |
+
out["pending"] = _kc().first_entry_of_type(rt, "odoo") is None
|
| 601 |
fields, why = _env_odoo_fields()
|
| 602 |
if not fields:
|
| 603 |
out["why"] = why
|
|
|
|
| 613 |
row, why = _kc().ensure_entry_of_type(
|
| 614 |
rt, "odoo", "Odoo (migrated from this deployment)", fields, "system")
|
| 615 |
if row is None:
|
| 616 |
+
out["why"] = ("Odoo credential migration is pending. Check the keychain and try again."
|
| 617 |
+
if out["pending"] else why)
|
| 618 |
return out
|
| 619 |
out["done"], out["entry"] = True, row["id"]
|
| 620 |
+
out["pending"] = False
|
| 621 |
if was_paused:
|
| 622 |
# The freeze followed the credential. Without this the connector silently RESUMES pulling
|
| 623 |
# live Odoo at the first boot after the migration.
|
|
|
|
| 632 |
return cur
|
| 633 |
|
| 634 |
try:
|
| 635 |
+
flags = rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync")
|
| 636 |
+
saved = flags.get(row["id"]) if isinstance(flags, dict) else None
|
| 637 |
+
if not isinstance(saved, dict) or saved.get("paused") is not True:
|
| 638 |
+
raise RuntimeError("The migrated connection pause was not confirmed.")
|
| 639 |
out["carried_pause"] = True
|
| 640 |
+
except Exception: # noqa: BLE001
|
| 641 |
+
# Keep the paused environment source authoritative when pause transfer fails.
|
| 642 |
+
try:
|
| 643 |
+
_kc().delete_entry(rt, row["id"])
|
| 644 |
+
out["done"], out["entry"] = False, ""
|
| 645 |
+
out["why"] = "Odoo credential migration is pending. The paused environment connection was kept."
|
| 646 |
+
except Exception:
|
| 647 |
+
out["why"] = "Odoo credential migration is pending. An administrator must restore the connection pause."
|
| 648 |
+
out["pending"] = True
|
| 649 |
return out
|
| 650 |
|
| 651 |
|
| 652 |
+
def env_odoo_migration_pending(rt):
|
| 653 |
+
"""Disclose an environment-only connection without decrypting or migrating during a read."""
|
| 654 |
+
if not env_odoo_available(rt):
|
| 655 |
+
return False
|
| 656 |
+
entries = _kc().list_entries(rt, "odoo")
|
| 657 |
+
if not entries:
|
| 658 |
+
return True
|
| 659 |
+
flags = rt.get(_CONNECTOR_FLAGS_KEY) or {}
|
| 660 |
+
return bool(_rel().frozen(rt) and (flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused") and any(
|
| 661 |
+
row.get("createdBy") == "system" and row.get("label") == "Odoo (migrated from this deployment)"
|
| 662 |
+
and not (flags.get(row["connection_id"]) or {}).get("paused") for row in entries))
|
| 663 |
+
|
| 664 |
+
|
| 665 |
def _own_row(session, entry_id):
|
| 666 |
"""The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the
|
| 667 |
caller cannot see: telling a member that somebody else's personal credential EXISTS is the
|
|
|
|
| 758 |
try:
|
| 759 |
from core import user_tables as ut
|
| 760 |
import routes_connectors as rc
|
| 761 |
+
import amazon_relational
|
| 762 |
# Instruction 30 / C13 names the Amazon Data Protection Policy, not every connector.
|
| 763 |
ut.register_credential_tables(
|
| 764 |
"amazon_sp", rc.CONNECTED_PREFIXES["amazon_sp"], on_revoke=ut.REVOKE_PURGE,
|
| 765 |
+
owner=amazon_relational.credential_table_owner,
|
| 766 |
why="The Amazon Data Protection Policy requires deleting the seller data "
|
| 767 |
"authorised by a revoked credential.")
|
| 768 |
# W32 R10's second sentence and D-259 require these grids to survive a reconnect.
|
|
|
|
| 788 |
def _credential_revocation(row, rt):
|
| 789 |
"""A failed census is UNKNOWN, never evidence of an empty seller footprint."""
|
| 790 |
registered = _register_credential_tables()
|
| 791 |
+
etype = str(row.get("type") or "")
|
| 792 |
+
cid = str(row.get("connection_id") or row.get("connectionId") or row["id"])
|
| 793 |
if registered:
|
| 794 |
try:
|
| 795 |
+
if etype == "amazon_sp":
|
| 796 |
+
import amazon_relational
|
| 797 |
+
return registered, amazon_relational.connection_footprint(rt, cid)
|
| 798 |
from core import user_tables as ut
|
| 799 |
return registered, ut.credential_footprint(etype, cid, rt=rt, st=rt)
|
| 800 |
except Exception as exc:
|
|
|
|
| 816 |
return []
|
| 817 |
return [{
|
| 818 |
"where": "The tenant DuckDB mirror",
|
| 819 |
+
"why": "Connection removal queues cleanup of this connection's Amazon data.",
|
| 820 |
"fix": "If automatic deletion could not be scheduled, after every Amazon credential is revoked an operator must run "
|
| 821 |
"platform/harness/amazon_store.py --purge --tenant <tenant> "
|
| 822 |
"--confirm-tenant <tenant> and retain the residue-free receipt.",
|
|
|
|
| 851 |
return {"warning": warning}
|
| 852 |
|
| 853 |
|
| 854 |
+
def _amazon_deletion_receipt(session):
|
| 855 |
try:
|
|
|
|
|
|
|
|
|
|
| 856 |
rt = session.runtime
|
| 857 |
+
if any(entry.get("type") == "amazon_sp" for entry in _kc().list_entries(rt)):
|
| 858 |
+
return {}
|
| 859 |
+
from harness import amazon_store as az
|
| 860 |
+
active = az._active_deletion_triggers(rt.key, az._ledger(rt.key, rt), az._utc())
|
| 861 |
+
if not any(trigger["kind"] == "revocation" for trigger in active):
|
| 862 |
+
az.record_deletion_trigger(rt.key, "revocation", rt=rt, principal=session.uname)
|
| 863 |
plan = az.scheduled_purge(rt.key, rt)
|
| 864 |
receipt = {"purgeScheduledAt": plan["purgeAt"], "deadlineAt": plan["deadlineAt"],
|
| 865 |
"residue": [{"where": "Amazon data in this workspace",
|
| 866 |
"why": az.deletion_schedule_message(plan),
|
| 867 |
+
"fix": ("Add the Amazon credential again before the scheduled date to cancel deletion."
|
| 868 |
+
if plan["trigger"]["kind"] in ("revocation", "authorization_loss") else
|
| 869 |
+
"Contact an administrator to track this deletion obligation.")}]}
|
| 870 |
deadline = az._utc(plan["deadlineAt"]).date().isoformat()
|
| 871 |
except Exception:
|
| 872 |
return {**_amazon_lifecycle_warning(session, "revocation"),
|
|
|
|
| 888 |
return receipt
|
| 889 |
|
| 890 |
|
| 891 |
+
def _queue_amazon_cleanup(session, connection_id):
|
| 892 |
+
"""Wake the resident writer while durable revocation metadata retains retry ownership."""
|
| 893 |
+
result = {"connectionId": connection_id, "cleanupPending": True,
|
| 894 |
+
"purged": {"tables": [], "rows": 0, "rowsKnown": False,
|
| 895 |
+
"refused": [], "failed": []}}
|
| 896 |
+
try:
|
| 897 |
+
from core import connector_sync
|
| 898 |
+
try:
|
| 899 |
+
connector_sync.request_sync("amazon_sp", session.runtime.key, requested_by=session.uname)
|
| 900 |
+
finally:
|
| 901 |
+
connector_sync.sync_wake_event().set()
|
| 902 |
+
except Exception:
|
| 903 |
+
result["warning"] = "Amazon cleanup is pending. Retry removal to wake the cleanup worker."
|
| 904 |
+
return result
|
| 905 |
+
|
| 906 |
+
|
| 907 |
def _write_revocation_warning(rt, entry_id, warning=None):
|
| 908 |
"""Prune expired consent on every write; failed persistence requires explicit confirmation."""
|
| 909 |
now = int(time.time())
|
|
|
|
| 1014 |
raise err(400, "bad_initial_selection", str(exc))
|
| 1015 |
except Exception:
|
| 1016 |
raise err(503, "store_unavailable", "The database choices were not saved. Try again.")
|
| 1017 |
+
existing_odoo = etype == "odoo" and kc.resolve_connection_id(session.runtime, "odoo") is not None
|
| 1018 |
try:
|
| 1019 |
row = kc.add_entry(session.runtime, body.get("label"), body.get("type"),
|
| 1020 |
body.get("fields"), session.uname)
|
|
|
|
| 1036 |
# of a side bucket is to publish a credential, not to hide one. So the second write is not
|
| 1037 |
# best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was
|
| 1038 |
# stored. `business` needs no row at all, so this branch is the only one that can be partial.
|
| 1039 |
+
# The first replacement key resumes frozen data. Adding a sibling preserves the existing freeze.
|
| 1040 |
+
if etype == "odoo" and not existing_odoo:
|
|
|
|
|
|
|
| 1041 |
try:
|
| 1042 |
_rel_reconnect(session.runtime)
|
| 1043 |
except Exception: # noqa: BLE001
|
|
|
|
| 1156 |
# another person's personal entry's 404, but ids have 12 random hex characters and are not
|
| 1157 |
# enumerable. A 404 on a lost response's retry would break this policy path's idempotence.
|
| 1158 |
if not any(entry["id"] == str(entry_id) for entry in entries):
|
| 1159 |
+
import amazon_relational
|
| 1160 |
+
revocations = rt.get(amazon_relational.CONNECTION_REVOCATIONS_KEY) or {}
|
| 1161 |
+
for cid, pending in revocations.items():
|
| 1162 |
+
if (isinstance(pending, dict) and pending.get("state") == "pending"
|
| 1163 |
+
and any(str(entry.get("id")) == str(entry_id)
|
| 1164 |
+
and str(entry.get("connection_id")) == str(cid)
|
| 1165 |
+
for entry in pending.get("entries", []))):
|
| 1166 |
+
if not session.admin:
|
| 1167 |
+
raise err(403, "admin_only", "only an administrator may retry Amazon cleanup")
|
| 1168 |
+
lifecycle = _amazon_deletion_receipt(session)
|
| 1169 |
+
cleanup = _queue_amazon_cleanup(session, cid)
|
| 1170 |
+
return {"ok": True, "alreadyGone": True, **lifecycle, **cleanup}
|
| 1171 |
return {"ok": True, "alreadyGone": True,
|
| 1172 |
"purged": {"tables": [], "rows": 0, "rowsKnown": True,
|
| 1173 |
"refused": [], "failed": []}}
|
|
|
|
| 1205 |
if stored else "The warning could not be saved. "
|
| 1206 |
"Pass confirm=purge to confirm explicitly.")
|
| 1207 |
raise err(409, "confirm_purge",
|
| 1208 |
+
(f"Removing this connection also deletes its rows from {len(tables):,} {kind} {noun}"
|
| 1209 |
+
if row["type"] == "amazon_sp" else
|
| 1210 |
+
f"Removing this connection also deletes {len(tables):,} {kind} {noun}")
|
| 1211 |
+ count + "That data cannot be recovered. " + next_step)
|
| 1212 |
|
| 1213 |
class _Ownership:
|
|
|
|
| 1220 |
return {"entries": {entry["id"]: entry for entry in entries}}
|
| 1221 |
return rt.get(name)
|
| 1222 |
|
| 1223 |
+
# Save Amazon ownership before removal so a restart can resume cleanup without credentials.
|
| 1224 |
+
if row["type"] == "amazon_sp":
|
| 1225 |
+
import amazon_relational
|
| 1226 |
+
try:
|
| 1227 |
+
cid = str(next(entry["connection_id"] for entry in entries
|
| 1228 |
+
if entry["id"] == str(entry_id)))
|
| 1229 |
+
amazon_relational.record_connection_revocation(rt, cid, entries_before=entries)
|
| 1230 |
+
except Exception:
|
| 1231 |
+
raise err(503, "store_unavailable", "Amazon cleanup was not recorded. The credential was kept. Try again.")
|
| 1232 |
+
|
| 1233 |
+
removal = _pause_odoo_removal(session, row) if row["type"] == "odoo" else {}
|
| 1234 |
try:
|
| 1235 |
kc.delete_entry(rt, entry_id)
|
| 1236 |
+
if row["type"] != "amazon_sp":
|
| 1237 |
+
_write_scope(rt, entry_id, DEFAULT_SCOPE, "")
|
| 1238 |
except Exception:
|
| 1239 |
raise err(503, "store_unavailable", "The delete did not land. Try again.")
|
| 1240 |
+
if row["type"] == "amazon_sp":
|
| 1241 |
+
scope_warning = {}
|
| 1242 |
+
try:
|
| 1243 |
+
_write_scope(rt, entry_id, DEFAULT_SCOPE, "")
|
| 1244 |
+
except Exception:
|
| 1245 |
+
scope_warning["warning"] = "The credential was removed, but its scope metadata could not be cleared."
|
| 1246 |
+
_write_revocation_warning(rt, entry_id)
|
| 1247 |
+
lifecycle = _amazon_deletion_receipt(session)
|
| 1248 |
+
cleanup = _queue_amazon_cleanup(session, cid)
|
| 1249 |
+
return {"ok": True, "registry": registered, "footprint": footprint,
|
| 1250 |
+
**scope_warning, **lifecycle, **cleanup}
|
| 1251 |
if not purges:
|
| 1252 |
_write_revocation_warning(rt, entry_id)
|
|
|
|
|
|
|
| 1253 |
if footprint["rowsKnown"]:
|
| 1254 |
+
return {"ok": True, **removal}
|
| 1255 |
# Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes
|
| 1256 |
# that incomplete cleanup explicit instead of returning an invented "nothing to purge".
|
| 1257 |
return {"ok": True, "registry": registered, "footprint": footprint,
|
| 1258 |
"purged": {"tables": [], "rows": 0, "rowsKnown": False,
|
| 1259 |
"refused": [], "failed": []},
|
| 1260 |
+
"residue": _revocation_residue(row["type"]), **removal}
|
| 1261 |
from core import user_tables as ut
|
| 1262 |
try:
|
| 1263 |
result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt)
|
|
|
|
| 1266 |
result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [],
|
| 1267 |
"failed": [{"key": key, "error": type(exc).__name__} for key in tables]}
|
| 1268 |
_write_revocation_warning(rt, entry_id)
|
|
|
|
|
|
|
|
|
|
| 1269 |
return {"ok": True,
|
| 1270 |
"purged": {"tables": result["purged"], "rows": result["rows"],
|
| 1271 |
"rowsKnown": result["rowsKnown"], "refused": result["refused"],
|
| 1272 |
"failed": result["failed"]},
|
| 1273 |
+
"residue": _revocation_residue(row["type"])}
|
| 1274 |
|
| 1275 |
|
| 1276 |
@router.post("/admin/keychain/{entry_id}/test")
|
|
|
|
| 1702 |
return f"the connection now points at the {server_db} database"
|
| 1703 |
|
| 1704 |
|
| 1705 |
+
def _pause_odoo_removal(session, entry):
|
| 1706 |
+
"""Pause the addressed mirror before deletion and block the last key's environment fallback."""
|
| 1707 |
+
rt = session.runtime
|
| 1708 |
+
cid = str(entry.get("connectionId") or entry.get("connection_id") or entry["id"])
|
| 1709 |
+
siblings = [row for row in _kc().list_entries(rt, "odoo") if row["id"] != entry["id"]
|
| 1710 |
+
and entry_scope(rt, row["id"])[0] != "personal"]
|
| 1711 |
+
last = not siblings and entry.get("scope") != "personal"
|
| 1712 |
+
keys = {cid}
|
| 1713 |
+
if last and str(rt.key) == ENV_ODOO_TENANT:
|
| 1714 |
+
keys.add(_ENV_ODOO_FLAG_KEY)
|
| 1715 |
+
|
| 1716 |
+
def pause(cur):
|
| 1717 |
+
cur = dict(cur or {})
|
| 1718 |
+
for key in keys:
|
| 1719 |
+
cur[key] = {**(cur.get(key) or {}), "paused": True}
|
| 1720 |
+
return cur
|
| 1721 |
+
|
| 1722 |
+
try:
|
| 1723 |
+
flags = rt.update(_CONNECTOR_FLAGS_KEY, pause, flush="sync")
|
| 1724 |
+
if not isinstance(flags, dict) or not all(
|
| 1725 |
+
isinstance(flags.get(key), dict) and flags[key].get("paused") is True for key in keys):
|
| 1726 |
+
raise RuntimeError("The Odoo pause was not confirmed.")
|
| 1727 |
+
if last:
|
| 1728 |
+
import datetime as dt
|
| 1729 |
+
|
| 1730 |
+
def freeze(cur):
|
| 1731 |
+
return {**(cur or {}), "frozen": True,
|
| 1732 |
+
"frozenAt": dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S"),
|
| 1733 |
+
"frozenBy": str(session.uname)}
|
| 1734 |
+
|
| 1735 |
+
config = rt.update(_rel().CONFIG_KEY, freeze, flush="sync")
|
| 1736 |
+
if not isinstance(config, dict) or config.get("frozen") is not True:
|
| 1737 |
+
raise RuntimeError("The Odoo freeze was not confirmed.")
|
| 1738 |
+
except Exception:
|
| 1739 |
+
raise err(503, "store_unavailable", "The Odoo pause could not be saved. No key was removed. Try again.") from None
|
| 1740 |
+
return {"pausedMirror": True, "connectionId": cid, "frozen": last or _rel().frozen(rt),
|
| 1741 |
+
"note": "This Odoo connection is paused. Its saved data stays available."}
|
| 1742 |
+
|
| 1743 |
+
|
| 1744 |
@router.post("/admin/connectors/odoo/disconnect")
|
| 1745 |
+
def odoo_disconnect(session: Session = Depends(require_session), connection_id: str | None = None):
|
| 1746 |
"""R10 — remove the credential and FREEZE the grids as static data.
|
| 1747 |
|
| 1748 |
⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the
|
|
|
|
| 1756 |
careful.
|
| 1757 |
"""
|
| 1758 |
_odoo_admin(session)
|
| 1759 |
+
if connection_id is not None:
|
| 1760 |
+
entry = _odoo_entry(session, connection_id)
|
| 1761 |
+
if entry is None:
|
| 1762 |
+
raise err(404, "no_entry", "no such Odoo connection")
|
| 1763 |
+
return delete_key(entry["id"], session)
|
| 1764 |
rel = _rel()
|
| 1765 |
import datetime as _dt
|
| 1766 |
entry = _odoo_entry(session)
|
|
|
|
| 1816 |
|
| 1817 |
paused_mirror = False
|
| 1818 |
if flag_key:
|
| 1819 |
+
pause_keys = {str(flag_key)} | {
|
| 1820 |
+
str(row["connection_id"]) for row in _kc().list_entries(session.runtime, "odoo")}
|
| 1821 |
+
if str(session.runtime.key) == ENV_ODOO_TENANT:
|
| 1822 |
+
pause_keys.add(_ENV_ODOO_FLAG_KEY)
|
| 1823 |
+
|
| 1824 |
def _pause(cur):
|
| 1825 |
cur = cur if isinstance(cur, dict) else {}
|
| 1826 |
+
for key in pause_keys:
|
| 1827 |
+
cur[key] = {**(cur.get(key) or {}), "paused": True}
|
| 1828 |
return cur
|
| 1829 |
try:
|
| 1830 |
flags = session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync")
|
| 1831 |
+
if not isinstance(flags, dict) or not all(
|
| 1832 |
+
isinstance(flags.get(key), dict) and flags[key].get("paused") is True
|
| 1833 |
+
for key in pause_keys):
|
| 1834 |
raise RuntimeError("the Odoo pause was not confirmed")
|
| 1835 |
paused_mirror = True
|
| 1836 |
except Exception: # noqa: BLE001
|
api/routes_nav.py
CHANGED
|
@@ -18,6 +18,7 @@ from datetime import datetime, timezone
|
|
| 18 |
from fastapi import APIRouter, Body, Depends
|
| 19 |
|
| 20 |
from deps import Session, err, perms, require_session
|
|
|
|
| 21 |
import application_registry
|
| 22 |
|
| 23 |
router = APIRouter(prefix="/api/v1")
|
|
@@ -879,15 +880,14 @@ def nav(session: Session = Depends(require_session)):
|
|
| 879 |
#
|
| 880 |
# ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the
|
| 881 |
# de5037f share-grant admission widened `may_open`, so a row's presence now includes
|
| 882 |
-
# databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete`
|
| 883 |
-
#
|
| 884 |
-
#
|
| 885 |
-
# controls — the honest shape (the old code offered rename to users the route 403'd).
|
| 886 |
key = str(p.get("key", ""))
|
| 887 |
if key.startswith("ut_"):
|
| 888 |
_creator = str((_ut_defs.get(key) or {}).get("createdBy") or "")
|
| 889 |
_mine = bool(session.admin or (_creator and _creator == session.uname))
|
| 890 |
-
p["manage"] =
|
| 891 |
# S73-T64: user_tables refuses to delete the activity log, so the rail offers no Delete.
|
| 892 |
p["canDelete"] = _mine and key != "ut_activity"
|
| 893 |
# ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL.
|
|
@@ -1017,44 +1017,14 @@ def nav_opened(body: dict = Body(default=None),
|
|
| 1017 |
return {"key": key, "at": stamp}
|
| 1018 |
|
| 1019 |
|
| 1020 |
-
|
| 1021 |
-
|
| 1022 |
-
session: Session = Depends(require_session)):
|
| 1023 |
-
"""WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide.
|
| 1024 |
-
|
| 1025 |
-
A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the
|
| 1026 |
-
asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so
|
| 1027 |
-
replacing the document whole is what makes a deleted folder stay deleted. This bucket is
|
| 1028 |
-
shared by every admin in the tenant: a wholesale write here means whoever saves last
|
| 1029 |
-
silently erases what the other one named while their tab was open.
|
| 1030 |
-
|
| 1031 |
-
THREE WALLS, all fail-closed:
|
| 1032 |
-
· the SESSION must be able to open the key at all (the same predicate the nav uses, so
|
| 1033 |
-
the rail and this route cannot disagree about what exists);
|
| 1034 |
-
· the WRITE is admin-only — renaming a database is a change every user in the tenant
|
| 1035 |
-
sees, which is the definition of an administrative act here;
|
| 1036 |
-
· `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry
|
| 1037 |
-
literals: honouring an override would leave this payload and `core/registry.py`
|
| 1038 |
-
calling the same module two different things, and the wave-16 lesson is that the
|
| 1039 |
-
client must not be the place that decides what a payload meant.
|
| 1040 |
-
|
| 1041 |
-
`icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an
|
| 1042 |
-
icon change two independent writes rather than a race between them.
|
| 1043 |
-
"""
|
| 1044 |
body = body if isinstance(body, dict) else {}
|
| 1045 |
key = str(body.get("key") or "").strip()
|
| 1046 |
if not key:
|
| 1047 |
raise err(400, "bad_request", "no database was named")
|
| 1048 |
-
|
| 1049 |
-
|
| 1050 |
-
#
|
| 1051 |
-
# · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what
|
| 1052 |
-
# `user_tables.may_open` already means. A database you made is yours to name; requiring
|
| 1053 |
-
# an admin for that was the C1 consequence B flagged and R14 resolved. `session.require`
|
| 1054 |
-
# is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key,
|
| 1055 |
-
# because a user table is not a module. Same split `/nav/schema/{key}` makes.
|
| 1056 |
-
# · everything else — admin only, and icon only. There is no owner of `customer_data` to
|
| 1057 |
-
# defer to, and its label is a compiled registry literal (refused below regardless).
|
| 1058 |
if key.startswith("ut_"):
|
| 1059 |
import core.user_tables as user_tables
|
| 1060 |
# This write edits metadata, never a record. The table wall therefore needs its
|
|
@@ -1065,9 +1035,6 @@ def save_nav_meta(body: dict = Body(default=None),
|
|
| 1065 |
key, session.uname, session.admin, st=definitions):
|
| 1066 |
raise err(403, "forbidden", "that database belongs to another user")
|
| 1067 |
else:
|
| 1068 |
-
if not session.admin:
|
| 1069 |
-
raise err(403, "forbidden",
|
| 1070 |
-
"only an administrator can change a built-in database's icon")
|
| 1071 |
session.require(key)
|
| 1072 |
|
| 1073 |
patch = {}
|
|
@@ -1090,6 +1057,14 @@ def save_nav_meta(body: dict = Body(default=None),
|
|
| 1090 |
patch["name"] = name
|
| 1091 |
if not patch:
|
| 1092 |
raise err(400, "bad_request", "nothing to change")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1093 |
|
| 1094 |
if not session.runtime.available():
|
| 1095 |
raise err(503, "store_unavailable",
|
|
@@ -1114,6 +1089,8 @@ def save_nav_meta(body: dict = Body(default=None),
|
|
| 1114 |
|
| 1115 |
try:
|
| 1116 |
session.runtime.update(_NAV_META_KEY, _up)
|
|
|
|
|
|
|
| 1117 |
except Exception:
|
| 1118 |
raise err(503, "store_unavailable",
|
| 1119 |
"the change was not saved: the store refused the write.")
|
|
|
|
| 18 |
from fastapi import APIRouter, Body, Depends
|
| 19 |
|
| 20 |
from deps import Session, err, perms, require_session
|
| 21 |
+
from core import view_audit
|
| 22 |
import application_registry
|
| 23 |
|
| 24 |
router = APIRouter(prefix="/api/v1")
|
|
|
|
| 880 |
#
|
| 881 |
# ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the
|
| 882 |
# de5037f share-grant admission widened `may_open`, so a row's presence now includes
|
| 883 |
+
# databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete`
|
| 884 |
+
# use separate walls: metadata requires an administrator, while deletion also allows
|
| 885 |
+
# the creator. A grantee sees the row without either control.
|
|
|
|
| 886 |
key = str(p.get("key", ""))
|
| 887 |
if key.startswith("ut_"):
|
| 888 |
_creator = str((_ut_defs.get(key) or {}).get("createdBy") or "")
|
| 889 |
_mine = bool(session.admin or (_creator and _creator == session.uname))
|
| 890 |
+
p["manage"] = bool(session.admin)
|
| 891 |
# S73-T64: user_tables refuses to delete the activity log, so the rail offers no Delete.
|
| 892 |
p["canDelete"] = _mine and key != "ut_activity"
|
| 893 |
# ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL.
|
|
|
|
| 1017 |
return {"key": key, "at": stamp}
|
| 1018 |
|
| 1019 |
|
| 1020 |
+
def _nav_meta_write_patch(body, session):
|
| 1021 |
+
"""Validate tenant-wide database metadata through the app's administration wall."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1022 |
body = body if isinstance(body, dict) else {}
|
| 1023 |
key = str(body.get("key") or "").strip()
|
| 1024 |
if not key:
|
| 1025 |
raise err(400, "bad_request", "no database was named")
|
| 1026 |
+
if not session.admin:
|
| 1027 |
+
raise err(403, "forbidden", "only an administrator can change a database's name or icon")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1028 |
if key.startswith("ut_"):
|
| 1029 |
import core.user_tables as user_tables
|
| 1030 |
# This write edits metadata, never a record. The table wall therefore needs its
|
|
|
|
| 1035 |
key, session.uname, session.admin, st=definitions):
|
| 1036 |
raise err(403, "forbidden", "that database belongs to another user")
|
| 1037 |
else:
|
|
|
|
|
|
|
|
|
|
| 1038 |
session.require(key)
|
| 1039 |
|
| 1040 |
patch = {}
|
|
|
|
| 1057 |
patch["name"] = name
|
| 1058 |
if not patch:
|
| 1059 |
raise err(400, "bad_request", "nothing to change")
|
| 1060 |
+
return key, patch
|
| 1061 |
+
|
| 1062 |
+
|
| 1063 |
+
@router.post("/nav/meta")
|
| 1064 |
+
def save_nav_meta(body: dict = Body(default=None),
|
| 1065 |
+
session: Session = Depends(require_session)):
|
| 1066 |
+
"""Patch supplied tenant-wide name/icon fields, with an explicit null icon clearing its override."""
|
| 1067 |
+
key, patch = _nav_meta_write_patch(body, session)
|
| 1068 |
|
| 1069 |
if not session.runtime.available():
|
| 1070 |
raise err(503, "store_unavailable",
|
|
|
|
| 1089 |
|
| 1090 |
try:
|
| 1091 |
session.runtime.update(_NAV_META_KEY, _up)
|
| 1092 |
+
except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable):
|
| 1093 |
+
raise
|
| 1094 |
except Exception:
|
| 1095 |
raise err(503, "store_unavailable",
|
| 1096 |
"the change was not saved: the store refused the write.")
|
api/routes_odoo_tables.py
CHANGED
|
@@ -25,6 +25,8 @@ import json
|
|
| 25 |
import math
|
| 26 |
import threading
|
| 27 |
import time
|
|
|
|
|
|
|
| 28 |
from contextlib import contextmanager
|
| 29 |
|
| 30 |
from fastapi import Body, Depends, HTTPException
|
|
@@ -108,7 +110,7 @@ def _table_cursor(rt, table_key=None):
|
|
| 108 |
`rt=None` is the unguarded legacy caller `whole_pool`/`population` already documents.
|
| 109 |
"""
|
| 110 |
from harness import datastore
|
| 111 |
-
if str(table_key or "").startswith("ut_amzn_"):
|
| 112 |
if rt is not None:
|
| 113 |
rt.assert_datastore_matches()
|
| 114 |
return datastore.ro_cursor()
|
|
@@ -219,8 +221,9 @@ def odoo_tables_status(brief: bool = Query(False),
|
|
| 219 |
definitions = ut.all_defs(st=session.runtime)
|
| 220 |
lent = ut.lend(session.runtime, **{ut.STORE_KEY: definitions})
|
| 221 |
for key, definition in definitions.items():
|
| 222 |
-
if (
|
| 223 |
-
and definition.get(
|
|
|
|
| 224 |
and perm_scope.may_read(session.user, key, st=lent)):
|
| 225 |
tables[key] = {"readThrough": True}
|
| 226 |
return {"ok": True, "tables": tables, "catchUp": catch_up,
|
|
@@ -1260,6 +1263,11 @@ def _odoo_sources():
|
|
| 1260 |
# S72-T43 / R107: the audited Odoo columns. Their presets are declared in `odoo_relational`.
|
| 1261 |
GRID_SOURCES = rel.merge_audit_cols(GRID_SOURCES)
|
| 1262 |
for key, spec in GRID_SOURCES.items():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1263 |
if key in rel.READ_THROUGH_KEYS:
|
| 1264 |
spec["mirror_only"] = True
|
| 1265 |
return GRID_SOURCES
|
|
@@ -1291,6 +1299,69 @@ def _amazon_sources():
|
|
| 1291 |
return {}
|
| 1292 |
|
| 1293 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1294 |
def require_amazon_mirror(table_key, session, definition=None, st=None):
|
| 1295 |
"""Admit one Amazon grid and return its binding, or None for a stored snapshot."""
|
| 1296 |
if not str(table_key).startswith("ut_amzn_"):
|
|
@@ -1317,6 +1388,7 @@ def require_amazon_mirror(table_key, session, definition=None, st=None):
|
|
| 1317 |
|
| 1318 |
register_source_provider(_odoo_sources)
|
| 1319 |
register_source_provider(_amazon_sources)
|
|
|
|
| 1320 |
register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned)
|
| 1321 |
|
| 1322 |
|
|
@@ -1543,6 +1615,10 @@ def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_looku
|
|
| 1543 |
"This Amazon database could not be resolved. Retry the request.") from exc
|
| 1544 |
if spec is not None and spec.get("missing"):
|
| 1545 |
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1546 |
return spec
|
| 1547 |
|
| 1548 |
|
|
@@ -1573,6 +1649,8 @@ def _source_for(cur, table_key, *, allow_lookups=True):
|
|
| 1573 |
# table and turns Products ordered into the wrong 404.
|
| 1574 |
if module_cache_source(key) is not None:
|
| 1575 |
return None
|
|
|
|
|
|
|
| 1576 |
if key.startswith("ut_amzn_"):
|
| 1577 |
import amazon_relational as amazon
|
| 1578 |
if key in amazon.SOURCE:
|
|
@@ -1632,6 +1710,8 @@ def _source_for(cur, table_key, *, allow_lookups=True):
|
|
| 1632 |
excluded = ", ".join(str(i) for i in ids) or "-1"
|
| 1633 |
cols = {}
|
| 1634 |
for key, (expr, cast) in spec["cols"].items():
|
|
|
|
|
|
|
| 1635 |
cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr,
|
| 1636 |
have_by_alias), cast)
|
| 1637 |
if str(table_key).startswith("ut_odoo_") and spec.get("table") and not spec.get("from_sql"):
|
|
@@ -1667,7 +1747,9 @@ FILTERED_SELECTION_SECONDS = 5 * 60
|
|
| 1667 |
#: `filter_sql` speaks the CLIENT's type vocabulary (`types.ts isNumericType`), and two of our
|
| 1668 |
#: field kinds are spelled differently there. Mapped in one place so the pushdown and the grid
|
| 1669 |
#: cannot disagree about whether a column is text or a number.
|
| 1670 |
-
_FILTER_TYPE = {"select": "status", "
|
|
|
|
|
|
|
| 1671 |
|
| 1672 |
|
| 1673 |
def _if_references(defn):
|
|
@@ -1743,6 +1825,8 @@ def _filter_columns(spec, fields):
|
|
| 1743 |
by_key = {f["key"]: f for f in fields}
|
| 1744 |
out = {}
|
| 1745 |
for key, (expr, _cast) in spec["cols"].items():
|
|
|
|
|
|
|
| 1746 |
ftype = str((by_key.get(key) or {}).get("type") or "text")
|
| 1747 |
out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False}
|
| 1748 |
if key == "customer_type" and (by_key.get(key) or {}).get("source") == "odoo":
|
|
@@ -1864,6 +1948,12 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=(
|
|
| 1864 |
trail = (*trail, table_key)
|
| 1865 |
handle = (st if isinstance(st, _ut()._Lent) and getattr(st, "schema_only", False)
|
| 1866 |
else _ut().lend_defs(st, schema_only=True))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1867 |
declared = {f["key"]: f for f in fields if isinstance(f, dict) and f.get("key")}
|
| 1868 |
columns = _filter_columns(spec, fields)
|
| 1869 |
guards, resolved, targets = [], {}, {}
|
|
@@ -1894,7 +1984,10 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=(
|
|
| 1894 |
allow_lookups=_may_open(session, amazon_relational.LISTINGS, st=handle))
|
| 1895 |
if not definition or target_spec is None or target_spec.get("missing"):
|
| 1896 |
refuse(field)
|
| 1897 |
-
target_fields =
|
|
|
|
|
|
|
|
|
|
| 1898 |
hidden = ps.hidden_keys(session.user, key, target_fields, st=handle)
|
| 1899 |
tree = ((ps.entry(session.user, key) or {}).get("filter")
|
| 1900 |
if ps.row_scope_applies(session.user, key) else None)
|
|
@@ -2031,8 +2124,16 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=(
|
|
| 2031 |
# Relation cells compare string identities. Blank source keys never match.
|
| 2032 |
match = (f"NULLIF(CAST({left['sql']} AS VARCHAR), '') = "
|
| 2033 |
f"NULLIF(CAST({right['sql']} AS VARCHAR), '')")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2034 |
allowed = f'{alias}."__wall_allowed"'
|
| 2035 |
-
good = (f"({left_ok}) AND NOT EXISTS (SELECT 1 FROM {source} WHERE {match} "
|
| 2036 |
f"AND NOT COALESCE(({allowed}) AND ({right_ok}), FALSE))")
|
| 2037 |
join = f"{match} AND {allowed} AND ({right_ok})"
|
| 2038 |
pid = target_context["id"]
|
|
@@ -2545,7 +2646,7 @@ class _RelationRows:
|
|
| 2545 |
|
| 2546 |
def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None,
|
| 2547 |
traversal_of=None, st=None, rev=..., normalize_match=False,
|
| 2548 |
-
compiled=None, stream=False, materialised=None): # `...` IS `_UNREAD`
|
| 2549 |
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
|
| 2550 |
|
| 2551 |
Join columns may be hidden on the wire. Only identities leave the resolver; the native
|
|
@@ -2571,6 +2672,8 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2571 |
_compiled_read(table, session, traversal_intermediate=True, st=st)
|
| 2572 |
if traversal_of is not None else _compiled_read(table, session, st=st))
|
| 2573 |
spec = read["spec"]
|
|
|
|
|
|
|
| 2574 |
bindings = dict(spec["cols"], pid=(spec["id"], int))
|
| 2575 |
if columns - set(bindings):
|
| 2576 |
raise err(409, "relation_target_unavailable", "A required relation column is unavailable.")
|
|
@@ -2580,6 +2683,12 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2580 |
if values is None:
|
| 2581 |
continue
|
| 2582 |
expr, cast = bindings[key]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2583 |
if normalize_match and key != "pid":
|
| 2584 |
expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))"
|
| 2585 |
cast = lambda value: str(value).strip().lower()
|
|
@@ -2659,12 +2768,16 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2659 |
normalize = ((lambda value: str(value).strip().lower()) if normalize_match
|
| 2660 |
else str)
|
| 2661 |
wanted = {normalize(value) for value in values}
|
| 2662 |
-
rows = [row for row in rows if normalize(
|
|
|
|
|
|
|
| 2663 |
if matches is not None:
|
| 2664 |
normalize = (lambda value: str(value).strip().lower()) if normalize_match else str
|
| 2665 |
wanted = {key: {normalize(value) for value in values} for key, values in matches.items()}
|
| 2666 |
rows = [row for row in rows if any(normalize(row.get(key)) in values
|
| 2667 |
for key, values in wanted.items())]
|
|
|
|
|
|
|
| 2668 |
return [{key: row.get(key) for key in columns} for row in rows]
|
| 2669 |
|
| 2670 |
|
|
@@ -2679,6 +2792,7 @@ def _relation_leg_membership(session, source, source_ids, entry, cur, *, source_
|
|
| 2679 |
"""Follow a scoped leg with SQL operands at every population-sized boundary."""
|
| 2680 |
import core.relations as relations
|
| 2681 |
target, link, via = str(entry['target']), entry.get('link') or {}, entry.get('via') or {}
|
|
|
|
| 2682 |
observed = {'rows': 0, 'complete': True, 'available': False}
|
| 2683 |
def source_values():
|
| 2684 |
for row in _relation_rows(session, source, [source_col], ids=source_ids, stream=True):
|
|
@@ -2689,7 +2803,11 @@ def _relation_leg_membership(session, source, source_ids, entry, cur, *, source_
|
|
| 2689 |
if (manual and not reverse) or (inverse and reverse):
|
| 2690 |
yield from _relation_cell_ids(row[source_col])
|
| 2691 |
else:
|
| 2692 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2693 |
values = _membership_relation(cur, source_values(), normalize=str, sql_type='VARCHAR')
|
| 2694 |
if source_col != relations.IDENTITY_COL and not observed['available']:
|
| 2695 |
if (source.startswith('ut_amzn_') and target.startswith('ut_amzn_')
|
|
@@ -2711,7 +2829,7 @@ def _relation_leg_membership(session, source, source_ids, entry, cur, *, source_
|
|
| 2711 |
rows = _relation_rows(session, target, [target_col], stream=True)
|
| 2712 |
return _relation_cell_membership(cur, rows, target_col, values)
|
| 2713 |
rows = _relation_rows(session, target, [target_col], match=(target_col, values),
|
| 2714 |
-
where=entry.get('where'), stream=True)
|
| 2715 |
return _membership_relation(cur, (row['pid'] for row in rows))
|
| 2716 |
|
| 2717 |
|
|
@@ -2737,7 +2855,10 @@ def _relation_leg_ids(session, source, source_ids, entry, membership_cur=None):
|
|
| 2737 |
return _relation_leg_membership(session, source, source_ids, entry, membership_cur,
|
| 2738 |
source_col=source_col, target_col=target_col, manual=manual, inverse=inverse, reverse=reverse)
|
| 2739 |
source_rows = _relation_rows(session, source, [source_col], ids=source_ids)
|
| 2740 |
-
|
|
|
|
|
|
|
|
|
|
| 2741 |
# A column-keyed edge cannot fall back to the wire record id. In particular,
|
| 2742 |
# product_data.pid is a stable hash while sale_order_line.product_id is an Odoo
|
| 2743 |
# identity. Treating a missing/hidden/blank source value as an empty match would
|
|
@@ -2764,7 +2885,7 @@ def _relation_leg_ids(session, source, source_ids, entry, membership_cur=None):
|
|
| 2764 |
rows = (row for row in rows if _relation_cell_ids(row.get(target_col)) & values)
|
| 2765 |
else:
|
| 2766 |
rows = _relation_rows(session, target, [target_col], match=(target_col, values),
|
| 2767 |
-
where=entry.get("where"), stream=True)
|
| 2768 |
return {int(row["pid"]) for row in rows}
|
| 2769 |
|
| 2770 |
|
|
@@ -3673,6 +3794,12 @@ def _may_see_view(session, vid, store):
|
|
| 3673 |
return shares.may_see("view", vid, session.uname, is_admin=session.admin, st=store.st)
|
| 3674 |
|
| 3675 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3676 |
def resolved_fields(table_key, defn):
|
| 3677 |
"""Resolve presets and authored fields for the same connection as the rows door."""
|
| 3678 |
rel = _rel()
|
|
@@ -3691,11 +3818,13 @@ def resolved_fields(table_key, defn):
|
|
| 3691 |
if not connected:
|
| 3692 |
raise err(404, "not_connected", "that database is not a connected grid")
|
| 3693 |
fields = (defn or {}).get("fields") or []
|
| 3694 |
-
if not fields:
|
| 3695 |
raise err(409, "schema_not_loaded", "This connected database has no saved fields yet.")
|
| 3696 |
declared = {field["key"]: field for field in fields if field.get("key")}
|
| 3697 |
declared.update({field["key"]: field for field in (defn or {}).get("fields") or []
|
| 3698 |
if field.get("key")})
|
|
|
|
|
|
|
| 3699 |
return list(declared.values())
|
| 3700 |
|
| 3701 |
|
|
@@ -3783,7 +3912,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 3783 |
# try, not inside it.
|
| 3784 |
try:
|
| 3785 |
cur, mismatch = (_mirror_cur(session, table_key)
|
| 3786 |
-
if (table_key.startswith("ut_amzn_")
|
| 3787 |
or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY)
|
| 3788 |
else _mirror_cur(session))
|
| 3789 |
except RuntimeError as e:
|
|
@@ -3818,6 +3947,8 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 3818 |
"or ask for a read-through binding to be added for it")
|
| 3819 |
|
| 3820 |
st = st if st is not None else session.runtime
|
|
|
|
|
|
|
| 3821 |
shared_fields = _so().fields(table_key, st=st)
|
| 3822 |
if shared_fields:
|
| 3823 |
declared = {field["key"]: field for field in fields}
|
|
@@ -3869,7 +4000,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 3869 |
_wall_nodes, _wall_conj = _fe().tree_parts(_wall_tree)
|
| 3870 |
if _wall_nodes:
|
| 3871 |
try:
|
| 3872 |
-
if str(table_key).startswith(("ut_odoo_", "ut_amzn_")):
|
| 3873 |
_wall_cols, _wall_guards = _read_wall_columns(
|
| 3874 |
table_key, session, cur, spec, fields, _wall_nodes,
|
| 3875 |
st if st is not None else session.runtime)
|
|
@@ -4242,8 +4373,103 @@ def _restrict_row_ids(read, ids):
|
|
| 4242 |
return narrowed
|
| 4243 |
|
| 4244 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4245 |
def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
|
| 4246 |
groupBy=None, groupKey=None, ids=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4247 |
from harness import datastore
|
| 4248 |
|
| 4249 |
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
|
|
@@ -4488,6 +4714,8 @@ def odoo_table_rows(table_key: str,
|
|
| 4488 |
# as /related pins. No unscoped SQL subquery can leak a hidden target's id.
|
| 4489 |
amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets,
|
| 4490 |
max_bytes=LINK_VALUE_MAX_BYTES, capacity_refusal=_link_capacity_refusal)
|
|
|
|
|
|
|
| 4491 |
overlays = {}
|
| 4492 |
|
| 4493 |
# ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
|
|
@@ -4536,12 +4764,23 @@ def odoo_table_rows(table_key: str,
|
|
| 4536 |
cells.update(overlays.get(str(row["pid"]), {}))
|
| 4537 |
cells.update({key: row.get(key, "") for key in if_keys if key not in _hidden})
|
| 4538 |
rows.extend(aios_grid.rows_from_pool([row], fields, {str(row["pid"]): cells}))
|
|
|
|
|
|
|
|
|
|
| 4539 |
|
| 4540 |
unfiltered = win["total"] if where == base_where else datastore.count_rows(
|
| 4541 |
where=base_where, params=tuple(wall_params), cur=cur, **frm)
|
| 4542 |
|
| 4543 |
from core import connector_spec
|
| 4544 |
rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4545 |
return {"ok": True, "fields": fields, "rows": rows,
|
| 4546 |
# ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`.
|
| 4547 |
"total": win["total"], "totalUnfiltered": unfiltered,
|
|
@@ -4610,6 +4849,52 @@ def _sql_link_cells(session, target, on, values, *, st, single, remaining, read=
|
|
| 4610 |
lease.close()
|
| 4611 |
|
| 4612 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4613 |
def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...):
|
| 4614 |
"""Resolve page link cells through scoped SQL and a complete-value byte budget."""
|
| 4615 |
from routes_grid import _may_open
|
|
|
|
| 25 |
import math
|
| 26 |
import threading
|
| 27 |
import time
|
| 28 |
+
from collections import OrderedDict
|
| 29 |
+
from copy import deepcopy
|
| 30 |
from contextlib import contextmanager
|
| 31 |
|
| 32 |
from fastapi import Body, Depends, HTTPException
|
|
|
|
| 110 |
`rt=None` is the unguarded legacy caller `whole_pool`/`population` already documents.
|
| 111 |
"""
|
| 112 |
from harness import datastore
|
| 113 |
+
if str(table_key or "").startswith(("ut_amzn_", "ut_meta_")):
|
| 114 |
if rt is not None:
|
| 115 |
rt.assert_datastore_matches()
|
| 116 |
return datastore.ro_cursor()
|
|
|
|
| 221 |
definitions = ut.all_defs(st=session.runtime)
|
| 222 |
lent = ut.lend(session.runtime, **{ut.STORE_KEY: definitions})
|
| 223 |
for key, definition in definitions.items():
|
| 224 |
+
if (isinstance(definition, dict)
|
| 225 |
+
and ((str(key).startswith('ut_amzn_') and definition.get('readThrough') is True)
|
| 226 |
+
or (str(key).startswith('ut_meta_') and key in sources))
|
| 227 |
and perm_scope.may_read(session.user, key, st=lent)):
|
| 228 |
tables[key] = {"readThrough": True}
|
| 229 |
return {"ok": True, "tables": tables, "catchUp": catch_up,
|
|
|
|
| 1263 |
# S72-T43 / R107: the audited Odoo columns. Their presets are declared in `odoo_relational`.
|
| 1264 |
GRID_SOURCES = rel.merge_audit_cols(GRID_SOURCES)
|
| 1265 |
for key, spec in GRID_SOURCES.items():
|
| 1266 |
+
owner = spec['id'].rsplit('.', 1)[0] + '.' if '.' in spec['id'] else ''
|
| 1267 |
+
spec['cols']['Store'] = (owner + 'Store', _s)
|
| 1268 |
+
spec['cols']['_connection_id'] = (owner + '_connection_id', _s)
|
| 1269 |
+
if 'odoo_id' in spec['cols']:
|
| 1270 |
+
spec['cols']['odoo_id'] = (owner + '_source_id', spec['cols']['odoo_id'][1])
|
| 1271 |
if key in rel.READ_THROUGH_KEYS:
|
| 1272 |
spec["mirror_only"] = True
|
| 1273 |
return GRID_SOURCES
|
|
|
|
| 1299 |
return {}
|
| 1300 |
|
| 1301 |
|
| 1302 |
+
def _meta_source(cur, key):
|
| 1303 |
+
"""Bind Meta's current mirror schema and the native connection-aware row identity."""
|
| 1304 |
+
import meta_relational as meta
|
| 1305 |
+
from core import connector_spec
|
| 1306 |
+
|
| 1307 |
+
if key not in meta.SOURCE:
|
| 1308 |
+
return None
|
| 1309 |
+
table = meta.SOURCE[key]
|
| 1310 |
+
columns = meta.mirror_columns(cur, table)
|
| 1311 |
+
if not columns:
|
| 1312 |
+
return None
|
| 1313 |
+
quote = lambda name: '"' + name.replace('"', '""') + '"'
|
| 1314 |
+
names = {name for name, _kind in columns}
|
| 1315 |
+
cols = {name: (quote(name), lambda value: '' if value is None else value)
|
| 1316 |
+
for name, _kind in columns}
|
| 1317 |
+
if '_row_key' in names:
|
| 1318 |
+
payload = "'[\"" + key + "\",[' || to_json(CAST(\"_row_key\" AS VARCHAR)) || ']]'"
|
| 1319 |
+
identity = f"CAST('0x' || substr(sha256({payload}), 1, 13) AS BIGINT)"
|
| 1320 |
+
elif '_connection_id' in names:
|
| 1321 |
+
raise err(409, 'meta_identity_unavailable', 'This Meta mirror needs connection-aware row identities.')
|
| 1322 |
+
elif meta.IDENTITY[key] == ('id',):
|
| 1323 |
+
identity = 'TRY_CAST("id" AS BIGINT)'
|
| 1324 |
+
else:
|
| 1325 |
+
values = ', '.join(f'COALESCE(CAST({quote(name)} AS VARCHAR), \'\')' for name in meta.IDENTITY[key])
|
| 1326 |
+
payload = "'[\"" + key + "\",' || to_json(list_value(" + values + ")) || ']'"
|
| 1327 |
+
identity = f"CAST('0x' || substr(sha256({payload}), 1, 13) AS BIGINT)"
|
| 1328 |
+
live = {target: {name for name, _kind in meta.mirror_columns(cur, source)}
|
| 1329 |
+
for target, source in meta.SOURCE.items()}
|
| 1330 |
+
links = connector_spec.system_link_fields('meta', live).get(key, [])
|
| 1331 |
+
scoped = {'account_id': '_account_key', 'campaign_id': '_campaign_key',
|
| 1332 |
+
'adset_id': '_adset_key', 'ad_id': '_ad_key', 'id': '_row_key'}
|
| 1333 |
+
for field in links:
|
| 1334 |
+
bag = field['link']
|
| 1335 |
+
target = bag['table']
|
| 1336 |
+
if '_connection_id' in names or '_connection_id' in live[target]:
|
| 1337 |
+
left, right = scoped.get(bag['from']), scoped.get(bag['on'])
|
| 1338 |
+
if left not in names or right not in live[target]:
|
| 1339 |
+
raise err(409, 'meta_identity_unavailable', 'This Meta relation needs connection-aware identities.')
|
| 1340 |
+
bag.update({'from': left, 'on': right})
|
| 1341 |
+
return {'table': table, 'id': identity, 'page_key': table + '.rowid', 'where': '',
|
| 1342 |
+
'cols': cols, 'mirror_only': True, '_fields': meta._fields_for(cur, key) + links}
|
| 1343 |
+
|
| 1344 |
+
|
| 1345 |
+
def _meta_sources():
|
| 1346 |
+
"""Declare Meta grids without retaining another tenant's mirror schema."""
|
| 1347 |
+
import meta_relational as meta
|
| 1348 |
+
return {key: {'table': table, 'id': 'id', 'where': '', 'cols': {}, 'mirror_only': True}
|
| 1349 |
+
for key, table in meta.SOURCE.items()}
|
| 1350 |
+
|
| 1351 |
+
|
| 1352 |
+
def meta_source_fields(spec, fields):
|
| 1353 |
+
"""Merge current Meta-owned columns while retaining authored field settings."""
|
| 1354 |
+
import meta_relational as meta
|
| 1355 |
+
declared = {field['key']: field for field in spec.get('_fields') or []}
|
| 1356 |
+
for field in fields:
|
| 1357 |
+
if field['key'] not in declared or not meta._machine_preset(field):
|
| 1358 |
+
declared[field['key']] = field
|
| 1359 |
+
else:
|
| 1360 |
+
declared[field['key']] = _ut().keep_user_edits(field, dict(
|
| 1361 |
+
declared[field['key']], automation=field.get('automation')))
|
| 1362 |
+
return list(declared.values())
|
| 1363 |
+
|
| 1364 |
+
|
| 1365 |
def require_amazon_mirror(table_key, session, definition=None, st=None):
|
| 1366 |
"""Admit one Amazon grid and return its binding, or None for a stored snapshot."""
|
| 1367 |
if not str(table_key).startswith("ut_amzn_"):
|
|
|
|
| 1388 |
|
| 1389 |
register_source_provider(_odoo_sources)
|
| 1390 |
register_source_provider(_amazon_sources)
|
| 1391 |
+
register_source_provider(_meta_sources)
|
| 1392 |
register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned)
|
| 1393 |
|
| 1394 |
|
|
|
|
| 1615 |
"This Amazon database could not be resolved. Retry the request.") from exc
|
| 1616 |
if spec is not None and spec.get("missing"):
|
| 1617 |
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
| 1618 |
+
if spec is not None and '_connection_id' in {
|
| 1619 |
+
name for name, _kind in amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)}:
|
| 1620 |
+
spec = {**spec, 'cols': {**spec['cols'], '_connection_id': (
|
| 1621 |
+
'"_connection_id"', lambda value: '' if value is None else str(value))}}
|
| 1622 |
return spec
|
| 1623 |
|
| 1624 |
|
|
|
|
| 1649 |
# table and turns Products ordered into the wrong 404.
|
| 1650 |
if module_cache_source(key) is not None:
|
| 1651 |
return None
|
| 1652 |
+
if key.startswith('ut_meta_'):
|
| 1653 |
+
return _meta_source(cur, key)
|
| 1654 |
if key.startswith("ut_amzn_"):
|
| 1655 |
import amazon_relational as amazon
|
| 1656 |
if key in amazon.SOURCE:
|
|
|
|
| 1710 |
excluded = ", ".join(str(i) for i in ids) or "-1"
|
| 1711 |
cols = {}
|
| 1712 |
for key, (expr, cast) in spec["cols"].items():
|
| 1713 |
+
if key == 'odoo_id' and expr.endswith('_source_id') and _degrade(expr, have_by_alias) == 'NULL':
|
| 1714 |
+
expr = spec['id']
|
| 1715 |
cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr,
|
| 1716 |
have_by_alias), cast)
|
| 1717 |
if str(table_key).startswith("ut_odoo_") and spec.get("table") and not spec.get("from_sql"):
|
|
|
|
| 1747 |
#: `filter_sql` speaks the CLIENT's type vocabulary (`types.ts isNumericType`), and two of our
|
| 1748 |
#: field kinds are spelled differently there. Mapped in one place so the pushdown and the grid
|
| 1749 |
#: cannot disagree about whether a column is text or a number.
|
| 1750 |
+
_FILTER_TYPE = {"select": "status", "float": "int"}
|
| 1751 |
+
_SOURCE_PRIVATE = frozenset({'_connection_id', '_row_key', '_source_id', '_account_key',
|
| 1752 |
+
'_campaign_key', '_adset_key', '_ad_key'})
|
| 1753 |
|
| 1754 |
|
| 1755 |
def _if_references(defn):
|
|
|
|
| 1825 |
by_key = {f["key"]: f for f in fields}
|
| 1826 |
out = {}
|
| 1827 |
for key, (expr, _cast) in spec["cols"].items():
|
| 1828 |
+
if key in _SOURCE_PRIVATE:
|
| 1829 |
+
continue
|
| 1830 |
ftype = str((by_key.get(key) or {}).get("type") or "text")
|
| 1831 |
out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False}
|
| 1832 |
if key == "customer_type" and (by_key.get(key) or {}).get("source") == "odoo":
|
|
|
|
| 1948 |
trail = (*trail, table_key)
|
| 1949 |
handle = (st if isinstance(st, _ut()._Lent) and getattr(st, "schema_only", False)
|
| 1950 |
else _ut().lend_defs(st, schema_only=True))
|
| 1951 |
+
# Source declarations can advance before the saved schema refreshes.
|
| 1952 |
+
definitions = dict(handle.get(_ut().STORE_KEY) or {})
|
| 1953 |
+
if isinstance(definitions.get(table_key), dict):
|
| 1954 |
+
definitions[table_key] = dict(definitions[table_key], fields=fields)
|
| 1955 |
+
handle = _ut().lend(handle, **{_ut().STORE_KEY: definitions})
|
| 1956 |
+
handle.schema_only = True
|
| 1957 |
declared = {f["key"]: f for f in fields if isinstance(f, dict) and f.get("key")}
|
| 1958 |
columns = _filter_columns(spec, fields)
|
| 1959 |
guards, resolved, targets = [], {}, {}
|
|
|
|
| 1984 |
allow_lookups=_may_open(session, amazon_relational.LISTINGS, st=handle))
|
| 1985 |
if not definition or target_spec is None or target_spec.get("missing"):
|
| 1986 |
refuse(field)
|
| 1987 |
+
target_fields = resolved_fields(key, definition)
|
| 1988 |
+
if target_spec.get('_fields') is not None:
|
| 1989 |
+
target_fields = meta_source_fields(target_spec, target_fields)
|
| 1990 |
+
definitions[key] = dict(definition, fields=target_fields)
|
| 1991 |
hidden = ps.hidden_keys(session.user, key, target_fields, st=handle)
|
| 1992 |
tree = ((ps.entry(session.user, key) or {}).get("filter")
|
| 1993 |
if ps.row_scope_applies(session.user, key) else None)
|
|
|
|
| 2124 |
# Relation cells compare string identities. Blank source keys never match.
|
| 2125 |
match = (f"NULLIF(CAST({left['sql']} AS VARCHAR), '') = "
|
| 2126 |
f"NULLIF(CAST({right['sql']} AS VARCHAR), '')")
|
| 2127 |
+
connection_ok = 'TRUE'
|
| 2128 |
+
if bag.get('connectionScope') == '_connection_id':
|
| 2129 |
+
source_connection = context['cols'].get('_connection_id', ('NULL',))[0]
|
| 2130 |
+
target_connection = target_context['cols'].get('_connection_id', ('NULL',))[0]
|
| 2131 |
+
match += (f" AND NULLIF(CAST({source_connection} AS VARCHAR), '') = "
|
| 2132 |
+
f"NULLIF(CAST({target_connection} AS VARCHAR), '')")
|
| 2133 |
+
connection_ok = (f"NULLIF(CAST({source_connection} AS VARCHAR), '') IS NOT NULL"
|
| 2134 |
+
if target_connection.strip().upper() != 'NULL' else 'FALSE')
|
| 2135 |
allowed = f'{alias}."__wall_allowed"'
|
| 2136 |
+
good = (f"({left_ok}) AND ({connection_ok}) AND NOT EXISTS (SELECT 1 FROM {source} WHERE {match} "
|
| 2137 |
f"AND NOT COALESCE(({allowed}) AND ({right_ok}), FALSE))")
|
| 2138 |
join = f"{match} AND {allowed} AND ({right_ok})"
|
| 2139 |
pid = target_context["id"]
|
|
|
|
| 2646 |
|
| 2647 |
def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None,
|
| 2648 |
traversal_of=None, st=None, rev=..., normalize_match=False,
|
| 2649 |
+
compiled=None, stream=False, materialised=None, connection_scope=None): # `...` IS `_UNREAD`
|
| 2650 |
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
|
| 2651 |
|
| 2652 |
Join columns may be hidden on the wire. Only identities leave the resolver; the native
|
|
|
|
| 2672 |
_compiled_read(table, session, traversal_intermediate=True, st=st)
|
| 2673 |
if traversal_of is not None else _compiled_read(table, session, st=st))
|
| 2674 |
spec = read["spec"]
|
| 2675 |
+
if '_connection_id' in spec['cols']:
|
| 2676 |
+
columns.add('_connection_id')
|
| 2677 |
bindings = dict(spec["cols"], pid=(spec["id"], int))
|
| 2678 |
if columns - set(bindings):
|
| 2679 |
raise err(409, "relation_target_unavailable", "A required relation column is unavailable.")
|
|
|
|
| 2683 |
if values is None:
|
| 2684 |
continue
|
| 2685 |
expr, cast = bindings[key]
|
| 2686 |
+
if connection_scope and key != 'pid':
|
| 2687 |
+
connection = bindings.get(connection_scope)
|
| 2688 |
+
if connection is None:
|
| 2689 |
+
raise err(409, 'relation_target_unavailable', 'This relation needs its account identity.')
|
| 2690 |
+
expr = f'to_json(list_value(CAST({connection[0]} AS VARCHAR), CAST({expr} AS VARCHAR)))'
|
| 2691 |
+
cast = str
|
| 2692 |
if normalize_match and key != "pid":
|
| 2693 |
expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))"
|
| 2694 |
cast = lambda value: str(value).strip().lower()
|
|
|
|
| 2768 |
normalize = ((lambda value: str(value).strip().lower()) if normalize_match
|
| 2769 |
else str)
|
| 2770 |
wanted = {normalize(value) for value in values}
|
| 2771 |
+
rows = [row for row in rows if normalize(
|
| 2772 |
+
json.dumps([str(row.get(connection_scope)), str(row.get(key))], separators=(',', ':'), ensure_ascii=False)
|
| 2773 |
+
if connection_scope and row.get(connection_scope) not in (None, '') else row.get(key)) in wanted]
|
| 2774 |
if matches is not None:
|
| 2775 |
normalize = (lambda value: str(value).strip().lower()) if normalize_match else str
|
| 2776 |
wanted = {key: {normalize(value) for value in values} for key, values in matches.items()}
|
| 2777 |
rows = [row for row in rows if any(normalize(row.get(key)) in values
|
| 2778 |
for key, values in wanted.items())]
|
| 2779 |
+
if any('_connection_id' in row for row in rows):
|
| 2780 |
+
columns.add('_connection_id')
|
| 2781 |
return [{key: row.get(key) for key in columns} for row in rows]
|
| 2782 |
|
| 2783 |
|
|
|
|
| 2792 |
"""Follow a scoped leg with SQL operands at every population-sized boundary."""
|
| 2793 |
import core.relations as relations
|
| 2794 |
target, link, via = str(entry['target']), entry.get('link') or {}, entry.get('via') or {}
|
| 2795 |
+
connection = link.get('connectionScope')
|
| 2796 |
observed = {'rows': 0, 'complete': True, 'available': False}
|
| 2797 |
def source_values():
|
| 2798 |
for row in _relation_rows(session, source, [source_col], ids=source_ids, stream=True):
|
|
|
|
| 2803 |
if (manual and not reverse) or (inverse and reverse):
|
| 2804 |
yield from _relation_cell_ids(row[source_col])
|
| 2805 |
else:
|
| 2806 |
+
if connection:
|
| 2807 |
+
if row.get(connection) not in (None, ''):
|
| 2808 |
+
yield json.dumps([str(row[connection]), str(row[source_col])], separators=(',', ':'), ensure_ascii=False)
|
| 2809 |
+
else:
|
| 2810 |
+
yield str(row[source_col])
|
| 2811 |
values = _membership_relation(cur, source_values(), normalize=str, sql_type='VARCHAR')
|
| 2812 |
if source_col != relations.IDENTITY_COL and not observed['available']:
|
| 2813 |
if (source.startswith('ut_amzn_') and target.startswith('ut_amzn_')
|
|
|
|
| 2829 |
rows = _relation_rows(session, target, [target_col], stream=True)
|
| 2830 |
return _relation_cell_membership(cur, rows, target_col, values)
|
| 2831 |
rows = _relation_rows(session, target, [target_col], match=(target_col, values),
|
| 2832 |
+
where=entry.get('where'), stream=True, connection_scope=connection)
|
| 2833 |
return _membership_relation(cur, (row['pid'] for row in rows))
|
| 2834 |
|
| 2835 |
|
|
|
|
| 2855 |
return _relation_leg_membership(session, source, source_ids, entry, membership_cur,
|
| 2856 |
source_col=source_col, target_col=target_col, manual=manual, inverse=inverse, reverse=reverse)
|
| 2857 |
source_rows = _relation_rows(session, source, [source_col], ids=source_ids)
|
| 2858 |
+
connection = link.get('connectionScope')
|
| 2859 |
+
values = {(json.dumps([str(row[connection]), str(row[source_col])], separators=(',', ':'), ensure_ascii=False)
|
| 2860 |
+
if connection else str(row[source_col])) for row in source_rows
|
| 2861 |
+
if row.get(source_col) not in (None, '') and (not connection or row.get(connection) not in (None, ''))}
|
| 2862 |
# A column-keyed edge cannot fall back to the wire record id. In particular,
|
| 2863 |
# product_data.pid is a stable hash while sale_order_line.product_id is an Odoo
|
| 2864 |
# identity. Treating a missing/hidden/blank source value as an empty match would
|
|
|
|
| 2885 |
rows = (row for row in rows if _relation_cell_ids(row.get(target_col)) & values)
|
| 2886 |
else:
|
| 2887 |
rows = _relation_rows(session, target, [target_col], match=(target_col, values),
|
| 2888 |
+
where=entry.get("where"), stream=True, connection_scope=connection)
|
| 2889 |
return {int(row["pid"]) for row in rows}
|
| 2890 |
|
| 2891 |
|
|
|
|
| 3794 |
return shares.may_see("view", vid, session.uname, is_admin=session.admin, st=store.st)
|
| 3795 |
|
| 3796 |
|
| 3797 |
+
def mirror_store_field():
|
| 3798 |
+
"""The Odoo Store label is a read-only mirror value, never an upstream field request."""
|
| 3799 |
+
return {'key': 'Store', 'label': 'Store', 'type': 'text', 'source': 'odoo',
|
| 3800 |
+
'default': True, 'readOnly': True, 'mirrorOnly': True}
|
| 3801 |
+
|
| 3802 |
+
|
| 3803 |
def resolved_fields(table_key, defn):
|
| 3804 |
"""Resolve presets and authored fields for the same connection as the rows door."""
|
| 3805 |
rel = _rel()
|
|
|
|
| 3818 |
if not connected:
|
| 3819 |
raise err(404, "not_connected", "that database is not a connected grid")
|
| 3820 |
fields = (defn or {}).get("fields") or []
|
| 3821 |
+
if not fields and not str(table_key).startswith('ut_meta_'):
|
| 3822 |
raise err(409, "schema_not_loaded", "This connected database has no saved fields yet.")
|
| 3823 |
declared = {field["key"]: field for field in fields if field.get("key")}
|
| 3824 |
declared.update({field["key"]: field for field in (defn or {}).get("fields") or []
|
| 3825 |
if field.get("key")})
|
| 3826 |
+
if str(base or '').startswith('ut_odoo_'):
|
| 3827 |
+
declared['Store'] = mirror_store_field()
|
| 3828 |
return list(declared.values())
|
| 3829 |
|
| 3830 |
|
|
|
|
| 3912 |
# try, not inside it.
|
| 3913 |
try:
|
| 3914 |
cur, mismatch = (_mirror_cur(session, table_key)
|
| 3915 |
+
if (table_key.startswith(("ut_amzn_", "ut_meta_"))
|
| 3916 |
or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY)
|
| 3917 |
else _mirror_cur(session))
|
| 3918 |
except RuntimeError as e:
|
|
|
|
| 3947 |
"or ask for a read-through binding to be added for it")
|
| 3948 |
|
| 3949 |
st = st if st is not None else session.runtime
|
| 3950 |
+
if spec.get('_fields') is not None:
|
| 3951 |
+
fields = meta_source_fields(spec, fields)
|
| 3952 |
shared_fields = _so().fields(table_key, st=st)
|
| 3953 |
if shared_fields:
|
| 3954 |
declared = {field["key"]: field for field in fields}
|
|
|
|
| 4000 |
_wall_nodes, _wall_conj = _fe().tree_parts(_wall_tree)
|
| 4001 |
if _wall_nodes:
|
| 4002 |
try:
|
| 4003 |
+
if str(table_key).startswith(("ut_odoo_", "ut_amzn_", "ut_meta_")):
|
| 4004 |
_wall_cols, _wall_guards = _read_wall_columns(
|
| 4005 |
table_key, session, cur, spec, fields, _wall_nodes,
|
| 4006 |
st if st is not None else session.runtime)
|
|
|
|
| 4373 |
return narrowed
|
| 4374 |
|
| 4375 |
|
| 4376 |
+
_DRILL_PAGES = OrderedDict()
|
| 4377 |
+
_DRILL_PAGES_LOCK = threading.Lock()
|
| 4378 |
+
_DRILL_PAGES_MAX = 64
|
| 4379 |
+
|
| 4380 |
+
|
| 4381 |
+
def _materialised_revision(session, table_key):
|
| 4382 |
+
"""Bind reusable selection IDs to current policy, authorship and mirror revisions."""
|
| 4383 |
+
import registry_source
|
| 4384 |
+
from core import sheet_binding, store_pg
|
| 4385 |
+
from harness import datastore, mirror_tokens
|
| 4386 |
+
|
| 4387 |
+
generation = registry_source.generation(session, table_key)
|
| 4388 |
+
if generation is None:
|
| 4389 |
+
return None
|
| 4390 |
+
runtime = session.runtime
|
| 4391 |
+
reader = registry_source.backend(runtime)
|
| 4392 |
+
revisions = None
|
| 4393 |
+
if isinstance(reader, store_pg.PgStore):
|
| 4394 |
+
# Derived fields can read another database's authored cells or workspace.
|
| 4395 |
+
revisions = store_pg._read(store_pg._pool(), store_pg._q(
|
| 4396 |
+
'SELECT key, rev, updated_at FROM {tbl} ORDER BY key', 'store_kv', reader.tenant_slug), ())
|
| 4397 |
+
mirror = None
|
| 4398 |
+
if callable(getattr(runtime, 'assert_datastore_matches', None)):
|
| 4399 |
+
from harness.runtime import mirror_cursor
|
| 4400 |
+
try:
|
| 4401 |
+
cur = mirror_cursor(runtime)
|
| 4402 |
+
except RuntimeError as exc:
|
| 4403 |
+
from core.request_timing import raise_if_resource_error
|
| 4404 |
+
raise_if_resource_error(exc)
|
| 4405 |
+
return None # A stored table remains readable while its mirror is unavailable.
|
| 4406 |
+
mirror = mirror_tokens._revision(cur, mirror_tokens._file_marks(cur),
|
| 4407 |
+
datastore.mirror_generation(), [])
|
| 4408 |
+
if mirror is None:
|
| 4409 |
+
return None
|
| 4410 |
+
elif table_key in {'customer_data', 'product_data'}:
|
| 4411 |
+
return None # Unmanaged mirrors cannot prove an unchanged projection.
|
| 4412 |
+
return sheet_binding.fingerprint([
|
| 4413 |
+
generation, revisions, mirror, _today(), getattr(runtime, 'key', None),
|
| 4414 |
+
getattr(runtime, 'store_key', str)('user_tables'), getattr(reader, 'tenant_slug', None),
|
| 4415 |
+
getattr(getattr(runtime, 'tenant', None), 'config', None)])
|
| 4416 |
+
|
| 4417 |
+
|
| 4418 |
def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
|
| 4419 |
groupBy=None, groupKey=None, ids=None):
|
| 4420 |
+
"""Reuse an unchanged page's selection, then hydrate it through current scoped readers."""
|
| 4421 |
+
import registry_source
|
| 4422 |
+
from core.sheet_binding import fingerprint
|
| 4423 |
+
from harness import datastore
|
| 4424 |
+
|
| 4425 |
+
if table_key == 'ut_activity':
|
| 4426 |
+
return _materialised_window_read(table_key, session, offset, limit, filters, filterConj,
|
| 4427 |
+
sorts, search, related, groupBy, groupKey, ids=ids)
|
| 4428 |
+
revision = _materialised_revision(session, table_key)
|
| 4429 |
+
owner = registry_source.backend(session.runtime)
|
| 4430 |
+
key = fingerprint([id(owner), table_key, revision, offset, limit, filters, filterConj,
|
| 4431 |
+
sorts, search, related, groupBy, groupKey, ids]) if revision is not None else None
|
| 4432 |
+
with _DRILL_PAGES_LOCK:
|
| 4433 |
+
cached = _DRILL_PAGES.get(key) if key is not None else None
|
| 4434 |
+
if cached is not None:
|
| 4435 |
+
_DRILL_PAGES.move_to_end(key)
|
| 4436 |
+
if cached is not None and cached[0] is owner:
|
| 4437 |
+
page_ids, summary = cached[1], deepcopy(cached[2])
|
| 4438 |
+
if table_key in {'customer_data', 'product_data'}:
|
| 4439 |
+
session.require(table_key)
|
| 4440 |
+
cur, mismatch = _mirror_cur(session)
|
| 4441 |
+
if mismatch:
|
| 4442 |
+
raise err(409, 'cross_tenant_store', mismatch)
|
| 4443 |
+
if cur is None:
|
| 4444 |
+
raise err(503, 'canonical_source_unavailable', 'The tenant mirror is unavailable.')
|
| 4445 |
+
with cur.cursor() as page_cur:
|
| 4446 |
+
payload = registry_source.canonical_page(session, page_cur, table_key, page_ids, related=related)
|
| 4447 |
+
else:
|
| 4448 |
+
payload = registry_source.stored_page(session, table_key, page_ids, related=related)
|
| 4449 |
+
if _materialised_revision(session, table_key) != revision:
|
| 4450 |
+
return _materialised_window_read(table_key, session, offset, limit, filters, filterConj,
|
| 4451 |
+
sorts, search, related, groupBy, groupKey, ids=ids)
|
| 4452 |
+
limits = list(payload.get('limits') or [])
|
| 4453 |
+
limits.extend(report for report in summary['limits'] if report not in limits)
|
| 4454 |
+
return {**payload, **summary, 'limits': limits}
|
| 4455 |
+
|
| 4456 |
+
answer = _materialised_window_read(table_key, session, offset, limit, filters, filterConj,
|
| 4457 |
+
sorts, search, related, groupBy, groupKey, ids=ids)
|
| 4458 |
+
if (key is not None and len(answer['rows']) <= datastore.WINDOW_MAX
|
| 4459 |
+
and _materialised_revision(session, table_key) == revision):
|
| 4460 |
+
summary = {name: deepcopy(answer[name]) for name in
|
| 4461 |
+
('ok', 'total', 'totalUnfiltered', 'offset', 'limit', 'limits', 'scope')}
|
| 4462 |
+
entry = (owner, tuple(int(row['pid']) for row in answer['rows']), summary)
|
| 4463 |
+
with _DRILL_PAGES_LOCK:
|
| 4464 |
+
_DRILL_PAGES[key] = entry
|
| 4465 |
+
_DRILL_PAGES.move_to_end(key)
|
| 4466 |
+
while len(_DRILL_PAGES) > _DRILL_PAGES_MAX:
|
| 4467 |
+
_DRILL_PAGES.popitem(last=False)
|
| 4468 |
+
return answer
|
| 4469 |
+
|
| 4470 |
+
|
| 4471 |
+
def _materialised_window_read(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
|
| 4472 |
+
groupBy=None, groupKey=None, ids=None):
|
| 4473 |
from harness import datastore
|
| 4474 |
|
| 4475 |
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
|
|
|
|
| 4714 |
# as /related pins. No unscoped SQL subquery can leak a hidden target's id.
|
| 4715 |
amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets,
|
| 4716 |
max_bytes=LINK_VALUE_MAX_BYTES, capacity_refusal=_link_capacity_refusal)
|
| 4717 |
+
if table_key.startswith('ut_meta_'):
|
| 4718 |
+
_fill_meta_link_cells(session, rows_src, fields, st=page_st, rev=page_rev)
|
| 4719 |
overlays = {}
|
| 4720 |
|
| 4721 |
# ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
|
|
|
|
| 4764 |
cells.update(overlays.get(str(row["pid"]), {}))
|
| 4765 |
cells.update({key: row.get(key, "") for key in if_keys if key not in _hidden})
|
| 4766 |
rows.extend(aios_grid.rows_from_pool([row], fields, {str(row["pid"]): cells}))
|
| 4767 |
+
fields = [field for field in fields if field['key'] not in _SOURCE_PRIVATE]
|
| 4768 |
+
rows = [{key: value for key, value in row.items() if key not in _SOURCE_PRIVATE}
|
| 4769 |
+
for row in rows]
|
| 4770 |
|
| 4771 |
unfiltered = win["total"] if where == base_where else datastore.count_rows(
|
| 4772 |
where=base_where, params=tuple(wall_params), cur=cur, **frm)
|
| 4773 |
|
| 4774 |
from core import connector_spec
|
| 4775 |
rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime)
|
| 4776 |
+
if rows:
|
| 4777 |
+
user_tables = _ut()
|
| 4778 |
+
notes = user_tables.cell_notes(
|
| 4779 |
+
table_key, [row["pid"] for row in rows],
|
| 4780 |
+
field_keys={field["key"] for field in fields} - _hidden, st=session.runtime)
|
| 4781 |
+
for row in rows:
|
| 4782 |
+
if str(row["pid"]) in notes:
|
| 4783 |
+
row[user_tables.CELL_NOTES_KEY] = notes[str(row["pid"])]
|
| 4784 |
return {"ok": True, "fields": fields, "rows": rows,
|
| 4785 |
# ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`.
|
| 4786 |
"total": win["total"], "totalUnfiltered": unfiltered,
|
|
|
|
| 4849 |
lease.close()
|
| 4850 |
|
| 4851 |
|
| 4852 |
+
def _fill_meta_link_cells(session, rows, fields, st=None, rev=...):
|
| 4853 |
+
"""Resolve Meta's scoped hierarchy keys within the existing complete-cell byte budget."""
|
| 4854 |
+
from routes_grid import _may_open
|
| 4855 |
+
remaining = LINK_VALUE_MAX_BYTES
|
| 4856 |
+
requests, edges = {}, []
|
| 4857 |
+
for field in fields:
|
| 4858 |
+
bag = field.get('link') or {}
|
| 4859 |
+
if field.get('type') != 'link' or not bag.get('from') or not bag.get('on'):
|
| 4860 |
+
continue
|
| 4861 |
+
target, source, on = bag['table'], bag['from'], bag['on']
|
| 4862 |
+
edges.append((field, target, source, on))
|
| 4863 |
+
values = {str(row[source]) for row in rows if row.get(source) not in (None, '')}
|
| 4864 |
+
requests.setdefault(target, {}).setdefault(on, set()).update(values)
|
| 4865 |
+
indexes = {}
|
| 4866 |
+
for target, matches in requests.items():
|
| 4867 |
+
index = {key: {} for key in matches}
|
| 4868 |
+
if _may_open(session, target, st=st) and any(matches.values()):
|
| 4869 |
+
iterator = iter(_relation_rows(session, target, list(matches), matches=matches,
|
| 4870 |
+
st=st, rev=rev, stream=True))
|
| 4871 |
+
try:
|
| 4872 |
+
for row in iterator:
|
| 4873 |
+
for key, values in matches.items():
|
| 4874 |
+
value = str(row.get(key))
|
| 4875 |
+
if value not in values:
|
| 4876 |
+
continue
|
| 4877 |
+
ids = index[key].setdefault(value, set())
|
| 4878 |
+
if int(row['pid']) not in ids:
|
| 4879 |
+
remaining -= len(str(row['pid'])) + 1
|
| 4880 |
+
if remaining < 0:
|
| 4881 |
+
raise _link_capacity_refusal()
|
| 4882 |
+
ids.add(int(row['pid']))
|
| 4883 |
+
finally:
|
| 4884 |
+
close = getattr(iterator, 'close', None)
|
| 4885 |
+
if callable(close):
|
| 4886 |
+
close()
|
| 4887 |
+
indexes[target] = index
|
| 4888 |
+
remaining = LINK_VALUE_MAX_BYTES
|
| 4889 |
+
for field, target, source, on in edges:
|
| 4890 |
+
for row in rows:
|
| 4891 |
+
value = ','.join(str(pid) for pid in sorted(indexes[target][on].get(str(row.get(source)), ())))
|
| 4892 |
+
remaining -= len(value) + (2 if value else 0)
|
| 4893 |
+
if remaining < 0:
|
| 4894 |
+
raise _link_capacity_refusal()
|
| 4895 |
+
row[field['key']] = value
|
| 4896 |
+
|
| 4897 |
+
|
| 4898 |
def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...):
|
| 4899 |
"""Resolve page link cells through scoped SQL and a complete-value byte budget."""
|
| 4900 |
from routes_grid import _may_open
|
api/routes_payouts.py
CHANGED
|
@@ -171,48 +171,35 @@ def _last_deposits(rows, as_of):
|
|
| 171 |
@router.get("/screen")
|
| 172 |
def read_screen(app: str = Query(..., min_length=1), currency: str | None = Query(None),
|
| 173 |
view: str = Query("all", pattern="^(all|unreconciled)$"), cursor: str | None = Query(None),
|
| 174 |
-
limit: int = Query(SCREEN_PAGE, ge=1), session: Session = Depends(_app_reader)):
|
| 175 |
"""The App screen in one read: the KPI strip, tab counts and one page of settlements, newest first. No cap."""
|
| 176 |
pay = _module()
|
| 177 |
-
offset = _offset(cursor)
|
| 178 |
tenant, st = _tenant(session), session.runtime
|
| 179 |
can_edit = _may_edit(session, _app(session, app))
|
| 180 |
base = {"canEdit": can_edit, "statuses": list(pay.STATUSES), "view": view, "currency": currency or None}
|
| 181 |
try:
|
| 182 |
-
|
| 183 |
except HTTPException as exc:
|
| 184 |
if not _empty(exc):
|
| 185 |
raise
|
| 186 |
return {**base, "empty": True, "currencies": [], "counts": {"all": 0, "unreconciled": 0},
|
| 187 |
"settlements": [], "cursor": None, "kpis": {}}
|
| 188 |
-
|
| 189 |
-
counts = {"all": len(listed), "unreconciled": sum(1 for r in listed if not r["reconciled"])}
|
| 190 |
-
rows = listed if view == "all" else [r for r in listed if not r["reconciled"]]
|
| 191 |
-
page = rows[offset:offset + limit]
|
| 192 |
-
opened = over.get("openSettlements") or []
|
| 193 |
-
open_now = next((o for o in opened if currency and o.get("currency") == currency.strip().upper()),
|
| 194 |
-
opened[0] if opened else None)
|
| 195 |
-
return {**base, "empty": not over.get("settlementCount"), "asOf": over.get("asOf"),
|
| 196 |
-
"currencies": over.get("currencies") or [], "economicsCurrency": over.get("economicsCurrency"),
|
| 197 |
-
"counts": counts, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page],
|
| 198 |
-
"cursor": str(offset + limit) if offset + limit < len(rows) else None,
|
| 199 |
-
"kpis": {"nextDeposit": over.get("nextDeposit"), "openSettlement": open_now,
|
| 200 |
-
"lastDeposit": _last_deposits(listed, over.get("asOf") or ""),
|
| 201 |
-
"unreconciled": counts["unreconciled"], "settlements": counts["all"]}}
|
| 202 |
|
| 203 |
|
| 204 |
@router.get("/screen/settlements/{settlement_id}")
|
| 205 |
def read_screen_settlement(settlement_id: str, app: str = Query(..., min_length=1),
|
|
|
|
|
|
|
| 206 |
session: Session = Depends(_app_reader)):
|
| 207 |
"""One settlement for the drawer: its waterfall, the tie line and, when unreconciled, its owned item."""
|
| 208 |
pay = _module()
|
| 209 |
tenant, st = _tenant(session), session.runtime
|
| 210 |
can_edit = _may_edit(session, _app(session, app))
|
| 211 |
-
detail = _call(pay.
|
| 212 |
item = None
|
| 213 |
if not detail.get("reconciled"):
|
| 214 |
-
|
| 215 |
-
item = next((i for i in held if str(i.get("id")) == str(settlement_id)), None)
|
| 216 |
return {"canEdit": can_edit, "statuses": list(pay.STATUSES), "settlement": detail, "item": item}
|
| 217 |
|
| 218 |
|
|
@@ -226,3 +213,49 @@ def read_settings(session: Session = Depends(_app_reader)):
|
|
| 226 |
def write_settings(defaultOwner: str | None = Body(None, embed=True), session: Session = Depends(_app_writer)):
|
| 227 |
pay = _module()
|
| 228 |
return _call(pay.set_settings, _tenant(session), default_owner=defaultOwner, st=session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
@router.get("/screen")
|
| 172 |
def read_screen(app: str = Query(..., min_length=1), currency: str | None = Query(None),
|
| 173 |
view: str = Query("all", pattern="^(all|unreconciled)$"), cursor: str | None = Query(None),
|
| 174 |
+
limit: int = Query(SCREEN_PAGE, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 175 |
"""The App screen in one read: the KPI strip, tab counts and one page of settlements, newest first. No cap."""
|
| 176 |
pay = _module()
|
|
|
|
| 177 |
tenant, st = _tenant(session), session.runtime
|
| 178 |
can_edit = _may_edit(session, _app(session, app))
|
| 179 |
base = {"canEdit": can_edit, "statuses": list(pay.STATUSES), "view": view, "currency": currency or None}
|
| 180 |
try:
|
| 181 |
+
page = _call(pay.screen, tenant, st=st, currency=currency, view=view, q=q, cursor=cursor, limit=limit)
|
| 182 |
except HTTPException as exc:
|
| 183 |
if not _empty(exc):
|
| 184 |
raise
|
| 185 |
return {**base, "empty": True, "currencies": [], "counts": {"all": 0, "unreconciled": 0},
|
| 186 |
"settlements": [], "cursor": None, "kpis": {}}
|
| 187 |
+
return {**base, **page, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page["settlements"]]}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 188 |
|
| 189 |
|
| 190 |
@router.get("/screen/settlements/{settlement_id}")
|
| 191 |
def read_screen_settlement(settlement_id: str, app: str = Query(..., min_length=1),
|
| 192 |
+
bucket: str | None = Query(None), cursor: str | None = Query(None),
|
| 193 |
+
limit: int = Query(SCREEN_PAGE, ge=1), q: str = Query(""),
|
| 194 |
session: Session = Depends(_app_reader)):
|
| 195 |
"""One settlement for the drawer: its waterfall, the tie line and, when unreconciled, its owned item."""
|
| 196 |
pay = _module()
|
| 197 |
tenant, st = _tenant(session), session.runtime
|
| 198 |
can_edit = _may_edit(session, _app(session, app))
|
| 199 |
+
detail = _call(pay.screen_settlement, tenant, settlement_id, st=st, bucket_key=bucket, cursor=cursor, limit=limit, q=q)
|
| 200 |
item = None
|
| 201 |
if not detail.get("reconciled"):
|
| 202 |
+
item = _call(pay.screen_item, tenant, detail, st=st)
|
|
|
|
| 203 |
return {"canEdit": can_edit, "statuses": list(pay.STATUSES), "settlement": detail, "item": item}
|
| 204 |
|
| 205 |
|
|
|
|
| 213 |
def write_settings(defaultOwner: str | None = Body(None, embed=True), session: Session = Depends(_app_writer)):
|
| 214 |
pay = _module()
|
| 215 |
return _call(pay.set_settings, _tenant(session), default_owner=defaultOwner, st=session.runtime)
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def _refresh_owned_items_once():
|
| 219 |
+
"""Persist settlement work independently of GETs and preserve its first-seen due date."""
|
| 220 |
+
import application_registry
|
| 221 |
+
from harness import runtime
|
| 222 |
+
for tenant in runtime.known_tenants():
|
| 223 |
+
try:
|
| 224 |
+
st = runtime.get_runtime(tenant)
|
| 225 |
+
apps = application_registry.load(st)["applications"].values()
|
| 226 |
+
if any(a.get("templateKind") == TEMPLATE_KIND and not a.get("retired") for a in apps):
|
| 227 |
+
_module().refresh_items(tenant, st=st)
|
| 228 |
+
except Exception:
|
| 229 |
+
import logging
|
| 230 |
+
logging.getLogger(__name__).exception("Payouts item refresh failed for %s", tenant)
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
_items_thread = None
|
| 234 |
+
_items_stop = None
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
@router.on_event("startup")
|
| 238 |
+
def _start_item_refresh():
|
| 239 |
+
global _items_thread, _items_stop
|
| 240 |
+
import process_role
|
| 241 |
+
import threading
|
| 242 |
+
if _items_thread is not None and _items_thread.is_alive():
|
| 243 |
+
return
|
| 244 |
+
_items_stop = threading.Event()
|
| 245 |
+
|
| 246 |
+
def run():
|
| 247 |
+
while not _items_stop.is_set():
|
| 248 |
+
_refresh_owned_items_once()
|
| 249 |
+
_items_stop.wait(30)
|
| 250 |
+
_items_thread = process_role.start("payouts-items", "heavy", run)
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
@router.on_event("shutdown")
|
| 254 |
+
def _stop_item_refresh():
|
| 255 |
+
if _items_stop is not None:
|
| 256 |
+
_items_stop.set()
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
import process_role as _process_role
|
| 260 |
+
if _process_role.role() == "worker":
|
| 261 |
+
_start_item_refresh()
|
api/routes_products.py
CHANGED
|
@@ -24,13 +24,15 @@ import json
|
|
| 24 |
import re
|
| 25 |
import time
|
| 26 |
|
| 27 |
-
from fastapi import Body, Depends
|
| 28 |
|
| 29 |
from fastapi import APIRouter
|
| 30 |
|
| 31 |
import scope_cache
|
| 32 |
from core import grid_events, user_tables
|
| 33 |
-
from deps import Session, err, module_gate
|
|
|
|
|
|
|
| 34 |
|
| 35 |
router = APIRouter(prefix="/api/v1")
|
| 36 |
|
|
@@ -614,6 +616,8 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_
|
|
| 614 |
"fields": fields, "views": views, "lists": lists,
|
| 615 |
"_permission_scope": permission_scope,
|
| 616 |
"derived": derived,
|
|
|
|
|
|
|
| 617 |
"measures": measures, "measure_sets": measure_sets, "today": today,
|
| 618 |
"team_id": team_id}
|
| 619 |
|
|
@@ -679,19 +683,23 @@ def product_aggregates(body: dict = Body(default=None),
|
|
| 679 |
|
| 680 |
|
| 681 |
@router.patch("/products/{pid}")
|
|
|
|
|
|
|
| 682 |
def patch_product(pid: int, body: dict = Body(default=None),
|
| 683 |
-
session: Session = Depends(
|
| 684 |
"""Write the product table's EDITABLE overlay stratum — the customers PATCH, on the product
|
| 685 |
topic's ctx. Routed through `core.grid_events.handle_one` so the per-key `permissions.edit`
|
| 686 |
wall, the pid wall and the truncation rules stay ONE implementation; the ctx's `table` ops
|
| 687 |
aim the write at the PRODUCT bucket."""
|
| 688 |
import modules.product_data as pd
|
| 689 |
-
from core import grid_events
|
| 690 |
|
| 691 |
updates = dict(body or {})
|
|
|
|
| 692 |
if not updates:
|
| 693 |
raise err(400, "empty_patch", "no fields to update")
|
| 694 |
-
g =
|
|
|
|
| 695 |
if pid not in g["pids"]:
|
| 696 |
# 403, not 404 — the code may exist; it is simply not in this session's catalogue.
|
| 697 |
raise err(403, "out_of_scope", "that product is not in your catalogue")
|
|
@@ -703,9 +711,13 @@ def patch_product(pid: int, body: dict = Body(default=None),
|
|
| 703 |
hidden_keys=perm_scope.hidden_keys(
|
| 704 |
session.user, MODULE, g["fields"], st=session.runtime),
|
| 705 |
scope_key="product", table=pd.table_ops(session.runtime), st=session.runtime)
|
|
|
|
|
|
|
|
|
|
| 706 |
try:
|
| 707 |
grid_events.handle_one(
|
| 708 |
-
{"id": f"patch:product:{pid}:{time.time_ns()}",
|
|
|
|
| 709 |
"pid": pid, "updates": updates}, ctx)
|
| 710 |
except grid_events.StoreUnavailable:
|
| 711 |
raise err(503, "store_unavailable",
|
|
@@ -744,7 +756,11 @@ def product_rows(assembly):
|
|
| 744 |
import aios_grid
|
| 745 |
rows = aios_grid.rows_from_pool(assembly["rows_src"], assembly["fields"],
|
| 746 |
assembly["ws"].get("overlays"), derived=assembly["derived"])
|
| 747 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 748 |
|
| 749 |
|
| 750 |
def _seed_shared(rows, rows_src, fields):
|
|
|
|
| 24 |
import re
|
| 25 |
import time
|
| 26 |
|
| 27 |
+
from fastapi import Body, Depends, Response
|
| 28 |
|
| 29 |
from fastapi import APIRouter
|
| 30 |
|
| 31 |
import scope_cache
|
| 32 |
from core import grid_events, user_tables
|
| 33 |
+
from deps import Session, err, module_gate, require_session
|
| 34 |
+
from routes_grid import row_patch_receipt
|
| 35 |
+
from routes_tables import _row_patch_identity, _note_delta, _patch_cell_notes, _project_cell_notes, _builtin_note_context
|
| 36 |
|
| 37 |
router = APIRouter(prefix="/api/v1")
|
| 38 |
|
|
|
|
| 616 |
"fields": fields, "views": views, "lists": lists,
|
| 617 |
"_permission_scope": permission_scope,
|
| 618 |
"derived": derived,
|
| 619 |
+
"_note_runtime": session.runtime,
|
| 620 |
+
"_note_hidden": hidden,
|
| 621 |
"measures": measures, "measure_sets": measure_sets, "today": today,
|
| 622 |
"team_id": team_id}
|
| 623 |
|
|
|
|
| 683 |
|
| 684 |
|
| 685 |
@router.patch("/products/{pid}")
|
| 686 |
+
@_row_patch_identity(module=MODULE)
|
| 687 |
+
@row_patch_receipt('product')
|
| 688 |
def patch_product(pid: int, body: dict = Body(default=None),
|
| 689 |
+
session: Session = Depends(require_session), response: Response = None):
|
| 690 |
"""Write the product table's EDITABLE overlay stratum — the customers PATCH, on the product
|
| 691 |
topic's ctx. Routed through `core.grid_events.handle_one` so the per-key `permissions.edit`
|
| 692 |
wall, the pid wall and the truncation rules stay ONE implementation; the ctx's `table` ops
|
| 693 |
aim the write at the PRODUCT bucket."""
|
| 694 |
import modules.product_data as pd
|
| 695 |
+
from core import grid_events, edit_receipts
|
| 696 |
|
| 697 |
updates = dict(body or {})
|
| 698 |
+
notes = _note_delta(updates)
|
| 699 |
if not updates:
|
| 700 |
raise err(400, "empty_patch", "no fields to update")
|
| 701 |
+
g = (_builtin_note_context(session, MODULE, pid) if notes is not None else
|
| 702 |
+
product_assembly(session, consume_corrections=False))
|
| 703 |
if pid not in g["pids"]:
|
| 704 |
# 403, not 404 — the code may exist; it is simply not in this session's catalogue.
|
| 705 |
raise err(403, "out_of_scope", "that product is not in your catalogue")
|
|
|
|
| 711 |
hidden_keys=perm_scope.hidden_keys(
|
| 712 |
session.user, MODULE, g["fields"], st=session.runtime),
|
| 713 |
scope_key="product", table=pd.table_ops(session.runtime), st=session.runtime)
|
| 714 |
+
if notes is not None:
|
| 715 |
+
return _patch_cell_notes(MODULE, pid, notes, ctx,
|
| 716 |
+
source_row_exists=lambda key, rid, admitted: key == MODULE and int(rid) in g['pids'])
|
| 717 |
try:
|
| 718 |
grid_events.handle_one(
|
| 719 |
+
{"id": edit_receipts.current_op_id() or f"patch:product:{pid}:{time.time_ns()}",
|
| 720 |
+
"type": "overlay_patch",
|
| 721 |
"pid": pid, "updates": updates}, ctx)
|
| 722 |
except grid_events.StoreUnavailable:
|
| 723 |
raise err(503, "store_unavailable",
|
|
|
|
| 756 |
import aios_grid
|
| 757 |
rows = aios_grid.rows_from_pool(assembly["rows_src"], assembly["fields"],
|
| 758 |
assembly["ws"].get("overlays"), derived=assembly["derived"])
|
| 759 |
+
rows = _seed_image(_seed_shared(rows, assembly["rows_src"], assembly["fields"]), assembly["fields"])
|
| 760 |
+
if assembly.get('_note_runtime') is not None:
|
| 761 |
+
rows = _project_cell_notes(MODULE, rows, assembly['fields'], assembly['_note_runtime'],
|
| 762 |
+
hidden=assembly.get('_note_hidden') or ())
|
| 763 |
+
return rows
|
| 764 |
|
| 765 |
|
| 766 |
def _seed_shared(rows, rows_src, fields):
|
api/routes_ranking_diagnostic.py
CHANGED
|
@@ -85,17 +85,19 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
|
|
| 85 |
@router.get("/screen")
|
| 86 |
def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 87 |
week: str | None = Query(None), cursor: str | None = Query(None),
|
|
|
|
| 88 |
session: Session = Depends(_app_reader)):
|
| 89 |
"""The week's drop list (the newest week by default), with counts, problems and settings."""
|
| 90 |
rd = _module()
|
| 91 |
record = _app(session, app)
|
| 92 |
st = session.runtime
|
| 93 |
held = _call(rd.weeks, _tenant(session), st=st)
|
| 94 |
-
listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor,
|
| 95 |
week=week or (held[-1] if held else None), st=st)
|
| 96 |
return {"canEdit": _may_edit(session, record), "week": listed["week"], "weeks": listed["weeks"],
|
| 97 |
"counts": listed["counts"], "items": listed["items"], "cursor": listed["cursor"],
|
| 98 |
-
"problems": listed["problems"], "statuses": list(rd.STATUSES),
|
|
|
|
| 99 |
"settings": _call(rd.settings, _tenant(session), st=st)}
|
| 100 |
|
| 101 |
|
|
@@ -116,10 +118,12 @@ def update_item(item_id: str, status: str = Body(...), note: str | None = Body(N
|
|
| 116 |
|
| 117 |
@router.get("/asins")
|
| 118 |
def asin_picker(week: str | None = Query(None), cursor: str | None = Query(None),
|
|
|
|
| 119 |
session: Session = Depends(_app_reader)):
|
| 120 |
"""The ASIN picker page. A flagged ASIN carries its cause label and item id for the cause card."""
|
| 121 |
rd = _module()
|
| 122 |
-
page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor,
|
|
|
|
| 123 |
labels = {key: label for key, label, _fix in (*rd.CAUSES, rd.UNEXPLAINED)}
|
| 124 |
for row in page["asins"]:
|
| 125 |
flagged = bool(row.get("cause") and page["week"])
|
|
|
|
| 85 |
@router.get("/screen")
|
| 86 |
def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 87 |
week: str | None = Query(None), cursor: str | None = Query(None),
|
| 88 |
+
limit: int = Query(200, ge=1, le=200), q: str = Query("", max_length=200),
|
| 89 |
session: Session = Depends(_app_reader)):
|
| 90 |
"""The week's drop list (the newest week by default), with counts, problems and settings."""
|
| 91 |
rd = _module()
|
| 92 |
record = _app(session, app)
|
| 93 |
st = session.runtime
|
| 94 |
held = _call(rd.weeks, _tenant(session), st=st)
|
| 95 |
+
listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor, limit=limit, q=q,
|
| 96 |
week=week or (held[-1] if held else None), st=st)
|
| 97 |
return {"canEdit": _may_edit(session, record), "week": listed["week"], "weeks": listed["weeks"],
|
| 98 |
"counts": listed["counts"], "items": listed["items"], "cursor": listed["cursor"],
|
| 99 |
+
"problems": listed["problems"], "statuses": list(rd.STATUSES), "total": listed["total"],
|
| 100 |
+
"sourceTotals": _call(rd.screen_totals, _tenant(session), listed["week"], st=st),
|
| 101 |
"settings": _call(rd.settings, _tenant(session), st=st)}
|
| 102 |
|
| 103 |
|
|
|
|
| 118 |
|
| 119 |
@router.get("/asins")
|
| 120 |
def asin_picker(week: str | None = Query(None), cursor: str | None = Query(None),
|
| 121 |
+
limit: int = Query(200, ge=1, le=200), q: str = Query("", max_length=200),
|
| 122 |
session: Session = Depends(_app_reader)):
|
| 123 |
"""The ASIN picker page. A flagged ASIN carries its cause label and item id for the cause card."""
|
| 124 |
rd = _module()
|
| 125 |
+
page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor, limit=limit, q=q,
|
| 126 |
+
st=session.runtime)
|
| 127 |
labels = {key: label for key, label, _fix in (*rd.CAUSES, rd.UNEXPLAINED)}
|
| 128 |
for row in page["asins"]:
|
| 129 |
flagged = bool(row.get("cause") and page["week"])
|
api/routes_reimbursements.py
CHANGED
|
@@ -83,18 +83,26 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
|
|
| 83 |
|
| 84 |
@router.get("/screen")
|
| 85 |
def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 86 |
-
cursor: str | None = Query(None),
|
|
|
|
| 87 |
rb = _module()
|
| 88 |
record = _app(session, app)
|
| 89 |
-
listed = _call(rb.items, _tenant(session), session.uname, status=status,
|
|
|
|
| 90 |
return {"canEdit": _may_edit(session, record), "counts": listed["counts"], "items": listed["items"],
|
| 91 |
-
"cursor": listed["cursor"], "
|
|
|
|
|
|
|
| 92 |
|
| 93 |
|
| 94 |
@router.get("/items/{item_id}")
|
| 95 |
-
def item_detail(item_id: str,
|
|
|
|
|
|
|
| 96 |
rb = _module()
|
| 97 |
-
return _call(rb.item_detail, _tenant(session), session.uname, item_id,
|
|
|
|
|
|
|
| 98 |
|
| 99 |
|
| 100 |
@router.patch("/items/{item_id}")
|
|
@@ -106,9 +114,10 @@ def update_item(item_id: str, status: str = Body(...), caseId: str | None = Body
|
|
| 106 |
|
| 107 |
|
| 108 |
@router.get("/paid")
|
| 109 |
-
def paid_rows(cursor: str | None = Query(None),
|
|
|
|
| 110 |
rb = _module()
|
| 111 |
-
return _call(rb.paid, _tenant(session), session.uname, cursor=cursor, st=session.runtime)
|
| 112 |
|
| 113 |
|
| 114 |
@router.get("/validate")
|
|
|
|
| 83 |
|
| 84 |
@router.get("/screen")
|
| 85 |
def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 86 |
+
cursor: str | None = Query(None), search: str = Query(""), limit: int = Query(200, ge=1),
|
| 87 |
+
session: Session = Depends(_app_reader)):
|
| 88 |
rb = _module()
|
| 89 |
record = _app(session, app)
|
| 90 |
+
listed = _call(rb.items, _tenant(session), session.uname, status=status, search=search, cursor=cursor,
|
| 91 |
+
limit=limit, st=session.runtime)
|
| 92 |
return {"canEdit": _may_edit(session, record), "counts": listed["counts"], "items": listed["items"],
|
| 93 |
+
"cursor": listed["cursor"], "total": listed["total"], "offset": listed["offset"],
|
| 94 |
+
"limit": listed["limit"], "paidTotal": listed["paidTotal"],
|
| 95 |
+
"problems": listed["problems"], "statuses": list(rb.STATUSES)}
|
| 96 |
|
| 97 |
|
| 98 |
@router.get("/items/{item_id}")
|
| 99 |
+
def item_detail(item_id: str, events_cursor: str | None = Query(None), returns_cursor: str | None = Query(None),
|
| 100 |
+
reimbursements_cursor: str | None = Query(None), history_cursor: str | None = Query(None),
|
| 101 |
+
limit: int = Query(200, ge=1), session: Session = Depends(_app_reader)):
|
| 102 |
rb = _module()
|
| 103 |
+
return _call(rb.item_detail, _tenant(session), session.uname, item_id, events_cursor=events_cursor,
|
| 104 |
+
returns_cursor=returns_cursor, reimbursements_cursor=reimbursements_cursor,
|
| 105 |
+
history_cursor=history_cursor, limit=limit, st=session.runtime)
|
| 106 |
|
| 107 |
|
| 108 |
@router.patch("/items/{item_id}")
|
|
|
|
| 114 |
|
| 115 |
|
| 116 |
@router.get("/paid")
|
| 117 |
+
def paid_rows(cursor: str | None = Query(None), search: str = Query(""), limit: int = Query(200, ge=1),
|
| 118 |
+
session: Session = Depends(_app_reader)):
|
| 119 |
rb = _module()
|
| 120 |
+
return _call(rb.paid, _tenant(session), session.uname, search=search, cursor=cursor, limit=limit, st=session.runtime)
|
| 121 |
|
| 122 |
|
| 123 |
@router.get("/validate")
|
api/routes_restock_planner.py
CHANGED
|
@@ -1,5 +1,9 @@
|
|
| 1 |
"""Restock planner App routes (C3): items, one SKU, settings, refresh and the rule 8 tie."""
|
|
|
|
|
|
|
|
|
|
| 2 |
from fastapi import APIRouter, Body, Depends, Query
|
|
|
|
| 3 |
|
| 4 |
from deps import Session, err, require_session
|
| 5 |
|
|
@@ -104,19 +108,23 @@ def thresholds(session):
|
|
| 104 |
@router.get("/items")
|
| 105 |
def list_items(app: str = Query(..., min_length=1), band: str | None = Query(None),
|
| 106 |
status: str | None = Query(None), cursor: str | None = Query(None),
|
| 107 |
-
limit: int = Query(200, ge=1
|
|
|
|
| 108 |
session: Session = Depends(_app_reader)):
|
| 109 |
rp = _module()
|
| 110 |
-
listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor,
|
| 111 |
-
band=band, q=q, st=session.runtime)
|
| 112 |
return {**listed, "owners": owners(session), "thresholds": thresholds(session),
|
| 113 |
"canEdit": _may_edit(session, _app(session, app))}
|
| 114 |
|
| 115 |
|
| 116 |
@router.get("/items/{item_id}")
|
| 117 |
-
def read_item(item_id: str, app: str = Query(..., min_length=1),
|
|
|
|
|
|
|
| 118 |
rp = _module()
|
| 119 |
-
detail = _call(rp.item_detail, _tenant(session), item_id,
|
|
|
|
| 120 |
return {**detail, "thresholds": thresholds(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 121 |
|
| 122 |
|
|
@@ -147,7 +155,80 @@ def write_settings(sendToFbaDays: float | None = Body(None), overstockDays: floa
|
|
| 147 |
@router.post("/refresh")
|
| 148 |
def refresh(session: Session = Depends(_app_writer)):
|
| 149 |
rp = _module()
|
| 150 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
|
| 152 |
|
| 153 |
@router.get("/validate")
|
|
|
|
| 1 |
"""Restock planner App routes (C3): items, one SKU, settings, refresh and the rule 8 tie."""
|
| 2 |
+
import time
|
| 3 |
+
import uuid
|
| 4 |
+
|
| 5 |
from fastapi import APIRouter, Body, Depends, Query
|
| 6 |
+
from fastapi.responses import JSONResponse
|
| 7 |
|
| 8 |
from deps import Session, err, require_session
|
| 9 |
|
|
|
|
| 108 |
@router.get("/items")
|
| 109 |
def list_items(app: str = Query(..., min_length=1), band: str | None = Query(None),
|
| 110 |
status: str | None = Query(None), cursor: str | None = Query(None),
|
| 111 |
+
limit: int = Query(200, ge=1, description="Requests above 200 use 200 items. limit reports the effective size."),
|
| 112 |
+
q: str | None = Query(None),
|
| 113 |
session: Session = Depends(_app_reader)):
|
| 114 |
rp = _module()
|
| 115 |
+
listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor,
|
| 116 |
+
limit=min(limit, rp.MAX_PAGE_SIZE), band=band, q=q, st=session.runtime)
|
| 117 |
return {**listed, "owners": owners(session), "thresholds": thresholds(session),
|
| 118 |
"canEdit": _may_edit(session, _app(session, app))}
|
| 119 |
|
| 120 |
|
| 121 |
@router.get("/items/{item_id}")
|
| 122 |
+
def read_item(item_id: str, app: str = Query(..., min_length=1), offset: int = Query(0, ge=0),
|
| 123 |
+
limit: int = Query(50, ge=1, description="Requests above 200 use 200 entries per collection. detailLimit reports the effective size."),
|
| 124 |
+
session: Session = Depends(_app_reader)):
|
| 125 |
rp = _module()
|
| 126 |
+
detail = _call(rp.item_detail, _tenant(session), item_id, offset=offset,
|
| 127 |
+
limit=min(limit, rp.MAX_PAGE_SIZE), st=session.runtime)
|
| 128 |
return {**detail, "thresholds": thresholds(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 129 |
|
| 130 |
|
|
|
|
| 155 |
@router.post("/refresh")
|
| 156 |
def refresh(session: Session = Depends(_app_writer)):
|
| 157 |
rp = _module()
|
| 158 |
+
identity, stamp = uuid.uuid4().hex, time.time()
|
| 159 |
+
|
| 160 |
+
def enqueue(doc):
|
| 161 |
+
doc = doc if isinstance(doc, dict) else {}
|
| 162 |
+
if doc.get("state") in ("queued", "running") and stamp - float(doc.get("requestedAt") or 0) < 600:
|
| 163 |
+
return doc
|
| 164 |
+
return {"state": "queued", "requestId": identity, "requestedAt": stamp}
|
| 165 |
+
|
| 166 |
+
queued = _call(session.runtime.update, rp.KEY_REFRESH, enqueue)
|
| 167 |
+
return JSONResponse(status_code=202, content=queued)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
@router.get("/refresh")
|
| 171 |
+
def refresh_status(session: Session = Depends(_app_reader)):
|
| 172 |
+
return _call(session.runtime.get, _module().KEY_REFRESH) or {"state": "idle"}
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def _drain_refresh(rt, *, now=None):
|
| 176 |
+
"""Claim one durable refresh intent in the sync worker and retain its terminal result."""
|
| 177 |
+
rp = _module()
|
| 178 |
+
stamp = time.time() if now is None else now
|
| 179 |
+
pending = rt.get(rp.KEY_REFRESH) or {}
|
| 180 |
+
if pending.get("state") != "queued" and not (
|
| 181 |
+
pending.get("state") == "running" and stamp - float(pending.get("startedAt") or 0) >= 600):
|
| 182 |
+
return
|
| 183 |
+
token, claimed = uuid.uuid4().hex, {}
|
| 184 |
+
|
| 185 |
+
def claim(doc):
|
| 186 |
+
if (doc or {}).get("requestId") == pending.get("requestId") and (
|
| 187 |
+
doc.get("state") == "queued" or (doc.get("state") == "running"
|
| 188 |
+
and stamp - float(doc.get("startedAt") or 0) >= 600)):
|
| 189 |
+
doc.update(state="running", startedAt=stamp, claim=token)
|
| 190 |
+
claimed.update(doc)
|
| 191 |
+
return doc
|
| 192 |
+
|
| 193 |
+
rt.update(rp.KEY_REFRESH, claim)
|
| 194 |
+
if not claimed:
|
| 195 |
+
return
|
| 196 |
+
try:
|
| 197 |
+
result = rp.refresh(rt.key, st=rt)
|
| 198 |
+
completion = {"state": "failed" if result.get("state") == "read_failed" else "done", "result": result}
|
| 199 |
+
except Exception: # noqa: BLE001
|
| 200 |
+
completion = {"state": "failed", "message": "The restock plan could not be refreshed. Try again shortly."}
|
| 201 |
+
|
| 202 |
+
def finish(doc):
|
| 203 |
+
if (doc or {}).get("claim") == token:
|
| 204 |
+
doc.update(completion, finishedAt=time.time())
|
| 205 |
+
return doc
|
| 206 |
+
rt.update(rp.KEY_REFRESH, finish)
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def _refresh_worker():
|
| 210 |
+
import threading
|
| 211 |
+
import process_role
|
| 212 |
+
from harness import runtime
|
| 213 |
+
stop = threading.Event()
|
| 214 |
+
while not stop.wait(1):
|
| 215 |
+
if not process_role.may_write():
|
| 216 |
+
continue
|
| 217 |
+
for tenant in runtime.known_tenants():
|
| 218 |
+
try:
|
| 219 |
+
_drain_refresh(runtime.get_runtime(tenant))
|
| 220 |
+
except Exception: # noqa: BLE001
|
| 221 |
+
continue
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def _start_refresh_worker():
|
| 225 |
+
import process_role
|
| 226 |
+
from core import data_binding
|
| 227 |
+
if not data_binding.gate_process():
|
| 228 |
+
process_role.start("restock-refresh", "heavy", _refresh_worker)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
_start_refresh_worker()
|
| 232 |
|
| 233 |
|
| 234 |
@router.get("/validate")
|
api/routes_returns_repeat.py
CHANGED
|
@@ -85,15 +85,16 @@ def _settings_writer(app_id: str = Path(..., min_length=1), session: Session = D
|
|
| 85 |
|
| 86 |
@router.get("/{app_id}/summary")
|
| 87 |
def read_summary(start: str | None = Query(None), end: str | None = Query(None),
|
|
|
|
| 88 |
session: Session = Depends(_app_reader)):
|
| 89 |
rr = _module()
|
| 90 |
-
out = _call(rr.summary, _tenant(session), start=start, end=end, st=session.runtime)
|
| 91 |
return {**out, "canEdit": _can_edit_items(session) and not _viewer(session)}
|
| 92 |
|
| 93 |
|
| 94 |
@router.get("/{app_id}/skus")
|
| 95 |
def list_skus(segment: str = Query("highest"), start: str | None = Query(None), end: str | None = Query(None),
|
| 96 |
-
cursor: str | None = Query(None), limit: int = Query(
|
| 97 |
session: Session = Depends(_app_reader)):
|
| 98 |
rr = _module()
|
| 99 |
return _call(rr.skus, _tenant(session), segment=segment, start=start, end=end, cursor=cursor,
|
|
@@ -102,9 +103,10 @@ def list_skus(segment: str = Query("highest"), start: str | None = Query(None),
|
|
| 102 |
|
| 103 |
@router.get("/{app_id}/sku")
|
| 104 |
def sku_detail(key: str = Query(..., min_length=1), start: str | None = Query(None),
|
| 105 |
-
end: str | None = Query(None),
|
|
|
|
| 106 |
rr = _module()
|
| 107 |
-
return _call(rr.detail, _tenant(session), key, start=start, end=end, st=session.runtime)
|
| 108 |
|
| 109 |
|
| 110 |
def owners(session):
|
|
@@ -127,7 +129,7 @@ def owners(session):
|
|
| 127 |
|
| 128 |
@router.get("/{app_id}/items")
|
| 129 |
def list_items(status: str | None = Query(None), cursor: str | None = Query(None),
|
| 130 |
-
limit: int = Query(
|
| 131 |
rr = _module()
|
| 132 |
listed = _call(rr.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit,
|
| 133 |
st=session.runtime)
|
|
|
|
| 85 |
|
| 86 |
@router.get("/{app_id}/summary")
|
| 87 |
def read_summary(start: str | None = Query(None), end: str | None = Query(None),
|
| 88 |
+
cursor: str | None = Query(None), limit: int = Query(24, ge=1),
|
| 89 |
session: Session = Depends(_app_reader)):
|
| 90 |
rr = _module()
|
| 91 |
+
out = _call(rr.summary, _tenant(session), start=start, end=end, cursor=cursor, limit=limit, st=session.runtime)
|
| 92 |
return {**out, "canEdit": _can_edit_items(session) and not _viewer(session)}
|
| 93 |
|
| 94 |
|
| 95 |
@router.get("/{app_id}/skus")
|
| 96 |
def list_skus(segment: str = Query("highest"), start: str | None = Query(None), end: str | None = Query(None),
|
| 97 |
+
cursor: str | None = Query(None), limit: int = Query(100, ge=1), q: str | None = Query(None),
|
| 98 |
session: Session = Depends(_app_reader)):
|
| 99 |
rr = _module()
|
| 100 |
return _call(rr.skus, _tenant(session), segment=segment, start=start, end=end, cursor=cursor,
|
|
|
|
| 103 |
|
| 104 |
@router.get("/{app_id}/sku")
|
| 105 |
def sku_detail(key: str = Query(..., min_length=1), start: str | None = Query(None),
|
| 106 |
+
end: str | None = Query(None), cursor: str | None = Query(None), limit: int = Query(25, ge=1),
|
| 107 |
+
session: Session = Depends(_app_reader)):
|
| 108 |
rr = _module()
|
| 109 |
+
return _call(rr.detail, _tenant(session), key, start=start, end=end, cursor=cursor, limit=limit, st=session.runtime)
|
| 110 |
|
| 111 |
|
| 112 |
def owners(session):
|
|
|
|
| 129 |
|
| 130 |
@router.get("/{app_id}/items")
|
| 131 |
def list_items(status: str | None = Query(None), cursor: str | None = Query(None),
|
| 132 |
+
limit: int = Query(100, ge=1), session: Session = Depends(_app_reader)):
|
| 133 |
rr = _module()
|
| 134 |
listed = _call(rr.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit,
|
| 135 |
st=session.runtime)
|
api/routes_sheet.py
CHANGED
|
@@ -3444,7 +3444,9 @@ def _serve_open_tab(session, book, target, *, databases=None):
|
|
| 3444 |
@router.patch("/sheets/{workbook_id}/tabs/{tab}/cells")
|
| 3445 |
def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None),
|
| 3446 |
session: Session = Depends(require_session)):
|
| 3447 |
-
"""Merge a RANGE of cells into one tab: body `{cells: {A1: Cell | null}, rev}` (contract C3).
|
|
|
|
|
|
|
| 3448 |
|
| 3449 |
⭐ `{"A1": null}` DELETES A1 and leaves its neighbours alone. After C2 an empty cell can carry
|
| 3450 |
a style (`kind: "empty"`), so a client needs a way to say "nothing at all lives here" that is
|
|
@@ -3470,10 +3472,49 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None
|
|
| 3470 |
whole workbook is recomputed IN MEMORY (never a second write, never a second `rev`, see
|
| 3471 |
`_recomputed_tab_cells`) and every FORMULA cell on THIS tab is handed back with its answer.
|
| 3472 |
"""
|
| 3473 |
-
from core import
|
| 3474 |
raw = dict(body or {})
|
|
|
|
|
|
|
|
|
|
| 3475 |
try:
|
| 3476 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3477 |
except sheets.SheetRefusal as exc:
|
| 3478 |
raise _refuse(exc)
|
| 3479 |
#: ⚠ `rev` ABSENT and `rev: null` are the same request and both opt out. A client that has
|
|
@@ -3481,7 +3522,7 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None
|
|
| 3481 |
#: imported workbook impossible.
|
| 3482 |
expect = raw.get("rev")
|
| 3483 |
try:
|
| 3484 |
-
out = sheets.patch_cells(
|
| 3485 |
expect_rev=expect, actor=session.uname,
|
| 3486 |
actor_name=session.user.get("name") or session.uname)
|
| 3487 |
except sheets.SheetRefusal as exc:
|
|
|
|
| 3444 |
@router.patch("/sheets/{workbook_id}/tabs/{tab}/cells")
|
| 3445 |
def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None),
|
| 3446 |
session: Session = Depends(require_session)):
|
| 3447 |
+
"""Merge a RANGE of cells into one tab: body `{cells: {A1: Cell | null}, rev, opId}` (contract C3).
|
| 3448 |
+
|
| 3449 |
+
A supplied opId retains the complete response for retries, including refusals and revisions.
|
| 3450 |
|
| 3451 |
⭐ `{"A1": null}` DELETES A1 and leaves its neighbours alone. After C2 an empty cell can carry
|
| 3452 |
a style (`kind: "empty"`), so a client needs a way to say "nothing at all lives here" that is
|
|
|
|
| 3472 |
whole workbook is recomputed IN MEMORY (never a second write, never a second `rev`, see
|
| 3473 |
`_recomputed_tab_cells`) and every FORMULA cell on THIS tab is handed back with its answer.
|
| 3474 |
"""
|
| 3475 |
+
from core import edit_receipts
|
| 3476 |
raw = dict(body or {})
|
| 3477 |
+
# Older clients omit opId until their outbox is available.
|
| 3478 |
+
if "opId" not in raw:
|
| 3479 |
+
return _apply_sheet_cell_patch(workbook_id, tab, raw, session, _store(session))
|
| 3480 |
try:
|
| 3481 |
+
identity = edit_receipts.op_id(raw.pop("opId"))
|
| 3482 |
+
except ValueError as exc:
|
| 3483 |
+
raise err(400, "bad_op_id", str(exc)) from exc
|
| 3484 |
+
|
| 3485 |
+
def apply():
|
| 3486 |
+
from dataclasses import replace
|
| 3487 |
+
checked_session = replace(
|
| 3488 |
+
session, runtime=edit_receipts.CheckedStore(_store(session)))
|
| 3489 |
+
try:
|
| 3490 |
+
answer = _apply_sheet_cell_patch(
|
| 3491 |
+
workbook_id, tab, raw, checked_session, checked_session.runtime)
|
| 3492 |
+
return {"answer": dict(answer, opId=identity)}
|
| 3493 |
+
except HTTPException as exc:
|
| 3494 |
+
if exc.status_code >= 500:
|
| 3495 |
+
raise
|
| 3496 |
+
detail = exc.detail if isinstance(exc.detail, dict) else {"message": exc.detail}
|
| 3497 |
+
return {"status": exc.status_code, "detail": dict(detail, opId=identity),
|
| 3498 |
+
"headers": exc.headers}
|
| 3499 |
+
|
| 3500 |
+
try:
|
| 3501 |
+
result = edit_receipts.run_once(
|
| 3502 |
+
_store(session), session.uname,
|
| 3503 |
+
edit_receipts.operation(workbook_id, "sheet_cells", tab), identity, apply)
|
| 3504 |
+
except edit_receipts.ReceiptUnavailable as exc:
|
| 3505 |
+
failure = err(503, "store_unavailable", str(exc))
|
| 3506 |
+
failure.detail["opId"] = identity
|
| 3507 |
+
raise failure from exc
|
| 3508 |
+
if "status" in result:
|
| 3509 |
+
raise HTTPException(result["status"], result["detail"], headers=result["headers"])
|
| 3510 |
+
return result["answer"]
|
| 3511 |
+
|
| 3512 |
+
|
| 3513 |
+
def _apply_sheet_cell_patch(workbook_id, tab, raw, session, st):
|
| 3514 |
+
"""Apply and compute a cell patch before retaining its complete response."""
|
| 3515 |
+
from core import sheets
|
| 3516 |
+
try:
|
| 3517 |
+
_require_editor(session, sheets.get_workbook(st, workbook_id))
|
| 3518 |
except sheets.SheetRefusal as exc:
|
| 3519 |
raise _refuse(exc)
|
| 3520 |
#: ⚠ `rev` ABSENT and `rev: null` are the same request and both opt out. A client that has
|
|
|
|
| 3522 |
#: imported workbook impossible.
|
| 3523 |
expect = raw.get("rev")
|
| 3524 |
try:
|
| 3525 |
+
out = sheets.patch_cells(st, workbook_id, tab, raw.get("cells"),
|
| 3526 |
expect_rev=expect, actor=session.uname,
|
| 3527 |
actor_name=session.user.get("name") or session.uname)
|
| 3528 |
except sheets.SheetRefusal as exc:
|
api/routes_tables.py
CHANGED
|
@@ -41,6 +41,7 @@ import time
|
|
| 41 |
import base64
|
| 42 |
import hmac
|
| 43 |
from functools import wraps
|
|
|
|
| 44 |
|
| 45 |
from fastapi import Body, Depends, Header, Query
|
| 46 |
from fastapi import APIRouter
|
|
@@ -49,13 +50,114 @@ from fastapi import APIRouter
|
|
| 49 |
# `rev_current` so the client can refetch that revision and retry rather than only toast.
|
| 50 |
# Bending `err` to take a fourth member would change the error shape of every route in the
|
| 51 |
# app; naming the exception here changes one.
|
| 52 |
-
from fastapi import HTTPException, Request
|
| 53 |
|
| 54 |
from deps import Session, err, require_session
|
|
|
|
| 55 |
|
| 56 |
router = APIRouter(prefix="/api/v1")
|
| 57 |
|
| 58 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
def _app_write_via(session, application_id):
|
| 60 |
"""Resolve write attribution through the viewer's stored App permissions."""
|
| 61 |
# Direct Python callers omit the header and receive FastAPI's parameter descriptor.
|
|
@@ -475,8 +577,52 @@ def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied=
|
|
| 475 |
return defn
|
| 476 |
|
| 477 |
|
| 478 |
-
def
|
| 479 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 480 |
# Share one snapshot across the read, editor and record-mode checks.
|
| 481 |
st = st if st is not None else _ut().lend(session.runtime)
|
| 482 |
# ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a
|
|
@@ -487,9 +633,14 @@ def _records_or_refuse(session, table_key, st=None):
|
|
| 487 |
if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st):
|
| 488 |
raise err(403, "database_read_only", "Can edit access is required to change this database.")
|
| 489 |
if not _ut().records_mutable(table_key, st=st):
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 493 |
return defn
|
| 494 |
|
| 495 |
|
|
@@ -906,7 +1057,8 @@ def scoped_pool(session: Session, table_key: str, st=None):
|
|
| 906 |
`workspace_wire` has run. `ut_assembly` applies it there, exactly where `grid_assembly` does.
|
| 907 |
"""
|
| 908 |
defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True)
|
| 909 |
-
fields_base = [dict(f) for f in (defn.get("fields") or [])]
|
|
|
|
| 910 |
field_keys = {f["key"] for f in fields_base}
|
| 911 |
if table_key == 'ut_activity':
|
| 912 |
rows_src = _activity_grid_rows(session)
|
|
@@ -956,6 +1108,46 @@ def scoped_pool(session: Session, table_key: str, st=None):
|
|
| 956 |
return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn
|
| 957 |
|
| 958 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 959 |
@router.patch("/tables/{table_key}/shared/{pid}")
|
| 960 |
@_c8_write
|
| 961 |
def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None),
|
|
@@ -981,8 +1173,9 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None),
|
|
| 981 |
key = str(body.get("field") or "").strip()
|
| 982 |
if not key:
|
| 983 |
raise err(400, "bad_request", "a field key is required")
|
| 984 |
-
|
| 985 |
-
|
|
|
|
| 986 |
if table_key == 'ut_activity':
|
| 987 |
raise err(403, 'records_read_only', 'Activity records are read-only.')
|
| 988 |
import core.shares as shares
|
|
@@ -996,8 +1189,13 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None),
|
|
| 996 |
except ValueError as exc:
|
| 997 |
raise err(400, 'bad_value', str(exc))
|
| 998 |
declared = {f.get('key') for f in definition.get('fields') or []}
|
| 999 |
-
|
| 1000 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1001 |
from core import grid_events
|
| 1002 |
from routes_grid import _ctx
|
| 1003 |
import uuid
|
|
@@ -1005,7 +1203,15 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None),
|
|
| 1005 |
if g.get('limits'):
|
| 1006 |
raise err(409, 'pid_scope_unresolved',
|
| 1007 |
'The permitted record set could not be resolved. This cell was not saved.')
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1008 |
ctx = _ctx(session, g['fields'], g['pids'], scope_key=table_key)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1009 |
event_id = uuid.uuid4().hex
|
| 1010 |
grid_events.handle_one({'id': event_id, 'type': 'overlay_patch', 'pid': pid,
|
| 1011 |
'updates': {key: body.get('value')}}, ctx)
|
|
@@ -1034,8 +1240,9 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None),
|
|
| 1034 |
# ⚠ COLUMNS THAT PREDATE THIS CARRY NO MARK and stay tenant-wide, unchanged. Reading
|
| 1035 |
# their absence as "granted to nobody" would blank every existing shared column at once.
|
| 1036 |
shared_overlay.put_field(table_key, key, {
|
| 1037 |
-
"key": key, "label": str(body.get("label") or key),
|
| 1038 |
-
|
|
|
|
| 1039 |
perm_scope.FIELD_GRANT_MARK: True,
|
| 1040 |
"createdBy": session.uname}, st=session.runtime)
|
| 1041 |
try:
|
|
@@ -1241,7 +1448,8 @@ def scoped_pids(session: Session, table_key: str, limits=None, st=None):
|
|
| 1241 |
# D-213 bought on the database-switch path is that this wall answers from the 0.1% document,
|
| 1242 |
# and handing it `lend()` would quietly take that back while looking like an optimisation.
|
| 1243 |
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 1244 |
-
fields_base = [dict(f) for f in (defn.get("fields") or [])]
|
|
|
|
| 1245 |
if table_key == 'ut_activity':
|
| 1246 |
if limits is not None:
|
| 1247 |
limits.append({'subject': 'pids', 'effect': 'unresolved',
|
|
@@ -1333,6 +1541,24 @@ def _ut_hidden(session, table_key, fields, st=None):
|
|
| 1333 |
return perm_scope.hidden_keys(session.user, table_key, fields, st=st)
|
| 1334 |
|
| 1335 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1336 |
def _ut_contract_fields(table_key, fields, st=None):
|
| 1337 |
"""The base contract every `ut_*` read door presents before its field wall.
|
| 1338 |
|
|
@@ -1341,7 +1567,7 @@ def _ut_contract_fields(table_key, fields, st=None):
|
|
| 1341 |
Keeping this boundary named prevents `/nav/schema` and the grid from attaching it on
|
| 1342 |
opposite sides of that wall.
|
| 1343 |
"""
|
| 1344 |
-
return _ut().with_zone_field(table_key, fields, st=st)
|
| 1345 |
|
| 1346 |
|
| 1347 |
def _ut_shared_fields(session, table_key, fields, workspace_document=None,
|
|
@@ -2941,6 +3167,8 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 2941 |
for zone in _ut().zones(table_key, st=_lent)]
|
| 2942 |
from core import connector_spec
|
| 2943 |
rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime)
|
|
|
|
|
|
|
| 2944 |
return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"],
|
| 2945 |
"refusals": metric_refusals,
|
| 2946 |
"zones": _zones,
|
|
@@ -3710,24 +3938,43 @@ def delete_row(table_key: str, rid: str, session: Session = Depends(require_sess
|
|
| 3710 |
|
| 3711 |
|
| 3712 |
@router.patch("/tables/{table_key}/rows/{pid}")
|
|
|
|
|
|
|
| 3713 |
def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
|
| 3714 |
session: Session = Depends(require_session),
|
| 3715 |
-
application_id: str | None = Header(default=None, alias="X-Loopable-App")
|
|
|
|
| 3716 |
"""Cell edits — the products PATCH on the user-table ctx. Routed through
|
| 3717 |
`core.grid_events.handle_one` so truncation and permission rules stay ONE implementation;
|
| 3718 |
the accepted values are read BACK from the bucket, never echoed from the request."""
|
| 3719 |
-
from core import grid_events
|
| 3720 |
|
| 3721 |
via = _app_write_via(session, application_id)
|
| 3722 |
updates = dict(body or {})
|
|
|
|
| 3723 |
if not updates:
|
| 3724 |
raise err(400, "empty_patch", "no fields to update")
|
| 3725 |
if _ut().ZONE_FIELD_KEY in updates or "zone" in updates:
|
| 3726 |
raise err(403, "zone_membership_read_only",
|
| 3727 |
"Zone membership is worked out from the record's current location")
|
| 3728 |
_managed_or_refuse(session, table_key, mutation=True)
|
| 3729 |
-
_records_or_refuse(session, table_key
|
| 3730 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3731 |
if pid not in g["pids"]:
|
| 3732 |
raise err(403, "out_of_scope", "that row is not in this database")
|
| 3733 |
ctx = grid_events.EventCtx(
|
|
@@ -3739,9 +3986,19 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
|
|
| 3739 |
hidden_keys=g.get("hidden") or frozenset(),
|
| 3740 |
st=session.runtime, # R6b (D-16)
|
| 3741 |
scope_key=table_key, table=_ops(session, table_key), via=via)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3742 |
try:
|
| 3743 |
refused = grid_events.handle_one(
|
| 3744 |
-
{"id": f"patch:{table_key}:{pid}:{time.time_ns()}",
|
|
|
|
| 3745 |
"pid": pid, "updates": updates}, ctx)
|
| 3746 |
except grid_events.StoreUnavailable:
|
| 3747 |
raise err(503, "store_unavailable",
|
|
@@ -3755,21 +4012,12 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
|
|
| 3755 |
"cell_write_refused", "row_outside_audience", "bad_row_id"):
|
| 3756 |
raise err(400, _refusal.get("code") or "bad_value",
|
| 3757 |
_refusal.get("message") or "the edit was refused")
|
| 3758 |
-
#
|
| 3759 |
-
#
|
| 3760 |
-
# ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH
|
| 3761 |
-
# STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket.
|
| 3762 |
-
# It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's
|
| 3763 |
-
# OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began
|
| 3764 |
-
# routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells`
|
| 3765 |
-
# (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring
|
| 3766 |
-
# claiming otherwise is what makes the next reader look for a merge bug that is not here.
|
| 3767 |
-
# ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY
|
| 3768 |
-
# by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took`
|
| 3769 |
-
# asks whether THIS write landed, and this write goes to the definition.
|
| 3770 |
-
# `get_record` answers the TABLE entry holding this one record; the read-back is the record.
|
| 3771 |
stored = dict(((_ut().get_record(table_key, pid, st=session.runtime) or {}).get("rows") or {})
|
| 3772 |
.get(str(pid)) or {})
|
|
|
|
|
|
|
|
|
|
| 3773 |
accepted = {k: stored.get(k) for k in updates if k in stored}
|
| 3774 |
|
| 3775 |
def _took(k):
|
|
|
|
| 41 |
import base64
|
| 42 |
import hmac
|
| 43 |
from functools import wraps
|
| 44 |
+
from inspect import signature
|
| 45 |
|
| 46 |
from fastapi import Body, Depends, Header, Query
|
| 47 |
from fastapi import APIRouter
|
|
|
|
| 50 |
# `rev_current` so the client can refetch that revision and retry rather than only toast.
|
| 51 |
# Bending `err` to take a fourth member would change the error shape of every route in the
|
| 52 |
# app; naming the exception here changes one.
|
| 53 |
+
from fastapi import HTTPException, Request, Response
|
| 54 |
|
| 55 |
from deps import Session, err, require_session
|
| 56 |
+
from routes_grid import row_patch_receipt
|
| 57 |
|
| 58 |
router = APIRouter(prefix="/api/v1")
|
| 59 |
|
| 60 |
|
| 61 |
+
def _row_patch_identity(fn=None, *, module=None):
|
| 62 |
+
"""Stamp authenticated row acknowledgements and refusals, including receipt replays."""
|
| 63 |
+
if fn is None:
|
| 64 |
+
return lambda handler: _row_patch_identity(handler, module=module)
|
| 65 |
+
sig = signature(fn)
|
| 66 |
+
|
| 67 |
+
@wraps(fn)
|
| 68 |
+
def wrapped(*args, **kwargs):
|
| 69 |
+
from deps import _TENANT_HEADER, _PRINCIPAL_HEADER, _principal
|
| 70 |
+
bound = sig.bind(*args, **kwargs)
|
| 71 |
+
session = bound.arguments['session']
|
| 72 |
+
headers = {_TENANT_HEADER: str(session.tenant),
|
| 73 |
+
_PRINCIPAL_HEADER: _principal(session.tenant, session.uname)}
|
| 74 |
+
response = bound.arguments.get('response')
|
| 75 |
+
if response is not None:
|
| 76 |
+
response.headers.update(headers)
|
| 77 |
+
try:
|
| 78 |
+
if module is not None:
|
| 79 |
+
session.require(module)
|
| 80 |
+
return fn(*args, **kwargs)
|
| 81 |
+
except HTTPException as exc:
|
| 82 |
+
exc.headers = {**(exc.headers or {}), **headers}
|
| 83 |
+
raise
|
| 84 |
+
return wrapped
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _note_delta(updates):
|
| 88 |
+
"""Reserve one note field per operation and refuse mixed scalar writes."""
|
| 89 |
+
key = _ut().CELL_NOTES_KEY
|
| 90 |
+
if key not in updates:
|
| 91 |
+
return None
|
| 92 |
+
notes = updates[key]
|
| 93 |
+
if (set(updates) != {key} or not isinstance(notes, dict) or len(notes) != 1
|
| 94 |
+
or any(not isinstance(field, str) or not field or field.startswith('_')
|
| 95 |
+
or (text is not None and not isinstance(text, str)) for field, text in notes.items())):
|
| 96 |
+
raise err(400, 'bad_cell_notes', 'Send one changed field note as text or null.')
|
| 97 |
+
return notes
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _patch_cell_notes(table_key, pid, notes, ctx, *, source_row_exists=None):
|
| 101 |
+
"""Use the outer row operation's runtime and receipt for the canonical note merge."""
|
| 102 |
+
from core import edit_receipts
|
| 103 |
+
identity = edit_receipts.current_op_id()
|
| 104 |
+
if identity is None:
|
| 105 |
+
raise err(400, 'bad_op_id', 'A note edit needs its opId.')
|
| 106 |
+
try:
|
| 107 |
+
return _ut().patch_cell_notes(table_key, pid, notes, ctx, op_id=identity,
|
| 108 |
+
source_row_exists=source_row_exists)
|
| 109 |
+
except _ut().CellNoteRefused as exc:
|
| 110 |
+
raise err(400 if exc.code == 'bad_cell_notes' else 403, exc.code, str(exc)) from exc
|
| 111 |
+
except edit_receipts.ReceiptUnavailable as exc:
|
| 112 |
+
raise err(503, 'store_unavailable', str(exc)) from exc
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _connected_note_context(session, table_key, pid):
|
| 116 |
+
"""Admit one mirrored note record through the shared SQL row and field walls."""
|
| 117 |
+
import core.perm_scope as perm_scope
|
| 118 |
+
import routes_odoo_tables as connected
|
| 119 |
+
read = connected._compiled_read(table_key, session)
|
| 120 |
+
try:
|
| 121 |
+
rows = connected._relation_rows(session, table_key, [], ids=[pid], compiled=read,
|
| 122 |
+
materialised=False)
|
| 123 |
+
return {'pids': frozenset(row['pid'] for row in rows), 'fields': read['fields'], 'limits': [],
|
| 124 |
+
'hidden': perm_scope.hidden_keys(session.user, table_key, read['fields'], st=session.runtime)}
|
| 125 |
+
finally:
|
| 126 |
+
read['cur'].close()
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _builtin_note_context(session, table_key, pid):
|
| 130 |
+
"""Admit one canonical customer or deduplicated product without reading its whole pool."""
|
| 131 |
+
from core import perm_scope
|
| 132 |
+
from registry_source import canonical_page
|
| 133 |
+
from routes_odoo_tables import _mirror_cur
|
| 134 |
+
try:
|
| 135 |
+
cursor, mismatch = _mirror_cur(session)
|
| 136 |
+
except RuntimeError as exc:
|
| 137 |
+
raise err(503, 'store_not_ready', str(exc)) from exc
|
| 138 |
+
if mismatch:
|
| 139 |
+
raise err(409, 'cross_tenant_store', mismatch)
|
| 140 |
+
try:
|
| 141 |
+
page = canonical_page(session, cursor, table_key, [pid])
|
| 142 |
+
return {'pids': frozenset(row['pid'] for row in page['rows']), 'fields': page['fields'],
|
| 143 |
+
'hidden': perm_scope.hidden_keys(session.user, table_key, page['fields'], st=session.runtime)}
|
| 144 |
+
finally:
|
| 145 |
+
cursor.close()
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _project_cell_notes(table_key, rows, fields, st, *, hidden=()):
|
| 149 |
+
"""Attach metadata only for this authorized page and its readable fields."""
|
| 150 |
+
if not rows:
|
| 151 |
+
return rows
|
| 152 |
+
visible = {field['key'] for field in fields} - set(hidden)
|
| 153 |
+
notes = _ut().cell_notes(table_key, [row['pid'] for row in rows], field_keys=visible, st=st)
|
| 154 |
+
for row in rows:
|
| 155 |
+
row.pop(_ut().CELL_NOTES_KEY, None)
|
| 156 |
+
if str(row['pid']) in notes:
|
| 157 |
+
row[_ut().CELL_NOTES_KEY] = notes[str(row['pid'])]
|
| 158 |
+
return rows
|
| 159 |
+
|
| 160 |
+
|
| 161 |
def _app_write_via(session, application_id):
|
| 162 |
"""Resolve write attribution through the viewer's stored App permissions."""
|
| 163 |
# Direct Python callers omit the header and receive FastAPI's parameter descriptor.
|
|
|
|
| 577 |
return defn
|
| 578 |
|
| 579 |
|
| 580 |
+
def _locked_user_field(field):
|
| 581 |
+
"""Admit authored values on locked databases without admitting source or computed cells."""
|
| 582 |
+
if not isinstance(field, dict):
|
| 583 |
+
return False
|
| 584 |
+
automation = field.get("automation") or {}
|
| 585 |
+
return (field.get("source") == "overlay"
|
| 586 |
+
and field.get("origin", "user") == "user"
|
| 587 |
+
and field.get("valueAuthority", "user-entered") == "user-entered"
|
| 588 |
+
and not field.get("preset")
|
| 589 |
+
and not field.get("derived")
|
| 590 |
+
and not (isinstance(automation, dict)
|
| 591 |
+
and (automation.get("flowId") or automation.get("preset")))
|
| 592 |
+
and field.get("type") in _ut().UT_FIELD_TYPES - {"automation"}
|
| 593 |
+
and not field.get("measure")
|
| 594 |
+
and not _ut().is_computed_cell(field))
|
| 595 |
+
|
| 596 |
+
|
| 597 |
+
def _records_lock_message(table_key, definition, st):
|
| 598 |
+
"""Name the database's declared source when its record lock refuses a write."""
|
| 599 |
+
from core import connector_spec, connector_sync, keychain
|
| 600 |
+
identity = _ut().connector_database_identity(table_key, st=st, definition=definition)
|
| 601 |
+
source = ""
|
| 602 |
+
if identity:
|
| 603 |
+
provider = identity["provider"]
|
| 604 |
+
provider = next((key for key, spec in connector_spec.CONNECTOR_SPEC_KEYS.items()
|
| 605 |
+
if spec == provider), provider)
|
| 606 |
+
source = (connector_sync.providers().get(provider) or {}).get("label", "")
|
| 607 |
+
if not source:
|
| 608 |
+
label = keychain.kind_label(provider)
|
| 609 |
+
source = label if label != "Connection" else ""
|
| 610 |
+
if not source:
|
| 611 |
+
import automation_engine as engine
|
| 612 |
+
for platform in engine.PLATFORMS:
|
| 613 |
+
try:
|
| 614 |
+
schema = engine.platform_schema(platform)
|
| 615 |
+
except ValueError:
|
| 616 |
+
continue
|
| 617 |
+
if table_key in schema["locked_tables"]:
|
| 618 |
+
source = schema["platform"]
|
| 619 |
+
break
|
| 620 |
+
name = f"{source} database" if source else "database"
|
| 621 |
+
return f"This {name} is locked, so only user-created field values can be edited."
|
| 622 |
+
|
| 623 |
+
|
| 624 |
+
def _records_or_refuse(session, table_key, st=None, *, cell_updates=None):
|
| 625 |
+
"""Require editor authority and reserve locked database edits for authored field values."""
|
| 626 |
# Share one snapshot across the read, editor and record-mode checks.
|
| 627 |
st = st if st is not None else _ut().lend(session.runtime)
|
| 628 |
# ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a
|
|
|
|
| 633 |
if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st):
|
| 634 |
raise err(403, "database_read_only", "Can edit access is required to change this database.")
|
| 635 |
if not _ut().records_mutable(table_key, st=st):
|
| 636 |
+
if cell_updates and table_key != "ut_activity":
|
| 637 |
+
from core import shared_overlay
|
| 638 |
+
fields = dict(shared_overlay.fields(table_key, st=st))
|
| 639 |
+
# A shared field cannot redefine a connector-owned column with the same key.
|
| 640 |
+
fields.update({field["key"]: field for field in defn.get("fields") or []})
|
| 641 |
+
if all(_locked_user_field(fields.get(key)) for key in cell_updates):
|
| 642 |
+
return defn
|
| 643 |
+
raise err(403, "records_read_only", _records_lock_message(table_key, defn, st))
|
| 644 |
return defn
|
| 645 |
|
| 646 |
|
|
|
|
| 1057 |
`workspace_wire` has run. `ut_assembly` applies it there, exactly where `grid_assembly` does.
|
| 1058 |
"""
|
| 1059 |
defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True)
|
| 1060 |
+
fields_base = _source_contract_fields(table_key, [dict(f) for f in (defn.get("fields") or [])],
|
| 1061 |
+
st=st if st is not None else session.runtime)
|
| 1062 |
field_keys = {f["key"] for f in fields_base}
|
| 1063 |
if table_key == 'ut_activity':
|
| 1064 |
rows_src = _activity_grid_rows(session)
|
|
|
|
| 1108 |
return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn
|
| 1109 |
|
| 1110 |
|
| 1111 |
+
def _new_locked_shared_field(session, table_key, pid, key, body):
|
| 1112 |
+
"""Admit an explicit authored declaration and its first value before creating any artifacts."""
|
| 1113 |
+
_managed_or_refuse(session, table_key, mutation=True)
|
| 1114 |
+
if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin,
|
| 1115 |
+
st=session.runtime):
|
| 1116 |
+
raise err(403, "database_read_only", "Can edit access is required to add a column.")
|
| 1117 |
+
proposed = dict(body, key=key)
|
| 1118 |
+
proposed.setdefault("source", "overlay")
|
| 1119 |
+
proposed.setdefault("type", "text")
|
| 1120 |
+
if (key in {"pid", "zone", _ut().ZONE_FIELD_KEY}
|
| 1121 |
+
or not _locked_user_field(proposed)
|
| 1122 |
+
or proposed.get("scope", "tenant") != "tenant"
|
| 1123 |
+
or any(proposed.get(name) for name in (
|
| 1124 |
+
"formula", "rollup", "if", "automation", "metric", "measure", "link"))):
|
| 1125 |
+
raise err(400, "bad_field", "Only a user-created value column can be added here.")
|
| 1126 |
+
fields = _ut().clean_fields([proposed])
|
| 1127 |
+
if not fields or fields[0]["key"] != key or not _locked_user_field(fields[0]):
|
| 1128 |
+
raise err(400, "bad_field", "The column name or type is invalid.")
|
| 1129 |
+
field = fields[0]
|
| 1130 |
+
reason = _ut().cell_type_refusal(field, body.get("value"))
|
| 1131 |
+
if reason:
|
| 1132 |
+
raise err(400, "bad_value", reason)
|
| 1133 |
+
if field["type"] == "json" and body.get("value") not in (None, ""):
|
| 1134 |
+
value = body["value"]
|
| 1135 |
+
try:
|
| 1136 |
+
json.loads(value if isinstance(value, str) else json.dumps(value))
|
| 1137 |
+
except (TypeError, ValueError):
|
| 1138 |
+
raise err(400, "bad_value", "This column holds JSON. Enter a valid JSON value.")
|
| 1139 |
+
# The write context resolves connected membership through the same row wall as existing edits.
|
| 1140 |
+
g = ut_write_ctx(session, table_key)
|
| 1141 |
+
if g.get("limits"):
|
| 1142 |
+
raise err(409, "pid_scope_unresolved",
|
| 1143 |
+
"The permitted record set could not be resolved. This cell was not saved.")
|
| 1144 |
+
if pid not in g["pids"]:
|
| 1145 |
+
raise err(403, "out_of_scope", "That row is not in this database.")
|
| 1146 |
+
if key in (g.get("hidden") or ()) or key in {f["key"] for f in g["fields"]}:
|
| 1147 |
+
raise err(403, "field_read_only", "This field is outside your edit access.")
|
| 1148 |
+
return field
|
| 1149 |
+
|
| 1150 |
+
|
| 1151 |
@router.patch("/tables/{table_key}/shared/{pid}")
|
| 1152 |
@_c8_write
|
| 1153 |
def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None),
|
|
|
|
| 1173 |
key = str(body.get("field") or "").strip()
|
| 1174 |
if not key:
|
| 1175 |
raise err(400, "bad_request", "a field key is required")
|
| 1176 |
+
connected = _ut().is_connected(table_key, st=session.runtime)
|
| 1177 |
+
locked = not _ut().records_mutable(table_key, st=session.runtime)
|
| 1178 |
+
definition = _defn_or_refuse(session, table_key, scope_applied=connected or locked)
|
| 1179 |
if table_key == 'ut_activity':
|
| 1180 |
raise err(403, 'records_read_only', 'Activity records are read-only.')
|
| 1181 |
import core.shares as shares
|
|
|
|
| 1189 |
except ValueError as exc:
|
| 1190 |
raise err(400, 'bad_value', str(exc))
|
| 1191 |
declared = {f.get('key') for f in definition.get('fields') or []}
|
| 1192 |
+
shared = shared_overlay.is_shared(table_key, key, st=session.runtime)
|
| 1193 |
+
new_field = None
|
| 1194 |
+
# A label explicitly declares a new column; a bare unknown-key edit still fails closed.
|
| 1195 |
+
if locked and key not in declared and not shared and body.get('label'):
|
| 1196 |
+
new_field = _new_locked_shared_field(session, table_key, pid, key, body)
|
| 1197 |
+
if new_field is None and (locked or (connected and (key in declared or shared))):
|
| 1198 |
+
_records_or_refuse(session, table_key, cell_updates={key: body.get('value')})
|
| 1199 |
from core import grid_events
|
| 1200 |
from routes_grid import _ctx
|
| 1201 |
import uuid
|
|
|
|
| 1203 |
if g.get('limits'):
|
| 1204 |
raise err(409, 'pid_scope_unresolved',
|
| 1205 |
'The permitted record set could not be resolved. This cell was not saved.')
|
| 1206 |
+
if pid not in g['pids']:
|
| 1207 |
+
raise err(403, 'out_of_scope', 'That row is not in this database.')
|
| 1208 |
+
if key not in {field['key'] for field in g['fields']} or key in (g.get('hidden') or ()):
|
| 1209 |
+
raise err(403, 'field_read_only', 'This field is outside your access.')
|
| 1210 |
ctx = _ctx(session, g['fields'], g['pids'], scope_key=table_key)
|
| 1211 |
+
ctx.hidden_keys = g.get('hidden') or frozenset()
|
| 1212 |
+
field = next(field for field in g['fields'] if field['key'] == key)
|
| 1213 |
+
if locked and not grid_events._may_edit_field_value(ctx, key, field):
|
| 1214 |
+
raise err(403, 'field_read_only', 'This field is outside your edit access.')
|
| 1215 |
event_id = uuid.uuid4().hex
|
| 1216 |
grid_events.handle_one({'id': event_id, 'type': 'overlay_patch', 'pid': pid,
|
| 1217 |
'updates': {key: body.get('value')}}, ctx)
|
|
|
|
| 1240 |
# ⚠ COLUMNS THAT PREDATE THIS CARRY NO MARK and stay tenant-wide, unchanged. Reading
|
| 1241 |
# their absence as "granted to nobody" would blank every existing shared column at once.
|
| 1242 |
shared_overlay.put_field(table_key, key, {
|
| 1243 |
+
**(new_field or {"key": key, "label": str(body.get("label") or key),
|
| 1244 |
+
"source": "overlay", "type": str(body.get("type") or "text")}),
|
| 1245 |
+
"shared": True,
|
| 1246 |
perm_scope.FIELD_GRANT_MARK: True,
|
| 1247 |
"createdBy": session.uname}, st=session.runtime)
|
| 1248 |
try:
|
|
|
|
| 1448 |
# D-213 bought on the database-switch path is that this wall answers from the 0.1% document,
|
| 1449 |
# and handing it `lend()` would quietly take that back while looking like an optimisation.
|
| 1450 |
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 1451 |
+
fields_base = _source_contract_fields(table_key, [dict(f) for f in (defn.get("fields") or [])],
|
| 1452 |
+
st=st if st is not None else session.runtime)
|
| 1453 |
if table_key == 'ut_activity':
|
| 1454 |
if limits is not None:
|
| 1455 |
limits.append({'subject': 'pids', 'effect': 'unresolved',
|
|
|
|
| 1541 |
return perm_scope.hidden_keys(session.user, table_key, fields, st=st)
|
| 1542 |
|
| 1543 |
|
| 1544 |
+
def _source_contract_fields(table_key, fields, st=None):
|
| 1545 |
+
"""Keep the Odoo mirror label on old saved definitions and fresh source contracts."""
|
| 1546 |
+
if str(table_key).startswith('ut_odoo_'):
|
| 1547 |
+
from routes_odoo_tables import _rel, _sources, mirror_store_field
|
| 1548 |
+
if _rel().base_table_key(table_key) in _sources():
|
| 1549 |
+
return [field for field in fields if field.get('key') != 'Store'] + [mirror_store_field()]
|
| 1550 |
+
if str(table_key).startswith('ut_meta_') and callable(getattr(st, 'assert_datastore_matches', None)):
|
| 1551 |
+
from routes_odoo_tables import _table_cursor, _meta_source, meta_source_fields, _SOURCE_PRIVATE
|
| 1552 |
+
cursor = _table_cursor(st, table_key)
|
| 1553 |
+
try:
|
| 1554 |
+
spec = _meta_source(cursor, table_key)
|
| 1555 |
+
if spec is not None:
|
| 1556 |
+
return [field for field in meta_source_fields(spec, fields) if field['key'] not in _SOURCE_PRIVATE]
|
| 1557 |
+
finally:
|
| 1558 |
+
cursor.close()
|
| 1559 |
+
return fields
|
| 1560 |
+
|
| 1561 |
+
|
| 1562 |
def _ut_contract_fields(table_key, fields, st=None):
|
| 1563 |
"""The base contract every `ut_*` read door presents before its field wall.
|
| 1564 |
|
|
|
|
| 1567 |
Keeping this boundary named prevents `/nav/schema` and the grid from attaching it on
|
| 1568 |
opposite sides of that wall.
|
| 1569 |
"""
|
| 1570 |
+
return _ut().with_zone_field(table_key, _source_contract_fields(table_key, fields, st=st), st=st)
|
| 1571 |
|
| 1572 |
|
| 1573 |
def _ut_shared_fields(session, table_key, fields, workspace_document=None,
|
|
|
|
| 3167 |
for zone in _ut().zones(table_key, st=_lent)]
|
| 3168 |
from core import connector_spec
|
| 3169 |
rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime)
|
| 3170 |
+
rows = _project_cell_notes(table_key, rows, projection_fields, session.runtime,
|
| 3171 |
+
hidden=g.get('hidden') or ())
|
| 3172 |
return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"],
|
| 3173 |
"refusals": metric_refusals,
|
| 3174 |
"zones": _zones,
|
|
|
|
| 3938 |
|
| 3939 |
|
| 3940 |
@router.patch("/tables/{table_key}/rows/{pid}")
|
| 3941 |
+
@_row_patch_identity
|
| 3942 |
+
@row_patch_receipt(lambda args: args['table_key'])
|
| 3943 |
def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
|
| 3944 |
session: Session = Depends(require_session),
|
| 3945 |
+
application_id: str | None = Header(default=None, alias="X-Loopable-App"),
|
| 3946 |
+
response: Response = None):
|
| 3947 |
"""Cell edits — the products PATCH on the user-table ctx. Routed through
|
| 3948 |
`core.grid_events.handle_one` so truncation and permission rules stay ONE implementation;
|
| 3949 |
the accepted values are read BACK from the bucket, never echoed from the request."""
|
| 3950 |
+
from core import grid_events, edit_receipts
|
| 3951 |
|
| 3952 |
via = _app_write_via(session, application_id)
|
| 3953 |
updates = dict(body or {})
|
| 3954 |
+
notes = _note_delta(updates)
|
| 3955 |
if not updates:
|
| 3956 |
raise err(400, "empty_patch", "no fields to update")
|
| 3957 |
if _ut().ZONE_FIELD_KEY in updates or "zone" in updates:
|
| 3958 |
raise err(403, "zone_membership_read_only",
|
| 3959 |
"Zone membership is worked out from the record's current location")
|
| 3960 |
_managed_or_refuse(session, table_key, mutation=True)
|
| 3961 |
+
definition = _records_or_refuse(session, table_key,
|
| 3962 |
+
st=_table_lend(session.runtime, table_key, defs_only=True) if notes is not None else None,
|
| 3963 |
+
cell_updates=notes if notes is not None else updates)
|
| 3964 |
+
connected = _ut().is_connected(table_key, st=session.runtime)
|
| 3965 |
+
if notes is not None and connected:
|
| 3966 |
+
from routes_odoo_tables import _sources
|
| 3967 |
+
mirror_note = (table_key in _sources()
|
| 3968 |
+
or not _ut().materialises(table_key, st=session.runtime, defn=definition))
|
| 3969 |
+
else:
|
| 3970 |
+
mirror_note = False
|
| 3971 |
+
g = (_connected_note_context(session, table_key, pid) if mirror_note else
|
| 3972 |
+
ut_write_ctx(session, table_key) if connected else
|
| 3973 |
+
ut_assembly(session, table_key, consume_corrections=False))
|
| 3974 |
+
if any(limit.get('subject') == 'pids' and limit.get('effect') == 'unresolved'
|
| 3975 |
+
for limit in g.get('limits') or ()):
|
| 3976 |
+
raise err(409, 'pid_scope_unresolved',
|
| 3977 |
+
'The permitted record set could not be resolved. This edit was not saved.')
|
| 3978 |
if pid not in g["pids"]:
|
| 3979 |
raise err(403, "out_of_scope", "that row is not in this database")
|
| 3980 |
ctx = grid_events.EventCtx(
|
|
|
|
| 3986 |
hidden_keys=g.get("hidden") or frozenset(),
|
| 3987 |
st=session.runtime, # R6b (D-16)
|
| 3988 |
scope_key=table_key, table=_ops(session, table_key), via=via)
|
| 3989 |
+
if notes is not None:
|
| 3990 |
+
return _patch_cell_notes(table_key, pid, notes, ctx,
|
| 3991 |
+
source_row_exists=lambda key, rid, admitted: key == table_key and int(rid) in g['pids'])
|
| 3992 |
+
if not _ut().records_mutable(table_key, st=session.runtime):
|
| 3993 |
+
fields = {field['key']: field for field in g['fields']}
|
| 3994 |
+
if any(key not in fields or key in ctx.hidden_keys
|
| 3995 |
+
or not grid_events._may_edit_field_value(ctx, key, fields[key])
|
| 3996 |
+
for key in updates):
|
| 3997 |
+
raise err(403, 'field_read_only', 'This field is outside your edit access.')
|
| 3998 |
try:
|
| 3999 |
refused = grid_events.handle_one(
|
| 4000 |
+
{"id": edit_receipts.current_op_id() or f"patch:{table_key}:{pid}:{time.time_ns()}",
|
| 4001 |
+
"type": "overlay_patch",
|
| 4002 |
"pid": pid, "updates": updates}, ctx)
|
| 4003 |
except grid_events.StoreUnavailable:
|
| 4004 |
raise err(503, "store_unavailable",
|
|
|
|
| 4012 |
"cell_write_refused", "row_outside_audience", "bad_row_id"):
|
| 4013 |
raise err(400, _refusal.get("code") or "bad_value",
|
| 4014 |
_refusal.get("message") or "the edit was refused")
|
| 4015 |
+
# Connected edits live in the authored overlay, while ordinary edits live in definition rows.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4016 |
stored = dict(((_ut().get_record(table_key, pid, st=session.runtime) or {}).get("rows") or {})
|
| 4017 |
.get(str(pid)) or {})
|
| 4018 |
+
if connected:
|
| 4019 |
+
from core import shared_overlay
|
| 4020 |
+
stored.update(shared_overlay.cells(table_key, [pid], st=session.runtime).get(str(pid)) or {})
|
| 4021 |
accepted = {k: stored.get(k) for k in updates if k in stored}
|
| 4022 |
|
| 4023 |
def _took(k):
|
api/routes_templates.py
CHANGED
|
@@ -37,12 +37,12 @@ router = APIRouter(prefix="/api/v1")
|
|
| 37 |
|
| 38 |
_TEMPLATE_CATEGORIES = {
|
| 39 |
"sheet": "Sheet", "data": "Table", "app": "App", "agents": "Automation", "saved": "Saved",
|
| 40 |
-
"route": "Operations", "catalog": "
|
| 41 |
}
|
| 42 |
# C1 connector keys for a shipped Sheet descriptor's `source` (F25: Amazon is `amazon_sp` there).
|
| 43 |
_SHEET_CONNECTORS = {"odoo": "odoo", "amazon_sp": "amazon"}
|
| 44 |
# C1 fields each App row carries so the client can pick its screen and marks.
|
| 45 |
-
_APP_WIRE_KEYS = ("view", "mark", "connector", "process", "purpose")
|
| 46 |
_TEMPLATE_PREVIEW_LABELS = {**_TEMPLATE_CATEGORIES, "route": "Routing", "catalog": "Catalog",
|
| 47 |
"competitor_watch": "Competitor watch"}
|
| 48 |
|
|
@@ -249,7 +249,7 @@ def apply_template(key: str, body: dict = Body(default=None),
|
|
| 249 |
# keeps the reference (the read projection in `grid_events.table_workspace` still masks it).
|
| 250 |
try:
|
| 251 |
import application_registry
|
| 252 |
-
raw_registry =
|
| 253 |
live_apps = (set(raw_registry["applications"]) if isinstance(raw_registry, dict)
|
| 254 |
and isinstance(raw_registry.get("applications"), dict) else None)
|
| 255 |
except Exception:
|
|
@@ -515,11 +515,12 @@ def _module_templates(session):
|
|
| 515 |
connector = "odoo" if row["source"].startswith("odoo_") else row["source"]
|
| 516 |
data.append(_catalog_row(row["key"], row["label"], row["desc"],
|
| 517 |
[requirement(database, connector)], module="data"))
|
| 518 |
-
#
|
| 519 |
-
#
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
**{key:
|
|
|
|
| 523 |
for row in application_templates(session=session)["templates"]]
|
| 524 |
# Owned copies are optional links. They never replace the template's create action.
|
| 525 |
owned = _own_template_apps(session)
|
|
|
|
| 37 |
|
| 38 |
_TEMPLATE_CATEGORIES = {
|
| 39 |
"sheet": "Sheet", "data": "Table", "app": "App", "agents": "Automation", "saved": "Saved",
|
| 40 |
+
"route": "Operations", "catalog": "Odoo", "competitor_watch": "Amazon",
|
| 41 |
}
|
| 42 |
# C1 connector keys for a shipped Sheet descriptor's `source` (F25: Amazon is `amazon_sp` there).
|
| 43 |
_SHEET_CONNECTORS = {"odoo": "odoo", "amazon_sp": "amazon"}
|
| 44 |
# C1 fields each App row carries so the client can pick its screen and marks.
|
| 45 |
+
_APP_WIRE_KEYS = ("view", "mark", "connector", "process", "purpose", "tags")
|
| 46 |
_TEMPLATE_PREVIEW_LABELS = {**_TEMPLATE_CATEGORIES, "route": "Routing", "catalog": "Catalog",
|
| 47 |
"competitor_watch": "Competitor watch"}
|
| 48 |
|
|
|
|
| 249 |
# keeps the reference (the read projection in `grid_events.table_workspace` still masks it).
|
| 250 |
try:
|
| 251 |
import application_registry
|
| 252 |
+
raw_registry = application_registry.load(session.runtime, strict=True)
|
| 253 |
live_apps = (set(raw_registry["applications"]) if isinstance(raw_registry, dict)
|
| 254 |
and isinstance(raw_registry.get("applications"), dict) else None)
|
| 255 |
except Exception:
|
|
|
|
| 515 |
connector = "odoo" if row["source"].startswith("odoo_") else row["source"]
|
| 516 |
data.append(_catalog_row(row["key"], row["label"], row["desc"],
|
| 517 |
[requirement(database, connector)], module="data"))
|
| 518 |
+
# Bound kinds keep their wizard callback; connector-gated Apps post through their template door.
|
| 519 |
+
# The registry's offer predicate checks the connector and source walls before listing an App.
|
| 520 |
+
app = [{**_catalog_row(row["kind"], row["label"], row.get("purpose") or row["label"], [], module="app",
|
| 521 |
+
creation_mode="wizard" if row["kind"] in app_templates.BOUND_KINDS else "post"),
|
| 522 |
+
**{key: (app_templates.BY_KIND[row["kind"]] if key == "tags" else row).get(key)
|
| 523 |
+
for key in _APP_WIRE_KEYS}}
|
| 524 |
for row in application_templates(session=session)["templates"]]
|
| 525 |
# Owned copies are optional links. They never replace the template's create action.
|
| 526 |
owned = _own_template_apps(session)
|
api/tool_groups/data.py
CHANGED
|
@@ -5,11 +5,40 @@ from core import external_tools
|
|
| 5 |
from . import legacy_action
|
| 6 |
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
READ_ARGUMENT_HELP = (
|
| 9 |
"Call table with action describe_database or query_database and an arguments object. "
|
| 10 |
"arguments.database must be a string. Put query options inside arguments. "
|
| 11 |
-
"fields must be an array of field keys.
|
| 12 |
-
"Read loopable_guide with group table for the selected action's input schema."
|
| 13 |
)
|
| 14 |
|
| 15 |
|
|
@@ -89,8 +118,8 @@ def query_syntax_section():
|
|
| 89 |
"one aggregate and no sorts (query_groups_invalid otherwise). Only a connected database, such as an "
|
| 90 |
"Odoo sales lines mirror, groups. Any other database answers 409 query_groups_unavailable: read its "
|
| 91 |
"records by page, or ask for aggregates without groupBy.",
|
| 92 |
-
"- Paging:
|
| 93 |
-
"
|
| 94 |
"accepted (query_arguments_invalid otherwise).",
|
| 95 |
"- Full pulls: no row cap applies to a connected source. Read every page at limit 500.",
|
| 96 |
]
|
|
@@ -212,7 +241,7 @@ def _delivery_photos(ctx, args):
|
|
| 212 |
|
| 213 |
GROUP = {
|
| 214 |
"name": "table", "title": "Table",
|
| 215 |
-
"guide": "Table: list databases, read fields, group and page records, propose changes, list an Odoo delivery's photos with delivery_photos, or report a limitation. Read loopable_guide(group='table') for actions and a grouping example.",
|
| 216 |
"actions": {
|
| 217 |
"operation_status": {
|
| 218 |
"read": True, "destructive": False, "confirm": False,
|
|
|
|
| 5 |
from . import legacy_action
|
| 6 |
|
| 7 |
|
| 8 |
+
QUERY_PAGING_HELP = (
|
| 9 |
+
'Send limit as a positive integer, for example {"limit": 100}. '
|
| 10 |
+
'A limit of 501 or more requests pages of at most 500 rows. '
|
| 11 |
+
'Send page.next_cursor as cursor with the same query until page.has_more is false. '
|
| 12 |
+
'There is no total row cap.')
|
| 13 |
+
DISCOVERY_HELP = (
|
| 14 |
+
"If your client exposes only the previous data tool, send the same action and arguments to that tool. "
|
| 15 |
+
"Refresh or reconnect the client to discover table. Read loopable_guide with group table for its actions."
|
| 16 |
+
)
|
| 17 |
+
READ_REFUSAL_FIXES = {
|
| 18 |
+
"assistant_snapshot_unavailable": (
|
| 19 |
+
"This database's current data is unavailable. Describe it to read its fields. "
|
| 20 |
+
"Try the same read again after its source sync completes."),
|
| 21 |
+
"assistant_store_not_ready": (
|
| 22 |
+
"This database is temporarily unavailable. Try the same read again shortly."),
|
| 23 |
+
"assistant_cross_tenant_store": (
|
| 24 |
+
"This connection cannot read that workspace. Reconnect to the intended workspace before reading."),
|
| 25 |
+
"assistant_scope_not_applicable": (
|
| 26 |
+
"This database's access restriction could not be applied. Ask an Administrator to repair it. "
|
| 27 |
+
"Do not remove or widen the restriction."),
|
| 28 |
+
"assistant_filter_not_answerable": (
|
| 29 |
+
"This filter cannot be applied. Use only the fields and operators returned by describe_database. "
|
| 30 |
+
"Keep the intended restriction when correcting the filter."),
|
| 31 |
+
"query_group_type": (
|
| 32 |
+
"Month grouping requires a date field. Use a date field returned by describe_database, "
|
| 33 |
+
"or omit unit to group by the field's values."),
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
READ_ARGUMENT_HELP = (
|
| 38 |
"Call table with action describe_database or query_database and an arguments object. "
|
| 39 |
"arguments.database must be a string. Put query options inside arguments. "
|
| 40 |
+
"fields must be an array of field keys. " + QUERY_PAGING_HELP + " "
|
| 41 |
+
"Read loopable_guide with group table for the selected action's input schema. " + DISCOVERY_HELP
|
| 42 |
)
|
| 43 |
|
| 44 |
|
|
|
|
| 118 |
"one aggregate and no sorts (query_groups_invalid otherwise). Only a connected database, such as an "
|
| 119 |
"Odoo sales lines mirror, groups. Any other database answers 409 query_groups_unavailable: read its "
|
| 120 |
"records by page, or ask for aggregates without groupBy.",
|
| 121 |
+
"- Paging: default 100. " + QUERY_PAGING_HELP
|
| 122 |
+
+ " Only fields, filters, sorts, limit, cursor, aggregates and groupBy are "
|
| 123 |
"accepted (query_arguments_invalid otherwise).",
|
| 124 |
"- Full pulls: no row cap applies to a connected source. Read every page at limit 500.",
|
| 125 |
]
|
|
|
|
| 241 |
|
| 242 |
GROUP = {
|
| 243 |
"name": "table", "title": "Table",
|
| 244 |
+
"guide": "Table: list databases, read fields, group and page records, propose changes, list an Odoo delivery's photos with delivery_photos, or report a limitation. Read loopable_guide(group='table') for actions and a grouping example. " + DISCOVERY_HELP,
|
| 245 |
"actions": {
|
| 246 |
"operation_status": {
|
| 247 |
"read": True, "destructive": False, "confirm": False,
|
api/tool_groups/sheet.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
"""Sheet actions over the existing workbook permissions and writers."""
|
| 2 |
from difflib import get_close_matches
|
|
|
|
| 3 |
|
| 4 |
from fastapi import HTTPException
|
| 5 |
from core import external_access, external_tools
|
|
@@ -8,6 +9,14 @@ from . import legacy_action
|
|
| 8 |
|
| 9 |
WORKBOOK_RESOURCE_URI = "ui://loopable/workbook.html"
|
| 10 |
WORKBOOK_RESOURCE_MIME = "text/html;profile=mcp-app"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
#: The next call after workbook_missing (S79-T11): list readable workbooks, then read one by its id.
|
| 12 |
WORKBOOK_MISSING_FIX = (
|
| 13 |
'Call sheet with {"action": "workbook.list", "arguments": {"limit": 100}} and follow cursor until it is null. '
|
|
@@ -103,6 +112,23 @@ def read_guide():
|
|
| 103 |
'A missing or hidden workbook returns workbook_missing. Readable workbooks return their current tabs.')
|
| 104 |
|
| 105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
def _workbook_view(ctx, args):
|
| 107 |
"""Project native workbook cells without exposing bindings or private workbook metadata."""
|
| 108 |
import copy
|
|
@@ -210,6 +236,7 @@ GROUP["guide"] += (
|
|
| 210 |
"Use write_batch for a checked sequence of supported writes. "
|
| 211 |
"Workbook export and named ranges have no grouped replacement in this build.")
|
| 212 |
GROUP["guide"] += ' Under MCP rule 9, ask "Do you want me to build a Sheet workbook in Loopable?".'
|
|
|
|
| 213 |
_BATCH_GUIDE = (
|
| 214 |
" A batch has 1 to 50 operations for one workbook. Use name and arguments for each operation. "
|
| 215 |
"Use sheet_add_tab, sheet_write_cells, sheet_set_binding, sheet_set_styles or sheet_set_dimensions "
|
|
|
|
| 1 |
"""Sheet actions over the existing workbook permissions and writers."""
|
| 2 |
from difflib import get_close_matches
|
| 3 |
+
import json
|
| 4 |
|
| 5 |
from fastapi import HTTPException
|
| 6 |
from core import external_access, external_tools
|
|
|
|
| 9 |
|
| 10 |
WORKBOOK_RESOURCE_URI = "ui://loopable/workbook.html"
|
| 11 |
WORKBOOK_RESOURCE_MIME = "text/html;profile=mcp-app"
|
| 12 |
+
WRITE_ABOUT_EXAMPLE = {
|
| 13 |
+
"title": "Sheet write", "description": "Update the requested workbook cells.", "tags": ["reporting"]}
|
| 14 |
+
WRITE_INTENT_EXAMPLE = "Update the requested workbook cells"
|
| 15 |
+
WRITE_CALL_EXAMPLE = {
|
| 16 |
+
"action": "write_cells", "arguments": {
|
| 17 |
+
"workbook_id": "<workbook_id>", "tab": "<tab_id>",
|
| 18 |
+
"cells": {"A1": {"kind": "literal", "value": "Weekly review"}}},
|
| 19 |
+
"intent": WRITE_INTENT_EXAMPLE, "about": WRITE_ABOUT_EXAMPLE}
|
| 20 |
#: The next call after workbook_missing (S79-T11): list readable workbooks, then read one by its id.
|
| 21 |
WORKBOOK_MISSING_FIX = (
|
| 22 |
'Call sheet with {"action": "workbook.list", "arguments": {"limit": 100}} and follow cursor until it is null. '
|
|
|
|
| 112 |
'A missing or hidden workbook returns workbook_missing. Readable workbooks return their current tabs.')
|
| 113 |
|
| 114 |
|
| 115 |
+
def write_guide():
|
| 116 |
+
"""Publish the metadata, argument shape and signed confirmation required for Sheet writes."""
|
| 117 |
+
return (
|
| 118 |
+
"Every Sheet write needs intent and about beside action and arguments. "
|
| 119 |
+
"If your client sends only action and arguments, put intent and about inside arguments. "
|
| 120 |
+
"Use intent as 1 to 500 characters explaining the write. "
|
| 121 |
+
"Use about with only title (1 to 240 characters), description (20 to 240 characters), "
|
| 122 |
+
"and tags (one or more distinct values). Use no control characters or long dashes in title or description. "
|
| 123 |
+
"Allowed tags: " + ", ".join(external_tools.BUILD_TAGS) + ". "
|
| 124 |
+
"Use workbook_id and tab from workbook.get. Call sheet with "
|
| 125 |
+
+ json.dumps(WRITE_CALL_EXAMPLE, separators=(", ", ": ")) + ". "
|
| 126 |
+
"The write returns preview_ready without changing cells. After the user approves, call confirm_change "
|
| 127 |
+
'with {"confirmation_token": "<returned signed token>", "intent": "Apply the approved Sheet write"}. '
|
| 128 |
+
"Confirmation uses the saved about."
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
def _workbook_view(ctx, args):
|
| 133 |
"""Project native workbook cells without exposing bindings or private workbook metadata."""
|
| 134 |
import copy
|
|
|
|
| 236 |
"Use write_batch for a checked sequence of supported writes. "
|
| 237 |
"Workbook export and named ranges have no grouped replacement in this build.")
|
| 238 |
GROUP["guide"] += ' Under MCP rule 9, ask "Do you want me to build a Sheet workbook in Loopable?".'
|
| 239 |
+
GROUP["guide"] += " " + write_guide()
|
| 240 |
_BATCH_GUIDE = (
|
| 241 |
" A batch has 1 to 50 operations for one workbook. Use name and arguments for each operation. "
|
| 242 |
"Use sheet_add_tab, sheet_write_cells, sheet_set_binding, sheet_set_styles or sheet_set_dimensions "
|
platform/ai_client_steps.json
CHANGED
|
@@ -24,116 +24,104 @@
|
|
| 24 |
"claude": {
|
| 25 |
"need": "Access to Claude.ai and your Loopable sign in.",
|
| 26 |
"failureAction": "Finish the Loopable sign in from Claude, then select Verify sign in again. Check your workspace policy if the connector is unavailable.",
|
| 27 |
-
"testNote": "
|
| 28 |
"verified": "2026-10-11",
|
| 29 |
"documentation": "https://support.claude.com/en/articles/11175166-getting-started-with-custom-connectors-using-remote-mcp",
|
| 30 |
"record": "AutoWiki/research/chatgpt-side-view-and-connector-setup.md#4-claude",
|
| 31 |
"provenance": "Documentation reviewed on 2026-10-11. No account walkthrough was performed.",
|
| 32 |
"steps": [
|
| 33 |
-
{ "text": "In Claude, open Customize, then Connectors.", "action": { "label": "Open Claude", "url": "https://claude.ai/" } },
|
| 34 |
-
{ "text": "
|
| 35 |
-
{ "
|
| 36 |
-
{ "text": "Review the detected authentication settings. Click Continue." },
|
| 37 |
-
{ "text": "Under Authentication, choose Sign in now to connect now. Sign in when needed and No sign in are also listed." },
|
| 38 |
-
{ "text": "Under OAuth client, keep Use Claude’s published identity (Recommended)." },
|
| 39 |
-
{ "text": "Click Add and complete any Loopable sign in requested." }
|
| 40 |
],
|
| 41 |
-
"caveat": "For Team or Enterprise, an Owner first adds the connector through Organization settings > Connectors > Add > Custom > Web. Members then click Connect. Free users are limited to one custom connector."
|
| 42 |
},
|
| 43 |
"chatgpt": {
|
| 44 |
"need": "Access to ChatGPT Plugins and your Loopable sign in.",
|
| 45 |
"failureAction": "Finish the Loopable sign in from ChatGPT, then select Verify sign in again. Check your workspace policy if the plugin is unavailable.",
|
| 46 |
-
"testNote": "
|
| 47 |
"verified": "2026-10-11",
|
| 48 |
"documentation": "https://developers.openai.com/plugins/deploy/connect-chatgpt",
|
| 49 |
"additionalDocumentation": ["https://developers.openai.com/plugins/quickstart"],
|
| 50 |
"record": "AutoWiki/research/chatgpt-side-view-and-connector-setup.md#3-chatgpt-setup-steps-today",
|
| 51 |
"provenance": "Documentation reviewed on 2026-10-11. No account walkthrough was performed.",
|
| 52 |
"steps": [
|
| 53 |
-
{ "text": "Open ChatGPT Plugins at chatgpt.com/plugins.", "action": { "label": "Open ChatGPT Plugins", "url": "https://chatgpt.com/plugins" } },
|
| 54 |
-
{ "text": "
|
| 55 |
-
{ "text": "
|
| 56 |
-
{ "text": "Under Connection, choose the public URL method and enter this MCP server URL.", "block": "<mcpUrl>" },
|
| 57 |
-
{ "text": "Configure OAuth authentication, review the risk warning, and select I understand and want to continue." },
|
| 58 |
-
{ "text": "Select Create as a plugin." },
|
| 59 |
-
{ "text": "Review the tools and metadata discovered from Loopable." },
|
| 60 |
-
{ "text": "Open Loopable under personal plugins and click the plus button to install it." },
|
| 61 |
-
{ "text": "In a new chat, type @ and select Loopable. Complete any Loopable sign in requested." }
|
| 62 |
],
|
| 63 |
-
"caveat": "Account and workspace policies apply to adding and using custom MCP servers."
|
| 64 |
},
|
| 65 |
"claude-code": {
|
| 66 |
"need": "Claude Code on your computer and your Loopable sign in or an active API key.",
|
| 67 |
"failureAction": "Check the server URL and finish signing in, or check your saved API key. Select the verification button for your chosen mode again.",
|
| 68 |
-
"testNote": "
|
| 69 |
"verified": "2026-10-11",
|
| 70 |
"documentation": "https://code.claude.com/docs/en/mcp",
|
| 71 |
"record": "AutoWiki/sdlc/sdlc64/lanes/E/context/S64-T12.md",
|
| 72 |
"provenance": "Checked against retained Loopable instructions on 2026-10-11. Provider documentation and account setup were not rechecked. The 2026-09-23 research also did not recheck this command.",
|
| 73 |
"steps": [
|
| 74 |
-
{ "text": "
|
| 75 |
-
{ "
|
| 76 |
],
|
| 77 |
"keySteps": [
|
| 78 |
-
{ "
|
| 79 |
-
{ "
|
| 80 |
-
{ "text": "Open /mcp to check the server.
|
| 81 |
],
|
| 82 |
"caveat": "Use a current Claude Code release. Organization policy can restrict MCP servers."
|
| 83 |
},
|
| 84 |
"codex": {
|
| 85 |
"need": "Codex on your computer and your Loopable sign in or an active API key.",
|
| 86 |
"failureAction": "Check the server URL and finish signing in, or check the API key in your local environment. Select the verification button again.",
|
| 87 |
-
"testNote": "
|
| 88 |
"verified": "2026-09-23",
|
| 89 |
"documentation": "https://learn.chatgpt.com/docs/extend/mcp",
|
| 90 |
"record": "AutoWiki/research/chatgpt-and-codex-mcp-connections.md",
|
| 91 |
"provenance": "Documentation reviewed on 2026-09-23. Remote add flags were measured with codex-cli 0.155.1 on the owner’s laptop that day. No new account walkthrough was performed.",
|
| 92 |
"steps": [
|
| 93 |
-
{ "text": "
|
| 94 |
-
{ "
|
| 95 |
],
|
| 96 |
"keySteps": [
|
| 97 |
-
{ "
|
| 98 |
-
{ "
|
| 99 |
],
|
| 100 |
"caveat": "Set up once for Codex CLI, the ChatGPT desktop app and the IDE extension on the same host through one ~/.codex/config.toml."
|
| 101 |
},
|
| 102 |
"cursor": {
|
| 103 |
"need": "Cursor on your computer and your Loopable sign in or an active API key.",
|
| 104 |
"failureAction": "Check the server configuration and sign in, or replace an inactive API key. Reopen the server and select the verification button again.",
|
| 105 |
-
"testNote": "
|
| 106 |
"verified": "2026-10-11",
|
| 107 |
"documentation": "https://cursor.com/docs/context/mcp",
|
| 108 |
"record": "AutoWiki/sdlc/sdlc64/lanes/E/context/S64-T12.md",
|
| 109 |
"provenance": "Checked against retained Loopable instructions on 2026-10-11. Provider documentation and account setup were not rechecked. Cursor was added in the SDLC 64 setup record.",
|
| 110 |
"steps": [
|
| 111 |
-
{ "
|
| 112 |
-
{ "text": "Add this server to ~/.cursor/mcp.json, keeping
|
| 113 |
-
{ "text": "Reopen the server and finish signing in to Loopable." }
|
| 114 |
],
|
| 115 |
"keySteps": [
|
| 116 |
-
{ "text": "
|
| 117 |
-
{ "text": "Replace <YOUR_API_KEY>
|
| 118 |
],
|
| 119 |
"caveat": "Cursor reads project servers from .cursor/mcp.json and personal servers from the home folder. An organization policy can block a server."
|
| 120 |
},
|
| 121 |
"muse": {
|
| 122 |
"need": "Access to Muse and an active Loopable API key.",
|
| 123 |
"failureAction": "Check the MCP address and locally saved bearer key. Select Verify API key again.",
|
| 124 |
-
"testNote": "
|
| 125 |
"verified": "2026-10-16",
|
| 126 |
"record": "AutoWiki/sdlc/sdlc76/PRD.md",
|
| 127 |
"provenance": "Loopable API key setup guide. Muse account setup has not been independently verified.",
|
| 128 |
"steps": [
|
| 129 |
-
{ "text": "Create a Loopable API key
|
| 130 |
-
{ "
|
| 131 |
-
{ "text": "Set Authorization: Bearer with your saved key locally. Never paste the key into chat." }
|
| 132 |
],
|
| 133 |
"keySteps": [
|
| 134 |
-
{ "text": "Create a Loopable API key
|
| 135 |
-
{ "
|
| 136 |
-
{ "text": "Set Authorization: Bearer with your saved key locally. Never paste the key into chat." }
|
| 137 |
],
|
| 138 |
"caveat": "Muse uses the same tenant bound MCP address and grants as other AI clients. Restricted and buyer data stay excluded."
|
| 139 |
}
|
|
|
|
| 24 |
"claude": {
|
| 25 |
"need": "Access to Claude.ai and your Loopable sign in.",
|
| 26 |
"failureAction": "Finish the Loopable sign in from Claude, then select Verify sign in again. Check your workspace policy if the connector is unavailable.",
|
| 27 |
+
"testNote": "",
|
| 28 |
"verified": "2026-10-11",
|
| 29 |
"documentation": "https://support.claude.com/en/articles/11175166-getting-started-with-custom-connectors-using-remote-mcp",
|
| 30 |
"record": "AutoWiki/research/chatgpt-side-view-and-connector-setup.md#4-claude",
|
| 31 |
"provenance": "Documentation reviewed on 2026-10-11. No account walkthrough was performed.",
|
| 32 |
"steps": [
|
| 33 |
+
{ "title": "Open Connectors", "text": "In Claude, open Customize, then Connectors. Click + Add, then Add custom connector.", "block": "Loopable", "action": { "label": "Open Claude", "url": "https://claude.ai/" } },
|
| 34 |
+
{ "title": "Add Loopable", "text": "Enter Loopable and this remote MCP server URL, then Continue.", "block": "<mcpUrl>" },
|
| 35 |
+
{ "title": "Sign in", "text": "Keep Claude’s published identity, choose Sign in now, then Add.", "block": "<base>" }
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
],
|
| 37 |
+
"caveat": "Review the detected authentication settings and click Continue. Authentication also lists Sign in when needed and No sign in. Keep Use Claude’s published identity (Recommended). Click Add and complete any Loopable sign in requested. For Team or Enterprise, an Owner first adds the connector through Organization settings > Connectors > Add > Custom > Web. Members then click Connect. Free users are limited to one custom connector."
|
| 38 |
},
|
| 39 |
"chatgpt": {
|
| 40 |
"need": "Access to ChatGPT Plugins and your Loopable sign in.",
|
| 41 |
"failureAction": "Finish the Loopable sign in from ChatGPT, then select Verify sign in again. Check your workspace policy if the plugin is unavailable.",
|
| 42 |
+
"testNote": "",
|
| 43 |
"verified": "2026-10-11",
|
| 44 |
"documentation": "https://developers.openai.com/plugins/deploy/connect-chatgpt",
|
| 45 |
"additionalDocumentation": ["https://developers.openai.com/plugins/quickstart"],
|
| 46 |
"record": "AutoWiki/research/chatgpt-side-view-and-connector-setup.md#3-chatgpt-setup-steps-today",
|
| 47 |
"provenance": "Documentation reviewed on 2026-10-11. No account walkthrough was performed.",
|
| 48 |
"steps": [
|
| 49 |
+
{ "title": "Open Plugins", "text": "Open ChatGPT Plugins at chatgpt.com/plugins. Choose Add custom MCP server.", "block": "Loopable", "action": { "label": "Open ChatGPT Plugins", "url": "https://chatgpt.com/plugins" } },
|
| 50 |
+
{ "title": "Add Loopable", "text": "Enter a name and description, then this public URL under Connection.", "block": "<mcpUrl>" },
|
| 51 |
+
{ "title": "Install plugin", "text": "Choose OAuth, Create as a plugin, then install Loopable and sign in.", "block": "@Loopable" }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
],
|
| 53 |
+
"caveat": "Select the plus button to add the server. Review the risk warning and select I understand and want to continue before Create as a plugin. Review the discovered tools and metadata, then open Loopable under personal plugins and click the plus button to install it. In a new chat, type @ and select Loopable. Complete any Loopable sign in requested. Account and workspace policies apply to adding and using custom MCP servers."
|
| 54 |
},
|
| 55 |
"claude-code": {
|
| 56 |
"need": "Claude Code on your computer and your Loopable sign in or an active API key.",
|
| 57 |
"failureAction": "Check the server URL and finish signing in, or check your saved API key. Select the verification button for your chosen mode again.",
|
| 58 |
+
"testNote": "",
|
| 59 |
"verified": "2026-10-11",
|
| 60 |
"documentation": "https://code.claude.com/docs/en/mcp",
|
| 61 |
"record": "AutoWiki/sdlc/sdlc64/lanes/E/context/S64-T12.md",
|
| 62 |
"provenance": "Checked against retained Loopable instructions on 2026-10-11. Provider documentation and account setup were not rechecked. The 2026-09-23 research also did not recheck this command.",
|
| 63 |
"steps": [
|
| 64 |
+
{ "title": "Add server", "text": "Run this command in your terminal.", "block": "claude mcp add --transport http loopable <mcpUrl>" },
|
| 65 |
+
{ "title": "Sign in", "text": "Run /mcp in Claude Code and finish the Loopable browser sign in.", "block": "/mcp" }
|
| 66 |
],
|
| 67 |
"keySteps": [
|
| 68 |
+
{ "title": "Remove old entry", "text": "Run this only if Loopable is already configured.", "block": "claude mcp remove loopable" },
|
| 69 |
+
{ "title": "Add server", "text": "Replace <YOUR_API_KEY> with your saved key locally, never in chat.", "block": "claude mcp add --transport http loopable <mcpUrl> --header \"Authorization: Bearer <YOUR_API_KEY>\"" },
|
| 70 |
+
{ "title": "Open server", "text": "Open /mcp to check the server.", "block": "/mcp" }
|
| 71 |
],
|
| 72 |
"caveat": "Use a current Claude Code release. Organization policy can restrict MCP servers."
|
| 73 |
},
|
| 74 |
"codex": {
|
| 75 |
"need": "Codex on your computer and your Loopable sign in or an active API key.",
|
| 76 |
"failureAction": "Check the server URL and finish signing in, or check the API key in your local environment. Select the verification button again.",
|
| 77 |
+
"testNote": "",
|
| 78 |
"verified": "2026-09-23",
|
| 79 |
"documentation": "https://learn.chatgpt.com/docs/extend/mcp",
|
| 80 |
"record": "AutoWiki/research/chatgpt-and-codex-mcp-connections.md",
|
| 81 |
"provenance": "Documentation reviewed on 2026-09-23. Remote add flags were measured with codex-cli 0.155.1 on the owner’s laptop that day. No new account walkthrough was performed.",
|
| 82 |
"steps": [
|
| 83 |
+
{ "title": "Add server", "text": "Run this command in your terminal.", "block": "codex mcp add loopable --url <mcpUrl>" },
|
| 84 |
+
{ "title": "Sign in", "text": "Sign in to Loopable in the browser.", "block": "codex mcp login loopable" }
|
| 85 |
],
|
| 86 |
"keySteps": [
|
| 87 |
+
{ "title": "Set saved key", "text": "Set LOOPABLE_API_KEY locally before opening Codex, never in chat.", "block": "LOOPABLE_API_KEY" },
|
| 88 |
+
{ "title": "Add server", "text": "Run this command in your terminal.", "block": "codex mcp add loopable --url <mcpUrl> --bearer-token-env-var LOOPABLE_API_KEY" }
|
| 89 |
],
|
| 90 |
"caveat": "Set up once for Codex CLI, the ChatGPT desktop app and the IDE extension on the same host through one ~/.codex/config.toml."
|
| 91 |
},
|
| 92 |
"cursor": {
|
| 93 |
"need": "Cursor on your computer and your Loopable sign in or an active API key.",
|
| 94 |
"failureAction": "Check the server configuration and sign in, or replace an inactive API key. Reopen the server and select the verification button again.",
|
| 95 |
+
"testNote": "",
|
| 96 |
"verified": "2026-10-11",
|
| 97 |
"documentation": "https://cursor.com/docs/context/mcp",
|
| 98 |
"record": "AutoWiki/sdlc/sdlc64/lanes/E/context/S64-T12.md",
|
| 99 |
"provenance": "Checked against retained Loopable instructions on 2026-10-11. Provider documentation and account setup were not rechecked. Cursor was added in the SDLC 64 setup record.",
|
| 100 |
"steps": [
|
| 101 |
+
{ "title": "Open settings", "text": "Open Settings, Tools and Integrations, then New MCP Server.", "block": "~/.cursor/mcp.json" },
|
| 102 |
+
{ "title": "Add server", "text": "Add this server to ~/.cursor/mcp.json, keeping existing servers.", "block": "{\n \"mcpServers\": {\n \"loopable\": {\n \"url\": \"<mcpUrl>\"\n }\n }\n}" },
|
| 103 |
+
{ "title": "Sign in", "text": "Reopen the server and finish signing in to Loopable.", "block": "<mcpUrl>" }
|
| 104 |
],
|
| 105 |
"keySteps": [
|
| 106 |
+
{ "title": "Add server", "text": "Add or update Loopable in ~/.cursor/mcp.json, keeping other servers.", "block": "{\n \"mcpServers\": {\n \"loopable\": {\n \"url\": \"<mcpUrl>\",\n \"headers\": {\n \"Authorization\": \"Bearer <YOUR_API_KEY>\"\n }\n }\n }\n}" },
|
| 107 |
+
{ "title": "Use saved key", "text": "Replace <YOUR_API_KEY> locally, then reopen the server. Keep the key out of chat.", "block": "~/.cursor/mcp.json" }
|
| 108 |
],
|
| 109 |
"caveat": "Cursor reads project servers from .cursor/mcp.json and personal servers from the home folder. An organization policy can block a server."
|
| 110 |
},
|
| 111 |
"muse": {
|
| 112 |
"need": "Access to Muse and an active Loopable API key.",
|
| 113 |
"failureAction": "Check the MCP address and locally saved bearer key. Select Verify API key again.",
|
| 114 |
+
"testNote": "",
|
| 115 |
"verified": "2026-10-16",
|
| 116 |
"record": "AutoWiki/sdlc/sdlc76/PRD.md",
|
| 117 |
"provenance": "Loopable API key setup guide. Muse account setup has not been independently verified.",
|
| 118 |
"steps": [
|
| 119 |
+
{ "title": "Save API key", "text": "Create or choose a Loopable API key. Never paste the key into chat.", "block": "Loopable" },
|
| 120 |
+
{ "title": "Add server", "text": "Add this MCP address in Muse over HTTP with your locally saved bearer key.", "block": "<mcpUrl>" }
|
|
|
|
| 121 |
],
|
| 122 |
"keySteps": [
|
| 123 |
+
{ "title": "Save API key", "text": "Create or choose a Loopable API key. Never paste the key into chat.", "block": "Loopable" },
|
| 124 |
+
{ "title": "Add server", "text": "Add this MCP address in Muse over HTTP with your locally saved bearer key.", "block": "<mcpUrl>" }
|
|
|
|
| 125 |
],
|
| 126 |
"caveat": "Muse uses the same tenant bound MCP address and grants as other AI clients. Restricted and buyer data stay excluded."
|
| 127 |
}
|
platform/aios_grid.py
CHANGED
|
@@ -1928,6 +1928,7 @@ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None):
|
|
| 1928 |
odoo_fields = [field for field in fields if field["source"] == "odoo"]
|
| 1929 |
overlay_fields = [field for field in fields if field["source"] == "overlay"]
|
| 1930 |
derived_keys = [field["key"] for field in fields if field.get("derived")]
|
|
|
|
| 1931 |
overlays = overlays or {}
|
| 1932 |
derived = derived or {}
|
| 1933 |
out = []
|
|
@@ -1940,6 +1941,14 @@ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None):
|
|
| 1940 |
# the same way: the Map VIEW's data, nullable, deliberately not a column.
|
| 1941 |
row = {"pid": pid, "_created": r.get("_created") or "",
|
| 1942 |
"lat": r.get("lat"), "lon": r.get("lon")}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1943 |
for field in odoo_fields:
|
| 1944 |
k = field["key"]
|
| 1945 |
if field.get("derived"):
|
|
|
|
| 1928 |
odoo_fields = [field for field in fields if field["source"] == "odoo"]
|
| 1929 |
overlay_fields = [field for field in fields if field["source"] == "overlay"]
|
| 1930 |
derived_keys = [field["key"] for field in fields if field.get("derived")]
|
| 1931 |
+
readable_keys = {field["key"] for field in fields}
|
| 1932 |
overlays = overlays or {}
|
| 1933 |
derived = derived or {}
|
| 1934 |
out = []
|
|
|
|
| 1941 |
# the same way: the Map VIEW's data, nullable, deliberately not a column.
|
| 1942 |
row = {"pid": pid, "_created": r.get("_created") or "",
|
| 1943 |
"lat": r.get("lat"), "lon": r.get("lon")}
|
| 1944 |
+
notes = {}
|
| 1945 |
+
for source in (r, saved):
|
| 1946 |
+
mapped = source.get('__cell_notes__')
|
| 1947 |
+
if isinstance(mapped, dict):
|
| 1948 |
+
notes.update({key: text for key, text in mapped.items()
|
| 1949 |
+
if key in readable_keys and isinstance(text, str)})
|
| 1950 |
+
if notes:
|
| 1951 |
+
row['__cell_notes__'] = notes
|
| 1952 |
for field in odoo_fields:
|
| 1953 |
k = field["key"]
|
| 1954 |
if field.get("derived"):
|
platform/aios_grid_fields.json
CHANGED
|
@@ -205,7 +205,7 @@
|
|
| 205 |
"type": "currency",
|
| 206 |
"source": "odoo",
|
| 207 |
"default": false,
|
| 208 |
-
"description": "Odoo
|
| 209 |
},
|
| 210 |
{
|
| 211 |
"_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
|
|
|
|
| 205 |
"type": "currency",
|
| 206 |
"source": "odoo",
|
| 207 |
"default": false,
|
| 208 |
+
"description": "Odoo open receivables past due, no grace; New Customer Hold due now. Credits and unapplied payments net per customer, tying to Odoo to the cent."
|
| 209 |
},
|
| 210 |
{
|
| 211 |
"_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
|
platform/core/activity_log.py
CHANGED
|
@@ -98,7 +98,7 @@ def _object(value, *, allow_key_only=False):
|
|
| 98 |
return kind, key, encoded
|
| 99 |
|
| 100 |
|
| 101 |
-
def _bounded_changes(changes):
|
| 102 |
if changes is None:
|
| 103 |
changes = {}
|
| 104 |
if not isinstance(changes, dict):
|
|
@@ -110,7 +110,7 @@ def _bounded_changes(changes):
|
|
| 110 |
if '\\u0000' in raw:
|
| 111 |
raise ValueError('changes cannot contain a null character.')
|
| 112 |
original_bytes = len(raw.encode('utf-8'))
|
| 113 |
-
if original_bytes <= CHANGES_MAX_BYTES:
|
| 114 |
return json.loads(raw), False, original_bytes
|
| 115 |
kept = {}
|
| 116 |
for key, pair in changes.items():
|
|
@@ -203,7 +203,8 @@ def append(rt, *, kind, object, by, via=None, on_behalf_of=None, changes=None, _
|
|
| 203 |
by = _text(by, 'by')
|
| 204 |
via = _text(via, 'via', optional=True)
|
| 205 |
on_behalf_of = _text(on_behalf_of, 'on_behalf_of', optional=True)
|
| 206 |
-
bounded, truncated, original_bytes = _bounded_changes(
|
|
|
|
| 207 |
now = _time(_delivery['occurred_at']) if _delivery else _now()
|
| 208 |
values = (now.isoformat(), kind, object_type, object_key, object_id, by, via,
|
| 209 |
on_behalf_of, _json(bounded), truncated, original_bytes)
|
|
@@ -240,6 +241,10 @@ def drain(rt, *, keys=None):
|
|
| 240 |
finally:
|
| 241 |
if delivered:
|
| 242 |
def acknowledge(document):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
queue = (document.get(DATABASE_KEY) or {}).get(OUTBOX_KEY) or {}
|
| 244 |
for key, version in delivered.items():
|
| 245 |
entry = queue.get(key)
|
|
@@ -303,7 +308,8 @@ def _pending_page(backend, pending, *, target, actor, since, until, before, limi
|
|
| 303 |
if prior is not None and prior[0] >= delivery['version']:
|
| 304 |
yield prior[1]
|
| 305 |
continue
|
| 306 |
-
changes, truncated, original_bytes = _bounded_changes(
|
|
|
|
| 307 |
yield _event((event_id, stamp, entry['kind'], kind, name, identity,
|
| 308 |
entry['by'], entry.get('via'), entry.get('on_behalf_of'),
|
| 309 |
changes, truncated, original_bytes),
|
|
@@ -447,6 +453,7 @@ class PgActivityStore:
|
|
| 447 |
|
| 448 |
def _connection(self):
|
| 449 |
from core import store_pg
|
|
|
|
| 450 |
return store_pg._pool().connection()
|
| 451 |
|
| 452 |
def _exists(self, con):
|
|
@@ -457,6 +464,20 @@ class PgActivityStore:
|
|
| 457 |
if self._ready:
|
| 458 |
return
|
| 459 |
with self._connection() as con:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 460 |
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
|
| 461 |
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} (
|
| 462 |
id BIGSERIAL PRIMARY KEY, occurred_at TIMESTAMPTZ NOT NULL,
|
|
@@ -487,7 +508,10 @@ class PgActivityStore:
|
|
| 487 |
):
|
| 488 |
con.execute(f'CREATE INDEX IF NOT EXISTS "{self.name}_{suffix}" '
|
| 489 |
f'ON {self.table} ({columns})')
|
| 490 |
-
|
|
|
|
|
|
|
|
|
|
| 491 |
|
| 492 |
def _prune(self, con, cutoff, limit):
|
| 493 |
return con.execute(f'''DELETE FROM {self.table} WHERE id IN (
|
|
|
|
| 98 |
return kind, key, encoded
|
| 99 |
|
| 100 |
|
| 101 |
+
def _bounded_changes(changes, *, cell_edit=False):
|
| 102 |
if changes is None:
|
| 103 |
changes = {}
|
| 104 |
if not isinstance(changes, dict):
|
|
|
|
| 110 |
if '\\u0000' in raw:
|
| 111 |
raise ValueError('changes cannot contain a null character.')
|
| 112 |
original_bytes = len(raw.encode('utf-8'))
|
| 113 |
+
if cell_edit or original_bytes <= CHANGES_MAX_BYTES:
|
| 114 |
return json.loads(raw), False, original_bytes
|
| 115 |
kept = {}
|
| 116 |
for key, pair in changes.items():
|
|
|
|
| 203 |
by = _text(by, 'by')
|
| 204 |
via = _text(via, 'via', optional=True)
|
| 205 |
on_behalf_of = _text(on_behalf_of, 'on_behalf_of', optional=True)
|
| 206 |
+
bounded, truncated, original_bytes = _bounded_changes(
|
| 207 |
+
changes, cell_edit=kind == 'cell' or bool(_delivery and _delivery.get('_cell_edit')))
|
| 208 |
now = _time(_delivery['occurred_at']) if _delivery else _now()
|
| 209 |
values = (now.isoformat(), kind, object_type, object_key, object_id, by, via,
|
| 210 |
on_behalf_of, _json(bounded), truncated, original_bytes)
|
|
|
|
| 241 |
finally:
|
| 242 |
if delivered:
|
| 243 |
def acknowledge(document):
|
| 244 |
+
# Delivery priority differs by writer. Lock every delivered row before reading any.
|
| 245 |
+
lock_records = getattr(document, 'lock_records', None)
|
| 246 |
+
if callable(lock_records):
|
| 247 |
+
lock_records({DATABASE_KEY: sorted(delivered)})
|
| 248 |
queue = (document.get(DATABASE_KEY) or {}).get(OUTBOX_KEY) or {}
|
| 249 |
for key, version in delivered.items():
|
| 250 |
entry = queue.get(key)
|
|
|
|
| 308 |
if prior is not None and prior[0] >= delivery['version']:
|
| 309 |
yield prior[1]
|
| 310 |
continue
|
| 311 |
+
changes, truncated, original_bytes = _bounded_changes(
|
| 312 |
+
entry.get('changes'), cell_edit=entry['kind'] == 'cell' or delivery.get('_cell_edit', False))
|
| 313 |
yield _event((event_id, stamp, entry['kind'], kind, name, identity,
|
| 314 |
entry['by'], entry.get('via'), entry.get('on_behalf_of'),
|
| 315 |
changes, truncated, original_bytes),
|
|
|
|
| 453 |
|
| 454 |
def _connection(self):
|
| 455 |
from core import store_pg
|
| 456 |
+
store_pg._effective_slug(self.tenant_slug)
|
| 457 |
return store_pg._pool().connection()
|
| 458 |
|
| 459 |
def _exists(self, con):
|
|
|
|
| 464 |
if self._ready:
|
| 465 |
return
|
| 466 |
with self._connection() as con:
|
| 467 |
+
from core import store_pg
|
| 468 |
+
if store_pg._CALLER_TRANSACTION.get() is not None and self._exists(con):
|
| 469 |
+
columns = con.execute('SELECT attname FROM pg_attribute WHERE attrelid = %s::regclass '
|
| 470 |
+
'AND attnum > 0 AND NOT attisdropped', (self.table,)).fetchall()
|
| 471 |
+
indexes = con.execute('SELECT indexname FROM pg_indexes WHERE schemaname = %s '
|
| 472 |
+
'AND tablename = %s', (self.schema, self.name)).fetchall()
|
| 473 |
+
constraint = con.execute('SELECT pg_get_constraintdef(oid) FROM pg_constraint '
|
| 474 |
+
'WHERE conrelid = %s::regclass AND conname = %s',
|
| 475 |
+
(self.table, f'{self.name}_kind_check')).fetchone()
|
| 476 |
+
needed = {f'{self.name}_{suffix}' for suffix in (
|
| 477 |
+
'delivery', 'time', 'object_time', 'object_id_time', 'person_time')}
|
| 478 |
+
if (set(_COLUMNS) | {'delivery_key', 'delivery_version'} <= {r[0] for r in columns}
|
| 479 |
+
and needed <= {r[0] for r in indexes} and constraint and "'view'" in constraint[0]):
|
| 480 |
+
return
|
| 481 |
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
|
| 482 |
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} (
|
| 483 |
id BIGSERIAL PRIMARY KEY, occurred_at TIMESTAMPTZ NOT NULL,
|
|
|
|
| 508 |
):
|
| 509 |
con.execute(f'CREATE INDEX IF NOT EXISTS "{self.name}_{suffix}" '
|
| 510 |
f'ON {self.table} ({columns})')
|
| 511 |
+
# A caller rollback can remove newly created indexes and the table itself.
|
| 512 |
+
from core import store_pg
|
| 513 |
+
if store_pg._CALLER_TRANSACTION.get() is None:
|
| 514 |
+
self._ready = True
|
| 515 |
|
| 516 |
def _prune(self, con, cutoff, limit):
|
| 517 |
return con.execute(f'''DELETE FROM {self.table} WHERE id IN (
|
platform/core/connector_directory.py
CHANGED
|
@@ -235,6 +235,7 @@ NATIVE_CATEGORIES = {
|
|
| 235 |
# same way Odoo is: it reaches orders, fees and listings, which is Data plus the two shelves
|
| 236 |
# Odoo itself sits under for the commerce side of the same story.
|
| 237 |
"amazon_sp": ("Data", "E-commerce", "Finance"),
|
|
|
|
| 238 |
"webhooks": ("Utilities", "Web Tools"),
|
| 239 |
"scraper": ("Scrapers", "Social Media"),
|
| 240 |
"stripe": ("Finance", "E-commerce"),
|
|
|
|
| 235 |
# same way Odoo is: it reaches orders, fees and listings, which is Data plus the two shelves
|
| 236 |
# Odoo itself sits under for the commerce side of the same story.
|
| 237 |
"amazon_sp": ("Data", "E-commerce", "Finance"),
|
| 238 |
+
"amazon_ads": ("Data", "E-commerce", "Finance"),
|
| 239 |
"webhooks": ("Utilities", "Web Tools"),
|
| 240 |
"scraper": ("Scrapers", "Social Media"),
|
| 241 |
"stripe": ("Finance", "E-commerce"),
|
platform/core/connector_spec.py
CHANGED
|
@@ -308,7 +308,10 @@ def system_link_fields(key, live_columns):
|
|
| 308 |
"key": edge["key"], "label": _link_field_label(edge["label"]), "type": "link",
|
| 309 |
"source": "overlay", "default": True,
|
| 310 |
"link": {"table": target, "on": edge["to_col"], "from": edge["from_col"],
|
| 311 |
-
"single": edge["cardinality"] == "many_to_one"
|
|
|
|
|
|
|
|
|
|
| 312 |
})
|
| 313 |
return out
|
| 314 |
|
|
@@ -336,7 +339,7 @@ AMAZON_TABLES = (
|
|
| 336 |
key=_AMZN + "orders",
|
| 337 |
native="GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL",
|
| 338 |
source="amzn_orders",
|
| 339 |
-
label="Amazon
|
| 340 |
pinned="amazon_order_id",
|
| 341 |
identity=("amazon_order_id",),
|
| 342 |
links=(
|
|
@@ -347,7 +350,7 @@ AMAZON_TABLES = (
|
|
| 347 |
key=_AMZN + "order_items",
|
| 348 |
native="GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL",
|
| 349 |
source="amzn_order_items",
|
| 350 |
-
label="Amazon
|
| 351 |
pinned="amazon_order_id",
|
| 352 |
identity=("order_item_id", "amazon_order_id"),
|
| 353 |
links=(
|
|
@@ -361,7 +364,7 @@ AMAZON_TABLES = (
|
|
| 361 |
key=_AMZN + "listings",
|
| 362 |
native="GET_MERCHANT_LISTINGS_ALL_DATA",
|
| 363 |
source="amzn_listings",
|
| 364 |
-
label="Amazon
|
| 365 |
pinned="seller_sku",
|
| 366 |
identity=("seller_sku", "asin1", "asin"),
|
| 367 |
links=(), # the product spine's HEAD: everything points at it, it points nowhere
|
|
@@ -370,7 +373,7 @@ AMAZON_TABLES = (
|
|
| 370 |
key=_AMZN + "settlements",
|
| 371 |
native="GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2",
|
| 372 |
source="amzn_settlements",
|
| 373 |
-
label="Amazon
|
| 374 |
pinned="settlement_id",
|
| 375 |
identity=("settlement_id",),
|
| 376 |
# ⚠ THE MONEY-TO-ORDER BRIDGE — what turns "an order happened" into "this is what we were
|
|
@@ -384,7 +387,7 @@ AMAZON_TABLES = (
|
|
| 384 |
key=_AMZN + "inventory",
|
| 385 |
native="GET_LEDGER_SUMMARY_VIEW_DATA",
|
| 386 |
source="amzn_inventory",
|
| 387 |
-
label="Amazon
|
| 388 |
pinned="asin",
|
| 389 |
identity=("asin", "fnsku", "msku"),
|
| 390 |
# ⚠ THE LEDGER KEYS ON `msku`, NOT `sku` — FBA calls the seller's own SKU the MSKU. Same
|
|
@@ -398,7 +401,7 @@ AMAZON_TABLES = (
|
|
| 398 |
key=_AMZN + "pricing",
|
| 399 |
native="GET_SALES_AND_TRAFFIC_REPORT",
|
| 400 |
source="amzn_pricing",
|
| 401 |
-
label="Amazon
|
| 402 |
pinned="parent_asin",
|
| 403 |
identity=("child_asin", "parent_asin"),
|
| 404 |
links=(
|
|
@@ -410,7 +413,7 @@ AMAZON_TABLES = (
|
|
| 410 |
key=_AMZN + "brand_analytics",
|
| 411 |
native="GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT",
|
| 412 |
source="amzn_brand_analytics",
|
| 413 |
-
label="Amazon
|
| 414 |
pinned="search_term",
|
| 415 |
identity=("department_name", "search_term", "clicked_asin", "click_share_rank",
|
| 416 |
"_period_start", "_period_end"),
|
|
@@ -421,14 +424,14 @@ AMAZON_TABLES = (
|
|
| 421 |
),
|
| 422 |
ConnectorTableSpec(
|
| 423 |
key=_AMZN + "economics", native="analytics_economics_2024_03_15",
|
| 424 |
-
source="amzn_economics", label="Amazon economics", pinned="msku",
|
| 425 |
identity=("start_date", "msku", "marketplace_id"),
|
| 426 |
links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 427 |
("child_asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 428 |
),
|
| 429 |
ConnectorTableSpec(
|
| 430 |
key=_AMZN + "transactions", native="finances/2024-06-19/transactions",
|
| 431 |
-
source="amzn_transactions", label="Amazon
|
| 432 |
identity=("transaction_id", "item_position"),
|
| 433 |
links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
|
| 434 |
("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
|
@@ -436,325 +439,325 @@ AMAZON_TABLES = (
|
|
| 436 |
),
|
| 437 |
ConnectorTableSpec(
|
| 438 |
key=_AMZN + "fba_fee_estimates", native="GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA",
|
| 439 |
-
source="amzn_fba_fee_estimates", label="Amazon
|
| 440 |
identity=("_snapshot_date", "sku", "amazon_store"),
|
| 441 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 442 |
("asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 443 |
),
|
| 444 |
ConnectorTableSpec(
|
| 445 |
key=_AMZN + "referral_fees", native="GET_REFERRAL_FEE_PREVIEW_REPORT",
|
| 446 |
-
source="amzn_referral_fees", label="Amazon
|
| 447 |
identity=("_snapshot_date", "seller_sku"),
|
| 448 |
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 449 |
("asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 450 |
),
|
| 451 |
ConnectorTableSpec(
|
| 452 |
key=_AMZN + "storage_fees", native="GET_FBA_STORAGE_FEE_CHARGES_DATA",
|
| 453 |
-
source="amzn_storage_fees", label="Amazon
|
| 454 |
identity=("month", "asin", "fnsku", "fulfillment_center"),
|
| 455 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 456 |
),
|
| 457 |
ConnectorTableSpec(
|
| 458 |
key=_AMZN + "search_query_performance", native="GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
|
| 459 |
-
source="amzn_search_query_performance", label="Amazon
|
| 460 |
pinned="search_query_data_search_query",
|
| 461 |
identity=("start_date", "end_date", "asin", "search_query_data_search_query"),
|
| 462 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 463 |
),
|
| 464 |
ConnectorTableSpec(
|
| 465 |
key=_AMZN + "long_term_storage_fees", native="GET_FBA_FULFILLMENT_LONGTERM_STORAGE_FEE_CHARGES_DATA",
|
| 466 |
-
source="amzn_long_term_storage_fees", label="Amazon
|
| 467 |
identity=("_period_start", "sku", "fnsku", "asin", "_report_row_number"),
|
| 468 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 469 |
),
|
| 470 |
ConnectorTableSpec(
|
| 471 |
key=_AMZN + "storage_overage_fees", native="GET_FBA_OVERAGE_FEE_CHARGES_DATA",
|
| 472 |
-
source="amzn_storage_overage_fees", label="Amazon
|
| 473 |
identity=("_period_start", "storage_type", "_report_row_number"),
|
| 474 |
links=(),
|
| 475 |
),
|
| 476 |
ConnectorTableSpec(
|
| 477 |
key=_AMZN + "reimbursements", native="GET_FBA_REIMBURSEMENTS_DATA",
|
| 478 |
-
source="amzn_reimbursements", label="Amazon
|
| 479 |
identity=("_period_start", "reimbursement_id", "sku", "asin", "amazon_order_id", "_report_row_number"),
|
| 480 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"), ("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),),
|
| 481 |
),
|
| 482 |
ConnectorTableSpec(
|
| 483 |
key=_AMZN + "fba_customer_returns", native="GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
|
| 484 |
-
source="amzn_fba_customer_returns", label="Amazon
|
| 485 |
identity=("_period_start", "order_id", "sku", "fnsku", "asin", "_report_row_number"),
|
| 486 |
links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 487 |
),
|
| 488 |
ConnectorTableSpec(
|
| 489 |
key=_AMZN + "replacements", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA",
|
| 490 |
-
source="amzn_replacements", label="Amazon
|
| 491 |
identity=("_period_start", "replacement_amazon_order_id", "original_amazon_order_id", "sku", "asin", "_report_row_number"),
|
| 492 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 493 |
),
|
| 494 |
ConnectorTableSpec(
|
| 495 |
key=_AMZN + "fba_shipped_sales", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_SALES_DATA",
|
| 496 |
-
source="amzn_fba_shipped_sales", label="Amazon FBA
|
| 497 |
identity=("_period_start", "amazon_order_id", "sku", "fnsku", "asin", "_report_row_number"),
|
| 498 |
links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 499 |
),
|
| 500 |
ConnectorTableSpec(
|
| 501 |
key=_AMZN + "fba_shipment_promotions", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA",
|
| 502 |
-
source="amzn_fba_shipment_promotions", label="Amazon FBA shipment promotions", pinned="amazon_order_id",
|
| 503 |
identity=("_period_start", "amazon_order_id", "sku", "_report_row_number"),
|
| 504 |
links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),),
|
| 505 |
),
|
| 506 |
ConnectorTableSpec(
|
| 507 |
key=_AMZN + "inventory_ledger_detail", native="GET_LEDGER_DETAIL_VIEW_DATA",
|
| 508 |
-
source="amzn_inventory_ledger_detail", label="Amazon
|
| 509 |
identity=("_period_start", "msku", "fnsku", "asin", "_report_row_number"),
|
| 510 |
links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 511 |
),
|
| 512 |
ConnectorTableSpec(
|
| 513 |
key=_AMZN + "inventory_planning", native="GET_FBA_INVENTORY_PLANNING_DATA",
|
| 514 |
-
source="amzn_inventory_planning", label="Amazon
|
| 515 |
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 516 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 517 |
),
|
| 518 |
ConnectorTableSpec(
|
| 519 |
key=_AMZN + "manage_inventory", native="GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA",
|
| 520 |
-
source="amzn_manage_inventory", label="Amazon
|
| 521 |
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 522 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 523 |
),
|
| 524 |
ConnectorTableSpec(
|
| 525 |
key=_AMZN + "reserved_inventory", native="GET_RESERVED_INVENTORY_DATA",
|
| 526 |
-
source="amzn_reserved_inventory", label="Amazon
|
| 527 |
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 528 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 529 |
),
|
| 530 |
ConnectorTableSpec(
|
| 531 |
key=_AMZN + "restock_recommendations", native="GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT",
|
| 532 |
-
source="amzn_restock_recommendations", label="Amazon
|
| 533 |
identity=("_snapshot_date", "merchant_sku", "fnsku", "asin", "_report_row_number"),
|
| 534 |
links=(("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 535 |
("asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 536 |
),
|
| 537 |
ConnectorTableSpec(
|
| 538 |
key=_AMZN + "stranded_inventory", native="GET_STRANDED_INVENTORY_UI_DATA",
|
| 539 |
-
source="amzn_stranded_inventory", label="Amazon
|
| 540 |
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 541 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 542 |
),
|
| 543 |
ConnectorTableSpec(
|
| 544 |
key=_AMZN + "inbound_noncompliance", native="GET_FBA_FULFILLMENT_INBOUND_NONCOMPLIANCE_DATA",
|
| 545 |
-
source="amzn_inbound_noncompliance", label="Amazon
|
| 546 |
identity=("_period_start", "shipment_id", "sku", "asin", "_report_row_number"),
|
| 547 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 548 |
),
|
| 549 |
ConnectorTableSpec(
|
| 550 |
key=_AMZN + "inventory_expiration_dates", native="GET_FBA_FULFILLMENT_INVENTORY_EXPIRATION_DATE_DATA",
|
| 551 |
-
source="amzn_inventory_expiration_dates", label="Amazon
|
| 552 |
identity=("_snapshot_date", "sku", "fnsku", "_report_row_number"),
|
| 553 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 554 |
),
|
| 555 |
ConnectorTableSpec(
|
| 556 |
key=_AMZN + "recommended_removals", native="GET_FBA_RECOMMENDED_REMOVAL_DATA",
|
| 557 |
-
source="amzn_recommended_removals", label="Amazon
|
| 558 |
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 559 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 560 |
),
|
| 561 |
ConnectorTableSpec(
|
| 562 |
key=_AMZN + "removal_orders", native="GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
|
| 563 |
-
source="amzn_removal_orders", label="Amazon
|
| 564 |
identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
|
| 565 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 566 |
),
|
| 567 |
ConnectorTableSpec(
|
| 568 |
key=_AMZN + "removal_shipments", native="GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
|
| 569 |
-
source="amzn_removal_shipments", label="Amazon
|
| 570 |
identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
|
| 571 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 572 |
),
|
| 573 |
ConnectorTableSpec(
|
| 574 |
key=_AMZN + "subscribe_save_forecast", native="GET_FBA_SNS_FORECAST_DATA",
|
| 575 |
-
source="amzn_subscribe_save_forecast", label="Amazon Subscribe and Save forecast", pinned="sku",
|
| 576 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 577 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 578 |
),
|
| 579 |
ConnectorTableSpec(
|
| 580 |
key=_AMZN + "subscribe_save_performance", native="GET_FBA_SNS_PERFORMANCE_DATA",
|
| 581 |
-
source="amzn_subscribe_save_performance", label="Amazon Subscribe and Save
|
| 582 |
identity=("_period_start", "sku", "asin", "_report_row_number"),
|
| 583 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 584 |
),
|
| 585 |
ConnectorTableSpec(
|
| 586 |
key=_AMZN + "remote_fulfillment_eligibility", native="GET_REMOTE_FULFILLMENT_ELIGIBILITY",
|
| 587 |
-
source="amzn_remote_fulfillment_eligibility", label="Amazon
|
| 588 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 589 |
links=(), # Measured header has no product key; do not declare a guessed join.
|
| 590 |
),
|
| 591 |
ConnectorTableSpec(
|
| 592 |
key=_AMZN + "search_catalog_performance", native="GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT",
|
| 593 |
-
source="amzn_search_catalog_performance", label="Amazon
|
| 594 |
identity=("_period_start", "asin", "_report_row_number"),
|
| 595 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 596 |
),
|
| 597 |
ConnectorTableSpec(
|
| 598 |
key=_AMZN + "market_basket", native="GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT",
|
| 599 |
-
source="amzn_market_basket", label="Amazon
|
| 600 |
identity=("_period_start", "asin", "purchased_with_asin", "_report_row_number"),
|
| 601 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 602 |
),
|
| 603 |
ConnectorTableSpec(
|
| 604 |
key=_AMZN + "repeat_purchase", native="GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT",
|
| 605 |
-
source="amzn_repeat_purchase", label="Amazon
|
| 606 |
identity=("_period_start", "asin", "_report_row_number"),
|
| 607 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 608 |
),
|
| 609 |
ConnectorTableSpec(
|
| 610 |
key=_AMZN + "promotion_performance", native="GET_PROMOTION_PERFORMANCE_REPORT",
|
| 611 |
-
source="amzn_promotion_performance", label="Amazon
|
| 612 |
identity=("_period_start", "promotion_id", "_report_row_number"),
|
| 613 |
links=(),
|
| 614 |
),
|
| 615 |
ConnectorTableSpec(
|
| 616 |
key=_AMZN + "coupon_performance", native="GET_COUPON_PERFORMANCE_REPORT",
|
| 617 |
-
source="amzn_coupon_performance", label="Amazon
|
| 618 |
identity=("_period_start", "coupon_id", "_report_row_number"),
|
| 619 |
links=(),
|
| 620 |
),
|
| 621 |
ConnectorTableSpec(
|
| 622 |
key=_AMZN + "account_health", native="GET_V2_SELLER_PERFORMANCE_REPORT",
|
| 623 |
-
source="amzn_account_health", label="Amazon
|
| 624 |
identity=("_snapshot_date", "marketplace_id", "_json_root", "_report_row_number"),
|
| 625 |
links=(),
|
| 626 |
),
|
| 627 |
ConnectorTableSpec(
|
| 628 |
key=_AMZN + "inactive_listings", native="GET_MERCHANT_LISTINGS_INACTIVE_DATA",
|
| 629 |
-
source="amzn_inactive_listings", label="Amazon
|
| 630 |
identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
|
| 631 |
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 632 |
("asin1", _AMZN + "listings", "asin1", "many_to_one")),
|
| 633 |
),
|
| 634 |
ConnectorTableSpec(
|
| 635 |
key=_AMZN + "cancelled_listings", native="GET_MERCHANT_CANCELLED_LISTINGS_DATA",
|
| 636 |
-
source="amzn_cancelled_listings", label="Amazon
|
| 637 |
identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
|
| 638 |
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 639 |
("asin1", _AMZN + "listings", "asin1", "many_to_one")),
|
| 640 |
),
|
| 641 |
ConnectorTableSpec(
|
| 642 |
key=_AMZN + "suppressed_listings", native="GET_MERCHANTS_LISTINGS_FYP_REPORT",
|
| 643 |
-
source="amzn_suppressed_listings", label="Amazon
|
| 644 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 645 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 646 |
),
|
| 647 |
ConnectorTableSpec(
|
| 648 |
key=_AMZN + "listing_defects", native="GET_MERCHANT_LISTINGS_DEFECT_DATA",
|
| 649 |
-
source="amzn_listing_defects", label="Amazon
|
| 650 |
identity=("_snapshot_date", "sku", "_report_row_number"),
|
| 651 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 652 |
),
|
| 653 |
ConnectorTableSpec(
|
| 654 |
key=_AMZN + "sold_listings", native="GET_CONVERGED_FLAT_FILE_SOLD_LISTINGS_DATA",
|
| 655 |
-
source="amzn_sold_listings", label="Amazon
|
| 656 |
identity=("_snapshot_date", "sku", "_report_row_number"),
|
| 657 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 658 |
),
|
| 659 |
ConnectorTableSpec(
|
| 660 |
key=_AMZN + "mfn_returns", native="GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
|
| 661 |
-
source="amzn_mfn_returns", label="Amazon
|
| 662 |
identity=("_period_start", "order_id", "sku", "asin", "_report_row_number"),
|
| 663 |
links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 664 |
),
|
| 665 |
ConnectorTableSpec(
|
| 666 |
key=_AMZN + "b2b_recommended", native="GET_B2B_PRODUCT_OPPORTUNITIES_RECOMMENDED_FOR_YOU",
|
| 667 |
-
source="amzn_b2b_recommended", label="Amazon B2B product opportunities", pinned="asin",
|
| 668 |
identity=("_snapshot_date", "asin", "_report_row_number"),
|
| 669 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 670 |
),
|
| 671 |
ConnectorTableSpec(
|
| 672 |
key=_AMZN + "b2b_not_yet_on_amazon", native="GET_B2B_PRODUCT_OPPORTUNITIES_NOT_YET_ON_AMAZON",
|
| 673 |
-
source="amzn_b2b_not_yet_on_amazon", label="Amazon B2B products not yet on Amazon", pinned="ean",
|
| 674 |
identity=("_snapshot_date", "ean", "upc", "part_number", "_report_row_number"),
|
| 675 |
links=(),
|
| 676 |
),
|
| 677 |
ConnectorTableSpec(
|
| 678 |
key=_AMZN + "geo_opportunities", native="GET_FLAT_FILE_GEO_OPPORTUNITIES",
|
| 679 |
-
source="amzn_geo_opportunities", label="Amazon
|
| 680 |
identity=("_snapshot_date", "asin", "_report_row_number"),
|
| 681 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 682 |
),
|
| 683 |
ConnectorTableSpec(
|
| 684 |
key=_AMZN + "out_of_stock_mfn", native="GET_OUT_OF_STOCK_MFN_INFORMATION",
|
| 685 |
-
source="amzn_out_of_stock_mfn", label="Amazon
|
| 686 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 687 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 688 |
),
|
| 689 |
ConnectorTableSpec(
|
| 690 |
key=_AMZN + "b2b_quotes", native="RFQD_BULK_DOWNLOAD",
|
| 691 |
-
source="amzn_b2b_quotes", label="Amazon B2B quantity discounts", pinned="sku",
|
| 692 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 693 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 694 |
),
|
| 695 |
ConnectorTableSpec(
|
| 696 |
key=_AMZN + "fee_discounts", native="FEE_DISCOUNTS_REPORT",
|
| 697 |
-
source="amzn_fee_discounts", label="Amazon
|
| 698 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 699 |
links=(), # The measured empty report supplies metadata only.
|
| 700 |
),
|
| 701 |
ConnectorTableSpec(
|
| 702 |
key=_AMZN + "order_metrics", native="sales/v1/orderMetrics",
|
| 703 |
-
source="amzn_order_metrics", label="Amazon
|
| 704 |
identity=("_period_start", "interval", "_report_row_number"),
|
| 705 |
links=(),
|
| 706 |
),
|
| 707 |
ConnectorTableSpec(
|
| 708 |
key=_AMZN + "fba_inventory", native="fba/inventory/v1/summaries",
|
| 709 |
-
source="amzn_fba_inventory", label="Amazon
|
| 710 |
identity=("_snapshot_date", "seller_sku", "fn_sku", "asin", "_report_row_number"),
|
| 711 |
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 712 |
),
|
| 713 |
ConnectorTableSpec(
|
| 714 |
key=_AMZN + "listing_items", native="listings/2021-08-01/items/{sellerId}",
|
| 715 |
-
source="amzn_listing_items", label="Amazon
|
| 716 |
identity=("sku",),
|
| 717 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 718 |
),
|
| 719 |
ConnectorTableSpec(
|
| 720 |
key=_AMZN + "catalog_items", native="catalog/2022-04-01/items",
|
| 721 |
-
source="amzn_catalog_items", label="Amazon
|
| 722 |
identity=("_snapshot_date", "asin", "_report_row_number"),
|
| 723 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 724 |
),
|
| 725 |
ConnectorTableSpec(
|
| 726 |
key=_AMZN + "fee_estimates_api", native="products/fees/v0/feesEstimate",
|
| 727 |
-
source="amzn_fee_estimates_api", label="Amazon
|
| 728 |
identity=("estimate_id",),
|
| 729 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 730 |
),
|
| 731 |
ConnectorTableSpec(
|
| 732 |
key=_AMZN + "outbound_fulfillment", native="fulfillmentOutbound/2020-07-01/fulfillmentOrders",
|
| 733 |
-
source="amzn_outbound_fulfillment", label="Amazon
|
| 734 |
identity=("seller_fulfillment_order_id", "seller_fulfillment_order_item_id"),
|
| 735 |
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 736 |
),
|
| 737 |
ConnectorTableSpec(
|
| 738 |
key=_AMZN + "inbound_shipments", native="inbound/fba/2024-03-20/inboundPlans",
|
| 739 |
-
source="amzn_inbound_shipments", label="Amazon
|
| 740 |
identity=("shipment_item_id",),
|
| 741 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 742 |
),
|
| 743 |
ConnectorTableSpec(
|
| 744 |
key=_AMZN + "replenishment", native="replenishment/2022-11-07/offers/metrics/search",
|
| 745 |
-
source="amzn_replenishment", label="Amazon
|
| 746 |
identity=("metric_id",),
|
| 747 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 748 |
),
|
| 749 |
ConnectorTableSpec(
|
| 750 |
key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance",
|
| 751 |
-
source="amzn_sku_keywords", label="Amazon
|
| 752 |
identity=("keyword_id",),
|
| 753 |
links=(("sku", _AMZN + "listing_items", "sku", "many_to_one"),),
|
| 754 |
),
|
| 755 |
ConnectorTableSpec(
|
| 756 |
key=_AMZN + "keyword_ranks", native="derived:apify:khadinakbar/amazon-search-scraper+sku_keywords",
|
| 757 |
-
source="amzn_keyword_ranks", label="Amazon
|
| 758 |
identity=("rank_id",),
|
| 759 |
links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),),
|
| 760 |
),
|
|
@@ -3570,6 +3573,11 @@ def table_group(key):
|
|
| 3570 |
return None
|
| 3571 |
|
| 3572 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3573 |
register("amazon", AMAZON_TABLES,
|
| 3574 |
mirror=ConnectorMirrorSpec(tuple(s.source for s in AMAZON_TABLES
|
| 3575 |
if not s.native.startswith("derived:"))
|
|
|
|
| 308 |
"key": edge["key"], "label": _link_field_label(edge["label"]), "type": "link",
|
| 309 |
"source": "overlay", "default": True,
|
| 310 |
"link": {"table": target, "on": edge["to_col"], "from": edge["from_col"],
|
| 311 |
+
"single": edge["cardinality"] == "many_to_one",
|
| 312 |
+
**({"connectionScope": "_connection_id"}
|
| 313 |
+
if source.startswith("ut_amzn_") and not source.startswith("ut_amzn_ads_")
|
| 314 |
+
and "store" in live_columns[source] else {})},
|
| 315 |
})
|
| 316 |
return out
|
| 317 |
|
|
|
|
| 339 |
key=_AMZN + "orders",
|
| 340 |
native="GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL",
|
| 341 |
source="amzn_orders",
|
| 342 |
+
label="Amazon Sales: Orders",
|
| 343 |
pinned="amazon_order_id",
|
| 344 |
identity=("amazon_order_id",),
|
| 345 |
links=(
|
|
|
|
| 350 |
key=_AMZN + "order_items",
|
| 351 |
native="GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL",
|
| 352 |
source="amzn_order_items",
|
| 353 |
+
label="Amazon Sales: Order items",
|
| 354 |
pinned="amazon_order_id",
|
| 355 |
identity=("order_item_id", "amazon_order_id"),
|
| 356 |
links=(
|
|
|
|
| 364 |
key=_AMZN + "listings",
|
| 365 |
native="GET_MERCHANT_LISTINGS_ALL_DATA",
|
| 366 |
source="amzn_listings",
|
| 367 |
+
label="Amazon Listings: All SKUs",
|
| 368 |
pinned="seller_sku",
|
| 369 |
identity=("seller_sku", "asin1", "asin"),
|
| 370 |
links=(), # the product spine's HEAD: everything points at it, it points nowhere
|
|
|
|
| 373 |
key=_AMZN + "settlements",
|
| 374 |
native="GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2",
|
| 375 |
source="amzn_settlements",
|
| 376 |
+
label="Amazon Money: Settlements",
|
| 377 |
pinned="settlement_id",
|
| 378 |
identity=("settlement_id",),
|
| 379 |
# ⚠ THE MONEY-TO-ORDER BRIDGE — what turns "an order happened" into "this is what we were
|
|
|
|
| 387 |
key=_AMZN + "inventory",
|
| 388 |
native="GET_LEDGER_SUMMARY_VIEW_DATA",
|
| 389 |
source="amzn_inventory",
|
| 390 |
+
label="Amazon Inventory: All",
|
| 391 |
pinned="asin",
|
| 392 |
identity=("asin", "fnsku", "msku"),
|
| 393 |
# ⚠ THE LEDGER KEYS ON `msku`, NOT `sku` — FBA calls the seller's own SKU the MSKU. Same
|
|
|
|
| 401 |
key=_AMZN + "pricing",
|
| 402 |
native="GET_SALES_AND_TRAFFIC_REPORT",
|
| 403 |
source="amzn_pricing",
|
| 404 |
+
label="Amazon Listings: Pricing and Buy Box",
|
| 405 |
pinned="parent_asin",
|
| 406 |
identity=("child_asin", "parent_asin"),
|
| 407 |
links=(
|
|
|
|
| 413 |
key=_AMZN + "brand_analytics",
|
| 414 |
native="GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT",
|
| 415 |
source="amzn_brand_analytics",
|
| 416 |
+
label="Amazon Marketing: Brand analytics",
|
| 417 |
pinned="search_term",
|
| 418 |
identity=("department_name", "search_term", "clicked_asin", "click_share_rank",
|
| 419 |
"_period_start", "_period_end"),
|
|
|
|
| 424 |
),
|
| 425 |
ConnectorTableSpec(
|
| 426 |
key=_AMZN + "economics", native="analytics_economics_2024_03_15",
|
| 427 |
+
source="amzn_economics", label="Amazon Sales: Unit economics", pinned="msku",
|
| 428 |
identity=("start_date", "msku", "marketplace_id"),
|
| 429 |
links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 430 |
("child_asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 431 |
),
|
| 432 |
ConnectorTableSpec(
|
| 433 |
key=_AMZN + "transactions", native="finances/2024-06-19/transactions",
|
| 434 |
+
source="amzn_transactions", label="Amazon Money: Transactions", pinned="transaction_id",
|
| 435 |
identity=("transaction_id", "item_position"),
|
| 436 |
links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
|
| 437 |
("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
|
|
|
| 439 |
),
|
| 440 |
ConnectorTableSpec(
|
| 441 |
key=_AMZN + "fba_fee_estimates", native="GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA",
|
| 442 |
+
source="amzn_fba_fee_estimates", label="Amazon Fees: FBA", pinned="sku",
|
| 443 |
identity=("_snapshot_date", "sku", "amazon_store"),
|
| 444 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 445 |
("asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 446 |
),
|
| 447 |
ConnectorTableSpec(
|
| 448 |
key=_AMZN + "referral_fees", native="GET_REFERRAL_FEE_PREVIEW_REPORT",
|
| 449 |
+
source="amzn_referral_fees", label="Amazon Fees: Referral", pinned="seller_sku",
|
| 450 |
identity=("_snapshot_date", "seller_sku"),
|
| 451 |
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 452 |
("asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 453 |
),
|
| 454 |
ConnectorTableSpec(
|
| 455 |
key=_AMZN + "storage_fees", native="GET_FBA_STORAGE_FEE_CHARGES_DATA",
|
| 456 |
+
source="amzn_storage_fees", label="Amazon Fees: Storage", pinned="asin",
|
| 457 |
identity=("month", "asin", "fnsku", "fulfillment_center"),
|
| 458 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 459 |
),
|
| 460 |
ConnectorTableSpec(
|
| 461 |
key=_AMZN + "search_query_performance", native="GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
|
| 462 |
+
source="amzn_search_query_performance", label="Amazon Marketing: Search terms",
|
| 463 |
pinned="search_query_data_search_query",
|
| 464 |
identity=("start_date", "end_date", "asin", "search_query_data_search_query"),
|
| 465 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 466 |
),
|
| 467 |
ConnectorTableSpec(
|
| 468 |
key=_AMZN + "long_term_storage_fees", native="GET_FBA_FULFILLMENT_LONGTERM_STORAGE_FEE_CHARGES_DATA",
|
| 469 |
+
source="amzn_long_term_storage_fees", label="Amazon Fees: Long term storage", pinned="sku",
|
| 470 |
identity=("_period_start", "sku", "fnsku", "asin", "_report_row_number"),
|
| 471 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 472 |
),
|
| 473 |
ConnectorTableSpec(
|
| 474 |
key=_AMZN + "storage_overage_fees", native="GET_FBA_OVERAGE_FEE_CHARGES_DATA",
|
| 475 |
+
source="amzn_storage_overage_fees", label="Amazon Fees: Storage overage", pinned="storage_type",
|
| 476 |
identity=("_period_start", "storage_type", "_report_row_number"),
|
| 477 |
links=(),
|
| 478 |
),
|
| 479 |
ConnectorTableSpec(
|
| 480 |
key=_AMZN + "reimbursements", native="GET_FBA_REIMBURSEMENTS_DATA",
|
| 481 |
+
source="amzn_reimbursements", label="Amazon Money: Reimbursements", pinned="reimbursement_id",
|
| 482 |
identity=("_period_start", "reimbursement_id", "sku", "asin", "amazon_order_id", "_report_row_number"),
|
| 483 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"), ("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),),
|
| 484 |
),
|
| 485 |
ConnectorTableSpec(
|
| 486 |
key=_AMZN + "fba_customer_returns", native="GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
|
| 487 |
+
source="amzn_fba_customer_returns", label="Amazon Returns: FBA", pinned="order_id",
|
| 488 |
identity=("_period_start", "order_id", "sku", "fnsku", "asin", "_report_row_number"),
|
| 489 |
links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 490 |
),
|
| 491 |
ConnectorTableSpec(
|
| 492 |
key=_AMZN + "replacements", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA",
|
| 493 |
+
source="amzn_replacements", label="Amazon Returns: Replacements", pinned="replacement_amazon_order_id",
|
| 494 |
identity=("_period_start", "replacement_amazon_order_id", "original_amazon_order_id", "sku", "asin", "_report_row_number"),
|
| 495 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 496 |
),
|
| 497 |
ConnectorTableSpec(
|
| 498 |
key=_AMZN + "fba_shipped_sales", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_SALES_DATA",
|
| 499 |
+
source="amzn_fba_shipped_sales", label="Amazon Sales: FBA shipments", pinned="amazon_order_id",
|
| 500 |
identity=("_period_start", "amazon_order_id", "sku", "fnsku", "asin", "_report_row_number"),
|
| 501 |
links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 502 |
),
|
| 503 |
ConnectorTableSpec(
|
| 504 |
key=_AMZN + "fba_shipment_promotions", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA",
|
| 505 |
+
source="amzn_fba_shipment_promotions", label="Amazon Marketing: FBA shipment promotions", pinned="amazon_order_id",
|
| 506 |
identity=("_period_start", "amazon_order_id", "sku", "_report_row_number"),
|
| 507 |
links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),),
|
| 508 |
),
|
| 509 |
ConnectorTableSpec(
|
| 510 |
key=_AMZN + "inventory_ledger_detail", native="GET_LEDGER_DETAIL_VIEW_DATA",
|
| 511 |
+
source="amzn_inventory_ledger_detail", label="Amazon Inventory: Ledger", pinned="msku",
|
| 512 |
identity=("_period_start", "msku", "fnsku", "asin", "_report_row_number"),
|
| 513 |
links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 514 |
),
|
| 515 |
ConnectorTableSpec(
|
| 516 |
key=_AMZN + "inventory_planning", native="GET_FBA_INVENTORY_PLANNING_DATA",
|
| 517 |
+
source="amzn_inventory_planning", label="Amazon Inventory: Planning", pinned="sku",
|
| 518 |
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 519 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 520 |
),
|
| 521 |
ConnectorTableSpec(
|
| 522 |
key=_AMZN + "manage_inventory", native="GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA",
|
| 523 |
+
source="amzn_manage_inventory", label="Amazon Inventory: Manage inventory", pinned="sku",
|
| 524 |
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 525 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 526 |
),
|
| 527 |
ConnectorTableSpec(
|
| 528 |
key=_AMZN + "reserved_inventory", native="GET_RESERVED_INVENTORY_DATA",
|
| 529 |
+
source="amzn_reserved_inventory", label="Amazon Inventory: Reserved", pinned="sku",
|
| 530 |
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 531 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 532 |
),
|
| 533 |
ConnectorTableSpec(
|
| 534 |
key=_AMZN + "restock_recommendations", native="GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT",
|
| 535 |
+
source="amzn_restock_recommendations", label="Amazon Inventory: Restock", pinned="merchant_sku",
|
| 536 |
identity=("_snapshot_date", "merchant_sku", "fnsku", "asin", "_report_row_number"),
|
| 537 |
links=(("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 538 |
("asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 539 |
),
|
| 540 |
ConnectorTableSpec(
|
| 541 |
key=_AMZN + "stranded_inventory", native="GET_STRANDED_INVENTORY_UI_DATA",
|
| 542 |
+
source="amzn_stranded_inventory", label="Amazon Inventory: Stranded", pinned="sku",
|
| 543 |
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 544 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 545 |
),
|
| 546 |
ConnectorTableSpec(
|
| 547 |
key=_AMZN + "inbound_noncompliance", native="GET_FBA_FULFILLMENT_INBOUND_NONCOMPLIANCE_DATA",
|
| 548 |
+
source="amzn_inbound_noncompliance", label="Amazon Inventory: Inbound issues", pinned="shipment_id",
|
| 549 |
identity=("_period_start", "shipment_id", "sku", "asin", "_report_row_number"),
|
| 550 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 551 |
),
|
| 552 |
ConnectorTableSpec(
|
| 553 |
key=_AMZN + "inventory_expiration_dates", native="GET_FBA_FULFILLMENT_INVENTORY_EXPIRATION_DATE_DATA",
|
| 554 |
+
source="amzn_inventory_expiration_dates", label="Amazon Inventory: Expiration dates", pinned="sku",
|
| 555 |
identity=("_snapshot_date", "sku", "fnsku", "_report_row_number"),
|
| 556 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 557 |
),
|
| 558 |
ConnectorTableSpec(
|
| 559 |
key=_AMZN + "recommended_removals", native="GET_FBA_RECOMMENDED_REMOVAL_DATA",
|
| 560 |
+
source="amzn_recommended_removals", label="Amazon Inventory: Recommended removals", pinned="sku",
|
| 561 |
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 562 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 563 |
),
|
| 564 |
ConnectorTableSpec(
|
| 565 |
key=_AMZN + "removal_orders", native="GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
|
| 566 |
+
source="amzn_removal_orders", label="Amazon Inventory: Removal orders", pinned="order_id",
|
| 567 |
identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
|
| 568 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 569 |
),
|
| 570 |
ConnectorTableSpec(
|
| 571 |
key=_AMZN + "removal_shipments", native="GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
|
| 572 |
+
source="amzn_removal_shipments", label="Amazon Inventory: Removal shipments", pinned="order_id",
|
| 573 |
identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
|
| 574 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 575 |
),
|
| 576 |
ConnectorTableSpec(
|
| 577 |
key=_AMZN + "subscribe_save_forecast", native="GET_FBA_SNS_FORECAST_DATA",
|
| 578 |
+
source="amzn_subscribe_save_forecast", label="Amazon Marketing: Subscribe and Save forecast", pinned="sku",
|
| 579 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 580 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 581 |
),
|
| 582 |
ConnectorTableSpec(
|
| 583 |
key=_AMZN + "subscribe_save_performance", native="GET_FBA_SNS_PERFORMANCE_DATA",
|
| 584 |
+
source="amzn_subscribe_save_performance", label="Amazon Marketing: Subscribe and Save", pinned="sku",
|
| 585 |
identity=("_period_start", "sku", "asin", "_report_row_number"),
|
| 586 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 587 |
),
|
| 588 |
ConnectorTableSpec(
|
| 589 |
key=_AMZN + "remote_fulfillment_eligibility", native="GET_REMOTE_FULFILLMENT_ELIGIBILITY",
|
| 590 |
+
source="amzn_remote_fulfillment_eligibility", label="Amazon Inventory: Remote fulfillment", pinned="sku",
|
| 591 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 592 |
links=(), # Measured header has no product key; do not declare a guessed join.
|
| 593 |
),
|
| 594 |
ConnectorTableSpec(
|
| 595 |
key=_AMZN + "search_catalog_performance", native="GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT",
|
| 596 |
+
source="amzn_search_catalog_performance", label="Amazon Marketing: Search funnel by ASIN", pinned="asin",
|
| 597 |
identity=("_period_start", "asin", "_report_row_number"),
|
| 598 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 599 |
),
|
| 600 |
ConnectorTableSpec(
|
| 601 |
key=_AMZN + "market_basket", native="GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT",
|
| 602 |
+
source="amzn_market_basket", label="Amazon Marketing: Bought together", pinned="asin",
|
| 603 |
identity=("_period_start", "asin", "purchased_with_asin", "_report_row_number"),
|
| 604 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 605 |
),
|
| 606 |
ConnectorTableSpec(
|
| 607 |
key=_AMZN + "repeat_purchase", native="GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT",
|
| 608 |
+
source="amzn_repeat_purchase", label="Amazon Marketing: Repeat purchases", pinned="asin",
|
| 609 |
identity=("_period_start", "asin", "_report_row_number"),
|
| 610 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 611 |
),
|
| 612 |
ConnectorTableSpec(
|
| 613 |
key=_AMZN + "promotion_performance", native="GET_PROMOTION_PERFORMANCE_REPORT",
|
| 614 |
+
source="amzn_promotion_performance", label="Amazon Marketing: Promotions", pinned="promotion_id",
|
| 615 |
identity=("_period_start", "promotion_id", "_report_row_number"),
|
| 616 |
links=(),
|
| 617 |
),
|
| 618 |
ConnectorTableSpec(
|
| 619 |
key=_AMZN + "coupon_performance", native="GET_COUPON_PERFORMANCE_REPORT",
|
| 620 |
+
source="amzn_coupon_performance", label="Amazon Marketing: Coupons", pinned="coupon_id",
|
| 621 |
identity=("_period_start", "coupon_id", "_report_row_number"),
|
| 622 |
links=(),
|
| 623 |
),
|
| 624 |
ConnectorTableSpec(
|
| 625 |
key=_AMZN + "account_health", native="GET_V2_SELLER_PERFORMANCE_REPORT",
|
| 626 |
+
source="amzn_account_health", label="Amazon Account health", pinned="marketplace_id",
|
| 627 |
identity=("_snapshot_date", "marketplace_id", "_json_root", "_report_row_number"),
|
| 628 |
links=(),
|
| 629 |
),
|
| 630 |
ConnectorTableSpec(
|
| 631 |
key=_AMZN + "inactive_listings", native="GET_MERCHANT_LISTINGS_INACTIVE_DATA",
|
| 632 |
+
source="amzn_inactive_listings", label="Amazon Listings: Inactive", pinned="seller_sku",
|
| 633 |
identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
|
| 634 |
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 635 |
("asin1", _AMZN + "listings", "asin1", "many_to_one")),
|
| 636 |
),
|
| 637 |
ConnectorTableSpec(
|
| 638 |
key=_AMZN + "cancelled_listings", native="GET_MERCHANT_CANCELLED_LISTINGS_DATA",
|
| 639 |
+
source="amzn_cancelled_listings", label="Amazon Listings: Cancelled", pinned="seller_sku",
|
| 640 |
identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
|
| 641 |
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 642 |
("asin1", _AMZN + "listings", "asin1", "many_to_one")),
|
| 643 |
),
|
| 644 |
ConnectorTableSpec(
|
| 645 |
key=_AMZN + "suppressed_listings", native="GET_MERCHANTS_LISTINGS_FYP_REPORT",
|
| 646 |
+
source="amzn_suppressed_listings", label="Amazon Listings: Suppressed", pinned="sku",
|
| 647 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 648 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 649 |
),
|
| 650 |
ConnectorTableSpec(
|
| 651 |
key=_AMZN + "listing_defects", native="GET_MERCHANT_LISTINGS_DEFECT_DATA",
|
| 652 |
+
source="amzn_listing_defects", label="Amazon Listings: Defects", pinned="sku",
|
| 653 |
identity=("_snapshot_date", "sku", "_report_row_number"),
|
| 654 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 655 |
),
|
| 656 |
ConnectorTableSpec(
|
| 657 |
key=_AMZN + "sold_listings", native="GET_CONVERGED_FLAT_FILE_SOLD_LISTINGS_DATA",
|
| 658 |
+
source="amzn_sold_listings", label="Amazon Listings: Sold", pinned="sku",
|
| 659 |
identity=("_snapshot_date", "sku", "_report_row_number"),
|
| 660 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 661 |
),
|
| 662 |
ConnectorTableSpec(
|
| 663 |
key=_AMZN + "mfn_returns", native="GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
|
| 664 |
+
source="amzn_mfn_returns", label="Amazon Returns: Seller fulfilled", pinned="order_id",
|
| 665 |
identity=("_period_start", "order_id", "sku", "asin", "_report_row_number"),
|
| 666 |
links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 667 |
),
|
| 668 |
ConnectorTableSpec(
|
| 669 |
key=_AMZN + "b2b_recommended", native="GET_B2B_PRODUCT_OPPORTUNITIES_RECOMMENDED_FOR_YOU",
|
| 670 |
+
source="amzn_b2b_recommended", label="Amazon Marketing: B2B product opportunities", pinned="asin",
|
| 671 |
identity=("_snapshot_date", "asin", "_report_row_number"),
|
| 672 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 673 |
),
|
| 674 |
ConnectorTableSpec(
|
| 675 |
key=_AMZN + "b2b_not_yet_on_amazon", native="GET_B2B_PRODUCT_OPPORTUNITIES_NOT_YET_ON_AMAZON",
|
| 676 |
+
source="amzn_b2b_not_yet_on_amazon", label="Amazon Marketing: B2B products not yet on Amazon", pinned="ean",
|
| 677 |
identity=("_snapshot_date", "ean", "upc", "part_number", "_report_row_number"),
|
| 678 |
links=(),
|
| 679 |
),
|
| 680 |
ConnectorTableSpec(
|
| 681 |
key=_AMZN + "geo_opportunities", native="GET_FLAT_FILE_GEO_OPPORTUNITIES",
|
| 682 |
+
source="amzn_geo_opportunities", label="Amazon Marketing: Geo opportunities", pinned="asin",
|
| 683 |
identity=("_snapshot_date", "asin", "_report_row_number"),
|
| 684 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 685 |
),
|
| 686 |
ConnectorTableSpec(
|
| 687 |
key=_AMZN + "out_of_stock_mfn", native="GET_OUT_OF_STOCK_MFN_INFORMATION",
|
| 688 |
+
source="amzn_out_of_stock_mfn", label="Amazon Listings: Out of stock MFN", pinned="sku",
|
| 689 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 690 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 691 |
),
|
| 692 |
ConnectorTableSpec(
|
| 693 |
key=_AMZN + "b2b_quotes", native="RFQD_BULK_DOWNLOAD",
|
| 694 |
+
source="amzn_b2b_quotes", label="Amazon Listings: B2B quantity discounts", pinned="sku",
|
| 695 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 696 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 697 |
),
|
| 698 |
ConnectorTableSpec(
|
| 699 |
key=_AMZN + "fee_discounts", native="FEE_DISCOUNTS_REPORT",
|
| 700 |
+
source="amzn_fee_discounts", label="Amazon Fees: Discounts", pinned="sku",
|
| 701 |
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 702 |
links=(), # The measured empty report supplies metadata only.
|
| 703 |
),
|
| 704 |
ConnectorTableSpec(
|
| 705 |
key=_AMZN + "order_metrics", native="sales/v1/orderMetrics",
|
| 706 |
+
source="amzn_order_metrics", label="Amazon Sales: Metrics by period", pinned="interval",
|
| 707 |
identity=("_period_start", "interval", "_report_row_number"),
|
| 708 |
links=(),
|
| 709 |
),
|
| 710 |
ConnectorTableSpec(
|
| 711 |
key=_AMZN + "fba_inventory", native="fba/inventory/v1/summaries",
|
| 712 |
+
source="amzn_fba_inventory", label="Amazon Inventory: FBA", pinned="seller_sku",
|
| 713 |
identity=("_snapshot_date", "seller_sku", "fn_sku", "asin", "_report_row_number"),
|
| 714 |
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 715 |
),
|
| 716 |
ConnectorTableSpec(
|
| 717 |
key=_AMZN + "listing_items", native="listings/2021-08-01/items/{sellerId}",
|
| 718 |
+
source="amzn_listing_items", label="Amazon Listings: Status and issues", pinned="sku",
|
| 719 |
identity=("sku",),
|
| 720 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 721 |
),
|
| 722 |
ConnectorTableSpec(
|
| 723 |
key=_AMZN + "catalog_items", native="catalog/2022-04-01/items",
|
| 724 |
+
source="amzn_catalog_items", label="Amazon ASINs", pinned="asin",
|
| 725 |
identity=("_snapshot_date", "asin", "_report_row_number"),
|
| 726 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 727 |
),
|
| 728 |
ConnectorTableSpec(
|
| 729 |
key=_AMZN + "fee_estimates_api", native="products/fees/v0/feesEstimate",
|
| 730 |
+
source="amzn_fee_estimates_api", label="Amazon Fees: Per-price estimates", pinned="sku",
|
| 731 |
identity=("estimate_id",),
|
| 732 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 733 |
),
|
| 734 |
ConnectorTableSpec(
|
| 735 |
key=_AMZN + "outbound_fulfillment", native="fulfillmentOutbound/2020-07-01/fulfillmentOrders",
|
| 736 |
+
source="amzn_outbound_fulfillment", label="Amazon Inventory: Outbound fulfillment orders", pinned="seller_fulfillment_order_id",
|
| 737 |
identity=("seller_fulfillment_order_id", "seller_fulfillment_order_item_id"),
|
| 738 |
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 739 |
),
|
| 740 |
ConnectorTableSpec(
|
| 741 |
key=_AMZN + "inbound_shipments", native="inbound/fba/2024-03-20/inboundPlans",
|
| 742 |
+
source="amzn_inbound_shipments", label="Amazon Inventory: Inbound shipments", pinned="sku",
|
| 743 |
identity=("shipment_item_id",),
|
| 744 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 745 |
),
|
| 746 |
ConnectorTableSpec(
|
| 747 |
key=_AMZN + "replenishment", native="replenishment/2022-11-07/offers/metrics/search",
|
| 748 |
+
source="amzn_replenishment", label="Amazon Inventory: Replenishment", pinned="sku",
|
| 749 |
identity=("metric_id",),
|
| 750 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 751 |
),
|
| 752 |
ConnectorTableSpec(
|
| 753 |
key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance",
|
| 754 |
+
source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku",
|
| 755 |
identity=("keyword_id",),
|
| 756 |
links=(("sku", _AMZN + "listing_items", "sku", "many_to_one"),),
|
| 757 |
),
|
| 758 |
ConnectorTableSpec(
|
| 759 |
key=_AMZN + "keyword_ranks", native="derived:apify:khadinakbar/amazon-search-scraper+sku_keywords",
|
| 760 |
+
source="amzn_keyword_ranks", label="Amazon Marketing: Keyword ranks", pinned="keyword",
|
| 761 |
identity=("rank_id",),
|
| 762 |
links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),),
|
| 763 |
),
|
|
|
|
| 3573 |
return None
|
| 3574 |
|
| 3575 |
|
| 3576 |
+
for _amazon_table in AMAZON_TABLES:
|
| 3577 |
+
AMAZON_FIELDS[_amazon_table.key] = (*AMAZON_FIELDS.get(_amazon_table.key, ()), {
|
| 3578 |
+
"name": "store", "type": "VARCHAR", "label": "Store", "source": "keychain.label",
|
| 3579 |
+
})
|
| 3580 |
+
|
| 3581 |
register("amazon", AMAZON_TABLES,
|
| 3582 |
mirror=ConnectorMirrorSpec(tuple(s.source for s in AMAZON_TABLES
|
| 3583 |
if not s.native.startswith("derived:"))
|
platform/core/connector_sync.py
CHANGED
|
@@ -206,10 +206,14 @@ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=Non
|
|
| 206 |
with _SYNC_REQUEST_LOCK:
|
| 207 |
previous = _SYNC_REQUESTS.get((provider, tenant))
|
| 208 |
if previous and previous["state"] in ("queued", "running"):
|
| 209 |
-
|
| 210 |
-
|
|
|
|
|
|
|
| 211 |
"requestedAt": time.time() if now is None else now,
|
| 212 |
-
"notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0)
|
|
|
|
|
|
|
| 213 |
"startedAt": None, "finishedAt": None, "detail": "",
|
| 214 |
"requestedBy": str(requested_by or ""), "accepted": True}
|
| 215 |
if requested_by == "first load" and previous and previous.get("firstLoad"):
|
|
@@ -220,7 +224,39 @@ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=Non
|
|
| 220 |
_arm_sync_wake(stamp)
|
| 221 |
if (receipt.get("notBefore") or 0) <= stamp:
|
| 222 |
_SYNC_WAKE.set()
|
| 223 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
|
| 225 |
|
| 226 |
def sync_request(provider, tenant):
|
|
@@ -231,22 +267,34 @@ def sync_request(provider, tenant):
|
|
| 231 |
receipt = next((row for row in rows if row["state"] == "running"), None)
|
| 232 |
if receipt is None:
|
| 233 |
queued = [row for row in rows if row["state"] == "queued"]
|
| 234 |
-
receipt = min(queued, key=
|
| 235 |
if receipt is None:
|
| 236 |
return None
|
| 237 |
-
return {**
|
| 238 |
|
| 239 |
|
| 240 |
-
def take_sync_requests(*, now=None):
|
| 241 |
-
"""Claim
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
with _SYNC_REQUEST_LOCK:
|
| 243 |
_SYNC_WAKE.clear()
|
| 244 |
claimed = []
|
| 245 |
stamp = time.time() if now is None else now
|
| 246 |
busy = {(row["provider"], row["tenant"]) for row in _SYNC_REQUESTS.values() if row["state"] == "running"}
|
| 247 |
-
for receipt in sorted(list(_SYNC_REQUESTS.values()), key=
|
|
|
|
|
|
|
| 248 |
pair = (receipt["provider"], receipt["tenant"])
|
| 249 |
-
if pair in busy or receipt["state"] != "queued"
|
|
|
|
| 250 |
continue
|
| 251 |
if receipt.get("tables") is not None:
|
| 252 |
rt = _SYNC_REQUEST_RUNTIMES[receipt["tenant"]]
|
|
@@ -260,7 +308,7 @@ def take_sync_requests(*, now=None):
|
|
| 260 |
return document
|
| 261 |
eligible = [row for row in rows if row["state"] == "queued"
|
| 262 |
and (row.get("notBefore") or 0) <= stamp]
|
| 263 |
-
row = min(eligible, key=
|
| 264 |
if row is not None and row["id"] == identity:
|
| 265 |
row.update(state="running", startedAt=stamp, claimToken=token, accepted=False)
|
| 266 |
return document
|
|
@@ -272,9 +320,12 @@ def take_sync_requests(*, now=None):
|
|
| 272 |
continue
|
| 273 |
else:
|
| 274 |
receipt.update(state="running", startedAt=stamp, accepted=False)
|
|
|
|
|
|
|
|
|
|
| 275 |
claimed.append(deepcopy(receipt))
|
| 276 |
busy.add(pair)
|
| 277 |
-
|
| 278 |
return claimed
|
| 279 |
|
| 280 |
|
|
@@ -310,6 +361,7 @@ def finish_sync_request(provider, tenant, *, ok, detail="", now=None, first_load
|
|
| 310 |
else:
|
| 311 |
receipt.update(completion)
|
| 312 |
result = deepcopy(receipt)
|
|
|
|
| 313 |
_wake_sync_requests(stamp)
|
| 314 |
connector = {"amazon_sp": "amazon", "meta_ads": "meta"}.get(provider)
|
| 315 |
if connector:
|
|
@@ -334,7 +386,7 @@ def sync_wake_event():
|
|
| 334 |
def queue_remote_request(rt, receipt, connection_id=None, *, now=None):
|
| 335 |
"""Web side: durably hand one accepted ordinary request to the sync worker process."""
|
| 336 |
stamp = time.time() if now is None else now
|
| 337 |
-
identity = uuid.uuid4().hex
|
| 338 |
entry = {"id": identity, "provider": receipt["provider"], "tenant": rt.key,
|
| 339 |
"requestedBy": str(receipt.get("requestedBy") or ""),
|
| 340 |
"requestedAt": float(receipt.get("requestedAt") or stamp),
|
|
@@ -365,10 +417,13 @@ def fail_remote_request(provider, tenant, detail, *, now=None):
|
|
| 365 |
def _remote_progress(entry, receipt):
|
| 366 |
"""The worker receipt's progress for one entry, or None when that run does not cover it."""
|
| 367 |
started = receipt.get("startedAt") if receipt else None
|
| 368 |
-
if
|
|
|
|
|
|
|
| 369 |
return None
|
| 370 |
progress = {"state": receipt["state"], "startedAt": started,
|
| 371 |
-
"finishedAt": receipt.get("finishedAt"), "detail": receipt.get("detail") or ""
|
|
|
|
| 372 |
if receipt.get("firstLoad"):
|
| 373 |
progress["firstLoad"] = dict(receipt["firstLoad"])
|
| 374 |
return progress
|
|
@@ -384,8 +439,7 @@ def poll_remote_requests(rt, *, now=None):
|
|
| 384 |
admitted, changes, pruned = [], {}, []
|
| 385 |
with _SYNC_REQUEST_LOCK:
|
| 386 |
document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
|
| 387 |
-
for identity, entry in sorted(document[REMOTE_MEMBER].items(),
|
| 388 |
-
key=lambda item: item[1].get("requestedAt") or 0):
|
| 389 |
key = (entry.get("provider"), rt.key)
|
| 390 |
if entry.get("tenant") != rt.key or key[0] not in _PROVIDERS:
|
| 391 |
continue
|
|
@@ -394,6 +448,16 @@ def poll_remote_requests(rt, *, now=None):
|
|
| 394 |
pruned.append(identity)
|
| 395 |
continue
|
| 396 |
receipt = _SYNC_REQUESTS.get(key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 397 |
progress = _remote_progress(entry, receipt)
|
| 398 |
if progress is not None:
|
| 399 |
if any(entry.get(name) != value for name, value in progress.items()):
|
|
@@ -402,17 +466,30 @@ def poll_remote_requests(rt, *, now=None):
|
|
| 402 |
# A "running" entry with no local run was abandoned by a worker restart: admit it again.
|
| 403 |
if (entry["state"] in ("queued", "running")
|
| 404 |
and not (receipt and receipt["state"] in ("queued", "running"))):
|
| 405 |
-
local = {"provider": key[0], "tenant": rt.key, "state": "queued",
|
| 406 |
"requestedAt": entry["requestedAt"],
|
| 407 |
"notBefore": max(float(entry.get("notBefore") or 0),
|
| 408 |
_SYNC_NOT_BEFORE.get(key, 0)) or None,
|
| 409 |
"startedAt": None, "finishedAt": None, "detail": "",
|
|
|
|
| 410 |
"requestedBy": entry.get("requestedBy") or "", "accepted": True}
|
| 411 |
if receipt and receipt.get("firstLoad") and local["requestedBy"] == "first load":
|
| 412 |
local.update(detail=receipt["detail"], firstLoad=dict(receipt["firstLoad"]))
|
| 413 |
_SYNC_REQUESTS[key] = local
|
| 414 |
_SYNC_REQUEST_RUNTIMES.setdefault(rt.key, rt)
|
| 415 |
admitted.append(deepcopy(entry))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 416 |
if changes or pruned:
|
| 417 |
def publish(current):
|
| 418 |
doc = _sync_request_document(current)
|
|
@@ -438,10 +515,11 @@ def apply_remote_progress(rt):
|
|
| 438 |
receipt = _SYNC_REQUESTS.get(key)
|
| 439 |
if _REMOTE_IDS.get(key) != identity or receipt is None:
|
| 440 |
continue
|
| 441 |
-
fields = {name: entry.get(name) for name in
|
|
|
|
| 442 |
if entry.get("firstLoad"):
|
| 443 |
fields["firstLoad"] = dict(entry["firstLoad"])
|
| 444 |
-
if
|
| 445 |
receipt.update(fields)
|
| 446 |
applied += 1
|
| 447 |
# Event receipts the worker claimed or finished, refreshed without requeueing anything.
|
|
|
|
| 206 |
with _SYNC_REQUEST_LOCK:
|
| 207 |
previous = _SYNC_REQUESTS.get((provider, tenant))
|
| 208 |
if previous and previous["state"] in ("queued", "running"):
|
| 209 |
+
if not (previous["state"] == "queued" and previous.get("requestedBy") == "first load"
|
| 210 |
+
and requested_by != "first load"):
|
| 211 |
+
return {**_request_wait(previous), "accepted": False}
|
| 212 |
+
receipt = {"id": uuid.uuid4().hex, "provider": provider, "tenant": tenant, "state": "queued",
|
| 213 |
"requestedAt": time.time() if now is None else now,
|
| 214 |
+
"notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0),
|
| 215 |
+
float(previous.get("notBefore") or 0) if previous
|
| 216 |
+
and previous["state"] == "queued" else 0) or None,
|
| 217 |
"startedAt": None, "finishedAt": None, "detail": "",
|
| 218 |
"requestedBy": str(requested_by or ""), "accepted": True}
|
| 219 |
if requested_by == "first load" and previous and previous.get("firstLoad"):
|
|
|
|
| 224 |
_arm_sync_wake(stamp)
|
| 225 |
if (receipt.get("notBefore") or 0) <= stamp:
|
| 226 |
_SYNC_WAKE.set()
|
| 227 |
+
return _request_wait(receipt, now=stamp)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _request_order(receipt):
|
| 231 |
+
"""Manual work precedes first-load continuations, with FIFO within each class."""
|
| 232 |
+
return receipt.get("requestedBy") == "first load", receipt.get("requestedAt") or 0
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def _request_wait(receipt, *, now=None):
|
| 236 |
+
"""Describe a queued receipt's executing competitor or provider floor without claiming a start."""
|
| 237 |
+
result = deepcopy(receipt)
|
| 238 |
+
result.setdefault("waitingFor", None)
|
| 239 |
+
if receipt["state"] != "queued":
|
| 240 |
+
result["waitingFor"] = None
|
| 241 |
+
return result
|
| 242 |
+
running = [row for row in _SYNC_REQUESTS.values()
|
| 243 |
+
if row["tenant"] == receipt["tenant"] and row["state"] == "running"
|
| 244 |
+
and row.get("startedAt") is not None]
|
| 245 |
+
if running:
|
| 246 |
+
active = max(running, key=lambda row: row["startedAt"])
|
| 247 |
+
started = active["startedAt"]
|
| 248 |
+
identity = active.get("id") or f"{active['provider']}:{started}"
|
| 249 |
+
result["waitingFor"] = {"id": identity, "provider": active["provider"], "startedAt": started}
|
| 250 |
+
stamp = _dt.datetime.fromtimestamp(started, _dt.timezone.utc).isoformat()
|
| 251 |
+
result["detail"] = f"Waiting for {active['provider']} receipt {identity}, started at {stamp}."
|
| 252 |
+
elif result.get("waitingFor"):
|
| 253 |
+
return result
|
| 254 |
+
elif (receipt.get("notBefore") or 0) > (time.time() if now is None else now):
|
| 255 |
+
stamp = _dt.datetime.fromtimestamp(receipt["notBefore"], _dt.timezone.utc).isoformat()
|
| 256 |
+
result["detail"] = f"Waiting for the provider not-before floor at {stamp}."
|
| 257 |
+
elif result.get("detail", "").startswith("Waiting for "):
|
| 258 |
+
result["detail"] = ""
|
| 259 |
+
return result
|
| 260 |
|
| 261 |
|
| 262 |
def sync_request(provider, tenant):
|
|
|
|
| 267 |
receipt = next((row for row in rows if row["state"] == "running"), None)
|
| 268 |
if receipt is None:
|
| 269 |
queued = [row for row in rows if row["state"] == "queued"]
|
| 270 |
+
receipt = min(queued, key=_request_order) if queued else _SYNC_REQUESTS.get((provider, tenant))
|
| 271 |
if receipt is None:
|
| 272 |
return None
|
| 273 |
+
return {**_request_wait(receipt), "accepted": False}
|
| 274 |
|
| 275 |
|
| 276 |
+
def take_sync_requests(*, now=None, limit=None, exclude_providers=(), refresh_remote=False):
|
| 277 |
+
"""Claim manual work first, preserving provider floors, tenant claims and event siblings."""
|
| 278 |
+
if refresh_remote:
|
| 279 |
+
with _SYNC_REQUEST_LOCK:
|
| 280 |
+
runtimes = list(_SYNC_REQUEST_RUNTIMES.values())
|
| 281 |
+
for rt in runtimes:
|
| 282 |
+
try:
|
| 283 |
+
poll_remote_requests(rt, now=now)
|
| 284 |
+
except Exception as exc:
|
| 285 |
+
logging.getLogger(__name__).warning("Sync desk refresh unavailable for %s (%s)",
|
| 286 |
+
rt.key, type(exc).__name__)
|
| 287 |
with _SYNC_REQUEST_LOCK:
|
| 288 |
_SYNC_WAKE.clear()
|
| 289 |
claimed = []
|
| 290 |
stamp = time.time() if now is None else now
|
| 291 |
busy = {(row["provider"], row["tenant"]) for row in _SYNC_REQUESTS.values() if row["state"] == "running"}
|
| 292 |
+
for receipt in sorted(list(_SYNC_REQUESTS.values()), key=_request_order):
|
| 293 |
+
if limit is not None and len(claimed) >= limit:
|
| 294 |
+
break
|
| 295 |
pair = (receipt["provider"], receipt["tenant"])
|
| 296 |
+
if (receipt["provider"] in exclude_providers or pair in busy or receipt["state"] != "queued"
|
| 297 |
+
or max(receipt.get("notBefore") or 0, _SYNC_NOT_BEFORE.get(pair, 0)) > stamp):
|
| 298 |
continue
|
| 299 |
if receipt.get("tables") is not None:
|
| 300 |
rt = _SYNC_REQUEST_RUNTIMES[receipt["tenant"]]
|
|
|
|
| 308 |
return document
|
| 309 |
eligible = [row for row in rows if row["state"] == "queued"
|
| 310 |
and (row.get("notBefore") or 0) <= stamp]
|
| 311 |
+
row = min(eligible, key=_request_order) if eligible else None
|
| 312 |
if row is not None and row["id"] == identity:
|
| 313 |
row.update(state="running", startedAt=stamp, claimToken=token, accepted=False)
|
| 314 |
return document
|
|
|
|
| 320 |
continue
|
| 321 |
else:
|
| 322 |
receipt.update(state="running", startedAt=stamp, accepted=False)
|
| 323 |
+
receipt["waitingFor"] = None
|
| 324 |
+
if receipt.get("detail", "").startswith("Waiting for "):
|
| 325 |
+
receipt["detail"] = ""
|
| 326 |
claimed.append(deepcopy(receipt))
|
| 327 |
busy.add(pair)
|
| 328 |
+
_wake_sync_requests(stamp)
|
| 329 |
return claimed
|
| 330 |
|
| 331 |
|
|
|
|
| 361 |
else:
|
| 362 |
receipt.update(completion)
|
| 363 |
result = deepcopy(receipt)
|
| 364 |
+
result["waitingFor"] = None
|
| 365 |
_wake_sync_requests(stamp)
|
| 366 |
connector = {"amazon_sp": "amazon", "meta_ads": "meta"}.get(provider)
|
| 367 |
if connector:
|
|
|
|
| 386 |
def queue_remote_request(rt, receipt, connection_id=None, *, now=None):
|
| 387 |
"""Web side: durably hand one accepted ordinary request to the sync worker process."""
|
| 388 |
stamp = time.time() if now is None else now
|
| 389 |
+
identity = receipt.get("id") or uuid.uuid4().hex
|
| 390 |
entry = {"id": identity, "provider": receipt["provider"], "tenant": rt.key,
|
| 391 |
"requestedBy": str(receipt.get("requestedBy") or ""),
|
| 392 |
"requestedAt": float(receipt.get("requestedAt") or stamp),
|
|
|
|
| 417 |
def _remote_progress(entry, receipt):
|
| 418 |
"""The worker receipt's progress for one entry, or None when that run does not cover it."""
|
| 419 |
started = receipt.get("startedAt") if receipt else None
|
| 420 |
+
if (started is None or entry["requestedAt"] > started
|
| 421 |
+
or receipt["state"] not in ("running", "done", "failed")
|
| 422 |
+
or (entry.get("requestedBy") != "first load" and receipt.get("requestedBy") == "first load")):
|
| 423 |
return None
|
| 424 |
progress = {"state": receipt["state"], "startedAt": started,
|
| 425 |
+
"finishedAt": receipt.get("finishedAt"), "detail": receipt.get("detail") or "",
|
| 426 |
+
"waitingFor": None}
|
| 427 |
if receipt.get("firstLoad"):
|
| 428 |
progress["firstLoad"] = dict(receipt["firstLoad"])
|
| 429 |
return progress
|
|
|
|
| 439 |
admitted, changes, pruned = [], {}, []
|
| 440 |
with _SYNC_REQUEST_LOCK:
|
| 441 |
document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
|
| 442 |
+
for identity, entry in sorted(document[REMOTE_MEMBER].items(), key=lambda item: _request_order(item[1])):
|
|
|
|
| 443 |
key = (entry.get("provider"), rt.key)
|
| 444 |
if entry.get("tenant") != rt.key or key[0] not in _PROVIDERS:
|
| 445 |
continue
|
|
|
|
| 448 |
pruned.append(identity)
|
| 449 |
continue
|
| 450 |
receipt = _SYNC_REQUESTS.get(key)
|
| 451 |
+
# Promote under the same lock as claims, so a continuation cannot win a manual hand-off race.
|
| 452 |
+
if (receipt and receipt["state"] == "queued" and receipt.get("requestedBy") == "first load"
|
| 453 |
+
and entry.get("requestedBy") != "first load"):
|
| 454 |
+
receipt.update(id=identity, requestedBy=entry.get("requestedBy") or "",
|
| 455 |
+
requestedAt=entry["requestedAt"],
|
| 456 |
+
connectionId=entry.get("connectionId"),
|
| 457 |
+
notBefore=max(float(receipt.get("notBefore") or 0),
|
| 458 |
+
float(entry.get("notBefore") or 0),
|
| 459 |
+
_SYNC_NOT_BEFORE.get(key, 0)) or None)
|
| 460 |
+
admitted.append(deepcopy(entry))
|
| 461 |
progress = _remote_progress(entry, receipt)
|
| 462 |
if progress is not None:
|
| 463 |
if any(entry.get(name) != value for name, value in progress.items()):
|
|
|
|
| 466 |
# A "running" entry with no local run was abandoned by a worker restart: admit it again.
|
| 467 |
if (entry["state"] in ("queued", "running")
|
| 468 |
and not (receipt and receipt["state"] in ("queued", "running"))):
|
| 469 |
+
local = {"id": identity, "provider": key[0], "tenant": rt.key, "state": "queued",
|
| 470 |
"requestedAt": entry["requestedAt"],
|
| 471 |
"notBefore": max(float(entry.get("notBefore") or 0),
|
| 472 |
_SYNC_NOT_BEFORE.get(key, 0)) or None,
|
| 473 |
"startedAt": None, "finishedAt": None, "detail": "",
|
| 474 |
+
"connectionId": entry.get("connectionId"),
|
| 475 |
"requestedBy": entry.get("requestedBy") or "", "accepted": True}
|
| 476 |
if receipt and receipt.get("firstLoad") and local["requestedBy"] == "first load":
|
| 477 |
local.update(detail=receipt["detail"], firstLoad=dict(receipt["firstLoad"]))
|
| 478 |
_SYNC_REQUESTS[key] = local
|
| 479 |
_SYNC_REQUEST_RUNTIMES.setdefault(rt.key, rt)
|
| 480 |
admitted.append(deepcopy(entry))
|
| 481 |
+
receipt = local
|
| 482 |
+
waiting = _request_wait({**entry, "state": "queued", "startedAt": None,
|
| 483 |
+
"finishedAt": None, "waitingFor": None,
|
| 484 |
+
"notBefore": max(float(entry.get("notBefore") or 0),
|
| 485 |
+
_SYNC_NOT_BEFORE.get(key, 0),
|
| 486 |
+
float(receipt.get("notBefore") or 0)
|
| 487 |
+
if receipt and receipt["state"] == "queued" else 0) or None},
|
| 488 |
+
now=stamp)
|
| 489 |
+
fields = {name: waiting.get(name) for name in
|
| 490 |
+
("state", "startedAt", "finishedAt", "notBefore", "detail", "waitingFor")}
|
| 491 |
+
if any(entry.get(name) != value for name, value in fields.items()):
|
| 492 |
+
changes[identity] = fields
|
| 493 |
if changes or pruned:
|
| 494 |
def publish(current):
|
| 495 |
doc = _sync_request_document(current)
|
|
|
|
| 515 |
receipt = _SYNC_REQUESTS.get(key)
|
| 516 |
if _REMOTE_IDS.get(key) != identity or receipt is None:
|
| 517 |
continue
|
| 518 |
+
fields = {name: entry.get(name) for name in
|
| 519 |
+
("state", "startedAt", "finishedAt", "detail", "notBefore", "waitingFor")}
|
| 520 |
if entry.get("firstLoad"):
|
| 521 |
fields["firstLoad"] = dict(entry["firstLoad"])
|
| 522 |
+
if any(receipt.get(k) != v for k, v in fields.items()):
|
| 523 |
receipt.update(fields)
|
| 524 |
applied += 1
|
| 525 |
# Event receipts the worker claimed or finished, refreshed without requeueing anything.
|
platform/core/edit_receipts.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Retain edit responses in the tenant store and serialize replays before applying them."""
|
| 2 |
+
from contextlib import contextmanager
|
| 3 |
+
from contextvars import ContextVar
|
| 4 |
+
from copy import deepcopy
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
import threading
|
| 8 |
+
import time
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
TABLE_NAME = 'edit_receipts'
|
| 12 |
+
_OFFLINE_LOCK = threading.RLock()
|
| 13 |
+
_ACTIVE = ContextVar('edit_receipt_operation', default=None)
|
| 14 |
+
# Reserve half of store_pg's eight connections for callbacks across all tenant handles.
|
| 15 |
+
_PG_SLOTS = threading.BoundedSemaphore(4)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class ReceiptUnavailable(RuntimeError):
|
| 19 |
+
"""The edit cannot receive a durable acknowledgement."""
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class CheckedStore:
|
| 23 |
+
"""Require synchronous acknowledged writes from the event's existing store handle."""
|
| 24 |
+
|
| 25 |
+
def __init__(self, st):
|
| 26 |
+
self.st = st
|
| 27 |
+
|
| 28 |
+
def __getattr__(self, name):
|
| 29 |
+
return getattr(self.st, name)
|
| 30 |
+
|
| 31 |
+
def _store(self):
|
| 32 |
+
return self.st._store() if callable(getattr(self.st, '_store', None)) else self.st
|
| 33 |
+
|
| 34 |
+
def update(self, name, fn, flush='sync', **kwargs):
|
| 35 |
+
result = self.st.update(name, fn, flush='sync', **kwargs)
|
| 36 |
+
if result is None or result is False:
|
| 37 |
+
raise ReceiptUnavailable('The edit write was not saved. Retry this opId.')
|
| 38 |
+
return result
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def op_id(value):
|
| 42 |
+
"""Keep the complete client identity without truncation or normalization."""
|
| 43 |
+
if not isinstance(value, str) or not value.strip():
|
| 44 |
+
raise ValueError('opId must be a non-empty string.')
|
| 45 |
+
return value
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def operation(scope, kind, pid=None):
|
| 49 |
+
return json.dumps([str(scope), str(kind), None if pid is None else str(pid)],
|
| 50 |
+
separators=(',', ':'))
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def current_op_id():
|
| 54 |
+
active = _ACTIVE.get()
|
| 55 |
+
return active['key'][3] if active is not None else None
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def observe_event(result):
|
| 59 |
+
active = _ACTIVE.get()
|
| 60 |
+
if active is not None:
|
| 61 |
+
active['event'] = deepcopy(result)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def current_event():
|
| 65 |
+
active = _ACTIVE.get()
|
| 66 |
+
return deepcopy(active.get('event')) if active is not None else None
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _backend(st):
|
| 70 |
+
from core import store, store_pg
|
| 71 |
+
bound = st._store() if callable(getattr(st, '_store', None)) else st
|
| 72 |
+
# The default runtime can hold the module facade instead of a PgStore.
|
| 73 |
+
if bound is store and store.backend() == 'pg':
|
| 74 |
+
bound = store._d()
|
| 75 |
+
if isinstance(bound, store_pg.PgStore):
|
| 76 |
+
if getattr(st, 'store_namespace', ''):
|
| 77 |
+
raise ReceiptUnavailable('Edit receipts require a dedicated tenant schema.')
|
| 78 |
+
if getattr(st, 'key', bound.tenant_slug) != bound.tenant_slug:
|
| 79 |
+
raise ReceiptUnavailable('The edit receipt store belongs to another tenant.')
|
| 80 |
+
return PgReceipts(bound)
|
| 81 |
+
# Detached stores retain their normal get/update contract without opening Postgres.
|
| 82 |
+
return DocumentReceipts(st)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def run_once(st, principal, action, identity, apply):
|
| 86 |
+
"""Return the first committed response and skip every later application of its identity."""
|
| 87 |
+
identity = op_id(identity)
|
| 88 |
+
key = (id(st), str(principal), str(action), identity)
|
| 89 |
+
if _ACTIVE.get() is not None and _ACTIVE.get()['key'] == key:
|
| 90 |
+
return apply()
|
| 91 |
+
|
| 92 |
+
def invoke():
|
| 93 |
+
token = _ACTIVE.set({'key': key})
|
| 94 |
+
try:
|
| 95 |
+
return apply()
|
| 96 |
+
finally:
|
| 97 |
+
_ACTIVE.reset(token)
|
| 98 |
+
|
| 99 |
+
return _backend(st).run(str(principal), str(action), identity, invoke)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
class DocumentReceipts:
|
| 103 |
+
"""Keep offline and legacy stores hermetic, with serialization within their process."""
|
| 104 |
+
|
| 105 |
+
def __init__(self, st):
|
| 106 |
+
self.st = st
|
| 107 |
+
|
| 108 |
+
def run(self, principal, action, identity, apply):
|
| 109 |
+
key = json.dumps([principal, action, identity], separators=(',', ':'))
|
| 110 |
+
with _OFFLINE_LOCK:
|
| 111 |
+
prior = (self.st.get(TABLE_NAME) or {}).get(key)
|
| 112 |
+
if prior is not None:
|
| 113 |
+
return deepcopy(prior['result'])
|
| 114 |
+
result = apply()
|
| 115 |
+
|
| 116 |
+
def save(current):
|
| 117 |
+
current = dict(current or {})
|
| 118 |
+
current[key] = {'result': deepcopy(result)}
|
| 119 |
+
return current
|
| 120 |
+
|
| 121 |
+
stored = self.st.update(TABLE_NAME, save, flush='sync')
|
| 122 |
+
if stored is True:
|
| 123 |
+
stored = self.st.get(TABLE_NAME)
|
| 124 |
+
if not isinstance(stored, dict) or key not in stored:
|
| 125 |
+
raise ReceiptUnavailable('The edit receipt was not saved. Retry this opId.')
|
| 126 |
+
return deepcopy(stored[key]['result'])
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
class PgReceipts:
|
| 130 |
+
"""Use the bound tenant schema and a transaction lock for each principal and operation."""
|
| 131 |
+
|
| 132 |
+
def __init__(self, bound):
|
| 133 |
+
from core import store_pg
|
| 134 |
+
self.slug = bound.tenant_slug
|
| 135 |
+
self.schema = store_pg._schema_for(self.slug)
|
| 136 |
+
self.table = f'"{self.schema}"."{TABLE_NAME}"'
|
| 137 |
+
|
| 138 |
+
def _connection(self):
|
| 139 |
+
from core import store_pg
|
| 140 |
+
return store_pg._pool().connection()
|
| 141 |
+
|
| 142 |
+
def ensure(self):
|
| 143 |
+
self._write_guard()
|
| 144 |
+
with self._connection() as con:
|
| 145 |
+
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
|
| 146 |
+
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} (
|
| 147 |
+
receipt_key BYTEA PRIMARY KEY,
|
| 148 |
+
principal TEXT NOT NULL, operation TEXT NOT NULL, op_id TEXT NOT NULL,
|
| 149 |
+
result JSONB NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now())''')
|
| 150 |
+
|
| 151 |
+
def _write_guard(self):
|
| 152 |
+
from core import store_pg
|
| 153 |
+
store_pg.check_write(self.slug, 'store edit receipt')
|
| 154 |
+
|
| 155 |
+
@contextmanager
|
| 156 |
+
def _admission(self):
|
| 157 |
+
from core import request_timing, store_pg
|
| 158 |
+
timeout = request_timing.remaining_budget(store_pg._pg_limits()['acquire'])
|
| 159 |
+
if not _PG_SLOTS.acquire(timeout=timeout):
|
| 160 |
+
request_timing.remaining_budget()
|
| 161 |
+
raise store_pg.PostgresOperationTimeout('Edit receipt admission timed out. Retry this opId.')
|
| 162 |
+
try:
|
| 163 |
+
request_timing.remaining_budget()
|
| 164 |
+
yield
|
| 165 |
+
finally:
|
| 166 |
+
_PG_SLOTS.release()
|
| 167 |
+
|
| 168 |
+
def run(self, principal, action, identity, apply):
|
| 169 |
+
from core import request_timing, store_pg
|
| 170 |
+
limits = store_pg._pg_limits()
|
| 171 |
+
deadline = time.monotonic() + limits['acquire'] + limits['statement']
|
| 172 |
+
with request_timing.budget_scope(deadline, store_pg.PostgresOperationTimeout):
|
| 173 |
+
with self._admission():
|
| 174 |
+
return self._run(principal, action, identity, apply)
|
| 175 |
+
|
| 176 |
+
def _run(self, principal, action, identity, apply):
|
| 177 |
+
from core import request_timing, store_pg
|
| 178 |
+
self.ensure()
|
| 179 |
+
store = store_pg.PgStore(self.slug)
|
| 180 |
+
lock = json.dumps([self.table, principal, action, identity], separators=(',', ':'))
|
| 181 |
+
receipt_key = hashlib.sha256(lock.encode('utf-8')).digest()
|
| 182 |
+
# Poll without keeping a pool connection that the current writer may need.
|
| 183 |
+
while True:
|
| 184 |
+
request_timing.remaining_budget()
|
| 185 |
+
with self._connection() as con:
|
| 186 |
+
acquired = con.execute(
|
| 187 |
+
'SELECT pg_try_advisory_xact_lock(hashtextextended(%s, 0))',
|
| 188 |
+
(lock,)).fetchone()[0]
|
| 189 |
+
if acquired:
|
| 190 |
+
# The callback may leave this transaction idle, within the existing database budget.
|
| 191 |
+
idle_ms = max(1, int(request_timing.remaining_budget(
|
| 192 |
+
store_pg._pg_limits()['idle']) * 1000))
|
| 193 |
+
con.execute("SELECT set_config('idle_in_transaction_session_timeout', %s, true)",
|
| 194 |
+
(f'{idle_ms}ms',))
|
| 195 |
+
prior = con.execute(f'SELECT result FROM {self.table} '
|
| 196 |
+
'WHERE receipt_key=%s', (receipt_key,)).fetchone()
|
| 197 |
+
if prior is not None:
|
| 198 |
+
result = deepcopy(prior[0])
|
| 199 |
+
break
|
| 200 |
+
with store.transaction(con):
|
| 201 |
+
result = apply()
|
| 202 |
+
request_timing.remaining_budget()
|
| 203 |
+
con.execute(f'INSERT INTO {self.table} '
|
| 204 |
+
'(receipt_key, principal, operation, op_id, result) '
|
| 205 |
+
'VALUES (%s, %s, %s, %s, %s::jsonb)',
|
| 206 |
+
(receipt_key, principal, action, identity, json.dumps(result)))
|
| 207 |
+
# The connection context commits before the caller receives the answer.
|
| 208 |
+
break
|
| 209 |
+
time.sleep(request_timing.remaining_budget(0.01))
|
| 210 |
+
# Deliver only committed activity. The durable intent survives a delivery outage.
|
| 211 |
+
from core import activity_log
|
| 212 |
+
try:
|
| 213 |
+
activity_log.drain(store)
|
| 214 |
+
except Exception as error:
|
| 215 |
+
print('[edit_receipts] activity delivery deferred: ' + type(error).__name__)
|
| 216 |
+
return deepcopy(result)
|
platform/core/external_tools.py
CHANGED
|
@@ -519,7 +519,7 @@ def _callback(ctx, name, args):
|
|
| 519 |
|
| 520 |
_DATABASE_ACTIONS = (
|
| 521 |
"list", "describe", "query", "record_create", "record_update", "record_delete",
|
| 522 |
-
"field_create", "field_update", "field_delete",
|
| 523 |
)
|
| 524 |
_VIEW_ACTIONS = ("upsert", "delete", "share_preview")
|
| 525 |
_SHEET_OBJECT_ACTIONS = {
|
|
@@ -670,10 +670,14 @@ TOOLS = {
|
|
| 670 |
additionalProperties=False),
|
| 671 |
lambda ctx, args: ctx.app_delete(args), write=True, grants=("ai_apps",)),
|
| 672 |
"database_object": _tool(
|
| 673 |
-
"Read a database or preview changes to its records
|
| 674 |
_object({
|
| 675 |
"action": {"enum": list(_DATABASE_ACTIONS)},
|
| 676 |
"database": _STRING,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 677 |
"record_id": {"oneOf": [_STRING, {"type": "integer"}]},
|
| 678 |
"values": {"type": "object"},
|
| 679 |
"field": {"type": "object"},
|
|
|
|
| 519 |
|
| 520 |
_DATABASE_ACTIONS = (
|
| 521 |
"list", "describe", "query", "record_create", "record_update", "record_delete",
|
| 522 |
+
"field_create", "field_update", "field_delete", "rename",
|
| 523 |
)
|
| 524 |
_VIEW_ACTIONS = ("upsert", "delete", "share_preview")
|
| 525 |
_SHEET_OBJECT_ACTIONS = {
|
|
|
|
| 670 |
additionalProperties=False),
|
| 671 |
lambda ctx, args: ctx.app_delete(args), write=True, grants=("ai_apps",)),
|
| 672 |
"database_object": _tool(
|
| 673 |
+
"Read a database or preview changes to its records, fields, name or icon, then confirm them. Name and icon changes require an Administrator.",
|
| 674 |
_object({
|
| 675 |
"action": {"enum": list(_DATABASE_ACTIONS)},
|
| 676 |
"database": _STRING,
|
| 677 |
+
"name": _STRING,
|
| 678 |
+
"icon": {"oneOf": [{"type": "null"}, _object({
|
| 679 |
+
"shape": _STRING, "tone": _STRING,
|
| 680 |
+
}, ("shape", "tone"), additionalProperties=False)]},
|
| 681 |
"record_id": {"oneOf": [_STRING, {"type": "integer"}]},
|
| 682 |
"values": {"type": "object"},
|
| 683 |
"field": {"type": "object"},
|