Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- Dockerfile +4 -2
- RELEASES.json +1 -1
- VERSION +1 -1
- api/amazon_relational.py +82 -17
- api/application_registry.py +2 -1
- api/assistant_mirror.py +41 -1
- api/automation_engine.py +185 -119
- api/connectors_amazon.py +3 -29
- api/deps.py +20 -8
- api/intelligence_store.py +1 -0
- api/loopy_bulk.py +8 -14
- api/main.py +44 -25
- api/odoo_relational.py +6 -1
- api/process_role.py +3 -8
- api/request_class.py +3 -5
- api/routes_amazon_events.py +180 -0
- api/routes_amazon_projects.py +318 -15
- api/routes_auth.py +2 -0
- api/routes_brand.py +23 -5
- api/routes_external.py +672 -77
- api/routes_feedback.py +12 -4
- api/routes_grid.py +80 -52
- api/routes_keychain.py +2 -0
- api/routes_odoo_tables.py +127 -12
- api/routes_products.py +16 -6
- api/routes_query.py +16 -1
- api/routes_shares.py +7 -5
- api/routes_sheet.py +77 -27
- api/routes_sku_actions.py +2 -2
- api/routes_tables.py +310 -157
- api/routes_usage.py +38 -1
- api/sync_worker.py +3 -0
- api/test_odoo_rank_refresh.py +177 -33
- api/tool_groups/__init__.py +4 -1
- api/tool_groups/data.py +11 -6
- api/tool_groups/metrics.py +7 -3
- api/usage_activity.py +491 -0
- platform/aios_grid.py +28 -1
- platform/core/amazon_project_tasks.py +725 -0
- platform/core/amazon_projects.py +336 -31
- platform/core/audit.py +1 -1
- platform/core/connector_spec.py +66 -1
- platform/core/external_tools.py +109 -4
- platform/core/filter_eval.py +827 -0
- platform/core/grid_events.py +802 -306
- platform/core/module_tables.py +17 -2
- platform/core/perm_scope.py +11 -10
- platform/core/store_pg.py +48 -4
- platform/core/table_store.py +446 -332
- platform/core/user_tables.py +240 -23
Dockerfile
CHANGED
|
@@ -126,8 +126,10 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
|
|
| 126 |
# S79-T37 (R25): the launcher sizes the web tier to the container. `request_class.py --env` reads
|
| 127 |
# the cgroup CPU and memory limits (never os.cpu_count(), which reports the host) and exports the
|
| 128 |
# uvicorn worker count, heavy concurrency and DuckDB caps. An env value set on the Space wins.
|
| 129 |
-
# Several workers
|
| 130 |
-
#
|
|
|
|
|
|
|
| 131 |
# Several workers load the app through a factory: uvicorn imports a plain app string in its
|
| 132 |
# supervisor too, and that process would run the boot threads without serving.
|
| 133 |
CMD ["sh", "-c", "eval \"$(python api/request_class.py --env)\"; W=1; if [ \"${AIOS_MIRROR_VERSIONS:-0}\" = 1 ]; then W=\"${AIOS_UVICORN_WORKERS:-1}\"; fi; export AIOS_UVICORN_WORKERS=\"$W\"; APP=api.main:app; if [ \"$W\" -gt 1 ]; then APP=\"--factory api.request_class:web_app\"; fi; exec uvicorn $APP --host 0.0.0.0 --port ${PORT} --workers \"$W\" --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"]
|
|
|
|
| 126 |
# S79-T37 (R25): the launcher sizes the web tier to the container. `request_class.py --env` reads
|
| 127 |
# the cgroup CPU and memory limits (never os.cpu_count(), which reports the host) and exports the
|
| 128 |
# uvicorn worker count, heavy concurrency and DuckDB caps. An env value set on the Space wins.
|
| 129 |
+
# Several workers require an explicit AIOS_MIRROR_VERSIONS=1 input and the web role.
|
| 130 |
+
# Without that opt-in, --env exports one worker and the shell keeps W at 1.
|
| 131 |
+
# Measure unloaded staging freshness and peak RSS before raising its worker count.
|
| 132 |
+
# On 2 vCPU, two workers also require AIOS_UVICORN_WORKERS=2.
|
| 133 |
# Several workers load the app through a factory: uvicorn imports a plain app string in its
|
| 134 |
# supervisor too, and that process would run the boot threads without serving.
|
| 135 |
CMD ["sh", "-c", "eval \"$(python api/request_class.py --env)\"; W=1; if [ \"${AIOS_MIRROR_VERSIONS:-0}\" = 1 ]; then W=\"${AIOS_UVICORN_WORKERS:-1}\"; fi; export AIOS_UVICORN_WORKERS=\"$W\"; APP=api.main:app; if [ \"$W\" -gt 1 ]; then APP=\"--factory api.request_class:web_app\"; fi; exec uvicorn $APP --host 0.0.0.0 --port ${PORT} --workers \"$W\" --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"]
|
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "0a741b9b3",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
0a741b9b3
|
api/amazon_relational.py
CHANGED
|
@@ -284,7 +284,8 @@ _MONEY_INFIX = ("_life_time_value_",)
|
|
| 284 |
#: multiplied by 100 in the read path (`grid_source`), so 0.15 reads "15%" and never "0.15%".
|
| 285 |
#: ⛔ A rate whose scale was not measured stays out of both sets (see `oos_rate_week_*`).
|
| 286 |
_PERCENT_SUFFIX = ("_percentage", "_percentage_b2_b")
|
| 287 |
-
_PERCENT = {"
|
|
|
|
| 288 |
"cart_add_data_asin_cart_add_share", "click_data_asin_click_share",
|
| 289 |
"impression_data_asin_impression_share", "purchase_data_asin_purchase_share",
|
| 290 |
"cart_add_data_total_cart_add_rate", "click_data_total_click_rate",
|
|
@@ -646,6 +647,8 @@ def _fields_for(cur, key, *, read_through=False):
|
|
| 646 |
"source": "overlay",
|
| 647 |
"default": name == pinned or name in shown or name in product or name == amazon_schema.STORE,
|
| 648 |
"pinned": name == pinned,
|
|
|
|
|
|
|
| 649 |
**({"format": fmt} if fmt else {}),
|
| 650 |
}, flow="amazon_relational"))
|
| 651 |
for lookup in derived.get(name, ()):
|
|
@@ -765,11 +768,53 @@ def restore_pending(rt=None):
|
|
| 765 |
return True
|
| 766 |
|
| 767 |
|
| 768 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 769 |
"""The ONE sentence every door gives for an Amazon table this server's mirror does not hold yet.
|
| 770 |
|
| 771 |
`pending` is a `restore_pending()` answer the caller already holds, so a listing reads the keychain once.
|
|
|
|
|
|
|
| 772 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 773 |
label = key.removeprefix(PREFIX).replace("_", " ")
|
| 774 |
if restore_pending(rt) if pending is None else pending:
|
| 775 |
return {"missing": f"Amazon {label} is still loading after a server restart, "
|
|
@@ -790,12 +835,15 @@ def _integer(value):
|
|
| 790 |
return None if value in (None, "") else int(value)
|
| 791 |
|
| 792 |
|
| 793 |
-
def grid_source(cur, key, *, columns=None, allow_lookups=True):
|
| 794 |
-
"""One binding from the current mirror schema, or an explicit unloaded-mirror reason.
|
|
|
|
|
|
|
|
|
|
| 795 |
columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns
|
| 796 |
columns = [(name, kind) for name, kind in columns if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS]
|
| 797 |
if not columns:
|
| 798 |
-
return missing_source(key)
|
| 799 |
names = {name for name, _dtype in columns}
|
| 800 |
table = key.removeprefix(PREFIX)
|
| 801 |
prepared = {row[0] for row in cur.execute(
|
|
@@ -886,6 +934,10 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True):
|
|
| 886 |
"needs_excluded": False, "cols": cols}
|
| 887 |
if table not in amazon_store.DERIVED_TABLES:
|
| 888 |
spec["page_key"] = f"{source}.rowid"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 889 |
return spec
|
| 890 |
|
| 891 |
|
|
@@ -894,12 +946,17 @@ def page_source(cur, key, definition=None, *, allow_lookups=True):
|
|
| 894 |
columns = mirror_columns(cur, SOURCE[key], strict=True)
|
| 895 |
mandatory = key in READ_THROUGH or bool((definition or {}).get("readThrough"))
|
| 896 |
if not columns:
|
| 897 |
-
return missing_source(key) if mandatory else None
|
|
|
|
| 898 |
if not mandatory:
|
| 899 |
count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 900 |
-
|
| 901 |
-
|
| 902 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 903 |
|
| 904 |
|
| 905 |
def grid_sources(cur=None):
|
|
@@ -913,7 +970,9 @@ def grid_sources(cur=None):
|
|
| 913 |
keys = set(READ_THROUGH)
|
| 914 |
for key in present - keys:
|
| 915 |
count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 916 |
-
|
|
|
|
|
|
|
| 917 |
keys.add(key)
|
| 918 |
return {key: grid_source(cur, key) for key in SOURCE if key in keys}
|
| 919 |
finally:
|
|
@@ -1144,7 +1203,9 @@ def plan_fingerprint(cur, built):
|
|
| 1144 |
canonical = sorted(rows, key=lambda row: json.dumps(
|
| 1145 |
row, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str))
|
| 1146 |
# S78-T02: an empty population differs from an absent table, so a stale copy is rewritten.
|
| 1147 |
-
|
|
|
|
|
|
|
| 1148 |
tables.append({"key": key, "fields": _fields_for(cur, key), "rows": canonical, **empty})
|
| 1149 |
payload = json.dumps(tables, sort_keys=True, ensure_ascii=True,
|
| 1150 |
separators=(",", ":"), default=str)
|
|
@@ -1377,6 +1438,9 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1377 |
# S79-T13: each Ads table's description states its history window and any recorded limit.
|
| 1378 |
ads_descriptions = {key: amazon_ads_store.table_description(SOURCE[key], cur_mirror)
|
| 1379 |
for key, *_rest in plans if key in _ADS_SPEC}
|
|
|
|
|
|
|
|
|
|
| 1380 |
for key, _label, _fields, rows in plans:
|
| 1381 |
if isinstance(rows, dict):
|
| 1382 |
continue
|
|
@@ -1513,17 +1577,18 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1513 |
saved = stored.get(row["_id"])
|
| 1514 |
if numeric and isinstance(saved, dict):
|
| 1515 |
saved.update({name: row[name] for name in numeric if name in row})
|
| 1516 |
-
|
| 1517 |
-
|
| 1518 |
-
|
| 1519 |
-
|
| 1520 |
-
|
| 1521 |
-
doc[key].pop("sourceStatus", None)
|
| 1522 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 1523 |
# mutable display label or from the process having imported this module.
|
| 1524 |
doc[key]["connector"] = source_attribution(key)
|
| 1525 |
if ads_descriptions.get(key):
|
| 1526 |
doc[key]["description"] = ads_descriptions[key]
|
|
|
|
|
|
|
| 1527 |
if key == PREFIX + "listing_items":
|
| 1528 |
doc[key]["amazonListingIdentity"] = listing_version
|
| 1529 |
return doc
|
|
|
|
| 284 |
#: multiplied by 100 in the read path (`grid_source`), so 0.15 reads "15%" and never "0.15%".
|
| 285 |
#: ⛔ A rate whose scale was not measured stays out of both sets (see `oos_rate_week_*`).
|
| 286 |
_PERCENT_SUFFIX = ("_percentage", "_percentage_b2_b")
|
| 287 |
+
_PERCENT = {"browse_node_metrics_occurrence_percentage_all_products",
|
| 288 |
+
"sales_by_asin_refund_rate", "sales_by_asin_refund_rate_b2_b",
|
| 289 |
"cart_add_data_asin_cart_add_share", "click_data_asin_click_share",
|
| 290 |
"impression_data_asin_impression_share", "purchase_data_asin_purchase_share",
|
| 291 |
"cart_add_data_total_cart_add_rate", "click_data_total_click_rate",
|
|
|
|
| 647 |
"source": "overlay",
|
| 648 |
"default": name == pinned or name in shown or name in product or name == amazon_schema.STORE,
|
| 649 |
"pinned": name == pinned,
|
| 650 |
+
**({"description": amazon_schema.REVIEW_UNAVAILABLE_NOTE}
|
| 651 |
+
if name == "asin" and key in (PREFIX + "review_topics", PREFIX + "review_trends") else {}),
|
| 652 |
**({"format": fmt} if fmt else {}),
|
| 653 |
}, flow="amazon_relational"))
|
| 654 |
for lookup in derived.get(name, ()):
|
|
|
|
| 768 |
return True
|
| 769 |
|
| 770 |
|
| 771 |
+
_UNRESOLVED = object()
|
| 772 |
+
|
| 773 |
+
|
| 774 |
+
def source_status(cur, key):
|
| 775 |
+
"""D-2643: `{empty, code, reason}` from `amazon_store.table_empty_status` for a present table.
|
| 776 |
+
|
| 777 |
+
An empty table always answers. A table with rows answers only beside a recorded Amazon
|
| 778 |
+
refusal, with `empty` False. None otherwise, and for keys outside the declared databases.
|
| 779 |
+
"""
|
| 780 |
+
table = key.removeprefix(PREFIX)
|
| 781 |
+
if key not in SOURCE or table not in amazon_store.DATABASE_TABLES:
|
| 782 |
+
return None
|
| 783 |
+
present = cur.execute(f"SELECT 1 FROM {_quoted(SOURCE[key])} LIMIT 1").fetchone() is not None
|
| 784 |
+
# `rows=1` marks a table that holds rows. The producer then answers a refusal only.
|
| 785 |
+
status = amazon_store.table_empty_status(cur, table, rows=int(present))
|
| 786 |
+
if not status.get("code") or (present and status["code"] != "refused_by_amazon"):
|
| 787 |
+
return None
|
| 788 |
+
return {"empty": not present, "code": status["code"], "reason": status.get("reason") or ""}
|
| 789 |
+
|
| 790 |
+
|
| 791 |
+
def missing_source(key, rt=None, *, pending=None, cur=None):
|
| 792 |
"""The ONE sentence every door gives for an Amazon table this server's mirror does not hold yet.
|
| 793 |
|
| 794 |
`pending` is a `restore_pending()` answer the caller already holds, so a listing reads the keychain once.
|
| 795 |
+
`cur` is the admitted mirror cursor. With it, a saved refusal or a measured empty report names
|
| 796 |
+
its own reason instead of the loading sentence.
|
| 797 |
"""
|
| 798 |
+
table = key.removeprefix(PREFIX)
|
| 799 |
+
# Schema-only probe first: a mirror without sync bookkeeping keeps the loading sentences.
|
| 800 |
+
if (cur is not None and key != PREFIX + "inventory"
|
| 801 |
+
and table in amazon_store.DATABASE_TABLES and mirror_columns(cur, "_sync_state")):
|
| 802 |
+
status = amazon_store.table_empty_status(cur, table, rows=None)
|
| 803 |
+
if (status.get("code") and status.get("reason")
|
| 804 |
+
and status["reason"] != amazon_store.loading_reason(table)):
|
| 805 |
+
return {"missing": status["reason"],
|
| 806 |
+
"sourceStatus": {"empty": True, "code": status["code"], "reason": status["reason"]}}
|
| 807 |
+
if key == PREFIX + "inventory":
|
| 808 |
+
if rt is None:
|
| 809 |
+
from harness import runtime
|
| 810 |
+
tenant = runtime.tenant_for_store_path()
|
| 811 |
+
rt = runtime.get_runtime(tenant) if tenant else None
|
| 812 |
+
definitions = rt.get_projection("user_tables", drop=("rows",)) if rt is not None else {}
|
| 813 |
+
status = ((definitions or {}).get(key) or {}).get("sourceStatus") or {}
|
| 814 |
+
if status.get("empty") is True and isinstance(status.get("reason"), str) and status["reason"].strip():
|
| 815 |
+
return {"missing": status["reason"]}
|
| 816 |
+
return {"missing": "Inventory request metadata is unavailable, "
|
| 817 |
+
"so retry Amazon sync to record the FBA ledger request."}
|
| 818 |
label = key.removeprefix(PREFIX).replace("_", " ")
|
| 819 |
if restore_pending(rt) if pending is None else pending:
|
| 820 |
return {"missing": f"Amazon {label} is still loading after a server restart, "
|
|
|
|
| 835 |
return None if value in (None, "") else int(value)
|
| 836 |
|
| 837 |
|
| 838 |
+
def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVED):
|
| 839 |
+
"""One binding from the current mirror schema, or an explicit unloaded-mirror reason.
|
| 840 |
+
|
| 841 |
+
`status` is a `source_status` answer the caller already holds.
|
| 842 |
+
"""
|
| 843 |
columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns
|
| 844 |
columns = [(name, kind) for name, kind in columns if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS]
|
| 845 |
if not columns:
|
| 846 |
+
return missing_source(key, cur=cur)
|
| 847 |
names = {name for name, _dtype in columns}
|
| 848 |
table = key.removeprefix(PREFIX)
|
| 849 |
prepared = {row[0] for row in cur.execute(
|
|
|
|
| 934 |
"needs_excluded": False, "cols": cols}
|
| 935 |
if table not in amazon_store.DERIVED_TABLES:
|
| 936 |
spec["page_key"] = f"{source}.rowid"
|
| 937 |
+
# D-2643: an empty table carries its code and reason to every door that reads this binding.
|
| 938 |
+
status = source_status(cur, key) if status is _UNRESOLVED else status
|
| 939 |
+
if status is not None:
|
| 940 |
+
spec["sourceStatus"] = status
|
| 941 |
return spec
|
| 942 |
|
| 943 |
|
|
|
|
| 946 |
columns = mirror_columns(cur, SOURCE[key], strict=True)
|
| 947 |
mandatory = key in READ_THROUGH or bool((definition or {}).get("readThrough"))
|
| 948 |
if not columns:
|
| 949 |
+
return missing_source(key, cur=cur) if mandatory else None
|
| 950 |
+
status = _UNRESOLVED
|
| 951 |
if not mandatory:
|
| 952 |
count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 953 |
+
# D-2643: an empty table, or one beside a recorded refusal, reads through, so its
|
| 954 |
+
# binding names its reason in the Data grid.
|
| 955 |
+
if count and not _read_through(cur, key, count, columns=columns):
|
| 956 |
+
status = source_status(cur, key)
|
| 957 |
+
if status is None:
|
| 958 |
+
return None
|
| 959 |
+
return grid_source(cur, key, columns=columns, allow_lookups=allow_lookups, status=status)
|
| 960 |
|
| 961 |
|
| 962 |
def grid_sources(cur=None):
|
|
|
|
| 970 |
keys = set(READ_THROUGH)
|
| 971 |
for key in present - keys:
|
| 972 |
count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 973 |
+
# D-2643: an empty or refused table registers too, so the grid opens it on the
|
| 974 |
+
# read-through door that carries its reason.
|
| 975 |
+
if _read_through(cur, key, count) or source_status(cur, key) is not None:
|
| 976 |
keys.add(key)
|
| 977 |
return {key: grid_source(cur, key) for key in SOURCE if key in keys}
|
| 978 |
finally:
|
|
|
|
| 1203 |
canonical = sorted(rows, key=lambda row: json.dumps(
|
| 1204 |
row, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str))
|
| 1205 |
# S78-T02: an empty population differs from an absent table, so a stale copy is rewritten.
|
| 1206 |
+
# D-2643: a changed empty reason is a changed publication too.
|
| 1207 |
+
empty = ({"empty": True, "sourceStatus": source_status(cur, key)
|
| 1208 |
+
if mirror_columns(cur, SOURCE[key]) else None} if built.get(key) == [] else {})
|
| 1209 |
tables.append({"key": key, "fields": _fields_for(cur, key), "rows": canonical, **empty})
|
| 1210 |
payload = json.dumps(tables, sort_keys=True, ensure_ascii=True,
|
| 1211 |
separators=(",", ":"), default=str)
|
|
|
|
| 1438 |
# S79-T13: each Ads table's description states its history window and any recorded limit.
|
| 1439 |
ads_descriptions = {key: amazon_ads_store.table_description(SOURCE[key], cur_mirror)
|
| 1440 |
for key, *_rest in plans if key in _ADS_SPEC}
|
| 1441 |
+
# D-2643: an emptied stored table keeps the code and reason its mirror answers with.
|
| 1442 |
+
empty_status = {key: source_status(cur_mirror, key) for key, _label, _fields, rows in plans
|
| 1443 |
+
if isinstance(rows, list) and not rows}
|
| 1444 |
for key, _label, _fields, rows in plans:
|
| 1445 |
if isinstance(rows, dict):
|
| 1446 |
continue
|
|
|
|
| 1577 |
saved = stored.get(row["_id"])
|
| 1578 |
if numeric and isinstance(saved, dict):
|
| 1579 |
saved.update({name: row[name] for name in numeric if name in row})
|
| 1580 |
+
status = rows.get("sourceStatus") if isinstance(rows, dict) else empty_status.get(key)
|
| 1581 |
+
if status is not None:
|
| 1582 |
+
doc[key]["sourceStatus"] = dict(status)
|
| 1583 |
+
else:
|
| 1584 |
+
doc[key].pop("sourceStatus", None)
|
|
|
|
| 1585 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 1586 |
# mutable display label or from the process having imported this module.
|
| 1587 |
doc[key]["connector"] = source_attribution(key)
|
| 1588 |
if ads_descriptions.get(key):
|
| 1589 |
doc[key]["description"] = ads_descriptions[key]
|
| 1590 |
+
if key in (PREFIX + "review_topics", PREFIX + "review_trends"):
|
| 1591 |
+
doc[key]["description"] = amazon_schema.REVIEW_UNAVAILABLE_NOTE
|
| 1592 |
if key == PREFIX + "listing_items":
|
| 1593 |
doc[key]["amazonListingIdentity"] = listing_version
|
| 1594 |
return doc
|
api/application_registry.py
CHANGED
|
@@ -1418,9 +1418,10 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 1418 |
def _nav_placement(app, index):
|
| 1419 |
"""Put default Project management entries first without rewriting stored placements."""
|
| 1420 |
placement = app.get("navPlacement")
|
|
|
|
|
|
|
| 1421 |
if (app.get("templateKind") == "amazon_profit"
|
| 1422 |
and (not placement or placement == {"section": "app", "order": 0})):
|
| 1423 |
-
# A zero placement already asks for first place. Break its default tie in the rail.
|
| 1424 |
return {"section": "app", "order": -1}
|
| 1425 |
return placement or {"section": "app", "order": index}
|
| 1426 |
|
|
|
|
| 1418 |
def _nav_placement(app, index):
|
| 1419 |
"""Put default Project management entries first without rewriting stored placements."""
|
| 1420 |
placement = app.get("navPlacement")
|
| 1421 |
+
# Every install stores the template default {"section": "app", "order": 0}, so order 0 is the
|
| 1422 |
+
# default here, not a choice; any other stored placement wins.
|
| 1423 |
if (app.get("templateKind") == "amazon_profit"
|
| 1424 |
and (not placement or placement == {"section": "app", "order": 0})):
|
|
|
|
| 1425 |
return {"section": "app", "order": -1}
|
| 1426 |
return placement or {"section": "app", "order": index}
|
| 1427 |
|
api/assistant_mirror.py
CHANGED
|
@@ -166,6 +166,14 @@ class ReadinessSnapshot:
|
|
| 166 |
return self._loading[key]
|
| 167 |
|
| 168 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
def unavailable_reason(table_key, definition=None, *, snapshot=None):
|
| 170 |
"""The ONE sentence `list_databases` and `query_database` both give when the mirror holds no rows.
|
| 171 |
|
|
@@ -177,6 +185,11 @@ def unavailable_reason(table_key, definition=None, *, snapshot=None):
|
|
| 177 |
if key.startswith("ut_amzn_"):
|
| 178 |
import amazon_relational as amazon
|
| 179 |
if key in amazon.SOURCE:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 180 |
pending = snapshot.amazon_pending() if snapshot is not None else None
|
| 181 |
return amazon.missing_source(key, pending=pending)["missing"]
|
| 182 |
spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key)
|
|
@@ -222,11 +235,38 @@ def binding(cur, table_key):
|
|
| 222 |
which is not the same as the grid being empty.
|
| 223 |
"""
|
| 224 |
try:
|
| 225 |
-
|
| 226 |
except Exception: # noqa: BLE001
|
| 227 |
# A spec that cannot be resolved is indistinguishable, to every caller, from a grid with
|
| 228 |
# no binding: neither can be read. Fail closed to the reportable answer.
|
| 229 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
|
| 231 |
|
| 232 |
def is_bound(cur, table_key, *, snapshot=None):
|
|
|
|
| 166 |
return self._loading[key]
|
| 167 |
|
| 168 |
|
| 169 |
+
def amazon_empty_reason(status):
|
| 170 |
+
"""The one-line reason in an Amazon `sourceStatus`, or "" when it names no code and reason."""
|
| 171 |
+
if not isinstance(status, dict) or not status.get("empty") or not status.get("code"):
|
| 172 |
+
return ""
|
| 173 |
+
reason = status.get("reason")
|
| 174 |
+
return " ".join(reason.split()) if isinstance(reason, str) else ""
|
| 175 |
+
|
| 176 |
+
|
| 177 |
def unavailable_reason(table_key, definition=None, *, snapshot=None):
|
| 178 |
"""The ONE sentence `list_databases` and `query_database` both give when the mirror holds no rows.
|
| 179 |
|
|
|
|
| 185 |
if key.startswith("ut_amzn_"):
|
| 186 |
import amazon_relational as amazon
|
| 187 |
if key in amazon.SOURCE:
|
| 188 |
+
# D-2643: the stored definition's empty reason, from the one Amazon producer.
|
| 189 |
+
status = (definition or {}).get("sourceStatus") if isinstance(definition, dict) else None
|
| 190 |
+
reason = amazon_empty_reason(status)
|
| 191 |
+
if reason:
|
| 192 |
+
return reason
|
| 193 |
pending = snapshot.amazon_pending() if snapshot is not None else None
|
| 194 |
return amazon.missing_source(key, pending=pending)["missing"]
|
| 195 |
spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key)
|
|
|
|
| 235 |
which is not the same as the grid being empty.
|
| 236 |
"""
|
| 237 |
try:
|
| 238 |
+
spec = _rot()._source_for(cur, str(table_key or ""))
|
| 239 |
except Exception: # noqa: BLE001
|
| 240 |
# A spec that cannot be resolved is indistinguishable, to every caller, from a grid with
|
| 241 |
# no binding: neither can be read. Fail closed to the reportable answer.
|
| 242 |
return None
|
| 243 |
+
return amazon_lag(cur, str(table_key or ""), spec)
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def amazon_lag(cur, table_key, spec):
|
| 247 |
+
"""D-2642: a dated Amazon binding with rows also carries its newest row date, cadence and lag line.
|
| 248 |
+
|
| 249 |
+
The keys come from `amazon_store.table_empty_status`, the one per-table producer. A trailing table
|
| 250 |
+
also gets a `source_behind` limit. A status that cannot be read carries the unknown lag line.
|
| 251 |
+
"""
|
| 252 |
+
if not spec or spec.get("missing") or not table_key.startswith("ut_amzn_"):
|
| 253 |
+
return spec
|
| 254 |
+
from harness import amazon_store
|
| 255 |
+
table = table_key.removeprefix("ut_amzn_")
|
| 256 |
+
if table not in amazon_store.LAG_DATE_COLUMNS:
|
| 257 |
+
return spec
|
| 258 |
+
try:
|
| 259 |
+
if cur.execute(f'SELECT 1 FROM "{amazon_store.table_name(table)}" LIMIT 1').fetchone() is None:
|
| 260 |
+
return spec
|
| 261 |
+
status = amazon_store.table_empty_status(cur, table, rows=1, lag=True)
|
| 262 |
+
dated = {name: status[name] for name in amazon_store.LAG_KEYS if name in status}
|
| 263 |
+
except Exception: # noqa: BLE001
|
| 264 |
+
dated = amazon_store.unknown_lag(table)
|
| 265 |
+
spec = {**spec, "sourceStatus": {"empty": False, **(spec.get("sourceStatus") or {}), **dated}}
|
| 266 |
+
if dated.get("lag"):
|
| 267 |
+
spec["limits"] = [*(spec.get("limits") or []),
|
| 268 |
+
{"effect": "source_behind", "code": dated.get("lagCode"), "cause": dated["lag"]}]
|
| 269 |
+
return spec
|
| 270 |
|
| 271 |
|
| 272 |
def is_bound(cur, table_key, *, snapshot=None):
|
api/automation_engine.py
CHANGED
|
@@ -1111,7 +1111,7 @@ def retire_automation_stage_fields(rt, tables=None):
|
|
| 1111 |
changed["tables"].add(str(table_key))
|
| 1112 |
return cur
|
| 1113 |
|
| 1114 |
-
|
| 1115 |
return {"tables": len(changed["tables"]), "fields": changed["fields"],
|
| 1116 |
"cells": changed["cells"]}
|
| 1117 |
|
|
@@ -1155,7 +1155,9 @@ def bind_unbound_fields(rt):
|
|
| 1155 |
"or delete it")
|
| 1156 |
return cur
|
| 1157 |
|
| 1158 |
-
|
|
|
|
|
|
|
| 1159 |
bound = sum(1 for v in plan.values() if v)
|
| 1160 |
return bound, len(plan) - bound
|
| 1161 |
|
|
@@ -1341,7 +1343,10 @@ def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="",
|
|
| 1341 |
t["recordMode"] = record_mode
|
| 1342 |
return cur
|
| 1343 |
|
| 1344 |
-
|
|
|
|
|
|
|
|
|
|
| 1345 |
return key
|
| 1346 |
|
| 1347 |
|
|
@@ -1353,7 +1358,7 @@ def ut_write_rows(rt, key, rows):
|
|
| 1353 |
if t is not None:
|
| 1354 |
t["rows"] = rows
|
| 1355 |
return cur
|
| 1356 |
-
|
| 1357 |
|
| 1358 |
|
| 1359 |
def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None):
|
|
@@ -1368,7 +1373,7 @@ def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None):
|
|
| 1368 |
if tt is not None:
|
| 1369 |
tt["rows"] = rws
|
| 1370 |
return cur
|
| 1371 |
-
|
| 1372 |
|
| 1373 |
|
| 1374 |
IG_FIELD_DESCRIPTIONS = {
|
|
@@ -3746,6 +3751,11 @@ def create(rt, raw, username="", notes=None):
|
|
| 3746 |
# against a PREVIOUS definition, and a create has none — R10's columns must exist before a run,
|
| 3747 |
# so this call stays unconditional.
|
| 3748 |
_store_update(rt, _up, flush="async")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3749 |
_presets_after_write(rt, defn, username) # R10: the columns exist before a run
|
| 3750 |
if defn.get("trigger"):
|
| 3751 |
_seed_event_state(rt, defn)
|
|
@@ -4104,6 +4114,23 @@ def remove(rt, auto_id):
|
|
| 4104 |
# ⚠ THOSE NUMBERS WERE UNOBSERVABLE UNTIL W35-T34: `verify_automation`'s delete-speed double
|
| 4105 |
# had no `get_projection`, so `all_defs` fell back and the gate measured 2 and 1 — the
|
| 4106 |
# PRE-WAVE-33 path — for two waves. The double is producer-faithful now and asserts the counts.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4107 |
try:
|
| 4108 |
import core.user_tables as _ut_defs # noqa: PLC0415
|
| 4109 |
_scan = dict(_ut_defs.all_defs(rt) or {})
|
|
@@ -4130,7 +4157,7 @@ def remove(rt, auto_id):
|
|
| 4130 |
# person waits. Async for the reason given above — the disable is visible to the next read
|
| 4131 |
# immediately; only the upload is deferred, and it coalesces with any other pending write to
|
| 4132 |
# the same key instead of racing it.
|
| 4133 |
-
|
| 4134 |
|
| 4135 |
|
| 4136 |
# ---------------------------------------------------------------------------------------------
|
|
@@ -5926,7 +5953,7 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
|
|
| 5926 |
if needs:
|
| 5927 |
break
|
| 5928 |
if needs:
|
| 5929 |
-
|
| 5930 |
return changed
|
| 5931 |
|
| 5932 |
|
|
@@ -6413,7 +6440,7 @@ def retract_foreign_presets(rt, log=print):
|
|
| 6413 |
for k in stats["kept"]))
|
| 6414 |
return stats
|
| 6415 |
stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []}
|
| 6416 |
-
|
| 6417 |
log(f"[aios-auto] retract: {stats['columns']} foreign column(s), {stats['cells']} cell(s) and "
|
| 6418 |
f"{stats['flags']} mis-declared flag(s) across {stats['tables']} database(s)")
|
| 6419 |
if stats["kept"]:
|
|
@@ -6796,7 +6823,7 @@ def backfill_post_snapshot_grain(rt, log=print):
|
|
| 6796 |
table.setdefault("rows", {}).setdefault(rid, {}).update(values)
|
| 6797 |
return cur
|
| 6798 |
|
| 6799 |
-
|
| 6800 |
log(f"[ig-migrate] post-measurement grain: {len(changes)} row(s) gained a post date/kind")
|
| 6801 |
return len(changes)
|
| 6802 |
|
|
@@ -6921,6 +6948,110 @@ def _as_derived_field(field):
|
|
| 6921 |
"type": "rollup", "rollup": _derived_profile_bag(key)}
|
| 6922 |
|
| 6923 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6924 |
def migrate_ig_tables(rt, log=print, only=""):
|
| 6925 |
"""Bring every Instagram Profile and canonical child table onto one current contract.
|
| 6926 |
|
|
@@ -6986,115 +7117,36 @@ def migrate_ig_tables(rt, log=print, only=""):
|
|
| 6986 |
if only and not targets:
|
| 6987 |
return stats
|
| 6988 |
for table_key in targets:
|
|
|
|
|
|
|
| 6989 |
tbl = ut_get(rt, table_key) or {}
|
| 6990 |
-
|
| 6991 |
-
rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()}
|
| 6992 |
-
# ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched,
|
| 6993 |
-
# so the cell conversions below can never run twice on the same value.
|
| 6994 |
-
stale = [f for f in fields
|
| 6995 |
-
if f.get("key") in want and f.get("type") != want[f["key"]]
|
| 6996 |
-
and f.get("type") in ("text", "", None)]
|
| 6997 |
-
stale_keys = {f["key"] for f in stale}
|
| 6998 |
-
has_platform = any(f.get("key") == "platform" for f in fields)
|
| 6999 |
-
needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()]
|
| 7000 |
-
# ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING.
|
| 7001 |
-
# `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the
|
| 7002 |
-
# two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in
|
| 7003 |
-
# production needs them stamped here, and that is the whole reason this branch exists.
|
| 7004 |
-
#
|
| 7005 |
-
# ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule,
|
| 7006 |
-
# and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which
|
| 7007 |
-
# is exactly why these are safe to run on every write where a value conversion would not be.
|
| 7008 |
-
pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \
|
| 7009 |
-
if PRESET_PINNED_KEY else None
|
| 7010 |
-
flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \
|
| 7011 |
-
if PRESET_FLAG_KEY else None
|
| 7012 |
-
# ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write
|
| 7013 |
-
# doors, and a migration is not exempt from it. If somebody has already flagged another
|
| 7014 |
-
# column on this table, stamping the handle too would leave the table in a state its own
|
| 7015 |
-
# validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would
|
| 7016 |
-
# silently bind to whichever came first in the field list. Leave it alone and pin only.
|
| 7017 |
-
other_profile = next((f for f in fields if f is not flag_f
|
| 7018 |
-
and isinstance(f.get("profile"), dict)), None)
|
| 7019 |
-
want_pin = pin_f is not None and pin_f.get("pinned") is not True
|
| 7020 |
-
want_flag = (flag_f is not None and other_profile is None
|
| 7021 |
-
and not isinstance(flag_f.get("profile"), dict))
|
| 7022 |
-
vestigial = _vestigial_name_field(fields, rows)
|
| 7023 |
-
retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"}
|
| 7024 |
-
if _retired_tracked_field(fields) is not None:
|
| 7025 |
-
retired.add(TRACKED_KEY)
|
| 7026 |
-
orphan_tracked = _orphan_tracked_rows(fields, rows)
|
| 7027 |
-
if (not stale_keys and has_platform and not needs_stamp
|
| 7028 |
-
and not want_pin and not want_flag and vestigial is None and not retired
|
| 7029 |
-
and not orphan_tracked):
|
| 7030 |
continue
|
| 7031 |
-
|
| 7032 |
-
|
| 7033 |
-
|
| 7034 |
-
stats["pinned"] += 1
|
| 7035 |
-
if want_flag:
|
| 7036 |
-
flag_f["profile"] = {"source": PROFILE_SOURCE_IG}
|
| 7037 |
-
stats["flagged"] += 1
|
| 7038 |
-
if vestigial is not None:
|
| 7039 |
-
# ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer
|
| 7040 |
-
# exists is invisible everywhere except the next export, where it reappears as a column
|
| 7041 |
-
# nobody declared. `_vestigial_name_field` has already proven every one of them blank.
|
| 7042 |
-
fields = [f for f in fields if f is not vestigial]
|
| 7043 |
-
for r in rows.values():
|
| 7044 |
-
r.pop("name", None)
|
| 7045 |
-
stats["droppedName"] += 1
|
| 7046 |
-
if retired:
|
| 7047 |
-
fields = [f for f in fields if f.get("key") not in retired]
|
| 7048 |
-
for r in rows.values():
|
| 7049 |
-
for key in retired:
|
| 7050 |
-
r.pop(key, None)
|
| 7051 |
-
stats["droppedRecentPosts"] += int("posts" in retired)
|
| 7052 |
-
stats["droppedPostHashtags"] += int("post_hashtags" in retired)
|
| 7053 |
-
stats["droppedTracked"] += int(TRACKED_KEY in retired)
|
| 7054 |
-
for rid in orphan_tracked:
|
| 7055 |
-
# D-74: a tick whose column was already gone. Counted apart from the column drop
|
| 7056 |
-
# because they answer different questions — "how many tables still declared it" and
|
| 7057 |
-
# "how many rows were still carrying it after the declaration went".
|
| 7058 |
-
rows[rid].pop(TRACKED_KEY, None)
|
| 7059 |
-
stats["droppedTrackedCells"] += 1
|
| 7060 |
-
|
| 7061 |
-
for key in stale_keys:
|
| 7062 |
-
conv = _MIGRATE_CONVERT.get(key)
|
| 7063 |
-
if not conv:
|
| 7064 |
-
continue # int/checkbox already store the right text shape
|
| 7065 |
-
for r in rows.values():
|
| 7066 |
-
if str(r.get(key) or "").strip():
|
| 7067 |
-
new = conv(r[key])
|
| 7068 |
-
# "" from a converter means UNREADABLE, and the cell keeps its original text.
|
| 7069 |
-
# A migration that empties a cell is indistinguishable from one that moved it.
|
| 7070 |
-
if new:
|
| 7071 |
-
r[key], stats["converted"] = new, stats["converted"] + 1
|
| 7072 |
-
for f in stale:
|
| 7073 |
-
f["type"] = want[f["key"]]
|
| 7074 |
-
stats["retyped"] += 1
|
| 7075 |
-
for rid in needs_stamp:
|
| 7076 |
-
rows[rid]["platform"] = PLATFORM_INSTAGRAM
|
| 7077 |
-
stats["stamped"] += 1
|
| 7078 |
-
if not has_platform:
|
| 7079 |
-
fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields
|
| 7080 |
-
rows, merged = _merge_candidates(rows)
|
| 7081 |
-
stats["merged"] += merged
|
| 7082 |
-
|
| 7083 |
-
field_changes = {}
|
| 7084 |
-
|
| 7085 |
-
def _up(cur, _f=fields, _r=rows):
|
| 7086 |
field_changes.clear()
|
|
|
|
| 7087 |
cur = cur or {}
|
| 7088 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7089 |
old_keys = _ut()._field_list_keys(t)
|
| 7090 |
-
t["fields"], t["rows"] =
|
| 7091 |
new_keys = _ut()._field_list_keys(t)
|
| 7092 |
field_changes.update(removed=[key for key in old_keys if key not in new_keys],
|
| 7093 |
added=[key for key in new_keys if key not in old_keys])
|
|
|
|
| 7094 |
cur[table_key] = t
|
| 7095 |
return cur
|
| 7096 |
|
| 7097 |
rt.update(UT_STORE_KEY, _up)
|
|
|
|
|
|
|
|
|
|
| 7098 |
_ut()._log_field_list_change(rt, table_key, 'automation', 'migrate_ig_tables', field_changes)
|
| 7099 |
# ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION.
|
| 7100 |
#
|
|
@@ -7750,7 +7802,7 @@ def append_ig_snapshots(rt, incoming, snap_key=IG_SNAPSHOTS_TABLE, log=print):
|
|
| 7750 |
cur[snap_key]["rows"] = merged
|
| 7751 |
return cur
|
| 7752 |
|
| 7753 |
-
|
| 7754 |
return written
|
| 7755 |
|
| 7756 |
|
|
@@ -8401,7 +8453,7 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 8401 |
_refresh_relations_inplace(cur, log=log)
|
| 8402 |
return cur
|
| 8403 |
|
| 8404 |
-
|
| 8405 |
queued = queue_pending_metric_snapshots(
|
| 8406 |
rt, str(defn.get("id") or ""), pending_metrics,
|
| 8407 |
workflow_id=str(defn.get("workflowId") or ""))
|
|
@@ -9153,8 +9205,7 @@ def assemble_statements(defn, log=print):
|
|
| 9153 |
if left > 0:
|
| 9154 |
# R6's second sentence: a limit that bites is REPORTED, with its cause and what to do.
|
| 9155 |
notes.append(f"{left} more customers matched but one batch holds "
|
| 9156 |
-
f"{MAX_STATEMENT_BATCH}. Send this batch, then run it again for the rest
|
| 9157 |
-
f"narrow the tier filter")
|
| 9158 |
return batch, notes
|
| 9159 |
|
| 9160 |
|
|
@@ -12805,7 +12856,7 @@ def _enrich_flush(rt, defn, username, acc, log):
|
|
| 12805 |
_refresh_relations_inplace(cur, log=log)
|
| 12806 |
return cur
|
| 12807 |
|
| 12808 |
-
|
| 12809 |
# LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which
|
| 12810 |
# is the failure a chart cannot show you.
|
| 12811 |
capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"]
|
|
@@ -12942,7 +12993,7 @@ def _tt_write_tables(rt, tag, username, snaps, posts, psnaps, comments, log=prin
|
|
| 12942 |
return cur
|
| 12943 |
|
| 12944 |
if written:
|
| 12945 |
-
|
| 12946 |
return inserted, capped, missing
|
| 12947 |
|
| 12948 |
|
|
@@ -13518,7 +13569,7 @@ def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments
|
|
| 13518 |
_refresh_relations_inplace(cur, log=log)
|
| 13519 |
return cur
|
| 13520 |
|
| 13521 |
-
|
| 13522 |
try:
|
| 13523 |
import ig_master
|
| 13524 |
status, note = ig_master.append_run(getattr(rt, "key", ""), [], idents, snapshots)
|
|
@@ -14084,9 +14135,9 @@ def compute_metric_cells(rt, table_key, today=None, tables=None, persist=False):
|
|
| 14084 |
if owned:
|
| 14085 |
_up(blob)
|
| 14086 |
if persist:
|
| 14087 |
-
|
| 14088 |
return len(changes)
|
| 14089 |
-
|
| 14090 |
return len(changes)
|
| 14091 |
|
| 14092 |
|
|
@@ -14108,6 +14159,17 @@ def compute_metric_cells(rt, table_key, today=None, tables=None, persist=False):
|
|
| 14108 |
# posts to fit, precisely because the store still holds them). ⛔ A rollup may only ever READ. The
|
| 14109 |
# moment one writes a number nothing else can re-derive, it has become a third copy.
|
| 14110 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14111 |
def _ut():
|
| 14112 |
"""`core.user_tables`, imported LAZILY — and the laziness is measured, not stylistic.
|
| 14113 |
|
|
@@ -15549,7 +15611,7 @@ def compute_relation_cells(rt, table_key, tables=None):
|
|
| 15549 |
tt.setdefault("rows", {}).setdefault(r, {}).update(vals)
|
| 15550 |
return cur
|
| 15551 |
|
| 15552 |
-
|
| 15553 |
return len(changes)
|
| 15554 |
|
| 15555 |
|
|
@@ -15702,7 +15764,7 @@ def refresh_relations(rt, log=print, tables=None, persist=True, *, target=None):
|
|
| 15702 |
return cur
|
| 15703 |
|
| 15704 |
# A cold process must see recovered cells before their old refusal disappears.
|
| 15705 |
-
saved =
|
| 15706 |
if saved is False:
|
| 15707 |
raise RuntimeError('The relation refresh was not committed.')
|
| 15708 |
_persist_relation_limits(rt, results)
|
|
@@ -15771,7 +15833,7 @@ def purge_subject(rt, handle):
|
|
| 15771 |
t["rows"] = nxt
|
| 15772 |
return cur
|
| 15773 |
|
| 15774 |
-
|
| 15775 |
import ig_master
|
| 15776 |
for bucket, n in (ig_master.purge_handle(subject) or {}).items():
|
| 15777 |
counts[f"master:{bucket}"] = n
|
|
@@ -16097,7 +16159,11 @@ def clean_trigger(raw, previous=None):
|
|
| 16097 |
return None, str(exc)
|
| 16098 |
if key in TRIGGER_TABLE_KEYS:
|
| 16099 |
table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip()
|
| 16100 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16101 |
return None, ("event triggers watch blank databases (ut_*) this wave. "
|
| 16102 |
f"{table!r} is not one")
|
| 16103 |
out["table"] = table
|
|
|
|
| 1111 |
changed["tables"].add(str(table_key))
|
| 1112 |
return cur
|
| 1113 |
|
| 1114 |
+
_ut_update(rt, _up, flush="sync")
|
| 1115 |
return {"tables": len(changed["tables"]), "fields": changed["fields"],
|
| 1116 |
"cells": changed["cells"]}
|
| 1117 |
|
|
|
|
| 1155 |
"or delete it")
|
| 1156 |
return cur
|
| 1157 |
|
| 1158 |
+
for _aid in sorted({a for a in plan.values() if a}):
|
| 1159 |
+
_ut().index_flow_tables(_aid, sorted({tk for (tk, _fk), a in plan.items() if a == _aid}), rt)
|
| 1160 |
+
_ut_update(rt, _up, flush="sync")
|
| 1161 |
bound = sum(1 for v in plan.values() if v)
|
| 1162 |
return bound, len(plan) - bound
|
| 1163 |
|
|
|
|
| 1343 |
t["recordMode"] = record_mode
|
| 1344 |
return cur
|
| 1345 |
|
| 1346 |
+
if flow_tag:
|
| 1347 |
+
# D-179: index first, so a failed stamp leaves a superset and never a missed table.
|
| 1348 |
+
_ut().index_flow_tables(flow_tag, (key,), rt)
|
| 1349 |
+
_ut_update(rt, _up, flush="sync")
|
| 1350 |
return key
|
| 1351 |
|
| 1352 |
|
|
|
|
| 1358 |
if t is not None:
|
| 1359 |
t["rows"] = rows
|
| 1360 |
return cur
|
| 1361 |
+
_ut_update(rt, _up, flush="sync")
|
| 1362 |
|
| 1363 |
|
| 1364 |
def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None):
|
|
|
|
| 1373 |
if tt is not None:
|
| 1374 |
tt["rows"] = rws
|
| 1375 |
return cur
|
| 1376 |
+
_ut_update(rt, _up, flush="sync")
|
| 1377 |
|
| 1378 |
|
| 1379 |
IG_FIELD_DESCRIPTIONS = {
|
|
|
|
| 3751 |
# against a PREVIOUS definition, and a create has none — R10's columns must exist before a run,
|
| 3752 |
# so this call stays unconditional.
|
| 3753 |
_store_update(rt, _up, flush="async")
|
| 3754 |
+
# D-179: a new flow has tagged nothing, so its flow-index entry starts complete and empty.
|
| 3755 |
+
try:
|
| 3756 |
+
_ut().index_flow_tables(defn["id"], (), rt, create=True)
|
| 3757 |
+
except Exception as e: # noqa: BLE001
|
| 3758 |
+
print(f"[aios-auto] D-179 flow index entry not started: {type(e).__name__}")
|
| 3759 |
_presets_after_write(rt, defn, username) # R10: the columns exist before a run
|
| 3760 |
if defn.get("trigger"):
|
| 3761 |
_seed_event_state(rt, defn)
|
|
|
|
| 4114 |
# ⚠ THOSE NUMBERS WERE UNOBSERVABLE UNTIL W35-T34: `verify_automation`'s delete-speed double
|
| 4115 |
# had no `get_projection`, so `all_defs` fell back and the gate measured 2 and 1 — the
|
| 4116 |
# PRE-WAVE-33 path — for two waves. The double is producer-faithful now and asserts the counts.
|
| 4117 |
+
# S85-T173 (D-179): a flow with an index entry skips the scan and patches only its tables,
|
| 4118 |
+
# through a write that declares them (`user_tables.update_tables`). A flow with no entry, one
|
| 4119 |
+
# that tagged before the index existed, keeps the scan and the whole write below.
|
| 4120 |
+
_indexed = _ut().flow_index_tables(aid, rt)
|
| 4121 |
+
if _indexed is not None:
|
| 4122 |
+
if not _indexed:
|
| 4123 |
+
return
|
| 4124 |
+
_note = f"its automation was deleted {_stamp(_authored_now())}"
|
| 4125 |
+
|
| 4126 |
+
def _disable_table(_tk, table):
|
| 4127 |
+
for f in (table.get("fields") or []):
|
| 4128 |
+
a = f.get("automation") if isinstance(f, dict) else None
|
| 4129 |
+
if isinstance(a, dict) and str(a.get("flowId") or "") == aid:
|
| 4130 |
+
a["disabled"] = True
|
| 4131 |
+
a["statusNote"] = _note
|
| 4132 |
+
_ut().update_tables(_indexed, _disable_table, rt, flush="async")
|
| 4133 |
+
return
|
| 4134 |
try:
|
| 4135 |
import core.user_tables as _ut_defs # noqa: PLC0415
|
| 4136 |
_scan = dict(_ut_defs.all_defs(rt) or {})
|
|
|
|
| 4157 |
# person waits. Async for the reason given above — the disable is visible to the next read
|
| 4158 |
# immediately; only the upload is deferred, and it coalesces with any other pending write to
|
| 4159 |
# the same key instead of racing it.
|
| 4160 |
+
_ut_update(rt, _disable, flush="async")
|
| 4161 |
|
| 4162 |
|
| 4163 |
# ---------------------------------------------------------------------------------------------
|
|
|
|
| 5953 |
if needs:
|
| 5954 |
break
|
| 5955 |
if needs:
|
| 5956 |
+
_ut_update(rt, _up, flush="sync")
|
| 5957 |
return changed
|
| 5958 |
|
| 5959 |
|
|
|
|
| 6440 |
for k in stats["kept"]))
|
| 6441 |
return stats
|
| 6442 |
stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []}
|
| 6443 |
+
_ut_update(rt, _up, flush="sync")
|
| 6444 |
log(f"[aios-auto] retract: {stats['columns']} foreign column(s), {stats['cells']} cell(s) and "
|
| 6445 |
f"{stats['flags']} mis-declared flag(s) across {stats['tables']} database(s)")
|
| 6446 |
if stats["kept"]:
|
|
|
|
| 6823 |
table.setdefault("rows", {}).setdefault(rid, {}).update(values)
|
| 6824 |
return cur
|
| 6825 |
|
| 6826 |
+
_ut_update(rt, _up, flush="sync")
|
| 6827 |
log(f"[ig-migrate] post-measurement grain: {len(changes)} row(s) gained a post date/kind")
|
| 6828 |
return len(changes)
|
| 6829 |
|
|
|
|
| 6948 |
"type": "rollup", "rollup": _derived_profile_bag(key)}
|
| 6949 |
|
| 6950 |
|
| 6951 |
+
def _ig_profile_plan(tbl, want):
|
| 6952 |
+
"""One profile table's migration as `(fields, rows, counts)`, or None when it has nothing to do.
|
| 6953 |
+
|
| 6954 |
+
It edits copies and reads no store, so `migrate_ig_tables` runs it on the LOCKED table inside
|
| 6955 |
+
the write: a field deleted after an earlier read is not in that table, so it is not re-added.
|
| 6956 |
+
"""
|
| 6957 |
+
counts = dict.fromkeys(("tables", "retyped", "converted", "stamped", "merged", "pinned",
|
| 6958 |
+
"flagged", "droppedName", "droppedRecentPosts",
|
| 6959 |
+
"droppedPostHashtags", "droppedTracked", "droppedTrackedCells"), 0)
|
| 6960 |
+
fields = [dict(f) for f in tbl.get("fields") or []]
|
| 6961 |
+
rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()}
|
| 6962 |
+
# ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched,
|
| 6963 |
+
# so the cell conversions below can never run twice on the same value.
|
| 6964 |
+
stale = [f for f in fields
|
| 6965 |
+
if f.get("key") in want and f.get("type") != want[f["key"]]
|
| 6966 |
+
and f.get("type") in ("text", "", None)]
|
| 6967 |
+
stale_keys = {f["key"] for f in stale}
|
| 6968 |
+
has_platform = any(f.get("key") == "platform" for f in fields)
|
| 6969 |
+
needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()]
|
| 6970 |
+
# ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING.
|
| 6971 |
+
# `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the
|
| 6972 |
+
# two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in
|
| 6973 |
+
# production needs them stamped here, and that is the whole reason this branch exists.
|
| 6974 |
+
#
|
| 6975 |
+
# ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule,
|
| 6976 |
+
# and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which
|
| 6977 |
+
# is exactly why these are safe to run on every write where a value conversion would not be.
|
| 6978 |
+
pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \
|
| 6979 |
+
if PRESET_PINNED_KEY else None
|
| 6980 |
+
flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \
|
| 6981 |
+
if PRESET_FLAG_KEY else None
|
| 6982 |
+
# ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write
|
| 6983 |
+
# doors, and a migration is not exempt from it. If somebody has already flagged another
|
| 6984 |
+
# column on this table, stamping the handle too would leave the table in a state its own
|
| 6985 |
+
# validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would
|
| 6986 |
+
# silently bind to whichever came first in the field list. Leave it alone and pin only.
|
| 6987 |
+
other_profile = next((f for f in fields if f is not flag_f
|
| 6988 |
+
and isinstance(f.get("profile"), dict)), None)
|
| 6989 |
+
want_pin = pin_f is not None and pin_f.get("pinned") is not True
|
| 6990 |
+
want_flag = (flag_f is not None and other_profile is None
|
| 6991 |
+
and not isinstance(flag_f.get("profile"), dict))
|
| 6992 |
+
vestigial = _vestigial_name_field(fields, rows)
|
| 6993 |
+
retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"}
|
| 6994 |
+
if _retired_tracked_field(fields) is not None:
|
| 6995 |
+
retired.add(TRACKED_KEY)
|
| 6996 |
+
orphan_tracked = _orphan_tracked_rows(fields, rows)
|
| 6997 |
+
if (not stale_keys and has_platform and not needs_stamp
|
| 6998 |
+
and not want_pin and not want_flag and vestigial is None and not retired
|
| 6999 |
+
and not orphan_tracked):
|
| 7000 |
+
return None
|
| 7001 |
+
counts["tables"] += 1
|
| 7002 |
+
if want_pin:
|
| 7003 |
+
pin_f["pinned"] = True
|
| 7004 |
+
counts["pinned"] += 1
|
| 7005 |
+
if want_flag:
|
| 7006 |
+
flag_f["profile"] = {"source": PROFILE_SOURCE_IG}
|
| 7007 |
+
counts["flagged"] += 1
|
| 7008 |
+
if vestigial is not None:
|
| 7009 |
+
# ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer
|
| 7010 |
+
# exists is invisible everywhere except the next export, where it reappears as a column
|
| 7011 |
+
# nobody declared. `_vestigial_name_field` has already proven every one of them blank.
|
| 7012 |
+
fields = [f for f in fields if f is not vestigial]
|
| 7013 |
+
for r in rows.values():
|
| 7014 |
+
r.pop("name", None)
|
| 7015 |
+
counts["droppedName"] += 1
|
| 7016 |
+
if retired:
|
| 7017 |
+
fields = [f for f in fields if f.get("key") not in retired]
|
| 7018 |
+
for r in rows.values():
|
| 7019 |
+
for key in retired:
|
| 7020 |
+
r.pop(key, None)
|
| 7021 |
+
counts["droppedRecentPosts"] += int("posts" in retired)
|
| 7022 |
+
counts["droppedPostHashtags"] += int("post_hashtags" in retired)
|
| 7023 |
+
counts["droppedTracked"] += int(TRACKED_KEY in retired)
|
| 7024 |
+
for rid in orphan_tracked:
|
| 7025 |
+
# D-74: a tick whose column was already gone. Counted apart from the column drop
|
| 7026 |
+
# because they answer different questions — "how many tables still declared it" and
|
| 7027 |
+
# "how many rows were still carrying it after the declaration went".
|
| 7028 |
+
rows[rid].pop(TRACKED_KEY, None)
|
| 7029 |
+
counts["droppedTrackedCells"] += 1
|
| 7030 |
+
|
| 7031 |
+
for key in stale_keys:
|
| 7032 |
+
conv = _MIGRATE_CONVERT.get(key)
|
| 7033 |
+
if not conv:
|
| 7034 |
+
continue # int/checkbox already store the right text shape
|
| 7035 |
+
for r in rows.values():
|
| 7036 |
+
if str(r.get(key) or "").strip():
|
| 7037 |
+
new = conv(r[key])
|
| 7038 |
+
# "" from a converter means UNREADABLE, and the cell keeps its original text.
|
| 7039 |
+
# A migration that empties a cell is indistinguishable from one that moved it.
|
| 7040 |
+
if new:
|
| 7041 |
+
r[key], counts["converted"] = new, counts["converted"] + 1
|
| 7042 |
+
for f in stale:
|
| 7043 |
+
f["type"] = want[f["key"]]
|
| 7044 |
+
counts["retyped"] += 1
|
| 7045 |
+
for rid in needs_stamp:
|
| 7046 |
+
rows[rid]["platform"] = PLATFORM_INSTAGRAM
|
| 7047 |
+
counts["stamped"] += 1
|
| 7048 |
+
if not has_platform:
|
| 7049 |
+
fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields
|
| 7050 |
+
rows, merged = _merge_candidates(rows)
|
| 7051 |
+
counts["merged"] += merged
|
| 7052 |
+
return fields, rows, counts
|
| 7053 |
+
|
| 7054 |
+
|
| 7055 |
def migrate_ig_tables(rt, log=print, only=""):
|
| 7056 |
"""Bring every Instagram Profile and canonical child table onto one current contract.
|
| 7057 |
|
|
|
|
| 7117 |
if only and not targets:
|
| 7118 |
return stats
|
| 7119 |
for table_key in targets:
|
| 7120 |
+
# D-118: this lenient read only decides whether there is work. `_up` plans again on the
|
| 7121 |
+
# locked table, so a field deleted after this read stays deleted and keeps its rows.
|
| 7122 |
tbl = ut_get(rt, table_key) or {}
|
| 7123 |
+
if _ig_profile_plan(tbl, want) is None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7124 |
continue
|
| 7125 |
+
field_changes, planned = {}, {}
|
| 7126 |
+
|
| 7127 |
+
def _up(cur):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7128 |
field_changes.clear()
|
| 7129 |
+
planned.clear()
|
| 7130 |
cur = cur or {}
|
| 7131 |
+
if table_key not in cur:
|
| 7132 |
+
return cur # deleted since the read: never re-create it
|
| 7133 |
+
t = dict(cur[table_key] or {})
|
| 7134 |
+
plan = _ig_profile_plan(t, want)
|
| 7135 |
+
if plan is None:
|
| 7136 |
+
return cur
|
| 7137 |
old_keys = _ut()._field_list_keys(t)
|
| 7138 |
+
t["fields"], t["rows"], counts = plan
|
| 7139 |
new_keys = _ut()._field_list_keys(t)
|
| 7140 |
field_changes.update(removed=[key for key in old_keys if key not in new_keys],
|
| 7141 |
added=[key for key in new_keys if key not in old_keys])
|
| 7142 |
+
planned.update(counts)
|
| 7143 |
cur[table_key] = t
|
| 7144 |
return cur
|
| 7145 |
|
| 7146 |
rt.update(UT_STORE_KEY, _up)
|
| 7147 |
+
for name, count in planned.items():
|
| 7148 |
+
stats[name] += count
|
| 7149 |
+
_ut()._log_store_write(rt, table_key, 'migrate_ig_tables')
|
| 7150 |
_ut()._log_field_list_change(rt, table_key, 'automation', 'migrate_ig_tables', field_changes)
|
| 7151 |
# ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION.
|
| 7152 |
#
|
|
|
|
| 7802 |
cur[snap_key]["rows"] = merged
|
| 7803 |
return cur
|
| 7804 |
|
| 7805 |
+
_ut_update(rt, _up, flush="sync")
|
| 7806 |
return written
|
| 7807 |
|
| 7808 |
|
|
|
|
| 8453 |
_refresh_relations_inplace(cur, log=log)
|
| 8454 |
return cur
|
| 8455 |
|
| 8456 |
+
_ut_update(rt, _up, flush="sync") # ONE coalesced update for all four tables
|
| 8457 |
queued = queue_pending_metric_snapshots(
|
| 8458 |
rt, str(defn.get("id") or ""), pending_metrics,
|
| 8459 |
workflow_id=str(defn.get("workflowId") or ""))
|
|
|
|
| 9205 |
if left > 0:
|
| 9206 |
# R6's second sentence: a limit that bites is REPORTED, with its cause and what to do.
|
| 9207 |
notes.append(f"{left} more customers matched but one batch holds "
|
| 9208 |
+
f"{MAX_STATEMENT_BATCH}. Send this batch, then run it again for the rest.")
|
|
|
|
| 9209 |
return batch, notes
|
| 9210 |
|
| 9211 |
|
|
|
|
| 12856 |
_refresh_relations_inplace(cur, log=log)
|
| 12857 |
return cur
|
| 12858 |
|
| 12859 |
+
_ut_update(rt, _up, flush="sync") # ONE coalesced update for all three tables
|
| 12860 |
# LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which
|
| 12861 |
# is the failure a chart cannot show you.
|
| 12862 |
capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"]
|
|
|
|
| 12993 |
return cur
|
| 12994 |
|
| 12995 |
if written:
|
| 12996 |
+
_ut_update(rt, _up, flush="sync") # ONE coalesced update for every table
|
| 12997 |
return inserted, capped, missing
|
| 12998 |
|
| 12999 |
|
|
|
|
| 13569 |
_refresh_relations_inplace(cur, log=log)
|
| 13570 |
return cur
|
| 13571 |
|
| 13572 |
+
_ut_update(rt, _up, flush="sync")
|
| 13573 |
try:
|
| 13574 |
import ig_master
|
| 13575 |
status, note = ig_master.append_run(getattr(rt, "key", ""), [], idents, snapshots)
|
|
|
|
| 14135 |
if owned:
|
| 14136 |
_up(blob)
|
| 14137 |
if persist:
|
| 14138 |
+
_ut_update(rt, _up, flush="sync")
|
| 14139 |
return len(changes)
|
| 14140 |
+
_ut_update(rt, _up, flush="sync")
|
| 14141 |
return len(changes)
|
| 14142 |
|
| 14143 |
|
|
|
|
| 14159 |
# posts to fit, precisely because the store still holds them). ⛔ A rollup may only ever READ. The
|
| 14160 |
# moment one writes a number nothing else can re-derive, it has become a third copy.
|
| 14161 |
|
| 14162 |
+
def _ut_update(rt, fn, **kw):
|
| 14163 |
+
"""Write the whole `user_tables` document and log the D-118 write line for it."""
|
| 14164 |
+
out = rt.update(UT_STORE_KEY, fn, **kw)
|
| 14165 |
+
try:
|
| 14166 |
+
import sys
|
| 14167 |
+
_ut()._log_store_write(rt, '*', sys._getframe(1).f_code.co_name)
|
| 14168 |
+
except Exception: # noqa: BLE001
|
| 14169 |
+
pass # the log line never fails the write it describes
|
| 14170 |
+
return out
|
| 14171 |
+
|
| 14172 |
+
|
| 14173 |
def _ut():
|
| 14174 |
"""`core.user_tables`, imported LAZILY — and the laziness is measured, not stylistic.
|
| 14175 |
|
|
|
|
| 15611 |
tt.setdefault("rows", {}).setdefault(r, {}).update(vals)
|
| 15612 |
return cur
|
| 15613 |
|
| 15614 |
+
_ut_update(rt, _up, flush="sync")
|
| 15615 |
return len(changes)
|
| 15616 |
|
| 15617 |
|
|
|
|
| 15764 |
return cur
|
| 15765 |
|
| 15766 |
# A cold process must see recovered cells before their old refusal disappears.
|
| 15767 |
+
saved = _ut_update(rt, _up, flush='sync')
|
| 15768 |
if saved is False:
|
| 15769 |
raise RuntimeError('The relation refresh was not committed.')
|
| 15770 |
_persist_relation_limits(rt, results)
|
|
|
|
| 15833 |
t["rows"] = nxt
|
| 15834 |
return cur
|
| 15835 |
|
| 15836 |
+
_ut_update(rt, _up, flush="sync")
|
| 15837 |
import ig_master
|
| 15838 |
for bucket, n in (ig_master.purge_handle(subject) or {}).items():
|
| 15839 |
counts[f"master:{bucket}"] = n
|
|
|
|
| 16159 |
return None, str(exc)
|
| 16160 |
if key in TRIGGER_TABLE_KEYS:
|
| 16161 |
table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip()
|
| 16162 |
+
native_conditions = key == "event_field" and table in ("customer", "product")
|
| 16163 |
+
if table and not table.startswith(UT_PREFIX) and not native_conditions:
|
| 16164 |
+
if key == "event_field":
|
| 16165 |
+
return None, ("Event conditions watch customer, product or blank databases (ut_*). "
|
| 16166 |
+
f"{table!r} is not one")
|
| 16167 |
return None, ("event triggers watch blank databases (ut_*) this wave. "
|
| 16168 |
f"{table!r} is not one")
|
| 16169 |
out["table"] = table
|
api/connectors_amazon.py
CHANGED
|
@@ -638,35 +638,9 @@ def database_keys():
|
|
| 638 |
|
| 639 |
|
| 640 |
def read_database_enablement(rt):
|
| 641 |
-
"""
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
Every stored value must be a real boolean. A malformed or unreadable document disables every row in the detail
|
| 645 |
-
response and marks the capability unavailable; it never guesses through corrupt tenant state.
|
| 646 |
-
"""
|
| 647 |
-
keys = database_keys()
|
| 648 |
-
try:
|
| 649 |
-
current = rt.get(ENABLEMENT_KEY) if rt is not None else None
|
| 650 |
-
except Exception as exc: # noqa: BLE001
|
| 651 |
-
raise_if_resource_error(exc)
|
| 652 |
-
return {"available": False, "databases": {key: False for key in keys}}
|
| 653 |
-
# Runtime/store readers use ``{}`` for a genuinely absent document, so both empty forms are
|
| 654 |
-
# the migration default rather than malformed state.
|
| 655 |
-
if current is None or current == {}:
|
| 656 |
-
return {"available": True, "databases": {key: True for key in keys}}
|
| 657 |
-
if not isinstance(current, dict):
|
| 658 |
-
return {"available": False, "databases": {key: False for key in keys}}
|
| 659 |
-
stored = current.get("databases")
|
| 660 |
-
if not isinstance(stored, dict):
|
| 661 |
-
return {"available": False, "databases": {key: False for key in keys}}
|
| 662 |
-
if any(type(value) is not bool for value in stored.values()):
|
| 663 |
-
return {"available": False, "databases": {key: False for key in keys}}
|
| 664 |
-
if set(stored) - set(keys):
|
| 665 |
-
logging.getLogger(__name__).warning("Ignoring unknown Amazon database enablement keys")
|
| 666 |
-
return {"available": True,
|
| 667 |
-
# Absent keys stay on so a later report addition does not silently disappear for
|
| 668 |
-
# every tenant that saved an older version of this document.
|
| 669 |
-
"databases": {key: stored.get(key, True) for key in keys}}
|
| 670 |
|
| 671 |
|
| 672 |
def set_database_enabled(rt, database_key, enabled):
|
|
|
|
| 638 |
|
| 639 |
|
| 640 |
def read_database_enablement(rt):
|
| 641 |
+
"""Use the sync engine's fail-closed evaluator for this tenant's database choices."""
|
| 642 |
+
from harness import amazon_store as az
|
| 643 |
+
return az.read_database_enablement(rt)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 644 |
|
| 645 |
|
| 646 |
def set_database_enabled(rt, database_key, enabled):
|
api/deps.py
CHANGED
|
@@ -776,10 +776,11 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 776 |
# (D-2630: describe answers the field list of a million-row table without counting it);
|
| 777 |
# ("aggregate", specs) runs the aggregates IN SQL over every permitted row (D-616, D-2631);
|
| 778 |
# ("page", offset, limit) pages IN SQL for a table past the one-pass ceiling (rule 1).
|
|
|
|
| 779 |
try:
|
| 780 |
if mode == "query_schema":
|
| 781 |
-
|
| 782 |
-
|
| 783 |
if len(mode) > 2:
|
| 784 |
values, total, groups, group_total, stamp = _assistant_mirror_groups(
|
| 785 |
session, database, cur, spec, fields, mode[1], mode[2],
|
|
@@ -792,9 +793,8 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 792 |
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys)
|
| 793 |
if isinstance(extra, dict):
|
| 794 |
extra["aggregates"], extra["total"] = values, total
|
| 795 |
-
|
| 796 |
-
|
| 797 |
-
if isinstance(mode, tuple) and mode and mode[0] == "page":
|
| 798 |
rows, stamp, total = assistant_mirror.read_page(
|
| 799 |
session, database, cur, spec, fields, offset=mode[1], limit=mode[2],
|
| 800 |
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
|
|
@@ -810,12 +810,19 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 810 |
_assistant_refuse(exc.status, exc.code, exc.message)
|
| 811 |
raise # unreachable; see above
|
| 812 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 813 |
# ⭐ PROVENANCE A CITATION CAN CARRY. `routes_query._system_prompt` runs
|
| 814 |
# `json.dumps(source_version, default=str)`, so this stays plain JSON: when the mirror last
|
| 815 |
# reconciled this table against its connector, and how many rows the answer was computed over.
|
| 816 |
# The count is here because the model is told the record count separately, and a reader
|
| 817 |
# checking one against the other should find the same integer.
|
| 818 |
-
source_version
|
|
|
|
| 819 |
return rows, fields, source_version, definition.get("label") or database
|
| 820 |
|
| 821 |
|
|
@@ -1294,6 +1301,10 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 1294 |
out["groups"] = copy.deepcopy(extra["groups"])
|
| 1295 |
out["total_groups"] = extra["group_total"]
|
| 1296 |
out["source_version"]["refreshed"] = copy.deepcopy(extra["group_stamp"])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1297 |
return out
|
| 1298 |
|
| 1299 |
|
|
@@ -1830,10 +1841,11 @@ def _view_as_user(request, actor, tenant, rt):
|
|
| 1830 |
if not target:
|
| 1831 |
raise _view_as_refused()
|
| 1832 |
rec = copy.deepcopy(_session_registry(target).get(target))
|
| 1833 |
-
if
|
| 1834 |
-
or str(rec.get("tenant") or "").strip().lower() != tenant):
|
| 1835 |
raise _view_as_refused()
|
| 1836 |
member = users._public(target, rec)
|
|
|
|
|
|
|
| 1837 |
audit_key = (tenant, actor["username"], target)
|
| 1838 |
if getattr(request.state, "_view_as_audited", None) != audit_key:
|
| 1839 |
from core import activity_log
|
|
|
|
| 776 |
# (D-2630: describe answers the field list of a million-row table without counting it);
|
| 777 |
# ("aggregate", specs) runs the aggregates IN SQL over every permitted row (D-616, D-2631);
|
| 778 |
# ("page", offset, limit) pages IN SQL for a table past the one-pass ceiling (rule 1).
|
| 779 |
+
source_version = None
|
| 780 |
try:
|
| 781 |
if mode == "query_schema":
|
| 782 |
+
rows, source_version = [], {"read": "connector mirror"}
|
| 783 |
+
elif isinstance(mode, tuple) and mode and mode[0] == "aggregate":
|
| 784 |
if len(mode) > 2:
|
| 785 |
values, total, groups, group_total, stamp = _assistant_mirror_groups(
|
| 786 |
session, database, cur, spec, fields, mode[1], mode[2],
|
|
|
|
| 793 |
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys)
|
| 794 |
if isinstance(extra, dict):
|
| 795 |
extra["aggregates"], extra["total"] = values, total
|
| 796 |
+
rows, source_version = [], {"read": "connector mirror", "rows": total}
|
| 797 |
+
elif isinstance(mode, tuple) and mode and mode[0] == "page":
|
|
|
|
| 798 |
rows, stamp, total = assistant_mirror.read_page(
|
| 799 |
session, database, cur, spec, fields, offset=mode[1], limit=mode[2],
|
| 800 |
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
|
|
|
|
| 810 |
_assistant_refuse(exc.status, exc.code, exc.message)
|
| 811 |
raise # unreachable; see above
|
| 812 |
|
| 813 |
+
if isinstance(extra, dict):
|
| 814 |
+
if isinstance(spec.get("sourceStatus"), dict):
|
| 815 |
+
extra["sourceStatus"] = copy.deepcopy(spec["sourceStatus"])
|
| 816 |
+
if isinstance(spec.get("limits"), list):
|
| 817 |
+
extra["limits"] = copy.deepcopy(spec["limits"])
|
| 818 |
+
|
| 819 |
# ⭐ PROVENANCE A CITATION CAN CARRY. `routes_query._system_prompt` runs
|
| 820 |
# `json.dumps(source_version, default=str)`, so this stays plain JSON: when the mirror last
|
| 821 |
# reconciled this table against its connector, and how many rows the answer was computed over.
|
| 822 |
# The count is here because the model is told the record count separately, and a reader
|
| 823 |
# checking one against the other should find the same integer.
|
| 824 |
+
if source_version is None:
|
| 825 |
+
source_version = {"read": "connector mirror", "refreshed": stamp, "rows": total}
|
| 826 |
return rows, fields, source_version, definition.get("label") or database
|
| 827 |
|
| 828 |
|
|
|
|
| 1301 |
out["groups"] = copy.deepcopy(extra["groups"])
|
| 1302 |
out["total_groups"] = extra["group_total"]
|
| 1303 |
out["source_version"]["refreshed"] = copy.deepcopy(extra["group_stamp"])
|
| 1304 |
+
if isinstance(extra.get("sourceStatus"), dict):
|
| 1305 |
+
out["sourceStatus"] = copy.deepcopy(extra["sourceStatus"])
|
| 1306 |
+
if isinstance(extra.get("limits"), list):
|
| 1307 |
+
out["limits"] = copy.deepcopy(extra["limits"])
|
| 1308 |
return out
|
| 1309 |
|
| 1310 |
|
|
|
|
| 1841 |
if not target:
|
| 1842 |
raise _view_as_refused()
|
| 1843 |
rec = copy.deepcopy(_session_registry(target).get(target))
|
| 1844 |
+
if not rec or not rec.get("active", True) or rec.get("role", "user") != "user":
|
|
|
|
| 1845 |
raise _view_as_refused()
|
| 1846 |
member = users._public(target, rec)
|
| 1847 |
+
if member.get("tenant") != tenant:
|
| 1848 |
+
raise _view_as_refused()
|
| 1849 |
audit_key = (tenant, actor["username"], target)
|
| 1850 |
if getattr(request.state, "_view_as_audited", None) != audit_key:
|
| 1851 |
from core import activity_log
|
api/intelligence_store.py
CHANGED
|
@@ -1628,6 +1628,7 @@ def decide_proposal(session, proposal_id, decision):
|
|
| 1628 |
raise IntelligenceError("person_required", "A person must review this change.", 403)
|
| 1629 |
if decision not in ("approve", "reject"):
|
| 1630 |
raise IntelligenceError("bad_decision", "Choose Approve or Decline.")
|
|
|
|
| 1631 |
|
| 1632 |
def decide(raw):
|
| 1633 |
saved.clear()
|
|
|
|
| 1628 |
raise IntelligenceError("person_required", "A person must review this change.", 403)
|
| 1629 |
if decision not in ("approve", "reject"):
|
| 1630 |
raise IntelligenceError("bad_decision", "Choose Approve or Decline.")
|
| 1631 |
+
saved = {}
|
| 1632 |
|
| 1633 |
def decide(raw):
|
| 1634 |
saved.clear()
|
api/loopy_bulk.py
CHANGED
|
@@ -5,7 +5,7 @@ from urllib.parse import quote
|
|
| 5 |
|
| 6 |
from loopy_chat_link_tools import _name_field, _phone_field
|
| 7 |
from loopy_chat_links import validate_message, wa_link
|
| 8 |
-
from loopy_diagnostics import ScopedOdoo, amount, bind_functions, identities, plain
|
| 9 |
from loopy_file_tools import _secure_link, read_all
|
| 10 |
|
| 11 |
CUSTOMERS = "customer_data"
|
|
@@ -72,25 +72,18 @@ def _group(rows, today):
|
|
| 72 |
def validate(session, records, overdue, grouped, *, today, partner_ids):
|
| 73 |
"""Tie each grouped amount and overdue age to independent, permission-bounded Odoo reads."""
|
| 74 |
import wakeups
|
| 75 |
-
from core import odoo as O
|
| 76 |
|
| 77 |
if session.tenant != session.runtime.key:
|
| 78 |
raise ValueError("The chase list tenant must match its runtime.")
|
| 79 |
-
def narrowed(database):
|
| 80 |
-
grant = perm_scope.assistant_entry(session.user, database) or {}
|
| 81 |
-
return (bool(grant.get("filter"))
|
| 82 |
-
or perm_scope.row_scope_applies(session.user, database, st=session.runtime)
|
| 83 |
-
or "All" not in perms.allowed_bu_labels(session.user)
|
| 84 |
-
or any(value is not None for value in perm_scope.derive_pool_scope(session.user, database)))
|
| 85 |
-
|
| 86 |
-
invoice_wall, customer_wall = narrowed(INVOICES), narrowed(CUSTOMERS)
|
| 87 |
-
ids = identities(records, "odoo_id")
|
| 88 |
shown_ids = identities(overdue, "odoo_id")
|
| 89 |
source = session.runtime.odoo_source()
|
| 90 |
if source is None:
|
| 91 |
raise ValueError("Connect Odoo before building a chase list.")
|
| 92 |
-
|
| 93 |
-
|
|
|
|
|
|
|
| 94 |
domain, _accepts = wakeups._ar_rule(today, 0)
|
| 95 |
with source._bound():
|
| 96 |
client = ScopedOdoo(O.get_odoo(), restrictions)
|
|
@@ -101,7 +94,8 @@ def validate(session, records, overdue, grouped, *, today, partner_ids):
|
|
| 101 |
partner = row.get("partner_id")
|
| 102 |
pid = partner[0] if isinstance(partner, (list, tuple)) and partner else partner
|
| 103 |
if (type(pid) is not int or pid <= 0
|
| 104 |
-
or
|
|
|
|
| 105 |
raise ValueError("Odoo returned an unreadable invoice.")
|
| 106 |
native_rows.append({"odoo_id": row["id"], "partner_id": pid, "residual": row.get("amount_residual_signed"),
|
| 107 |
"due_date": row.get("invoice_date_due")})
|
|
|
|
| 5 |
|
| 6 |
from loopy_chat_link_tools import _name_field, _phone_field
|
| 7 |
from loopy_chat_links import validate_message, wa_link
|
| 8 |
+
from loopy_diagnostics import ScopedOdoo, amount, ar_scope, bind_functions, identities, plain
|
| 9 |
from loopy_file_tools import _secure_link, read_all
|
| 10 |
|
| 11 |
CUSTOMERS = "customer_data"
|
|
|
|
| 72 |
def validate(session, records, overdue, grouped, *, today, partner_ids):
|
| 73 |
"""Tie each grouped amount and overdue age to independent, permission-bounded Odoo reads."""
|
| 74 |
import wakeups
|
| 75 |
+
from core import odoo as O
|
| 76 |
|
| 77 |
if session.tenant != session.runtime.key:
|
| 78 |
raise ValueError("The chase list tenant must match its runtime.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
shown_ids = identities(overdue, "odoo_id")
|
| 80 |
source = session.runtime.odoo_source()
|
| 81 |
if source is None:
|
| 82 |
raise ValueError("Connect Odoo before building a chase list.")
|
| 83 |
+
# Unrestricted validation must see invoices absent from the permitted mirror snapshot.
|
| 84 |
+
restrictions = ar_scope(session, customers=True, native=True, records={
|
| 85 |
+
INVOICES: records, CUSTOMERS: [{"pid": pid} for pid in partner_ids]})
|
| 86 |
+
walls = {field: set(values) for field, _operator, values in restrictions["account.move"]}
|
| 87 |
domain, _accepts = wakeups._ar_rule(today, 0)
|
| 88 |
with source._bound():
|
| 89 |
client = ScopedOdoo(O.get_odoo(), restrictions)
|
|
|
|
| 94 |
partner = row.get("partner_id")
|
| 95 |
pid = partner[0] if isinstance(partner, (list, tuple)) and partner else partner
|
| 96 |
if (type(pid) is not int or pid <= 0
|
| 97 |
+
or "partner_id" in walls and pid not in walls["partner_id"]
|
| 98 |
+
or "id" in walls and row["id"] not in walls["id"]):
|
| 99 |
raise ValueError("Odoo returned an unreadable invoice.")
|
| 100 |
native_rows.append({"odoo_id": row["id"], "partner_id": pid, "residual": row.get("amount_residual_signed"),
|
| 101 |
"due_date": row.get("invoice_date_due")})
|
api/main.py
CHANGED
|
@@ -388,27 +388,22 @@ def public_freeze_notice():
|
|
| 388 |
|
| 389 |
|
| 390 |
@app.get("/api/health")
|
| 391 |
-
|
| 392 |
-
"""
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
W-265 adds a possession challenge for the deployer. It reveals only whether this process was
|
| 403 |
-
born after one high-entropy, single-deploy nonce was stamped; it never returns the nonce,
|
| 404 |
-
version, Space id, or configuration. A normal liveness request keeps the exact
|
| 405 |
-
``{"ok": true}`` response shape.
|
| 406 |
-
"""
|
| 407 |
if challenge:
|
| 408 |
expected = os.environ.get("AIOS_DEPLOY_NONCE") or ""
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
|
|
|
| 412 |
|
| 413 |
|
| 414 |
def _mirror_answers(rt=None):
|
|
@@ -1195,9 +1190,10 @@ def _pull_meta(why, tenant_key=None):
|
|
| 1195 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 1196 |
log=lambda *_a: None, insights_days=7)
|
| 1197 |
outcomes[key] = mirror_store._sync_result("meta", rep)[1]
|
| 1198 |
-
|
|
|
|
| 1199 |
for problem in rep.get("problems") or []:
|
| 1200 |
-
print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {problem}")
|
| 1201 |
except Exception as exc:
|
| 1202 |
print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
|
| 1203 |
_failed = globals().get("_publish_connector_status")
|
|
@@ -1526,6 +1522,20 @@ def _sync_now_cause(cause):
|
|
| 1526 |
return "unclassified", "The connector reported a failure. Inspect its sync report and retry."
|
| 1527 |
raw = str(cause)
|
| 1528 |
text = raw.lower()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1529 |
causes = (
|
| 1530 |
("not_offered", ("not_offered", "not offered", "does not offer this report"),
|
| 1531 |
"Amazon does not offer this report to this seller. Its last published rows are kept."),
|
|
@@ -3215,15 +3225,17 @@ def _entity_source_counter(ds):
|
|
| 3215 |
"""
|
| 3216 |
import inspect as _inspect_counts
|
| 3217 |
try:
|
| 3218 |
-
if "after_entity" not in _inspect_counts.signature(ds.sync_all).parameters:
|
| 3219 |
-
return None
|
| 3220 |
rt = ds._resident_odoo_runtime()
|
|
|
|
| 3221 |
except Exception: # noqa: BLE001
|
| 3222 |
return None
|
| 3223 |
tenant = getattr(rt, "key", None)
|
| 3224 |
if not tenant:
|
| 3225 |
return None
|
|
|
|
| 3226 |
counted = _PASS_SOURCE_COUNTS[tenant] = {}
|
|
|
|
|
|
|
| 3227 |
per_connection = {}
|
| 3228 |
|
| 3229 |
def after_entity(key):
|
|
@@ -3380,6 +3392,11 @@ def _store_resync_loop():
|
|
| 3380 |
return
|
| 3381 |
import time as _t
|
| 3382 |
from core import connector_sync
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3383 |
_restore_sync_request_desks()
|
| 3384 |
# Restore/rebuild precedes cadence/source hydration as well as the Odoo backfill.
|
| 3385 |
import amazon_relational as _ar_boot
|
|
@@ -4155,6 +4172,8 @@ def _seed_mirror_if_absent():
|
|
| 4155 |
|
| 4156 |
def restore(tenant_key, rt):
|
| 4157 |
try:
|
|
|
|
|
|
|
| 4158 |
result = _ds.restore_odoo_mirror(tenant_key, rt)
|
| 4159 |
except Exception as exc:
|
| 4160 |
result = refused(tenant_key, exc)
|
|
@@ -4336,7 +4355,7 @@ def _boot_mirrors():
|
|
| 4336 |
# With several web workers only the primary restores the mirror. The others read it.
|
| 4337 |
_seed_mirror_if_absent()
|
| 4338 |
if _role_boot.role() == "web":
|
| 4339 |
-
# Readiness
|
| 4340 |
_role_boot.mark_ready("mirror", True)
|
| 4341 |
# Every connector but Odoo reaches the web process through the worker's hand-off.
|
| 4342 |
print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
|
|
@@ -4348,7 +4367,7 @@ def _boot_mirrors():
|
|
| 4348 |
_restore_amazon_mirror("boot")
|
| 4349 |
_restore_declared_mirrors("boot")
|
| 4350 |
finally:
|
| 4351 |
-
#
|
| 4352 |
_role_boot.note_boot_restore()
|
| 4353 |
return
|
| 4354 |
_restore_amazon_mirror("boot")
|
|
|
|
| 388 |
|
| 389 |
|
| 390 |
@app.get("/api/health")
|
| 391 |
+
def health(challenge: str = ""):
|
| 392 |
+
"""Check a strict store read without disclosing tenants, configuration or exception text."""
|
| 393 |
+
# A cached availability flag cannot detect a connection lost after a successful first read.
|
| 394 |
+
try:
|
| 395 |
+
ok = bool(store.available())
|
| 396 |
+
if ok:
|
| 397 |
+
store._read_strict("sync_settings")
|
| 398 |
+
except Exception:
|
| 399 |
+
ok = False
|
| 400 |
+
body = {"ok": ok}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 401 |
if challenge:
|
| 402 |
expected = os.environ.get("AIOS_DEPLOY_NONCE") or ""
|
| 403 |
+
body["deployReady"] = bool(ok and expected and hmac.compare_digest(challenge, expected))
|
| 404 |
+
if not ok:
|
| 405 |
+
return JSONResponse(body, status_code=503, headers={"Cache-Control": "no-store"})
|
| 406 |
+
return body
|
| 407 |
|
| 408 |
|
| 409 |
def _mirror_answers(rt=None):
|
|
|
|
| 1190 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 1191 |
log=lambda *_a: None, insights_days=7)
|
| 1192 |
outcomes[key] = mirror_store._sync_result("meta", rep)[1]
|
| 1193 |
+
detail = _sync_now_failure(" ".join(rep.get("problems") or [])) if not outcomes[key] else ""
|
| 1194 |
+
_status(rt, "meta_ads", "done" if outcomes[key] else "failed", detail=detail)
|
| 1195 |
for problem in rep.get("problems") or []:
|
| 1196 |
+
print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {_sync_now_failure(problem)}")
|
| 1197 |
except Exception as exc:
|
| 1198 |
print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
|
| 1199 |
_failed = globals().get("_publish_connector_status")
|
|
|
|
| 1522 |
return "unclassified", "The connector reported a failure. Inspect its sync report and retry."
|
| 1523 |
raw = str(cause)
|
| 1524 |
text = raw.lower()
|
| 1525 |
+
# Authored transport causes precede generic timeout, credential and connection matching.
|
| 1526 |
+
meta_transport = (
|
| 1527 |
+
("meta_dns", "Meta Ads could not resolve the provider's host name. Check the server's DNS connection."),
|
| 1528 |
+
("meta_certificate", "Meta Ads could not verify the server certificate. Check the server's trusted certificates and clock."),
|
| 1529 |
+
("meta_tls", "Meta Ads could not establish a secure connection. Check the server's TLS connection."),
|
| 1530 |
+
("timeout", "The Meta Ads request timed out. Retry the sync after checking the server's network connection."),
|
| 1531 |
+
("meta_refused", "The connection to Meta Ads was refused. Check the server's firewall and proxy settings."),
|
| 1532 |
+
("meta_reset", "The connection to Meta Ads was reset. Retry the sync after checking the server's network connection."),
|
| 1533 |
+
("meta_route", "The server cannot reach the Meta Ads network. Check its network route and firewall settings."),
|
| 1534 |
+
("meta_network", "The network request to Meta Ads failed. Check the server's network and proxy settings, then retry the sync."),
|
| 1535 |
+
)
|
| 1536 |
+
for classifier, sentence in meta_transport:
|
| 1537 |
+
if sentence.split(". ", 1)[0].lower() + "." in text:
|
| 1538 |
+
return classifier, sentence
|
| 1539 |
causes = (
|
| 1540 |
("not_offered", ("not_offered", "not offered", "does not offer this report"),
|
| 1541 |
"Amazon does not offer this report to this seller. Its last published rows are kept."),
|
|
|
|
| 3225 |
"""
|
| 3226 |
import inspect as _inspect_counts
|
| 3227 |
try:
|
|
|
|
|
|
|
| 3228 |
rt = ds._resident_odoo_runtime()
|
| 3229 |
+
supports_hook = "after_entity" in _inspect_counts.signature(ds.sync_all).parameters
|
| 3230 |
except Exception: # noqa: BLE001
|
| 3231 |
return None
|
| 3232 |
tenant = getattr(rt, "key", None)
|
| 3233 |
if not tenant:
|
| 3234 |
return None
|
| 3235 |
+
# A new pass cannot inherit a failed pass's counts when its adapter lacks the hook.
|
| 3236 |
counted = _PASS_SOURCE_COUNTS[tenant] = {}
|
| 3237 |
+
if not supports_hook:
|
| 3238 |
+
return None
|
| 3239 |
per_connection = {}
|
| 3240 |
|
| 3241 |
def after_entity(key):
|
|
|
|
| 3392 |
return
|
| 3393 |
import time as _t
|
| 3394 |
from core import connector_sync
|
| 3395 |
+
from harness import amazon_store as _phase_amazon
|
| 3396 |
+
import odoo_relational as _phase_odoo
|
| 3397 |
+
import sync_alerts as _phase_alerts
|
| 3398 |
+
if _phase_alerts.environment() in {"LIVE", "STAGING"}:
|
| 3399 |
+
_phase_amazon.register_schedule_offset(_phase_odoo.slot_offset_seconds)
|
| 3400 |
_restore_sync_request_desks()
|
| 3401 |
# Restore/rebuild precedes cadence/source hydration as well as the Odoo backfill.
|
| 3402 |
import amazon_relational as _ar_boot
|
|
|
|
| 4172 |
|
| 4173 |
def restore(tenant_key, rt):
|
| 4174 |
try:
|
| 4175 |
+
if _ds.odoo_mirror_restored(tenant_key, rt):
|
| 4176 |
+
return
|
| 4177 |
result = _ds.restore_odoo_mirror(tenant_key, rt)
|
| 4178 |
except Exception as exc:
|
| 4179 |
result = refused(tenant_key, exc)
|
|
|
|
| 4355 |
# With several web workers only the primary restores the mirror. The others read it.
|
| 4356 |
_seed_mirror_if_absent()
|
| 4357 |
if _role_boot.role() == "web":
|
| 4358 |
+
# Readiness checks this process's Odoo mirror and reads while worker catch-up continues.
|
| 4359 |
_role_boot.mark_ready("mirror", True)
|
| 4360 |
# Every connector but Odoo reaches the web process through the worker's hand-off.
|
| 4361 |
print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
|
|
|
|
| 4367 |
_restore_amazon_mirror("boot")
|
| 4368 |
_restore_declared_mirrors("boot")
|
| 4369 |
finally:
|
| 4370 |
+
# Publish restore progress without holding the web's installed Odoo reads.
|
| 4371 |
_role_boot.note_boot_restore()
|
| 4372 |
return
|
| 4373 |
_restore_amazon_mirror("boot")
|
api/odoo_relational.py
CHANGED
|
@@ -7819,12 +7819,15 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
|
|
| 7819 |
key = table_key_for(base_key, connection_id)
|
| 7820 |
source_count = None
|
| 7821 |
source_mode = "unavailable"
|
|
|
|
| 7822 |
source_failure = None
|
| 7823 |
if source_counts and bucket in source_counts:
|
| 7824 |
source_count, source_mode = source_counts[bucket]
|
|
|
|
| 7825 |
elif source_client is not None:
|
| 7826 |
try:
|
| 7827 |
source_count, source_mode = _source_domain_count(source_client, bucket)
|
|
|
|
| 7828 |
except Exception as exc: # noqa: BLE001
|
| 7829 |
source_failure = {
|
| 7830 |
"code": "source_reconciliation_failed", "type": type(exc).__name__,
|
|
@@ -7887,6 +7890,7 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
|
|
| 7887 |
receipts[key] = {
|
| 7888 |
**identity, "state": "partial" if partial else "succeeded",
|
| 7889 |
"sourceCount": source_count, "sourceCountMode": source_mode,
|
|
|
|
| 7890 |
"sourceReconciliation": source_reconciliation,
|
| 7891 |
"mirrorCount": count, "materializedCount": materialized_count,
|
| 7892 |
"servingCount": count, "readThrough": read_through, "duplicateCount": 0,
|
|
@@ -7908,7 +7912,8 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
|
|
| 7908 |
errors[key] = failure
|
| 7909 |
receipts[key] = {
|
| 7910 |
**identity, "state": "failed", "sourceCount": source_count,
|
| 7911 |
-
"sourceCountMode": source_mode, "
|
|
|
|
| 7912 |
"mirrorCount": None, "materializedCount": None, "servingCount": None,
|
| 7913 |
"readThrough": is_read_through_key(key), "duplicateCount": None,
|
| 7914 |
"freshness": {"state": "failed", "capturedAt": captured_at},
|
|
|
|
| 7819 |
key = table_key_for(base_key, connection_id)
|
| 7820 |
source_count = None
|
| 7821 |
source_mode = "unavailable"
|
| 7822 |
+
source_basis = "unavailable"
|
| 7823 |
source_failure = None
|
| 7824 |
if source_counts and bucket in source_counts:
|
| 7825 |
source_count, source_mode = source_counts[bucket]
|
| 7826 |
+
source_basis = "mirror_pull"
|
| 7827 |
elif source_client is not None:
|
| 7828 |
try:
|
| 7829 |
source_count, source_mode = _source_domain_count(source_client, bucket)
|
| 7830 |
+
source_basis = "reconciliation"
|
| 7831 |
except Exception as exc: # noqa: BLE001
|
| 7832 |
source_failure = {
|
| 7833 |
"code": "source_reconciliation_failed", "type": type(exc).__name__,
|
|
|
|
| 7890 |
receipts[key] = {
|
| 7891 |
**identity, "state": "partial" if partial else "succeeded",
|
| 7892 |
"sourceCount": source_count, "sourceCountMode": source_mode,
|
| 7893 |
+
"sourceCountBasis": source_basis,
|
| 7894 |
"sourceReconciliation": source_reconciliation,
|
| 7895 |
"mirrorCount": count, "materializedCount": materialized_count,
|
| 7896 |
"servingCount": count, "readThrough": read_through, "duplicateCount": 0,
|
|
|
|
| 7912 |
errors[key] = failure
|
| 7913 |
receipts[key] = {
|
| 7914 |
**identity, "state": "failed", "sourceCount": source_count,
|
| 7915 |
+
"sourceCountMode": source_mode, "sourceCountBasis": source_basis,
|
| 7916 |
+
"sourceReconciliation": "unconfirmed",
|
| 7917 |
"mirrorCount": None, "materializedCount": None, "servingCount": None,
|
| 7918 |
"readThrough": is_read_through_key(key), "duplicateCount": None,
|
| 7919 |
"freshness": {"state": "failed", "capturedAt": captured_at},
|
api/process_role.py
CHANGED
|
@@ -439,8 +439,7 @@ _READY_DEFAULT = {"mirror": "The mirror is not installed yet.",
|
|
| 439 |
"tenant_keys": "The tenant data keys have not been checked yet."}
|
| 440 |
_READINESS = {}
|
| 441 |
_BOOT_RESTORE = {}
|
| 442 |
-
#
|
| 443 |
-
# must not pull a warm web process out of rotation.
|
| 444 |
_READY_LATCH = {"at": None}
|
| 445 |
|
| 446 |
|
|
@@ -622,15 +621,11 @@ def readiness(mirror_live=None, role_name=None, root=None):
|
|
| 622 |
continue
|
| 623 |
row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")}
|
| 624 |
checks[name], detail[name] = bool(row["ok"]), row["detail"]
|
| 625 |
-
boot = role_name == "web" and _READY_LATCH["at"] is None
|
| 626 |
if "mirror" in checks and checks["mirror"]:
|
| 627 |
if mirror_live is False:
|
| 628 |
checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading."
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
elif boot and pending_handoffs(root):
|
| 632 |
-
checks["mirror"] = False
|
| 633 |
-
detail["mirror"] = "Mirror hand-offs are not installed yet: " + ", ".join(pending_handoffs(root)) + "."
|
| 634 |
failing = [name for name in checks if not checks[name]]
|
| 635 |
if not failing and _READY_LATCH["at"] is None:
|
| 636 |
_READY_LATCH["at"] = time.time()
|
|
|
|
| 439 |
"tenant_keys": "The tenant data keys have not been checked yet."}
|
| 440 |
_READINESS = {}
|
| 441 |
_BOOT_RESTORE = {}
|
| 442 |
+
# Record the first completed local readiness check for diagnostics.
|
|
|
|
| 443 |
_READY_LATCH = {"at": None}
|
| 444 |
|
| 445 |
|
|
|
|
| 621 |
continue
|
| 622 |
row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")}
|
| 623 |
checks[name], detail[name] = bool(row["ok"]), row["detail"]
|
|
|
|
| 624 |
if "mirror" in checks and checks["mirror"]:
|
| 625 |
if mirror_live is False:
|
| 626 |
checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading."
|
| 627 |
+
# The web serves its installed Odoo snapshot while worker restores and catch-up continue.
|
| 628 |
+
# Connector reads enforce their own mirror readiness before serving that connector's rows.
|
|
|
|
|
|
|
|
|
|
| 629 |
failing = [name for name in checks if not checks[name]]
|
| 630 |
if not failing and _READY_LATCH["at"] is None:
|
| 631 |
_READY_LATCH["at"] = time.time()
|
api/request_class.py
CHANGED
|
@@ -536,7 +536,7 @@ def _replay(msgs, receive):
|
|
| 536 |
|
| 537 |
|
| 538 |
# ----------------------------------------------------------------------------- box sizing (R25)
|
| 539 |
-
#: The 2 vCPU
|
| 540 |
TODAY = {"workers": 1, "heavyConcurrency": 2, "webDuckdbMemory": "6GB", "webDuckdbThreads": 2,
|
| 541 |
"workerDuckdbMemory": "4GB", "workerDuckdbThreads": 1}
|
| 542 |
#: Launch env name for each sizing value. An explicit env value always wins over the derived one.
|
|
@@ -655,15 +655,13 @@ def launch_role(environ=None):
|
|
| 655 |
|
| 656 |
|
| 657 |
def launch_env(environ=None, box=None):
|
| 658 |
-
"""
|
| 659 |
-
without the read-only mirror: AIOS_MIRROR_VERSIONS=0 or a role other than web holds one worker.
|
| 660 |
-
"""
|
| 661 |
environ = os.environ if environ is None else environ
|
| 662 |
box = box or detect_box()
|
| 663 |
explicit = {key for key, name in SIZING_ENV.items() if str(environ.get(name) or "").strip()}
|
| 664 |
forced = web_workers(environ) if "workers" in explicit else None
|
| 665 |
role = launch_role(environ)
|
| 666 |
-
if str(environ.get(VERSIONS_ENV) or "").strip() =
|
| 667 |
# Only the web role versions its mirror writes. Every other role restores in place.
|
| 668 |
forced = 1
|
| 669 |
values = sizing(box["cpus"], box["memoryBytes"], workers=forced)
|
|
|
|
| 536 |
|
| 537 |
|
| 538 |
# ----------------------------------------------------------------------------- box sizing (R25)
|
| 539 |
+
#: The nominal 2 vCPU profile, independent of the container's measured memory limit.
|
| 540 |
TODAY = {"workers": 1, "heavyConcurrency": 2, "webDuckdbMemory": "6GB", "webDuckdbThreads": 2,
|
| 541 |
"workerDuckdbMemory": "4GB", "workerDuckdbThreads": 1}
|
| 542 |
#: Launch env name for each sizing value. An explicit env value always wins over the derived one.
|
|
|
|
| 655 |
|
| 656 |
|
| 657 |
def launch_env(environ=None, box=None):
|
| 658 |
+
"""Return launch exports, allowing several workers only for web with AIOS_MIRROR_VERSIONS=1."""
|
|
|
|
|
|
|
| 659 |
environ = os.environ if environ is None else environ
|
| 660 |
box = box or detect_box()
|
| 661 |
explicit = {key for key, name in SIZING_ENV.items() if str(environ.get(name) or "").strip()}
|
| 662 |
forced = web_workers(environ) if "workers" in explicit else None
|
| 663 |
role = launch_role(environ)
|
| 664 |
+
if str(environ.get(VERSIONS_ENV) or "").strip() != "1" or role != "web":
|
| 665 |
# Only the web role versions its mirror writes. Every other role restores in place.
|
| 666 |
forced = 1
|
| 667 |
values = sizing(box["cpus"], box["memoryBytes"], workers=forced)
|
api/routes_amazon_events.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Receive signed Amazon listing events and administer tenant subscriptions."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import hashlib
|
| 5 |
+
import hmac
|
| 6 |
+
import json
|
| 7 |
+
import os
|
| 8 |
+
import time
|
| 9 |
+
|
| 10 |
+
from fastapi import APIRouter, Body, Depends, HTTPException, Request
|
| 11 |
+
|
| 12 |
+
from deps import Session, err
|
| 13 |
+
from routes_admin import admin_gate
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
router = APIRouter()
|
| 17 |
+
NOTIFICATION_TYPE = "BRANDED_ITEM_CONTENT_CHANGE"
|
| 18 |
+
AWS_REGION = "us-east-2"
|
| 19 |
+
AWS_ACCOUNT_SHA256 = "ef5273b4f80cbdf7da2c921e9679b69e0772f2d585e9c47a13425022750a96b5"
|
| 20 |
+
SUBSCRIPTIONS_FIELD = "amazonNotifications"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def verify_event(headers, body: bytes, *, now=None):
|
| 24 |
+
"""Verify the producer's timestamp and exact body before JSON decoding."""
|
| 25 |
+
token = os.environ.get("AIOS_AMAZON_EVENT_TOKEN")
|
| 26 |
+
if not token:
|
| 27 |
+
return False, 503, "amazon_event_token_not_configured"
|
| 28 |
+
lowered = {str(key).lower(): str(value) for key, value in headers.items()}
|
| 29 |
+
stamp = lowered.get("x-aios-timestamp", "")
|
| 30 |
+
signature = lowered.get("x-aios-signature", "")
|
| 31 |
+
try:
|
| 32 |
+
valid_stamp = abs((time.time() if now is None else float(now)) - int(stamp)) <= 300
|
| 33 |
+
except ValueError:
|
| 34 |
+
valid_stamp = False
|
| 35 |
+
expected = hmac.new(token.encode(), stamp.encode() + b"\n" + body, hashlib.sha256).hexdigest()
|
| 36 |
+
if not valid_stamp or not hmac.compare_digest(expected.encode(), signature.encode()):
|
| 37 |
+
return False, 403, "amazon_event_signature_invalid"
|
| 38 |
+
return True, 200, ""
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _partner_bus(fields, result):
|
| 42 |
+
"""Read the returned destination's partner bus name for the owner's bind step."""
|
| 43 |
+
from harness import amazon_client as client
|
| 44 |
+
|
| 45 |
+
if result.get("eventBusName"):
|
| 46 |
+
return result["eventBusName"]
|
| 47 |
+
token = client.notifications_token(fields)
|
| 48 |
+
destinations = client._request(
|
| 49 |
+
client._NOTIFICATION_DESTINATIONS_PATH, token,
|
| 50 |
+
region=fields.get("region") or client.DEFAULT_REGION,
|
| 51 |
+
sandbox=client.is_sandbox(fields)).get("payload") or []
|
| 52 |
+
for destination in destinations:
|
| 53 |
+
if destination.get("destinationId") == result["destinationId"]:
|
| 54 |
+
return ((destination.get("resource") or {}).get("eventBridge") or {}).get("name")
|
| 55 |
+
return None
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@router.post("/connectors/amazon_sp/notifications")
|
| 59 |
+
def setup_notifications(body: dict = Body(...), session: Session = Depends(admin_gate)):
|
| 60 |
+
"""Ensure subscriptions and save their IDs on the selected tenant connection."""
|
| 61 |
+
from core import keychain
|
| 62 |
+
from harness import amazon_client as client
|
| 63 |
+
|
| 64 |
+
account = str(body.get("awsAccountId") or "")
|
| 65 |
+
if (len(account) != 12 or not account.isascii() or not account.isdigit()
|
| 66 |
+
or hashlib.sha256(account.encode()).hexdigest() != AWS_ACCOUNT_SHA256
|
| 67 |
+
or body.get("awsRegion", AWS_REGION) != AWS_REGION):
|
| 68 |
+
raise err(422, "amazon_notification_destination_invalid",
|
| 69 |
+
"Use the designated AIOS AWS account in us-east-2.")
|
| 70 |
+
roles = body.get("grantedRoles", [])
|
| 71 |
+
if not isinstance(roles, list) or any(not isinstance(role, str) for role in roles):
|
| 72 |
+
raise err(422, "amazon_notification_roles_invalid", "grantedRoles must be a list of role names.")
|
| 73 |
+
rt = session.runtime
|
| 74 |
+
connection_id = keychain.resolve_connection_id(rt, "amazon_sp", body.get("connectionId"))
|
| 75 |
+
row = keychain._row_for(rt, connection_id, "amazon_sp")
|
| 76 |
+
if row is None:
|
| 77 |
+
raise err(404, "amazon_connection_not_found", "This tenant has no matching Amazon connection.")
|
| 78 |
+
try:
|
| 79 |
+
fields = keychain.amazon_creds(rt, connection_id=connection_id)
|
| 80 |
+
if not fields:
|
| 81 |
+
raise err(404, "amazon_connection_not_found", "This Amazon connection has no credential.")
|
| 82 |
+
result = client.ensure_notifications(fields, account, AWS_REGION, granted_roles=roles)
|
| 83 |
+
except client.AmazonError:
|
| 84 |
+
raise err(503, "amazon_notifications_unavailable", "Amazon notification setup could not complete.") from None
|
| 85 |
+
except keychain.KeychainLocked:
|
| 86 |
+
raise err(503, "keychain_locked", "The tenant keychain is locked.") from None
|
| 87 |
+
try:
|
| 88 |
+
event_bus_name = _partner_bus(fields, result)
|
| 89 |
+
except client.AmazonError:
|
| 90 |
+
# Subscription IDs remain durable if the subsequent destination read fails.
|
| 91 |
+
event_bus_name = None
|
| 92 |
+
|
| 93 |
+
def save(bucket):
|
| 94 |
+
entry = (bucket.get("entries") or {}).get(row["id"])
|
| 95 |
+
if (not isinstance(entry, dict) or entry.get("type") != "amazon_sp"
|
| 96 |
+
or keychain.connection_id_of(entry, row["id"]) != connection_id):
|
| 97 |
+
raise err(409, "amazon_connection_changed", "The Amazon connection changed during setup.")
|
| 98 |
+
previous = entry.get(SUBSCRIPTIONS_FIELD) or {}
|
| 99 |
+
subscriptions = dict(previous.get("subscriptions") or {})
|
| 100 |
+
# Preserve prior IDs when a destination-only pass skips all subscription types.
|
| 101 |
+
for kind, state in result["subscriptions"].items():
|
| 102 |
+
if state.get("subscriptionId"):
|
| 103 |
+
subscriptions[kind] = dict(state)
|
| 104 |
+
entry[SUBSCRIPTIONS_FIELD] = {
|
| 105 |
+
"connectionId": connection_id, "destinationId": result["destinationId"],
|
| 106 |
+
"eventBusName": event_bus_name, "subscriptions": subscriptions}
|
| 107 |
+
return bucket
|
| 108 |
+
|
| 109 |
+
try:
|
| 110 |
+
rt.update(keychain.KEY, save)
|
| 111 |
+
except HTTPException:
|
| 112 |
+
raise
|
| 113 |
+
except Exception:
|
| 114 |
+
raise err(503, "store_unavailable", "Amazon subscription IDs could not be saved.") from None
|
| 115 |
+
if not event_bus_name:
|
| 116 |
+
raise err(503, "amazon_partner_bus_name_missing",
|
| 117 |
+
"Amazon did not return the partner event bus name. Subscription IDs were saved.")
|
| 118 |
+
return {**result, "connectionId": connection_id, "eventBusName": event_bus_name}
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def tenant_for_subscription(subscription_id):
|
| 122 |
+
"""Resolve one live tenant connection by subscription metadata without decrypting credentials."""
|
| 123 |
+
from core import keychain
|
| 124 |
+
from harness.runtime import get_runtime, known_tenants
|
| 125 |
+
|
| 126 |
+
matches = []
|
| 127 |
+
for tenant in known_tenants():
|
| 128 |
+
rt = get_runtime(tenant)
|
| 129 |
+
bucket = rt.get(keychain.KEY, fresh=True) or {}
|
| 130 |
+
for entry_id, entry in (bucket.get("entries") or {}).items():
|
| 131 |
+
if not isinstance(entry, dict) or entry.get("type") != "amazon_sp":
|
| 132 |
+
continue
|
| 133 |
+
connection_id = keychain.connection_id_of(entry, entry_id)
|
| 134 |
+
if keychain._row_for(rt, connection_id, "amazon_sp") is None:
|
| 135 |
+
continue
|
| 136 |
+
states = (entry.get(SUBSCRIPTIONS_FIELD) or {}).get("subscriptions") or {}
|
| 137 |
+
if any(state.get("subscriptionId") == subscription_id for state in states.values()):
|
| 138 |
+
matches.append(rt)
|
| 139 |
+
if len(matches) > 1:
|
| 140 |
+
raise err(409, "amazon_subscription_ambiguous", "The subscription belongs to multiple connections.")
|
| 141 |
+
return matches[0] if matches else None
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
@router.post("/connectors/amazon_sp/events")
|
| 145 |
+
async def listing_event(request: Request):
|
| 146 |
+
"""Record listing attributes only after signature, type and subscription admission."""
|
| 147 |
+
from harness import amazon_listing_changes
|
| 148 |
+
|
| 149 |
+
raw = await request.body()
|
| 150 |
+
valid, status, reason = verify_event(request.headers, raw)
|
| 151 |
+
if not valid:
|
| 152 |
+
raise err(status, reason, "Amazon event authentication failed.")
|
| 153 |
+
try:
|
| 154 |
+
envelope = json.loads(raw)
|
| 155 |
+
detail = envelope.get("detail") if isinstance(envelope, dict) else None
|
| 156 |
+
if not isinstance(detail, dict):
|
| 157 |
+
raise ValueError
|
| 158 |
+
except (ValueError, UnicodeDecodeError):
|
| 159 |
+
raise err(422, "amazon_event_envelope_invalid", "A full EventBridge envelope is required.") from None
|
| 160 |
+
if detail.get("NotificationType") != NOTIFICATION_TYPE:
|
| 161 |
+
raise err(422, "amazon_notification_type_unsupported", "Only BRANDED_ITEM_CONTENT_CHANGE is supported.")
|
| 162 |
+
metadata, payload = detail.get("NotificationMetadata"), detail.get("Payload")
|
| 163 |
+
if (not isinstance(metadata, dict) or not isinstance(payload, dict)
|
| 164 |
+
or not isinstance(metadata.get("SubscriptionId"), str) or not metadata["SubscriptionId"]):
|
| 165 |
+
raise err(422, "amazon_event_payload_invalid", "Notification metadata and listing payload are required.")
|
| 166 |
+
try:
|
| 167 |
+
rt = tenant_for_subscription(metadata["SubscriptionId"])
|
| 168 |
+
if rt is None:
|
| 169 |
+
raise err(404, "amazon_subscription_unknown", "The Amazon notification subscription is not registered.")
|
| 170 |
+
count = amazon_listing_changes.record(rt, {
|
| 171 |
+
"asin": payload.get("Asin"), "marketplaceId": payload.get("MarketplaceId"),
|
| 172 |
+
"attributesChanged": payload.get("AttributesChanged"), "source": "notification",
|
| 173 |
+
"notificationId": metadata.get("NotificationId"), "publishedAt": metadata.get("PublishTime")})
|
| 174 |
+
except HTTPException:
|
| 175 |
+
raise
|
| 176 |
+
except ValueError:
|
| 177 |
+
raise err(422, "amazon_listing_change_invalid", "The listing change fields are incomplete or invalid.") from None
|
| 178 |
+
except Exception:
|
| 179 |
+
raise err(503, "store_unavailable", "The Amazon listing change could not be recorded.") from None
|
| 180 |
+
return {"ok": True, "recorded": count}
|
api/routes_amazon_projects.py
CHANGED
|
@@ -15,10 +15,12 @@ from typing import Any
|
|
| 15 |
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
| 16 |
|
| 17 |
from core import amazon_projects as projects
|
|
|
|
| 18 |
from core import amazon_project_results as project_results
|
| 19 |
-
from core import users
|
| 20 |
from deps import Session, err, require_session
|
| 21 |
from modules import amazon_profit
|
|
|
|
| 22 |
from modules import sku_action_results
|
| 23 |
import routes_amazon_profit
|
| 24 |
import routes_sku_actions
|
|
@@ -32,7 +34,7 @@ NOT_FOUND = "That project does not exist."
|
|
| 32 |
#: Contract field name to store field.
|
| 33 |
FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": "owner",
|
| 34 |
"startOn": "start_on", "targetOn": "target_on", "progress": "progress", "notes": "notes",
|
| 35 |
-
"completedOn": "completed_on", "resultConfigs": "result_configs"}
|
| 36 |
#: Project JSON keys a client may echo back. They are ignored, never written.
|
| 37 |
READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
|
| 38 |
"rollup", "impact", "via"))
|
|
@@ -49,6 +51,8 @@ def _call(fn, *args, **kwargs):
|
|
| 49 |
raise err(503, "projects_store_unavailable", "Projects need the workspace database.") from None
|
| 50 |
except ValueError as exc:
|
| 51 |
raise err(400, "invalid_project", str(exc)) from None
|
|
|
|
|
|
|
| 52 |
except Exception as exc: # noqa: BLE001
|
| 53 |
print(f"[amazon-projects] {getattr(fn, '__name__', fn)} failed: {type(exc).__name__}: {exc}",
|
| 54 |
file=sys.stderr)
|
|
@@ -118,7 +122,7 @@ def _project(record, today):
|
|
| 118 |
"targetCount": record.get("target_count", 0), "createdAt": record["created_at"],
|
| 119 |
"updatedAt": record["updated_at"], "createdBy": record["created_by"],
|
| 120 |
"rollup": record.get("rollup"), "impact": impact or None,
|
| 121 |
-
"resultConfigs": record.get("result_configs", [])}
|
| 122 |
|
| 123 |
|
| 124 |
def _item(grain, key, listing, pictures):
|
|
@@ -158,6 +162,20 @@ def _owner_candidates(session):
|
|
| 158 |
|
| 159 |
def _owner_fields(body, session, values, previous=None):
|
| 160 |
"""Resolve new assignments against the current tenant roster on every write."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
if "ownerUsername" not in body:
|
| 162 |
if values.get("owner") is not None:
|
| 163 |
old = (previous or {}).get("owner")
|
|
@@ -175,7 +193,7 @@ def _owner_fields(body, session, values, previous=None):
|
|
| 175 |
old = projects.owner_identity((previous or {}).get("owner"), session.tenant)
|
| 176 |
if old and old["username"] == username:
|
| 177 |
return {**values, "owner": previous["owner"]}
|
| 178 |
-
raise err(400, "invalid_project", "
|
| 179 |
return {**values, "owner": _call(projects.owner_value, session.tenant, identity)}
|
| 180 |
|
| 181 |
|
|
@@ -496,14 +514,14 @@ def row_projects(grain: str | None = None, key: str | None = None,
|
|
| 496 |
today = projects._today()
|
| 497 |
if grain is None and key is None:
|
| 498 |
records = _call(projects.list_projects, runtime)
|
| 499 |
-
return {"grain": None, "key": None, "canEdit": can_edit,
|
| 500 |
"projects": [_project(r, today) for r in records], "summary": projects.summary(records)}
|
| 501 |
pair = _pair(grain, key)
|
| 502 |
above, cause = _mirror(runtime, lambda cur: projects.lineage(cur, *pair), [])
|
| 503 |
pairs = [pair] + [p for p in above if p != pair]
|
| 504 |
listed = _distinct(_call(projects.projects_for, runtime, pairs), pairs)
|
| 505 |
out = [{**_project(record, today), "via": {"grain": via[0], "key": via[1]}} for via, record in listed]
|
| 506 |
-
return _with_cause({"grain": pair[0], "key": pair[1], "canEdit": can_edit, "projects": out,
|
| 507 |
"summary": projects.summary([record for _via, record in listed])}, cause)
|
| 508 |
|
| 509 |
|
|
@@ -545,12 +563,53 @@ def one_project(project_id: str, session: Session = Depends(require_session)):
|
|
| 545 |
runtime = _reader(session)
|
| 546 |
record = _call(projects.get, runtime, project_id)
|
| 547 |
targets, cause = _targets(runtime, project_id)
|
| 548 |
-
activity =
|
| 549 |
return _with_cause({"canEdit": bool(routes_amazon_profit._may_edit(session)),
|
| 550 |
"project": _project(record, projects._today()), "targets": targets,
|
| 551 |
"activity": activity}, cause)
|
| 552 |
|
| 553 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 554 |
@router.get("/{project_id}/results")
|
| 555 |
def project_result_figures(project_id: str, session: Session = Depends(require_session)):
|
| 556 |
"""Serve current target results only after effective grants and independent total validation."""
|
|
@@ -606,6 +665,7 @@ def project_result_series(project_id: str, start: str | None = Query(None, alias
|
|
| 606 |
available = _result_sources(session, targets + selected_targets)
|
| 607 |
con = routes_sku_actions._results_mirror(runtime)
|
| 608 |
reason = None
|
|
|
|
| 609 |
try:
|
| 610 |
skus = projects.resolve_target_skus(con, targets)
|
| 611 |
if selected:
|
|
@@ -617,6 +677,7 @@ def project_result_series(project_id: str, start: str | None = Query(None, alias
|
|
| 617 |
reason = "Today is incomplete. No completed days are available in this range."
|
| 618 |
except projects.MirrorUnreadable as exc:
|
| 619 |
skus, reason = [], str(exc)
|
|
|
|
| 620 |
figures = {}
|
| 621 |
|
| 622 |
def read(key):
|
|
@@ -627,8 +688,10 @@ def project_result_series(project_id: str, start: str | None = Query(None, alias
|
|
| 627 |
figures[metric] = _result_figure(con, {"metric": metric}, bounds, skus, available)
|
| 628 |
return _series_figure(key, figures.get(metric, {}), bounds)
|
| 629 |
|
| 630 |
-
|
| 631 |
independent=lambda key: _series_economics_total(con, key, bounds, skus, available))
|
|
|
|
|
|
|
| 632 |
except HTTPException:
|
| 633 |
raise
|
| 634 |
except (project_results.ValidationFailed, sku_action_results.ValidationFailed):
|
|
@@ -681,23 +744,263 @@ def _target_answer(runtime, project_id, result):
|
|
| 681 |
return _with_cause({"project": _project(result["project"], projects._today()), "targets": targets}, cause)
|
| 682 |
|
| 683 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 684 |
@router.post("/{project_id}/targets")
|
| 685 |
def add_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 686 |
"""Assign the project to more SKUs, ASINs or Parent ASINs."""
|
| 687 |
runtime = _writer(session)
|
| 688 |
-
|
| 689 |
-
|
|
|
|
|
|
|
| 690 |
_repaint(runtime, pairs)
|
| 691 |
-
return _target_answer(runtime, project_id, result)
|
| 692 |
|
| 693 |
|
| 694 |
@router.post("/{project_id}/targets/remove")
|
| 695 |
def remove_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 696 |
-
"""Unassign
|
| 697 |
runtime = _writer(session)
|
| 698 |
if not isinstance(body, dict):
|
| 699 |
raise err(400, "invalid_target", "Name the grain and key to remove.")
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
|
|
|
| 703 |
return _target_answer(runtime, project_id, result)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
| 16 |
|
| 17 |
from core import amazon_projects as projects
|
| 18 |
+
from core import amazon_project_tasks as project_tasks
|
| 19 |
from core import amazon_project_results as project_results
|
| 20 |
+
from core import perm_scope, users
|
| 21 |
from deps import Session, err, require_session
|
| 22 |
from modules import amazon_profit
|
| 23 |
+
from modules import amazon_project_filters as project_filters
|
| 24 |
from modules import sku_action_results
|
| 25 |
import routes_amazon_profit
|
| 26 |
import routes_sku_actions
|
|
|
|
| 34 |
#: Contract field name to store field.
|
| 35 |
FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": "owner",
|
| 36 |
"startOn": "start_on", "targetOn": "target_on", "progress": "progress", "notes": "notes",
|
| 37 |
+
"completedOn": "completed_on", "resultConfigs": "result_configs", "contributors": "contributors"}
|
| 38 |
#: Project JSON keys a client may echo back. They are ignored, never written.
|
| 39 |
READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
|
| 40 |
"rollup", "impact", "via"))
|
|
|
|
| 51 |
raise err(503, "projects_store_unavailable", "Projects need the workspace database.") from None
|
| 52 |
except ValueError as exc:
|
| 53 |
raise err(400, "invalid_project", str(exc)) from None
|
| 54 |
+
except perm_scope.Denied as exc:
|
| 55 |
+
raise err(403, "forbidden", str(exc)) from None
|
| 56 |
except Exception as exc: # noqa: BLE001
|
| 57 |
print(f"[amazon-projects] {getattr(fn, '__name__', fn)} failed: {type(exc).__name__}: {exc}",
|
| 58 |
file=sys.stderr)
|
|
|
|
| 122 |
"targetCount": record.get("target_count", 0), "createdAt": record["created_at"],
|
| 123 |
"updatedAt": record["updated_at"], "createdBy": record["created_by"],
|
| 124 |
"rollup": record.get("rollup"), "impact": impact or None,
|
| 125 |
+
"resultConfigs": record.get("result_configs", []), "contributors": record.get("contributors", [])}
|
| 126 |
|
| 127 |
|
| 128 |
def _item(grain, key, listing, pictures):
|
|
|
|
| 162 |
|
| 163 |
def _owner_fields(body, session, values, previous=None):
|
| 164 |
"""Resolve new assignments against the current tenant roster on every write."""
|
| 165 |
+
if "contributors" in body:
|
| 166 |
+
if not isinstance(body["contributors"], list):
|
| 167 |
+
raise err(400, "invalid_project", "Send contributors as a list of workspace usernames.")
|
| 168 |
+
candidates = {item["username"]: item for item in _owner_candidates(session)}
|
| 169 |
+
members = []
|
| 170 |
+
for username in body["contributors"]:
|
| 171 |
+
if not isinstance(username, str) or not username.strip():
|
| 172 |
+
raise err(400, "invalid_project", "Each contributor must be a workspace username.")
|
| 173 |
+
username = username.strip().lower()
|
| 174 |
+
identity = candidates.get(username)
|
| 175 |
+
if identity is None:
|
| 176 |
+
raise err(400, "invalid_project", f"Workspace user '{username}' cannot contribute to this project.")
|
| 177 |
+
members.append({"tenant": session.tenant, "username": identity["username"], "name": identity["name"]})
|
| 178 |
+
values = {**values, "contributors": members}
|
| 179 |
if "ownerUsername" not in body:
|
| 180 |
if values.get("owner") is not None:
|
| 181 |
old = (previous or {}).get("owner")
|
|
|
|
| 193 |
old = projects.owner_identity((previous or {}).get("owner"), session.tenant)
|
| 194 |
if old and old["username"] == username:
|
| 195 |
return {**values, "owner": previous["owner"]}
|
| 196 |
+
raise err(400, "invalid_project", f"Workspace user '{username}' cannot own this project.")
|
| 197 |
return {**values, "owner": _call(projects.owner_value, session.tenant, identity)}
|
| 198 |
|
| 199 |
|
|
|
|
| 514 |
today = projects._today()
|
| 515 |
if grain is None and key is None:
|
| 516 |
records = _call(projects.list_projects, runtime)
|
| 517 |
+
return {"grain": None, "key": None, "canEdit": can_edit, "today": today.isoformat(),
|
| 518 |
"projects": [_project(r, today) for r in records], "summary": projects.summary(records)}
|
| 519 |
pair = _pair(grain, key)
|
| 520 |
above, cause = _mirror(runtime, lambda cur: projects.lineage(cur, *pair), [])
|
| 521 |
pairs = [pair] + [p for p in above if p != pair]
|
| 522 |
listed = _distinct(_call(projects.projects_for, runtime, pairs), pairs)
|
| 523 |
out = [{**_project(record, today), "via": {"grain": via[0], "key": via[1]}} for via, record in listed]
|
| 524 |
+
return _with_cause({"grain": pair[0], "key": pair[1], "canEdit": can_edit, "today": today.isoformat(), "projects": out,
|
| 525 |
"summary": projects.summary([record for _via, record in listed])}, cause)
|
| 526 |
|
| 527 |
|
|
|
|
| 563 |
runtime = _reader(session)
|
| 564 |
record = _call(projects.get, runtime, project_id)
|
| 565 |
targets, cause = _targets(runtime, project_id)
|
| 566 |
+
activity = _activity_page(session, project_id)["activity"]
|
| 567 |
return _with_cause({"canEdit": bool(routes_amazon_profit._may_edit(session)),
|
| 568 |
"project": _project(record, projects._today()), "targets": targets,
|
| 569 |
"activity": activity}, cause)
|
| 570 |
|
| 571 |
|
| 572 |
+
def _changes_for(runtime, pairs, since):
|
| 573 |
+
"""This tenant's listing changes for the project targets, empty when the reader cannot answer."""
|
| 574 |
+
from harness import amazon_listing_changes
|
| 575 |
+
try:
|
| 576 |
+
return amazon_listing_changes.for_runtime(runtime).changes_for(pairs, since)
|
| 577 |
+
except Exception as exc: # noqa: BLE001
|
| 578 |
+
# An absent or unreadable mirror omits Amazon rows and keeps app activity readable.
|
| 579 |
+
print(f"[amazon-projects] listing changes unavailable: {type(exc).__name__}: {exc}", file=sys.stderr)
|
| 580 |
+
return []
|
| 581 |
+
|
| 582 |
+
|
| 583 |
+
def _activity_page(session, project_id, cursor=None):
|
| 584 |
+
"""Use one session-bound cursor and the same activity normalization for both project doors."""
|
| 585 |
+
from routes_tables import _activity_cursor
|
| 586 |
+
runtime = session.runtime
|
| 587 |
+
pairs = _call(projects.targets, runtime, project_id)
|
| 588 |
+
query = {"amazonProject": project_id, "targets": pairs}
|
| 589 |
+
position = _activity_cursor(session, query, token=cursor) if cursor is not None else None
|
| 590 |
+
actors = {username: str(record.get("name") or username)
|
| 591 |
+
for username, record in users._tenant_accounts(session.tenant)}
|
| 592 |
+
tasks = _call(project_tasks.list_tasks, runtime, project_id)
|
| 593 |
+
page = _call(projects.activity_page, runtime, project_id, cursor=position,
|
| 594 |
+
actor_names=actors, task_names={task["id"]: task["name"] for task in tasks},
|
| 595 |
+
read_changes=lambda since: _changes_for(runtime, pairs, since))
|
| 596 |
+
for entry in page["activity"]:
|
| 597 |
+
field = entry.get("field") or ""
|
| 598 |
+
if field == "Done by filter" or field.endswith("no longer meets the filter"):
|
| 599 |
+
entry.update(action=field, field="tasks")
|
| 600 |
+
page["nextCursor"] = _activity_cursor(session, query, position=page.pop("next_cursor"))
|
| 601 |
+
return page
|
| 602 |
+
|
| 603 |
+
|
| 604 |
+
@router.get("/{project_id}/activity")
|
| 605 |
+
def project_activity(project_id: str, cursor: str | None = Query(None),
|
| 606 |
+
session: Session = Depends(require_session)):
|
| 607 |
+
"""Page every project activity entry newest first through the Command Center read permission."""
|
| 608 |
+
runtime = _reader(session)
|
| 609 |
+
_call(projects.get, runtime, project_id)
|
| 610 |
+
return _activity_page(session, project_id, cursor)
|
| 611 |
+
|
| 612 |
+
|
| 613 |
@router.get("/{project_id}/results")
|
| 614 |
def project_result_figures(project_id: str, session: Session = Depends(require_session)):
|
| 615 |
"""Serve current target results only after effective grants and independent total validation."""
|
|
|
|
| 665 |
available = _result_sources(session, targets + selected_targets)
|
| 666 |
con = routes_sku_actions._results_mirror(runtime)
|
| 667 |
reason = None
|
| 668 |
+
sku_count_reason = None
|
| 669 |
try:
|
| 670 |
skus = projects.resolve_target_skus(con, targets)
|
| 671 |
if selected:
|
|
|
|
| 677 |
reason = "Today is incomplete. No completed days are available in this range."
|
| 678 |
except projects.MirrorUnreadable as exc:
|
| 679 |
skus, reason = [], str(exc)
|
| 680 |
+
sku_count_reason = reason
|
| 681 |
figures = {}
|
| 682 |
|
| 683 |
def read(key):
|
|
|
|
| 688 |
figures[metric] = _result_figure(con, {"metric": metric}, bounds, skus, available)
|
| 689 |
return _series_figure(key, figures.get(metric, {}), bounds)
|
| 690 |
|
| 691 |
+
answer = project_results.series(record, bounds=bounds, read=read,
|
| 692 |
independent=lambda key: _series_economics_total(con, key, bounds, skus, available))
|
| 693 |
+
return {**answer, "skuCount": len(skus) if sku_count_reason is None else None,
|
| 694 |
+
"skuCountReason": sku_count_reason}
|
| 695 |
except HTTPException:
|
| 696 |
raise
|
| 697 |
except (project_results.ValidationFailed, sku_action_results.ValidationFailed):
|
|
|
|
| 744 |
return _with_cause({"project": _project(result["project"], projects._today()), "targets": targets}, cause)
|
| 745 |
|
| 746 |
|
| 747 |
+
def _matching_targets(session, body):
|
| 748 |
+
"""Resolve the whole permitted filter population before selecting a response page."""
|
| 749 |
+
if not isinstance(body, dict) or "filter" not in body:
|
| 750 |
+
raise err(400, "invalid_target", "Send a grain and filter tree.")
|
| 751 |
+
grain = body.get("grain")
|
| 752 |
+
key = _call(project_filters.table_key, grain)
|
| 753 |
+
session.require(key)
|
| 754 |
+
try:
|
| 755 |
+
keys = project_filters.matching_keys(_cells_store(session.runtime), grain, body["filter"],
|
| 756 |
+
user=session.user)
|
| 757 |
+
except perm_scope.Denied:
|
| 758 |
+
raise err(403, "forbidden", f"Your account cannot read {key}.") from None
|
| 759 |
+
except perm_scope.UnknownTable:
|
| 760 |
+
raise err(503, "project_targets_unavailable", f"Target table {key} is unavailable.") from None
|
| 761 |
+
except perm_scope.Unresolvable as exc:
|
| 762 |
+
raise err(503, "project_targets_unavailable",
|
| 763 |
+
f"Cannot read {key}: {exc.cause} {exc.recommendation}") from None
|
| 764 |
+
except ValueError as exc:
|
| 765 |
+
raise err(400, "invalid_target", str(exc)) from None
|
| 766 |
+
except Exception as exc: # noqa: BLE001
|
| 767 |
+
print(f"[amazon-projects] filter read failed: {type(exc).__name__}", file=sys.stderr)
|
| 768 |
+
raise err(503, "project_targets_unavailable", f"Target table {key} could not be read. Try again shortly.") from None
|
| 769 |
+
return [(grain, key) for key in keys]
|
| 770 |
+
|
| 771 |
+
|
| 772 |
+
def _require_target_grains(session, pairs):
|
| 773 |
+
for grain in dict.fromkeys(grain for grain, _key in pairs):
|
| 774 |
+
session.require(project_filters.table_key(grain))
|
| 775 |
+
|
| 776 |
+
|
| 777 |
+
def _remove_target_pairs(body):
|
| 778 |
+
return _pairs(body["targets"]) if "targets" in body else [_pair(body.get("grain"), body.get("key"))]
|
| 779 |
+
|
| 780 |
+
|
| 781 |
+
def _add_target_pairs(session, body):
|
| 782 |
+
"""Keep valid explicit targets and return each malformed target's refusal."""
|
| 783 |
+
if isinstance(body, dict) and "filter" in body:
|
| 784 |
+
if "targets" in body:
|
| 785 |
+
raise err(400, "invalid_target", "Send either targets or a grain and filter tree.")
|
| 786 |
+
return _matching_targets(session, body), []
|
| 787 |
+
values = body.get("targets") if isinstance(body, dict) else None
|
| 788 |
+
if not isinstance(values, list) or not values:
|
| 789 |
+
raise err(400, "invalid_target", "Send targets as a list of grain and key.")
|
| 790 |
+
pairs, refused = [], []
|
| 791 |
+
for value in values:
|
| 792 |
+
if not isinstance(value, dict):
|
| 793 |
+
refused.append({"key": None, "reason": "Send each target as a grain and key."})
|
| 794 |
+
continue
|
| 795 |
+
try:
|
| 796 |
+
pairs.append(projects.target(value.get("grain"), value.get("key")))
|
| 797 |
+
except ValueError as exc:
|
| 798 |
+
refused.append({"key": value.get("key"), "reason": str(exc)})
|
| 799 |
+
_require_target_grains(session, pairs)
|
| 800 |
+
return pairs, refused
|
| 801 |
+
|
| 802 |
+
|
| 803 |
+
@router.post("/{project_id}/targets/match")
|
| 804 |
+
def match_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 805 |
+
"""Count every permitted filter match and return one page of its keys."""
|
| 806 |
+
runtime = _reader(session)
|
| 807 |
+
_call(projects.get, runtime, project_id)
|
| 808 |
+
limit = body.get("limit", project_filters.MATCH_PAGE) if isinstance(body, dict) else project_filters.MATCH_PAGE
|
| 809 |
+
offset = body.get("offset", 0) if isinstance(body, dict) else 0
|
| 810 |
+
if type(limit) is not int or limit < 1 or type(offset) is not int or offset < 0:
|
| 811 |
+
raise err(400, "invalid_target", "Use a positive whole limit and a nonnegative whole offset.")
|
| 812 |
+
pairs = _matching_targets(session, body)
|
| 813 |
+
return {"grain": body["grain"], "total": len(pairs), "keys": [key for _grain, key in pairs[offset:offset + limit]]}
|
| 814 |
+
|
| 815 |
+
|
| 816 |
@router.post("/{project_id}/targets")
|
| 817 |
def add_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 818 |
"""Assign the project to more SKUs, ASINs or Parent ASINs."""
|
| 819 |
runtime = _writer(session)
|
| 820 |
+
_call(projects.get, runtime, project_id)
|
| 821 |
+
pairs, refused = _add_target_pairs(session, body)
|
| 822 |
+
result = (_call(projects.add_targets, runtime, project_id, pairs, by=_by(session)) if pairs
|
| 823 |
+
else {"project": _call(projects.get, runtime, project_id)})
|
| 824 |
_repaint(runtime, pairs)
|
| 825 |
+
return {**_target_answer(runtime, project_id, result), "refused": refused}
|
| 826 |
|
| 827 |
|
| 828 |
@router.post("/{project_id}/targets/remove")
|
| 829 |
def remove_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 830 |
+
"""Unassign a target list or the legacy grain and key pair from the project."""
|
| 831 |
runtime = _writer(session)
|
| 832 |
if not isinstance(body, dict):
|
| 833 |
raise err(400, "invalid_target", "Name the grain and key to remove.")
|
| 834 |
+
pairs = _remove_target_pairs(body)
|
| 835 |
+
_require_target_grains(session, pairs)
|
| 836 |
+
result = _call(projects.remove_targets, runtime, project_id, pairs, by=_by(session))
|
| 837 |
+
_repaint(runtime, pairs)
|
| 838 |
return _target_answer(runtime, project_id, result)
|
| 839 |
+
|
| 840 |
+
|
| 841 |
+
# --------------------------------------------------------------------------- tasks and dependencies
|
| 842 |
+
|
| 843 |
+
TASK_FIELDS = {"name": "name", "assignee": "assignee", "startOn": "start_on", "dueOn": "due_on",
|
| 844 |
+
"done": "done", "contingencyFor": "contingency_for", "parentId": "parent_id",
|
| 845 |
+
"position": "position", "targets": "targets", "targetFilter": "target_filter",
|
| 846 |
+
"completionFilter": "completion_filter"}
|
| 847 |
+
TASK_EDITS = frozenset(TASK_FIELDS) - {"parentId", "position"}
|
| 848 |
+
DEPENDENCY_FIELDS = {"fromTask": "from_task", "toTask": "to_task", "type": "type", "lagDays": "lag_days"}
|
| 849 |
+
|
| 850 |
+
|
| 851 |
+
def _task(record):
|
| 852 |
+
result = {"id": record["id"], "projectId": record["project_id"], "parentId": record["parent_id"],
|
| 853 |
+
"position": record["position"], "name": record["name"], "assignee": record["assignee"],
|
| 854 |
+
"startOn": record["start_on"], "dueOn": record["due_on"], "done": record["done"],
|
| 855 |
+
"doneAt": record["done_at"], "contingencyFor": record["contingency_for"],
|
| 856 |
+
"active": record["active"], "createdBy": record["created_by"]}
|
| 857 |
+
if any(field in record for field in project_tasks.TARGET_FIELDS):
|
| 858 |
+
result.update(targets=record.get("targets"), targetFilter=record.get("target_filter"),
|
| 859 |
+
completionFilter=record.get("completion_filter"), filterStatus=record.get("filter_status"),
|
| 860 |
+
filterClosedAt=record.get("filter_closed_at"))
|
| 861 |
+
return result
|
| 862 |
+
|
| 863 |
+
|
| 864 |
+
def _task_filters(session, record):
|
| 865 |
+
"""Use current source permissions before saving task targets or completion conditions."""
|
| 866 |
+
pairs = record.get("targets") or []
|
| 867 |
+
grains = list(dict.fromkeys(pair["grain"] for pair in pairs))
|
| 868 |
+
saved = record.get("target_filter")
|
| 869 |
+
if saved is not None:
|
| 870 |
+
grains.append(saved["grain"])
|
| 871 |
+
for grain in dict.fromkeys(grains):
|
| 872 |
+
session.require(project_filters.table_key(grain))
|
| 873 |
+
for tree in ([saved["filter"]] if saved and grain == saved["grain"] else []) + (
|
| 874 |
+
[record["completion_filter"]] if record.get("completion_filter") is not None else []):
|
| 875 |
+
_call(project_filters.check_task_filter, _cells_store(session.runtime), grain, tree, user=session.user)
|
| 876 |
+
|
| 877 |
+
|
| 878 |
+
def _task_matcher(session):
|
| 879 |
+
def match(_runtime, grain, tree, *, user):
|
| 880 |
+
key = project_filters.table_key(grain)
|
| 881 |
+
session.require(key)
|
| 882 |
+
st = _cells_store(session.runtime)
|
| 883 |
+
project_filters.validate_task_tree(tree, perm_scope.scoped_fields(session.user, key, st=st))
|
| 884 |
+
return project_filters.matching_keys(st, grain, tree, user=user)
|
| 885 |
+
return match
|
| 886 |
+
|
| 887 |
+
|
| 888 |
+
def _dependency(record):
|
| 889 |
+
return {"id": record["id"], "projectId": record["project_id"], "fromTask": record["from_task"],
|
| 890 |
+
"toTask": record["to_task"], "type": record["type"], "lagDays": record["lag_days"]}
|
| 891 |
+
|
| 892 |
+
|
| 893 |
+
def _planning_fields(body, fields, kind, required=()):
|
| 894 |
+
if not isinstance(body, dict) or not body:
|
| 895 |
+
raise err(400, f"invalid_{kind}", f"Send the {kind} fields as an object.")
|
| 896 |
+
unknown = sorted(set(body) - set(fields))
|
| 897 |
+
if unknown:
|
| 898 |
+
raise err(400, f"invalid_{kind}", f"Unknown {kind} fields: {', '.join(unknown)}.")
|
| 899 |
+
if set(required) - set(body):
|
| 900 |
+
raise err(400, f"invalid_{kind}", f"Send these {kind} fields: {', '.join(required)}.")
|
| 901 |
+
return {fields[key]: value for key, value in body.items()}
|
| 902 |
+
|
| 903 |
+
|
| 904 |
+
def _task_answer(runtime, project_id, record, shifted=()):
|
| 905 |
+
# Task writes commit leaf progress before the target cells read it.
|
| 906 |
+
project = _call(projects.get, runtime, project_id)
|
| 907 |
+
_repaint(runtime, _target_pairs(runtime, project_id))
|
| 908 |
+
return {"task": _task(record), "shifted": [_task(row) for row in shifted],
|
| 909 |
+
"project": _project(project, projects._today())}
|
| 910 |
+
|
| 911 |
+
|
| 912 |
+
@router.get("/{project_id}/tasks")
|
| 913 |
+
def list_project_tasks(project_id: str, session: Session = Depends(require_session)):
|
| 914 |
+
"""Read every task as a flat row and every dependency in this tenant's project."""
|
| 915 |
+
runtime = _reader(session)
|
| 916 |
+
return {"tasks": [_task(row) for row in _call(project_tasks.list_tasks, runtime, project_id,
|
| 917 |
+
matcher=_task_matcher(session), today=projects._today(),
|
| 918 |
+
by=amazon_profit.WRITER)],
|
| 919 |
+
"dependencies": [_dependency(row) for row in
|
| 920 |
+
_call(project_tasks.list_dependencies, runtime, project_id)]}
|
| 921 |
+
|
| 922 |
+
|
| 923 |
+
@router.post("/{project_id}/tasks")
|
| 924 |
+
def create_project_task(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 925 |
+
"""Create a task under an optional parent with one assignee and its own dates."""
|
| 926 |
+
runtime = _writer(session)
|
| 927 |
+
values = _planning_fields(body, TASK_FIELDS, "task", required=("name",))
|
| 928 |
+
validated = _call(project_tasks._fields, values)
|
| 929 |
+
_task_filters(session, validated)
|
| 930 |
+
record = _call(project_tasks.create, runtime, project_id, by=_by(session), **values)
|
| 931 |
+
return _task_answer(runtime, project_id, record)
|
| 932 |
+
|
| 933 |
+
|
| 934 |
+
@router.patch("/{project_id}/tasks/{task_id}")
|
| 935 |
+
def edit_project_task(project_id: str, task_id: str, body: Any = Body(default=None),
|
| 936 |
+
session: Session = Depends(require_session)):
|
| 937 |
+
"""Reschedule dates first and retain every shifted dependent through the other field edits."""
|
| 938 |
+
runtime = _writer(session)
|
| 939 |
+
values = _planning_fields(body, {key: TASK_FIELDS[key] for key in TASK_EDITS}, "task")
|
| 940 |
+
# Reschedule accepts only a tenant task id, so bind it to the URL's project first.
|
| 941 |
+
previous = _call(project_tasks.get, runtime, project_id, task_id)
|
| 942 |
+
validated = _call(project_tasks._fields, values, previous)
|
| 943 |
+
_task_filters(session, validated)
|
| 944 |
+
if "contingency_for" in values:
|
| 945 |
+
rows = {row["id"]: row for row in _call(project_tasks.list_tasks, runtime, project_id)}
|
| 946 |
+
_call(project_tasks._check_contingency, rows, task_id, validated["contingency_for"])
|
| 947 |
+
shifted = []
|
| 948 |
+
if {"start_on", "due_on"} & values.keys():
|
| 949 |
+
result = _call(project_tasks.reschedule, runtime, task_id,
|
| 950 |
+
values.pop("start_on", previous["start_on"]),
|
| 951 |
+
values.pop("due_on", previous["due_on"]), _by(session))
|
| 952 |
+
record, shifted = result["task"], result["shifted"]
|
| 953 |
+
else:
|
| 954 |
+
record = previous
|
| 955 |
+
if values:
|
| 956 |
+
record = _call(project_tasks.update, runtime, project_id, task_id, values, by=_by(session))
|
| 957 |
+
return _task_answer(runtime, project_id, record, shifted)
|
| 958 |
+
|
| 959 |
+
|
| 960 |
+
@router.post("/{project_id}/tasks/{task_id}")
|
| 961 |
+
def activate_project_task(project_id: str, task_id: str, session: Session = Depends(require_session)):
|
| 962 |
+
"""Activate a dormant backup through an explicit editor action."""
|
| 963 |
+
runtime = _writer(session)
|
| 964 |
+
record = _call(project_tasks.activate, runtime, project_id, task_id, by=_by(session))
|
| 965 |
+
return _task_answer(runtime, project_id, record)
|
| 966 |
+
|
| 967 |
+
|
| 968 |
+
@router.post("/{project_id}/tasks/{task_id}/move")
|
| 969 |
+
def move_project_task(project_id: str, task_id: str, body: Any = Body(default=None),
|
| 970 |
+
session: Session = Depends(require_session)):
|
| 971 |
+
"""Reparent or reorder a task through the core's ancestor and position checks."""
|
| 972 |
+
runtime = _writer(session)
|
| 973 |
+
values = _planning_fields(body, {key: TASK_FIELDS[key] for key in ("parentId", "position")}, "task")
|
| 974 |
+
record = _call(project_tasks.move, runtime, project_id, task_id, by=_by(session), **values)
|
| 975 |
+
return _task_answer(runtime, project_id, record)
|
| 976 |
+
|
| 977 |
+
|
| 978 |
+
@router.delete("/{project_id}/tasks/{task_id}")
|
| 979 |
+
def delete_project_task(project_id: str, task_id: str, session: Session = Depends(require_session)):
|
| 980 |
+
"""Delete the task and its subtree, then repaint its project's target cells."""
|
| 981 |
+
runtime = _writer(session)
|
| 982 |
+
result = _call(project_tasks.delete, runtime, project_id, task_id, by=_by(session))
|
| 983 |
+
_repaint(runtime, _target_pairs(runtime, project_id))
|
| 984 |
+
return {"ok": True, "id": result["deleted"], "taskIds": result["task_ids"], "progress": result["progress"]}
|
| 985 |
+
|
| 986 |
+
|
| 987 |
+
@router.post("/{project_id}/dependencies")
|
| 988 |
+
def add_project_dependency(project_id: str, body: Any = Body(default=None),
|
| 989 |
+
session: Session = Depends(require_session)):
|
| 990 |
+
"""Link two project tasks with a dependency type and signed day lag."""
|
| 991 |
+
runtime = _writer(session)
|
| 992 |
+
values = _planning_fields(body, DEPENDENCY_FIELDS, "dependency", required=("fromTask", "toTask"))
|
| 993 |
+
for field in ("from_task", "to_task"):
|
| 994 |
+
_call(project_tasks.get, runtime, project_id, values[field])
|
| 995 |
+
record = _call(project_tasks.add_dependency, runtime, project_id, by=_by(session), **values)
|
| 996 |
+
return {"dependency": _dependency(record), "shifted": [_task(row) for row in record["shifted"]]}
|
| 997 |
+
|
| 998 |
+
|
| 999 |
+
@router.delete("/{project_id}/dependencies")
|
| 1000 |
+
def delete_project_dependency(project_id: str, body: Any = Body(default=None),
|
| 1001 |
+
session: Session = Depends(require_session)):
|
| 1002 |
+
"""Unlink a dependency by id within this tenant's project."""
|
| 1003 |
+
runtime = _writer(session)
|
| 1004 |
+
values = _planning_fields(body, {"id": "dependency_id"}, "dependency", required=("id",))
|
| 1005 |
+
record = _call(project_tasks.delete_dependency, runtime, project_id, by=_by(session), **values)
|
| 1006 |
+
return {"ok": True, "id": record["id"]}
|
api/routes_auth.py
CHANGED
|
@@ -75,6 +75,8 @@ def _verify(uname, pw):
|
|
| 75 |
HOTFIX 2026-09-28: a store `PoolTimeout` used to read as "no such account", so a correct
|
| 76 |
password got 401 `invalid_credentials` and was charged to the lockout budget."""
|
| 77 |
try:
|
|
|
|
|
|
|
| 78 |
return users.verify(uname, pw) if (uname and pw) else None
|
| 79 |
except users.RegistryUnavailable as exc:
|
| 80 |
raise _auth_store_down("verify", exc)
|
|
|
|
| 75 |
HOTFIX 2026-09-28: a store `PoolTimeout` used to read as "no such account", so a correct
|
| 76 |
password got 401 `invalid_credentials` and was charged to the lockout budget."""
|
| 77 |
try:
|
| 78 |
+
if uname and pw and not users.store.available():
|
| 79 |
+
raise users.RegistryUnavailable("The account store is unavailable")
|
| 80 |
return users.verify(uname, pw) if (uname and pw) else None
|
| 81 |
except users.RegistryUnavailable as exc:
|
| 82 |
raise _auth_store_down("verify", exc)
|
api/routes_brand.py
CHANGED
|
@@ -35,6 +35,7 @@ from __future__ import annotations
|
|
| 35 |
|
| 36 |
import os
|
| 37 |
import re
|
|
|
|
| 38 |
from pathlib import Path
|
| 39 |
|
| 40 |
from fastapi import APIRouter, Depends
|
|
@@ -49,8 +50,7 @@ router = APIRouter(prefix="/api/v1")
|
|
| 49 |
#: path are refused at the door (400) rather than resolved and then defended against.
|
| 50 |
_KEY_RE = re.compile(r"^[a-z][a-z0-9_]{0,31}$")
|
| 51 |
|
| 52 |
-
#:
|
| 53 |
-
#: with the same name, so a rebrand ships with a deploy and a hard reload, not with a cache bust.
|
| 54 |
#: ⚠ Deliberately the same string as `routes_assets._IMMUTABLE` — two asset doors telling a browser
|
| 55 |
#: two different things about the same class of file would be a defect nobody could see.
|
| 56 |
_IMMUTABLE = "public, max-age=31536000, immutable"
|
|
@@ -86,6 +86,21 @@ def _roots():
|
|
| 86 |
return out
|
| 87 |
|
| 88 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
def brand_path(key):
|
| 90 |
"""The file for a brand key, or None. Pure and importable, so the gate answers this question
|
| 91 |
without a request — the same shape `routes_connectors.directory` uses for the same reason."""
|
|
@@ -97,8 +112,11 @@ def brand_path(key):
|
|
| 97 |
p = root / f"{k}{ext}"
|
| 98 |
# ⚠ `is_file()` and not `exists()`: a DIRECTORY named `instagram.png` would otherwise
|
| 99 |
# resolve and then fail inside FileResponse as a 500.
|
| 100 |
-
|
| 101 |
-
|
|
|
|
|
|
|
|
|
|
| 102 |
return None
|
| 103 |
|
| 104 |
|
|
@@ -111,7 +129,7 @@ def available():
|
|
| 111 |
if not root.is_dir():
|
| 112 |
continue
|
| 113 |
for p in root.iterdir():
|
| 114 |
-
if p.
|
| 115 |
keys.add(p.stem)
|
| 116 |
return sorted(keys)
|
| 117 |
|
|
|
|
| 35 |
|
| 36 |
import os
|
| 37 |
import re
|
| 38 |
+
from xml.etree import ElementTree
|
| 39 |
from pathlib import Path
|
| 40 |
|
| 41 |
from fastapi import APIRouter, Depends
|
|
|
|
| 50 |
#: path are refused at the door (400) rather than resolved and then defended against.
|
| 51 |
_KEY_RE = re.compile(r"^[a-z][a-z0-9_]{0,31}$")
|
| 52 |
|
| 53 |
+
#: The client versions each artwork URL before it can enter this immutable cache.
|
|
|
|
| 54 |
#: ⚠ Deliberately the same string as `routes_assets._IMMUTABLE` — two asset doors telling a browser
|
| 55 |
#: two different things about the same class of file would be a defect nobody could see.
|
| 56 |
_IMMUTABLE = "public, max-age=31536000, immutable"
|
|
|
|
| 86 |
return out
|
| 87 |
|
| 88 |
|
| 89 |
+
def _has_artwork_magic(path):
|
| 90 |
+
"""Accept PNG signatures or an SVG document root, never a Git LFS pointer."""
|
| 91 |
+
try:
|
| 92 |
+
with path.open("rb") as stream:
|
| 93 |
+
if path.suffix == ".png":
|
| 94 |
+
return stream.read(8) == b"\x89PNG\r\n\x1a\n"
|
| 95 |
+
if path.suffix == ".svg":
|
| 96 |
+
return ElementTree.parse(stream).getroot().tag in (
|
| 97 |
+
"svg", "{http://www.w3.org/2000/svg}svg",
|
| 98 |
+
)
|
| 99 |
+
except (OSError, ElementTree.ParseError, ValueError, LookupError):
|
| 100 |
+
return False
|
| 101 |
+
return False
|
| 102 |
+
|
| 103 |
+
|
| 104 |
def brand_path(key):
|
| 105 |
"""The file for a brand key, or None. Pure and importable, so the gate answers this question
|
| 106 |
without a request — the same shape `routes_connectors.directory` uses for the same reason."""
|
|
|
|
| 112 |
p = root / f"{k}{ext}"
|
| 113 |
# ⚠ `is_file()` and not `exists()`: a DIRECTORY named `instagram.png` would otherwise
|
| 114 |
# resolve and then fail inside FileResponse as a 500.
|
| 115 |
+
try:
|
| 116 |
+
if p.is_file() and p.stat().st_size > 129 and _has_artwork_magic(p):
|
| 117 |
+
return p
|
| 118 |
+
except OSError:
|
| 119 |
+
continue
|
| 120 |
return None
|
| 121 |
|
| 122 |
|
|
|
|
| 129 |
if not root.is_dir():
|
| 130 |
continue
|
| 131 |
for p in root.iterdir():
|
| 132 |
+
if p.suffix in _MEDIA and _KEY_RE.match(p.stem) and brand_path(p.stem) is not None:
|
| 133 |
keys.add(p.stem)
|
| 134 |
return sorted(keys)
|
| 135 |
|
api/routes_external.py
CHANGED
|
@@ -14,6 +14,8 @@ import re
|
|
| 14 |
import time
|
| 15 |
import uuid
|
| 16 |
from contextlib import contextmanager
|
|
|
|
|
|
|
| 17 |
from urllib.parse import parse_qs, quote, unquote, urlencode, urlsplit, urlunsplit
|
| 18 |
|
| 19 |
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response
|
|
@@ -49,6 +51,19 @@ _QUERY_MAX_FILTER_SIBLINGS = 50
|
|
| 49 |
_QUERY_KEYS = frozenset({"fields", "filters", "sorts", "limit", "cursor", "aggregates", "groupBy"})
|
| 50 |
|
| 51 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
def _register_query_paging_contract():
|
| 53 |
"""Keep the legacy and grouped query schemas aligned with the API's bounded page reader."""
|
| 54 |
from tool_groups.data import GROUP
|
|
@@ -68,6 +83,7 @@ _VIEW_DRAFT_CONFIG_KEYS = frozenset({
|
|
| 68 |
"filters", "filterConj", "sorts", "groupBy", "visible", "order", "display",
|
| 69 |
"memberPids",
|
| 70 |
})
|
|
|
|
| 71 |
_CONFIRMATION_BUCKET = "external_confirmations"
|
| 72 |
_CONFIRMATION_TTL_SECONDS = 5 * 60
|
| 73 |
_OPERATION_RETENTION_SECONDS = 24 * 60 * 60
|
|
@@ -780,16 +796,21 @@ def _encode_query_cursor(session: Session, query_digest: str,
|
|
| 780 |
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
| 781 |
|
| 782 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 783 |
def _read_query_cursor(session: Session, raw) -> dict:
|
| 784 |
"""Authenticate a cursor before reading the source publication."""
|
| 785 |
if not isinstance(raw, str) or not raw or len(raw) > _QUERY_MAX_CURSOR:
|
| 786 |
-
raise
|
| 787 |
try:
|
| 788 |
padded = raw + "=" * (-len(raw) % 4)
|
| 789 |
payload = json.loads(base64.b64decode(
|
| 790 |
padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8"))
|
| 791 |
except (ValueError, TypeError, UnicodeError, json.JSONDecodeError) as exc:
|
| 792 |
-
raise
|
| 793 |
if (not isinstance(payload, dict)
|
| 794 |
or set(payload) not in ({"v", "q", "s", "o", "p"}, {"v", "q", "s", "o", "p", "i"})
|
| 795 |
or type(payload.get("v")) is not int or payload["v"] != 1
|
|
@@ -798,15 +819,15 @@ def _read_query_cursor(session: Session, raw) -> dict:
|
|
| 798 |
or not isinstance(payload.get("q"), str)
|
| 799 |
or not isinstance(payload.get("s"), str)
|
| 800 |
or not isinstance(payload.get("p"), str)):
|
| 801 |
-
raise
|
| 802 |
publication = payload.get("i")
|
| 803 |
if "i" in payload and (not isinstance(publication, dict)
|
| 804 |
or set(publication) != {"source", "snapshot_id"}
|
| 805 |
or any(not isinstance(value, str) or not value for value in publication.values())):
|
| 806 |
-
raise
|
| 807 |
expected = _cursor_proof(session, payload["q"], payload["s"], payload["o"], publication)
|
| 808 |
if not payload["p"].isascii() or not hmac.compare_digest(payload["p"], expected):
|
| 809 |
-
raise
|
| 810 |
return payload
|
| 811 |
|
| 812 |
|
|
@@ -1847,7 +1868,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str:
|
|
| 1847 |
"",
|
| 1848 |
])
|
| 1849 |
|
| 1850 |
-
lines.extend(query_syntax_section())
|
| 1851 |
if public:
|
| 1852 |
lines.extend([
|
| 1853 |
"## Your read scope",
|
|
@@ -2709,6 +2730,40 @@ def list_databases(session: Session = Depends(require_external_analyst)):
|
|
| 2709 |
"write_scope": "Use the listed tools to make changes with your own permissions."}
|
| 2710 |
|
| 2711 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2712 |
@router.get("/databases/{database}/schema")
|
| 2713 |
def database_schema(database: str, session: Session = Depends(require_external_analyst)):
|
| 2714 |
"""Return the field contract after resolving the ordinary strict Assistant read wall."""
|
|
@@ -2718,9 +2773,20 @@ def database_schema(database: str, session: Session = Depends(require_external_a
|
|
| 2718 |
session = copy.copy(session)
|
| 2719 |
session.runtime = user_tables.lend(session.runtime)
|
| 2720 |
_assert_database_scope(session, database)
|
| 2721 |
-
|
| 2722 |
-
|
| 2723 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2724 |
schema = copy.deepcopy(resolved[0]["schema"])
|
| 2725 |
schema["fields"] = [{**field, **filter_field_contract(field)} for field in schema.get("fields", [])]
|
| 2726 |
return external_freshness.add(schema, session)
|
|
@@ -2788,6 +2854,49 @@ def query_as(rt, tenant, username, database, payload, scope=None) -> dict:
|
|
| 2788 |
return {"ok": False, "error": {"code": refused["code"], "message": message}}
|
| 2789 |
|
| 2790 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2791 |
def _query_database(session: Session, database: str, payload: dict):
|
| 2792 |
try:
|
| 2793 |
if isinstance(payload, dict):
|
|
@@ -2804,6 +2913,16 @@ def _query_database(session: Session, database: str, payload: dict):
|
|
| 2804 |
return result
|
| 2805 |
except HTTPException as exc:
|
| 2806 |
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2807 |
cause = {"assistant_store_not_ready": deps._assistant_source_cause_of(info.get("message")) or "syncing",
|
| 2808 |
"store_not_ready": "syncing",
|
| 2809 |
"amazon_mirror_not_loaded": deps._assistant_source_cause_of(info.get("message")) or "loading"}.get(info.get("code"))
|
|
@@ -2813,7 +2932,11 @@ def _query_database(session: Session, database: str, payload: dict):
|
|
| 2813 |
# S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules.
|
| 2814 |
info["message"] = bad_filter_message(info.get("message"))
|
| 2815 |
if info.get("code") in QUERY_SYNTAX_CODES and isinstance(info.get("message"), str):
|
| 2816 |
-
info["message"] = with_query_syntax(info["message"])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2817 |
if info.get("code") == "query_cursor_stale":
|
| 2818 |
fix = _MCP_FIXES["query_cursor_stale"]
|
| 2819 |
if fix not in info["message"]:
|
|
@@ -2922,6 +3045,90 @@ def _typed_selected_stop_identity(session, database, filters):
|
|
| 2922 |
return identity
|
| 2923 |
|
| 2924 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2925 |
def _query_database_page(session: Session, database: str, payload: dict):
|
| 2926 |
if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS:
|
| 2927 |
raise err(400, "query_arguments_invalid",
|
|
@@ -2955,6 +3162,7 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 2955 |
scope_filters = None if identity is not None else payload.get("filters")
|
| 2956 |
scoped = _external_read_scope(
|
| 2957 |
session, database, fields=None, filters=scope_filters, mode="query_schema")
|
|
|
|
| 2958 |
if identity is not None and (
|
| 2959 |
scoped.get("source_kind") != "runtime_cache"
|
| 2960 |
or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters):
|
|
@@ -3002,16 +3210,19 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 3002 |
if groups:
|
| 3003 |
if not aggregates or sorts:
|
| 3004 |
raise err(400, "query_groups_invalid",
|
| 3005 |
-
"Grouped queries require aggregates and
|
| 3006 |
-
if not mirror:
|
| 3007 |
-
raise err(409, "query_groups_unavailable",
|
| 3008 |
-
"Server grouping requires a connector mirror. Use a mirror database.")
|
| 3009 |
digest = _json_digest({"database": database, **payload, "groupBy": groups,
|
| 3010 |
"limit": limit, "cursor": None})
|
| 3011 |
options = {"groupBy": groups, "offset": 0, "limit": limit}
|
| 3012 |
-
|
| 3013 |
-
|
| 3014 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3015 |
snap = _json_digest({"source_version": summary.get("source_version"),
|
| 3016 |
"fields": visible_fields,
|
| 3017 |
"total_groups": summary["total_groups"]})
|
|
@@ -3021,9 +3232,7 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 3021 |
raise err(409, "query_cursor_stale",
|
| 3022 |
"The permitted result set changed. Start again without a cursor.")
|
| 3023 |
if offset:
|
| 3024 |
-
summary =
|
| 3025 |
-
session, database, filters=payload.get("filters"),
|
| 3026 |
-
mode=("aggregate", aggregates, {**options, "offset": offset}))
|
| 3027 |
if _json_digest({"source_version": summary.get("source_version"),
|
| 3028 |
"fields": visible_fields,
|
| 3029 |
"total_groups": summary["total_groups"]}) != snap:
|
|
@@ -3040,17 +3249,19 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 3040 |
sorts=[], page={"limit": limit, "returned": returned, "offset": offset,
|
| 3041 |
"has_more": next_cursor is not None,
|
| 3042 |
"next_cursor": next_cursor},
|
| 3043 |
-
caps={"max_page_size": _QUERY_MAX_LIMIT, "max_group_fields": 4,
|
| 3044 |
"max_sorts": _QUERY_MAX_SORTS, "max_aggregates": _QUERY_MAX_AGGREGATES,
|
| 3045 |
"max_filter_depth": _QUERY_MAX_FILTER_DEPTH,
|
| 3046 |
"max_filter_nodes": _QUERY_MAX_FILTER_NODES,
|
| 3047 |
"max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS})
|
|
|
|
| 3048 |
external_freshness.add(result, session)
|
| 3049 |
_assert_finite_query_value(result)
|
| 3050 |
external_activity.record_ok(session, database)
|
| 3051 |
return result
|
| 3052 |
|
| 3053 |
server_aggregates = None
|
|
|
|
| 3054 |
paged_total = None
|
| 3055 |
if mirror:
|
| 3056 |
# ⭐⭐ S67-T34 (D-616, D-2631): THE PERMISSION FILTER IS INSIDE THE QUERY. The aggregates
|
|
@@ -3124,6 +3335,7 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 3124 |
"max_filter_nodes": _QUERY_MAX_FILTER_NODES,
|
| 3125 |
"max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS},
|
| 3126 |
}
|
|
|
|
| 3127 |
external_freshness.add(result, session)
|
| 3128 |
_assert_finite_query_value(result)
|
| 3129 |
external_activity.record_ok(session, database)
|
|
@@ -3204,13 +3416,14 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 3204 |
"max_filter_nodes": _QUERY_MAX_FILTER_NODES,
|
| 3205 |
"max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS},
|
| 3206 |
}
|
|
|
|
| 3207 |
external_freshness.add(result, session)
|
| 3208 |
_assert_finite_query_value(result)
|
| 3209 |
external_activity.record_ok(session, database)
|
| 3210 |
return result
|
| 3211 |
|
| 3212 |
|
| 3213 |
-
def _view_context(session, database, event):
|
| 3214 |
import routes_grid
|
| 3215 |
|
| 3216 |
scope = routes_grid._scope_or_400(
|
|
@@ -3233,18 +3446,21 @@ def _view_context(session, database, event):
|
|
| 3233 |
from routes_products import MODULE, product_assembly
|
| 3234 |
|
| 3235 |
session.require(MODULE)
|
| 3236 |
-
g = product_assembly(session, consume_corrections=False
|
|
|
|
| 3237 |
else:
|
| 3238 |
from routes_customers import grid_assembly
|
| 3239 |
|
| 3240 |
session.require(routes_grid.MODULE)
|
| 3241 |
-
g = grid_assembly(session, scope=scope, consume_corrections=False
|
|
|
|
| 3242 |
except HTTPException as exc:
|
| 3243 |
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 3244 |
raise err(exc.status_code, info.get("code", "view_refused"),
|
| 3245 |
f"Database {database}: {info.get('message') or 'The app refused access.'}") from exc
|
| 3246 |
ctx = routes_grid._ctx(
|
| 3247 |
session, g["fields"], g["pids"], scope_key=scope,
|
|
|
|
| 3248 |
measure_keys=frozenset(m["key"] for m in g["measures"]),
|
| 3249 |
resolved_ids=frozenset(g["measure_sets"]),
|
| 3250 |
cohort_ids=frozenset(c["id"] for c in g["lists"]),
|
|
@@ -3331,7 +3547,53 @@ def _validate_personal_view_draft(session: Session, database: str, raw: dict) ->
|
|
| 3331 |
_external_read_scope(session, database, fields=[], filters=None)
|
| 3332 |
|
| 3333 |
|
| 3334 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3335 |
"""Convert declared query groups to native View groups and refuse any lossy normalization."""
|
| 3336 |
value = config.get("filters", [])
|
| 3337 |
_assert_query_filter_bounds([value] if isinstance(value, dict) else value)
|
|
@@ -3351,22 +3613,29 @@ def _saved_view_filters(config):
|
|
| 3351 |
or not isinstance(raw[key], list) or not raw[key]):
|
| 3352 |
refuse()
|
| 3353 |
return {"conj": raw.get("conj", "and"), "children": [node(child) for child in raw[key]]}
|
| 3354 |
-
if any(key in raw for key in ("valueRef", "value2Ref")):
|
| 3355 |
-
refuse("Cell references cannot be saved through this tool because workbook and cell access "
|
| 3356 |
-
"cannot be verified here. Use literal values or a readable rhs field instead.")
|
| 3357 |
op = raw.get("op")
|
| 3358 |
if not isinstance(op, str) or op not in aios_grid.FILTER_OPS:
|
| 3359 |
name = json.dumps(op[:120]) if isinstance(op, str) else "(missing or invalid)"
|
| 3360 |
refuse(f"Unsupported filter operator {name}. Use a supported operator: "
|
| 3361 |
+ ", ".join(sorted(aios_grid.FILTER_OPS)) + ".")
|
| 3362 |
if (set(raw) - {"colId", "op", "value", "value2", "values", "id", "locked",
|
| 3363 |
-
"dateWindow", "dateMode", "rhs"}
|
| 3364 |
or not isinstance(raw.get("colId"), str)):
|
| 3365 |
refuse()
|
| 3366 |
for key in ("value", "value2"):
|
| 3367 |
value = raw.get(key)
|
| 3368 |
if isinstance(value, (dict, list)) or value is not None and len(str(value)) > 500:
|
| 3369 |
refuse()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3370 |
return copy.deepcopy(raw)
|
| 3371 |
|
| 3372 |
if isinstance(value, dict):
|
|
@@ -3471,6 +3740,108 @@ def _view_share_preview(session: Session, args: dict):
|
|
| 3471 |
}
|
| 3472 |
|
| 3473 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3474 |
def _view_event(session, kind, args):
|
| 3475 |
from core import grid_events
|
| 3476 |
|
|
@@ -3498,23 +3869,52 @@ def _view_event(session, kind, args):
|
|
| 3498 |
if kind == "view_upsert":
|
| 3499 |
# Resolve the ordinary View/database wall before the additional T31 draft validation so
|
| 3500 |
# the established refusal vocabulary and connector record-scope guard stay intact.
|
| 3501 |
-
raw = {**raw, "config": _saved_view_filters(
|
|
|
|
| 3502 |
event["view"] = raw
|
| 3503 |
_validate_personal_view_draft(session, database, raw)
|
| 3504 |
existing = ctx.table.find_view(view_id, prefer_owner=session.uname)
|
| 3505 |
if (existing and str(existing[0]).strip().lower()
|
| 3506 |
!= str(session.uname).strip().lower()):
|
| 3507 |
raise err(404, "view_unavailable",
|
| 3508 |
-
f"View {view_id}: another user's personal draft is read-only.")
|
| 3509 |
from core import shares
|
| 3510 |
share_record = shares.grants("view", view_id, st=session.runtime)
|
| 3511 |
if share_record.get("owner") or share_record.get("entries"):
|
| 3512 |
raise err(409, "view_is_shared",
|
| 3513 |
"This View has entered the sharing workflow and cannot be changed by the "
|
| 3514 |
"personal-draft tool. Use a separately confirmed View change.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3515 |
try:
|
| 3516 |
before = ctx.table.find_view(view_id, prefer_owner=session.uname)
|
| 3517 |
-
rerender =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3518 |
except grid_events.StoreUnavailable as exc:
|
| 3519 |
raise err(503, "store_unavailable", "The View is unavailable right now.") from exc
|
| 3520 |
if ctx.out.refusals:
|
|
@@ -3525,7 +3925,7 @@ def _view_event(session, kind, args):
|
|
| 3525 |
raise err(403, "view_refused", f"View {view_id}: {ctx.out.toast}")
|
| 3526 |
after = ctx.table.find_view(view_id, prefer_owner=session.uname)
|
| 3527 |
changed = before != after
|
| 3528 |
-
if not changed:
|
| 3529 |
raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.")
|
| 3530 |
# ⛔⛔ `handle_one`'s BOOL IS A RERENDER FLAG, NOT A SUCCESS FLAG, and reading it as success is
|
| 3531 |
# the mistake this comment exists to stop the next reader making. Its own contract block
|
|
@@ -3543,8 +3943,30 @@ def _view_event(session, kind, args):
|
|
| 3543 |
(not after or str(after[0]).strip().lower()
|
| 3544 |
!= str(session.uname).strip().lower())):
|
| 3545 |
raise err(409, "view_not_personal", f"View {view_id}: the personal draft was not saved.")
|
| 3546 |
-
|
| 3547 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3548 |
|
| 3549 |
|
| 3550 |
def _trusted_activity_via(session):
|
|
@@ -4031,6 +4453,54 @@ def _field_link_tables(*fields):
|
|
| 4031 |
return tables
|
| 4032 |
|
| 4033 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4034 |
def _plan_database_write(session, args):
|
| 4035 |
import routes_tables
|
| 4036 |
from core import user_tables
|
|
@@ -4044,6 +4514,10 @@ def _plan_database_write(session, args):
|
|
| 4044 |
_assert_write_scope(session, "database_object")
|
| 4045 |
if action == "rename":
|
| 4046 |
return _plan_database_rename(session, args)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4047 |
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 4048 |
if action == "record_create":
|
| 4049 |
routes_tables._adds_or_refuse(database)
|
|
@@ -4051,8 +4525,14 @@ def _plan_database_write(session, args):
|
|
| 4051 |
tables_before = _database_write_tables(session.runtime, (database,))
|
| 4052 |
table = copy.deepcopy(tables_before.get(database) or {})
|
| 4053 |
if not table or not user_tables.is_user_table(database, st=session.runtime):
|
| 4054 |
-
|
| 4055 |
-
"Records from connected sources are read only.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4056 |
revision = _bucket_revision(_database_write_source(session.runtime), user_tables.STORE_KEY,
|
| 4057 |
tables_before)
|
| 4058 |
if type(revision.get("rev")) is not int:
|
|
@@ -4136,11 +4616,13 @@ def _plan_database_write(session, args):
|
|
| 4136 |
if not isinstance(raw, dict):
|
| 4137 |
raise err(400, "tool_arguments_invalid", "field must be an object.")
|
| 4138 |
have = list(table.get("fields") or [])
|
|
|
|
|
|
|
| 4139 |
field = user_tables._clean_field(
|
| 4140 |
raw, recurrence_validator=recurrence_validator,
|
| 4141 |
valid_keys={str(item.get("key")) for item in have if item.get("key")})
|
| 4142 |
if (not field or any(item.get("key") == field.get("key") for item in have)
|
| 4143 |
-
or len(have) >=
|
| 4144 |
or not user_tables.dynamic_link_filter_valid(field, have)
|
| 4145 |
or not user_tables.flow_bound(field.get("automation"), session.runtime)):
|
| 4146 |
raise err(400, "field_refused", "The field does not pass the database's schema rules.")
|
|
@@ -4228,6 +4710,8 @@ def _execute_database_object(session, args, *, plan=None):
|
|
| 4228 |
return database_schema(database, session)
|
| 4229 |
if action == "query":
|
| 4230 |
return _query_database(session, database, args.get("query") or {})
|
|
|
|
|
|
|
| 4231 |
plan = plan or _plan_database_write(session, args)
|
| 4232 |
if action == "rename":
|
| 4233 |
import routes_nav
|
|
@@ -4881,7 +5365,7 @@ def _is_website_note(name, arguments):
|
|
| 4881 |
|
| 4882 |
|
| 4883 |
def _preview_external_write(session, name, arguments, *, issue_token=True, execution=None, purpose=None, about=None,
|
| 4884 |
-
intent=None):
|
| 4885 |
from core import write_execution
|
| 4886 |
if isinstance(arguments, dict) and "about" in arguments:
|
| 4887 |
if about is not None:
|
|
@@ -4904,20 +5388,32 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
|
|
| 4904 |
tenant=session.tenant, user=copy.deepcopy(session.user),
|
| 4905 |
claims=copy.deepcopy(session.claims), runtime=preview_runtime)
|
| 4906 |
execution = execution or {'seed': uuid.uuid4().hex, 'timestamp': time.time()}
|
| 4907 |
-
|
| 4908 |
-
|
| 4909 |
-
|
| 4910 |
-
|
| 4911 |
-
|
| 4912 |
-
|
| 4913 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4914 |
if isinstance(result, dict) and result.get("refused"):
|
| 4915 |
raise err(409, "write_partial_refused",
|
| 4916 |
"Part of this change could not be accepted. Nothing was previewed.")
|
| 4917 |
|
| 4918 |
remote_note = _is_website_note(name, normalised)
|
| 4919 |
-
|
| 4920 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4921 |
diff = copy.deepcopy(plan.get("diff") or [])
|
| 4922 |
warnings = list(plan.get("warnings") or [])
|
| 4923 |
affected = copy.deepcopy(plan.get("affected") or {})
|
|
@@ -4934,7 +5430,9 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
|
|
| 4934 |
affected = _affected_for(name, normalised, diff, result)
|
| 4935 |
current_revision = {
|
| 4936 |
key: revision for key, revision in preview_runtime.base_revisions().items()
|
| 4937 |
-
|
|
|
|
|
|
|
| 4938 |
}
|
| 4939 |
if not diff:
|
| 4940 |
raise err(409, "write_not_changed", "The requested write would not change anything.")
|
|
@@ -4955,7 +5453,7 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
|
|
| 4955 |
|
| 4956 |
_canonical_json(diff, capped=False)
|
| 4957 |
diff_digest = _json_digest(diff)
|
| 4958 |
-
shown_diff, diff_summary = (diff, None) if plan is not None else _summarise_diff(diff, projected)
|
| 4959 |
if diff_summary is not None:
|
| 4960 |
warnings.append(f"The preview shows {diff_summary['sample']} of {diff_summary['changes']} changes. "
|
| 4961 |
"confirm_change applies all of them.")
|
|
@@ -4978,6 +5476,8 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
|
|
| 4978 |
"execution": execution,
|
| 4979 |
"about": about, "about_digest": _json_digest(about), "buildKind": build_kind,
|
| 4980 |
}
|
|
|
|
|
|
|
| 4981 |
if purpose is not None:
|
| 4982 |
payload.update(purpose=purpose, purpose_digest=_json_digest(purpose))
|
| 4983 |
if remote_note:
|
|
@@ -5096,7 +5596,7 @@ def _store_operation(session, payload, preview, staged, intent=None):
|
|
| 5096 |
app = ((doc or {}).get("applications") or {}).get(app_id) or {}
|
| 5097 |
access["databases"].extend(_databases_in_arguments(app))
|
| 5098 |
# Foreign handlers own their object permissions. Pin their consulted documents without calling them on replay.
|
| 5099 |
-
if payload["tool"] not in external_tools.TOOLS:
|
| 5100 |
access["documents"] = {key: _json_digest(value) for key, value in staged._initial.items()
|
| 5101 |
if key not in {external_access.BUCKET, external_oauth.BUCKET}}
|
| 5102 |
access["after_documents"] = {key: _json_digest(staged._docs[key]) for key in access["documents"]}
|
|
@@ -5181,7 +5681,7 @@ def _operation_access(session, item, *, confirming=False):
|
|
| 5181 |
"Ask the owner to restore access before checking again.")
|
| 5182 |
for database in set(access["databases"]):
|
| 5183 |
_assert_database_scope(session, database)
|
| 5184 |
-
if _is_website_note(name, args):
|
| 5185 |
_external_read_scope(session, database, fields=[], mode="schema")
|
| 5186 |
else:
|
| 5187 |
_external_read_scope(session, database, fields=[])
|
|
@@ -5192,23 +5692,29 @@ def _operation_access(session, item, *, confirming=False):
|
|
| 5192 |
import routes_tables
|
| 5193 |
action = args["action"]
|
| 5194 |
record_pid = str(args.get("record_id") or "").strip()
|
| 5195 |
-
|
| 5196 |
-
|
|
|
|
|
|
|
| 5197 |
if action in {"record_create", "record_update"} and item["status"] == "committed":
|
| 5198 |
if any(int(rid) not in g["pids"] for rid in item["affected"]["ids"]):
|
| 5199 |
raise err(403, "operation_unavailable",
|
| 5200 |
"This connection cannot read every record covered by the operation. "
|
| 5201 |
"Query the database with current access, or ask the owner to restore access before checking again.")
|
| 5202 |
fields = set(args.get("values") or {})
|
|
|
|
|
|
|
| 5203 |
if args.get("field_key") and action != "field_delete":
|
| 5204 |
fields.add(args["field_key"])
|
| 5205 |
if not fields <= {field["key"] for field in g["fields"]}:
|
| 5206 |
raise err(403, "operation_unavailable",
|
| 5207 |
"This connection cannot read every field covered by the operation. "
|
| 5208 |
"Call data with action describe_database, then ask the owner to restore required access before checking again.")
|
| 5209 |
-
if name in {"view_upsert", "view_delete", "view_object"}:
|
| 5210 |
-
|
| 5211 |
-
|
|
|
|
|
|
|
| 5212 |
found = ctx_view.table.find_view(view_id, prefer_owner=session.uname)
|
| 5213 |
deleting = name == "view_delete" or args.get("action") == "delete"
|
| 5214 |
if (found and str(found[0]).strip().lower() != session.uname.lower()) or (not found and not deleting):
|
|
@@ -5248,7 +5754,7 @@ def _operation_access(session, item, *, confirming=False):
|
|
| 5248 |
"This connection cannot read the operation's item with its current access. "
|
| 5249 |
"Read the item with this connection, or ask the owner to restore access before checking again.")
|
| 5250 |
return session
|
| 5251 |
-
if access["documents"] and _json_digest(user) != access["user"]:
|
| 5252 |
raise err(403, "operation_unavailable",
|
| 5253 |
"The account changed since the preview, so receipt access cannot be verified. "
|
| 5254 |
"Read the item with this connection before previewing another change.")
|
|
@@ -5475,6 +5981,30 @@ def _expired_fix(payload):
|
|
| 5475 |
+ str(_CONFIRMATION_TTL_SECONDS // 60) + " minutes.")
|
| 5476 |
|
| 5477 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5478 |
def _confirm_fix(args):
|
| 5479 |
"""Name what a confirm_change call got wrong and the shape that succeeds."""
|
| 5480 |
allowed = ("confirmation_token", "confirmation_phrase", "intent")
|
|
@@ -5507,7 +6037,7 @@ def _confirm_change(session, args):
|
|
| 5507 |
if payload.get("environment") != _write_environment():
|
| 5508 |
raise err(409, "confirmation_stale", "The environment changed after the preview, " + _STALE_FIX)
|
| 5509 |
if int(payload.get("expiry") or 0) <= int(time.time()):
|
| 5510 |
-
raise
|
| 5511 |
if payload.get("strong") and args.get("confirmation_phrase") != _CONFIRMATION_PHRASE:
|
| 5512 |
raise err(400, "confirmation_phrase_required",
|
| 5513 |
"Enter the exact stronger confirmation phrase shown in the preview.")
|
|
@@ -5522,17 +6052,23 @@ def _confirm_change(session, args):
|
|
| 5522 |
_confirmation_claim(fresh_session, operation_id)
|
| 5523 |
except _OperationAlreadyClaimed:
|
| 5524 |
return _operation_status(fresh_session, {"operation_id": operation_id})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5525 |
remote_note = _is_website_note(tool, arguments)
|
| 5526 |
capability = None
|
| 5527 |
try:
|
| 5528 |
recomputed = _preview_external_write(
|
| 5529 |
fresh_session, tool, arguments, issue_token=False, execution=execution,
|
| 5530 |
-
purpose=payload.get("purpose"), about=payload.get("about")
|
|
|
|
| 5531 |
current = recomputed["_confirmation"]
|
| 5532 |
if (payload.get("arguments_digest") != current.get("arguments_digest")
|
| 5533 |
or payload.get("environment") != current.get("environment")
|
| 5534 |
or payload.get("revision_digest") != current.get("revision_digest")
|
| 5535 |
or payload.get("diff_digest") != current.get("diff_digest")
|
|
|
|
| 5536 |
or (remote_note and any(payload.get(key) != current.get(key)
|
| 5537 |
for key in ("remote_plan_digest", "target_digest", "source_fingerprint")))
|
| 5538 |
or any((key in payload, payload.get(key)) != (key in current, current.get(key))
|
|
@@ -5546,12 +6082,20 @@ def _confirm_change(session, args):
|
|
| 5546 |
ctx, recomputed["_plan"], confirmation_token=args["confirmation_token"])
|
| 5547 |
result = website_tools.execute_confirmed_note(ctx, recomputed["_plan"], capability=capability)
|
| 5548 |
return _finish_website_note(fresh_session, payload, recomputed["_plan"], result)
|
| 5549 |
-
if tool == "database_object":
|
| 5550 |
result = _execute_database_object(
|
| 5551 |
fresh_session, arguments, plan=recomputed.get("_plan"))
|
| 5552 |
elif _is_members_write(tool, arguments):
|
| 5553 |
# The account writer compares the pinned expect itself, so no runtime guard is needed.
|
| 5554 |
result = _dispatch_external_tool(_tool_context(fresh_session), tool, arguments, about=payload["about"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5555 |
else:
|
| 5556 |
from datetime import datetime, timezone
|
| 5557 |
commit_started = datetime.now(timezone.utc)
|
|
@@ -5578,7 +6122,7 @@ def _confirm_change(session, args):
|
|
| 5578 |
_confirmation_status(fresh_session, operation_id, "failed")
|
| 5579 |
raise
|
| 5580 |
before_revision = copy.deepcopy(recomputed.get("current_revision") or {})
|
| 5581 |
-
if tool == "database_object":
|
| 5582 |
from core import user_tables
|
| 5583 |
import routes_nav
|
| 5584 |
bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY
|
|
@@ -5589,6 +6133,8 @@ def _confirm_change(session, args):
|
|
| 5589 |
after_revision = {"rev": produced_rev}
|
| 5590 |
elif _is_members_write(tool, arguments):
|
| 5591 |
after_revision = copy.deepcopy(result["revision"])
|
|
|
|
|
|
|
| 5592 |
else:
|
| 5593 |
after_revision = copy.deepcopy(guarded_session.runtime._expected)
|
| 5594 |
after_revision_digest = _json_digest(after_revision)
|
|
@@ -5624,7 +6170,7 @@ def _confirm_change(session, args):
|
|
| 5624 |
_confirmation_status(fresh_session, operation_id, "committed", response)
|
| 5625 |
if isinstance(stored.get("arguments_ref"), dict):
|
| 5626 |
_drop_operation_documents(fresh_session, [stored["arguments_ref"].get("key")])
|
| 5627 |
-
if (tool != "database_object" and not _is_members_write(tool, arguments)
|
| 5628 |
and receipt.get("persisted") is True):
|
| 5629 |
try:
|
| 5630 |
targets, native_ids = _committed_build_evidence(fresh_session, guarded_session.runtime, arguments,
|
|
@@ -5665,13 +6211,13 @@ _MCP_CODE_GROUPS = (
|
|
| 5665 |
tab_missing folder_missing unknown_table unknown_field view_missing view_not_found
|
| 5666 |
sheet_structure_tab_missing unknown_sheet_template cell_missing
|
| 5667 |
"""),
|
| 5668 |
-
("
|
| 5669 |
tool_unavailable tool_capability_unsupported capability_unavailable external_access_unavailable
|
| 5670 |
store_unavailable store_not_ready store_catching_up oauth_unavailable not_connected
|
| 5671 |
schema_not_loaded query_groups_unavailable confirmation_revision_unavailable
|
| 5672 |
assistant_source_too_large amazon_mirror_not_loaded query_link_unavailable
|
| 5673 |
"""),
|
| 5674 |
-
("
|
| 5675 |
tool_refused write_partial_refused field_refused view_refused refused
|
| 5676 |
connected_source_read_only computed_field records_read_only loopable_managed
|
| 5677 |
preset_field_locked zone_membership_read_only not_a_user_table
|
|
@@ -5740,6 +6286,39 @@ MCP_RESULT_CODES = {
|
|
| 5740 |
code: reason for reason, codes in _MCP_CODE_GROUPS for code in codes.split()
|
| 5741 |
}
|
| 5742 |
MCP_RESULT_CODES.update({
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5743 |
"workbook_name_ambiguous": "More than one readable workbook has this exact name. Use one of the candidate ids.",
|
| 5744 |
"view_filters_unsupported": "These filters cannot be saved without changing their meaning. Use colId, op and value leaves, with conj and children groups.",
|
| 5745 |
"confirmation_session_required": "Read this confirmation through its authenticated connection.",
|
|
@@ -5792,6 +6371,8 @@ MCP_RESULT_CODES.update({
|
|
| 5792 |
"request_rate_limited": "This AI has reached its hourly request cap. Try again later.",
|
| 5793 |
})
|
| 5794 |
MCP_RESULT_CODES.update(READ_REFUSAL_FIXES)
|
|
|
|
|
|
|
| 5795 |
_MCP_INTERNAL_PROSE = re.compile(
|
| 5796 |
r"\b(?:DuckDB|Postgres(?:ql)?|SQLite|Polars|Pandas|SQLAlchemy|PyArrow|HyperFormula)\b|"
|
| 5797 |
r"\b(?:api|web|platform)_[a-z_]+\b|\b(?:verify_\w+|\w+Error)\b|"
|
|
@@ -5881,14 +6462,14 @@ class _MCPGuideText(str):
|
|
| 5881 |
|
| 5882 |
|
| 5883 |
_MCP_RESULT_OPAQUE = frozenset({
|
| 5884 |
-
"records", "rows", "cells", "values", "value", "before", "after", "arguments", "diff",
|
| 5885 |
"confirmation_token", "confirmation_phrase", "diff_digest", "current_revision",
|
| 5886 |
-
"revision_before", "revision_after", "filters", "sorts", "config", "binding", "bindings",
|
| 5887 |
})
|
| 5888 |
_MCP_RESULT_IDENTIFIERS = frozenset({
|
| 5889 |
"resourceUri", "deepLink", "pdfLink", "downloadUrl", "src", "websiteUrl", "fileId", "link", "href",
|
| 5890 |
"id", "ids", "operation_id", "key", "keys", "pid", "pids", "database", "databases", "sourceDatabase",
|
| 5891 |
-
"compatibleDatabases", "field", "fields", "field_key", "colId", "metric_key", "groupBy",
|
| 5892 |
"workbook_id", "worksheet", "tab", "view_id", "app_id", "catalog_id", "chart_id", "record_id", "contextId",
|
| 5893 |
"path", "paths", "address", "addresses", "range", "cursor", "next_cursor", "token",
|
| 5894 |
"source_version", "code", "tool", "action", "person", "agent", "owner", "author", "from", "to",
|
|
@@ -5899,7 +6480,7 @@ _MCP_RESULT_OBJECTS = frozenset({
|
|
| 5899 |
"data", "error", "databases", "fields", "tab", "tabs", "workbook", "workbooks", "worksheet",
|
| 5900 |
"worksheets", "file", "files", "request", "requests", "template", "templates", "application",
|
| 5901 |
"applications", "view", "views", "from", "to", "source", "affected", "aggregates", "tools",
|
| 5902 |
-
"annotations", "serverInfo", "environment",
|
| 5903 |
})
|
| 5904 |
# A link descriptor names its target database key. It is an identifier, never prose (D-9307).
|
| 5905 |
_MCP_RESULT_DESCRIPTORS = frozenset({"link"})
|
|
@@ -5924,7 +6505,7 @@ def _mcp_result_copy(value, labels=None, *, field=None, metadata=False):
|
|
| 5924 |
return copy.deepcopy(value)
|
| 5925 |
result = {}
|
| 5926 |
for key, child in value.items():
|
| 5927 |
-
if key == "retry_query" and value.get("code")
|
| 5928 |
# These are the caller's original arguments, not explanatory metadata.
|
| 5929 |
result[key] = copy.deepcopy(child)
|
| 5930 |
elif key == "about":
|
|
@@ -7034,11 +7615,12 @@ def _dispatch_mcp(req, session: Session, base_url: str = ""):
|
|
| 7034 |
|
| 7035 |
#: Known refusal codes whose MCP answer carries a repair.
|
| 7036 |
_MCP_NEXT_CALL = {"workbook_missing": WORKBOOK_MISSING_FIX, "assistant_unknown_database": UNKNOWN_DATABASE_FIX,
|
|
|
|
| 7037 |
**READ_REFUSAL_FIXES}
|
| 7038 |
|
| 7039 |
|
| 7040 |
def _mcp_next_call(code, message):
|
| 7041 |
-
"""Append the supported repair once for a known
|
| 7042 |
text = str(message or "").rstrip()
|
| 7043 |
fix = _MCP_NEXT_CALL.get(code, "")
|
| 7044 |
if code == "assistant_unknown_database":
|
|
@@ -7182,6 +7764,7 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 7182 |
if purpose is not None and not (writing and (confirm or direct_wiki)):
|
| 7183 |
raise err(400, "tool_arguments_invalid", "Send purpose only with a previewed build.")
|
| 7184 |
if name == "query_database" and isinstance(arguments, dict):
|
|
|
|
| 7185 |
external_tools.validate_query_filter_operators(arguments.get("filters"))
|
| 7186 |
_assert_tool_database_scope(session, name, scoped_arguments)
|
| 7187 |
if name == "confirm_change":
|
|
@@ -7244,8 +7827,8 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 7244 |
import routes_sheet
|
| 7245 |
if exc.code == "tool_arguments_invalid" and name == "confirm_change":
|
| 7246 |
exc = sheets.SheetRefusal(exc.code, _confirm_fix(arguments))
|
| 7247 |
-
if exc.code
|
| 7248 |
-
exc = sheets.SheetRefusal(exc.code, with_query_syntax(exc.message))
|
| 7249 |
if exc.code in _MCP_NEXT_CALL:
|
| 7250 |
exc = sheets.SheetRefusal(exc.code, _mcp_next_call(exc.code, exc.message))
|
| 7251 |
translated = routes_sheet._refuse(exc)
|
|
@@ -7255,9 +7838,11 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 7255 |
except HTTPException as exc:
|
| 7256 |
if exc.status_code == 401:
|
| 7257 |
_auth_refusal()
|
| 7258 |
-
info = exc.detail.get("error",
|
|
|
|
| 7259 |
retry = ({"retry_query": info["retry_query"]}
|
| 7260 |
-
if info.get("code")
|
|
|
|
| 7261 |
details = info.get("details")
|
| 7262 |
if info.get("code") in {"build_about_required", "build_about_invalid", "call_intent_required",
|
| 7263 |
"call_intent_invalid"} and isinstance(details, dict):
|
|
@@ -7269,12 +7854,22 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 7269 |
retry["suggestions"] = details.get("suggestions", [])
|
| 7270 |
if info.get("code") == "workbook_name_ambiguous" and isinstance(details, dict):
|
| 7271 |
retry["candidates"] = details.get("candidates", [])
|
| 7272 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7273 |
if info.get("code") in {"build_about_required", "build_about_invalid"} and isinstance(message, str):
|
| 7274 |
message += _mcp_about_fix(called)
|
| 7275 |
if info.get("code") in _MCP_NEXT_CALL:
|
| 7276 |
message = _mcp_next_call(info["code"], message)
|
| 7277 |
-
out = _tool_refusal(name, info.get("code"
|
| 7278 |
except (TypeError, ValueError, KeyError, AttributeError):
|
| 7279 |
out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
|
| 7280 |
except Exception as exc:
|
|
|
|
| 14 |
import time
|
| 15 |
import uuid
|
| 16 |
from contextlib import contextmanager
|
| 17 |
+
from contextvars import ContextVar
|
| 18 |
+
from http.client import responses as _HTTP_STATUS_REASONS
|
| 19 |
from urllib.parse import parse_qs, quote, unquote, urlencode, urlsplit, urlunsplit
|
| 20 |
|
| 21 |
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response
|
|
|
|
| 51 |
_QUERY_KEYS = frozenset({"fields", "filters", "sorts", "limit", "cursor", "aggregates", "groupBy"})
|
| 52 |
|
| 53 |
|
| 54 |
+
def _query_grouping_help(text):
|
| 55 |
+
"""Describe native Table grouping in the external query's existing syntax help."""
|
| 56 |
+
help_text = ("A stored Table groups one field without a month unit, using the Table grid's aggregate reader. "
|
| 57 |
+
"Its group order is count descending, then key ascending. "
|
| 58 |
+
"A query_groups_unavailable refusal names its cause and a query_database call without groupBy that reads the rows.")
|
| 59 |
+
return text.replace(
|
| 60 |
+
"Only a connected database groups. Any other database answers query_groups_unavailable.",
|
| 61 |
+
help_text).replace(
|
| 62 |
+
"Only a connected database, such as an Odoo sales lines mirror, groups. "
|
| 63 |
+
"Any other database answers 409 query_groups_unavailable: read its records by page, "
|
| 64 |
+
"or ask for aggregates without groupBy.", help_text)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
def _register_query_paging_contract():
|
| 68 |
"""Keep the legacy and grouped query schemas aligned with the API's bounded page reader."""
|
| 69 |
from tool_groups.data import GROUP
|
|
|
|
| 83 |
"filters", "filterConj", "sorts", "groupBy", "visible", "order", "display",
|
| 84 |
"memberPids",
|
| 85 |
})
|
| 86 |
+
_VIEW_OPERATION_PLAN = ContextVar("external_view_operation_plan", default=None)
|
| 87 |
_CONFIRMATION_BUCKET = "external_confirmations"
|
| 88 |
_CONFIRMATION_TTL_SECONDS = 5 * 60
|
| 89 |
_OPERATION_RETENTION_SECONDS = 24 * 60 * 60
|
|
|
|
| 796 |
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
| 797 |
|
| 798 |
|
| 799 |
+
def _invalid_query_cursor(cause: str):
|
| 800 |
+
return err(400, "query_cursor_invalid", cause + " Repeat the original query without cursor "
|
| 801 |
+
"for a fresh first page. Reuse each returned next_cursor unchanged with the same account and connection.")
|
| 802 |
+
|
| 803 |
+
|
| 804 |
def _read_query_cursor(session: Session, raw) -> dict:
|
| 805 |
"""Authenticate a cursor before reading the source publication."""
|
| 806 |
if not isinstance(raw, str) or not raw or len(raw) > _QUERY_MAX_CURSOR:
|
| 807 |
+
raise _invalid_query_cursor("The query cursor must be a nonempty next_cursor string within the cursor size limit.")
|
| 808 |
try:
|
| 809 |
padded = raw + "=" * (-len(raw) % 4)
|
| 810 |
payload = json.loads(base64.b64decode(
|
| 811 |
padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8"))
|
| 812 |
except (ValueError, TypeError, UnicodeError, json.JSONDecodeError) as exc:
|
| 813 |
+
raise _invalid_query_cursor("The query cursor is not a complete, unchanged next_cursor value.") from exc
|
| 814 |
if (not isinstance(payload, dict)
|
| 815 |
or set(payload) not in ({"v", "q", "s", "o", "p"}, {"v", "q", "s", "o", "p", "i"})
|
| 816 |
or type(payload.get("v")) is not int or payload["v"] != 1
|
|
|
|
| 819 |
or not isinstance(payload.get("q"), str)
|
| 820 |
or not isinstance(payload.get("s"), str)
|
| 821 |
or not isinstance(payload.get("p"), str)):
|
| 822 |
+
raise _invalid_query_cursor("The query cursor has an unsupported format.")
|
| 823 |
publication = payload.get("i")
|
| 824 |
if "i" in payload and (not isinstance(publication, dict)
|
| 825 |
or set(publication) != {"source", "snapshot_id"}
|
| 826 |
or any(not isinstance(value, str) or not value for value in publication.values())):
|
| 827 |
+
raise _invalid_query_cursor("The query cursor has an unsupported format.")
|
| 828 |
expected = _cursor_proof(session, payload["q"], payload["s"], payload["o"], publication)
|
| 829 |
if not payload["p"].isascii() or not hmac.compare_digest(payload["p"], expected):
|
| 830 |
+
raise _invalid_query_cursor("The query cursor cannot be verified for this account and connection.")
|
| 831 |
return payload
|
| 832 |
|
| 833 |
|
|
|
|
| 1868 |
"",
|
| 1869 |
])
|
| 1870 |
|
| 1871 |
+
lines.extend(_query_grouping_help(line) for line in query_syntax_section())
|
| 1872 |
if public:
|
| 1873 |
lines.extend([
|
| 1874 |
"## Your read scope",
|
|
|
|
| 2730 |
"write_scope": "Use the listed tools to make changes with your own permissions."}
|
| 2731 |
|
| 2732 |
|
| 2733 |
+
_SCHEMA_READ_REFUSAL_FIXES = {
|
| 2734 |
+
"refused": (
|
| 2735 |
+
"The schema read was refused by the database's source. Ask an Administrator to check the source before retrying."),
|
| 2736 |
+
"not_read_through": (
|
| 2737 |
+
"This database has no connected read path. Ask an Administrator to repair its source connection, then describe it again."),
|
| 2738 |
+
"cross_tenant_store": (
|
| 2739 |
+
"This connection cannot read that workspace. Reconnect to the intended workspace before describing the database."),
|
| 2740 |
+
"store_not_ready": (
|
| 2741 |
+
"This database is temporarily unavailable. Try describing it again shortly."),
|
| 2742 |
+
"schema_unavailable": (
|
| 2743 |
+
"This database's field definitions are temporarily unavailable. Try describing it again shortly."),
|
| 2744 |
+
"schema_not_loaded": (
|
| 2745 |
+
"This connected database has no saved fields yet. Describe it again after its source sync completes."),
|
| 2746 |
+
"not_connected": (
|
| 2747 |
+
"This database has no connected source. Call table.list_databases and choose a returned database key."),
|
| 2748 |
+
"scope_not_applicable": (
|
| 2749 |
+
"This database's access restriction could not be applied. Ask an Administrator to repair it without widening access."),
|
| 2750 |
+
"filter_not_applicable": (
|
| 2751 |
+
"This database's access filter could not be applied. Ask an Administrator to repair it without removing the restriction."),
|
| 2752 |
+
"filter_unsupported": (
|
| 2753 |
+
"This database's access filter cannot be evaluated. Ask an Administrator to repair it without removing the restriction."),
|
| 2754 |
+
"forbidden_table": (
|
| 2755 |
+
"Current permissions do not allow this database. Call table.list_databases and choose a readable database."),
|
| 2756 |
+
"odoo_definition_not_published": (
|
| 2757 |
+
"This database's field definitions have not been published. Describe it again after its source sync completes."),
|
| 2758 |
+
"odoo_mirror_binding_incomplete": (
|
| 2759 |
+
"This database's source binding is incomplete. Ask an Administrator to repair the source, then describe it again."),
|
| 2760 |
+
"odoo_mirror_not_loaded": (
|
| 2761 |
+
"This database's source is not loaded. Describe it again after its source sync completes."),
|
| 2762 |
+
"canonical_source_unavailable": (
|
| 2763 |
+
"This database's current source is unavailable. Try describing it again after its source sync completes."),
|
| 2764 |
+
}
|
| 2765 |
+
|
| 2766 |
+
|
| 2767 |
@router.get("/databases/{database}/schema")
|
| 2768 |
def database_schema(database: str, session: Session = Depends(require_external_analyst)):
|
| 2769 |
"""Return the field contract after resolving the ordinary strict Assistant read wall."""
|
|
|
|
| 2773 |
session = copy.copy(session)
|
| 2774 |
session.runtime = user_tables.lend(session.runtime)
|
| 2775 |
_assert_database_scope(session, database)
|
| 2776 |
+
try:
|
| 2777 |
+
resolved = _resolve_external_scope(
|
| 2778 |
+
session, databases=[database], include_schema=True, strict_schema=True,
|
| 2779 |
+
)
|
| 2780 |
+
except HTTPException as exc:
|
| 2781 |
+
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 2782 |
+
repair = _SCHEMA_READ_REFUSAL_FIXES.get(info.get("code"))
|
| 2783 |
+
if not repair:
|
| 2784 |
+
raise
|
| 2785 |
+
message = _mcp_plain(info.get("message"), repair)
|
| 2786 |
+
if repair not in message:
|
| 2787 |
+
message = (message.rstrip() + " " + repair).strip()
|
| 2788 |
+
raise HTTPException(status_code=exc.status_code, headers=exc.headers,
|
| 2789 |
+
detail={**exc.detail, "error": {**info, "message": message}}) from exc
|
| 2790 |
schema = copy.deepcopy(resolved[0]["schema"])
|
| 2791 |
schema["fields"] = [{**field, **filter_field_contract(field)} for field in schema.get("fields", [])]
|
| 2792 |
return external_freshness.add(schema, session)
|
|
|
|
| 2854 |
return {"ok": False, "error": {"code": refused["code"], "message": message}}
|
| 2855 |
|
| 2856 |
|
| 2857 |
+
def _query_field_labels(payload, fields):
|
| 2858 |
+
"""Resolve exact, unique readable labels while preserving canonical keys and query values."""
|
| 2859 |
+
keys = {field["key"] for field in fields if isinstance(field.get("key"), str)}
|
| 2860 |
+
labels = {}
|
| 2861 |
+
for field in fields:
|
| 2862 |
+
label = field.get("label")
|
| 2863 |
+
if isinstance(label, str) and label and field.get("key") in keys:
|
| 2864 |
+
labels.setdefault(label, set()).add(field["key"])
|
| 2865 |
+
missing = []
|
| 2866 |
+
|
| 2867 |
+
def resolve(name):
|
| 2868 |
+
if isinstance(name, str) and name not in keys:
|
| 2869 |
+
matches = labels.get(name, set())
|
| 2870 |
+
if len(matches) == 1:
|
| 2871 |
+
return next(iter(matches))
|
| 2872 |
+
missing.append(name)
|
| 2873 |
+
return name
|
| 2874 |
+
|
| 2875 |
+
def filters(node):
|
| 2876 |
+
if isinstance(node, list):
|
| 2877 |
+
for child in node:
|
| 2878 |
+
filters(child)
|
| 2879 |
+
elif isinstance(node, dict):
|
| 2880 |
+
if "colId" in node:
|
| 2881 |
+
node["colId"] = resolve(node["colId"])
|
| 2882 |
+
for name in ("nodes", "children"):
|
| 2883 |
+
if name in node:
|
| 2884 |
+
filters(node[name])
|
| 2885 |
+
|
| 2886 |
+
resolved = copy.deepcopy(payload)
|
| 2887 |
+
if isinstance(resolved.get("fields"), list):
|
| 2888 |
+
resolved["fields"] = [resolve(name) for name in resolved["fields"]]
|
| 2889 |
+
filters(resolved.get("filters"))
|
| 2890 |
+
for name in ("sorts", "aggregates", "groupBy"):
|
| 2891 |
+
if isinstance(resolved.get(name), list):
|
| 2892 |
+
for index, item in enumerate(resolved[name]):
|
| 2893 |
+
if name == "groupBy" and isinstance(item, str):
|
| 2894 |
+
resolved[name][index] = resolve(item)
|
| 2895 |
+
elif isinstance(item, dict) and "field" in item:
|
| 2896 |
+
item["field"] = resolve(item["field"])
|
| 2897 |
+
return resolved, missing, keys
|
| 2898 |
+
|
| 2899 |
+
|
| 2900 |
def _query_database(session: Session, database: str, payload: dict):
|
| 2901 |
try:
|
| 2902 |
if isinstance(payload, dict):
|
|
|
|
| 2913 |
return result
|
| 2914 |
except HTTPException as exc:
|
| 2915 |
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 2916 |
+
if info.get("code") == "assistant_unknown_field" and isinstance(payload, dict):
|
| 2917 |
+
schema = database_schema(database, session)
|
| 2918 |
+
resolved, missing, visible_keys = _query_field_labels(payload, schema.get("fields", []))
|
| 2919 |
+
if resolved != payload:
|
| 2920 |
+
return _query_database(session, database, resolved)
|
| 2921 |
+
message = (deps.assistant_unknown_field_message(missing, visible_keys)
|
| 2922 |
+
if missing else info.get("message", ""))
|
| 2923 |
+
info["message"] = message.replace(
|
| 2924 |
+
"Use a field key returned by describe_database.",
|
| 2925 |
+
"Call data.describe_database for this database and use a returned field key.")
|
| 2926 |
cause = {"assistant_store_not_ready": deps._assistant_source_cause_of(info.get("message")) or "syncing",
|
| 2927 |
"store_not_ready": "syncing",
|
| 2928 |
"amazon_mirror_not_loaded": deps._assistant_source_cause_of(info.get("message")) or "loading"}.get(info.get("code"))
|
|
|
|
| 2932 |
# S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules.
|
| 2933 |
info["message"] = bad_filter_message(info.get("message"))
|
| 2934 |
if info.get("code") in QUERY_SYNTAX_CODES and isinstance(info.get("message"), str):
|
| 2935 |
+
info["message"] = _query_grouping_help(with_query_syntax(info["message"]))
|
| 2936 |
+
if info.get("code") == "query_groups_unavailable":
|
| 2937 |
+
info["retry_query"] = {"database": database, **{
|
| 2938 |
+
key: copy.deepcopy(value) for key, value in payload.items()
|
| 2939 |
+
if key not in {"groupBy", "aggregates", "cursor"}}}
|
| 2940 |
if info.get("code") == "query_cursor_stale":
|
| 2941 |
fix = _MCP_FIXES["query_cursor_stale"]
|
| 2942 |
if fix not in info["message"]:
|
|
|
|
| 3045 |
return identity
|
| 3046 |
|
| 3047 |
|
| 3048 |
+
def _query_source_metadata(*sources):
|
| 3049 |
+
"""Copy supplied canonical status and merge supplied limits without inferring a source cause."""
|
| 3050 |
+
out = {}
|
| 3051 |
+
for source in sources:
|
| 3052 |
+
if isinstance(source.get("sourceStatus"), dict):
|
| 3053 |
+
out["sourceStatus"] = copy.deepcopy(source["sourceStatus"])
|
| 3054 |
+
if isinstance(source.get("limits"), list):
|
| 3055 |
+
limits = out.setdefault("limits", [])
|
| 3056 |
+
for limitation in source["limits"]:
|
| 3057 |
+
if limitation not in limits:
|
| 3058 |
+
limits.append(copy.deepcopy(limitation))
|
| 3059 |
+
return out
|
| 3060 |
+
|
| 3061 |
+
|
| 3062 |
+
def _query_table_groups(session, database, scoped, aggregates, groups, offset, limit):
|
| 3063 |
+
"""Adapt the Table grid's scoped aggregate without reading a row window."""
|
| 3064 |
+
import registry_source
|
| 3065 |
+
import routes_tables
|
| 3066 |
+
from core import relations
|
| 3067 |
+
|
| 3068 |
+
def unavailable(cause):
|
| 3069 |
+
raise err(409, "query_groups_unavailable", cause + " Call table with action query_database "
|
| 3070 |
+
"using retry_query, without groupBy or aggregates, then follow page.next_cursor.")
|
| 3071 |
+
|
| 3072 |
+
if scoped.get("source_kind") != "materialised_user_table":
|
| 3073 |
+
unavailable("This source has no Table aggregate reader.")
|
| 3074 |
+
if len(groups) != 1 or groups[0].get("unit"):
|
| 3075 |
+
unavailable("The Table aggregate reader groups one field at a time without a month unit.")
|
| 3076 |
+
if relations.current_query_link_pin() is not None:
|
| 3077 |
+
unavailable("The Table aggregate reader cannot apply this link pin.")
|
| 3078 |
+
grant = perm_scope.assistant_entry(session.user, database)
|
| 3079 |
+
if session.admin and (grant or {}).get("filter"):
|
| 3080 |
+
unavailable("The Table grid bypasses administrator row filters, so it cannot answer this scoped grouping.")
|
| 3081 |
+
|
| 3082 |
+
aggs, claimed = [], {}
|
| 3083 |
+
for item in aggregates:
|
| 3084 |
+
field = item.get("field")
|
| 3085 |
+
if field is None:
|
| 3086 |
+
continue
|
| 3087 |
+
fn = "average" if item["fn"] == "avg" else item["fn"]
|
| 3088 |
+
if field in claimed and claimed[field] != fn:
|
| 3089 |
+
unavailable("The Table aggregate reader returns one summary per field. Send separate grouped calls for each summary.")
|
| 3090 |
+
if field not in claimed:
|
| 3091 |
+
aggs.append({"colId": field, "agg": fn})
|
| 3092 |
+
claimed[field] = fn
|
| 3093 |
+
|
| 3094 |
+
tree = scoped.get("filters") or {}
|
| 3095 |
+
body = {"filters": copy.deepcopy(tree.get("nodes") or []), "filterConj": tree.get("conj", "and"),
|
| 3096 |
+
"groupBy": groups[0]["field"], "groupOffset": offset, "groupLimit": limit, "aggs": aggs}
|
| 3097 |
+
before = registry_source.generation(session, database)
|
| 3098 |
+
if before is None:
|
| 3099 |
+
unavailable("This Table store cannot track changes for stable group paging.")
|
| 3100 |
+
out = routes_tables.table_aggregates(database, body=body, session=session)
|
| 3101 |
+
registry_source.require_generation(session, database, before)
|
| 3102 |
+
source_metadata = _query_source_metadata(scoped, out)
|
| 3103 |
+
limits = source_metadata.get("limits", [])
|
| 3104 |
+
for limitation in limits:
|
| 3105 |
+
# The standing editable-row budget does not mean this aggregate was refused.
|
| 3106 |
+
if (limitation.get("effect") in {"aggregate_ignored", "group_ignored", "filter_ignored", "filter_unresolved"}
|
| 3107 |
+
or (limitation.get("subject"), limitation.get("effect")) in {
|
| 3108 |
+
("pids", "unresolved"), ("bytes", "refused")}):
|
| 3109 |
+
unavailable(str(limitation.get("cause") or "The Table aggregate reader cannot compute this grouping or summary.")
|
| 3110 |
+
+ " " + str(limitation.get("recommendation") or "Use describe_database to choose supported fields."))
|
| 3111 |
+
if out.get("groups") is None:
|
| 3112 |
+
unavailable("The Table aggregate reader returned no grouped result.")
|
| 3113 |
+
|
| 3114 |
+
def values(container, count):
|
| 3115 |
+
if any(field not in (container or {}) for field in claimed):
|
| 3116 |
+
unavailable("The Table aggregate reader returned an incomplete requested summary.")
|
| 3117 |
+
return [{**item, "value": count if item.get("field") is None
|
| 3118 |
+
else container[item["field"]]} for item in aggregates]
|
| 3119 |
+
|
| 3120 |
+
return {**{key: copy.deepcopy(value) for key, value in scoped.items() if key != "records"},
|
| 3121 |
+
**source_metadata,
|
| 3122 |
+
"source_version": {"table_generation": _json_digest(before)},
|
| 3123 |
+
"limits": limits,
|
| 3124 |
+
"total_rows": out["total_rows"], "total_groups": out["total_groups"],
|
| 3125 |
+
"server_aggregates": values(out["aggregates"], out["total_rows"]),
|
| 3126 |
+
"groups": [{"keys": {groups[0]["field"]: group["key"]},
|
| 3127 |
+
"aggregates": values(group["aggregates"], group["count"])}
|
| 3128 |
+
for group in out["groups"]],
|
| 3129 |
+
"group_order": "count_desc_key_asc"}
|
| 3130 |
+
|
| 3131 |
+
|
| 3132 |
def _query_database_page(session: Session, database: str, payload: dict):
|
| 3133 |
if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS:
|
| 3134 |
raise err(400, "query_arguments_invalid",
|
|
|
|
| 3162 |
scope_filters = None if identity is not None else payload.get("filters")
|
| 3163 |
scoped = _external_read_scope(
|
| 3164 |
session, database, fields=None, filters=scope_filters, mode="query_schema")
|
| 3165 |
+
schema_metadata = _query_source_metadata(scoped)
|
| 3166 |
if identity is not None and (
|
| 3167 |
scoped.get("source_kind") != "runtime_cache"
|
| 3168 |
or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters):
|
|
|
|
| 3210 |
if groups:
|
| 3211 |
if not aggregates or sorts:
|
| 3212 |
raise err(400, "query_groups_invalid",
|
| 3213 |
+
"Grouped queries require aggregates and no sorts.")
|
|
|
|
|
|
|
|
|
|
| 3214 |
digest = _json_digest({"database": database, **payload, "groupBy": groups,
|
| 3215 |
"limit": limit, "cursor": None})
|
| 3216 |
options = {"groupBy": groups, "offset": 0, "limit": limit}
|
| 3217 |
+
|
| 3218 |
+
def group_page(offset):
|
| 3219 |
+
if not mirror:
|
| 3220 |
+
return _query_table_groups(session, database, scoped, aggregates, groups, offset, limit)
|
| 3221 |
+
return _external_read_scope(
|
| 3222 |
+
session, database, filters=payload.get("filters"),
|
| 3223 |
+
mode=("aggregate", aggregates, {**options, "offset": offset}))
|
| 3224 |
+
|
| 3225 |
+
summary = group_page(0)
|
| 3226 |
snap = _json_digest({"source_version": summary.get("source_version"),
|
| 3227 |
"fields": visible_fields,
|
| 3228 |
"total_groups": summary["total_groups"]})
|
|
|
|
| 3232 |
raise err(409, "query_cursor_stale",
|
| 3233 |
"The permitted result set changed. Start again without a cursor.")
|
| 3234 |
if offset:
|
| 3235 |
+
summary = group_page(offset)
|
|
|
|
|
|
|
| 3236 |
if _json_digest({"source_version": summary.get("source_version"),
|
| 3237 |
"fields": visible_fields,
|
| 3238 |
"total_groups": summary["total_groups"]}) != snap:
|
|
|
|
| 3249 |
sorts=[], page={"limit": limit, "returned": returned, "offset": offset,
|
| 3250 |
"has_more": next_cursor is not None,
|
| 3251 |
"next_cursor": next_cursor},
|
| 3252 |
+
caps={"max_page_size": _QUERY_MAX_LIMIT, "max_group_fields": 4 if mirror else 1,
|
| 3253 |
"max_sorts": _QUERY_MAX_SORTS, "max_aggregates": _QUERY_MAX_AGGREGATES,
|
| 3254 |
"max_filter_depth": _QUERY_MAX_FILTER_DEPTH,
|
| 3255 |
"max_filter_nodes": _QUERY_MAX_FILTER_NODES,
|
| 3256 |
"max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS})
|
| 3257 |
+
result.update(_query_source_metadata(schema_metadata, summary))
|
| 3258 |
external_freshness.add(result, session)
|
| 3259 |
_assert_finite_query_value(result)
|
| 3260 |
external_activity.record_ok(session, database)
|
| 3261 |
return result
|
| 3262 |
|
| 3263 |
server_aggregates = None
|
| 3264 |
+
summary = {}
|
| 3265 |
paged_total = None
|
| 3266 |
if mirror:
|
| 3267 |
# ⭐⭐ S67-T34 (D-616, D-2631): THE PERMISSION FILTER IS INSIDE THE QUERY. The aggregates
|
|
|
|
| 3335 |
"max_filter_nodes": _QUERY_MAX_FILTER_NODES,
|
| 3336 |
"max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS},
|
| 3337 |
}
|
| 3338 |
+
result.update(_query_source_metadata(schema_metadata, summary, first, page))
|
| 3339 |
external_freshness.add(result, session)
|
| 3340 |
_assert_finite_query_value(result)
|
| 3341 |
external_activity.record_ok(session, database)
|
|
|
|
| 3416 |
"max_filter_nodes": _QUERY_MAX_FILTER_NODES,
|
| 3417 |
"max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS},
|
| 3418 |
}
|
| 3419 |
+
result.update(_query_source_metadata(schema_metadata, scoped))
|
| 3420 |
external_freshness.add(result, session)
|
| 3421 |
_assert_finite_query_value(result)
|
| 3422 |
external_activity.record_ok(session, database)
|
| 3423 |
return result
|
| 3424 |
|
| 3425 |
|
| 3426 |
+
def _view_context(session, database, event, *, schema_only=False):
|
| 3427 |
import routes_grid
|
| 3428 |
|
| 3429 |
scope = routes_grid._scope_or_400(
|
|
|
|
| 3446 |
from routes_products import MODULE, product_assembly
|
| 3447 |
|
| 3448 |
session.require(MODULE)
|
| 3449 |
+
g = product_assembly(session, consume_corrections=False,
|
| 3450 |
+
**({"with_rows": False} if schema_only else {}))
|
| 3451 |
else:
|
| 3452 |
from routes_customers import grid_assembly
|
| 3453 |
|
| 3454 |
session.require(routes_grid.MODULE)
|
| 3455 |
+
g = grid_assembly(session, scope=scope, consume_corrections=False,
|
| 3456 |
+
**({"with_rows": False} if schema_only else {}))
|
| 3457 |
except HTTPException as exc:
|
| 3458 |
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 3459 |
raise err(exc.status_code, info.get("code", "view_refused"),
|
| 3460 |
f"Database {database}: {info.get('message') or 'The app refused access.'}") from exc
|
| 3461 |
ctx = routes_grid._ctx(
|
| 3462 |
session, g["fields"], g["pids"], scope_key=scope,
|
| 3463 |
+
**({"permission_fields": (g.get("_permission_scope") or {}).get("fields")} if schema_only else {}),
|
| 3464 |
measure_keys=frozenset(m["key"] for m in g["measures"]),
|
| 3465 |
resolved_ids=frozenset(g["measure_sets"]),
|
| 3466 |
cohort_ids=frozenset(c["id"] for c in g["lists"]),
|
|
|
|
| 3547 |
_external_read_scope(session, database, fields=[], filters=None)
|
| 3548 |
|
| 3549 |
|
| 3550 |
+
def _resolve_saved_view_cell_ref(session, reference):
|
| 3551 |
+
"""Authorize a saved operand through the current workbook and scoped cell read doors."""
|
| 3552 |
+
from core import sheets
|
| 3553 |
+
from core.sheet_structure import col_to_index
|
| 3554 |
+
|
| 3555 |
+
session.require("sheets")
|
| 3556 |
+
ctx = _tool_context(session)
|
| 3557 |
+
try:
|
| 3558 |
+
book = sheets.get_workbook(session.runtime, reference["workbook_id"],
|
| 3559 |
+
before_normalize=ctx.require_reader)
|
| 3560 |
+
ctx.require_reader(book)
|
| 3561 |
+
except (HTTPException, sheets.SheetRefusal) as exc:
|
| 3562 |
+
code = (exc.detail.get("error", {}).get("code")
|
| 3563 |
+
if isinstance(exc, HTTPException) else exc.code)
|
| 3564 |
+
if code == "workbook_missing":
|
| 3565 |
+
raise err(404, "workbook_missing", "The referenced workbook is unavailable or unreadable. "
|
| 3566 |
+
"Choose a workbook you can read, or use a literal value.") from exc
|
| 3567 |
+
raise
|
| 3568 |
+
tab = sheets.find_tab(book, reference["tab"])
|
| 3569 |
+
address = reference["ref"].replace("$", "").upper()
|
| 3570 |
+
match = re.fullmatch(r"([A-Z]+)([1-9][0-9]*)", address)
|
| 3571 |
+
column, row = col_to_index(match[1]), int(match[2])
|
| 3572 |
+
|
| 3573 |
+
def refuse(cause):
|
| 3574 |
+
raise err(403, "view_filters_unsupported", "Cell reference refused: " + cause +
|
| 3575 |
+
". Choose a readable workbook and cell, or use a literal value.")
|
| 3576 |
+
|
| 3577 |
+
if column > tab["cols"] or row > tab["rows"]:
|
| 3578 |
+
refuse("the cell is outside the tab dimensions")
|
| 3579 |
+
if (tab.get("visibility", "visible") != "visible" or row in tab.get("hiddenRows", [])
|
| 3580 |
+
or column in tab.get("hiddenCols", [])):
|
| 3581 |
+
refuse("the cell is hidden")
|
| 3582 |
+
# The external projection masks out-of-key source values before recomputing formulas.
|
| 3583 |
+
cell = tab.get("cells", {}).get(address, {})
|
| 3584 |
+
sources = ([cell] if cell.get("kind") != "formula" else
|
| 3585 |
+
[item for sheet in book.get("sheets", []) for item in sheet.get("cells", {}).values()])
|
| 3586 |
+
if any(item.get("kind") == "bound" and not external_access.database_allowed(
|
| 3587 |
+
_key_scope(session)[1], (item.get("binding") or {}).get("database")) for item in sources):
|
| 3588 |
+
refuse("the cell source is outside this connection's database scope")
|
| 3589 |
+
projected = sheets.find_tab(ctx.project_book(book), tab["id"])
|
| 3590 |
+
cell = projected.get("cells", {}).get(address, {})
|
| 3591 |
+
if cell.get("state") == "refused" or cell.get("error") or cell.get("refusal"):
|
| 3592 |
+
refuse("the cell is unreadable under the current source permissions or needs Refresh")
|
| 3593 |
+
return True
|
| 3594 |
+
|
| 3595 |
+
|
| 3596 |
+
def _saved_view_filters(config, *, resolve_cell_ref=None):
|
| 3597 |
"""Convert declared query groups to native View groups and refuse any lossy normalization."""
|
| 3598 |
value = config.get("filters", [])
|
| 3599 |
_assert_query_filter_bounds([value] if isinstance(value, dict) else value)
|
|
|
|
| 3613 |
or not isinstance(raw[key], list) or not raw[key]):
|
| 3614 |
refuse()
|
| 3615 |
return {"conj": raw.get("conj", "and"), "children": [node(child) for child in raw[key]]}
|
|
|
|
|
|
|
|
|
|
| 3616 |
op = raw.get("op")
|
| 3617 |
if not isinstance(op, str) or op not in aios_grid.FILTER_OPS:
|
| 3618 |
name = json.dumps(op[:120]) if isinstance(op, str) else "(missing or invalid)"
|
| 3619 |
refuse(f"Unsupported filter operator {name}. Use a supported operator: "
|
| 3620 |
+ ", ".join(sorted(aios_grid.FILTER_OPS)) + ".")
|
| 3621 |
if (set(raw) - {"colId", "op", "value", "value2", "values", "id", "locked",
|
| 3622 |
+
"dateWindow", "dateMode", "rhs", "valueRef", "value2Ref"}
|
| 3623 |
or not isinstance(raw.get("colId"), str)):
|
| 3624 |
refuse()
|
| 3625 |
for key in ("value", "value2"):
|
| 3626 |
value = raw.get(key)
|
| 3627 |
if isinstance(value, (dict, list)) or value is not None and len(str(value)) > 500:
|
| 3628 |
refuse()
|
| 3629 |
+
for key in ("valueRef", "value2Ref"):
|
| 3630 |
+
if key not in raw:
|
| 3631 |
+
continue
|
| 3632 |
+
reference = aios_grid.clean_filter_cell_ref(raw[key])
|
| 3633 |
+
if reference is None:
|
| 3634 |
+
refuse(f"{key} needs {{workbook_id, tab, ref: 'A1'}} and optional boolean locked. "
|
| 3635 |
+
"Use one cell address, or a literal value.")
|
| 3636 |
+
if resolve_cell_ref is None or resolve_cell_ref(reference) is not True:
|
| 3637 |
+
refuse("Cell references require a caller-authorised workbook and cell resolver. "
|
| 3638 |
+
"Use a readable workbook and cell, or a literal value.")
|
| 3639 |
return copy.deepcopy(raw)
|
| 3640 |
|
| 3641 |
if isinstance(value, dict):
|
|
|
|
| 3740 |
}
|
| 3741 |
|
| 3742 |
|
| 3743 |
+
def _view_filter_members(nodes, *, new=False, parent=None, seed=None):
|
| 3744 |
+
"""Address filter nodes by their C1 identity without cleaning their values."""
|
| 3745 |
+
members = {}
|
| 3746 |
+
for index, node in enumerate(nodes):
|
| 3747 |
+
node_id = node.get("id")
|
| 3748 |
+
if not node_id:
|
| 3749 |
+
if not new:
|
| 3750 |
+
raise err(409, "view_refused", "This View needs stable filter ids before its filters can be edited. "
|
| 3751 |
+
"Open it in Table after the View update is available.")
|
| 3752 |
+
node_id = uuid.uuid5(uuid.NAMESPACE_URL, f"external-view-filter:{seed}:{parent}:{index}").hex
|
| 3753 |
+
if not isinstance(node_id, str) or not node_id or "filter:" + node_id in members:
|
| 3754 |
+
raise err(400, "view_filters_unsupported", "Each filter needs its own text id.")
|
| 3755 |
+
value = {**copy.deepcopy(node), "id": node_id}
|
| 3756 |
+
children = value.pop("children", None)
|
| 3757 |
+
conjunction = value.pop("conj", "and") if children is not None else None
|
| 3758 |
+
members["filter:" + node_id] = {"node": value, "parent": parent}
|
| 3759 |
+
if children is not None:
|
| 3760 |
+
value["children"] = []
|
| 3761 |
+
members["filterConj:" + node_id] = conjunction
|
| 3762 |
+
nested = _view_filter_members(children, new=new, parent=node_id, seed=seed)
|
| 3763 |
+
if members.keys() & nested.keys():
|
| 3764 |
+
raise err(400, "view_filters_unsupported", "Each filter needs its own text id.")
|
| 3765 |
+
members.update(nested)
|
| 3766 |
+
return members
|
| 3767 |
+
|
| 3768 |
+
|
| 3769 |
+
def _view_members(view, config_keys, fields, *, new_filters=False, seed=None, stored=False):
|
| 3770 |
+
"""Translate the published draft config to C1 members, leaving validation to Table.
|
| 3771 |
+
|
| 3772 |
+
`stored` reads positions as the document holds them, with no seed for a legacy rule."""
|
| 3773 |
+
config = view.get("config") or {}
|
| 3774 |
+
members = {key: copy.deepcopy(view[key]) for key in ("name", "note") if key in view}
|
| 3775 |
+
if "visible" in config_keys:
|
| 3776 |
+
visible = set(config.get("visible", fields))
|
| 3777 |
+
members.update({"hidden:" + key: key not in visible for key in fields})
|
| 3778 |
+
if "sorts" in config_keys:
|
| 3779 |
+
for index, rule in enumerate(config.get("sorts") or []):
|
| 3780 |
+
pos = rule.get("pos") or (None if stored else f"{index + 1:08x}")
|
| 3781 |
+
members["sort:" + rule["colId"]] = {"dir": rule["dir"], **({"pos": pos} if pos else {})}
|
| 3782 |
+
if "order" in config_keys:
|
| 3783 |
+
positions = config.get("positions") or {}
|
| 3784 |
+
members.update({"position:" + key: positions.get(key) or f"{index + 1:08x}"
|
| 3785 |
+
for index, key in enumerate(config.get("order") or [])
|
| 3786 |
+
if positions.get(key) or not stored})
|
| 3787 |
+
if "filters" in config_keys:
|
| 3788 |
+
members.update(_view_filter_members(config.get("filters") or [], new=new_filters, seed=seed))
|
| 3789 |
+
if "memberPids" in config_keys:
|
| 3790 |
+
members.update({"memberPid:" + str(pid): True for pid in config.get("memberPids") or []})
|
| 3791 |
+
for key, member in (("groupBy", "group"), ("display", "display"), ("filterConj", "filterConj")):
|
| 3792 |
+
if key in config_keys:
|
| 3793 |
+
members[member] = copy.deepcopy(config.get(key, "and" if key == "filterConj" else None))
|
| 3794 |
+
return members
|
| 3795 |
+
|
| 3796 |
+
|
| 3797 |
+
def _view_operation_plan(database, owner, before, raw, fields, *, execution_id=None):
|
| 3798 |
+
"""Pin only requested member changes so confirmation never re-diffs a stale View snapshot."""
|
| 3799 |
+
from core import write_execution
|
| 3800 |
+
execution_id = execution_id or write_execution.new_id()
|
| 3801 |
+
keys = set(raw["config"])
|
| 3802 |
+
old_config = before.get("config") or {}
|
| 3803 |
+
for key in ("filters", "order"):
|
| 3804 |
+
if raw["config"].get(key, []) == old_config.get(key, []):
|
| 3805 |
+
keys.discard(key)
|
| 3806 |
+
old_sorts = old_config.get("sorts") or []
|
| 3807 |
+
if raw["config"].get("sorts", []) == [
|
| 3808 |
+
{key: value for key, value in rule.items() if key != "pos"} for rule in old_sorts]:
|
| 3809 |
+
keys.discard("sorts")
|
| 3810 |
+
prior = _view_members(before, keys, fields)
|
| 3811 |
+
# The preview shows the stored value. A seeded position the document never held is not a before value.
|
| 3812 |
+
shown = _view_members(before, keys, fields, stored=True)
|
| 3813 |
+
desired = _view_members(raw, keys, fields, new_filters=True, seed=execution_id)
|
| 3814 |
+
if "sorts" in keys:
|
| 3815 |
+
same_order = [rule["colId"] for rule in raw["config"].get("sorts") or []] == [
|
| 3816 |
+
rule["colId"] for rule in old_sorts]
|
| 3817 |
+
for index, rule in enumerate(raw["config"].get("sorts") or []):
|
| 3818 |
+
if same_order:
|
| 3819 |
+
desired["sort:" + rule["colId"]]["pos"] = prior["sort:" + rule["colId"]]["pos"]
|
| 3820 |
+
if "locked" in rule and bool(rule["locked"]) != next((bool(old.get("locked")) for old in old_sorts
|
| 3821 |
+
if old["colId"] == rule["colId"]), False):
|
| 3822 |
+
raise err(409, "view_refused", "Change sort locks in Table before editing this View through the assistant.")
|
| 3823 |
+
# Position members have no remove spelling. Omitted columns keep their stored positions.
|
| 3824 |
+
for member in prior.keys() - desired.keys():
|
| 3825 |
+
if member.startswith("position:") or member.startswith("filterConj:"):
|
| 3826 |
+
desired[member] = copy.deepcopy(prior[member])
|
| 3827 |
+
# An omitted draft property leaves that member alone.
|
| 3828 |
+
if "note" not in raw:
|
| 3829 |
+
desired.pop("note", None)
|
| 3830 |
+
prior.pop("note", None)
|
| 3831 |
+
ops = []
|
| 3832 |
+
for member in sorted(prior.keys() | desired.keys()):
|
| 3833 |
+
if (member in prior, prior.get(member)) == (member in desired, desired.get(member)):
|
| 3834 |
+
continue
|
| 3835 |
+
ops.append({"view_id": raw["id"], "member": member,
|
| 3836 |
+
"op": "set" if member in desired else "remove",
|
| 3837 |
+
"value": copy.deepcopy(desired.get(member)), "client_op_id": uuid.uuid5(
|
| 3838 |
+
uuid.NAMESPACE_URL, f"external-view-op:{execution_id}:{raw['id']}:{member}").hex})
|
| 3839 |
+
return {"database": database, "view_id": raw["id"], "owner": owner,
|
| 3840 |
+
"createdBy": before.get("createdBy"), "ops": ops,
|
| 3841 |
+
"before": {op["member"]: copy.deepcopy(shown.get(op["member"])) for op in ops},
|
| 3842 |
+
"config_keys": sorted(keys)}
|
| 3843 |
+
|
| 3844 |
+
|
| 3845 |
def _view_event(session, kind, args):
|
| 3846 |
from core import grid_events
|
| 3847 |
|
|
|
|
| 3869 |
if kind == "view_upsert":
|
| 3870 |
# Resolve the ordinary View/database wall before the additional T31 draft validation so
|
| 3871 |
# the established refusal vocabulary and connector record-scope guard stay intact.
|
| 3872 |
+
raw = {**raw, "config": _saved_view_filters(
|
| 3873 |
+
raw["config"], resolve_cell_ref=lambda ref: _resolve_saved_view_cell_ref(session, ref))}
|
| 3874 |
event["view"] = raw
|
| 3875 |
_validate_personal_view_draft(session, database, raw)
|
| 3876 |
existing = ctx.table.find_view(view_id, prefer_owner=session.uname)
|
| 3877 |
if (existing and str(existing[0]).strip().lower()
|
| 3878 |
!= str(session.uname).strip().lower()):
|
| 3879 |
raise err(404, "view_unavailable",
|
| 3880 |
+
f"View {view_id}: another user's personal draft is read-only. Ask the view's owner for edit access.")
|
| 3881 |
from core import shares
|
| 3882 |
share_record = shares.grants("view", view_id, st=session.runtime)
|
| 3883 |
if share_record.get("owner") or share_record.get("entries"):
|
| 3884 |
raise err(409, "view_is_shared",
|
| 3885 |
"This View has entered the sharing workflow and cannot be changed by the "
|
| 3886 |
"personal-draft tool. Use a separately confirmed View change.")
|
| 3887 |
+
plan = _VIEW_OPERATION_PLAN.get()
|
| 3888 |
+
if plan is not None and (kind != "view_upsert" or not existing
|
| 3889 |
+
or plan["database"] != database or plan["view_id"] != view_id
|
| 3890 |
+
or plan["owner"] != existing[0] or plan["createdBy"] != existing[1].get("createdBy")):
|
| 3891 |
+
raise err(409, "view_refused", "This View is no longer available for the previewed edit. Open it and preview again.")
|
| 3892 |
+
if kind == "view_upsert" and existing:
|
| 3893 |
+
readable = {field["key"] for field in ctx.fields}
|
| 3894 |
+
config_keys = set(plan["config_keys"] if plan is not None else raw["config"])
|
| 3895 |
+
prior_config = {key: value for key, value in existing[1].get("config", {}).items() if key in config_keys}
|
| 3896 |
+
if set(_view_referenced_fields(prior_config)) - readable:
|
| 3897 |
+
raise err(403, "field_unavailable", "This View includes a field you can no longer read. "
|
| 3898 |
+
"Open it in Table with current access before previewing this edit.")
|
| 3899 |
+
if "memberPids" in config_keys:
|
| 3900 |
+
allowed_pids = {str(pid) for pid in ctx.allowed_pids}
|
| 3901 |
+
if any(str(pid) not in allowed_pids for pid in prior_config.get("memberPids") or []):
|
| 3902 |
+
raise err(403, "out_of_scope", "This View includes records outside your current access. "
|
| 3903 |
+
"Open it with current record access before previewing this edit.")
|
| 3904 |
+
if not callable(getattr(grid_events, "clean_view_member", None)):
|
| 3905 |
+
raise err(503, "view_refused", "View edits are unavailable until Table's View update is installed. Try again after the update.")
|
| 3906 |
+
plan = plan or _view_operation_plan(database, existing[0], existing[1], raw,
|
| 3907 |
+
[field["key"] for field in ctx.fields], execution_id=event["id"])
|
| 3908 |
+
if not plan["ops"]:
|
| 3909 |
+
raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.")
|
| 3910 |
try:
|
| 3911 |
before = ctx.table.find_view(view_id, prefer_owner=session.uname)
|
| 3912 |
+
rerender = False
|
| 3913 |
+
for item in ([{"id": op["client_op_id"], "type": "view_op", **op} for op in plan["ops"]]
|
| 3914 |
+
if plan is not None else [event]):
|
| 3915 |
+
rerender = grid_events.handle_one(item, ctx) or rerender
|
| 3916 |
+
if ctx.out.refusals or ctx.out.toast:
|
| 3917 |
+
break
|
| 3918 |
except grid_events.StoreUnavailable as exc:
|
| 3919 |
raise err(503, "store_unavailable", "The View is unavailable right now.") from exc
|
| 3920 |
if ctx.out.refusals:
|
|
|
|
| 3925 |
raise err(403, "view_refused", f"View {view_id}: {ctx.out.toast}")
|
| 3926 |
after = ctx.table.find_view(view_id, prefer_owner=session.uname)
|
| 3927 |
changed = before != after
|
| 3928 |
+
if not changed and plan is None:
|
| 3929 |
raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.")
|
| 3930 |
# ⛔⛔ `handle_one`'s BOOL IS A RERENDER FLAG, NOT A SUCCESS FLAG, and reading it as success is
|
| 3931 |
# the mistake this comment exists to stop the next reader making. Its own contract block
|
|
|
|
| 3943 |
(not after or str(after[0]).strip().lower()
|
| 3944 |
!= str(session.uname).strip().lower())):
|
| 3945 |
raise err(409, "view_not_personal", f"View {view_id}: the personal draft was not saved.")
|
| 3946 |
+
result = {"database": database, "view_id": view_id, "rerender": bool(rerender),
|
| 3947 |
+
"draft": "personal", "shared": False}
|
| 3948 |
+
if plan is not None:
|
| 3949 |
+
observed = _view_members(after[1], plan["config_keys"], [field["key"] for field in ctx.fields])
|
| 3950 |
+
# A different member's diff cannot acknowledge an ignored operation.
|
| 3951 |
+
for op in plan["ops"]:
|
| 3952 |
+
member = op["member"]
|
| 3953 |
+
expected = (True, op["value"]) if op["op"] == "set" else (False, None)
|
| 3954 |
+
if (member in observed, observed.get(member)) != expected:
|
| 3955 |
+
# A handler that wrote nothing keeps the published silent no-op refusal.
|
| 3956 |
+
if not changed:
|
| 3957 |
+
raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.")
|
| 3958 |
+
raise err(409, "view_refused", f"View {view_id}: a requested change was not saved. "
|
| 3959 |
+
"Open it and preview again.")
|
| 3960 |
+
result["_view_operation_plan"] = plan
|
| 3961 |
+
result["_view_operation_diff"] = [
|
| 3962 |
+
{"path": "/views/" + view_id + "/" + op["member"],
|
| 3963 |
+
"before": copy.deepcopy(plan["before"].get(op["member"])),
|
| 3964 |
+
"after": copy.deepcopy(observed.get(op["member"]))} for op in plan["ops"]]
|
| 3965 |
+
result["_view_operation_diff"] = [item for item in result["_view_operation_diff"]
|
| 3966 |
+
if item["before"] != item["after"]]
|
| 3967 |
+
if not result["_view_operation_diff"]:
|
| 3968 |
+
raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.")
|
| 3969 |
+
return result
|
| 3970 |
|
| 3971 |
|
| 3972 |
def _trusted_activity_via(session):
|
|
|
|
| 4453 |
return tables
|
| 4454 |
|
| 4455 |
|
| 4456 |
+
_CONNECTED_SOURCE_FIELD_REASON = (
|
| 4457 |
+
"This database's source fields come from its connector and cannot be added, changed or deleted here."
|
| 4458 |
+
)
|
| 4459 |
+
_CONNECTED_SOURCE_FIELD_FIX = (
|
| 4460 |
+
"Add the field to a writable Table database instead. Call table with action list_databases to find one. "
|
| 4461 |
+
"Then call table with action database.field_create and arguments containing action field_create, "
|
| 4462 |
+
"its database key and the same field."
|
| 4463 |
+
)
|
| 4464 |
+
|
| 4465 |
+
|
| 4466 |
+
def _connected_formula_create(session, args):
|
| 4467 |
+
"""Identify formula creation that belongs to the connected database's workspace."""
|
| 4468 |
+
from core import user_tables
|
| 4469 |
+
|
| 4470 |
+
raw = args.get("field")
|
| 4471 |
+
return (args.get("action") == "field_create" and isinstance(raw, dict)
|
| 4472 |
+
and raw.get("type") == "formula"
|
| 4473 |
+
and not user_tables.is_user_table(str(args.get("database") or "").strip(), st=session.runtime))
|
| 4474 |
+
|
| 4475 |
+
|
| 4476 |
+
def _create_connected_formula(session, args):
|
| 4477 |
+
"""Use the app's field event to admit and persist a connected-source formula definition."""
|
| 4478 |
+
import routes_tables
|
| 4479 |
+
from core import grid_events, write_execution
|
| 4480 |
+
|
| 4481 |
+
database = str(args.get("database") or "").strip()
|
| 4482 |
+
_assert_database_scope(session, database)
|
| 4483 |
+
_assert_write_scope(session, "database_object")
|
| 4484 |
+
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 4485 |
+
event = {"id": write_execution.new_id(), "type": "field_upsert", "field": args["field"]}
|
| 4486 |
+
ctx = _view_context(session, database, event, schema_only=True)
|
| 4487 |
+
key = str(args["field"].get("key") or "")[:80]
|
| 4488 |
+
if key in {field["key"] for field in ctx.fields} or key in ctx.hidden_keys:
|
| 4489 |
+
raise err(400, "field_refused", "That field key is already in use or unavailable. Choose a new custom_ key.")
|
| 4490 |
+
try:
|
| 4491 |
+
grid_events.handle_one(event, ctx)
|
| 4492 |
+
except grid_events.StoreUnavailable as exc:
|
| 4493 |
+
raise err(503, "store_unavailable", "The formula field could not be saved. Try again.") from exc
|
| 4494 |
+
if ctx.out.refusals:
|
| 4495 |
+
refusal = ctx.out.refusals[0]
|
| 4496 |
+
raise err(400, "field_refused", refusal.get("message") or "The app refused this formula field.")
|
| 4497 |
+
written = ctx.out.writes.get(event["id"], {})
|
| 4498 |
+
if written.get("refused") or not isinstance(written.get("value"), dict):
|
| 4499 |
+
raise err(409, "field_refused", "The formula field was not saved. Check its key and definition.")
|
| 4500 |
+
return {"database": database, "action": "field_create", "field_key": key,
|
| 4501 |
+
"id": key, "_connected_formula": True}
|
| 4502 |
+
|
| 4503 |
+
|
| 4504 |
def _plan_database_write(session, args):
|
| 4505 |
import routes_tables
|
| 4506 |
from core import user_tables
|
|
|
|
| 4514 |
_assert_write_scope(session, "database_object")
|
| 4515 |
if action == "rename":
|
| 4516 |
return _plan_database_rename(session, args)
|
| 4517 |
+
if _connected_formula_create(session, args):
|
| 4518 |
+
if not isinstance(session.runtime, _PreviewRuntime):
|
| 4519 |
+
session = Session(session.tenant, session.user, session.claims, _PreviewRuntime(session.runtime))
|
| 4520 |
+
return _create_connected_formula(session, args)
|
| 4521 |
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 4522 |
if action == "record_create":
|
| 4523 |
routes_tables._adds_or_refuse(database)
|
|
|
|
| 4525 |
tables_before = _database_write_tables(session.runtime, (database,))
|
| 4526 |
table = copy.deepcopy(tables_before.get(database) or {})
|
| 4527 |
if not table or not user_tables.is_user_table(database, st=session.runtime):
|
| 4528 |
+
reason = (_CONNECTED_SOURCE_FIELD_REASON if action.startswith("field_")
|
| 4529 |
+
else "Records from connected sources are read only.")
|
| 4530 |
+
raw_field = args.get("field") if isinstance(args.get("field"), dict) else {}
|
| 4531 |
+
if action == "field_create" and str(raw_field.get("type") or "").strip().lower() == "link":
|
| 4532 |
+
# R22 keeps this door closed. The administrator's registry Link door is the REST one.
|
| 4533 |
+
reason += (" An administrator creates a Link field on a connected registry table "
|
| 4534 |
+
"through POST /api/v1/tables/{key}/fields.")
|
| 4535 |
+
raise err(403, "connected_source_read_only", reason)
|
| 4536 |
revision = _bucket_revision(_database_write_source(session.runtime), user_tables.STORE_KEY,
|
| 4537 |
tables_before)
|
| 4538 |
if type(revision.get("rev")) is not int:
|
|
|
|
| 4616 |
if not isinstance(raw, dict):
|
| 4617 |
raise err(400, "tool_arguments_invalid", "field must be an object.")
|
| 4618 |
have = list(table.get("fields") or [])
|
| 4619 |
+
# The preview definition's own ceiling, the one the REST door reads. None means no cap.
|
| 4620 |
+
field_cap = user_tables.field_limit(database, session.runtime, definition=table)
|
| 4621 |
field = user_tables._clean_field(
|
| 4622 |
raw, recurrence_validator=recurrence_validator,
|
| 4623 |
valid_keys={str(item.get("key")) for item in have if item.get("key")})
|
| 4624 |
if (not field or any(item.get("key") == field.get("key") for item in have)
|
| 4625 |
+
or (field_cap is not None and len(have) >= field_cap)
|
| 4626 |
or not user_tables.dynamic_link_filter_valid(field, have)
|
| 4627 |
or not user_tables.flow_bound(field.get("automation"), session.runtime)):
|
| 4628 |
raise err(400, "field_refused", "The field does not pass the database's schema rules.")
|
|
|
|
| 4710 |
return database_schema(database, session)
|
| 4711 |
if action == "query":
|
| 4712 |
return _query_database(session, database, args.get("query") or {})
|
| 4713 |
+
if _connected_formula_create(session, args):
|
| 4714 |
+
return _create_connected_formula(session, args)
|
| 4715 |
plan = plan or _plan_database_write(session, args)
|
| 4716 |
if action == "rename":
|
| 4717 |
import routes_nav
|
|
|
|
| 5365 |
|
| 5366 |
|
| 5367 |
def _preview_external_write(session, name, arguments, *, issue_token=True, execution=None, purpose=None, about=None,
|
| 5368 |
+
intent=None, view_operation_plan=None):
|
| 5369 |
from core import write_execution
|
| 5370 |
if isinstance(arguments, dict) and "about" in arguments:
|
| 5371 |
if about is not None:
|
|
|
|
| 5388 |
tenant=session.tenant, user=copy.deepcopy(session.user),
|
| 5389 |
claims=copy.deepcopy(session.claims), runtime=preview_runtime)
|
| 5390 |
execution = execution or {'seed': uuid.uuid4().hex, 'timestamp': time.time()}
|
| 5391 |
+
view_binding = _VIEW_OPERATION_PLAN.set(view_operation_plan)
|
| 5392 |
+
try:
|
| 5393 |
+
with write_execution.replay(execution):
|
| 5394 |
+
if _is_website_note(name, normalised):
|
| 5395 |
+
import website_tools
|
| 5396 |
+
result = website_tools.plan_internal_note(_tool_context(preview_session, preview=True), normalised["arguments"])
|
| 5397 |
+
normalised = {"action": "add_internal_note", "arguments": copy.deepcopy(result["arguments"])}
|
| 5398 |
+
else:
|
| 5399 |
+
result = _dispatch_external_tool(_tool_context(preview_session, preview=True), name, normalised, about=about)
|
| 5400 |
+
finally:
|
| 5401 |
+
_VIEW_OPERATION_PLAN.reset(view_binding)
|
| 5402 |
if isinstance(result, dict) and result.get("refused"):
|
| 5403 |
raise err(409, "write_partial_refused",
|
| 5404 |
"Part of this change could not be accepted. Nothing was previewed.")
|
| 5405 |
|
| 5406 |
remote_note = _is_website_note(name, normalised)
|
| 5407 |
+
view_plan = result.get("_view_operation_plan") if isinstance(result, dict) else None
|
| 5408 |
+
plan = result if ((name == "database_object" and not result.get("_connected_formula"))
|
| 5409 |
+
or remote_note or _is_members_write(name, normalised)) else None
|
| 5410 |
+
if view_plan is not None:
|
| 5411 |
+
diff = copy.deepcopy(result["_view_operation_diff"])
|
| 5412 |
+
warnings = []
|
| 5413 |
+
affected = {"count": len(view_plan["ops"]), "ids": [view_plan["view_id"]]}
|
| 5414 |
+
current_revision = {key: copy.deepcopy(view_plan[key])
|
| 5415 |
+
for key in ("database", "view_id", "owner", "createdBy")}
|
| 5416 |
+
elif plan is not None:
|
| 5417 |
diff = copy.deepcopy(plan.get("diff") or [])
|
| 5418 |
warnings = list(plan.get("warnings") or [])
|
| 5419 |
affected = copy.deepcopy(plan.get("affected") or {})
|
|
|
|
| 5430 |
affected = _affected_for(name, normalised, diff, result)
|
| 5431 |
current_revision = {
|
| 5432 |
key: revision for key, revision in preview_runtime.base_revisions().items()
|
| 5433 |
+
# Formula promotion also updates its personal workspace when there is nothing to remove.
|
| 5434 |
+
if key in documents or (isinstance(result, dict) and result.get("_connected_formula")
|
| 5435 |
+
and preview_runtime._local_revisions.get(key))
|
| 5436 |
}
|
| 5437 |
if not diff:
|
| 5438 |
raise err(409, "write_not_changed", "The requested write would not change anything.")
|
|
|
|
| 5453 |
|
| 5454 |
_canonical_json(diff, capped=False)
|
| 5455 |
diff_digest = _json_digest(diff)
|
| 5456 |
+
shown_diff, diff_summary = (diff, None) if plan is not None or view_plan is not None else _summarise_diff(diff, projected)
|
| 5457 |
if diff_summary is not None:
|
| 5458 |
warnings.append(f"The preview shows {diff_summary['sample']} of {diff_summary['changes']} changes. "
|
| 5459 |
"confirm_change applies all of them.")
|
|
|
|
| 5476 |
"execution": execution,
|
| 5477 |
"about": about, "about_digest": _json_digest(about), "buildKind": build_kind,
|
| 5478 |
}
|
| 5479 |
+
if view_plan is not None:
|
| 5480 |
+
payload["view_operation_plan"] = copy.deepcopy(view_plan)
|
| 5481 |
if purpose is not None:
|
| 5482 |
payload.update(purpose=purpose, purpose_digest=_json_digest(purpose))
|
| 5483 |
if remote_note:
|
|
|
|
| 5596 |
app = ((doc or {}).get("applications") or {}).get(app_id) or {}
|
| 5597 |
access["databases"].extend(_databases_in_arguments(app))
|
| 5598 |
# Foreign handlers own their object permissions. Pin their consulted documents without calling them on replay.
|
| 5599 |
+
if payload["tool"] not in external_tools.TOOLS and "view_operation_plan" not in payload:
|
| 5600 |
access["documents"] = {key: _json_digest(value) for key, value in staged._initial.items()
|
| 5601 |
if key not in {external_access.BUCKET, external_oauth.BUCKET}}
|
| 5602 |
access["after_documents"] = {key: _json_digest(staged._docs[key]) for key in access["documents"]}
|
|
|
|
| 5681 |
"Ask the owner to restore access before checking again.")
|
| 5682 |
for database in set(access["databases"]):
|
| 5683 |
_assert_database_scope(session, database)
|
| 5684 |
+
if _is_website_note(name, args) or (name == "database_object" and _connected_formula_create(session, args)):
|
| 5685 |
_external_read_scope(session, database, fields=[], mode="schema")
|
| 5686 |
else:
|
| 5687 |
_external_read_scope(session, database, fields=[])
|
|
|
|
| 5692 |
import routes_tables
|
| 5693 |
action = args["action"]
|
| 5694 |
record_pid = str(args.get("record_id") or "").strip()
|
| 5695 |
+
formula = _connected_formula_create(session, args)
|
| 5696 |
+
g = ({"fields": _view_context(session, args["database"], {}, schema_only=True).fields}
|
| 5697 |
+
if formula else routes_tables.ut_write_ctx(session, args["database"],
|
| 5698 |
+
pid=int(record_pid) if action in {"record_update", "record_delete"} and record_pid.isdigit() else None))
|
| 5699 |
if action in {"record_create", "record_update"} and item["status"] == "committed":
|
| 5700 |
if any(int(rid) not in g["pids"] for rid in item["affected"]["ids"]):
|
| 5701 |
raise err(403, "operation_unavailable",
|
| 5702 |
"This connection cannot read every record covered by the operation. "
|
| 5703 |
"Query the database with current access, or ask the owner to restore access before checking again.")
|
| 5704 |
fields = set(args.get("values") or {})
|
| 5705 |
+
if formula and item["status"] == "committed":
|
| 5706 |
+
fields.add(str(args["field"]["key"])[:80])
|
| 5707 |
if args.get("field_key") and action != "field_delete":
|
| 5708 |
fields.add(args["field_key"])
|
| 5709 |
if not fields <= {field["key"] for field in g["fields"]}:
|
| 5710 |
raise err(403, "operation_unavailable",
|
| 5711 |
"This connection cannot read every field covered by the operation. "
|
| 5712 |
"Call data with action describe_database, then ask the owner to restore required access before checking again.")
|
| 5713 |
+
if name in {"view_upsert", "view_delete", "view_object"} or "view_operation_plan" in payload:
|
| 5714 |
+
view_plan = payload.get("view_operation_plan")
|
| 5715 |
+
view_id = view_plan["view_id"] if view_plan else args.get("view_id") or (args.get("view") or {}).get("id")
|
| 5716 |
+
database = view_plan["database"] if view_plan else args["database"]
|
| 5717 |
+
ctx_view = _view_context(session, database, {})
|
| 5718 |
found = ctx_view.table.find_view(view_id, prefer_owner=session.uname)
|
| 5719 |
deleting = name == "view_delete" or args.get("action") == "delete"
|
| 5720 |
if (found and str(found[0]).strip().lower() != session.uname.lower()) or (not found and not deleting):
|
|
|
|
| 5754 |
"This connection cannot read the operation's item with its current access. "
|
| 5755 |
"Read the item with this connection, or ask the owner to restore access before checking again.")
|
| 5756 |
return session
|
| 5757 |
+
if (access["documents"] or "view_operation_plan" in payload) and _json_digest(user) != access["user"]:
|
| 5758 |
raise err(403, "operation_unavailable",
|
| 5759 |
"The account changed since the preview, so receipt access cannot be verified. "
|
| 5760 |
"Read the item with this connection before previewing another change.")
|
|
|
|
| 5981 |
+ str(_CONFIRMATION_TTL_SECONDS // 60) + " minutes.")
|
| 5982 |
|
| 5983 |
|
| 5984 |
+
def _expired_confirmation(session, payload, *, intent=None):
|
| 5985 |
+
"""Return expiry context and a newly scoped preview without committing the change."""
|
| 5986 |
+
window = _CONFIRMATION_TTL_SECONDS
|
| 5987 |
+
age = max(0, int(time.time()) - int(payload["expiry"]) + window)
|
| 5988 |
+
details = {"preview_age_seconds": age, "confirmation_window_seconds": window}
|
| 5989 |
+
message = (f"This preview is {age} seconds old. The confirmation window is {window} seconds. "
|
| 5990 |
+
+ _expired_fix(payload))
|
| 5991 |
+
try:
|
| 5992 |
+
preview = _preview_external_write(
|
| 5993 |
+
session, payload["tool"], copy.deepcopy(payload["arguments"]),
|
| 5994 |
+
purpose=payload.get("purpose"), about=payload.get("about"), intent=intent)
|
| 5995 |
+
except HTTPException as exc:
|
| 5996 |
+
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 5997 |
+
details["fresh_preview_error"] = info.get("code", "tool_unavailable")
|
| 5998 |
+
message += " A fresh preview could not be prepared with current access and data."
|
| 5999 |
+
except sheets.SheetRefusal as exc:
|
| 6000 |
+
details["fresh_preview_error"] = exc.code
|
| 6001 |
+
message += " A fresh preview could not be prepared with current access and data."
|
| 6002 |
+
else:
|
| 6003 |
+
details["fresh_preview"] = {key: value for key, value in preview.items() if not key.startswith("_")}
|
| 6004 |
+
message += " Review error.fresh_preview before confirming its new confirmation_token."
|
| 6005 |
+
return err(409, "confirmation_expired", message, details)
|
| 6006 |
+
|
| 6007 |
+
|
| 6008 |
def _confirm_fix(args):
|
| 6009 |
"""Name what a confirm_change call got wrong and the shape that succeeds."""
|
| 6010 |
allowed = ("confirmation_token", "confirmation_phrase", "intent")
|
|
|
|
| 6037 |
if payload.get("environment") != _write_environment():
|
| 6038 |
raise err(409, "confirmation_stale", "The environment changed after the preview, " + _STALE_FIX)
|
| 6039 |
if int(payload.get("expiry") or 0) <= int(time.time()):
|
| 6040 |
+
raise _expired_confirmation(fresh_session, payload, intent=stored.get("intent"))
|
| 6041 |
if payload.get("strong") and args.get("confirmation_phrase") != _CONFIRMATION_PHRASE:
|
| 6042 |
raise err(400, "confirmation_phrase_required",
|
| 6043 |
"Enter the exact stronger confirmation phrase shown in the preview.")
|
|
|
|
| 6052 |
_confirmation_claim(fresh_session, operation_id)
|
| 6053 |
except _OperationAlreadyClaimed:
|
| 6054 |
return _operation_status(fresh_session, {"operation_id": operation_id})
|
| 6055 |
+
except HTTPException as exc:
|
| 6056 |
+
if isinstance(exc.detail, dict) and exc.detail.get("error", {}).get("code") == "confirmation_expired":
|
| 6057 |
+
raise _expired_confirmation(fresh_session, payload, intent=stored.get("intent")) from None
|
| 6058 |
+
raise
|
| 6059 |
remote_note = _is_website_note(tool, arguments)
|
| 6060 |
capability = None
|
| 6061 |
try:
|
| 6062 |
recomputed = _preview_external_write(
|
| 6063 |
fresh_session, tool, arguments, issue_token=False, execution=execution,
|
| 6064 |
+
purpose=payload.get("purpose"), about=payload.get("about"),
|
| 6065 |
+
view_operation_plan=payload.get("view_operation_plan"))
|
| 6066 |
current = recomputed["_confirmation"]
|
| 6067 |
if (payload.get("arguments_digest") != current.get("arguments_digest")
|
| 6068 |
or payload.get("environment") != current.get("environment")
|
| 6069 |
or payload.get("revision_digest") != current.get("revision_digest")
|
| 6070 |
or payload.get("diff_digest") != current.get("diff_digest")
|
| 6071 |
+
or payload.get("view_operation_plan") != current.get("view_operation_plan")
|
| 6072 |
or (remote_note and any(payload.get(key) != current.get(key)
|
| 6073 |
for key in ("remote_plan_digest", "target_digest", "source_fingerprint")))
|
| 6074 |
or any((key in payload, payload.get(key)) != (key in current, current.get(key))
|
|
|
|
| 6082 |
ctx, recomputed["_plan"], confirmation_token=args["confirmation_token"])
|
| 6083 |
result = website_tools.execute_confirmed_note(ctx, recomputed["_plan"], capability=capability)
|
| 6084 |
return _finish_website_note(fresh_session, payload, recomputed["_plan"], result)
|
| 6085 |
+
if tool == "database_object" and recomputed.get("_plan") is not None:
|
| 6086 |
result = _execute_database_object(
|
| 6087 |
fresh_session, arguments, plan=recomputed.get("_plan"))
|
| 6088 |
elif _is_members_write(tool, arguments):
|
| 6089 |
# The account writer compares the pinned expect itself, so no runtime guard is needed.
|
| 6090 |
result = _dispatch_external_tool(_tool_context(fresh_session), tool, arguments, about=payload["about"])
|
| 6091 |
+
elif payload.get("view_operation_plan") is not None:
|
| 6092 |
+
# Only signed C1 ops use Table's member merge. Other writers retain document CAS.
|
| 6093 |
+
view_binding = _VIEW_OPERATION_PLAN.set(payload["view_operation_plan"])
|
| 6094 |
+
try:
|
| 6095 |
+
with write_execution.replay(execution):
|
| 6096 |
+
result = _dispatch_external_tool(_tool_context(fresh_session), tool, arguments, about=payload["about"])
|
| 6097 |
+
finally:
|
| 6098 |
+
_VIEW_OPERATION_PLAN.reset(view_binding)
|
| 6099 |
else:
|
| 6100 |
from datetime import datetime, timezone
|
| 6101 |
commit_started = datetime.now(timezone.utc)
|
|
|
|
| 6122 |
_confirmation_status(fresh_session, operation_id, "failed")
|
| 6123 |
raise
|
| 6124 |
before_revision = copy.deepcopy(recomputed.get("current_revision") or {})
|
| 6125 |
+
if tool == "database_object" and recomputed.get("_plan") is not None:
|
| 6126 |
from core import user_tables
|
| 6127 |
import routes_nav
|
| 6128 |
bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY
|
|
|
|
| 6133 |
after_revision = {"rev": produced_rev}
|
| 6134 |
elif _is_members_write(tool, arguments):
|
| 6135 |
after_revision = copy.deepcopy(result["revision"])
|
| 6136 |
+
elif payload.get("view_operation_plan") is not None:
|
| 6137 |
+
after_revision = {"view_id": result["view_id"], "members": copy.deepcopy(result["_view_operation_diff"])}
|
| 6138 |
else:
|
| 6139 |
after_revision = copy.deepcopy(guarded_session.runtime._expected)
|
| 6140 |
after_revision_digest = _json_digest(after_revision)
|
|
|
|
| 6170 |
_confirmation_status(fresh_session, operation_id, "committed", response)
|
| 6171 |
if isinstance(stored.get("arguments_ref"), dict):
|
| 6172 |
_drop_operation_documents(fresh_session, [stored["arguments_ref"].get("key")])
|
| 6173 |
+
if ((tool != "database_object" or recomputed.get("_plan") is None) and not _is_members_write(tool, arguments)
|
| 6174 |
and receipt.get("persisted") is True):
|
| 6175 |
try:
|
| 6176 |
targets, native_ids = _committed_build_evidence(fresh_session, guarded_session.runtime, arguments,
|
|
|
|
| 6211 |
tab_missing folder_missing unknown_table unknown_field view_missing view_not_found
|
| 6212 |
sheet_structure_tab_missing unknown_sheet_template cell_missing
|
| 6213 |
"""),
|
| 6214 |
+
("The requested service or data source is unavailable. Try again shortly.", """
|
| 6215 |
tool_unavailable tool_capability_unsupported capability_unavailable external_access_unavailable
|
| 6216 |
store_unavailable store_not_ready store_catching_up oauth_unavailable not_connected
|
| 6217 |
schema_not_loaded query_groups_unavailable confirmation_revision_unavailable
|
| 6218 |
assistant_source_too_large amazon_mirror_not_loaded query_link_unavailable
|
| 6219 |
"""),
|
| 6220 |
+
("This operation conflicts with the item's editing rules. Read its settings before trying again.", """
|
| 6221 |
tool_refused write_partial_refused field_refused view_refused refused
|
| 6222 |
connected_source_read_only computed_field records_read_only loopable_managed
|
| 6223 |
preset_field_locked zone_membership_read_only not_a_user_table
|
|
|
|
| 6286 |
code: reason for reason, codes in _MCP_CODE_GROUPS for code in codes.split()
|
| 6287 |
}
|
| 6288 |
MCP_RESULT_CODES.update({
|
| 6289 |
+
"forbidden": "This connection does not have permission for this operation. Ask an Administrator to check its access.",
|
| 6290 |
+
"tool_not_found": "The requested tool is not published on this connection. Read loopable_guide for the available tools.",
|
| 6291 |
+
"tool_unavailable": "The tool could not complete this request. Try the same call again shortly.",
|
| 6292 |
+
"tool_refused": "The request does not meet this tool's requirements. Read loopable_guide for its accepted actions and arguments.",
|
| 6293 |
+
"tool_capability_unsupported": "This tool does not support the requested capability. Read its guide for supported actions.",
|
| 6294 |
+
"capability_unavailable": "The requested capability is not available to this connection. Read loopable_guide for supported actions.",
|
| 6295 |
+
"external_access_unavailable": "This connection's access could not be checked. Try again shortly.",
|
| 6296 |
+
"store_unavailable": "The workspace's saved data could not be read. Try the same call again shortly.",
|
| 6297 |
+
"store_not_ready": "The workspace's saved data is not ready for this request. Try again after loading completes.",
|
| 6298 |
+
"store_catching_up": "The workspace is still receiving its saved data. Try again after loading completes.",
|
| 6299 |
+
"oauth_unavailable": "This connection's authorization service is unavailable. Try connecting again shortly.",
|
| 6300 |
+
"not_connected": "This database has no connected source. Call table with action list_databases and choose a connected database.",
|
| 6301 |
+
"schema_not_loaded": "This database's saved fields are not loaded. Try again after its source sync completes.",
|
| 6302 |
+
"query_groups_unavailable": "This query's grouping or summaries could not be calculated. Read its permitted records by page instead.",
|
| 6303 |
+
"confirmation_revision_unavailable": "The database version could not be checked. Nothing was saved. Read it and preview the change again.",
|
| 6304 |
+
"assistant_source_too_large": "This source exceeds the read size this action supports. Use a paged Table query instead.",
|
| 6305 |
+
"amazon_mirror_not_loaded": "This Amazon database's saved data is not loaded. Try again after its source sync completes.",
|
| 6306 |
+
"query_link_unavailable": "This query's linked database could not be read. Describe both databases and check their access before retrying.",
|
| 6307 |
+
"connected_source_read_only": _CONNECTED_SOURCE_FIELD_REASON + " " + _CONNECTED_SOURCE_FIELD_FIX,
|
| 6308 |
+
"write_partial_refused": "Some requested edits were refused by the fields' editing rules. Read the refused items before previewing a corrected change.",
|
| 6309 |
+
"field_refused": "This field's editing rules prevent the requested change. Read its definition before previewing a corrected change.",
|
| 6310 |
+
"view_refused": "This View's settings or access rules prevent the requested change. Read the View before previewing a corrected change.",
|
| 6311 |
+
"refused": "This operation could not satisfy the item's access or validation rules. Read the item's settings before retrying.",
|
| 6312 |
+
"computed_field": "This field is calculated from other values and cannot be edited directly. Change its permitted inputs instead.",
|
| 6313 |
+
"records_read_only": "This database's records cannot be edited through this connection. Use a writable Table database for new records.",
|
| 6314 |
+
"loopable_managed": "Loopable manages this system database. Use its dedicated App tools to change it.",
|
| 6315 |
+
"preset_field_locked": "This field is defined by the database's preset and cannot be changed here. Add a separate field to a writable Table database.",
|
| 6316 |
+
"zone_membership_read_only": "Zone membership is managed by its assignment rules. Use the Zone tools to change those rules.",
|
| 6317 |
+
"not_a_user_table": "This operation requires a writable Table database. Call table with action list_databases to choose one.",
|
| 6318 |
+
"metric_unavailable": ("This metric has no validated value within your current access and window. "
|
| 6319 |
+
"Ask an Administrator to check source availability and metric access, or choose another window."),
|
| 6320 |
+
"metric_window_invalid": ("The dates do not form a valid named window. Use real ISO dates in order, "
|
| 6321 |
+
"and a complete calendar month for calendar_month."),
|
| 6322 |
"workbook_name_ambiguous": "More than one readable workbook has this exact name. Use one of the candidate ids.",
|
| 6323 |
"view_filters_unsupported": "These filters cannot be saved without changing their meaning. Use colId, op and value leaves, with conj and children groups.",
|
| 6324 |
"confirmation_session_required": "Read this confirmation through its authenticated connection.",
|
|
|
|
| 6371 |
"request_rate_limited": "This AI has reached its hourly request cap. Try again later.",
|
| 6372 |
})
|
| 6373 |
MCP_RESULT_CODES.update(READ_REFUSAL_FIXES)
|
| 6374 |
+
for _schema_code, _schema_fix in _SCHEMA_READ_REFUSAL_FIXES.items():
|
| 6375 |
+
MCP_RESULT_CODES.setdefault(_schema_code, _schema_fix)
|
| 6376 |
_MCP_INTERNAL_PROSE = re.compile(
|
| 6377 |
r"\b(?:DuckDB|Postgres(?:ql)?|SQLite|Polars|Pandas|SQLAlchemy|PyArrow|HyperFormula)\b|"
|
| 6378 |
r"\b(?:api|web|platform)_[a-z_]+\b|\b(?:verify_\w+|\w+Error)\b|"
|
|
|
|
| 6462 |
|
| 6463 |
|
| 6464 |
_MCP_RESULT_OPAQUE = frozenset({
|
| 6465 |
+
"records", "rows", "groups", "cells", "values", "value", "before", "after", "arguments", "diff",
|
| 6466 |
"confirmation_token", "confirmation_phrase", "diff_digest", "current_revision",
|
| 6467 |
+
"revision_before", "revision_after", "filters", "sorts", "config", "binding", "bindings", "sourceStatus",
|
| 6468 |
})
|
| 6469 |
_MCP_RESULT_IDENTIFIERS = frozenset({
|
| 6470 |
"resourceUri", "deepLink", "pdfLink", "downloadUrl", "src", "websiteUrl", "fileId", "link", "href",
|
| 6471 |
"id", "ids", "operation_id", "key", "keys", "pid", "pids", "database", "databases", "sourceDatabase",
|
| 6472 |
+
"compatibleDatabases", "field", "fields", "field_key", "colId", "metric_key", "groupBy", "group_order",
|
| 6473 |
"workbook_id", "worksheet", "tab", "view_id", "app_id", "catalog_id", "chart_id", "record_id", "contextId",
|
| 6474 |
"path", "paths", "address", "addresses", "range", "cursor", "next_cursor", "token",
|
| 6475 |
"source_version", "code", "tool", "action", "person", "agent", "owner", "author", "from", "to",
|
|
|
|
| 6480 |
"data", "error", "databases", "fields", "tab", "tabs", "workbook", "workbooks", "worksheet",
|
| 6481 |
"worksheets", "file", "files", "request", "requests", "template", "templates", "application",
|
| 6482 |
"applications", "view", "views", "from", "to", "source", "affected", "aggregates", "tools",
|
| 6483 |
+
"annotations", "serverInfo", "environment", "fresh_preview",
|
| 6484 |
})
|
| 6485 |
# A link descriptor names its target database key. It is an identifier, never prose (D-9307).
|
| 6486 |
_MCP_RESULT_DESCRIPTORS = frozenset({"link"})
|
|
|
|
| 6505 |
return copy.deepcopy(value)
|
| 6506 |
result = {}
|
| 6507 |
for key, child in value.items():
|
| 6508 |
+
if key == "retry_query" and value.get("code") in {"query_cursor_stale", "query_groups_unavailable"}:
|
| 6509 |
# These are the caller's original arguments, not explanatory metadata.
|
| 6510 |
result[key] = copy.deepcopy(child)
|
| 6511 |
elif key == "about":
|
|
|
|
| 7615 |
|
| 7616 |
#: Known refusal codes whose MCP answer carries a repair.
|
| 7617 |
_MCP_NEXT_CALL = {"workbook_missing": WORKBOOK_MISSING_FIX, "assistant_unknown_database": UNKNOWN_DATABASE_FIX,
|
| 7618 |
+
"connected_source_read_only": _CONNECTED_SOURCE_FIELD_FIX,
|
| 7619 |
**READ_REFUSAL_FIXES}
|
| 7620 |
|
| 7621 |
|
| 7622 |
def _mcp_next_call(code, message):
|
| 7623 |
+
"""Append the supported repair once for a known refusal."""
|
| 7624 |
text = str(message or "").rstrip()
|
| 7625 |
fix = _MCP_NEXT_CALL.get(code, "")
|
| 7626 |
if code == "assistant_unknown_database":
|
|
|
|
| 7764 |
if purpose is not None and not (writing and (confirm or direct_wiki)):
|
| 7765 |
raise err(400, "tool_arguments_invalid", "Send purpose only with a previewed build.")
|
| 7766 |
if name == "query_database" and isinstance(arguments, dict):
|
| 7767 |
+
_assert_query_filter_bounds(arguments.get("filters"))
|
| 7768 |
external_tools.validate_query_filter_operators(arguments.get("filters"))
|
| 7769 |
_assert_tool_database_scope(session, name, scoped_arguments)
|
| 7770 |
if name == "confirm_change":
|
|
|
|
| 7827 |
import routes_sheet
|
| 7828 |
if exc.code == "tool_arguments_invalid" and name == "confirm_change":
|
| 7829 |
exc = sheets.SheetRefusal(exc.code, _confirm_fix(arguments))
|
| 7830 |
+
if exc.code in {"tool_arguments_invalid", "query_aggregates_invalid"} and _is_query_call(name, arguments):
|
| 7831 |
+
exc = sheets.SheetRefusal(exc.code, _query_grouping_help(with_query_syntax(exc.message)))
|
| 7832 |
if exc.code in _MCP_NEXT_CALL:
|
| 7833 |
exc = sheets.SheetRefusal(exc.code, _mcp_next_call(exc.code, exc.message))
|
| 7834 |
translated = routes_sheet._refuse(exc)
|
|
|
|
| 7838 |
except HTTPException as exc:
|
| 7839 |
if exc.status_code == 401:
|
| 7840 |
_auth_refusal()
|
| 7841 |
+
info = exc.detail.get("error", exc.detail) if isinstance(exc.detail, dict) else {}
|
| 7842 |
+
info = info if isinstance(info, dict) else {}
|
| 7843 |
retry = ({"retry_query": info["retry_query"]}
|
| 7844 |
+
if info.get("code") in {"query_cursor_stale", "query_groups_unavailable"}
|
| 7845 |
+
and "retry_query" in info else {})
|
| 7846 |
details = info.get("details")
|
| 7847 |
if info.get("code") in {"build_about_required", "build_about_invalid", "call_intent_required",
|
| 7848 |
"call_intent_invalid"} and isinstance(details, dict):
|
|
|
|
| 7854 |
retry["suggestions"] = details.get("suggestions", [])
|
| 7855 |
if info.get("code") == "workbook_name_ambiguous" and isinstance(details, dict):
|
| 7856 |
retry["candidates"] = details.get("candidates", [])
|
| 7857 |
+
if info.get("code") == "confirmation_expired" and isinstance(details, dict):
|
| 7858 |
+
retry.update({key: details[key] for key in (
|
| 7859 |
+
"preview_age_seconds", "confirmation_window_seconds", "fresh_preview", "fresh_preview_error")
|
| 7860 |
+
if key in details})
|
| 7861 |
+
public_code = _mcp_public_code(info.get("code"), exc.status_code)
|
| 7862 |
+
detail = exc.detail if isinstance(exc.detail, str) else None
|
| 7863 |
+
if detail == _HTTP_STATUS_REASONS.get(exc.status_code):
|
| 7864 |
+
detail = None
|
| 7865 |
+
message = info.get("message") or info.get("reason") or detail
|
| 7866 |
+
if not isinstance(message, str) or not message.strip():
|
| 7867 |
+
message = MCP_RESULT_CODES[public_code]
|
| 7868 |
if info.get("code") in {"build_about_required", "build_about_invalid"} and isinstance(message, str):
|
| 7869 |
message += _mcp_about_fix(called)
|
| 7870 |
if info.get("code") in _MCP_NEXT_CALL:
|
| 7871 |
message = _mcp_next_call(info["code"], message)
|
| 7872 |
+
out = _tool_refusal(name, info.get("code") or public_code, message, exc.status_code, **retry)
|
| 7873 |
except (TypeError, ValueError, KeyError, AttributeError):
|
| 7874 |
out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
|
| 7875 |
except Exception as exc:
|
api/routes_feedback.py
CHANGED
|
@@ -375,6 +375,11 @@ font-size: var(--lp-fs-xs); }
|
|
| 375 |
"""
|
| 376 |
|
| 377 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 378 |
@router.get("/feedback/log")
|
| 379 |
def feedback_log(tenant: str = "", session: Session = Depends(require_session)):
|
| 380 |
"""Render the existing operator log with one row per distinct AI limitation."""
|
|
@@ -386,7 +391,7 @@ def feedback_log(tenant: str = "", session: Session = Depends(require_session)):
|
|
| 386 |
rows = []
|
| 387 |
for row in data["rows"]:
|
| 388 |
stamp = datetime.fromtimestamp(float(row["ts"]), timezone.utc)
|
| 389 |
-
|
| 390 |
limitation = row.get("category") == "limitation"
|
| 391 |
question = row.get("question") if limitation else row.get("text")
|
| 392 |
detail = (f"{row.get('tool', '')}.{row.get('action', '')}" if limitation
|
|
@@ -404,10 +409,12 @@ def feedback_log(tenant: str = "", session: Session = Depends(require_session)):
|
|
| 404 |
filer_details.append(
|
| 405 |
f'<span class="feedback-detail">{escape(str(user))}: {count:,} {label}, last '
|
| 406 |
f'<time datetime="{escape(last.isoformat(), quote=True)}">'
|
| 407 |
-
f'{
|
|
|
|
| 408 |
rows.append(
|
| 409 |
f'<tr data-limitation="{escape(str(row["id"]), quote=True)}">'
|
| 410 |
-
f'<td><time datetime="{stamp.isoformat()}">{
|
|
|
|
| 411 |
f'<td>{escape(str(row["tenant"]))}</td>'
|
| 412 |
f'<td class="feedback-question">{escape(str(question or ""))}'
|
| 413 |
f'<span class="feedback-detail">{escape(detail)}</span>{"".join(filer_details)}</td>'
|
|
@@ -417,7 +424,8 @@ def feedback_log(tenant: str = "", session: Session = Depends(require_session)):
|
|
| 417 |
'<h1>Team feedback</h1>'
|
| 418 |
f'<p class="feedback-summary">{data["total"]:,} {item_label}</p>')
|
| 419 |
if rows:
|
| 420 |
-
body += ('<div class="table-scroll"><table><thead><tr><th scope="col">
|
|
|
|
| 421 |
'<th scope="col">Workspace</th><th scope="col">Question</th>'
|
| 422 |
'<th scope="col">Code</th><th scope="col" class="number">Count</th></tr></thead>'
|
| 423 |
'<tbody>' + "".join(rows) + '</tbody></table></div>')
|
|
|
|
| 375 |
"""
|
| 376 |
|
| 377 |
|
| 378 |
+
def _feedback_date(stamp):
|
| 379 |
+
"""Format an operator date without a leading zero on the day."""
|
| 380 |
+
return f"{stamp:%b} {stamp.day}, {stamp.year}"
|
| 381 |
+
|
| 382 |
+
|
| 383 |
@router.get("/feedback/log")
|
| 384 |
def feedback_log(tenant: str = "", session: Session = Depends(require_session)):
|
| 385 |
"""Render the existing operator log with one row per distinct AI limitation."""
|
|
|
|
| 391 |
rows = []
|
| 392 |
for row in data["rows"]:
|
| 393 |
stamp = datetime.fromtimestamp(float(row["ts"]), timezone.utc)
|
| 394 |
+
date = _feedback_date(stamp)
|
| 395 |
limitation = row.get("category") == "limitation"
|
| 396 |
question = row.get("question") if limitation else row.get("text")
|
| 397 |
detail = (f"{row.get('tool', '')}.{row.get('action', '')}" if limitation
|
|
|
|
| 409 |
filer_details.append(
|
| 410 |
f'<span class="feedback-detail">{escape(str(user))}: {count:,} {label}, last '
|
| 411 |
f'<time datetime="{escape(last.isoformat(), quote=True)}">'
|
| 412 |
+
f'{_feedback_date(last)}</time> '
|
| 413 |
+
f'<span class="feedback-time">{last:%H:%M:%S} UTC</span></span>')
|
| 414 |
rows.append(
|
| 415 |
f'<tr data-limitation="{escape(str(row["id"]), quote=True)}">'
|
| 416 |
+
f'<td><time datetime="{stamp.isoformat()}">{date}</time></td>'
|
| 417 |
+
f'<td>{stamp:%H:%M:%S} UTC</td>'
|
| 418 |
f'<td>{escape(str(row["tenant"]))}</td>'
|
| 419 |
f'<td class="feedback-question">{escape(str(question or ""))}'
|
| 420 |
f'<span class="feedback-detail">{escape(detail)}</span>{"".join(filer_details)}</td>'
|
|
|
|
| 424 |
'<h1>Team feedback</h1>'
|
| 425 |
f'<p class="feedback-summary">{data["total"]:,} {item_label}</p>')
|
| 426 |
if rows:
|
| 427 |
+
body += ('<div class="table-scroll"><table><thead><tr><th scope="col">Date</th>'
|
| 428 |
+
'<th scope="col">Time</th>'
|
| 429 |
'<th scope="col">Workspace</th><th scope="col">Question</th>'
|
| 430 |
'<th scope="col">Code</th><th scope="col" class="number">Count</th></tr></thead>'
|
| 431 |
'<tbody>' + "".join(rows) + '</tbody></table></div>')
|
api/routes_grid.py
CHANGED
|
@@ -28,6 +28,7 @@ MODULE = "customer_data"
|
|
| 28 |
#: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a
|
| 29 |
#: legitimate client — refuse it rather than doing 500 store writes on one request.
|
| 30 |
_MAX_EVENTS = 24
|
|
|
|
| 31 |
|
| 32 |
|
| 33 |
def row_patch_receipt(scope):
|
|
@@ -105,16 +106,17 @@ def _ctx(session: Session, fields, pids, *, permission_fields=None, **kw):
|
|
| 105 |
# Getting any one of them from the other topic is the "validated against the wrong field
|
| 106 |
# contract" near-miss the wave-15 routes_products header warned about.
|
| 107 |
scope_key = str(kw.get("scope_key") or "")
|
|
|
|
| 108 |
if scope_key == "product":
|
| 109 |
import modules.product_data as pd
|
| 110 |
from routes_products import MODULE as _PMOD, pd_fields
|
| 111 |
from registry_source import MetadataStore
|
| 112 |
|
| 113 |
-
module, canonical = _PMOD, pd_fields(consolidated=True, st=MetadataStore(
|
| 114 |
kw.setdefault("table", pd.table_ops(session.runtime))
|
| 115 |
hidden = perm_scope.hidden_keys(session.user, module,
|
| 116 |
canonical if permission_fields is None else permission_fields,
|
| 117 |
-
st=
|
| 118 |
elif scope_key.startswith("ut_"):
|
| 119 |
# Wave 18 C3-UT: a user table has no module in the permission wall (its wall is
|
| 120 |
# `user_tables.may_open`, already applied by the assembly this route ran first), so
|
|
@@ -129,7 +131,7 @@ def _ctx(session: Session, fields, pids, *, permission_fields=None, **kw):
|
|
| 129 |
import core.table_store as table_store
|
| 130 |
kw.setdefault("table", table_store.make(
|
| 131 |
"customer_table_workspace", st=session.runtime))
|
| 132 |
-
hidden = perm_scope.hidden_keys(session.user, module, canonical, st=
|
| 133 |
return grid_events.EventCtx(
|
| 134 |
uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin,
|
| 135 |
# C-PERM: the write wall's field half, on the EVENTS transport too. This route takes the
|
|
@@ -406,44 +408,20 @@ def _workspace_values(values, session, module, fields):
|
|
| 406 |
|
| 407 |
|
| 408 |
def _published_rev(ws):
|
| 409 |
-
"""
|
| 410 |
rev = ws.get("rev") if isinstance(ws, dict) else None
|
| 411 |
return rev if isinstance(rev, int) and not isinstance(rev, bool) else None
|
| 412 |
|
| 413 |
|
| 414 |
-
def _settle_refused_view_write(ctx, event):
|
| 415 |
-
"""Store a no-op receipt for a refused stale view write, so its resend re-reads and stops.
|
| 416 |
-
|
| 417 |
-
The refusal rolls back the event's own receipt, and the outbox resends the same id. Without
|
| 418 |
-
this receipt every resend is refused again and the browser's save queue never moves (D-634).
|
| 419 |
-
"""
|
| 420 |
-
from core import edit_receipts, grid_events
|
| 421 |
-
|
| 422 |
-
if not getattr(ctx, "durable_receipts", False) or not isinstance(event, dict):
|
| 423 |
-
return
|
| 424 |
-
identity = event.get("opId", event.get("id"))
|
| 425 |
-
if not identity:
|
| 426 |
-
return
|
| 427 |
-
try:
|
| 428 |
-
identity = edit_receipts.op_id(identity)
|
| 429 |
-
dropped = {"rerender": True, "write": {"opId": identity}, "refusals": [],
|
| 430 |
-
"doc": None, "toast": None}
|
| 431 |
-
edit_receipts.run_once(
|
| 432 |
-
grid_events._store_of(ctx), ctx.uname,
|
| 433 |
-
edit_receipts.operation(ctx.scope_key, event.get("type"), event.get("pid")),
|
| 434 |
-
identity, lambda: dropped)
|
| 435 |
-
except Exception as exc: # noqa: BLE001
|
| 436 |
-
print(f"[grid/events] refused view write {identity!r}: resend receipt not saved "
|
| 437 |
-
f"({type(exc).__name__})")
|
| 438 |
-
|
| 439 |
-
|
| 440 |
def _workspace_filter_removals(workspace, fields, measures, hidden=()):
|
| 441 |
-
"""Clean
|
| 442 |
from copy import deepcopy
|
|
|
|
| 443 |
import aios_grid
|
| 444 |
|
| 445 |
workspace = dict(workspace)
|
| 446 |
-
|
|
|
|
| 447 |
cohort_ids = {str(entry['id']) for entry in workspace.get('lists') or () if entry.get('id')}
|
| 448 |
view_ids = {str(view['id']) for view in workspace.get('views') or () if view.get('id')}
|
| 449 |
hidden = set(hidden or ())
|
|
@@ -457,6 +435,7 @@ def _workspace_filter_removals(workspace, fields, measures, hidden=()):
|
|
| 457 |
cfg['filters'] = aios_grid.clean_filter_tree(
|
| 458 |
cfg.get('filters'), valid_keys, cohort_ids=cohort_ids,
|
| 459 |
visible_view_ids=view_ids, report=reports)
|
|
|
|
| 460 |
removals = []
|
| 461 |
for report in [*(view.get('filterRemovals') or ()), *reports]:
|
| 462 |
if (isinstance(report, dict) and report.get('colId') not in hidden
|
|
@@ -501,6 +480,8 @@ def workspace(scope: str = "customer",
|
|
| 501 |
# session (its wall is the table's own, inside the assembly).
|
| 502 |
if not (scope == "product" or scope.startswith("ut_")):
|
| 503 |
session.require(MODULE)
|
|
|
|
|
|
|
| 504 |
# ── Wave 18 C3-UT: a USER TABLE'S workspace — the third topic through the one wire. The
|
| 505 |
# storage key has no (bu, agent) because a user table has no Odoo scope; per-user by the
|
| 506 |
# table's own wall (creator or admin, `user_tables.may_open`).
|
|
@@ -599,14 +580,22 @@ def workspace(scope: str = "customer",
|
|
| 599 |
# R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own
|
| 600 |
# first-cohort contract change. Same reason as the customer branch below.
|
| 601 |
workspace["fields"] = g["fields"]
|
| 602 |
-
#
|
| 603 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 604 |
workspace["measures"] = g["measures"]
|
| 605 |
workspace["measureSets"] = g["measure_sets"]
|
| 606 |
-
#
|
| 607 |
-
# reason: `product_assembly` builds this topic's cohort cells and this route discarded
|
| 608 |
-
# them. The measure half stays empty on this topic by descope, which is a different
|
| 609 |
-
# statement and one the offer already makes.
|
| 610 |
workspace["derived"] = _workspace_values(g["derived"], session, PRODUCT_MODULE, g["fields"])
|
| 611 |
workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
|
| 612 |
try:
|
|
@@ -2114,7 +2103,7 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2114 |
Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the
|
| 2115 |
same sessions the read door (`/workspace`) admits, or a tenant without the customer module
|
| 2116 |
can SEE its own user tables and not write to them."""
|
| 2117 |
-
from core import grid_events, shared_overlay
|
| 2118 |
from routes_customers import grid_assembly
|
| 2119 |
from routes_tables import _app_write_via, _managed_or_refuse
|
| 2120 |
|
|
@@ -2124,7 +2113,11 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2124 |
events = [body] # a single event object, the legacy shape
|
| 2125 |
if not isinstance(events, list):
|
| 2126 |
raise err(400, "bad_events", "expected {events: [...]}")
|
| 2127 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2128 |
raise err(400, "too_many_events",
|
| 2129 |
f"at most {_MAX_EVENTS} events per request (the client's resend window)")
|
| 2130 |
|
|
@@ -2147,11 +2140,20 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2147 |
# and lands in the product bucket. The measure/cohort context is honestly EMPTY there:
|
| 2148 |
# `clean_measure_field` refuses measure creates on this surface by construction (the
|
| 2149 |
# customer-grain descope), which is the fail-closed shape, not an accident.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2150 |
if scope == "product":
|
| 2151 |
from routes_products import MODULE as PRODUCT_MODULE, product_assembly
|
| 2152 |
|
| 2153 |
session.require(PRODUCT_MODULE) # the product wall (dependency is session-only, C4)
|
| 2154 |
-
g = product_assembly(
|
| 2155 |
elif scope.startswith("ut_"):
|
| 2156 |
# Wave 18 C3-UT — the user-table wall (creator/admin) is inside the assembly; the
|
| 2157 |
# measure/cohort context is honestly EMPTY (customer-grain machinery, no meaning here).
|
|
@@ -2163,9 +2165,10 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2163 |
# permission wall and the admission context — and `ut_write_ctx` returns the SAME six keys
|
| 2164 |
# this route reads, with a pid set derived from exactly the row ids `scoped_pool` would
|
| 2165 |
# have kept. What it does not do is materialise the rows nobody here looks at.
|
| 2166 |
-
from routes_tables import ut_write_ctx
|
| 2167 |
|
| 2168 |
-
g =
|
|
|
|
| 2169 |
# ⭐⭐ W31-T20 / D-174 — A WRITE THAT NEEDS THE ROW SET REFUSES OUT LOUD, and this is the
|
| 2170 |
# half that is easy to skip because the code already "fails closed" without it. When a
|
| 2171 |
# read-through grid's population exceeds one window the pid set is EMPTY, and every
|
|
@@ -2188,7 +2191,18 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2188 |
f"{lim.get('cause') or 'the row set is unresolved'}. "
|
| 2189 |
f"{lim.get('recommendation') or ''}".strip())
|
| 2190 |
else:
|
| 2191 |
-
g = grid_assembly(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2192 |
# ⚠ THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`,
|
| 2193 |
# `clean_measure_field` had an empty admission list and every measure-column create over
|
| 2194 |
# HTTP was silently refused; without `measure_keys`, `clean_filter_tree` stripped every
|
|
@@ -2198,6 +2212,8 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2198 |
# The product and customer assemblies read `ws` with these pids and no consume: lend it once.
|
| 2199 |
lend = None if scope.startswith("ut_") else g.get("ws")
|
| 2200 |
ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, via=via,
|
|
|
|
|
|
|
| 2201 |
durable_receipts=True,
|
| 2202 |
workspace_lend=lend if isinstance(lend, dict) else None,
|
| 2203 |
permission_fields=(g.get('_permission_scope') or {}).get('fields'),
|
|
@@ -2211,9 +2227,25 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2211 |
|
| 2212 |
# Per-event results so the client can tell which of a batch landed — the component's own
|
| 2213 |
# bridge has no response channel at all, so this is strictly more than the embed gets.
|
| 2214 |
-
results = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2215 |
try:
|
| 2216 |
-
for one in events:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2217 |
eid = str(one.get("opId", one.get("id")) or "") if isinstance(one, dict) else ""
|
| 2218 |
# ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler
|
| 2219 |
# appends to a shared list, so the refusals belonging to THIS event are exactly the
|
|
@@ -2221,11 +2253,7 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2221 |
# answer "the batch was refused" and leave the caller to guess which member, which is
|
| 2222 |
# the same class of unfalsifiable answer the channel exists to end.
|
| 2223 |
_before = len(ctx.out.refusals)
|
| 2224 |
-
|
| 2225 |
-
rerender = grid_events.handle_one(one, ctx)
|
| 2226 |
-
except view_audit.ViewRevisionStale:
|
| 2227 |
-
_settle_refused_view_write(ctx, one)
|
| 2228 |
-
raise
|
| 2229 |
row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid,
|
| 2230 |
"rerender": bool(rerender)}
|
| 2231 |
mine = ctx.out.refusals[_before:]
|
|
@@ -2233,7 +2261,7 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2233 |
# `refused` is the SHAPE a caller branches on; the first reason is the one that
|
| 2234 |
# stopped this write (a handler returns at its first refusal).
|
| 2235 |
row["refused"] = mine[0]
|
| 2236 |
-
results
|
| 2237 |
except (shared_overlay.WriteUnavailable, grid_events.StoreUnavailable) as exc:
|
| 2238 |
refusal = err(503, "store_unavailable",
|
| 2239 |
"This edit could not be confirmed as saved. Retry with the same op id. "
|
|
|
|
| 28 |
#: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a
|
| 29 |
#: legitimate client — refuse it rather than doing 500 store writes on one request.
|
| 30 |
_MAX_EVENTS = 24
|
| 31 |
+
_MAX_VIEW_OPS = 200
|
| 32 |
|
| 33 |
|
| 34 |
def row_patch_receipt(scope):
|
|
|
|
| 106 |
# Getting any one of them from the other topic is the "validated against the wrong field
|
| 107 |
# contract" near-miss the wave-15 routes_products header warned about.
|
| 108 |
scope_key = str(kw.get("scope_key") or "")
|
| 109 |
+
admission = kw.get('admission') or session.runtime
|
| 110 |
if scope_key == "product":
|
| 111 |
import modules.product_data as pd
|
| 112 |
from routes_products import MODULE as _PMOD, pd_fields
|
| 113 |
from registry_source import MetadataStore
|
| 114 |
|
| 115 |
+
module, canonical = _PMOD, pd_fields(consolidated=True, st=MetadataStore(admission))
|
| 116 |
kw.setdefault("table", pd.table_ops(session.runtime))
|
| 117 |
hidden = perm_scope.hidden_keys(session.user, module,
|
| 118 |
canonical if permission_fields is None else permission_fields,
|
| 119 |
+
st=admission)
|
| 120 |
elif scope_key.startswith("ut_"):
|
| 121 |
# Wave 18 C3-UT: a user table has no module in the permission wall (its wall is
|
| 122 |
# `user_tables.may_open`, already applied by the assembly this route ran first), so
|
|
|
|
| 131 |
import core.table_store as table_store
|
| 132 |
kw.setdefault("table", table_store.make(
|
| 133 |
"customer_table_workspace", st=session.runtime))
|
| 134 |
+
hidden = perm_scope.hidden_keys(session.user, module, canonical, st=admission)
|
| 135 |
return grid_events.EventCtx(
|
| 136 |
uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin,
|
| 137 |
# C-PERM: the write wall's field half, on the EVENTS transport too. This route takes the
|
|
|
|
| 408 |
|
| 409 |
|
| 410 |
def _published_rev(ws):
|
| 411 |
+
"""The store change revision published with this workspace snapshot."""
|
| 412 |
rev = ws.get("rev") if isinstance(ws, dict) else None
|
| 413 |
return rev if isinstance(rev, int) and not isinstance(rev, bool) else None
|
| 414 |
|
| 415 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 416 |
def _workspace_filter_removals(workspace, fields, measures, hidden=()):
|
| 417 |
+
"""Clean dead View members on a detached wire copy, retaining filter removal reasons."""
|
| 418 |
from copy import deepcopy
|
| 419 |
+
from core.table_store import prune_view_field_members
|
| 420 |
import aios_grid
|
| 421 |
|
| 422 |
workspace = dict(workspace)
|
| 423 |
+
field_keys = {field['key'] for field in fields}
|
| 424 |
+
valid_keys = field_keys | {measure['key'] for measure in measures}
|
| 425 |
cohort_ids = {str(entry['id']) for entry in workspace.get('lists') or () if entry.get('id')}
|
| 426 |
view_ids = {str(view['id']) for view in workspace.get('views') or () if view.get('id')}
|
| 427 |
hidden = set(hidden or ())
|
|
|
|
| 435 |
cfg['filters'] = aios_grid.clean_filter_tree(
|
| 436 |
cfg.get('filters'), valid_keys, cohort_ids=cohort_ids,
|
| 437 |
visible_view_ids=view_ids, report=reports)
|
| 438 |
+
prune_view_field_members(view, live_keys=field_keys, prune_filters=False)
|
| 439 |
removals = []
|
| 440 |
for report in [*(view.get('filterRemovals') or ()), *reports]:
|
| 441 |
if (isinstance(report, dict) and report.get('colId') not in hidden
|
|
|
|
| 480 |
# session (its wall is the table's own, inside the assembly).
|
| 481 |
if not (scope == "product" or scope.startswith("ut_")):
|
| 482 |
session.require(MODULE)
|
| 483 |
+
if not session.runtime.available():
|
| 484 |
+
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 485 |
# ── Wave 18 C3-UT: a USER TABLE'S workspace — the third topic through the one wire. The
|
| 486 |
# storage key has no (bu, agent) because a user table has no Odoo scope; per-user by the
|
| 487 |
# table's own wall (creator or admin, `user_tables.may_open`).
|
|
|
|
| 580 |
# R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own
|
| 581 |
# first-cohort contract change. Same reason as the customer branch below.
|
| 582 |
workspace["fields"] = g["fields"]
|
| 583 |
+
# Count workspace-wide references using the same producer as Customers and user tables.
|
| 584 |
+
try:
|
| 585 |
+
from routes_tables import _field_usage
|
| 586 |
+
admission = g.get('_admission', session.runtime)
|
| 587 |
+
usage = _field_usage(session, PRODUCT_MODULE, g['fields'], None,
|
| 588 |
+
admission, 'product_table_workspace', st=admission)
|
| 589 |
+
except Exception:
|
| 590 |
+
usage = {}
|
| 591 |
+
if usage:
|
| 592 |
+
workspace['fields'] = [dict(field, usage=dict(usage[field['key']]))
|
| 593 |
+
if usage.get(field.get('key')) else field
|
| 594 |
+
for field in g['fields']]
|
| 595 |
+
# The model owns this session's permitted Product catalogue.
|
| 596 |
workspace["measures"] = g["measures"]
|
| 597 |
workspace["measureSets"] = g["measure_sets"]
|
| 598 |
+
# Product cohort and measure cells share the existing read-only derived channel.
|
|
|
|
|
|
|
|
|
|
| 599 |
workspace["derived"] = _workspace_values(g["derived"], session, PRODUCT_MODULE, g["fields"])
|
| 600 |
workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
|
| 601 |
try:
|
|
|
|
| 2103 |
Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the
|
| 2104 |
same sessions the read door (`/workspace`) admits, or a tenant without the customer module
|
| 2105 |
can SEE its own user tables and not write to them."""
|
| 2106 |
+
from core import grid_events, shared_overlay
|
| 2107 |
from routes_customers import grid_assembly
|
| 2108 |
from routes_tables import _app_write_via, _managed_or_refuse
|
| 2109 |
|
|
|
|
| 2113 |
events = [body] # a single event object, the legacy shape
|
| 2114 |
if not isinstance(events, list):
|
| 2115 |
raise err(400, "bad_events", "expected {events: [...]}")
|
| 2116 |
+
view_ops = [event for event in events
|
| 2117 |
+
if isinstance(event, dict) and event.get("type") == "view_op"]
|
| 2118 |
+
if len(view_ops) > _MAX_VIEW_OPS:
|
| 2119 |
+
raise err(400, "too_many_view_ops", f"at most {_MAX_VIEW_OPS} View operations per request")
|
| 2120 |
+
if len(events) - len(view_ops) > _MAX_EVENTS:
|
| 2121 |
raise err(400, "too_many_events",
|
| 2122 |
f"at most {_MAX_EVENTS} events per request (the client's resend window)")
|
| 2123 |
|
|
|
|
| 2140 |
# and lands in the product bucket. The measure/cohort context is honestly EMPTY there:
|
| 2141 |
# `clean_measure_field` refuses measure creates on this surface by construction (the
|
| 2142 |
# customer-grain descope), which is the fail-closed shape, not an accident.
|
| 2143 |
+
view_only = bool(events) and all(
|
| 2144 |
+
isinstance(event, dict) and event.get('type') in (
|
| 2145 |
+
'view_op', 'view_select', 'folder_create', 'folder_rename', 'folder_delete',
|
| 2146 |
+
'folder_duplicate', 'item_move', 'folder_reorder')
|
| 2147 |
+
and event.get('pid') is None and not event.get('pids') for event in events)
|
| 2148 |
+
table_key = (scope + '_table_workspace' if scope.startswith('ut_') else
|
| 2149 |
+
'product_table_workspace' if scope == 'product' else 'customer_table_workspace')
|
| 2150 |
+
actor = grid_events.request_admission(session, table_key,
|
| 2151 |
+
schema_only=view_only and not scope.startswith('ut_'))
|
| 2152 |
if scope == "product":
|
| 2153 |
from routes_products import MODULE as PRODUCT_MODULE, product_assembly
|
| 2154 |
|
| 2155 |
session.require(PRODUCT_MODULE) # the product wall (dependency is session-only, C4)
|
| 2156 |
+
g = product_assembly(actor, consume_corrections=False, with_rows=not view_only)
|
| 2157 |
elif scope.startswith("ut_"):
|
| 2158 |
# Wave 18 C3-UT — the user-table wall (creator/admin) is inside the assembly; the
|
| 2159 |
# measure/cohort context is honestly EMPTY (customer-grain machinery, no meaning here).
|
|
|
|
| 2165 |
# permission wall and the admission context — and `ut_write_ctx` returns the SAME six keys
|
| 2166 |
# this route reads, with a pid set derived from exactly the row ids `scoped_pool` would
|
| 2167 |
# have kept. What it does not do is materialise the rows nobody here looks at.
|
| 2168 |
+
from routes_tables import ut_assembly, ut_write_ctx
|
| 2169 |
|
| 2170 |
+
g = (ut_assembly(actor, scope, with_rows=False, fields_only=True) if view_only else
|
| 2171 |
+
ut_write_ctx(actor, scope))
|
| 2172 |
# ⭐⭐ W31-T20 / D-174 — A WRITE THAT NEEDS THE ROW SET REFUSES OUT LOUD, and this is the
|
| 2173 |
# half that is easy to skip because the code already "fails closed" without it. When a
|
| 2174 |
# read-through grid's population exceeds one window the pid set is EMPTY, and every
|
|
|
|
| 2191 |
f"{lim.get('cause') or 'the row set is unresolved'}. "
|
| 2192 |
f"{lim.get('recommendation') or ''}".strip())
|
| 2193 |
else:
|
| 2194 |
+
g = grid_assembly(actor, scope=scope, consume_corrections=False, with_rows=not view_only)
|
| 2195 |
+
record_admitter = None
|
| 2196 |
+
if view_only:
|
| 2197 |
+
from routes_tables import _builtin_note_context, ut_write_ctx
|
| 2198 |
+
checked_pids = {}
|
| 2199 |
+
def record_admitter(pid):
|
| 2200 |
+
if pid not in checked_pids:
|
| 2201 |
+
permitted = (ut_write_ctx(actor, scope, pid=pid) if scope.startswith('ut_') else
|
| 2202 |
+
_builtin_note_context(
|
| 2203 |
+
actor, 'product_data' if scope == 'product' else MODULE, pid))
|
| 2204 |
+
checked_pids[pid] = pid in permitted['pids']
|
| 2205 |
+
return checked_pids[pid]
|
| 2206 |
# ⚠ THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`,
|
| 2207 |
# `clean_measure_field` had an empty admission list and every measure-column create over
|
| 2208 |
# HTTP was silently refused; without `measure_keys`, `clean_filter_tree` stripped every
|
|
|
|
| 2212 |
# The product and customer assemblies read `ws` with these pids and no consume: lend it once.
|
| 2213 |
lend = None if scope.startswith("ut_") else g.get("ws")
|
| 2214 |
ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, via=via,
|
| 2215 |
+
admission=g.get('_admission', actor.runtime),
|
| 2216 |
+
record_admitter=record_admitter,
|
| 2217 |
durable_receipts=True,
|
| 2218 |
workspace_lend=lend if isinstance(lend, dict) else None,
|
| 2219 |
permission_fields=(g.get('_permission_scope') or {}).get('fields'),
|
|
|
|
| 2227 |
|
| 2228 |
# Per-event results so the client can tell which of a batch landed — the component's own
|
| 2229 |
# bridge has no response channel at all, so this is strictly more than the embed gets.
|
| 2230 |
+
results = [None] * len(events)
|
| 2231 |
+
grouped = {}
|
| 2232 |
+
for index, one in enumerate(events):
|
| 2233 |
+
if isinstance(one, dict) and one.get("type") == "view_op":
|
| 2234 |
+
grouped.setdefault(str(one.get("view_id") or "").strip(), []).append((index, one))
|
| 2235 |
+
dispatched = set()
|
| 2236 |
try:
|
| 2237 |
+
for index, one in enumerate(events):
|
| 2238 |
+
if isinstance(one, dict) and one.get("type") == "view_op":
|
| 2239 |
+
view_id = str(one.get("view_id") or "").strip()
|
| 2240 |
+
if view_id in dispatched:
|
| 2241 |
+
continue
|
| 2242 |
+
dispatched.add(view_id)
|
| 2243 |
+
batch = grouped[view_id]
|
| 2244 |
+
eid = one.get("client_op_id", one.get("opId", one.get("id")))
|
| 2245 |
+
rows = grid_events.handle_view_ops([event for _, event in batch], ctx)
|
| 2246 |
+
for (slot, _), row in zip(batch, rows):
|
| 2247 |
+
results[slot] = row
|
| 2248 |
+
continue
|
| 2249 |
eid = str(one.get("opId", one.get("id")) or "") if isinstance(one, dict) else ""
|
| 2250 |
# ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler
|
| 2251 |
# appends to a shared list, so the refusals belonging to THIS event are exactly the
|
|
|
|
| 2253 |
# answer "the batch was refused" and leave the caller to guess which member, which is
|
| 2254 |
# the same class of unfalsifiable answer the channel exists to end.
|
| 2255 |
_before = len(ctx.out.refusals)
|
| 2256 |
+
rerender = grid_events.handle_one(one, ctx)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2257 |
row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid,
|
| 2258 |
"rerender": bool(rerender)}
|
| 2259 |
mine = ctx.out.refusals[_before:]
|
|
|
|
| 2261 |
# `refused` is the SHAPE a caller branches on; the first reason is the one that
|
| 2262 |
# stopped this write (a handler returns at its first refusal).
|
| 2263 |
row["refused"] = mine[0]
|
| 2264 |
+
results[index] = row
|
| 2265 |
except (shared_overlay.WriteUnavailable, grid_events.StoreUnavailable) as exc:
|
| 2266 |
refusal = err(503, "store_unavailable",
|
| 2267 |
"This edit could not be confirmed as saved. Retry with the same op id. "
|
api/routes_keychain.py
CHANGED
|
@@ -2224,5 +2224,7 @@ def live_limits_put(body: dict = Body(default=None),
|
|
| 2224 |
|
| 2225 |
|
| 2226 |
from routes_amazon_deletion import router as amazon_deletion_router # noqa: E402
|
|
|
|
| 2227 |
|
| 2228 |
router.include_router(amazon_deletion_router)
|
|
|
|
|
|
| 2224 |
|
| 2225 |
|
| 2226 |
from routes_amazon_deletion import router as amazon_deletion_router # noqa: E402
|
| 2227 |
+
from routes_amazon_events import router as amazon_events_router # noqa: E402
|
| 2228 |
|
| 2229 |
router.include_router(amazon_deletion_router)
|
| 2230 |
+
router.include_router(amazon_events_router)
|
api/routes_odoo_tables.py
CHANGED
|
@@ -1633,10 +1633,11 @@ def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_looku
|
|
| 1633 |
spec = amazon.grid_source(cur, key, columns=columns, allow_lookups=allow_lookups)
|
| 1634 |
else:
|
| 1635 |
spec = amazon.page_source(cur, key, definition, allow_lookups=allow_lookups)
|
| 1636 |
-
if key == amazon.PREFIX + "inventory" and spec is not None
|
|
|
|
| 1637 |
status = amazon.amazon_store.inventory_table_status(cur)
|
| 1638 |
spec = {**spec, 'sourceStatus': {
|
| 1639 |
-
name: status[name] for name in ('empty', 'code', 'reason') if name in status}}
|
| 1640 |
except Exception as exc: # noqa: BLE001
|
| 1641 |
from core.request_timing import raise_if_resource_error
|
| 1642 |
raise_if_resource_error(exc)
|
|
@@ -2749,11 +2750,11 @@ def _user_tables_token(rt):
|
|
| 2749 |
raise err(503, "schema_unavailable", "table definition revisions are unavailable") from exc
|
| 2750 |
|
| 2751 |
|
| 2752 |
-
def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
|
| 2753 |
"""`read()`, reused while this tenant's `user_tables` revision is unchanged.
|
| 2754 |
|
| 2755 |
-
⛔ `clone`
|
| 2756 |
-
cannot edit the next request's answer. `keep(value)` decides what may be held: an empty
|
| 2757 |
answer never is (it is the shape a failed read takes, and caching it would make one outage
|
| 2758 |
outlive itself), and neither is the whole-document fallback `all_defs` takes on a failure.
|
| 2759 |
`token` is one this REQUEST already read (`_page_defs`), so a page pays one revision round
|
|
@@ -2771,7 +2772,7 @@ def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
|
|
| 2771 |
return clone(hit[1])
|
| 2772 |
value = read()
|
| 2773 |
if token is not None and tenant and keep(value):
|
| 2774 |
-
held = (token, clone(value))
|
| 2775 |
with _REV_MEMO_LOCK:
|
| 2776 |
if slot not in _REV_MEMO and len(_REV_MEMO) >= _REV_MEMO_MAX:
|
| 2777 |
_REV_MEMO.pop(next(iter(_REV_MEMO)), None)
|
|
@@ -2779,11 +2780,111 @@ def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
|
|
| 2779 |
return value
|
| 2780 |
|
| 2781 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2782 |
def _page_defs(session):
|
| 2783 |
"""Return one schema-only `(snapshot, revision)` for a page and its scoped targets.
|
| 2784 |
|
| 2785 |
Shared metadata and source plans belong to the fresh lend, never to the revision memo.
|
| 2786 |
Projection and revision failures refuse instead of falling back to a whole document.
|
|
|
|
| 2787 |
"""
|
| 2788 |
import copy
|
| 2789 |
import core.store as store
|
|
@@ -2799,7 +2900,7 @@ def _page_defs(session):
|
|
| 2799 |
|
| 2800 |
try:
|
| 2801 |
defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt, schema_only=True),
|
| 2802 |
-
keep=keep, clone=copy.deepcopy, token=token)
|
| 2803 |
except Exception as exc: # noqa: BLE001
|
| 2804 |
from core.request_timing import raise_if_resource_error
|
| 2805 |
raise_if_resource_error(exc)
|
|
@@ -4068,6 +4169,24 @@ def resolved_fields(table_key, defn):
|
|
| 4068 |
return list(declared.values())
|
| 4069 |
|
| 4070 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4071 |
def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
|
| 4072 |
search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False,
|
| 4073 |
st=None):
|
|
@@ -4282,7 +4401,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 4282 |
# that number would tell a scoped reader exactly how many rows they are not allowed to see.
|
| 4283 |
base_where = (f'({spec["where"]}) AND {wall_sql}'
|
| 4284 |
if (spec["where"] and wall_sql) else (wall_sql or spec["where"]))
|
| 4285 |
-
limits =
|
| 4286 |
if tree is _TREE_UNSET:
|
| 4287 |
tree = _json_arg(filters, "filters")
|
| 4288 |
sort_spec = _json_arg(sorts, "sorts") or []
|
|
@@ -5189,10 +5308,6 @@ def odoo_table_rows(table_key: str,
|
|
| 5189 |
defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"]
|
| 5190 |
cols, limits, frm = read["cols"], read["limits"], read["frm"]
|
| 5191 |
source_status = spec.get('sourceStatus') or {}
|
| 5192 |
-
if source_status.get('empty') and source_status.get('reason'):
|
| 5193 |
-
limits = [*limits, {'subject': table_key, 'effect': 'source_empty',
|
| 5194 |
-
'code': source_status.get('code'), 'cause': source_status['reason'],
|
| 5195 |
-
'recommendation': ''}]
|
| 5196 |
base_where, wall_params = read["base_where"], read["wall_params"]
|
| 5197 |
where, params, sort_spec = read["where"], read["params"], read["sort_spec"]
|
| 5198 |
_gsql, _gparams = _group_pin(cols, groupBy, groupKey)
|
|
|
|
| 1633 |
spec = amazon.grid_source(cur, key, columns=columns, allow_lookups=allow_lookups)
|
| 1634 |
else:
|
| 1635 |
spec = amazon.page_source(cur, key, definition, allow_lookups=allow_lookups)
|
| 1636 |
+
if (key == amazon.PREFIX + "inventory" and spec is not None
|
| 1637 |
+
and not spec.get("missing") and 'sourceStatus' not in spec):
|
| 1638 |
status = amazon.amazon_store.inventory_table_status(cur)
|
| 1639 |
spec = {**spec, 'sourceStatus': {
|
| 1640 |
+
name: status[name] for name in ('empty', 'code', 'reason', 'recommendation') if name in status}}
|
| 1641 |
except Exception as exc: # noqa: BLE001
|
| 1642 |
from core.request_timing import raise_if_resource_error
|
| 1643 |
raise_if_resource_error(exc)
|
|
|
|
| 2750 |
raise err(503, "schema_unavailable", "table definition revisions are unavailable") from exc
|
| 2751 |
|
| 2752 |
|
| 2753 |
+
def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD, hold=None):
|
| 2754 |
"""`read()`, reused while this tenant's `user_tables` revision is unchanged.
|
| 2755 |
|
| 2756 |
+
⛔ `hold` (default `clone`) copies the value on the way in and `clone` runs on every hit, so a
|
| 2757 |
+
caller that edits what it was handed cannot edit the next request's answer. `keep(value)` decides what may be held: an empty
|
| 2758 |
answer never is (it is the shape a failed read takes, and caching it would make one outage
|
| 2759 |
outlive itself), and neither is the whole-document fallback `all_defs` takes on a failure.
|
| 2760 |
`token` is one this REQUEST already read (`_page_defs`), so a page pays one revision round
|
|
|
|
| 2772 |
return clone(hit[1])
|
| 2773 |
value = read()
|
| 2774 |
if token is not None and tenant and keep(value):
|
| 2775 |
+
held = (token, (hold or clone)(value))
|
| 2776 |
with _REV_MEMO_LOCK:
|
| 2777 |
if slot not in _REV_MEMO and len(_REV_MEMO) >= _REV_MEMO_MAX:
|
| 2778 |
_REV_MEMO.pop(next(iter(_REV_MEMO)), None)
|
|
|
|
| 2780 |
return value
|
| 2781 |
|
| 2782 |
|
| 2783 |
+
class _DefsView(dict):
|
| 2784 |
+
"""One request's view of the held definitions: a table is deep-copied the first time it is read.
|
| 2785 |
+
|
| 2786 |
+
Keys, `in` and `len` read the held mapping directly. Every door that hands out a definition
|
| 2787 |
+
(`[]`, `get`, `values`, `items`, `pop`, `dict(view)`, `copy`, `deepcopy`) hands out this
|
| 2788 |
+
request's own copy, so a caller edit never reaches the memo or another request.
|
| 2789 |
+
"""
|
| 2790 |
+
|
| 2791 |
+
# `user_tables._with_activity` copies a mapping with `dict()` unless it is marked lazy.
|
| 2792 |
+
lazy_tables = True
|
| 2793 |
+
__slots__ = ("_mine",)
|
| 2794 |
+
|
| 2795 |
+
def __init__(self, held):
|
| 2796 |
+
super().__init__(held)
|
| 2797 |
+
self._mine = set()
|
| 2798 |
+
|
| 2799 |
+
def __getitem__(self, key):
|
| 2800 |
+
value = dict.__getitem__(self, key)
|
| 2801 |
+
if key not in self._mine:
|
| 2802 |
+
value = deepcopy(value)
|
| 2803 |
+
dict.__setitem__(self, key, value)
|
| 2804 |
+
self._mine.add(key)
|
| 2805 |
+
return value
|
| 2806 |
+
|
| 2807 |
+
def __setitem__(self, key, value):
|
| 2808 |
+
dict.__setitem__(self, key, value)
|
| 2809 |
+
self._mine.add(key)
|
| 2810 |
+
|
| 2811 |
+
def __delitem__(self, key):
|
| 2812 |
+
dict.__delitem__(self, key)
|
| 2813 |
+
self._mine.discard(key)
|
| 2814 |
+
|
| 2815 |
+
# A Python `__iter__` sends `dict(view)`, `{**view}` and `|` through `keys()` and `__getitem__`.
|
| 2816 |
+
def __iter__(self):
|
| 2817 |
+
return iter(dict.keys(self))
|
| 2818 |
+
|
| 2819 |
+
def get(self, key, default=None):
|
| 2820 |
+
return self[key] if dict.__contains__(self, key) else default
|
| 2821 |
+
|
| 2822 |
+
def setdefault(self, key, default=None):
|
| 2823 |
+
if not dict.__contains__(self, key):
|
| 2824 |
+
self[key] = default
|
| 2825 |
+
return self[key]
|
| 2826 |
+
|
| 2827 |
+
def pop(self, key, *default):
|
| 2828 |
+
if not dict.__contains__(self, key):
|
| 2829 |
+
if default:
|
| 2830 |
+
return default[0]
|
| 2831 |
+
raise KeyError(key)
|
| 2832 |
+
value = self[key]
|
| 2833 |
+
del self[key]
|
| 2834 |
+
return value
|
| 2835 |
+
|
| 2836 |
+
def popitem(self):
|
| 2837 |
+
if not self:
|
| 2838 |
+
raise KeyError("popitem(): dictionary is empty")
|
| 2839 |
+
key = next(reversed(dict.keys(self)))
|
| 2840 |
+
return key, self.pop(key)
|
| 2841 |
+
|
| 2842 |
+
def update(self, *args, **kwargs):
|
| 2843 |
+
for key, value in dict(*args, **kwargs).items():
|
| 2844 |
+
self[key] = value
|
| 2845 |
+
|
| 2846 |
+
def values(self):
|
| 2847 |
+
return [self[key] for key in dict.keys(self)]
|
| 2848 |
+
|
| 2849 |
+
def items(self):
|
| 2850 |
+
return [(key, self[key]) for key in dict.keys(self)]
|
| 2851 |
+
|
| 2852 |
+
def copy(self):
|
| 2853 |
+
return dict(self.items())
|
| 2854 |
+
|
| 2855 |
+
__copy__ = copy
|
| 2856 |
+
|
| 2857 |
+
def __deepcopy__(self, memo):
|
| 2858 |
+
return {key: deepcopy(dict.__getitem__(self, key), memo) for key in dict.keys(self)}
|
| 2859 |
+
|
| 2860 |
+
def __reduce__(self):
|
| 2861 |
+
return dict, (self.copy(),)
|
| 2862 |
+
|
| 2863 |
+
def __or__(self, other):
|
| 2864 |
+
if not isinstance(other, dict):
|
| 2865 |
+
return NotImplemented
|
| 2866 |
+
merged = self.copy()
|
| 2867 |
+
merged.update(other)
|
| 2868 |
+
return merged
|
| 2869 |
+
|
| 2870 |
+
def __ror__(self, other):
|
| 2871 |
+
if not isinstance(other, dict):
|
| 2872 |
+
return NotImplemented
|
| 2873 |
+
merged = dict(other)
|
| 2874 |
+
merged.update(self.items())
|
| 2875 |
+
return merged
|
| 2876 |
+
|
| 2877 |
+
def __ior__(self, other):
|
| 2878 |
+
self.update(other)
|
| 2879 |
+
return self
|
| 2880 |
+
|
| 2881 |
+
|
| 2882 |
def _page_defs(session):
|
| 2883 |
"""Return one schema-only `(snapshot, revision)` for a page and its scoped targets.
|
| 2884 |
|
| 2885 |
Shared metadata and source plans belong to the fresh lend, never to the revision memo.
|
| 2886 |
Projection and revision failures refuse instead of falling back to a whole document.
|
| 2887 |
+
A memo hit lends a `_DefsView`, so a page copies only the definitions it reads.
|
| 2888 |
"""
|
| 2889 |
import copy
|
| 2890 |
import core.store as store
|
|
|
|
| 2900 |
|
| 2901 |
try:
|
| 2902 |
defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt, schema_only=True),
|
| 2903 |
+
keep=keep, clone=_DefsView, hold=copy.deepcopy, token=token)
|
| 2904 |
except Exception as exc: # noqa: BLE001
|
| 2905 |
from core.request_timing import raise_if_resource_error
|
| 2906 |
raise_if_resource_error(exc)
|
|
|
|
| 4169 |
return list(declared.values())
|
| 4170 |
|
| 4171 |
|
| 4172 |
+
def _source_status_limits(table_key, spec):
|
| 4173 |
+
"""Carry a declared source cause through the existing row and aggregate metadata.
|
| 4174 |
+
|
| 4175 |
+
An empty source names why it is empty. A source that keeps rows beside a refusal (`empty`
|
| 4176 |
+
false, with `code` and `reason`) names the refusal, so the count hover is not silent about it.
|
| 4177 |
+
"""
|
| 4178 |
+
status = spec.get('sourceStatus') or {}
|
| 4179 |
+
if not status.get('reason'):
|
| 4180 |
+
return []
|
| 4181 |
+
if status.get('empty'):
|
| 4182 |
+
return [{'subject': table_key, 'effect': 'source_empty', 'code': status.get('code'),
|
| 4183 |
+
'cause': status['reason'], 'recommendation': status.get('recommendation') or ''}]
|
| 4184 |
+
if not status.get('code'):
|
| 4185 |
+
return []
|
| 4186 |
+
return [{'subject': table_key, 'effect': 'source_refused', 'code': status['code'],
|
| 4187 |
+
'cause': status['reason'], 'recommendation': status.get('recommendation') or ''}]
|
| 4188 |
+
|
| 4189 |
+
|
| 4190 |
def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
|
| 4191 |
search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False,
|
| 4192 |
st=None):
|
|
|
|
| 4401 |
# that number would tell a scoped reader exactly how many rows they are not allowed to see.
|
| 4402 |
base_where = (f'({spec["where"]}) AND {wall_sql}'
|
| 4403 |
if (spec["where"] and wall_sql) else (wall_sql or spec["where"]))
|
| 4404 |
+
limits = _source_status_limits(table_key, spec)
|
| 4405 |
if tree is _TREE_UNSET:
|
| 4406 |
tree = _json_arg(filters, "filters")
|
| 4407 |
sort_spec = _json_arg(sorts, "sorts") or []
|
|
|
|
| 5308 |
defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"]
|
| 5309 |
cols, limits, frm = read["cols"], read["limits"], read["frm"]
|
| 5310 |
source_status = spec.get('sourceStatus') or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5311 |
base_where, wall_params = read["base_where"], read["wall_params"]
|
| 5312 |
where, params, sort_spec = read["where"], read["params"], read["sort_spec"]
|
| 5313 |
_gsql, _gparams = _group_pin(cols, groupBy, groupKey)
|
api/routes_products.py
CHANGED
|
@@ -433,6 +433,11 @@ def _project_enabled_raw_rows(rows, enabled, *, drop_source=False):
|
|
| 433 |
return out
|
| 434 |
|
| 435 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
def _duplicate_code_limits(rows_src, fields):
|
| 437 |
"""Report collapsed products only through permitted rows and the visible code field."""
|
| 438 |
import modules.product_data as pd
|
|
@@ -446,19 +451,21 @@ def _duplicate_code_limits(rows_src, fields):
|
|
| 446 |
codes = ', '.join(report['codes'])
|
| 447 |
return [{'subject': 'code', 'effect': 'duplicate_codes',
|
| 448 |
'cause': f"Shared SKU codes {codes}: {report['gap']} fewer rows than active Odoo products for these codes.",
|
| 449 |
-
'recommendation':
|
| 450 |
|
| 451 |
|
| 452 |
def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
| 453 |
consume_corrections: bool = False, *, with_rows: bool = True, needed=None):
|
| 454 |
from routes_customers import grid_assembly
|
| 455 |
from core import perm_scope
|
|
|
|
| 456 |
team_id, _agent = perm_scope.derive_pool_scope(session.user, MODULE)
|
| 457 |
-
|
| 458 |
build=lambda borrowed: _compute_product_assembly(
|
| 459 |
borrowed, scope, storage_key, False, with_rows=with_rows, needed=needed),
|
| 460 |
pool_stamp=lambda: _measure_stamp(session.runtime, team_id) if with_rows else 0,
|
| 461 |
with_rows=with_rows, needed=needed)
|
|
|
|
| 462 |
|
| 463 |
|
| 464 |
def _compute_product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
|
@@ -772,12 +779,15 @@ def patch_product(pid: int, body: dict = Body(default=None),
|
|
| 772 |
raise err(403, "out_of_scope", "that product is not in your catalogue")
|
| 773 |
import core.perm_scope as perm_scope
|
| 774 |
|
|
|
|
|
|
|
|
|
|
| 775 |
ctx = grid_events.EventCtx(
|
| 776 |
uname=session.uname, allowed_pids=g["pids"], fields=g["fields"],
|
| 777 |
admin=session.admin, fallback_ws=None, seen_ids={},
|
| 778 |
hidden_keys=perm_scope.hidden_keys(
|
| 779 |
-
session.user, MODULE, g["fields"], st=session.runtime),
|
| 780 |
-
scope_key="product", table=
|
| 781 |
if notes is not None:
|
| 782 |
return _patch_cell_notes(MODULE, pid, notes, ctx,
|
| 783 |
source_row_exists=lambda key, rid, admitted: key == MODULE and int(rid) in g['pids'])
|
|
@@ -797,8 +807,8 @@ def patch_product(pid: int, body: dict = Body(default=None),
|
|
| 797 |
# only the per-user stratum would find nothing and report every accepted shared write as
|
| 798 |
# `refused` — a correct write reported as a failure, which is the read-path-cannot-witness-
|
| 799 |
# the-write-path shape ([[read-path-cannot-witness-write-path]]).
|
| 800 |
-
stored = (
|
| 801 |
-
|
| 802 |
stored = {**stored, **((pd.shared_cells([pid], st=session.runtime).get(str(pid))) or {})}
|
| 803 |
accepted = {k: stored.get(k) for k in updates if k in stored}
|
| 804 |
refused = sorted(k for k in updates
|
|
|
|
| 433 |
return out
|
| 434 |
|
| 435 |
|
| 436 |
+
#: R27 (D-3503): the pool serves the oldest active record and flags newer ones. Nobody picks one.
|
| 437 |
+
DUPLICATE_CODE_RECOMMENDATION = ('The oldest active Odoo record is the canonical record for each '
|
| 438 |
+
'shared SKU. Newer records are flagged for review.')
|
| 439 |
+
|
| 440 |
+
|
| 441 |
def _duplicate_code_limits(rows_src, fields):
|
| 442 |
"""Report collapsed products only through permitted rows and the visible code field."""
|
| 443 |
import modules.product_data as pd
|
|
|
|
| 451 |
codes = ', '.join(report['codes'])
|
| 452 |
return [{'subject': 'code', 'effect': 'duplicate_codes',
|
| 453 |
'cause': f"Shared SKU codes {codes}: {report['gap']} fewer rows than active Odoo products for these codes.",
|
| 454 |
+
'recommendation': DUPLICATE_CODE_RECOMMENDATION}]
|
| 455 |
|
| 456 |
|
| 457 |
def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
| 458 |
consume_corrections: bool = False, *, with_rows: bool = True, needed=None):
|
| 459 |
from routes_customers import grid_assembly
|
| 460 |
from core import perm_scope
|
| 461 |
+
session = grid_events.request_admission(session, 'product_table_workspace', schema_only=not with_rows)
|
| 462 |
team_id, _agent = perm_scope.derive_pool_scope(session.user, MODULE)
|
| 463 |
+
result = grid_assembly(session, scope, storage_key,
|
| 464 |
build=lambda borrowed: _compute_product_assembly(
|
| 465 |
borrowed, scope, storage_key, False, with_rows=with_rows, needed=needed),
|
| 466 |
pool_stamp=lambda: _measure_stamp(session.runtime, team_id) if with_rows else 0,
|
| 467 |
with_rows=with_rows, needed=needed)
|
| 468 |
+
return dict(result, _admission=session.runtime)
|
| 469 |
|
| 470 |
|
| 471 |
def _compute_product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
|
|
|
| 779 |
raise err(403, "out_of_scope", "that product is not in your catalogue")
|
| 780 |
import core.perm_scope as perm_scope
|
| 781 |
|
| 782 |
+
table = pd.table_ops(session.runtime)
|
| 783 |
+
table.overlay_readback = {}
|
| 784 |
+
admission = g.get('_admission')
|
| 785 |
ctx = grid_events.EventCtx(
|
| 786 |
uname=session.uname, allowed_pids=g["pids"], fields=g["fields"],
|
| 787 |
admin=session.admin, fallback_ws=None, seen_ids={},
|
| 788 |
hidden_keys=perm_scope.hidden_keys(
|
| 789 |
+
session.user, MODULE, g["fields"], st=admission or session.runtime),
|
| 790 |
+
scope_key="product", table=table, st=session.runtime, admission=admission)
|
| 791 |
if notes is not None:
|
| 792 |
return _patch_cell_notes(MODULE, pid, notes, ctx,
|
| 793 |
source_row_exists=lambda key, rid, admitted: key == MODULE and int(rid) in g['pids'])
|
|
|
|
| 807 |
# only the per-user stratum would find nothing and report every accepted shared write as
|
| 808 |
# `refused` — a correct write reported as a failure, which is the read-path-cannot-witness-
|
| 809 |
# the-write-path shape ([[read-path-cannot-witness-write-path]]).
|
| 810 |
+
stored = dict(((g.get('ws') or {}).get('overlays') or {}).get(str(pid)) or {})
|
| 811 |
+
stored.update(table.overlay_readback.get(str(pid)) or {})
|
| 812 |
stored = {**stored, **((pd.shared_cells([pid], st=session.runtime).get(str(pid))) or {})}
|
| 813 |
accepted = {k: stored.get(k) for k in updates if k in stored}
|
| 814 |
refused = sorted(k for k in updates
|
api/routes_query.py
CHANGED
|
@@ -1089,6 +1089,15 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s
|
|
| 1089 |
chosen = None
|
| 1090 |
offered = {tool["function"]["name"] for tool in tools}
|
| 1091 |
steps = 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1092 |
while True:
|
| 1093 |
# Joined messages remain transient and are admitted before every model step.
|
| 1094 |
checkpoint = _turn_checkpoint(turn, messages)
|
|
@@ -1122,6 +1131,8 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s
|
|
| 1122 |
return None, None, provider, "stopped"
|
| 1123 |
if spec is not None and not reason and snapshot is not None:
|
| 1124 |
turn["toolCalls"].append({"name": "build_view", "args": _safe(spec), "effect": "read", "ok": False, "ref": None})
|
|
|
|
|
|
|
| 1125 |
return spec, said, provider, reason
|
| 1126 |
calls = spec["tool_calls"]
|
| 1127 |
if not isinstance(calls, list) or not calls:
|
|
@@ -1130,7 +1141,7 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s
|
|
| 1130 |
continue
|
| 1131 |
if closed == "stopped":
|
| 1132 |
return None, None, provider, "stopped"
|
| 1133 |
-
return None, said or "The assistant did not request a tool.", provider, None
|
| 1134 |
if any(not isinstance(call, dict) or not isinstance(call.get("name"), str) for call in calls):
|
| 1135 |
closed = _close_model_turn(turn)
|
| 1136 |
if closed == "joined":
|
|
@@ -1206,6 +1217,10 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s
|
|
| 1206 |
for recorded in turn["toolCalls"]:
|
| 1207 |
if recorded.get("name") == "remember" and recorded.get("ref") == result["id"]:
|
| 1208 |
recorded.pop("linkNote", None)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1209 |
if isinstance(result, dict) and result.get("stopped"):
|
| 1210 |
return None, None, provider, "stopped"
|
| 1211 |
content = json.dumps(result, default=str, ensure_ascii=False)
|
|
|
|
| 1089 |
chosen = None
|
| 1090 |
offered = {tool["function"]["name"] for tool in tools}
|
| 1091 |
steps = 0
|
| 1092 |
+
message_deliveries = []
|
| 1093 |
+
|
| 1094 |
+
def completed_reply(said):
|
| 1095 |
+
# A delivery receipt owns the copy only when this turn did nothing else.
|
| 1096 |
+
if (message_deliveries and all(message_deliveries)
|
| 1097 |
+
and len(turn["intentTexts"]) == 1 and not turn.get("pendingDelete")):
|
| 1098 |
+
return "Sent"
|
| 1099 |
+
return said
|
| 1100 |
+
|
| 1101 |
while True:
|
| 1102 |
# Joined messages remain transient and are admitted before every model step.
|
| 1103 |
checkpoint = _turn_checkpoint(turn, messages)
|
|
|
|
| 1131 |
return None, None, provider, "stopped"
|
| 1132 |
if spec is not None and not reason and snapshot is not None:
|
| 1133 |
turn["toolCalls"].append({"name": "build_view", "args": _safe(spec), "effect": "read", "ok": False, "ref": None})
|
| 1134 |
+
if spec is None and not reason:
|
| 1135 |
+
said = completed_reply(said)
|
| 1136 |
return spec, said, provider, reason
|
| 1137 |
calls = spec["tool_calls"]
|
| 1138 |
if not isinstance(calls, list) or not calls:
|
|
|
|
| 1141 |
continue
|
| 1142 |
if closed == "stopped":
|
| 1143 |
return None, None, provider, "stopped"
|
| 1144 |
+
return None, completed_reply(said) or "The assistant did not request a tool.", provider, None
|
| 1145 |
if any(not isinstance(call, dict) or not isinstance(call.get("name"), str) for call in calls):
|
| 1146 |
closed = _close_model_turn(turn)
|
| 1147 |
if closed == "joined":
|
|
|
|
| 1217 |
for recorded in turn["toolCalls"]:
|
| 1218 |
if recorded.get("name") == "remember" and recorded.get("ref") == result["id"]:
|
| 1219 |
recorded.pop("linkNote", None)
|
| 1220 |
+
message_deliveries.append(
|
| 1221 |
+
name == "agent_message" and isinstance(result, dict) and result.get("ok") is True
|
| 1222 |
+
and bool(result.get("request_id"))
|
| 1223 |
+
and not any(result.get(key) for key in ("refused", "error", "pending", "pendingDelete")))
|
| 1224 |
if isinstance(result, dict) and result.get("stopped"):
|
| 1225 |
return None, None, provider, "stopped"
|
| 1226 |
content = json.dumps(result, default=str, ensure_ascii=False)
|
api/routes_shares.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
|
| 2 |
|
| 3 |
-
GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
|
| 4 |
PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
|
| 5 |
PUT /api/v1/share/{kind}/{oid}/owner <- {owner: "<username>"} (R6d — `field` ONLY)
|
| 6 |
GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
|
|
@@ -60,9 +60,8 @@ SECOND ONE IS GONE RATHER THAN SOFTENED.** It read *"'Widen a users-scoped view
|
|
| 60 |
NOT closed — an `edit` grantee may add `*` — it is CAPPED at Can view"*, and that was an accurate
|
| 61 |
account of R4 that the owner overruled the moment it was put in front of them. *"Grant themselves
|
| 62 |
ownership and lock you out"* stays closed by (3) and `set_grants`' sticky owner. A `view` grantee
|
| 63 |
-
still cannot share at all, and still gets `403`. The client
|
| 64 |
-
|
| 65 |
-
checks are the wall.
|
| 66 |
|
| 67 |
⚠ **WHAT `GET` NOW DISCLOSES.** The grant record carries `by`, and this route returns the record
|
| 68 |
verbatim — so anybody who may read a view's grant list also learns who added each person, a
|
|
@@ -1247,7 +1246,8 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
|
| 1247 |
# registry has no owner to return from `may_administer`, even though the same caller may safely
|
| 1248 |
# claim their own object through the PUT path below. Reflect that exact claim predicate here:
|
| 1249 |
# a Member who owns an unshared View receives the people picker; a collaborator still does not.
|
| 1250 |
-
|
|
|
|
| 1251 |
may_admin = True
|
| 1252 |
# ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
|
| 1253 |
# who owns it, everyone it is granted to, and the tenant's whole username↔name directory —
|
|
@@ -1279,6 +1279,8 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
|
| 1279 |
**rec,
|
| 1280 |
"role": role,
|
| 1281 |
"mayAdminister": may_admin,
|
|
|
|
|
|
|
| 1282 |
"ownerLabel": owner_label,
|
| 1283 |
# ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
|
| 1284 |
# them. `assignable_people` serves bare display names because `user`-kind CELLS store
|
|
|
|
| 1 |
"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
|
| 2 |
|
| 3 |
+
GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, maxGrantableRole, people}
|
| 4 |
PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
|
| 5 |
PUT /api/v1/share/{kind}/{oid}/owner <- {owner: "<username>"} (R6d — `field` ONLY)
|
| 6 |
GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
|
|
|
|
| 60 |
NOT closed — an `edit` grantee may add `*` — it is CAPPED at Can view"*, and that was an accurate
|
| 61 |
account of R4 that the owner overruled the moment it was put in front of them. *"Grant themselves
|
| 62 |
ownership and lock you out"* stays closed by (3) and `set_grants`' sticky owner. A `view` grantee
|
| 63 |
+
still cannot share at all, and still gets `403`. The client caps both role pickers with
|
| 64 |
+
`maxGrantableRole`; these checks remain the wall.
|
|
|
|
| 65 |
|
| 66 |
⚠ **WHAT `GET` NOW DISCLOSES.** The grant record carries `by`, and this route returns the record
|
| 67 |
verbatim — so anybody who may read a view's grant list also learns who added each person, a
|
|
|
|
| 1246 |
# registry has no owner to return from `may_administer`, even though the same caller may safely
|
| 1247 |
# claim their own object through the PUT path below. Reflect that exact claim predicate here:
|
| 1248 |
# a Member who owns an unshared View receives the people picker; a collaborator still does not.
|
| 1249 |
+
claiming = not rec["owner"] and _owns_object(session, kind, oid)
|
| 1250 |
+
if claiming:
|
| 1251 |
may_admin = True
|
| 1252 |
# ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
|
| 1253 |
# who owns it, everyone it is granted to, and the tenant's whole username↔name directory —
|
|
|
|
| 1279 |
**rec,
|
| 1280 |
"role": role,
|
| 1281 |
"mayAdminister": may_admin,
|
| 1282 |
+
"maxGrantableRole": "edit" if claiming else shares.max_grantable_role(
|
| 1283 |
+
kind, oid, session.uname, is_admin=_admin_share, st=session.runtime),
|
| 1284 |
"ownerLabel": owner_label,
|
| 1285 |
# ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
|
| 1286 |
# them. `assignable_people` serves bare display names because `user`-kind CELLS store
|
api/routes_sheet.py
CHANGED
|
@@ -2190,6 +2190,16 @@ def _refresh_context(book, binding, memo=None, *, tab=None, session=None):
|
|
| 2190 |
payload["inputs"] = memo["inputs"]
|
| 2191 |
# Relative filters and measure windows can change their answer at midnight without a write.
|
| 2192 |
encoded = json.dumps(binding, sort_keys=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2193 |
if (_binding_is_volatile(binding, graph, closure)
|
| 2194 |
or any(word in encoded for word in ('"window"', '"within"', '"ending"', '"view_id"'))):
|
| 2195 |
payload["today"] = _sheet_today(session)
|
|
@@ -7573,7 +7583,7 @@ def _asked_key(asked, home, fr):
|
|
| 7573 |
|
| 7574 |
@router.post("/sheets/{workbook_id}/resolve-bindings")
|
| 7575 |
def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
|
| 7576 |
-
session: Session = Depends(require_session)):
|
| 7577 |
"""Values or named refusals for the NAMED bound cells only (contract C1, rulings R2 and R48).
|
| 7578 |
|
| 7579 |
⭐⭐ THIS IS THE DOOR THE COMMIT PATH USES, AND ITS WHOLE POINT IS THAT IT IS SMALL.
|
|
@@ -7629,7 +7639,10 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
|
|
| 7629 |
pairs in it rather than by the number of cells.
|
| 7630 |
"""
|
| 7631 |
with _sheet_request_scope(session):
|
| 7632 |
-
|
|
|
|
|
|
|
|
|
|
| 7633 |
|
| 7634 |
|
| 7635 |
def _resolve_bindings_scoped(workbook_id: str, body, session: Session):
|
|
@@ -7637,7 +7650,8 @@ def _resolve_bindings_scoped(workbook_id: str, body, session: Session):
|
|
| 7637 |
import formula_runtime as fr
|
| 7638 |
|
| 7639 |
try:
|
| 7640 |
-
book = sheets.get_workbook(_store(session), workbook_id
|
|
|
|
| 7641 |
except sheets.SheetRefusal as exc:
|
| 7642 |
raise _refuse(exc)
|
| 7643 |
|
|
@@ -7655,29 +7669,65 @@ def _resolve_bindings_scoped(workbook_id: str, body, session: Session):
|
|
| 7655 |
view_memo = {}
|
| 7656 |
by_binding = {}
|
| 7657 |
answer = {}
|
| 7658 |
-
|
| 7659 |
-
|
| 7660 |
-
|
| 7661 |
-
|
| 7662 |
-
|
| 7663 |
-
|
| 7664 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7665 |
binding = cell.get("binding") or {}
|
| 7666 |
-
|
| 7667 |
-
slot =
|
| 7668 |
-
|
| 7669 |
-
|
| 7670 |
-
|
| 7671 |
-
|
| 7672 |
-
|
| 7673 |
-
|
| 7674 |
-
|
| 7675 |
-
|
| 7676 |
-
|
| 7677 |
-
|
| 7678 |
-
|
| 7679 |
-
|
| 7680 |
-
|
| 7681 |
-
|
| 7682 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7683 |
return answer
|
|
|
|
| 2190 |
payload["inputs"] = memo["inputs"]
|
| 2191 |
# Relative filters and measure windows can change their answer at midnight without a write.
|
| 2192 |
encoded = json.dumps(binding, sort_keys=True)
|
| 2193 |
+
scope = binding.get("scope") or {}
|
| 2194 |
+
if session is not None and scope.get("view_id"):
|
| 2195 |
+
if memo is None:
|
| 2196 |
+
memo = {}
|
| 2197 |
+
try:
|
| 2198 |
+
payload["view"] = _view_filter(session, binding.get("database"), scope["view_id"],
|
| 2199 |
+
scope.get("view_name", ""), memo=memo.setdefault("views", {}))
|
| 2200 |
+
except HTTPException as exc:
|
| 2201 |
+
# Resolution owns the per-cell refusal when a saved view disappears.
|
| 2202 |
+
payload["view"] = {"refusal": exc.detail}
|
| 2203 |
if (_binding_is_volatile(binding, graph, closure)
|
| 2204 |
or any(word in encoded for word in ('"window"', '"within"', '"ending"', '"view_id"'))):
|
| 2205 |
payload["today"] = _sheet_today(session)
|
|
|
|
| 7583 |
|
| 7584 |
@router.post("/sheets/{workbook_id}/resolve-bindings")
|
| 7585 |
def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
|
| 7586 |
+
session: Session = Depends(require_session), response: Response = None):
|
| 7587 |
"""Values or named refusals for the NAMED bound cells only (contract C1, rulings R2 and R48).
|
| 7588 |
|
| 7589 |
⭐⭐ THIS IS THE DOOR THE COMMIT PATH USES, AND ITS WHOLE POINT IS THAT IT IS SMALL.
|
|
|
|
| 7639 |
pairs in it rather than by the number of cells.
|
| 7640 |
"""
|
| 7641 |
with _sheet_request_scope(session):
|
| 7642 |
+
answer = _resolve_bindings_scoped(workbook_id, body, session)
|
| 7643 |
+
if response is not None:
|
| 7644 |
+
response.headers["X-Sheet-Resolved-Cells"] = str(len(answer))
|
| 7645 |
+
return answer
|
| 7646 |
|
| 7647 |
|
| 7648 |
def _resolve_bindings_scoped(workbook_id: str, body, session: Session):
|
|
|
|
| 7650 |
import formula_runtime as fr
|
| 7651 |
|
| 7652 |
try:
|
| 7653 |
+
book = sheets.get_workbook(_store(session), workbook_id,
|
| 7654 |
+
before_normalize=lambda doc: _require_reader(session, doc))
|
| 7655 |
except sheets.SheetRefusal as exc:
|
| 7656 |
raise _refuse(exc)
|
| 7657 |
|
|
|
|
| 7669 |
view_memo = {}
|
| 7670 |
by_binding = {}
|
| 7671 |
answer = {}
|
| 7672 |
+
requested = {str(asked): _asked_key(asked, home, fr)
|
| 7673 |
+
for asked in payload.get("cells") or () if str(asked or "")}
|
| 7674 |
+
selected = set(requested.values()) & bound.keys()
|
| 7675 |
+
graph = _read_graph(session, book, fr, None)
|
| 7676 |
+
previous = getattr(session, "_sheet_refresh_input", None)
|
| 7677 |
+
cache = None
|
| 7678 |
+
|
| 7679 |
+
def viewer_input(key):
|
| 7680 |
+
nonlocal cache
|
| 7681 |
+
if key in selected:
|
| 7682 |
+
return None
|
| 7683 |
+
from core import sheet_binding
|
| 7684 |
+
if cache is None:
|
| 7685 |
+
cache = sheet_binding.read_refresh_cache(_store(session), workbook_id)
|
| 7686 |
+
cell = bound[key]
|
| 7687 |
binding = cell.get("binding") or {}
|
| 7688 |
+
database = _require_database(session, binding.get("database"))
|
| 7689 |
+
slot = _binding_scope_key(session, database, binding)
|
| 7690 |
+
_require_aggregate_field(session, binding)
|
| 7691 |
+
own_tab, _, addr = key.rpartition("!")
|
| 7692 |
+
cell_key = str(sheets.find_tab(book, own_tab)["id"]) + "!" + addr
|
| 7693 |
+
hit = (cache.get("refreshCells") or {}).get(cell_key, {}).get(slot)
|
| 7694 |
+
# A viewer's filter request never resolves an unnamed input or borrows an owner's value.
|
| 7695 |
+
if not hit or hit["cell"].get("state") == "refused":
|
| 7696 |
+
raise err(409, "sheet_refresh_required", "Refresh this input cell before refreshing its dependents.")
|
| 7697 |
+
return _public_refresh_snapshot(hit["cell"])
|
| 7698 |
+
|
| 7699 |
+
viewer = _role_on_workbook(session, book) == "view"
|
| 7700 |
+
if viewer:
|
| 7701 |
+
session._sheet_refresh_input = viewer_input
|
| 7702 |
+
try:
|
| 7703 |
+
for label, key in requested.items():
|
| 7704 |
+
cell = bound.get(key)
|
| 7705 |
+
if cell is None:
|
| 7706 |
+
continue
|
| 7707 |
+
binding = cell.get("binding") or {}
|
| 7708 |
+
own_tab = key.rpartition("!")[0]
|
| 7709 |
+
slot = json.dumps([own_tab, binding], sort_keys=True, default=str)
|
| 7710 |
+
if slot not in by_binding:
|
| 7711 |
+
try:
|
| 7712 |
+
resolved = graph["bound_values"].get(key)
|
| 7713 |
+
if isinstance(resolved, HTTPException):
|
| 7714 |
+
raise resolved
|
| 7715 |
+
if resolved is None:
|
| 7716 |
+
resolved = _resolve_cell_binding(session, binding, book, own_tab,
|
| 7717 |
+
view_memo=view_memo)
|
| 7718 |
+
graph["bound_values"][key] = resolved
|
| 7719 |
+
by_binding[slot] = resolved["value"]
|
| 7720 |
+
except HTTPException as exc:
|
| 7721 |
+
detail = getattr(exc, "detail", None) or {}
|
| 7722 |
+
info = detail.get("error") if isinstance(detail, dict) else None
|
| 7723 |
+
by_binding[slot] = {
|
| 7724 |
+
"value": None,
|
| 7725 |
+
"refusal": {
|
| 7726 |
+
"code": (info or {}).get("code", "sheet_unresolved"),
|
| 7727 |
+
"message": (info or {}).get("message", "This cell could not be refreshed."),
|
| 7728 |
+
},
|
| 7729 |
+
}
|
| 7730 |
+
answer[label] = by_binding[slot]
|
| 7731 |
+
finally:
|
| 7732 |
+
session._sheet_refresh_input = previous
|
| 7733 |
return answer
|
api/routes_sku_actions.py
CHANGED
|
@@ -173,12 +173,12 @@ def _results_sources(session):
|
|
| 173 |
def _results_mirror(runtime):
|
| 174 |
"""Open a private cursor only after asserting the caller's tenant mirror binding."""
|
| 175 |
from pathlib import Path
|
| 176 |
-
from harness import
|
| 177 |
try:
|
| 178 |
runtime.assert_datastore_matches()
|
| 179 |
if not Path(runtime.datastore_path).is_file():
|
| 180 |
raise err(503, 'sku_action_results_source_unavailable', 'This workspace has no Amazon mirror yet.')
|
| 181 |
-
return
|
| 182 |
except runtime_module.DatastoreMismatch:
|
| 183 |
raise err(503, 'sku_action_results_store_mismatch', 'The workspace mirror binding does not match.') from None
|
| 184 |
except HTTPException:
|
|
|
|
| 173 |
def _results_mirror(runtime):
|
| 174 |
"""Open a private cursor only after asserting the caller's tenant mirror binding."""
|
| 175 |
from pathlib import Path
|
| 176 |
+
from harness import runtime as runtime_module
|
| 177 |
try:
|
| 178 |
runtime.assert_datastore_matches()
|
| 179 |
if not Path(runtime.datastore_path).is_file():
|
| 180 |
raise err(503, 'sku_action_results_source_unavailable', 'This workspace has no Amazon mirror yet.')
|
| 181 |
+
return runtime_module.mirror_cursor(runtime, require_ready=False)
|
| 182 |
except runtime_module.DatastoreMismatch:
|
| 183 |
raise err(503, 'sku_action_results_store_mismatch', 'The workspace mirror binding does not match.') from None
|
| 184 |
except HTTPException:
|
api/routes_tables.py
CHANGED
|
@@ -54,6 +54,7 @@ from fastapi import APIRouter
|
|
| 54 |
from fastapi import HTTPException, Request, Response
|
| 55 |
|
| 56 |
from deps import Session, err, require_session
|
|
|
|
| 57 |
from routes_grid import row_patch_receipt
|
| 58 |
|
| 59 |
router = APIRouter(prefix="/api/v1")
|
|
@@ -773,6 +774,31 @@ def query_activity(session: Session, *, object=None, by=None, since=None, until=
|
|
| 773 |
_with_workspace=_with_workspace)
|
| 774 |
|
| 775 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 776 |
def _query_activity_page(session: Session, *, object=None, by=None, since=None, until=None,
|
| 777 |
cursor=None, limit=100, scan_limit=10000, _with_workspace=False):
|
| 778 |
"""Resolve one activity page within its caller's request scope."""
|
|
@@ -884,8 +910,11 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
|
|
| 884 |
if 'app' in obj:
|
| 885 |
from routes_assets import application_index
|
| 886 |
if applications is None:
|
|
|
|
|
|
|
|
|
|
| 887 |
applications = {app['id'] for app in
|
| 888 |
-
application_index(session=
|
| 889 |
admitted = obj['app'] in applications
|
| 890 |
elif 'workbook' in obj:
|
| 891 |
import routes_sheet
|
|
@@ -1140,6 +1169,8 @@ def scoped_pool(session: Session, table_key: str, st=None):
|
|
| 1140 |
# short one. `scoped_pids` below takes the same refusal and answers a different
|
| 1141 |
# question with it, because an envelope needs no row.
|
| 1142 |
raise err(409, "window_required", str(e))
|
|
|
|
|
|
|
| 1143 |
except RuntimeError as e:
|
| 1144 |
raise err(503, "store_not_ready", str(e))
|
| 1145 |
rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base,
|
|
@@ -1181,7 +1212,7 @@ def _new_locked_shared_field(session, table_key, pid, key, body):
|
|
| 1181 |
or any(proposed.get(name) for name in (
|
| 1182 |
"formula", "rollup", "if", "automation", "metric", "measure", "link"))):
|
| 1183 |
raise err(400, "bad_field", "Only a user-created value column can be added here.")
|
| 1184 |
-
fields = _ut().clean_fields([proposed])
|
| 1185 |
if not fields or fields[0]["key"] != key or not _locked_user_field(fields[0]):
|
| 1186 |
raise err(400, "bad_field", "The column name or type is invalid.")
|
| 1187 |
field = fields[0]
|
|
@@ -1536,6 +1567,8 @@ def scoped_pids(session: Session, table_key: str, limits=None, st=None):
|
|
| 1536 |
if limits is not None:
|
| 1537 |
limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)})
|
| 1538 |
return frozenset(), fields_base, defn
|
|
|
|
|
|
|
| 1539 |
except RuntimeError as e:
|
| 1540 |
raise err(503, "store_not_ready", str(e))
|
| 1541 |
# ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection
|
|
@@ -1750,7 +1783,7 @@ _USAGE_KEYS = ("views", "filters", "rollups", "automations", "permRules")
|
|
| 1750 |
_USAGE_DISPLAY_REFS = ("dateField", "stackField", "titleField",
|
| 1751 |
"colorField", "sizeField", "coordField")
|
| 1752 |
|
| 1753 |
-
#: `{(tenant, table_key): (token, {field_key: usage})}` — see `_field_usage` for why this is a
|
| 1754 |
#: memo rather than three fresh documents per assembly. Bounded: cleared past the cap rather than
|
| 1755 |
#: grown, the same rule `core.measure_resolve`'s memos keep.
|
| 1756 |
_USAGE_MEMO = {}
|
|
@@ -1948,7 +1981,8 @@ def _field_usage(session, table_key, fields, defn, ws_st, ws_key, st=None):
|
|
| 1948 |
if not keys:
|
| 1949 |
return {}
|
| 1950 |
lent = st if st is not None else session.runtime
|
| 1951 |
-
|
|
|
|
| 1952 |
token = _usage_token(lent, ws_key)
|
| 1953 |
if token is not None:
|
| 1954 |
hit = _USAGE_MEMO.get(memo_key)
|
|
@@ -2043,9 +2077,61 @@ def _field_usage(session, table_key, fields, defn, ws_st, ws_key, st=None):
|
|
| 2043 |
return tally
|
| 2044 |
|
| 2045 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2046 |
def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 2047 |
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 2048 |
-
*, fields_only: bool = False, needed=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2049 |
if not with_rows and st is None:
|
| 2050 |
if not fields_only:
|
| 2051 |
import routes_odoo_tables as connected
|
|
@@ -2062,7 +2148,7 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
|
| 2062 |
borrowed, table_key, storage_key, False, False, fields_only=fields_only, needed=needed),
|
| 2063 |
pool_stamp=lambda: 0, with_rows=not fields_only, needed=needed)
|
| 2064 |
return _compute_ut_assembly(session, table_key, storage_key, False, with_rows, st,
|
| 2065 |
-
fields_only=fields_only, needed=needed)
|
| 2066 |
|
| 2067 |
|
| 2068 |
def _metadata_pids(session, table_key, ctx, st):
|
|
@@ -2112,7 +2198,7 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 2112 |
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 2113 |
*, fields_only: bool = False, needed=None, include_usage: bool = True,
|
| 2114 |
include_measures: bool = True, include_views: bool = True,
|
| 2115 |
-
metadata_only: bool = True):
|
| 2116 |
"""The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so
|
| 2117 |
`/workspace` and the events route consume any of the three interchangeably.
|
| 2118 |
|
|
@@ -2145,7 +2231,10 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 2145 |
# object then serves `object_shares` to the grant legs downstream, so the assembly stops
|
| 2146 |
# reading that bucket once per question asked of it.
|
| 2147 |
if st is None:
|
| 2148 |
-
|
|
|
|
|
|
|
|
|
|
| 2149 |
metadata_membership = False
|
| 2150 |
if fields_only:
|
| 2151 |
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
|
@@ -2156,9 +2245,8 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 2156 |
fields_base = list(defn.get("fields") or [])
|
| 2157 |
pids, rows_src = frozenset(), []
|
| 2158 |
elif with_rows:
|
| 2159 |
-
|
| 2160 |
-
|
| 2161 |
-
pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st)
|
| 2162 |
else:
|
| 2163 |
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 2164 |
metadata_membership = (metadata_only and table_key != "ut_activity" and
|
|
@@ -2368,6 +2456,7 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 2368 |
workspace['canAddField'] = _ut().may_edit_database(
|
| 2369 |
table_key, session.uname, is_admin=session.admin, st=st)
|
| 2370 |
return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
|
|
|
|
| 2371 |
"fields": fields, "views": views, "lists": lists, "hidden": hidden,
|
| 2372 |
"derived": derived,
|
| 2373 |
# ⭐ W38-T16 — ALWAYS PRESENT, `{}` when this database shares nothing. A key a
|
|
@@ -2685,6 +2774,13 @@ def list_tables(brief: bool = Query(False),
|
|
| 2685 |
"created": t.get("created") or "",
|
| 2686 |
"rowCount": (counts.get(key) if ut.materialises(key, defn=t)
|
| 2687 |
and type(counts.get(key)) is int and counts[key] >= 0 else None)})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2688 |
out.append(row)
|
| 2689 |
if brief:
|
| 2690 |
# ⚠ RE-SORTED, not appended. The loop above is already label-ordered; extending it and
|
|
@@ -2865,9 +2961,11 @@ def create_table(body: dict = Body(default=None),
|
|
| 2865 |
recurrence_validator = lambda raw: govern_date_recurrence(
|
| 2866 |
session, "same", base_fields, raw)
|
| 2867 |
try:
|
|
|
|
| 2868 |
key = ut.create(label, session.uname, fields=raw_fields,
|
| 2869 |
source=source, st=session.runtime,
|
| 2870 |
-
recurrence_validator=recurrence_validator
|
|
|
|
| 2871 |
except ValueError as exc:
|
| 2872 |
raise err(400, getattr(exc, 'code', 'bad_recurrence'), str(exc))
|
| 2873 |
except Exception:
|
|
@@ -2911,7 +3009,7 @@ def table_footprint(table_key: str, session: Session = Depends(require_session))
|
|
| 2911 |
like the delete itself: only someone who could delete may case the joint.
|
| 2912 |
|
| 2913 |
⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row
|
| 2914 |
-
COUNT needs
|
| 2915 |
defn = _defn_or_refuse(session, table_key, defs_only=True)
|
| 2916 |
if not (session.admin or defn.get("createdBy") == session.uname):
|
| 2917 |
raise err(403, "forbidden", "only the database's creator or an admin can delete it")
|
|
@@ -2949,18 +3047,29 @@ def table_footprint(table_key: str, session: Session = Depends(require_session))
|
|
| 2949 |
auto.append({"id": str(aid), "name": d.get("name") or str(aid)})
|
| 2950 |
except Exception:
|
| 2951 |
pass
|
| 2952 |
-
# ⛔ THE ROW COUNT IS THE ONE FIELD THAT NEEDS THE WHOLE DOCUMENT, and it needs it only where
|
| 2953 |
-
# the rows are actually stored here. A read-through database keeps `rows: {}` by construction,
|
| 2954 |
-
# so `len(...)` answered **0** for it before this change and answers 0 now — identical, and the
|
| 2955 |
-
# projection is not what makes it wrong.
|
| 2956 |
-
# ⚠ 0 IS A WRONG NUMBER FOR A READ-THROUGH GRID and always was (`ut_odoo_gl_lines` would say 0
|
| 2957 |
-
# in a dialog headed "what dies with this database"). Booked rather than fixed here: this
|
| 2958 |
-
# ticket is a read-path change and correcting it means asking the mirror for a `count(*)`
|
| 2959 |
-
# inside a confirm dialog. See the `PENDING:` line in `mailbox/A.md`.
|
| 2960 |
if _ut().materialises(table_key, st=s, defn=defn):
|
| 2961 |
rows = len((_ut().get_one(table_key, st=s) or {}).get("rows") or {})
|
| 2962 |
else:
|
| 2963 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2964 |
return {"rows": rows, "fields": fields, "views": len(views),
|
| 2965 |
"sharedUsers": len(g.get("entries") or []),
|
| 2966 |
"automations": sorted(auto, key=lambda a: a["name"].lower())}
|
|
@@ -3136,15 +3245,98 @@ def _amazon_profit_period_cells(table_key, fields, merged, st):
|
|
| 3136 |
if not diagnostic_pids:
|
| 3137 |
return merged
|
| 3138 |
label = _amazon_recorded_period_label(st)
|
|
|
|
|
|
|
| 3139 |
out = dict(merged)
|
| 3140 |
for pid in diagnostic_pids:
|
| 3141 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3142 |
return out
|
| 3143 |
|
| 3144 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3145 |
@router.get("/tables/{table_key}/rows")
|
| 3146 |
def table_rows(table_key: str, session: Session = Depends(require_session),
|
| 3147 |
-
related: str = None, cursor: str = None, limit: int =
|
| 3148 |
"""The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at,
|
| 3149 |
identity}` — so the client's generic topic fetch consumes it with zero new parsing.
|
| 3150 |
|
|
@@ -3158,7 +3350,8 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 3158 |
if table_key == 'ut_activity':
|
| 3159 |
if related:
|
| 3160 |
raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.')
|
| 3161 |
-
page = query_activity(session, cursor=cursor, limit=limit
|
|
|
|
| 3162 |
g = page.pop('_grid')
|
| 3163 |
json_keys = {field['key'] for field in g['fields'] if field.get('type') == 'json'}
|
| 3164 |
rows = [{name: json.dumps(value, ensure_ascii=False, separators=(',', ':'))
|
|
@@ -3183,42 +3376,21 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 3183 |
# ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a
|
| 3184 |
# PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the
|
| 3185 |
# value it replaced (contract C5).
|
| 3186 |
-
|
| 3187 |
-
|
| 3188 |
-
|
| 3189 |
-
|
| 3190 |
-
|
| 3191 |
-
|
| 3192 |
-
|
| 3193 |
-
|
| 3194 |
-
|
| 3195 |
-
|
| 3196 |
-
|
| 3197 |
-
|
| 3198 |
-
|
| 3199 |
-
|
| 3200 |
-
|
| 3201 |
-
if entry_key in assembly_rt.docs:
|
| 3202 |
-
# A cold build already read this entry. The page only borrows it within this request.
|
| 3203 |
-
entry = assembly_rt.docs[entry_key]
|
| 3204 |
-
else:
|
| 3205 |
-
# A warm memo holds no records. Read the original runtime, avoiding a whole-entry copy.
|
| 3206 |
-
entry = _table_entry(original_rt, table_key)
|
| 3207 |
-
_lent = _ut().lend(assembly_rt, user_tables={str(table_key): entry} if entry else {})
|
| 3208 |
-
# The page borrows the same envelope as workspace; only records and shared cells are fresh.
|
| 3209 |
-
pids, rows_src, _fields_base, _defn = scoped_pool(session, table_key, st=_lent)
|
| 3210 |
-
g['pids'] = pids
|
| 3211 |
-
from core import perm_scope
|
| 3212 |
-
g['rows_src'] = [perm_scope.strip_row(row, g['hidden']) for row in rows_src
|
| 3213 |
-
if row['pid'] in g['pids']]
|
| 3214 |
-
g['shared_cells'] = {
|
| 3215 |
-
pid: {key: value for key, value in cells.items() if key not in g['hidden']}
|
| 3216 |
-
for pid, cells in _ut_shared_cells(session, table_key, g['pids']).items()}
|
| 3217 |
-
if table_key in _AMAZON_PROFIT_TABLES:
|
| 3218 |
-
# Schema metadata omits personal cells; restore them after row admission through the bound reader.
|
| 3219 |
-
personal = _ops(session, table_key, st=original_rt).workspace(
|
| 3220 |
-
session.uname, consume_corrections=False)
|
| 3221 |
-
g['ws'] = dict(g.get('ws') or {}, overlays=personal.get('overlays') or {})
|
| 3222 |
# ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge
|
| 3223 |
# this line does not perform since the route was written (owner item 3, 2026-08-09:
|
| 3224 |
# *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*).
|
|
@@ -3287,7 +3459,10 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 3287 |
# `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`.
|
| 3288 |
_stamp_ai_states(table_key, g["fields"], rows, st=_lent)
|
| 3289 |
from routes_odoo_tables import narrow_related_rows
|
| 3290 |
-
|
|
|
|
|
|
|
|
|
|
| 3291 |
rows, g['fields'], metric_refusals = scope_linked_metrics(
|
| 3292 |
session, table_key, rows, g['fields'], g['measures'], related)
|
| 3293 |
if related:
|
|
@@ -3305,7 +3480,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 3305 |
_walled = [] if rows or not perm_scope.row_scope_applies(session.user, table_key) else [{
|
| 3306 |
"subject": "rows", "effect": "filtered",
|
| 3307 |
"cause": "an access rule an administrator set for your account matches no record in "
|
| 3308 |
-
"this
|
| 3309 |
"recommendation": "ask an administrator to review your access rule"}]
|
| 3310 |
# ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the
|
| 3311 |
# only thing that ever set this key; the write door never stopped working and every client
|
|
@@ -3322,10 +3497,12 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 3322 |
for zone in _ut().zones(table_key, st=_lent)]
|
| 3323 |
from core import connector_spec
|
| 3324 |
rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime)
|
| 3325 |
-
rows = _project_cell_notes(table_key, rows, projection_fields,
|
|
|
|
| 3326 |
hidden=g.get('hidden') or ())
|
| 3327 |
rows = _exclude_sourceless(rows, projection_fields)
|
| 3328 |
return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"],
|
|
|
|
| 3329 |
"refusals": metric_refusals,
|
| 3330 |
"zones": _zones,
|
| 3331 |
"docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname,
|
|
@@ -3571,16 +3748,25 @@ def import_rows(table_key: str, body: dict = Body(default=None),
|
|
| 3571 |
# could not see a column the user had just created — it reads the DEFINITION. Same defect shape
|
| 3572 |
# as the rename (item 6a): two places to look, and the surfaces disagreed silently.
|
| 3573 |
|
| 3574 |
-
def
|
| 3575 |
-
"""
|
|
|
|
|
|
|
|
|
|
| 3576 |
|
| 3577 |
-
|
| 3578 |
-
|
| 3579 |
-
|
| 3580 |
-
⚠ It is a SCHEMA wall on a write route, not a row-write wall — the distinction contract C5
|
| 3581 |
-
draws is about the snapshot reaching a read-BACK, and this one does not escape the function."""
|
| 3582 |
-
defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
|
| 3583 |
ut = _ut()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3584 |
if not ut.is_user_table(table_key, st=session.runtime):
|
| 3585 |
raise err(400, "not_a_user_table",
|
| 3586 |
"only a user-created database has an editable schema. A connected source "
|
|
@@ -3635,15 +3821,16 @@ def _admit_if_sources(session, table_key, body, definition, fkey=''):
|
|
| 3635 |
raise err(403 if exc.code == 'if_source_forbidden' else 400, exc.code, str(exc))
|
| 3636 |
|
| 3637 |
|
| 3638 |
-
def _admit_link_selection(session, body):
|
| 3639 |
"""S76-T33: a Link condition may name only target fields this session can read."""
|
| 3640 |
raw = body if isinstance(body, dict) else {}
|
| 3641 |
bag = raw.get("link")
|
| 3642 |
if not isinstance(bag, dict) or not isinstance(bag.get("selectionFilter"), dict):
|
| 3643 |
return
|
| 3644 |
-
target =
|
| 3645 |
-
|
| 3646 |
-
|
|
|
|
| 3647 |
import core.perm_scope as perm_scope
|
| 3648 |
if not perm_scope.may_read(session.user, target, st=session.runtime):
|
| 3649 |
raise err(403, "link_selection_forbidden",
|
|
@@ -3656,21 +3843,52 @@ def _admit_link_selection(session, body):
|
|
| 3656 |
stack.extend(node.get("children") or [])
|
| 3657 |
rhs = node.get("rhs") if isinstance(node.get("rhs"), dict) else {}
|
| 3658 |
named.update(ref for ref in (node.get("colId"), rhs.get("colId")) if isinstance(ref, str) and ref)
|
| 3659 |
-
|
| 3660 |
-
|
| 3661 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3662 |
"The link condition names a field you cannot read in the linked database.")
|
| 3663 |
|
| 3664 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3665 |
@router.post("/tables/{table_key}/fields", status_code=201)
|
| 3666 |
@_c8_write
|
| 3667 |
def add_field(table_key: str, body: dict = Body(default=None),
|
| 3668 |
session: Session = Depends(require_session)):
|
| 3669 |
_managed_or_refuse(session, table_key, mutation=True)
|
| 3670 |
-
definition = _field_or_refuse(session, table_key)
|
| 3671 |
_admit_if_sources(session, table_key, body, definition)
|
| 3672 |
-
_admit_link_selection(session, body)
|
| 3673 |
ut = _ut()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3674 |
from routes_grid import govern_date_recurrence
|
| 3675 |
# Only recurrence admission needs the merged workspace. Plain creates need no row audience.
|
| 3676 |
visible_fields = (ut_write_ctx(session, table_key)['fields']
|
|
@@ -3788,8 +4006,12 @@ def _refusal_sentence(ut, session, body, table_key="", fkey=""):
|
|
| 3788 |
if kind and kind not in ut.UT_FIELD_TYPES:
|
| 3789 |
return (f"{kind!r} is not a column type here (types: "
|
| 3790 |
f"{', '.join(sorted(ut.UT_FIELD_TYPES))})")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3791 |
return (f"the column was refused. Check the name and type, or the table may be at its "
|
| 3792 |
-
f"{
|
| 3793 |
|
| 3794 |
|
| 3795 |
@router.patch("/tables/{table_key}/fields/{fkey}")
|
|
@@ -4423,37 +4645,13 @@ def view_audit_trail(table_key: str, since: float = Query(default=0),
|
|
| 4423 |
|
| 4424 |
@router.delete("/tables/{table_key}/views/{view_id}")
|
| 4425 |
def delete_table_view(table_key: str, view_id: str, request: Request,
|
| 4426 |
-
rev: int = Query(default=None),
|
| 4427 |
body: dict = Body(default=None),
|
| 4428 |
session: Session = Depends(require_session)):
|
| 4429 |
-
"""Delete
|
| 4430 |
-
|
| 4431 |
-
⭐⭐ W50-T25 — THIS ROUTE IS WHERE THE 409 BECOMES OBSERVABLE. C3's refusal is
|
| 4432 |
-
`409 view_revision_stale {rev_current}`; the events channel cannot answer it yet because
|
| 4433 |
-
`api/main.py` (lane F's file) is where a core exception becomes an HTTP status. Landing the
|
| 4434 |
-
same refusal on a route this ticket owns means the shape is proven in this wave rather than
|
| 4435 |
-
asserted, and the mapping the combine copies into `main.py` is a copy of something running.
|
| 4436 |
-
|
| 4437 |
-
`rev` may arrive as a query parameter or in the body. ABSENT MEANS NO CHECK, and the delete
|
| 4438 |
-
is accepted and logged with `rev: null` — which is what keeps the whole change safe before
|
| 4439 |
-
W50-T28 wires the client: nothing on the wire sends a `rev` today, so nothing can be refused.
|
| 4440 |
-
|
| 4441 |
-
⚡ W50-T26 OWNS TRASH. When its `<owner>.trash[<id>]` door lands, this route keeps its wall,
|
| 4442 |
-
its guard and its audit row and gains the move; nothing here has to be unpicked for it.
|
| 4443 |
-
"""
|
| 4444 |
-
from core import view_audit
|
| 4445 |
-
|
| 4446 |
_managed_or_refuse(session, table_key, mutation=True)
|
| 4447 |
vid = str(view_id or "").strip()
|
| 4448 |
if not vid:
|
| 4449 |
raise err(400, "view_required", "choose a view to delete")
|
| 4450 |
-
payload = body if isinstance(body, dict) else {}
|
| 4451 |
-
sent = rev if rev is not None else payload.get("rev")
|
| 4452 |
-
try:
|
| 4453 |
-
sent = None if sent is None else int(sent)
|
| 4454 |
-
except (TypeError, ValueError):
|
| 4455 |
-
raise err(400, "bad_revision", "the view revision must be a whole number")
|
| 4456 |
-
|
| 4457 |
ops = _ops(session, table_key)
|
| 4458 |
# ⛔ `prefer_owner` — W48C-T02's trap, and an audit row is exactly where it would hurt.
|
| 4459 |
# A bare first-hit scan returns whichever stratum the document yielded first, so an admin
|
|
@@ -4465,17 +4663,7 @@ def delete_table_view(table_key: str, view_id: str, request: Request,
|
|
| 4465 |
owner = found[0]
|
| 4466 |
if owner != session.uname and not session.admin:
|
| 4467 |
raise err(403, "view_not_yours", "only the owner can delete this saved view")
|
| 4468 |
-
|
| 4469 |
-
ops.delete_view(owner, vid, actor=session.uname,
|
| 4470 |
-
origin=_audit_origin(request), rev=sent)
|
| 4471 |
-
except view_audit.ViewRevisionStale as stale:
|
| 4472 |
-
raise HTTPException(status_code=409, detail={"error": {
|
| 4473 |
-
"code": "view_revision_stale",
|
| 4474 |
-
"message": "someone else changed this view list while you had it open. "
|
| 4475 |
-
"Reload and try again.",
|
| 4476 |
-
"rev_current": stale.rev_current}})
|
| 4477 |
-
except view_audit.RevisionUnverifiable as unknown:
|
| 4478 |
-
raise err(503, "view_revision_unverifiable", str(unknown))
|
| 4479 |
from core import shares
|
| 4480 |
shares.drop_objects([('view', vid)], st=ops.st)
|
| 4481 |
return {"deleted": vid, "owner": owner}
|
|
@@ -4557,32 +4745,20 @@ def list_view_trash(table_key: str, session: Session = Depends(require_session))
|
|
| 4557 |
|
| 4558 |
@router.post("/tables/{table_key}/views/{view_id}/restore")
|
| 4559 |
def restore_table_view(table_key: str, view_id: str, request: Request,
|
| 4560 |
-
|
| 4561 |
-
body: dict = Body(default=None),
|
| 4562 |
session: Session = Depends(require_session)):
|
| 4563 |
"""Put one trashed view back, with every assignment its delete cascaded away.
|
| 4564 |
|
| 4565 |
-
Guarded by the same revision the caller read, refused with the same `409 view_revision_stale`
|
| 4566 |
-
body as the delete beside it, and audited as C3's `restore` action.
|
| 4567 |
-
|
| 4568 |
⛔ AN ID COLLISION IS A 409 THAT NAMES THE LIVE VIEW, never a silent re-key: a restored view
|
| 4569 |
under a fresh id would leave every grant, assignment and Sheet cell that names the old id
|
| 4570 |
pointing at something else, and it would look like it worked.
|
| 4571 |
"""
|
| 4572 |
from core import table_store as ts
|
| 4573 |
-
from core import view_audit
|
| 4574 |
|
| 4575 |
_managed_or_refuse(session, table_key, mutation=True)
|
| 4576 |
vid = str(view_id or "").strip()
|
| 4577 |
if not vid:
|
| 4578 |
raise err(400, "view_required", "choose a view to restore")
|
| 4579 |
-
payload = body if isinstance(body, dict) else {}
|
| 4580 |
-
sent = rev if rev is not None else payload.get("rev")
|
| 4581 |
-
try:
|
| 4582 |
-
sent = None if sent is None else int(sent)
|
| 4583 |
-
except (TypeError, ValueError):
|
| 4584 |
-
raise err(400, "bad_revision", "the view revision must be a whole number")
|
| 4585 |
-
|
| 4586 |
ops = _ops(session, table_key)
|
| 4587 |
# ⛔ AN UNREADABLE DOCUMENT IS A 503, NOT A 404. `find_trashed` deliberately does not swallow
|
| 4588 |
# its read (unlike the older `find_view` beside it), because answering `None` here would make
|
|
@@ -4605,7 +4781,7 @@ def restore_table_view(table_key: str, view_id: str, request: Request,
|
|
| 4605 |
raise err(403, "view_not_yours", "only the owner can restore this saved view")
|
| 4606 |
try:
|
| 4607 |
landed = ops.restore_view(owner, vid, actor=session.uname,
|
| 4608 |
-
origin=_audit_origin(request)
|
| 4609 |
except ts.ViewIdInUse as clash:
|
| 4610 |
raise HTTPException(status_code=409, detail={"error": {
|
| 4611 |
"code": "view_id_in_use",
|
|
@@ -4614,14 +4790,6 @@ def restore_table_view(table_key: str, view_id: str, request: Request,
|
|
| 4614 |
"name": clash.name}})
|
| 4615 |
except ts.ViewNotInTrash as missing:
|
| 4616 |
raise err(404, "view_not_in_trash", str(missing))
|
| 4617 |
-
except view_audit.ViewRevisionStale as stale:
|
| 4618 |
-
raise HTTPException(status_code=409, detail={"error": {
|
| 4619 |
-
"code": "view_revision_stale",
|
| 4620 |
-
"message": "someone else changed this view list while you had it open. "
|
| 4621 |
-
"Reload and try again.",
|
| 4622 |
-
"rev_current": stale.rev_current}})
|
| 4623 |
-
except view_audit.RevisionUnverifiable as unknown:
|
| 4624 |
-
raise err(503, "view_revision_unverifiable", str(unknown))
|
| 4625 |
return {"restored": vid, "owner": owner, "name": landed.get("name")}
|
| 4626 |
|
| 4627 |
|
|
@@ -4687,30 +4855,15 @@ def preview_view_purge(table_key: str, view_id: str,
|
|
| 4687 |
|
| 4688 |
@router.delete("/tables/{table_key}/views/{view_id}/purge")
|
| 4689 |
def purge_table_view(table_key: str, view_id: str,
|
| 4690 |
-
rev: int = Query(default=None),
|
| 4691 |
session: Session = Depends(require_session)):
|
| 4692 |
-
"""Destroy one trashed view
|
| 4693 |
-
from core import view_audit
|
| 4694 |
-
|
| 4695 |
ops, vid, owner, _entry = _trash_owner_or_refuse(
|
| 4696 |
session, table_key, view_id, mutation=True)
|
| 4697 |
-
try:
|
| 4698 |
-
sent = None if rev is None else int(rev)
|
| 4699 |
-
except (TypeError, ValueError):
|
| 4700 |
-
raise err(400, "bad_revision", "the view revision must be a whole number")
|
| 4701 |
from core import table_store as ts
|
| 4702 |
try:
|
| 4703 |
-
name, adopted = ops.purge_view(owner, vid,
|
| 4704 |
except ts.ViewNotInTrash as missing:
|
| 4705 |
raise err(404, "view_not_in_trash", str(missing))
|
| 4706 |
-
except view_audit.ViewRevisionStale as stale:
|
| 4707 |
-
raise HTTPException(status_code=409, detail={"error": {
|
| 4708 |
-
"code": "view_revision_stale",
|
| 4709 |
-
"message": "someone else changed this view list while you had it open. "
|
| 4710 |
-
"Reload and try again.",
|
| 4711 |
-
"rev_current": stale.rev_current}})
|
| 4712 |
-
except view_audit.RevisionUnverifiable as unknown:
|
| 4713 |
-
raise err(503, "view_revision_unverifiable", str(unknown))
|
| 4714 |
except Exception as exc: # noqa: BLE001
|
| 4715 |
# ⛔ THE ADOPTION FAILED, SO NOTHING WAS DESTROYED. `purge_view` copies the filter before
|
| 4716 |
# it touches the bucket, so this refusal leaves the view exactly where it was.
|
|
|
|
| 54 |
from fastapi import HTTPException, Request, Response
|
| 55 |
|
| 56 |
from deps import Session, err, require_session
|
| 57 |
+
from harness.runtime import DatastoreMismatch
|
| 58 |
from routes_grid import row_patch_receipt
|
| 59 |
|
| 60 |
router = APIRouter(prefix="/api/v1")
|
|
|
|
| 774 |
_with_workspace=_with_workspace)
|
| 775 |
|
| 776 |
|
| 777 |
+
class _ActivityAppReads:
|
| 778 |
+
"""One page's App registry reads, shared by the index's two registry loads (S85-T169)."""
|
| 779 |
+
|
| 780 |
+
_READS = frozenset(('get', 'get_many', 'list_keys', 'get_projection', 'get_entry', 'exists'))
|
| 781 |
+
|
| 782 |
+
def __init__(self, runtime):
|
| 783 |
+
self._runtime, self._memo = runtime, {}
|
| 784 |
+
|
| 785 |
+
def __getattr__(self, name):
|
| 786 |
+
attr = getattr(self._runtime, name)
|
| 787 |
+
if name not in self._READS or not callable(attr):
|
| 788 |
+
return attr
|
| 789 |
+
|
| 790 |
+
def read(*args, **kwargs):
|
| 791 |
+
if kwargs.get('fresh'):
|
| 792 |
+
return attr(*args, **kwargs)
|
| 793 |
+
identity = repr((name, args, sorted(kwargs.items())))
|
| 794 |
+
if identity not in self._memo:
|
| 795 |
+
self._memo[identity] = attr(*args, **kwargs)
|
| 796 |
+
from copy import deepcopy
|
| 797 |
+
return deepcopy(self._memo[identity])
|
| 798 |
+
|
| 799 |
+
return read
|
| 800 |
+
|
| 801 |
+
|
| 802 |
def _query_activity_page(session: Session, *, object=None, by=None, since=None, until=None,
|
| 803 |
cursor=None, limit=100, scan_limit=10000, _with_workspace=False):
|
| 804 |
"""Resolve one activity page within its caller's request scope."""
|
|
|
|
| 910 |
if 'app' in obj:
|
| 911 |
from routes_assets import application_index
|
| 912 |
if applications is None:
|
| 913 |
+
# The index loads the registry twice. This page lists and reads it once.
|
| 914 |
+
reader = copy(session)
|
| 915 |
+
reader.runtime = _ActivityAppReads(session.runtime)
|
| 916 |
applications = {app['id'] for app in
|
| 917 |
+
application_index(session=reader)['applications']}
|
| 918 |
admitted = obj['app'] in applications
|
| 919 |
elif 'workbook' in obj:
|
| 920 |
import routes_sheet
|
|
|
|
| 1169 |
# short one. `scoped_pids` below takes the same refusal and answers a different
|
| 1170 |
# question with it, because an envelope needs no row.
|
| 1171 |
raise err(409, "window_required", str(e))
|
| 1172 |
+
except DatastoreMismatch as e:
|
| 1173 |
+
raise err(409, "cross_tenant_store", str(e)) from e
|
| 1174 |
except RuntimeError as e:
|
| 1175 |
raise err(503, "store_not_ready", str(e))
|
| 1176 |
rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base,
|
|
|
|
| 1212 |
or any(proposed.get(name) for name in (
|
| 1213 |
"formula", "rollup", "if", "automation", "metric", "measure", "link"))):
|
| 1214 |
raise err(400, "bad_field", "Only a user-created value column can be added here.")
|
| 1215 |
+
fields = _ut().clean_fields([proposed], table_key=table_key, st=session.runtime)
|
| 1216 |
if not fields or fields[0]["key"] != key or not _locked_user_field(fields[0]):
|
| 1217 |
raise err(400, "bad_field", "The column name or type is invalid.")
|
| 1218 |
field = fields[0]
|
|
|
|
| 1567 |
if limits is not None:
|
| 1568 |
limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)})
|
| 1569 |
return frozenset(), fields_base, defn
|
| 1570 |
+
except DatastoreMismatch as e:
|
| 1571 |
+
raise err(409, "cross_tenant_store", str(e)) from e
|
| 1572 |
except RuntimeError as e:
|
| 1573 |
raise err(503, "store_not_ready", str(e))
|
| 1574 |
# ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection
|
|
|
|
| 1783 |
_USAGE_DISPLAY_REFS = ("dateField", "stackField", "titleField",
|
| 1784 |
"colorField", "sizeField", "coordField")
|
| 1785 |
|
| 1786 |
+
#: `{(tenant, table_key, field_keys): (token, {field_key: usage})}` — see `_field_usage` for why this is a
|
| 1787 |
#: memo rather than three fresh documents per assembly. Bounded: cleared past the cap rather than
|
| 1788 |
#: grown, the same rule `core.measure_resolve`'s memos keep.
|
| 1789 |
_USAGE_MEMO = {}
|
|
|
|
| 1981 |
if not keys:
|
| 1982 |
return {}
|
| 1983 |
lent = st if st is not None else session.runtime
|
| 1984 |
+
# Registry callers lend already-walled fields, so a narrower reader cannot seed missing counts.
|
| 1985 |
+
memo_key = (str(getattr(session, "tenant", "") or ""), str(table_key), tuple(sorted(keys)))
|
| 1986 |
token = _usage_token(lent, ws_key)
|
| 1987 |
if token is not None:
|
| 1988 |
hit = _USAGE_MEMO.get(memo_key)
|
|
|
|
| 2077 |
return tally
|
| 2078 |
|
| 2079 |
|
| 2080 |
+
def _table_page_store(runtime):
|
| 2081 |
+
"""Keep SQL pages on the bound store while preserving explicit snapshot readers."""
|
| 2082 |
+
from core import grid_events
|
| 2083 |
+
from routes_customers import _AssemblyStore
|
| 2084 |
+
while isinstance(runtime, (grid_events.AdmissionSnapshot, _AssemblyStore)):
|
| 2085 |
+
runtime = runtime._runtime
|
| 2086 |
+
return runtime
|
| 2087 |
+
|
| 2088 |
+
|
| 2089 |
+
def _scoped_table_rows(session, table_key, st, *, cursor=None, limit=None):
|
| 2090 |
+
"""Read one scoped page or follow every continuation for a complete assembly."""
|
| 2091 |
+
from core import perm_scope, store, store_pg
|
| 2092 |
+
try:
|
| 2093 |
+
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 2094 |
+
fields = _source_contract_fields(table_key, [dict(f) for f in (defn.get('fields') or [])], st=st)
|
| 2095 |
+
runtime = _table_page_store(session.runtime)
|
| 2096 |
+
sql_pages = isinstance(perm_scope._window_store(runtime), store_pg.PgStore)
|
| 2097 |
+
reader = runtime if sql_pages and ('rows' not in defn or store.is_projected(defn)) else st
|
| 2098 |
+
rows = []
|
| 2099 |
+
while True:
|
| 2100 |
+
page = perm_scope.scoped_table_page(session.user, table_key, st=reader,
|
| 2101 |
+
cursor=cursor, limit=1024 if limit is None else limit)
|
| 2102 |
+
rows.extend(page['rows'])
|
| 2103 |
+
cursor = page['nextCursor']
|
| 2104 |
+
# An empty permitted page can still have later readable records.
|
| 2105 |
+
if limit is not None or cursor is None:
|
| 2106 |
+
break
|
| 2107 |
+
except ValueError as exc:
|
| 2108 |
+
raise err(400, 'bad_table_page', str(exc)) from exc
|
| 2109 |
+
except perm_scope.Denied as exc:
|
| 2110 |
+
raise err(403, 'forbidden', str(exc)) from exc
|
| 2111 |
+
except perm_scope.UnknownTable as exc:
|
| 2112 |
+
raise err(404, 'unknown_table', str(exc)) from exc
|
| 2113 |
+
except perm_scope.Unresolvable as exc:
|
| 2114 |
+
raise err(409, 'scope_unavailable', str(exc)) from exc
|
| 2115 |
+
except DatastoreMismatch as exc:
|
| 2116 |
+
raise err(409, 'cross_tenant_store', str(exc)) from exc
|
| 2117 |
+
except RuntimeError as exc:
|
| 2118 |
+
raise err(503, 'store_not_ready', str(exc)) from exc
|
| 2119 |
+
return frozenset(row['pid'] for row in rows), rows, fields, defn, cursor
|
| 2120 |
+
|
| 2121 |
+
|
| 2122 |
def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 2123 |
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 2124 |
+
*, fields_only: bool = False, needed=None, cursor=None, limit=None):
|
| 2125 |
+
from core import grid_events
|
| 2126 |
+
actor = grid_events.request_admission(session, table_key + '_table_workspace')
|
| 2127 |
+
result = _ut_assembly(actor, table_key, storage_key, consume_corrections, with_rows, st,
|
| 2128 |
+
fields_only=fields_only, needed=needed, cursor=cursor, limit=limit)
|
| 2129 |
+
return dict(result, _admission=actor.runtime)
|
| 2130 |
+
|
| 2131 |
+
|
| 2132 |
+
def _ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 2133 |
+
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 2134 |
+
*, fields_only: bool = False, needed=None, cursor=None, limit=None):
|
| 2135 |
if not with_rows and st is None:
|
| 2136 |
if not fields_only:
|
| 2137 |
import routes_odoo_tables as connected
|
|
|
|
| 2148 |
borrowed, table_key, storage_key, False, False, fields_only=fields_only, needed=needed),
|
| 2149 |
pool_stamp=lambda: 0, with_rows=not fields_only, needed=needed)
|
| 2150 |
return _compute_ut_assembly(session, table_key, storage_key, False, with_rows, st,
|
| 2151 |
+
fields_only=fields_only, needed=needed, cursor=cursor, limit=limit)
|
| 2152 |
|
| 2153 |
|
| 2154 |
def _metadata_pids(session, table_key, ctx, st):
|
|
|
|
| 2198 |
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 2199 |
*, fields_only: bool = False, needed=None, include_usage: bool = True,
|
| 2200 |
include_measures: bool = True, include_views: bool = True,
|
| 2201 |
+
metadata_only: bool = True, cursor=None, limit=None):
|
| 2202 |
"""The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so
|
| 2203 |
`/workspace` and the events route consume any of the three interchangeably.
|
| 2204 |
|
|
|
|
| 2231 |
# object then serves `object_shares` to the grant legs downstream, so the assembly stops
|
| 2232 |
# reading that bucket once per question asked of it.
|
| 2233 |
if st is None:
|
| 2234 |
+
from core import perm_scope, store_pg
|
| 2235 |
+
sql_pages = isinstance(perm_scope._window_store(_table_page_store(session.runtime)), store_pg.PgStore)
|
| 2236 |
+
st = _table_lend(session.runtime, table_key, defs_only=not with_rows or sql_pages)
|
| 2237 |
+
next_cursor = None
|
| 2238 |
metadata_membership = False
|
| 2239 |
if fields_only:
|
| 2240 |
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
|
|
|
| 2245 |
fields_base = list(defn.get("fields") or [])
|
| 2246 |
pids, rows_src = frozenset(), []
|
| 2247 |
elif with_rows:
|
| 2248 |
+
pids, rows_src, fields_base, defn, next_cursor = _scoped_table_rows(
|
| 2249 |
+
session, table_key, st, cursor=cursor, limit=limit)
|
|
|
|
| 2250 |
else:
|
| 2251 |
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 2252 |
metadata_membership = (metadata_only and table_key != "ut_activity" and
|
|
|
|
| 2456 |
workspace['canAddField'] = _ut().may_edit_database(
|
| 2457 |
table_key, session.uname, is_admin=session.admin, st=st)
|
| 2458 |
return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
|
| 2459 |
+
"nextCursor": next_cursor,
|
| 2460 |
"fields": fields, "views": views, "lists": lists, "hidden": hidden,
|
| 2461 |
"derived": derived,
|
| 2462 |
# ⭐ W38-T16 — ALWAYS PRESENT, `{}` when this database shares nothing. A key a
|
|
|
|
| 2774 |
"created": t.get("created") or "",
|
| 2775 |
"rowCount": (counts.get(key) if ut.materialises(key, defn=t)
|
| 2776 |
and type(counts.get(key)) is int and counts[key] >= 0 else None)})
|
| 2777 |
+
# The stored definition's empty-source cause, from the definition already in hand.
|
| 2778 |
+
# No sourceStatus adds no key, so the row keeps its shape.
|
| 2779 |
+
if isinstance(t.get("sourceStatus"), dict):
|
| 2780 |
+
from routes_odoo_tables import _source_status_limits
|
| 2781 |
+
limits = _source_status_limits(key, t)
|
| 2782 |
+
if limits:
|
| 2783 |
+
row["limits"] = limits
|
| 2784 |
out.append(row)
|
| 2785 |
if brief:
|
| 2786 |
# ⚠ RE-SORTED, not appended. The loop above is already label-ordered; extending it and
|
|
|
|
| 2961 |
recurrence_validator = lambda raw: govern_date_recurrence(
|
| 2962 |
session, "same", base_fields, raw)
|
| 2963 |
try:
|
| 2964 |
+
# A top-level `subject` is a claim `ut.create` refuses. Never drop it silently.
|
| 2965 |
key = ut.create(label, session.uname, fields=raw_fields,
|
| 2966 |
source=source, st=session.runtime,
|
| 2967 |
+
recurrence_validator=recurrence_validator,
|
| 2968 |
+
subject=body.get("subject"))
|
| 2969 |
except ValueError as exc:
|
| 2970 |
raise err(400, getattr(exc, 'code', 'bad_recurrence'), str(exc))
|
| 2971 |
except Exception:
|
|
|
|
| 3009 |
like the delete itself: only someone who could delete may case the joint.
|
| 3010 |
|
| 3011 |
⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row
|
| 3012 |
+
COUNT needs stored rows on a materialised table, or the mirror on a read-through table."""
|
| 3013 |
defn = _defn_or_refuse(session, table_key, defs_only=True)
|
| 3014 |
if not (session.admin or defn.get("createdBy") == session.uname):
|
| 3015 |
raise err(403, "forbidden", "only the database's creator or an admin can delete it")
|
|
|
|
| 3047 |
auto.append({"id": str(aid), "name": d.get("name") or str(aid)})
|
| 3048 |
except Exception:
|
| 3049 |
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3050 |
if _ut().materialises(table_key, st=s, defn=defn):
|
| 3051 |
rows = len((_ut().get_one(table_key, st=s) or {}).get("rows") or {})
|
| 3052 |
else:
|
| 3053 |
+
from routes_odoo_tables import _mirror_cur, population
|
| 3054 |
+
cur = None
|
| 3055 |
+
try:
|
| 3056 |
+
cur, mismatch = _mirror_cur(session, table_key)
|
| 3057 |
+
if mismatch:
|
| 3058 |
+
raise err(409, 'cross_tenant_store', mismatch)
|
| 3059 |
+
rows = population(table_key, cur=cur, rt=s)
|
| 3060 |
+
if rows is None:
|
| 3061 |
+
raise err(503, 'footprint_count_unavailable',
|
| 3062 |
+
'The row count is not held here. This database has no readable mirror binding.')
|
| 3063 |
+
except HTTPException:
|
| 3064 |
+
raise
|
| 3065 |
+
except RuntimeError as exc:
|
| 3066 |
+
raise err(503, 'store_not_ready', str(exc)) from exc
|
| 3067 |
+
except Exception as exc:
|
| 3068 |
+
raise err(503, 'footprint_count_unavailable',
|
| 3069 |
+
'The mirror row count could not be read. Retry when the mirror is available.') from exc
|
| 3070 |
+
finally:
|
| 3071 |
+
if cur is not None:
|
| 3072 |
+
cur.close()
|
| 3073 |
return {"rows": rows, "fields": fields, "views": len(views),
|
| 3074 |
"sharedUsers": len(g.get("entries") or []),
|
| 3075 |
"automations": sorted(auto, key=lambda a: a["name"].lower())}
|
|
|
|
| 3245 |
if not diagnostic_pids:
|
| 3246 |
return merged
|
| 3247 |
label = _amazon_recorded_period_label(st)
|
| 3248 |
+
from modules.amazon_profit import BLANK_REASONS
|
| 3249 |
+
keys = {field.get("key") for field in fields}
|
| 3250 |
out = dict(merged)
|
| 3251 |
for pid in diagnostic_pids:
|
| 3252 |
+
cells = dict(merged[pid], period=label)
|
| 3253 |
+
# A legacy refusal blanked figures and lived only in Period. An empty reason cell beside a
|
| 3254 |
+
# blank figure carries it, so the blank still says why.
|
| 3255 |
+
for note, figures in BLANK_REASONS.items():
|
| 3256 |
+
if (note in keys and not str(cells.get(note) or "").strip()
|
| 3257 |
+
and any(key in keys and cells.get(key) in (None, "") for key in figures)):
|
| 3258 |
+
cells[note] = merged[pid]["period"]
|
| 3259 |
+
out[pid] = cells
|
| 3260 |
return out
|
| 3261 |
|
| 3262 |
|
| 3263 |
+
def _stored_link_pin_ids(session, target, related):
|
| 3264 |
+
"""Target ids of an offered one-hop stored link, read from the cell the source rows door serves.
|
| 3265 |
+
|
| 3266 |
+
None when the pin is not that shape, so the caller keeps the original refusal. The offer, both
|
| 3267 |
+
grants, the source row wall and the source field wall all apply, as they do for the source grid.
|
| 3268 |
+
"""
|
| 3269 |
+
import core.relations as relations
|
| 3270 |
+
import routes_odoo_tables as connected
|
| 3271 |
+
from routes_grid import _may_open, related_path_entries
|
| 3272 |
+
|
| 3273 |
+
if isinstance(related, str):
|
| 3274 |
+
try:
|
| 3275 |
+
related = json.loads(related)
|
| 3276 |
+
except ValueError:
|
| 3277 |
+
return None
|
| 3278 |
+
if not isinstance(related, dict):
|
| 3279 |
+
return None
|
| 3280 |
+
source, via = str(related.get("table") or ""), related.get("via")
|
| 3281 |
+
path = connected._via_path(via)
|
| 3282 |
+
if path is not None and len(path) != 1:
|
| 3283 |
+
return None
|
| 3284 |
+
try:
|
| 3285 |
+
record = int(related.get("record"))
|
| 3286 |
+
except (TypeError, ValueError):
|
| 3287 |
+
return None
|
| 3288 |
+
ut = _ut()
|
| 3289 |
+
if (not source.startswith(ut.KEY_PREFIX) or not _may_open(session, source)
|
| 3290 |
+
or not _may_open(session, target)):
|
| 3291 |
+
return None
|
| 3292 |
+
entries = related_path_entries(session, source, 1)
|
| 3293 |
+
entry = relations.pick_path(entries, path) if path else relations.pick(entries, via)
|
| 3294 |
+
if entry is None or str(entry.get("target")) != str(target) or not entry.get("served"):
|
| 3295 |
+
return None
|
| 3296 |
+
bag, link = entry.get("via") or {}, entry.get("link") or {}
|
| 3297 |
+
field = str(bag.get("field") or "")
|
| 3298 |
+
# The same manual-link predicate `routes_odoo_tables._relation_leg_ids` applies.
|
| 3299 |
+
manual = bool(field and link and not link.get("inverse")
|
| 3300 |
+
and (link.get("selectionMode") == "manual" or not link.get("on")))
|
| 3301 |
+
if bag.get("direction") != "forward" or not manual:
|
| 3302 |
+
return None
|
| 3303 |
+
if not ut.materialises(source, st=session.runtime):
|
| 3304 |
+
return None
|
| 3305 |
+
g = ut_assembly(session, source, consume_corrections=False)
|
| 3306 |
+
if (record not in g["pids"] or field in (g.get("hidden") or ())
|
| 3307 |
+
or field not in {f.get("key") for f in g["fields"]}):
|
| 3308 |
+
return None
|
| 3309 |
+
base = next((r for r in g["rows_src"] if str(r.get("pid")) == str(record)), None)
|
| 3310 |
+
if base is None:
|
| 3311 |
+
return None
|
| 3312 |
+
# The merge order of `table_rows`: definition row, then this user's overlay, then shared cells.
|
| 3313 |
+
cells = {k: v for k, v in base.items() if k != "pid"}
|
| 3314 |
+
overlay = ((g.get("ws") or {}).get("overlays") or {}).get(str(record))
|
| 3315 |
+
if isinstance(overlay, dict):
|
| 3316 |
+
cells.update(overlay)
|
| 3317 |
+
shared = (g.get("shared_cells") or {}).get(str(record))
|
| 3318 |
+
if isinstance(shared, dict):
|
| 3319 |
+
cells.update(shared)
|
| 3320 |
+
return {int(pid) for pid in connected._relation_cell_ids(cells.get(field))}
|
| 3321 |
+
|
| 3322 |
+
|
| 3323 |
+
def _offered_link_rows(session, table_key, rows, related, refusal):
|
| 3324 |
+
"""Narrow by a stored link edge `/related` offers when the shared resolver refused it (D-13545).
|
| 3325 |
+
|
| 3326 |
+
Any other refusal, or a pin of another shape, re-raises `refusal` unchanged.
|
| 3327 |
+
"""
|
| 3328 |
+
detail = refusal.detail if isinstance(refusal.detail, dict) else {}
|
| 3329 |
+
if refusal.status_code != 400 or (detail.get("error") or {}).get("code") != "unknown_relation":
|
| 3330 |
+
raise refusal
|
| 3331 |
+
ids = _stored_link_pin_ids(session, table_key, related)
|
| 3332 |
+
if ids is None:
|
| 3333 |
+
raise refusal
|
| 3334 |
+
return [row for row in rows if int(row["pid"]) in ids]
|
| 3335 |
+
|
| 3336 |
+
|
| 3337 |
@router.get("/tables/{table_key}/rows")
|
| 3338 |
def table_rows(table_key: str, session: Session = Depends(require_session),
|
| 3339 |
+
related: str = None, cursor: str = None, limit: int = None):
|
| 3340 |
"""The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at,
|
| 3341 |
identity}` — so the client's generic topic fetch consumes it with zero new parsing.
|
| 3342 |
|
|
|
|
| 3350 |
if table_key == 'ut_activity':
|
| 3351 |
if related:
|
| 3352 |
raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.')
|
| 3353 |
+
page = query_activity(session, cursor=cursor, limit=100 if limit is None else limit,
|
| 3354 |
+
_with_workspace=True)
|
| 3355 |
g = page.pop('_grid')
|
| 3356 |
json_keys = {field['key'] for field in g['fields'] if field.get('type') == 'json'}
|
| 3357 |
rows = [{name: json.dumps(value, ensure_ascii=False, separators=(',', ':'))
|
|
|
|
| 3376 |
# ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a
|
| 3377 |
# PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the
|
| 3378 |
# value it replaced (contract C5).
|
| 3379 |
+
from core import perm_scope, store_pg
|
| 3380 |
+
try:
|
| 3381 |
+
sql_pages = isinstance(perm_scope._window_store(_table_page_store(session.runtime)), store_pg.PgStore)
|
| 3382 |
+
_lent = _table_lend(session.runtime, table_key, defs_only=sql_pages)
|
| 3383 |
+
except DatastoreMismatch as exc:
|
| 3384 |
+
raise err(409, 'cross_tenant_store', str(exc)) from exc
|
| 3385 |
+
except RuntimeError as exc:
|
| 3386 |
+
raise err(503, 'store_not_ready', str(exc)) from exc
|
| 3387 |
+
# Bare GET callers consume a complete book; a cursor alone requests one default page.
|
| 3388 |
+
page_limit = 100 if cursor is not None and limit is None else limit
|
| 3389 |
+
# Canonical evaluators lend one bounded batch and must retain every row in that batch.
|
| 3390 |
+
if hasattr(session.runtime, '_canonical_needed'):
|
| 3391 |
+
page_limit = None
|
| 3392 |
+
g = ut_assembly(session, table_key, st=_lent, consume_corrections=False,
|
| 3393 |
+
needed=source_needed, cursor=cursor, limit=page_limit)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3394 |
# ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge
|
| 3395 |
# this line does not perform since the route was written (owner item 3, 2026-08-09:
|
| 3396 |
# *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*).
|
|
|
|
| 3459 |
# `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`.
|
| 3460 |
_stamp_ai_states(table_key, g["fields"], rows, st=_lent)
|
| 3461 |
from routes_odoo_tables import narrow_related_rows
|
| 3462 |
+
try:
|
| 3463 |
+
rows = narrow_related_rows(session, table_key, rows, related)
|
| 3464 |
+
except HTTPException as exc:
|
| 3465 |
+
rows = _offered_link_rows(session, table_key, rows, related, exc)
|
| 3466 |
rows, g['fields'], metric_refusals = scope_linked_metrics(
|
| 3467 |
session, table_key, rows, g['fields'], g['measures'], related)
|
| 3468 |
if related:
|
|
|
|
| 3480 |
_walled = [] if rows or not perm_scope.row_scope_applies(session.user, table_key) else [{
|
| 3481 |
"subject": "rows", "effect": "filtered",
|
| 3482 |
"cause": "an access rule an administrator set for your account matches no record in "
|
| 3483 |
+
"this page",
|
| 3484 |
"recommendation": "ask an administrator to review your access rule"}]
|
| 3485 |
# ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the
|
| 3486 |
# only thing that ever set this key; the write door never stopped working and every client
|
|
|
|
| 3497 |
for zone in _ut().zones(table_key, st=_lent)]
|
| 3498 |
from core import connector_spec
|
| 3499 |
rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime)
|
| 3500 |
+
rows = _project_cell_notes(table_key, rows, projection_fields,
|
| 3501 |
+
session.runtime if sql_pages else _lent,
|
| 3502 |
hidden=g.get('hidden') or ())
|
| 3503 |
rows = _exclude_sourceless(rows, projection_fields)
|
| 3504 |
return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"],
|
| 3505 |
+
"nextCursor": g['nextCursor'],
|
| 3506 |
"refusals": metric_refusals,
|
| 3507 |
"zones": _zones,
|
| 3508 |
"docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname,
|
|
|
|
| 3748 |
# could not see a column the user had just created — it reads the DEFINITION. Same defect shape
|
| 3749 |
# as the rename (item 6a): two places to look, and the surfaces disagreed silently.
|
| 3750 |
|
| 3751 |
+
def _registry_field_schema(session, table_key):
|
| 3752 |
+
"""Read the caller's complete registry schema without loading connector records."""
|
| 3753 |
+
from routes_grid import workspace
|
| 3754 |
+
scope = {'product_data': 'product', 'customer_data': 'customer'}[table_key]
|
| 3755 |
+
return workspace(scope=scope, session=session, fields_only=True)['workspace']['fields']
|
| 3756 |
|
| 3757 |
+
|
| 3758 |
+
def _field_or_refuse(session, table_key, fkey="", *, raw=None, create=False):
|
| 3759 |
+
"""Admit schema edits through table and field walls, with administrator registry Link creates as the sole exception."""
|
|
|
|
|
|
|
|
|
|
| 3760 |
ut = _ut()
|
| 3761 |
+
if create and not fkey and table_key in ut.LINKABLE_MODULE_KEYS:
|
| 3762 |
+
if not isinstance(raw, dict) or str(raw.get('type') or '').strip().lower() != 'link':
|
| 3763 |
+
raise err(400, 'not_a_user_table',
|
| 3764 |
+
'A connected database accepts Link fields here. Its source owns the other column types.')
|
| 3765 |
+
if not session.admin:
|
| 3766 |
+
raise err(403, 'forbidden', 'Only an administrator can add a Link field to a connected database.')
|
| 3767 |
+
session.require(table_key)
|
| 3768 |
+
return {'fields': _registry_field_schema(session, table_key)}
|
| 3769 |
+
defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
|
| 3770 |
if not ut.is_user_table(table_key, st=session.runtime):
|
| 3771 |
raise err(400, "not_a_user_table",
|
| 3772 |
"only a user-created database has an editable schema. A connected source "
|
|
|
|
| 3821 |
raise err(403 if exc.code == 'if_source_forbidden' else 400, exc.code, str(exc))
|
| 3822 |
|
| 3823 |
|
| 3824 |
+
def _admit_link_selection(session, body, *, hidden_code='link_selection_forbidden'):
|
| 3825 |
"""S76-T33: a Link condition may name only target fields this session can read."""
|
| 3826 |
raw = body if isinstance(body, dict) else {}
|
| 3827 |
bag = raw.get("link")
|
| 3828 |
if not isinstance(bag, dict) or not isinstance(bag.get("selectionFilter"), dict):
|
| 3829 |
return
|
| 3830 |
+
target = _ut()._field_key(bag.get("table"))
|
| 3831 |
+
registry_target = target in _ut().LINKABLE_MODULE_KEYS
|
| 3832 |
+
if not target.startswith(_ut().KEY_PREFIX) and not registry_target:
|
| 3833 |
+
return
|
| 3834 |
import core.perm_scope as perm_scope
|
| 3835 |
if not perm_scope.may_read(session.user, target, st=session.runtime):
|
| 3836 |
raise err(403, "link_selection_forbidden",
|
|
|
|
| 3843 |
stack.extend(node.get("children") or [])
|
| 3844 |
rhs = node.get("rhs") if isinstance(node.get("rhs"), dict) else {}
|
| 3845 |
named.update(ref for ref in (node.get("colId"), rhs.get("colId")) if isinstance(ref, str) and ref)
|
| 3846 |
+
if registry_target:
|
| 3847 |
+
session.require(target)
|
| 3848 |
+
fields = _registry_field_schema(session, target)
|
| 3849 |
+
hidden = named - {field['key'] for field in fields}
|
| 3850 |
+
else:
|
| 3851 |
+
fields = (_table_entry(session.runtime, target) or {}).get("fields") or []
|
| 3852 |
+
hidden = _ut_hidden(session, target, fields, st=session.runtime) or ()
|
| 3853 |
+
if named & set(hidden):
|
| 3854 |
+
raise err(403, hidden_code,
|
| 3855 |
"The link condition names a field you cannot read in the linked database.")
|
| 3856 |
|
| 3857 |
|
| 3858 |
+
def _add_registry_link(session, table_key, body, definition):
|
| 3859 |
+
"""Store an admitted registry Link in the canonical shared overlay without changing source columns."""
|
| 3860 |
+
from core import shared_overlay
|
| 3861 |
+
from modules import product_data
|
| 3862 |
+
from routes_customers import _shared_key
|
| 3863 |
+
|
| 3864 |
+
ut = _ut()
|
| 3865 |
+
bucket = product_data.TABLE_KEY if table_key == 'product_data' else _shared_key()
|
| 3866 |
+
have = list(definition.get('fields') or []) + list(shared_overlay.fields(bucket, st=session.runtime).values())
|
| 3867 |
+
try:
|
| 3868 |
+
field = ut._clean_field(body)
|
| 3869 |
+
if (not field or any(prior.get('key') == field['key'] for prior in have)
|
| 3870 |
+
or not ut.dynamic_link_filter_valid(field, have)):
|
| 3871 |
+
raise ValueError('The Link field needs a unique name and a valid linked database.')
|
| 3872 |
+
field.update(custom=True, shared=True, createdBy=session.uname)
|
| 3873 |
+
field = shared_overlay.put_field(bucket, field['key'], field, st=session.runtime)
|
| 3874 |
+
except ValueError as exc:
|
| 3875 |
+
raise err(400, getattr(exc, 'code', 'bad_field'), str(exc)) from exc
|
| 3876 |
+
return _with_dropped(ut, {'ok': True, 'value': field, 'refused': False, 'reason': '', 'field': field}, body)
|
| 3877 |
+
|
| 3878 |
+
|
| 3879 |
@router.post("/tables/{table_key}/fields", status_code=201)
|
| 3880 |
@_c8_write
|
| 3881 |
def add_field(table_key: str, body: dict = Body(default=None),
|
| 3882 |
session: Session = Depends(require_session)):
|
| 3883 |
_managed_or_refuse(session, table_key, mutation=True)
|
| 3884 |
+
definition = _field_or_refuse(session, table_key, raw=body, create=True)
|
| 3885 |
_admit_if_sources(session, table_key, body, definition)
|
|
|
|
| 3886 |
ut = _ut()
|
| 3887 |
+
registry_link = table_key in ut.LINKABLE_MODULE_KEYS
|
| 3888 |
+
_admit_link_selection(session, body, hidden_code=(
|
| 3889 |
+
'link_condition_hidden' if registry_link else 'link_selection_forbidden'))
|
| 3890 |
+
if registry_link:
|
| 3891 |
+
return _add_registry_link(session, table_key, body, definition)
|
| 3892 |
from routes_grid import govern_date_recurrence
|
| 3893 |
# Only recurrence admission needs the merged workspace. Plain creates need no row audience.
|
| 3894 |
visible_fields = (ut_write_ctx(session, table_key)['fields']
|
|
|
|
| 4006 |
if kind and kind not in ut.UT_FIELD_TYPES:
|
| 4007 |
return (f"{kind!r} is not a column type here (types: "
|
| 4008 |
f"{', '.join(sorted(ut.UT_FIELD_TYPES))})")
|
| 4009 |
+
limit = ut.field_limit(table_key, session.runtime)
|
| 4010 |
+
if limit is None:
|
| 4011 |
+
return (f"the column was refused. Check the name and type "
|
| 4012 |
+
f"(types: {', '.join(sorted(ut.UT_FIELD_TYPES))})")
|
| 4013 |
return (f"the column was refused. Check the name and type, or the table may be at its "
|
| 4014 |
+
f"{limit}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})")
|
| 4015 |
|
| 4016 |
|
| 4017 |
@router.patch("/tables/{table_key}/fields/{fkey}")
|
|
|
|
| 4645 |
|
| 4646 |
@router.delete("/tables/{table_key}/views/{view_id}")
|
| 4647 |
def delete_table_view(table_key: str, view_id: str, request: Request,
|
|
|
|
| 4648 |
body: dict = Body(default=None),
|
| 4649 |
session: Session = Depends(require_session)):
|
| 4650 |
+
"""Delete an owned saved View and accept ignored legacy request metadata."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4651 |
_managed_or_refuse(session, table_key, mutation=True)
|
| 4652 |
vid = str(view_id or "").strip()
|
| 4653 |
if not vid:
|
| 4654 |
raise err(400, "view_required", "choose a view to delete")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4655 |
ops = _ops(session, table_key)
|
| 4656 |
# ⛔ `prefer_owner` — W48C-T02's trap, and an audit row is exactly where it would hurt.
|
| 4657 |
# A bare first-hit scan returns whichever stratum the document yielded first, so an admin
|
|
|
|
| 4663 |
owner = found[0]
|
| 4664 |
if owner != session.uname and not session.admin:
|
| 4665 |
raise err(403, "view_not_yours", "only the owner can delete this saved view")
|
| 4666 |
+
ops.delete_view(owner, vid, actor=session.uname, origin=_audit_origin(request))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4667 |
from core import shares
|
| 4668 |
shares.drop_objects([('view', vid)], st=ops.st)
|
| 4669 |
return {"deleted": vid, "owner": owner}
|
|
|
|
| 4745 |
|
| 4746 |
@router.post("/tables/{table_key}/views/{view_id}/restore")
|
| 4747 |
def restore_table_view(table_key: str, view_id: str, request: Request,
|
| 4748 |
+
body: dict = Body(default=None),
|
|
|
|
| 4749 |
session: Session = Depends(require_session)):
|
| 4750 |
"""Put one trashed view back, with every assignment its delete cascaded away.
|
| 4751 |
|
|
|
|
|
|
|
|
|
|
| 4752 |
⛔ AN ID COLLISION IS A 409 THAT NAMES THE LIVE VIEW, never a silent re-key: a restored view
|
| 4753 |
under a fresh id would leave every grant, assignment and Sheet cell that names the old id
|
| 4754 |
pointing at something else, and it would look like it worked.
|
| 4755 |
"""
|
| 4756 |
from core import table_store as ts
|
|
|
|
| 4757 |
|
| 4758 |
_managed_or_refuse(session, table_key, mutation=True)
|
| 4759 |
vid = str(view_id or "").strip()
|
| 4760 |
if not vid:
|
| 4761 |
raise err(400, "view_required", "choose a view to restore")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4762 |
ops = _ops(session, table_key)
|
| 4763 |
# ⛔ AN UNREADABLE DOCUMENT IS A 503, NOT A 404. `find_trashed` deliberately does not swallow
|
| 4764 |
# its read (unlike the older `find_view` beside it), because answering `None` here would make
|
|
|
|
| 4781 |
raise err(403, "view_not_yours", "only the owner can restore this saved view")
|
| 4782 |
try:
|
| 4783 |
landed = ops.restore_view(owner, vid, actor=session.uname,
|
| 4784 |
+
origin=_audit_origin(request))
|
| 4785 |
except ts.ViewIdInUse as clash:
|
| 4786 |
raise HTTPException(status_code=409, detail={"error": {
|
| 4787 |
"code": "view_id_in_use",
|
|
|
|
| 4790 |
"name": clash.name}})
|
| 4791 |
except ts.ViewNotInTrash as missing:
|
| 4792 |
raise err(404, "view_not_in_trash", str(missing))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4793 |
return {"restored": vid, "owner": owner, "name": landed.get("name")}
|
| 4794 |
|
| 4795 |
|
|
|
|
| 4855 |
|
| 4856 |
@router.delete("/tables/{table_key}/views/{view_id}/purge")
|
| 4857 |
def purge_table_view(table_key: str, view_id: str,
|
|
|
|
| 4858 |
session: Session = Depends(require_session)):
|
| 4859 |
+
"""Destroy one trashed view after its dependent cells have taken its filter."""
|
|
|
|
|
|
|
| 4860 |
ops, vid, owner, _entry = _trash_owner_or_refuse(
|
| 4861 |
session, table_key, view_id, mutation=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4862 |
from core import table_store as ts
|
| 4863 |
try:
|
| 4864 |
+
name, adopted = ops.purge_view(owner, vid, actor=session.uname)
|
| 4865 |
except ts.ViewNotInTrash as missing:
|
| 4866 |
raise err(404, "view_not_in_trash", str(missing))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4867 |
except Exception as exc: # noqa: BLE001
|
| 4868 |
# ⛔ THE ADOPTION FAILED, SO NOTHING WAS DESTROYED. `purge_view` copies the filter before
|
| 4869 |
# it touches the bucket, so this refusal leaves the view exactly where it was.
|
api/routes_usage.py
CHANGED
|
@@ -12,15 +12,52 @@ reporting feature into a wall the owner explicitly did not ask for this wave.
|
|
| 12 |
`?scope=tenant` folds every account and is refused for a non-admin — how much AI another employee
|
| 13 |
used is not a fact this product hands out sideways.
|
| 14 |
"""
|
| 15 |
-
from fastapi import APIRouter, Depends
|
| 16 |
|
| 17 |
import usage_ledger
|
| 18 |
import turn_audit
|
|
|
|
| 19 |
from deps import Session, err, require_session
|
| 20 |
|
| 21 |
router = APIRouter(prefix="/api/v1")
|
| 22 |
|
| 23 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
@router.get("/usage/turns")
|
| 25 |
def turns(scope: str = "me", week: str = None, offset: int = 0, limit: int = 50,
|
| 26 |
session: Session = Depends(require_session)):
|
|
|
|
| 12 |
`?scope=tenant` folds every account and is refused for a non-admin — how much AI another employee
|
| 13 |
used is not a fact this product hands out sideways.
|
| 14 |
"""
|
| 15 |
+
from fastapi import APIRouter, Depends, Query
|
| 16 |
|
| 17 |
import usage_ledger
|
| 18 |
import turn_audit
|
| 19 |
+
import usage_activity
|
| 20 |
from deps import Session, err, require_session
|
| 21 |
|
| 22 |
router = APIRouter(prefix="/api/v1")
|
| 23 |
|
| 24 |
|
| 25 |
+
@router.get("/usage/activity/mcp")
|
| 26 |
+
def mcp_activity(from_: str = Query(default=None, alias="from"),
|
| 27 |
+
to: str = None, user: str = None, tool: str = None,
|
| 28 |
+
cursor: str = None, limit: int = 100,
|
| 29 |
+
session: Session = Depends(require_session)):
|
| 30 |
+
"""Return complete MCP tool totals and one page of value-free event metadata."""
|
| 31 |
+
if not session.admin:
|
| 32 |
+
raise err(403, "admin_required", "Usage activity is available to administrators only.")
|
| 33 |
+
try:
|
| 34 |
+
return usage_activity.read_mcp(session, from_=from_, to=to, user=user,
|
| 35 |
+
tool=tool, cursor=cursor, limit=limit)
|
| 36 |
+
except PermissionError:
|
| 37 |
+
raise err(403, "forbidden", "The usage runtime must belong to this workspace.") from None
|
| 38 |
+
except usage_activity.McpCursorInvalid:
|
| 39 |
+
raise err(400, "usage_mcp_cursor_invalid", "Reload MCP activity to continue.") from None
|
| 40 |
+
except usage_activity.McpCursorStale:
|
| 41 |
+
raise err(409, "usage_mcp_cursor_stale", "Retained MCP activity changed. Reload to continue.") from None
|
| 42 |
+
except ValueError as exc:
|
| 43 |
+
raise err(400, "bad_request", str(exc)) from None
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@router.get("/usage/activity")
|
| 47 |
+
def activity(from_: str = Query(default=None, alias="from"),
|
| 48 |
+
to: str = None, user: str = None, kind: str = None,
|
| 49 |
+
session: Session = Depends(require_session)):
|
| 50 |
+
"""Return retained activity counts for this tenant's administrators."""
|
| 51 |
+
if not session.admin:
|
| 52 |
+
raise err(403, "admin_required", "Usage activity is available to administrators only.")
|
| 53 |
+
try:
|
| 54 |
+
return usage_activity.read(session, from_=from_, to=to, user=user, kind=kind)
|
| 55 |
+
except PermissionError:
|
| 56 |
+
raise err(403, "forbidden", "The usage runtime must belong to this workspace.") from None
|
| 57 |
+
except ValueError as exc:
|
| 58 |
+
raise err(400, "bad_request", str(exc)) from None
|
| 59 |
+
|
| 60 |
+
|
| 61 |
@router.get("/usage/turns")
|
| 62 |
def turns(scope: str = "me", week: str = None, offset: int = 0, limit: int = 50,
|
| 63 |
session: Session = Depends(require_session)):
|
api/sync_worker.py
CHANGED
|
@@ -7,6 +7,7 @@ It keeps its own DuckDB files, so it never opens the web process's royal.duckdb.
|
|
| 7 |
Its heavy jobs wait until it holds the store's sync writer lease, so two Spaces never both sync.
|
| 8 |
It loads the app as `api.main`, the name the web uses, so the process holds one main module (D-781).
|
| 9 |
"""
|
|
|
|
| 10 |
import importlib
|
| 11 |
import os
|
| 12 |
import sys
|
|
@@ -115,6 +116,8 @@ def boot_app_migrations(app_main):
|
|
| 115 |
|
| 116 |
|
| 117 |
def main(argv=None):
|
|
|
|
|
|
|
| 118 |
import sync_runtime
|
| 119 |
args = sys.argv[1:] if argv is None else argv
|
| 120 |
mode = next((sync_runtime.JOB_FLAGS[flag] for flag in args if flag in sync_runtime.JOB_FLAGS), None)
|
|
|
|
| 7 |
Its heavy jobs wait until it holds the store's sync writer lease, so two Spaces never both sync.
|
| 8 |
It loads the app as `api.main`, the name the web uses, so the process holds one main module (D-781).
|
| 9 |
"""
|
| 10 |
+
import faulthandler
|
| 11 |
import importlib
|
| 12 |
import os
|
| 13 |
import sys
|
|
|
|
| 116 |
|
| 117 |
|
| 118 |
def main(argv=None):
|
| 119 |
+
# Enable before app imports so native failures include every Python thread's stack.
|
| 120 |
+
faulthandler.enable(all_threads=True)
|
| 121 |
import sync_runtime
|
| 122 |
args = sys.argv[1:] if argv is None else argv
|
| 123 |
mode = next((sync_runtime.JOB_FLAGS[flag] for flag in args if flag in sync_runtime.JOB_FLAGS), None)
|
api/test_odoo_rank_refresh.py
CHANGED
|
@@ -232,7 +232,144 @@ def _with_collaborators(source, nodes, ns):
|
|
| 232 |
FREEZE_OFF = SimpleNamespace(enabled=lambda: False)
|
| 233 |
|
| 234 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
class RankRefreshTests(unittest.TestCase):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
def setUp(self):
|
| 237 |
self.stack = ExitStack()
|
| 238 |
self.addCleanup(self.stack.close)
|
|
@@ -921,7 +1058,8 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 921 |
return datetime.fromisoformat(text.replace('Z', '+00:00')).timestamp()
|
| 922 |
|
| 923 |
def run_slot_loop(self, memory, *, boot, max_sleeps, tenant='royal-imports', config=None,
|
| 924 |
-
reports=None, checkpoints=None, take=None, slot_flags=None, status=None
|
|
|
|
| 925 |
"""Drive the real loop and its real slot resolver on a clock that advances at each wait."""
|
| 926 |
import odoo_relational as rel
|
| 927 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
|
@@ -939,8 +1077,9 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 939 |
state = {}
|
| 940 |
|
| 941 |
def wait(seconds):
|
|
|
|
| 942 |
sleeps.append(seconds)
|
| 943 |
-
if len(sleeps) >= max_sleeps:
|
| 944 |
raise StopIteration('bounded synthetic resident loop')
|
| 945 |
now[0] += seconds
|
| 946 |
return False
|
|
@@ -1010,22 +1149,21 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1010 |
last = self.utc('2026-10-23T06:05:00Z')
|
| 1011 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1012 |
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1013 |
-
boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=
|
|
|
|
| 1014 |
slot, next_slot = self.utc('2026-10-24T06:00:00Z'), self.utc('2026-10-25T06:00:00Z')
|
| 1015 |
boot_work = [e for e in events if e[1] < slot]
|
| 1016 |
# Restore only at boot: the saved Amazon mirror, no mirror pass, no count, no store half.
|
| 1017 |
self.assertEqual(boot_work, [('amazon', self.utc('2026-10-24T02:30:00Z'), 'boot', False)])
|
| 1018 |
self.assertIn('2026-10-24T06:00:00Z (13:00 Asia/Jakarta)', self.loop_output)
|
| 1019 |
-
#
|
| 1020 |
-
self.assertEqual(sleeps
|
| 1021 |
-
self.
|
| 1022 |
-
self.
|
| 1023 |
-
# No second Odoo pass before the next day's slot: the wait after the pass targets it.
|
| 1024 |
-
self.assertEqual(sleeps[1], 86400)
|
| 1025 |
# The slot is stamped BEFORE the mirror pass reads Odoo, and the status names the next one.
|
| 1026 |
import odoo_relational as rel
|
| 1027 |
stamp = self.slot_store.docs[rel.CONFIG_KEY][rel.RESYNC_MEMORY_FIELD]['slotPassAt']
|
| 1028 |
-
self.assertEqual(rel._memory_epoch(stamp), next_slot
|
| 1029 |
self.assertEqual(self.slot_state['nextPassAt'], '2026-10-26T06:00:00Z')
|
| 1030 |
|
| 1031 |
def test_h73_a_missed_slot_runs_once_at_boot_then_at_the_next_slot(self):
|
|
@@ -1034,11 +1172,11 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1034 |
boot = self.utc('2026-10-24T09:00:00Z')
|
| 1035 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1036 |
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1037 |
-
boot=boot, max_sleeps=
|
| 1038 |
next_slot = self.utc('2026-10-25T06:00:00Z')
|
| 1039 |
mirror = [e[1] for e in odoo if e[0] == 'mirror']
|
| 1040 |
-
self.assertEqual(mirror, [boot, next_slot
|
| 1041 |
-
self.assertEqual(sleeps,
|
| 1042 |
self.assertNotIn(('amazon', boot, 'boot', False), events)
|
| 1043 |
|
| 1044 |
def test_h73_a_restart_inside_a_slot_pass_does_not_run_it_twice(self):
|
|
@@ -1046,7 +1184,6 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1046 |
import odoo_relational as rel
|
| 1047 |
stamp = self.utc('2026-10-24T06:00:01Z')
|
| 1048 |
yesterday = self.utc('2026-10-23T06:30:00Z')
|
| 1049 |
-
to_next_slot = self.utc('2026-10-25T06:00:01Z') - self.utc('2026-10-24T06:40:00Z')
|
| 1050 |
base = {'passAt': yesterday, 'fullRebuildAt': yesterday, 'slotPassAt': stamp,
|
| 1051 |
'fingerprints': {'res_partner': (3, 7)}}
|
| 1052 |
# Ended: `slotDoneAt` after the claim, or (older memory) `passAt` after it. No Odoo call.
|
|
@@ -1054,7 +1191,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1054 |
{'passAt': self.utc('2026-10-24T06:30:00Z')}):
|
| 1055 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1056 |
dict(base, **ended), boot=self.utc('2026-10-24T06:40:00Z'), max_sleeps=1)
|
| 1057 |
-
self.assertEqual((odoo, sleeps), ([], [
|
| 1058 |
# Killed: the end marker predates the claim. One pass at boot, stamped start and end.
|
| 1059 |
boot = self.utc('2026-10-24T06:40:00Z')
|
| 1060 |
events, sleeps, odoo = self.run_slot_loop(
|
|
@@ -1080,7 +1217,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1080 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1081 |
dict(base, slotPassAt=boot, slotDoneAt=yesterday, slotResumedFor=slot_at,
|
| 1082 |
slotResumes=3), boot=fourth, max_sleeps=1)
|
| 1083 |
-
self.assertEqual((odoo, sleeps), ([], [
|
| 1084 |
self.assertNotIn('never finished', self.loop_output)
|
| 1085 |
self.assertIn('slot_pass_interrupted: the 13:00 Asia/Jakarta slot pass was interrupted by '
|
| 1086 |
'4 restarts and waits for the next slot at 2026-10-25T06:00:00Z',
|
|
@@ -1110,7 +1247,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1110 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 1111 |
self.assertIn('the 13:00 Asia/Jakarta slot pass started at 2026-10-27T06:00:01Z never '
|
| 1112 |
'finished; running it again now (resume 1 of 3)', self.loop_output)
|
| 1113 |
-
self.assertEqual(sleeps, [
|
| 1114 |
|
| 1115 |
def test_h1_slot_resumes_are_bounded_and_reset_by_a_new_slot(self):
|
| 1116 |
"""H1: a killed slot pass resumes while its count is under 3, then waits; a new slot starts over"""
|
|
@@ -1145,7 +1282,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1145 |
last = self.utc('2026-10-23T06:05:00Z')
|
| 1146 |
events, _sleeps, _odoo = self.run_slot_loop(
|
| 1147 |
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1148 |
-
boot=self.utc('2026-10-
|
| 1149 |
slot = self.utc('2026-10-24T06:00:00Z')
|
| 1150 |
kinds = [e[0] for e in events if e[1] > slot]
|
| 1151 |
amazon = [i for i, e in enumerate(e for e in events if e[1] > slot)
|
|
@@ -1164,7 +1301,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1164 |
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1165 |
boot=boot, max_sleeps=1, tenant='acme')
|
| 1166 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 1167 |
-
self.assertEqual(sleeps, [
|
| 1168 |
self.assertNotIn('nextPassAt', self.slot_state)
|
| 1169 |
|
| 1170 |
def test_h73_manual_tenant0_has_no_slot(self):
|
|
@@ -1197,7 +1334,8 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1197 |
take = lambda **kwargs: held.append(tuple(kwargs.get('exclude_requested_by') or ())) or []
|
| 1198 |
_events, _sleeps, odoo = self.run_slot_loop(
|
| 1199 |
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1200 |
-
boot=self.utc('2026-10-24T04:30:00Z'), max_sleeps=
|
|
|
|
| 1201 |
self.assertTrue(odoo, 'the slot pass ran')
|
| 1202 |
# Boot 90 min before the slot: the drain outside the slot holds first load, and so does the claim.
|
| 1203 |
self.assertTrue(len(held) >= 2 and all(h == ('first load',) for h in held), held)
|
|
@@ -1223,21 +1361,23 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1223 |
self.assertTrue(held and all('first load' in exclusions for exclusions in held))
|
| 1224 |
self.assertEqual(job, {'state': 'running', 'rowsDone': 10})
|
| 1225 |
|
| 1226 |
-
def
|
| 1227 |
-
"""
|
| 1228 |
last = self.utc('2026-10-23T06:05:00Z')
|
| 1229 |
held = []
|
| 1230 |
-
take = lambda **kwargs: held.append(
|
| 1231 |
self.run_slot_loop({'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1232 |
boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=1, take=take)
|
| 1233 |
-
self.
|
|
|
|
|
|
|
| 1234 |
|
| 1235 |
-
def
|
| 1236 |
-
"""
|
| 1237 |
last = self.utc('2026-10-23T06:05:00Z')
|
| 1238 |
flags = []
|
| 1239 |
self.run_slot_loop({'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1240 |
-
boot=self.utc('2026-10-
|
| 1241 |
resync = [flag for args, flag in flags if args[:1] == ('resync',)]
|
| 1242 |
self.assertEqual(resync, [True])
|
| 1243 |
self.assertTrue(all(not flag for args, flag in flags if args[:1] != ('resync',)), flags)
|
|
@@ -1245,8 +1385,13 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1245 |
from datetime import datetime, timezone
|
| 1246 |
now = datetime(2026, 10, 29, 10, 16, tzinfo=timezone.utc)
|
| 1247 |
ticks = {'orders': {'state': 'done', 'updated_at': '2026-10-28T18:20:03+00:00'}}
|
| 1248 |
-
|
| 1249 |
-
self.assertEqual(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1250 |
|
| 1251 |
def test_slot_near_and_late(self):
|
| 1252 |
"""The quiet window remains two hours; the slot clock alone cannot make a pass late."""
|
|
@@ -1296,7 +1441,6 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1296 |
import odoo_relational as rel
|
| 1297 |
last = self.utc('2026-10-22T06:05:00Z')
|
| 1298 |
boot = self.utc('2026-10-24T07:00:00Z')
|
| 1299 |
-
next_slot = self.utc('2026-10-25T06:00:00Z')
|
| 1300 |
memory = {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}
|
| 1301 |
held = {'account_move_line': {'phase': 'backfill', 'pulled': 0}}
|
| 1302 |
marker = 'account_move_line.cols.amount_residual'
|
|
@@ -1307,7 +1451,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1307 |
memory, boot=boot, max_sleeps=2, reports=[held, held],
|
| 1308 |
checkpoints=[before, after, after, after])
|
| 1309 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot, boot + 60])
|
| 1310 |
-
self.assertEqual(sleeps, [60.0,
|
| 1311 |
self.assertEqual([e for e in events if e[0] in ('meta', 'amazon', 'rebuild')], [])
|
| 1312 |
stamps = rel.read_resync_memory(self.slot_store)
|
| 1313 |
self.assertEqual((stamps['slotPassAt'], stamps['slotDoneAt']), (boot, boot))
|
|
@@ -1318,7 +1462,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1318 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1319 |
memory, boot=boot, max_sleeps=1, reports=[held], checkpoints=[before, before])
|
| 1320 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 1321 |
-
self.assertEqual(sleeps, [
|
| 1322 |
self.assertNotIn('odoo backfill continues', self.loop_output)
|
| 1323 |
|
| 1324 |
# ── Hotfix 10-31: a good pass after the slot satisfies it, so a killed claim does not alarm ──
|
|
@@ -1373,7 +1517,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1373 |
memory, boot=boot, max_sleeps=1, config=self.stored_config(memory))
|
| 1374 |
# The loop reached the slot block: it named the next pass and waits for that slot.
|
| 1375 |
self.assertEqual(self.slot_state['nextPassAt'], '2026-11-01T06:00:00Z')
|
| 1376 |
-
self.assertEqual(sleeps, [
|
| 1377 |
self.assertEqual(odoo, [])
|
| 1378 |
self.assertNotIn('never finished', self.loop_output)
|
| 1379 |
self.assertNotIn('slot_pass_interrupted', self.loop_output)
|
|
|
|
| 232 |
FREEZE_OFF = SimpleNamespace(enabled=lambda: False)
|
| 233 |
|
| 234 |
|
| 235 |
+
def observe_s85_t113_growing_source(*, capture=True, mirror_rows=7706, hook=True,
|
| 236 |
+
prior_counts=None):
|
| 237 |
+
"""Reconcile a synthetic pull after Amazon takes 45 minutes and Odoo grows by 69 rows."""
|
| 238 |
+
from contextlib import nullcontext
|
| 239 |
+
from datetime import datetime, timedelta, timezone
|
| 240 |
+
import _fake_store
|
| 241 |
+
import automation_engine
|
| 242 |
+
import connector_onboarding
|
| 243 |
+
import core.odoo as odoo
|
| 244 |
+
import core.user_tables as ut
|
| 245 |
+
import odoo_relational as rel
|
| 246 |
+
import rollup_sql
|
| 247 |
+
import routes_connectors as connectors
|
| 248 |
+
import routes_odoo_tables
|
| 249 |
+
from harness import runtime
|
| 250 |
+
|
| 251 |
+
key = rel.STOCK_QUANT_KEY
|
| 252 |
+
source = {'count': mirror_rows, 'seconds': 0, 'calls': []}
|
| 253 |
+
|
| 254 |
+
class GrowingSource:
|
| 255 |
+
def search_count(self, model, domain):
|
| 256 |
+
assert model == 'stock.quant'
|
| 257 |
+
source['calls'].append((source['seconds'], copy.deepcopy(domain), source['count']))
|
| 258 |
+
# A count of all locations must fail the internal-stock reconciliation.
|
| 259 |
+
return source['count'] if domain == [('location_id.usage', '=', 'internal')] else 300000
|
| 260 |
+
|
| 261 |
+
def __getattr__(self, method):
|
| 262 |
+
raise AssertionError('unexpected source method: ' + method)
|
| 263 |
+
|
| 264 |
+
rt = _fake_store.FakeStore()
|
| 265 |
+
rt.key = rt.tenant = 's85_t113_fixture'
|
| 266 |
+
rt.odoo_source = lambda: SimpleNamespace(_bound=nullcontext)
|
| 267 |
+
rt.assert_datastore_matches = lambda: None
|
| 268 |
+
old = (datetime.now(timezone.utc) - timedelta(days=2)).isoformat()
|
| 269 |
+
rt.put(rel.CONFIG_KEY, {'grids': {key: True}, 'lastGoodSyncAt': old})
|
| 270 |
+
|
| 271 |
+
def sync_all(*, after_entity=None):
|
| 272 |
+
if after_entity:
|
| 273 |
+
after_entity('stock_quant')
|
| 274 |
+
|
| 275 |
+
def plain_sync_all():
|
| 276 |
+
return None
|
| 277 |
+
|
| 278 |
+
fixture_ds = SimpleNamespace(sync_all=sync_all if hook else plain_sync_all,
|
| 279 |
+
_resident_odoo_runtime=lambda: rt, ENTITIES={'stock_quant': {'model': 'stock.quant'}})
|
| 280 |
+
tree = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 281 |
+
wanted = {'_entity_source_counter', '_capture_resync_source_counts', '_rebuild_odoo_relational'}
|
| 282 |
+
nodes = [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in wanted]
|
| 283 |
+
namespace = {'routes_connectors': connectors, '_resync_tenant': lambda: (rt.key, 'fixture'),
|
| 284 |
+
'_rebuild_meta_relational': lambda *a, **k: {},
|
| 285 |
+
'_rebuild_amazon_relational': lambda *a, **k: {},
|
| 286 |
+
'_PASS_SOURCE_COUNTS': {rt.key: dict(prior_counts or {})}}
|
| 287 |
+
exec(compile(ast.Module(body=nodes, type_ignores=[]), 'main.py:S85-T113', 'exec'), namespace)
|
| 288 |
+
|
| 289 |
+
def plan(*_a, **_k):
|
| 290 |
+
return {'stock_quant': [{'_id': str(i)} for i in range(1, mirror_rows + 1)]}
|
| 291 |
+
|
| 292 |
+
def amazon(why):
|
| 293 |
+
assert why == 'resync'
|
| 294 |
+
source['seconds'] += 45 * 60
|
| 295 |
+
source['count'] += 69
|
| 296 |
+
|
| 297 |
+
with ExitStack() as stack, redirect_stdout(io.StringIO()) as output:
|
| 298 |
+
stack.enter_context(patch.object(socket.socket, 'connect', side_effect=AssertionError('no network')))
|
| 299 |
+
stack.enter_context(_fake_store.installed(rt))
|
| 300 |
+
stack.enter_context(patch.object(runtime, 'get_runtime', return_value=rt))
|
| 301 |
+
stack.enter_context(patch.object(odoo, 'get_odoo', return_value=GrowingSource()))
|
| 302 |
+
stack.enter_context(patch.object(rel, 'default_connection_id', return_value=None))
|
| 303 |
+
stack.enter_context(patch.object(rel, 'available', return_value=True))
|
| 304 |
+
stack.enter_context(patch.object(rel, 'enabled_buckets', return_value={'stock_quant'}))
|
| 305 |
+
stack.enter_context(patch.object(rel, 'plan', side_effect=plan))
|
| 306 |
+
stack.enter_context(patch.object(rel, 'apply_plan', side_effect=lambda *_a, **_k: {key: {'rows': 0}}))
|
| 307 |
+
stack.enter_context(patch.object(rel, '_canonical_mirror_ready', return_value=False))
|
| 308 |
+
stack.enter_context(patch.object(rel, 'sync_discovered_models', return_value={}))
|
| 309 |
+
stack.enter_context(patch.object(rel, '_refresh_declared_schema', return_value={}))
|
| 310 |
+
stack.enter_context(patch.object(rel, 'authoritative_schema_snapshot', return_value=({}, [])))
|
| 311 |
+
stack.enter_context(patch.object(ut, 'register_connected'))
|
| 312 |
+
stack.enter_context(patch.object(ut, 'record_odoo_schema_snapshot'))
|
| 313 |
+
stack.enter_context(patch.object(connector_onboarding, 'mirror_cursor', return_value=object()))
|
| 314 |
+
stack.enter_context(patch.object(routes_odoo_tables, 'population', return_value=mirror_rows))
|
| 315 |
+
stack.enter_context(patch.object(automation_engine, 'refresh_relations'))
|
| 316 |
+
stack.enter_context(patch.object(rollup_sql, 'compute'))
|
| 317 |
+
real_refresh, reports = rel.refresh, []
|
| 318 |
+
|
| 319 |
+
def refresh(*args, **kwargs):
|
| 320 |
+
reports.append((kwargs, real_refresh(*args, **kwargs)))
|
| 321 |
+
return reports[-1][1]
|
| 322 |
+
|
| 323 |
+
stack.enter_context(patch.object(rel, 'refresh', side_effect=refresh))
|
| 324 |
+
after_entity = namespace['_entity_source_counter'](fixture_ds) if capture else None
|
| 325 |
+
fixture_ds.sync_all(after_entity=after_entity) if hook else fixture_ds.sync_all()
|
| 326 |
+
counts = namespace['_capture_resync_source_counts'](rt.key) if capture else None
|
| 327 |
+
amazon('resync')
|
| 328 |
+
ok = namespace['_rebuild_odoo_relational']('resync', None, {}, tenant=rt.key,
|
| 329 |
+
cadence_seconds=86400, source_counts=counts)
|
| 330 |
+
kwargs, report = reports[-1] if reports else ({}, {})
|
| 331 |
+
failures = rel.blocking_failures(
|
| 332 |
+
connectors._odoo_result_database_failures(rt, None, rel.health_view(report))[0], report)
|
| 333 |
+
freshness = rel.sync_freshness(rt)
|
| 334 |
+
return SimpleNamespace(counts=counts, ok=ok, freshness=freshness, calls=source['calls'], rt=rt,
|
| 335 |
+
refresh_kwargs=kwargs, failures=failures, receipt=(report.get('databases') or {}).get(key) or {},
|
| 336 |
+
seconds=source['seconds'], source_count=source['count'], log=output.getvalue(),
|
| 337 |
+
remaining_counts=namespace['_PASS_SOURCE_COUNTS'])
|
| 338 |
+
|
| 339 |
+
|
| 340 |
class RankRefreshTests(unittest.TestCase):
|
| 341 |
+
def test_s85_t113_freshness_uses_the_pull_count_after_a_growing_source(self):
|
| 342 |
+
import odoo_relational as rel
|
| 343 |
+
fixed = observe_s85_t113_growing_source()
|
| 344 |
+
control = observe_s85_t113_growing_source(capture=False)
|
| 345 |
+
self.assertEqual(fixed.seconds, 2700)
|
| 346 |
+
self.assertEqual(fixed.source_count, 7775)
|
| 347 |
+
self.assertEqual(fixed.calls, [(0, [('location_id.usage', '=', 'internal')], 7706)])
|
| 348 |
+
self.assertEqual(fixed.counts, {'stock_quant': (7706, 'search_count')})
|
| 349 |
+
self.assertEqual(fixed.remaining_counts, {})
|
| 350 |
+
self.assertEqual(fixed.refresh_kwargs['source_counts'], fixed.counts)
|
| 351 |
+
self.assertEqual((fixed.receipt['sourceReconciliation'], fixed.receipt['sourceCount'],
|
| 352 |
+
fixed.receipt['mirrorCount'], fixed.receipt['sourceCountBasis']),
|
| 353 |
+
('matched', 7706, 7706, 'mirror_pull'))
|
| 354 |
+
self.assertTrue(fixed.ok)
|
| 355 |
+
self.assertFalse(fixed.freshness['stale'])
|
| 356 |
+
self.assertEqual(fixed.rt.get(rel._ut().ODOO_SYNC_STATUS_KEY)['phase'], 'ok')
|
| 357 |
+
self.assertEqual(control.calls, [(2700, [('location_id.usage', '=', 'internal')], 7775)])
|
| 358 |
+
self.assertEqual((control.receipt['sourceReconciliation'], control.receipt['sourceCountBasis']),
|
| 359 |
+
('mismatch', 'reconciliation'))
|
| 360 |
+
self.assertFalse(control.ok)
|
| 361 |
+
self.assertEqual(control.rt.get(rel._ut().ODOO_SYNC_STATUS_KEY)['phase'], 'error')
|
| 362 |
+
self.assertTrue(control.freshness['stale'])
|
| 363 |
+
self.assertEqual(control.failures[rel.STOCK_QUANT_KEY]['code'], 'reconciliation_incomplete')
|
| 364 |
+
|
| 365 |
+
def test_s85_t113_freshness_never_reuses_an_unfinished_pass_count_without_a_hook(self):
|
| 366 |
+
observed = observe_s85_t113_growing_source(hook=False,
|
| 367 |
+
prior_counts={'stock_quant': (1, 'search_count')})
|
| 368 |
+
self.assertTrue(observed.ok)
|
| 369 |
+
self.assertEqual(observed.counts, {'stock_quant': (7706, 'search_count')})
|
| 370 |
+
self.assertEqual(observed.calls, [(0, [('location_id.usage', '=', 'internal')], 7706)])
|
| 371 |
+
self.assertEqual(observed.remaining_counts, {})
|
| 372 |
+
|
| 373 |
def setUp(self):
|
| 374 |
self.stack = ExitStack()
|
| 375 |
self.addCleanup(self.stack.close)
|
|
|
|
| 1058 |
return datetime.fromisoformat(text.replace('Z', '+00:00')).timestamp()
|
| 1059 |
|
| 1060 |
def run_slot_loop(self, memory, *, boot, max_sleeps, tenant='royal-imports', config=None,
|
| 1061 |
+
reports=None, checkpoints=None, take=None, slot_flags=None, status=None,
|
| 1062 |
+
stop_at=None):
|
| 1063 |
"""Drive the real loop and its real slot resolver on a clock that advances at each wait."""
|
| 1064 |
import odoo_relational as rel
|
| 1065 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
|
|
|
| 1077 |
state = {}
|
| 1078 |
|
| 1079 |
def wait(seconds):
|
| 1080 |
+
self.assertLessEqual(seconds, 60.0, 'the Amazon check tick must remain at most 60 seconds')
|
| 1081 |
sleeps.append(seconds)
|
| 1082 |
+
if len(sleeps) >= max_sleeps or stop_at is not None and now[0] + seconds >= stop_at:
|
| 1083 |
raise StopIteration('bounded synthetic resident loop')
|
| 1084 |
now[0] += seconds
|
| 1085 |
return False
|
|
|
|
| 1149 |
last = self.utc('2026-10-23T06:05:00Z')
|
| 1150 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1151 |
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1152 |
+
boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=4000,
|
| 1153 |
+
stop_at=self.utc('2026-10-25T06:00:01Z'))
|
| 1154 |
slot, next_slot = self.utc('2026-10-24T06:00:00Z'), self.utc('2026-10-25T06:00:00Z')
|
| 1155 |
boot_work = [e for e in events if e[1] < slot]
|
| 1156 |
# Restore only at boot: the saved Amazon mirror, no mirror pass, no count, no store half.
|
| 1157 |
self.assertEqual(boot_work, [('amazon', self.utc('2026-10-24T02:30:00Z'), 'boot', False)])
|
| 1158 |
self.assertIn('2026-10-24T06:00:00Z (13:00 Asia/Jakarta)', self.loop_output)
|
| 1159 |
+
# Every real 60-second check runs before and between the two daily Odoo passes.
|
| 1160 |
+
self.assertEqual(set(sleeps), {60.0})
|
| 1161 |
+
self.assertGreater(len(sleeps), 1440)
|
| 1162 |
+
self.assertEqual([e[1] for e in odoo], [slot, slot, next_slot, next_slot])
|
|
|
|
|
|
|
| 1163 |
# The slot is stamped BEFORE the mirror pass reads Odoo, and the status names the next one.
|
| 1164 |
import odoo_relational as rel
|
| 1165 |
stamp = self.slot_store.docs[rel.CONFIG_KEY][rel.RESYNC_MEMORY_FIELD]['slotPassAt']
|
| 1166 |
+
self.assertEqual(rel._memory_epoch(stamp), next_slot)
|
| 1167 |
self.assertEqual(self.slot_state['nextPassAt'], '2026-10-26T06:00:00Z')
|
| 1168 |
|
| 1169 |
def test_h73_a_missed_slot_runs_once_at_boot_then_at_the_next_slot(self):
|
|
|
|
| 1172 |
boot = self.utc('2026-10-24T09:00:00Z')
|
| 1173 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1174 |
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1175 |
+
boot=boot, max_sleeps=4000, stop_at=self.utc('2026-10-25T06:00:01Z'))
|
| 1176 |
next_slot = self.utc('2026-10-25T06:00:00Z')
|
| 1177 |
mirror = [e[1] for e in odoo if e[0] == 'mirror']
|
| 1178 |
+
self.assertEqual(mirror, [boot, next_slot])
|
| 1179 |
+
self.assertEqual(set(sleeps), {60.0})
|
| 1180 |
self.assertNotIn(('amazon', boot, 'boot', False), events)
|
| 1181 |
|
| 1182 |
def test_h73_a_restart_inside_a_slot_pass_does_not_run_it_twice(self):
|
|
|
|
| 1184 |
import odoo_relational as rel
|
| 1185 |
stamp = self.utc('2026-10-24T06:00:01Z')
|
| 1186 |
yesterday = self.utc('2026-10-23T06:30:00Z')
|
|
|
|
| 1187 |
base = {'passAt': yesterday, 'fullRebuildAt': yesterday, 'slotPassAt': stamp,
|
| 1188 |
'fingerprints': {'res_partner': (3, 7)}}
|
| 1189 |
# Ended: `slotDoneAt` after the claim, or (older memory) `passAt` after it. No Odoo call.
|
|
|
|
| 1191 |
{'passAt': self.utc('2026-10-24T06:30:00Z')}):
|
| 1192 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1193 |
dict(base, **ended), boot=self.utc('2026-10-24T06:40:00Z'), max_sleeps=1)
|
| 1194 |
+
self.assertEqual((odoo, sleeps), ([], [60.0]))
|
| 1195 |
# Killed: the end marker predates the claim. One pass at boot, stamped start and end.
|
| 1196 |
boot = self.utc('2026-10-24T06:40:00Z')
|
| 1197 |
events, sleeps, odoo = self.run_slot_loop(
|
|
|
|
| 1217 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1218 |
dict(base, slotPassAt=boot, slotDoneAt=yesterday, slotResumedFor=slot_at,
|
| 1219 |
slotResumes=3), boot=fourth, max_sleeps=1)
|
| 1220 |
+
self.assertEqual((odoo, sleeps), ([], [60.0]))
|
| 1221 |
self.assertNotIn('never finished', self.loop_output)
|
| 1222 |
self.assertIn('slot_pass_interrupted: the 13:00 Asia/Jakarta slot pass was interrupted by '
|
| 1223 |
'4 restarts and waits for the next slot at 2026-10-25T06:00:00Z',
|
|
|
|
| 1247 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 1248 |
self.assertIn('the 13:00 Asia/Jakarta slot pass started at 2026-10-27T06:00:01Z never '
|
| 1249 |
'finished; running it again now (resume 1 of 3)', self.loop_output)
|
| 1250 |
+
self.assertEqual(sleeps, [60.0])
|
| 1251 |
|
| 1252 |
def test_h1_slot_resumes_are_bounded_and_reset_by_a_new_slot(self):
|
| 1253 |
"""H1: a killed slot pass resumes while its count is under 3, then waits; a new slot starts over"""
|
|
|
|
| 1282 |
last = self.utc('2026-10-23T06:05:00Z')
|
| 1283 |
events, _sleeps, _odoo = self.run_slot_loop(
|
| 1284 |
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1285 |
+
boot=self.utc('2026-10-24T05:59:30Z'), max_sleeps=2)
|
| 1286 |
slot = self.utc('2026-10-24T06:00:00Z')
|
| 1287 |
kinds = [e[0] for e in events if e[1] > slot]
|
| 1288 |
amazon = [i for i, e in enumerate(e for e in events if e[1] > slot)
|
|
|
|
| 1301 |
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1302 |
boot=boot, max_sleeps=1, tenant='acme')
|
| 1303 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 1304 |
+
self.assertEqual(sleeps, [60.0])
|
| 1305 |
self.assertNotIn('nextPassAt', self.slot_state)
|
| 1306 |
|
| 1307 |
def test_h73_manual_tenant0_has_no_slot(self):
|
|
|
|
| 1334 |
take = lambda **kwargs: held.append(tuple(kwargs.get('exclude_requested_by') or ())) or []
|
| 1335 |
_events, _sleeps, odoo = self.run_slot_loop(
|
| 1336 |
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1337 |
+
boot=self.utc('2026-10-24T04:30:00Z'), max_sleeps=100,
|
| 1338 |
+
stop_at=self.utc('2026-10-24T06:00:01Z'), take=take)
|
| 1339 |
self.assertTrue(odoo, 'the slot pass ran')
|
| 1340 |
# Boot 90 min before the slot: the drain outside the slot holds first load, and so does the claim.
|
| 1341 |
self.assertTrue(len(held) >= 2 and all(h == ('first load',) for h in held), held)
|
|
|
|
| 1361 |
self.assertTrue(held and all('first load' in exclusions for exclusions in held))
|
| 1362 |
self.assertEqual(job, {'state': 'running', 'rowsDone': 10})
|
| 1363 |
|
| 1364 |
+
def test_first_load_stays_with_its_actor_far_from_the_slot(self):
|
| 1365 |
+
"""The independent actor retains first-load work while the resident loop polls its desk."""
|
| 1366 |
last = self.utc('2026-10-23T06:05:00Z')
|
| 1367 |
held = []
|
| 1368 |
+
take = lambda **kwargs: held.append(kwargs) or []
|
| 1369 |
self.run_slot_loop({'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1370 |
boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=1, take=take)
|
| 1371 |
+
self.assertTrue(held)
|
| 1372 |
+
self.assertTrue(all('amazon_sp' in call['exclude_providers'] for call in held))
|
| 1373 |
+
self.assertTrue(all(call['exclude_requested_by'] == ('first load',) for call in held))
|
| 1374 |
|
| 1375 |
+
def test_the_slot_pass_preserves_each_amazon_tables_cadence(self):
|
| 1376 |
+
"""A slot pass retains Amazon's per-table cadence and the resident slot flag."""
|
| 1377 |
last = self.utc('2026-10-23T06:05:00Z')
|
| 1378 |
flags = []
|
| 1379 |
self.run_slot_loop({'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 1380 |
+
boot=self.utc('2026-10-24T05:59:30Z'), max_sleeps=2, slot_flags=flags)
|
| 1381 |
resync = [flag for args, flag in flags if args[:1] == ('resync',)]
|
| 1382 |
self.assertEqual(resync, [True])
|
| 1383 |
self.assertTrue(all(not flag for args, flag in flags if args[:1] != ('resync',)), flags)
|
|
|
|
| 1385 |
from datetime import datetime, timezone
|
| 1386 |
now = datetime(2026, 10, 29, 10, 16, tzinfo=timezone.utc)
|
| 1387 |
ticks = {'orders': {'state': 'done', 'updated_at': '2026-10-28T18:20:03+00:00'}}
|
| 1388 |
+
due = az.sync_plan(ticks, ['orders'], now=now)
|
| 1389 |
+
self.assertEqual(due['tables'], ['orders'])
|
| 1390 |
+
self.assertEqual(az.sync_plan(ticks, ['orders'], now=now, slot_pass=True), due)
|
| 1391 |
+
recent = {'orders': {'state': 'done', 'updated_at': now.isoformat()}}
|
| 1392 |
+
deferred = az.sync_plan(recent, ['orders'], now=now)
|
| 1393 |
+
self.assertIn('orders', deferred['deferred'])
|
| 1394 |
+
self.assertEqual(az.sync_plan(recent, ['orders'], now=now, slot_pass=True), deferred)
|
| 1395 |
|
| 1396 |
def test_slot_near_and_late(self):
|
| 1397 |
"""The quiet window remains two hours; the slot clock alone cannot make a pass late."""
|
|
|
|
| 1441 |
import odoo_relational as rel
|
| 1442 |
last = self.utc('2026-10-22T06:05:00Z')
|
| 1443 |
boot = self.utc('2026-10-24T07:00:00Z')
|
|
|
|
| 1444 |
memory = {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}
|
| 1445 |
held = {'account_move_line': {'phase': 'backfill', 'pulled': 0}}
|
| 1446 |
marker = 'account_move_line.cols.amount_residual'
|
|
|
|
| 1451 |
memory, boot=boot, max_sleeps=2, reports=[held, held],
|
| 1452 |
checkpoints=[before, after, after, after])
|
| 1453 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot, boot + 60])
|
| 1454 |
+
self.assertEqual(sleeps, [60.0, 60.0])
|
| 1455 |
self.assertEqual([e for e in events if e[0] in ('meta', 'amazon', 'rebuild')], [])
|
| 1456 |
stamps = rel.read_resync_memory(self.slot_store)
|
| 1457 |
self.assertEqual((stamps['slotPassAt'], stamps['slotDoneAt']), (boot, boot))
|
|
|
|
| 1462 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1463 |
memory, boot=boot, max_sleeps=1, reports=[held], checkpoints=[before, before])
|
| 1464 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 1465 |
+
self.assertEqual(sleeps, [60.0])
|
| 1466 |
self.assertNotIn('odoo backfill continues', self.loop_output)
|
| 1467 |
|
| 1468 |
# ── Hotfix 10-31: a good pass after the slot satisfies it, so a killed claim does not alarm ──
|
|
|
|
| 1517 |
memory, boot=boot, max_sleeps=1, config=self.stored_config(memory))
|
| 1518 |
# The loop reached the slot block: it named the next pass and waits for that slot.
|
| 1519 |
self.assertEqual(self.slot_state['nextPassAt'], '2026-11-01T06:00:00Z')
|
| 1520 |
+
self.assertEqual(sleeps, [60.0])
|
| 1521 |
self.assertEqual(odoo, [])
|
| 1522 |
self.assertNotIn('never finished', self.loop_output)
|
| 1523 |
self.assertNotIn('slot_pass_interrupted', self.loop_output)
|
api/tool_groups/__init__.py
CHANGED
|
@@ -197,7 +197,10 @@ def resolve(name, arguments):
|
|
| 197 |
}, "required": ["action"], "additionalProperties": False,
|
| 198 |
})
|
| 199 |
args = arguments.get("arguments", {})
|
| 200 |
-
|
|
|
|
|
|
|
|
|
|
| 201 |
except sheets.SheetRefusal as exc:
|
| 202 |
if exc.code != "tool_arguments_invalid" or entry is None:
|
| 203 |
raise
|
|
|
|
| 197 |
}, "required": ["action"], "additionalProperties": False,
|
| 198 |
})
|
| 199 |
args = arguments.get("arguments", {})
|
| 200 |
+
if entry.get("legacy_name") in {"query_database", "list_databases"}:
|
| 201 |
+
args = external_tools.table_read_arguments(entry["legacy_name"], args, entry["schema"])
|
| 202 |
+
else:
|
| 203 |
+
external_tools.validate_arguments(args, entry["schema"])
|
| 204 |
except sheets.SheetRefusal as exc:
|
| 205 |
if exc.code != "tool_arguments_invalid" or entry is None:
|
| 206 |
raise
|
api/tool_groups/data.py
CHANGED
|
@@ -45,11 +45,18 @@ READ_ARGUMENT_HELP = (
|
|
| 45 |
#: The one skill.md section every query refusal names.
|
| 46 |
QUERY_SYNTAX_SECTION = "Query syntax"
|
| 47 |
QUERY_FILTER_EXAMPLE = {"conj": "and", "nodes": [{"colId": "<field key>", "op": "eq", "value": "<value>"}]}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
QUERY_SYNTAX_FIX = (
|
| 49 |
"Filters use {conj, nodes: [{colId, op, value}]}, for example "
|
| 50 |
+ json.dumps(QUERY_FILTER_EXAMPLE, separators=(", ", ": ")) + ". "
|
| 51 |
"groupBy is a list of 1 to 4 field keys, or {field, unit: month}, with at least one aggregate and no sorts. "
|
| 52 |
-
|
| 53 |
"See " + QUERY_SYNTAX_SECTION + " in skill.md."
|
| 54 |
)
|
| 55 |
#: Query refusal codes that carry QUERY_SYNTAX_FIX.
|
|
@@ -115,9 +122,7 @@ def query_syntax_section():
|
|
| 115 |
"- aggregates: [{fn, field}], fn count, count_distinct, sum, avg, min or max, at most 16. "
|
| 116 |
"count may omit field. sum and avg need a numeric field (query_aggregate_type otherwise).",
|
| 117 |
"- groupBy: 1 to 4 field keys, or {field, unit: month} for a date field. A grouped query needs at least "
|
| 118 |
-
"one aggregate and no sorts (query_groups_invalid otherwise).
|
| 119 |
-
"Odoo sales lines mirror, groups. Any other database answers 409 query_groups_unavailable: read its "
|
| 120 |
-
"records by page, or ask for aggregates without groupBy.",
|
| 121 |
"- Paging: default 100. " + QUERY_PAGING_HELP
|
| 122 |
+ " Only fields, filters, sorts, limit, cursor, aggregates and groupBy are "
|
| 123 |
"accepted (query_arguments_invalid otherwise).",
|
|
@@ -158,8 +163,8 @@ def query_guide(rows):
|
|
| 158 |
"Only use operators declared for that field. Combine date bounds in an AND group's nodes.", "",
|
| 159 |
"## Grouping and paging", "",
|
| 160 |
"Use groupBy with 1 to 4 readable field keys. For calendar months, use an object with "
|
| 161 |
-
"field set to a readable date field's key and unit set to month.
|
| 162 |
-
"at least one aggregate and no sorts.
|
| 163 |
"The top-level aggregates cover the whole permitted, filtered result, never one page. "
|
| 164 |
"Each groups entry has keys and aggregates for that entire group. "
|
| 165 |
"page.returned counts groups on the current page. records is empty for grouped queries. "
|
|
|
|
| 45 |
#: The one skill.md section every query refusal names.
|
| 46 |
QUERY_SYNTAX_SECTION = "Query syntax"
|
| 47 |
QUERY_FILTER_EXAMPLE = {"conj": "and", "nodes": [{"colId": "<field key>", "op": "eq", "value": "<value>"}]}
|
| 48 |
+
QUERY_GROUPING_HELP = (
|
| 49 |
+
"Connector mirrors group up to four fields and support month units. Their groups arrive in key order. "
|
| 50 |
+
"A stored Table groups one field without a month unit, using the Table grid's aggregate reader. "
|
| 51 |
+
"Its group order is count descending, then key ascending. "
|
| 52 |
+
"A query_groups_unavailable refusal names its cause. Call table with action query_database "
|
| 53 |
+
"using error.retry_query, without groupBy or aggregates, then follow page.next_cursor."
|
| 54 |
+
)
|
| 55 |
QUERY_SYNTAX_FIX = (
|
| 56 |
"Filters use {conj, nodes: [{colId, op, value}]}, for example "
|
| 57 |
+ json.dumps(QUERY_FILTER_EXAMPLE, separators=(", ", ": ")) + ". "
|
| 58 |
"groupBy is a list of 1 to 4 field keys, or {field, unit: month}, with at least one aggregate and no sorts. "
|
| 59 |
+
+ QUERY_GROUPING_HELP + " "
|
| 60 |
"See " + QUERY_SYNTAX_SECTION + " in skill.md."
|
| 61 |
)
|
| 62 |
#: Query refusal codes that carry QUERY_SYNTAX_FIX.
|
|
|
|
| 122 |
"- aggregates: [{fn, field}], fn count, count_distinct, sum, avg, min or max, at most 16. "
|
| 123 |
"count may omit field. sum and avg need a numeric field (query_aggregate_type otherwise).",
|
| 124 |
"- groupBy: 1 to 4 field keys, or {field, unit: month} for a date field. A grouped query needs at least "
|
| 125 |
+
"one aggregate and no sorts (query_groups_invalid otherwise). " + QUERY_GROUPING_HELP,
|
|
|
|
|
|
|
| 126 |
"- Paging: default 100. " + QUERY_PAGING_HELP
|
| 127 |
+ " Only fields, filters, sorts, limit, cursor, aggregates and groupBy are "
|
| 128 |
"accepted (query_arguments_invalid otherwise).",
|
|
|
|
| 163 |
"Only use operators declared for that field. Combine date bounds in an AND group's nodes.", "",
|
| 164 |
"## Grouping and paging", "",
|
| 165 |
"Use groupBy with 1 to 4 readable field keys. For calendar months, use an object with "
|
| 166 |
+
"field set to a readable date field's key and unit set to month. "
|
| 167 |
+
"Grouping needs at least one aggregate and no sorts. " + QUERY_GROUPING_HELP,
|
| 168 |
"The top-level aggregates cover the whole permitted, filtered result, never one page. "
|
| 169 |
"Each groups entry has keys and aggregates for that entire group. "
|
| 170 |
"page.returned counts groups on the current page. records is empty for grouped queries. "
|
api/tool_groups/metrics.py
CHANGED
|
@@ -64,8 +64,12 @@ class _LimitRefused(_Unavailable):
|
|
| 64 |
LAST_VALIDATED_KEY = "metric_last_validated"
|
| 65 |
#: The newest entries kept. An evicted key is refused until it validates again (rule 8).
|
| 66 |
LAST_VALIDATED_ENTRIES = 2000
|
| 67 |
-
_UNAVAILABLE = "This metric has no validated value within your current access and window."
|
| 68 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
_WARNINGS_LIMITED = "Warnings were not checked because the live call limit is reached."
|
| 70 |
|
| 71 |
|
|
@@ -186,7 +190,7 @@ def _window(window):
|
|
| 186 |
or end.day != calendar.monthrange(start.year, start.month)[1]):
|
| 187 |
raise ValueError
|
| 188 |
except ValueError:
|
| 189 |
-
raise deps.err(400, "metric_window_invalid",
|
| 190 |
return copy.deepcopy(window)
|
| 191 |
|
| 192 |
|
|
|
|
| 64 |
LAST_VALIDATED_KEY = "metric_last_validated"
|
| 65 |
#: The newest entries kept. An evicted key is refused until it validates again (rule 8).
|
| 66 |
LAST_VALIDATED_ENTRIES = 2000
|
| 67 |
+
_UNAVAILABLE = ("This metric has no validated value within your current access and window. "
|
| 68 |
+
"Ask an Administrator to check source availability and metric access, or choose another window.")
|
| 69 |
+
_LIMITED = ("This window has no validated value yet, and the live call limit is reached. "
|
| 70 |
+
"Retry after the live call limit resets.")
|
| 71 |
+
_WINDOW_INVALID = ("The dates do not form a valid named window. Use real ISO dates in order, "
|
| 72 |
+
"and a complete calendar month for calendar_month.")
|
| 73 |
_WARNINGS_LIMITED = "Warnings were not checked because the live call limit is reached."
|
| 74 |
|
| 75 |
|
|
|
|
| 190 |
or end.day != calendar.monthrange(start.year, start.month)[1]):
|
| 191 |
raise ValueError
|
| 192 |
except ValueError:
|
| 193 |
+
raise deps.err(400, "metric_window_invalid", _WINDOW_INVALID) from None
|
| 194 |
return copy.deepcopy(window)
|
| 195 |
|
| 196 |
|
api/usage_activity.py
ADDED
|
@@ -0,0 +1,491 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Join tenant activity counts without changing authored state or hiding source failures."""
|
| 2 |
+
|
| 3 |
+
from calendar import monthrange
|
| 4 |
+
from collections import Counter
|
| 5 |
+
from datetime import date, datetime, time, timedelta, timezone
|
| 6 |
+
import base64
|
| 7 |
+
import binascii
|
| 8 |
+
import hashlib
|
| 9 |
+
import hmac
|
| 10 |
+
import json
|
| 11 |
+
import math
|
| 12 |
+
|
| 13 |
+
from core import activity_log, audit, history_rows, user_tables, view_audit
|
| 14 |
+
import turn_audit
|
| 15 |
+
|
| 16 |
+
KINDS = ("mcp", "api", "sign_in", "record_edit", "field_edit", "cell_edit", "view_edit", "ai_turn")
|
| 17 |
+
SOURCE_KINDS = {
|
| 18 |
+
"security_events": ("mcp", "api", "sign_in"),
|
| 19 |
+
"activity_events": ("record_edit", "field_edit"),
|
| 20 |
+
"value_history": ("cell_edit",),
|
| 21 |
+
"view_audit": ("view_edit",),
|
| 22 |
+
"turn_audit": ("ai_turn",),
|
| 23 |
+
}
|
| 24 |
+
PAGE = 200
|
| 25 |
+
UTC = timezone.utc
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _now():
|
| 29 |
+
return datetime.now(UTC)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _timestamp(value):
|
| 33 |
+
if isinstance(value, (float, int)) and not isinstance(value, bool) and math.isfinite(value):
|
| 34 |
+
return datetime.fromtimestamp(value, UTC)
|
| 35 |
+
if isinstance(value, str):
|
| 36 |
+
value = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
| 37 |
+
if not isinstance(value, datetime) or value.tzinfo is None:
|
| 38 |
+
raise ValueError("Activity timestamps must include a timezone.")
|
| 39 |
+
return value.astimezone(UTC)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _months_before(day, months):
|
| 43 |
+
year, month0 = divmod(day.year * 12 + day.month - 1 - months, 12)
|
| 44 |
+
month = month0 + 1
|
| 45 |
+
return day.replace(year=year, month=month, day=min(day.day, monthrange(year, month)[1]))
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def date_range(from_=None, to=None, *, now=None):
|
| 49 |
+
"""Return an inclusive UTC date range within the last thirteen calendar months."""
|
| 50 |
+
now = _timestamp(now) if now is not None else _now()
|
| 51 |
+
try:
|
| 52 |
+
end = date.fromisoformat(to) if to else now.date()
|
| 53 |
+
start = date.fromisoformat(from_) if from_ else end - timedelta(days=29)
|
| 54 |
+
except (TypeError, ValueError):
|
| 55 |
+
raise ValueError("Use dates in YYYY-MM-DD format.") from None
|
| 56 |
+
if start > end:
|
| 57 |
+
raise ValueError("From must be on or before To.")
|
| 58 |
+
if start < _months_before(now.date(), 13) or end > now.date():
|
| 59 |
+
raise ValueError("Choose dates within the last 13 months.")
|
| 60 |
+
return start, end, now
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _unlimited_prefix(reader):
|
| 64 |
+
"""Expand readers without a cursor until a short result proves exhaustion."""
|
| 65 |
+
limit = PAGE
|
| 66 |
+
while True:
|
| 67 |
+
rows = reader(limit)
|
| 68 |
+
if len(rows) < limit:
|
| 69 |
+
return rows
|
| 70 |
+
limit *= 2
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _tenant_row(row, tenant):
|
| 74 |
+
# Schema-bound sources can omit a stamp, but an explicit foreign stamp is never admitted.
|
| 75 |
+
return row.get("tenant", tenant) == tenant
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def _security_record_mode(runtime):
|
| 79 |
+
"""Use the canonical read-only layout decision for bound PostgreSQL security events."""
|
| 80 |
+
from core import account_rows
|
| 81 |
+
backend = account_rows.records_backend(runtime)
|
| 82 |
+
if not backend:
|
| 83 |
+
return False
|
| 84 |
+
pg, slug = backend
|
| 85 |
+
with account_rows.read_snapshot(slug) as con:
|
| 86 |
+
return account_rows._record_mode(con, pg, slug, account_rows.EVENTS_KEY)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _security(session, start, stop, now, source):
|
| 90 |
+
rows = _unlimited_prefix(lambda limit: audit.read(
|
| 91 |
+
session.tenant, limit=limit, now=now.timestamp(),
|
| 92 |
+
kinds={"external.query", "external.write", "auth.login"}, viewer=session.user))
|
| 93 |
+
source["retention"] = {"days": 400, "legacyRowsPerPartition": audit.MAX_EVENTS}
|
| 94 |
+
if not _security_record_mode(session.runtime):
|
| 95 |
+
stored = (session.runtime.get(audit.BUCKET) or {}).get("events", [])
|
| 96 |
+
for external in (True, False):
|
| 97 |
+
partition = [row for row in stored if _tenant_row(row, session.tenant)
|
| 98 |
+
and (row.get("kind") in audit.EXTERNAL_KINDS) == external]
|
| 99 |
+
if len(partition) >= audit.MAX_EVENTS:
|
| 100 |
+
source["limitations"].append({"code": "legacy_ring", "cause":
|
| 101 |
+
"Legacy security documents retain at most 2000 events per partition."})
|
| 102 |
+
held = min(_timestamp(row["ts"]) for row in partition).isoformat()
|
| 103 |
+
source["heldFrom"] = max(source["heldFrom"], held)
|
| 104 |
+
for row in rows:
|
| 105 |
+
if not _tenant_row(row, session.tenant):
|
| 106 |
+
continue
|
| 107 |
+
kind = "sign_in" if row["kind"] == "auth.login" else (
|
| 108 |
+
"mcp" if row.get("detail", {}).get("via") == "mcp" else "api")
|
| 109 |
+
yield _timestamp(row["ts"]), row.get("principal") or "", kind
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def _activity(session, start, stop, now, source):
|
| 113 |
+
cursor = None
|
| 114 |
+
source["retention"] = {"months": activity_log.RETENTION_MONTHS}
|
| 115 |
+
with activity_log.read_scope(session.runtime):
|
| 116 |
+
while True:
|
| 117 |
+
page = activity_log.read(session.runtime, can_read_object=lambda obj: True,
|
| 118 |
+
since=start, until=min(stop, now), cursor=cursor, limit=PAGE)
|
| 119 |
+
for row in page["records"]:
|
| 120 |
+
if _tenant_row(row, session.tenant) and row["kind"] in {"record", "field"}:
|
| 121 |
+
yield _timestamp(row["occurred_at"]), row.get("by") or "", row["kind"] + "_edit"
|
| 122 |
+
next_cursor = page["next_cursor"]
|
| 123 |
+
if next_cursor is None:
|
| 124 |
+
break
|
| 125 |
+
if next_cursor == cursor:
|
| 126 |
+
raise RuntimeError("Activity cursor did not advance.")
|
| 127 |
+
cursor = next_cursor
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _history_tables(rt, now):
|
| 131 |
+
"""Include retained history for removed tables without reading their former record values."""
|
| 132 |
+
tables = set(user_tables.all_defs(rt, schema_only=True))
|
| 133 |
+
backend = history_rows.backend(rt)
|
| 134 |
+
since = now - timedelta(days=history_rows.RETENTION_DAYS)
|
| 135 |
+
if isinstance(backend, history_rows.PgHistoryStore):
|
| 136 |
+
try:
|
| 137 |
+
with backend._connection() as con:
|
| 138 |
+
cursor = con.execute(f'SELECT DISTINCT "table" FROM {backend.table} WHERE at >= %s', (since,))
|
| 139 |
+
while rows := cursor.fetchmany(PAGE):
|
| 140 |
+
tables.update(row[0] for row in rows)
|
| 141 |
+
except Exception as exc:
|
| 142 |
+
if getattr(exc, "sqlstate", None) != "42P01":
|
| 143 |
+
raise
|
| 144 |
+
elif callable(getattr(backend, "table_keys", None)):
|
| 145 |
+
tables.update(backend.table_keys(since=since))
|
| 146 |
+
return sorted(tables)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def _cells(session, start, stop, now, source):
|
| 150 |
+
source["retention"] = {"days": history_rows.RETENTION_DAYS}
|
| 151 |
+
for table in _history_tables(session.runtime, now):
|
| 152 |
+
if table == activity_log.DATABASE_KEY:
|
| 153 |
+
continue
|
| 154 |
+
history = user_tables.value_history(table, st=session.runtime,
|
| 155 |
+
retention_days=history_rows.RETENTION_DAYS, extended_entitled=True)
|
| 156 |
+
for row in history["events"]:
|
| 157 |
+
if _tenant_row(row, session.tenant) and row.get("kind") in {"edit", "restore"}:
|
| 158 |
+
for _ in row.get("changes", {}):
|
| 159 |
+
yield _timestamp(row["at"]), row.get("actor") or "", "cell_edit"
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def _views(session, start, stop, now, source):
|
| 163 |
+
source["retention"] = {"days": None, "documentRowsPerTable": view_audit.RING_CAP}
|
| 164 |
+
from core.view_templates import _WS_KEYS
|
| 165 |
+
keys = set(_WS_KEYS.values()) | {key for key in session.runtime.list_keys("ut_")
|
| 166 |
+
if key.endswith("_table_workspace")}
|
| 167 |
+
for table in sorted(keys):
|
| 168 |
+
home = view_audit.home(session.runtime, table)
|
| 169 |
+
data = None
|
| 170 |
+
if home == view_audit.DOCUMENT:
|
| 171 |
+
# Read outside rows(), whose legacy document fallback swallows store failures.
|
| 172 |
+
data = session.runtime.get(table) or {}
|
| 173 |
+
ring = data.get(view_audit.AUDIT_KEY) or []
|
| 174 |
+
if len(ring) >= view_audit.RING_CAP:
|
| 175 |
+
held = min(_timestamp(row["ts"]) for row in ring).isoformat()
|
| 176 |
+
source["heldFrom"] = max(source["heldFrom"], held)
|
| 177 |
+
source["limitations"].append({"code": "view_ring", "cause":
|
| 178 |
+
"A view document retains only its newest 2000 audit rows."})
|
| 179 |
+
rows = _unlimited_prefix(lambda limit: view_audit.rows(session.runtime,
|
| 180 |
+
table_key=table, since=math.nextafter(start.timestamp(), -math.inf), limit=limit, data=data))
|
| 181 |
+
for row in rows:
|
| 182 |
+
if _tenant_row(row, session.tenant):
|
| 183 |
+
yield _timestamp(row["ts"]), row.get("actor") or "", "view_edit"
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _turns(session, start, stop, now, source):
|
| 187 |
+
source["retention"] = {"days": None, "linesPerWeek": turn_audit.MAX_LINES,
|
| 188 |
+
"bytesPerWeek": turn_audit.MAX_BYTES}
|
| 189 |
+
monday = start.date() - timedelta(days=start.weekday())
|
| 190 |
+
while monday <= min(stop, now).date():
|
| 191 |
+
year, week, _ = monday.isocalendar()
|
| 192 |
+
week = f"{year}-W{week:02d}"
|
| 193 |
+
offset = 0
|
| 194 |
+
capped, oldest = None, None
|
| 195 |
+
while True:
|
| 196 |
+
page = turn_audit.read(session.runtime, session.uname, tenant_wide=True,
|
| 197 |
+
week=week, offset=offset, limit=PAGE)
|
| 198 |
+
if offset == 0 and page["overflow"]:
|
| 199 |
+
capped = {"code": "turn_week_cap", "week": week,
|
| 200 |
+
"lost": page["overflow"], "limit": turn_audit.MAX_LINES,
|
| 201 |
+
"cause": "Weekly turn documents evict oldest rows above the line or byte limit."}
|
| 202 |
+
source["limitations"].append(capped)
|
| 203 |
+
for row in page["lines"]:
|
| 204 |
+
if _tenant_row(row, session.tenant):
|
| 205 |
+
stamp = _timestamp(row["ts"])
|
| 206 |
+
oldest = min(oldest, stamp) if oldest is not None else stamp
|
| 207 |
+
yield stamp, row.get("user") or "", "ai_turn"
|
| 208 |
+
if not page["hasMore"]:
|
| 209 |
+
break
|
| 210 |
+
if not page["lines"]:
|
| 211 |
+
raise RuntimeError("Turn cursor did not advance.")
|
| 212 |
+
offset += len(page["lines"])
|
| 213 |
+
if capped is not None:
|
| 214 |
+
held = oldest or datetime.combine(monday + timedelta(days=7), time.min, UTC)
|
| 215 |
+
capped["heldFrom"] = held.isoformat()
|
| 216 |
+
source["heldFrom"] = max(source["heldFrom"], held.isoformat())
|
| 217 |
+
monday += timedelta(days=7)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
READERS = {"security_events": _security, "activity_events": _activity,
|
| 221 |
+
"value_history": _cells, "view_audit": _views, "turn_audit": _turns}
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def _source_error(exc):
|
| 225 |
+
# Exception text can hold database addresses, credentials or customer values.
|
| 226 |
+
from core.tenant_keys import TenantKeyLocked
|
| 227 |
+
if isinstance(exc, TenantKeyLocked):
|
| 228 |
+
return {"code": "tenant_data_key_unavailable", "type": "TenantKeyLocked",
|
| 229 |
+
"message": "The tenant data key could not be read. Retry after restoring its configured backend."}
|
| 230 |
+
return {"code": "source_unreadable", "type": type(exc).__name__,
|
| 231 |
+
"message": "This activity source could not be read. Retry."}
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
class McpCursorInvalid(ValueError):
|
| 235 |
+
"""The cursor does not belong to this caller and query, or has an invalid shape."""
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
class McpCursorStale(ValueError):
|
| 239 |
+
"""The retained selection changed since the first page and requires a fresh read."""
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def _mcp_cursor(session, query, *, position=None, token=None):
|
| 243 |
+
"""Sign pagination state against the current principal, tenant, permissions and query."""
|
| 244 |
+
import aios_session
|
| 245 |
+
# Cookie refresh changes the idle deadline on each GET, but preserves the authenticated session.
|
| 246 |
+
claims = {key: value for key, value in (getattr(session, "claims", None) or {}).items() if key != "i"}
|
| 247 |
+
binding = json.dumps([session.tenant, session.uname, claims,
|
| 248 |
+
session.user, query], sort_keys=True, default=str)
|
| 249 |
+
|
| 250 |
+
def proof(value):
|
| 251 |
+
body = json.dumps(value, sort_keys=True, separators=(",", ":"))
|
| 252 |
+
return aios_session._sign(("usage-mcp:v1:" + binding + body).encode())
|
| 253 |
+
|
| 254 |
+
if token is not None:
|
| 255 |
+
try:
|
| 256 |
+
if not isinstance(token, str) or not 1 <= len(token) <= 4096:
|
| 257 |
+
raise ValueError()
|
| 258 |
+
raw = base64.b64decode(token + "=" * (-len(token) % 4), altchars=b"-_", validate=True)
|
| 259 |
+
value, signature = json.loads(raw)
|
| 260 |
+
if not isinstance(signature, str) or not hmac.compare_digest(signature, proof(value)):
|
| 261 |
+
raise ValueError()
|
| 262 |
+
if (not isinstance(value, dict)
|
| 263 |
+
or set(value) != {"asOf", "snapshot", "offset", "total"}
|
| 264 |
+
or not isinstance(value["snapshot"], str) or len(value["snapshot"]) != 64
|
| 265 |
+
or any(char not in "0123456789abcdef" for char in value["snapshot"])
|
| 266 |
+
or type(value["offset"]) is not int or type(value["total"]) is not int
|
| 267 |
+
or not 0 < value["offset"] < value["total"]
|
| 268 |
+
or not isinstance(value["asOf"], str)):
|
| 269 |
+
raise ValueError()
|
| 270 |
+
_timestamp(value["asOf"])
|
| 271 |
+
return value
|
| 272 |
+
except (ValueError, TypeError, UnicodeError, binascii.Error, OverflowError, RecursionError):
|
| 273 |
+
raise McpCursorInvalid() from None
|
| 274 |
+
raw = json.dumps([position, proof(position)], separators=(",", ":")).encode()
|
| 275 |
+
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _mcp_events(session, now, source):
|
| 279 |
+
"""Exhaust canonical audit reads without changing the source's retention policy."""
|
| 280 |
+
source["retention"] = {"days": audit.RETENTION_SECONDS // 86400,
|
| 281 |
+
"legacyRowsPerPartition": audit.MAX_EVENTS}
|
| 282 |
+
rows = _unlimited_prefix(lambda limit: audit.read(session.tenant, limit=limit,
|
| 283 |
+
now=now.timestamp(), kinds=audit.EXTERNAL_KINDS, viewer=session.user))
|
| 284 |
+
if not _security_record_mode(session.runtime):
|
| 285 |
+
stored = (session.runtime.get(audit.BUCKET) or {}).get("events", [])
|
| 286 |
+
partition = [row for row in stored if _tenant_row(row, session.tenant)
|
| 287 |
+
and row.get("kind") in audit.EXTERNAL_KINDS]
|
| 288 |
+
if len(partition) >= audit.MAX_EVENTS:
|
| 289 |
+
source["limitations"].append({"code": "legacy_ring", "cause":
|
| 290 |
+
"Legacy security documents retain at most 2000 events per partition."})
|
| 291 |
+
held = min(_timestamp(row["ts"]) for row in partition).isoformat()
|
| 292 |
+
source["heldFrom"] = max(source["heldFrom"], held)
|
| 293 |
+
return rows
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def _mcp_text(value):
|
| 297 |
+
"""Expose only bounded metadata text with the canonical credential redaction."""
|
| 298 |
+
if not isinstance(value, str) or not value.strip():
|
| 299 |
+
return "unknown"
|
| 300 |
+
clean = audit.redact_intent(value)
|
| 301 |
+
return "unknown" if clean == "[invalid]" else clean
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def _mcp_metadata(row):
|
| 305 |
+
"""Project the explicit metadata allowlist and leave absent historical fields unknown."""
|
| 306 |
+
detail = row.get("detail") or {}
|
| 307 |
+
return {"tool": _mcp_text(detail.get("tool")), "action": _mcp_text(detail.get("action")),
|
| 308 |
+
"outcome": _mcp_text(detail.get("outcome")), "code": _mcp_text(detail.get("code")),
|
| 309 |
+
"time": _timestamp(row["ts"]).isoformat(), "user": _mcp_text(row.get("principal")),
|
| 310 |
+
"database": _mcp_text(detail.get("database")), "keyId": _mcp_text(detail.get("key_id")),
|
| 311 |
+
"keyLabel": _mcp_text(detail.get("key_label"))}
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def read_mcp(session, *, from_=None, to=None, user=None, tool=None, cursor=None, limit=100, now=None):
|
| 315 |
+
"""Return full selected MCP totals and a page pinned to a signed, ephemeral source fingerprint."""
|
| 316 |
+
if not session.admin:
|
| 317 |
+
raise PermissionError("Usage activity requires a tenant administrator.")
|
| 318 |
+
if not session.tenant or view_audit.tenant_of(session.runtime) != session.tenant:
|
| 319 |
+
raise PermissionError("The usage runtime must belong to the session tenant.")
|
| 320 |
+
if type(limit) is not int or not 1 <= limit <= PAGE:
|
| 321 |
+
raise ValueError("Limit must be 1 to 200.")
|
| 322 |
+
user = str(user).strip() if user else None
|
| 323 |
+
if user and (len(user) > 256 or "\x00" in user):
|
| 324 |
+
raise ValueError("Choose a valid user.")
|
| 325 |
+
tool = str(tool).strip() if tool else None
|
| 326 |
+
if tool and (len(tool) > 500 or "\x00" in tool):
|
| 327 |
+
raise ValueError("Choose a valid tool.")
|
| 328 |
+
query = {"from": from_ or None, "to": to or None, "user": user, "tool": tool, "limit": limit}
|
| 329 |
+
position = _mcp_cursor(session, query, token=cursor) if cursor is not None else None
|
| 330 |
+
now = _timestamp(now) if now is not None else _now()
|
| 331 |
+
as_of = _timestamp(position["asOf"]) if position else now
|
| 332 |
+
if as_of > now:
|
| 333 |
+
raise McpCursorInvalid()
|
| 334 |
+
first, last, _ = date_range(from_, to, now=as_of)
|
| 335 |
+
start = datetime.combine(first, time.min, UTC)
|
| 336 |
+
stop = datetime.combine(last + timedelta(days=1), time.min, UTC)
|
| 337 |
+
floor = now - timedelta(seconds=audit.RETENTION_SECONDS)
|
| 338 |
+
source = {"source": "security_events", "kinds": ["mcp"], "heldFrom": floor.isoformat(),
|
| 339 |
+
"retention": {"days": audit.RETENTION_SECONDS // 86400,
|
| 340 |
+
"legacyRowsPerPartition": audit.MAX_EVENTS},
|
| 341 |
+
"status": "ok", "error": None, "limitations": [], "total": None}
|
| 342 |
+
if floor > start:
|
| 343 |
+
source["limitations"].append({"code": "retention", "cause":
|
| 344 |
+
"This source no longer holds the beginning of the requested range."})
|
| 345 |
+
report = {"tenant": session.tenant, "from": first.isoformat(), "to": last.isoformat(),
|
| 346 |
+
"timezone": "UTC", "filters": {"user": user, "tool": tool}, "asOf": as_of.isoformat(),
|
| 347 |
+
"limit": limit, "complete": False, "total": None, "byTool": [], "events": [],
|
| 348 |
+
"nextCursor": None, "source": source}
|
| 349 |
+
try:
|
| 350 |
+
rows = _mcp_events(session, now, source)
|
| 351 |
+
selected = []
|
| 352 |
+
for row in rows:
|
| 353 |
+
if (not _tenant_row(row, session.tenant) or row.get("kind") not in audit.EXTERNAL_KINDS
|
| 354 |
+
or row.get("detail", {}).get("via") != "mcp"):
|
| 355 |
+
continue
|
| 356 |
+
at = _timestamp(row["ts"])
|
| 357 |
+
if not (start <= at < stop and at <= as_of) or (user is not None and row.get("principal") != user):
|
| 358 |
+
continue
|
| 359 |
+
metadata = _mcp_metadata(row)
|
| 360 |
+
if tool is None or metadata["tool"] == tool:
|
| 361 |
+
selected.append(metadata)
|
| 362 |
+
except Exception as exc:
|
| 363 |
+
source.update(status="error", error=_source_error(exc))
|
| 364 |
+
return report
|
| 365 |
+
|
| 366 |
+
import aios_session
|
| 367 |
+
counts, occurrences = Counter(), Counter()
|
| 368 |
+
events = []
|
| 369 |
+
for metadata in selected:
|
| 370 |
+
body = json.dumps([session.tenant, metadata], sort_keys=True, separators=(",", ":"))
|
| 371 |
+
identity = aios_session._sign(("usage-mcp-event:v1:" + body).encode())
|
| 372 |
+
occurrences[identity] += 1
|
| 373 |
+
events.append({"id": f"{identity}.{occurrences[identity]}", **metadata})
|
| 374 |
+
counts[(metadata["tool"], metadata["outcome"])] += 1
|
| 375 |
+
# Identical events retain distinct occurrence ids, so pagination never deduplicates source rows.
|
| 376 |
+
events.sort(key=lambda row: (row["time"], row["id"]), reverse=True)
|
| 377 |
+
snapshot = hashlib.sha256(json.dumps([row["id"] for row in events], separators=(",", ":")).encode()).hexdigest()
|
| 378 |
+
if position and (position["snapshot"] != snapshot or position["total"] != len(events)):
|
| 379 |
+
raise McpCursorStale()
|
| 380 |
+
offset = position["offset"] if position else 0
|
| 381 |
+
following = offset + limit
|
| 382 |
+
if following < len(events):
|
| 383 |
+
report["nextCursor"] = _mcp_cursor(session, query, position={
|
| 384 |
+
"asOf": as_of.isoformat(), "snapshot": snapshot, "offset": following, "total": len(events)})
|
| 385 |
+
source.update(total=len(events), status="partial" if source["limitations"] else "ok")
|
| 386 |
+
report.update(complete=source["status"] == "ok", total=len(events), events=events[offset:following],
|
| 387 |
+
byTool=[{"tool": name, "total": sum(n for (t, _), n in counts.items() if t == name),
|
| 388 |
+
"byOutcome": {outcome: counts[(name, outcome)] for t, outcome in sorted(counts) if t == name}}
|
| 389 |
+
for name in sorted({t for t, _ in counts})])
|
| 390 |
+
return report
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
def read(session, *, from_=None, to=None, user=None, kind=None, now=None):
|
| 394 |
+
"""Return a count cube and separate retention and error metadata for every selected source."""
|
| 395 |
+
if not session.admin:
|
| 396 |
+
raise PermissionError("Usage activity requires a tenant administrator.")
|
| 397 |
+
if not session.tenant or view_audit.tenant_of(session.runtime) != session.tenant:
|
| 398 |
+
raise PermissionError("The usage runtime must belong to the session tenant.")
|
| 399 |
+
first, last, now = date_range(from_, to, now=now)
|
| 400 |
+
start = datetime.combine(first, time.min, UTC)
|
| 401 |
+
stop = datetime.combine(last + timedelta(days=1), time.min, UTC)
|
| 402 |
+
wanted = str(kind or "").strip().lower()
|
| 403 |
+
if wanted and wanted not in KINDS:
|
| 404 |
+
raise ValueError("Kind must be one of: " + ", ".join(KINDS) + ".")
|
| 405 |
+
user = str(user).strip() if user else None
|
| 406 |
+
if user and (len(user) > 256 or "\x00" in user):
|
| 407 |
+
raise ValueError("Choose a valid user.")
|
| 408 |
+
selected = (wanted,) if wanted else KINDS
|
| 409 |
+
floors = {"security_events": now - timedelta(seconds=audit.RETENTION_SECONDS),
|
| 410 |
+
"activity_events": activity_log.retention_cutoff(now),
|
| 411 |
+
"value_history": now - timedelta(days=history_rows.RETENTION_DAYS),
|
| 412 |
+
"view_audit": start, "turn_audit": start}
|
| 413 |
+
sources, counts = [], Counter()
|
| 414 |
+
totals = {key: 0 for key in selected}
|
| 415 |
+
for name, reader in READERS.items():
|
| 416 |
+
kinds = [key for key in SOURCE_KINDS[name] if key in selected]
|
| 417 |
+
if not kinds:
|
| 418 |
+
continue
|
| 419 |
+
source = {"source": name, "kinds": kinds, "heldFrom": floors[name].isoformat(),
|
| 420 |
+
"status": "ok", "error": None, "limitations": [], "total": None}
|
| 421 |
+
if floors[name] > start:
|
| 422 |
+
source["limitations"].append({"code": "retention", "cause":
|
| 423 |
+
"This source no longer holds the beginning of the requested range."})
|
| 424 |
+
local = Counter()
|
| 425 |
+
try:
|
| 426 |
+
for at, actor, event_kind in reader(session, start, stop, now, source):
|
| 427 |
+
if (start <= at < stop and at <= now and event_kind in kinds
|
| 428 |
+
and (user is None or actor == user)):
|
| 429 |
+
local[(at.date().isoformat(), actor, event_kind)] += 1
|
| 430 |
+
except Exception as exc:
|
| 431 |
+
source.update(status="error", error=_source_error(exc))
|
| 432 |
+
for key in kinds:
|
| 433 |
+
totals[key] = None
|
| 434 |
+
else:
|
| 435 |
+
source["total"] = sum(local.values())
|
| 436 |
+
source["byKind"] = {key: sum(n for (_, _, k), n in local.items() if k == key) for key in kinds}
|
| 437 |
+
source["status"] = "partial" if source["limitations"] else "ok"
|
| 438 |
+
counts.update(local)
|
| 439 |
+
totals.update(source["byKind"])
|
| 440 |
+
sources.append(source)
|
| 441 |
+
complete = all(source["status"] == "ok" for source in sources)
|
| 442 |
+
users = sorted({actor for _, actor, _ in counts})
|
| 443 |
+
by_user = [{"user": actor, "byKind": {
|
| 444 |
+
key: None if totals[key] is None else sum(n for (_, a, k), n in counts.items() if a == actor and k == key)
|
| 445 |
+
for key in selected}} for actor in users]
|
| 446 |
+
by_day = []
|
| 447 |
+
day = first
|
| 448 |
+
while day <= last:
|
| 449 |
+
values = {key: None if totals[key] is None else sum(
|
| 450 |
+
n for (d, _, k), n in counts.items() if d == day.isoformat() and k == key) for key in selected}
|
| 451 |
+
for source in sources:
|
| 452 |
+
if day < _timestamp(source["heldFrom"]).date():
|
| 453 |
+
for key in source["kinds"]:
|
| 454 |
+
values[key] = None
|
| 455 |
+
by_day.append({"day": day.isoformat(), "byKind": values})
|
| 456 |
+
day += timedelta(days=1)
|
| 457 |
+
return {"tenant": session.tenant, "from": first.isoformat(), "to": last.isoformat(), "timezone": "UTC",
|
| 458 |
+
"filters": {"user": user, "kind": wanted or None}, "complete": complete,
|
| 459 |
+
"total": sum(totals.values()) if all(n is not None for n in totals.values()) else None,
|
| 460 |
+
"availableTotal": sum(counts.values()), "byKind": totals, "byUser": by_user,
|
| 461 |
+
"byDay": by_day, "counts": [{"day": day, "user": actor, "kind": key, "count": n}
|
| 462 |
+
for (day, actor, key), n in sorted(counts.items())], "sources": sources}
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
def validate(report, direct_counts):
|
| 466 |
+
"""Reconcile every kind and day-user-kind cell against independently read source counts."""
|
| 467 |
+
actual = Counter({(row["day"], row["user"], row["kind"]): row["count"] for row in report["counts"]})
|
| 468 |
+
differences = {kind: {"served": report["byKind"][kind],
|
| 469 |
+
"source": sum(n for (_, _, key), n in direct_counts.items() if key == kind)}
|
| 470 |
+
for kind in report["byKind"]}
|
| 471 |
+
readable = all(source["status"] != "error" for source in report["sources"])
|
| 472 |
+
users_match = {row["user"] for row in report["byUser"]} == {actor for _, actor, _ in direct_counts}
|
| 473 |
+
for row in report["byUser"]:
|
| 474 |
+
for kind, value in row["byKind"].items():
|
| 475 |
+
users_match = users_match and value == sum(
|
| 476 |
+
n for (_, actor, key), n in direct_counts.items() if actor == row["user"] and key == kind)
|
| 477 |
+
days_match = True
|
| 478 |
+
for row in report["byDay"]:
|
| 479 |
+
for kind, value in row["byKind"].items():
|
| 480 |
+
missing = any(kind in source["kinds"] and (source["status"] == "error" or
|
| 481 |
+
row["day"] < _timestamp(source["heldFrom"]).date().isoformat()) for source in report["sources"])
|
| 482 |
+
expected = None if missing else sum(n for (day, _, key), n in direct_counts.items()
|
| 483 |
+
if day == row["day"] and key == kind)
|
| 484 |
+
days_match = days_match and value == expected
|
| 485 |
+
aggregates_match = (report["availableTotal"] == sum(direct_counts.values())
|
| 486 |
+
and report["total"] == sum(direct_counts.values()))
|
| 487 |
+
return {"ok": readable and actual == direct_counts and all(
|
| 488 |
+
pair["served"] == pair["source"] for pair in differences.values())
|
| 489 |
+
and users_match and days_match and aggregates_match,
|
| 490 |
+
"byKind": differences, "cubeMatches": actual == direct_counts, "sourcesReadable": readable,
|
| 491 |
+
"usersMatch": users_match, "daysMatch": days_match, "aggregatesMatch": aggregates_match}
|
platform/aios_grid.py
CHANGED
|
@@ -2893,12 +2893,20 @@ def _clean_display(raw, valid_keys):
|
|
| 2893 |
# next server round trip.
|
| 2894 |
if mode == 'grid':
|
| 2895 |
coord_field = raw.get('coordField')
|
| 2896 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2897 |
# Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown
|
| 2898 |
# mode is still rejected rather than accidentally aliased into a real one. Every already
|
| 2899 |
# saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'.
|
| 2900 |
mode = _LEGACY_MODES.get(mode, mode)
|
| 2901 |
out = {'mode': mode}
|
|
|
|
|
|
|
| 2902 |
# `coordField` is the older per-View fallback. Keep it in the regular field-ref loop so a
|
| 2903 |
# deleted source fails safely; the current source is served separately as database setup.
|
| 2904 |
for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField', 'coordField'):
|
|
@@ -3332,6 +3340,18 @@ def view_filter_cycle(view_id, nodes, filters_of):
|
|
| 3332 |
return walk(view_id, [view_id], {view_id})
|
| 3333 |
|
| 3334 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3335 |
def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None,
|
| 3336 |
visible_view_ids=None, report=None):
|
| 3337 |
"""Recursively validate an UNTRUSTED filter tree (conditions + nested groups).
|
|
@@ -3494,6 +3514,7 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None,
|
|
| 3494 |
leaf['rankScope'] = rank_scope
|
| 3495 |
rhs = _clean_rhs(node.get('rhs'), valid_keys)
|
| 3496 |
if isinstance(node.get('rhs'), dict) and rhs is None:
|
|
|
|
| 3497 |
if report is not None:
|
| 3498 |
report.append({'id': leaf.get('id', ''), 'colId': column,
|
| 3499 |
'rhsColId': node['rhs'].get('colId'),
|
|
@@ -3501,6 +3522,12 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None,
|
|
| 3501 |
continue
|
| 3502 |
if rhs is not None:
|
| 3503 |
leaf['rhs'] = rhs
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3504 |
# C3: a record lock is explicit on the leaf, not a view-wide presentation flag.
|
| 3505 |
# Once stored, grid_events preserves the containing filter config unless it is copied
|
| 3506 |
# to a new View; a later caller cannot turn this `True` into an editable-looking false.
|
|
|
|
| 2893 |
# next server round trip.
|
| 2894 |
if mode == 'grid':
|
| 2895 |
coord_field = raw.get('coordField')
|
| 2896 |
+
if raw.get('groupSummary') is not True:
|
| 2897 |
+
return {'mode': 'grid', 'coordField': coord_field} if coord_field in valid_keys else None
|
| 2898 |
+
# The C1 display member carries the grid's group summary toggle. Store only True.
|
| 2899 |
+
out = {'mode': 'grid', 'groupSummary': True}
|
| 2900 |
+
if coord_field in valid_keys:
|
| 2901 |
+
out['coordField'] = coord_field
|
| 2902 |
+
return out
|
| 2903 |
# Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown
|
| 2904 |
# mode is still rejected rather than accidentally aliased into a real one. Every already
|
| 2905 |
# saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'.
|
| 2906 |
mode = _LEGACY_MODES.get(mode, mode)
|
| 2907 |
out = {'mode': mode}
|
| 2908 |
+
if raw.get('groupSummary') is True:
|
| 2909 |
+
out['groupSummary'] = True
|
| 2910 |
# `coordField` is the older per-View fallback. Keep it in the regular field-ref loop so a
|
| 2911 |
# deleted source fails safely; the current source is served separately as database setup.
|
| 2912 |
for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField', 'coordField'):
|
|
|
|
| 3340 |
return walk(view_id, [view_id], {view_id})
|
| 3341 |
|
| 3342 |
|
| 3343 |
+
def clean_filter_cell_ref(raw):
|
| 3344 |
+
"""Validate a saved external cell address without choosing its reader or changing its shape."""
|
| 3345 |
+
if (not isinstance(raw, dict) or set(raw) - {'workbook_id', 'tab', 'ref', 'locked'}
|
| 3346 |
+
or any(not isinstance(raw.get(key), str) or not raw[key].strip()
|
| 3347 |
+
or raw[key] != raw[key].strip() or len(raw[key]) > 120
|
| 3348 |
+
for key in ('workbook_id', 'tab', 'ref'))
|
| 3349 |
+
or not re.fullmatch(r'\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}', raw['ref'])
|
| 3350 |
+
or 'locked' in raw and not isinstance(raw['locked'], bool)):
|
| 3351 |
+
return None
|
| 3352 |
+
return copy.deepcopy(raw)
|
| 3353 |
+
|
| 3354 |
+
|
| 3355 |
def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None,
|
| 3356 |
visible_view_ids=None, report=None):
|
| 3357 |
"""Recursively validate an UNTRUSTED filter tree (conditions + nested groups).
|
|
|
|
| 3514 |
leaf['rankScope'] = rank_scope
|
| 3515 |
rhs = _clean_rhs(node.get('rhs'), valid_keys)
|
| 3516 |
if isinstance(node.get('rhs'), dict) and rhs is None:
|
| 3517 |
+
# An invalid comparison must not become an inactive empty literal.
|
| 3518 |
if report is not None:
|
| 3519 |
report.append({'id': leaf.get('id', ''), 'colId': column,
|
| 3520 |
'rhsColId': node['rhs'].get('colId'),
|
|
|
|
| 3522 |
continue
|
| 3523 |
if rhs is not None:
|
| 3524 |
leaf['rhs'] = rhs
|
| 3525 |
+
# The save host authorizes these addresses before this structural cleaner runs.
|
| 3526 |
+
refs = {key: clean_filter_cell_ref(node[key])
|
| 3527 |
+
for key in ('valueRef', 'value2Ref') if key in node}
|
| 3528 |
+
if any(ref is None for ref in refs.values()):
|
| 3529 |
+
continue
|
| 3530 |
+
leaf.update(refs)
|
| 3531 |
# C3: a record lock is explicit on the leaf, not a view-wide presentation flag.
|
| 3532 |
# Once stored, grid_events preserves the containing filter config unless it is copied
|
| 3533 |
# to a new View; a later caller cannot turn this `True` into an editable-looking false.
|
platform/core/amazon_project_tasks.py
ADDED
|
@@ -0,0 +1,725 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tenant-bound task trees and leaf progress for Amazon projects."""
|
| 2 |
+
|
| 3 |
+
from collections import defaultdict, deque
|
| 4 |
+
from contextlib import contextmanager
|
| 5 |
+
from datetime import date, timedelta
|
| 6 |
+
from decimal import Decimal, InvalidOperation, localcontext
|
| 7 |
+
import json
|
| 8 |
+
import uuid
|
| 9 |
+
|
| 10 |
+
from core import amazon_projects as projects
|
| 11 |
+
|
| 12 |
+
TASKS = 'amazon_project_tasks'
|
| 13 |
+
DEPENDENCIES = 'amazon_project_dependencies'
|
| 14 |
+
FILTERS = 'amazon_project_task_filters'
|
| 15 |
+
TABLES = (TASKS, DEPENDENCIES, FILTERS)
|
| 16 |
+
TASK_COLUMNS = ('id', 'project_id', 'parent_id', 'position', 'name', 'assignee', 'start_on',
|
| 17 |
+
'due_on', 'done_at', 'contingency_for', 'active', 'created_by')
|
| 18 |
+
DEPENDENCY_COLUMNS = ('id', 'project_id', 'from_task', 'to_task', 'type', 'lag_days')
|
| 19 |
+
DEPENDENCY_ENDPOINTS = {'FS': ('due_on', 'start_on'), 'SS': ('start_on', 'start_on'),
|
| 20 |
+
'FF': ('due_on', 'due_on'), 'SF': ('start_on', 'due_on')}
|
| 21 |
+
TARGET_FIELDS = ('targets', 'target_filter', 'completion_filter')
|
| 22 |
+
FILTER_STATE = ('filter_status', 'filter_closed_at', 'closed_targets', 'filter_missing')
|
| 23 |
+
EDITABLE = frozenset(('name', 'assignee', 'start_on', 'due_on', 'done', 'contingency_for', *TARGET_FIELDS))
|
| 24 |
+
_UNCHANGED = object()
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _tree(value, *, empty=True):
|
| 28 |
+
nodes = value if isinstance(value, list) else value.get('nodes') if isinstance(value, dict) else None
|
| 29 |
+
if not isinstance(nodes, list) or (not empty and not nodes):
|
| 30 |
+
raise ValueError('Send a saved filter tree with conditions.')
|
| 31 |
+
try:
|
| 32 |
+
return json.loads(json.dumps(value, allow_nan=False))
|
| 33 |
+
except (TypeError, ValueError):
|
| 34 |
+
raise ValueError('Send a saved filter tree with JSON values.') from None
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _target_fields(current):
|
| 38 |
+
values = current.get('targets')
|
| 39 |
+
if values is not None:
|
| 40 |
+
if not isinstance(values, list):
|
| 41 |
+
raise ValueError('Send targets as a list of grain and key pairs.')
|
| 42 |
+
pairs, seen = [], set()
|
| 43 |
+
for value in values:
|
| 44 |
+
if not isinstance(value, dict) or set(value) != {'grain', 'key'}:
|
| 45 |
+
raise ValueError('Send each task target as a grain and key.')
|
| 46 |
+
pair = projects.target(value['grain'], value['key'])
|
| 47 |
+
if pair[0] not in ('sku', 'asin'):
|
| 48 |
+
raise ValueError('Task targets use sku or asin.')
|
| 49 |
+
if pair not in seen:
|
| 50 |
+
seen.add(pair)
|
| 51 |
+
pairs.append(pair)
|
| 52 |
+
current['targets'] = projects._pair_list(pairs)
|
| 53 |
+
saved = current.get('target_filter')
|
| 54 |
+
if saved is not None:
|
| 55 |
+
if not isinstance(saved, dict) or set(saved) != {'grain', 'filter'} or saved['grain'] not in ('sku', 'asin'):
|
| 56 |
+
raise ValueError('Send targetFilter as a sku or asin grain and filter tree.')
|
| 57 |
+
current['target_filter'] = dict(grain=saved['grain'], filter=_tree(saved['filter']))
|
| 58 |
+
if current.get('targets') and saved is not None:
|
| 59 |
+
raise ValueError('Send either a target list or one saved target filter.')
|
| 60 |
+
if current.get('completion_filter') is not None:
|
| 61 |
+
current['completion_filter'] = _tree(current['completion_filter'], empty=False)
|
| 62 |
+
if not current.get('targets') and saved is None:
|
| 63 |
+
raise ValueError('A completion filter needs task targets.')
|
| 64 |
+
return current
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _position(value):
|
| 68 |
+
try:
|
| 69 |
+
number = Decimal(str(value))
|
| 70 |
+
except (InvalidOperation, ValueError):
|
| 71 |
+
raise ValueError('position must be a finite number.') from None
|
| 72 |
+
if isinstance(value, bool) or not number.is_finite():
|
| 73 |
+
raise ValueError('position must be a finite number.')
|
| 74 |
+
return number
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _fields(values, previous=None):
|
| 78 |
+
current = dict(previous or {})
|
| 79 |
+
current.update(values)
|
| 80 |
+
current['name'] = projects._text(current.get('name'), 'name')
|
| 81 |
+
current['assignee'] = projects._text(current.get('assignee'), 'assignee', optional=True)
|
| 82 |
+
for field in ('start_on', 'due_on'):
|
| 83 |
+
current[field] = projects._date(current.get(field), field)
|
| 84 |
+
if current['start_on'] and current['due_on'] and current['due_on'] < current['start_on']:
|
| 85 |
+
raise ValueError('due_on must be on or after start_on.')
|
| 86 |
+
current['contingency_for'] = projects._text(current.get('contingency_for'), 'contingency_for', optional=True)
|
| 87 |
+
if 'done' in values:
|
| 88 |
+
if type(values['done']) is not bool:
|
| 89 |
+
raise ValueError('done must be true or false.')
|
| 90 |
+
current['done_at'] = (current.get('done_at') or projects._now()) if values['done'] else None
|
| 91 |
+
else:
|
| 92 |
+
current.setdefault('done_at', None)
|
| 93 |
+
current.pop('done', None)
|
| 94 |
+
return _target_fields(current)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _record(values):
|
| 98 |
+
row = dict(zip(TASK_COLUMNS, values))
|
| 99 |
+
row['position'] = str(_position(row['position']))
|
| 100 |
+
row['active'] = bool(row['active'])
|
| 101 |
+
row['done'] = row['done_at'] is not None
|
| 102 |
+
for field in ('start_on', 'due_on'):
|
| 103 |
+
row[field] = projects._date(row[field], field)
|
| 104 |
+
return row
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _check_parent(rows, task_id, parent_id):
|
| 108 |
+
"""Walk ancestors without a Python call stack or a fixed depth ceiling."""
|
| 109 |
+
visited = {task_id}
|
| 110 |
+
while parent_id is not None:
|
| 111 |
+
if parent_id in visited:
|
| 112 |
+
raise ValueError('A task cannot move under itself or its descendant.')
|
| 113 |
+
if parent_id not in rows:
|
| 114 |
+
raise ValueError('The parent task must belong to this project.')
|
| 115 |
+
visited.add(parent_id)
|
| 116 |
+
parent_id = rows[parent_id]['parent_id']
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _check_contingency(rows, task_id, contingency_for):
|
| 120 |
+
if contingency_for is not None and (contingency_for == task_id or contingency_for not in rows):
|
| 121 |
+
raise ValueError('The backup task must name another task in this project.')
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _dependency_fields(kind, lag_days):
|
| 125 |
+
if not isinstance(kind, str) or kind not in DEPENDENCY_ENDPOINTS:
|
| 126 |
+
raise ValueError('Dependency type must be FS, SS, FF or SF.')
|
| 127 |
+
if type(lag_days) is not int or not -2147483648 <= lag_days <= 2147483647:
|
| 128 |
+
raise ValueError('lag_days must be an integer number of days.')
|
| 129 |
+
return kind, lag_days
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _check_dependency(rows, dependencies, from_task, to_task):
|
| 133 |
+
if from_task not in rows or to_task not in rows:
|
| 134 |
+
raise ValueError('Both dependency tasks must belong to this project.')
|
| 135 |
+
if from_task == to_task:
|
| 136 |
+
raise ValueError('A task cannot depend on itself.')
|
| 137 |
+
outgoing = defaultdict(list)
|
| 138 |
+
for dependency in dependencies:
|
| 139 |
+
outgoing[dependency['from_task']].append(dependency['to_task'])
|
| 140 |
+
pending, seen = [to_task], set()
|
| 141 |
+
while pending:
|
| 142 |
+
task_id = pending.pop()
|
| 143 |
+
if task_id == from_task:
|
| 144 |
+
source, target = rows[from_task], rows[to_task]
|
| 145 |
+
raise ValueError(f'Dependency cycle between "{source["name"]}" ({from_task}) '
|
| 146 |
+
f'and "{target["name"]}" ({to_task}).')
|
| 147 |
+
if task_id not in seen:
|
| 148 |
+
seen.add(task_id)
|
| 149 |
+
pending.extend(outgoing[task_id])
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def _constraint_gap(rows, dependency):
|
| 153 |
+
source, target = rows[dependency['from_task']], rows[dependency['to_task']]
|
| 154 |
+
source_field, target_field = DEPENDENCY_ENDPOINTS[dependency['type']]
|
| 155 |
+
if source[source_field] is None or target[target_field] is None:
|
| 156 |
+
raise ValueError(f'Dependency "{source["name"]}" to "{target["name"]}" '
|
| 157 |
+
f'requires predecessor {source_field} and successor {target_field}.')
|
| 158 |
+
return ((date.fromisoformat(source[source_field]) - date.fromisoformat(target[target_field])).days
|
| 159 |
+
+ dependency['lag_days'])
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def _propagate(rows, dependencies, task_id):
|
| 163 |
+
"""Shift each affected task once after all its predecessors, preserving its duration."""
|
| 164 |
+
incoming, outgoing, indegree = defaultdict(list), defaultdict(list), dict.fromkeys(rows, 0)
|
| 165 |
+
for dependency in dependencies:
|
| 166 |
+
incoming[dependency['to_task']].append(dependency)
|
| 167 |
+
outgoing[dependency['from_task']].append(dependency['to_task'])
|
| 168 |
+
indegree[dependency['to_task']] += 1
|
| 169 |
+
pending, affected = [task_id], set()
|
| 170 |
+
while pending:
|
| 171 |
+
current = pending.pop()
|
| 172 |
+
if current not in affected:
|
| 173 |
+
affected.add(current)
|
| 174 |
+
pending.extend(outgoing[current])
|
| 175 |
+
ready, shifted, visited = deque(key for key, count in indegree.items() if count == 0), [], 0
|
| 176 |
+
while ready:
|
| 177 |
+
current = ready.popleft()
|
| 178 |
+
visited += 1
|
| 179 |
+
if current in affected:
|
| 180 |
+
days = max([0] + [_constraint_gap(rows, dependency) for dependency in incoming[current]])
|
| 181 |
+
if days:
|
| 182 |
+
changed = dict(rows[current])
|
| 183 |
+
try:
|
| 184 |
+
for field in ('start_on', 'due_on'):
|
| 185 |
+
if changed[field] is not None:
|
| 186 |
+
changed[field] = (date.fromisoformat(changed[field]) + timedelta(days=days)).isoformat()
|
| 187 |
+
except OverflowError:
|
| 188 |
+
raise ValueError(f'Dependency shift exceeds the calendar for task "{changed["name"]}".') from None
|
| 189 |
+
rows[current] = changed
|
| 190 |
+
shifted.append(changed)
|
| 191 |
+
for successor in outgoing[current]:
|
| 192 |
+
indegree[successor] -= 1
|
| 193 |
+
if indegree[successor] == 0:
|
| 194 |
+
ready.append(successor)
|
| 195 |
+
if visited != len(rows):
|
| 196 |
+
raise ValueError('The stored task dependencies contain a cycle.')
|
| 197 |
+
return shifted
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _subtree(rows, task_id):
|
| 201 |
+
children = defaultdict(list)
|
| 202 |
+
for row in rows.values():
|
| 203 |
+
children[row['parent_id']].append(row['id'])
|
| 204 |
+
pending, found = [task_id], []
|
| 205 |
+
seen = set()
|
| 206 |
+
while pending:
|
| 207 |
+
current = pending.pop()
|
| 208 |
+
if current not in seen:
|
| 209 |
+
seen.add(current)
|
| 210 |
+
found.append(current)
|
| 211 |
+
pending.extend(children[current])
|
| 212 |
+
return found
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _leaf_progress(rows, fallback):
|
| 216 |
+
"""Count completed active leaves, excluding every dormant backup subtree."""
|
| 217 |
+
children = defaultdict(list)
|
| 218 |
+
for row in rows:
|
| 219 |
+
children[row['parent_id']].append(row)
|
| 220 |
+
pending, leaves = list(children[None]), []
|
| 221 |
+
while pending:
|
| 222 |
+
row = pending.pop()
|
| 223 |
+
if not row['active']:
|
| 224 |
+
continue
|
| 225 |
+
active_children = [child for child in children[row['id']] if child['active']]
|
| 226 |
+
if active_children:
|
| 227 |
+
pending.extend(active_children)
|
| 228 |
+
else:
|
| 229 |
+
leaves.append(row)
|
| 230 |
+
done, total = sum(row['done'] for row in leaves), len(leaves)
|
| 231 |
+
return {'done': done, 'total': total,
|
| 232 |
+
'progress': round(100 * done / total) if total else (0 if rows else fallback)}
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def _backend(rt):
|
| 236 |
+
return PgTaskStore(projects._backend(rt))
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def list_tasks(rt, project_id, *, matcher=None, today=None, by=None):
|
| 240 |
+
"""Read every task as flat rows with parent ids and exact fractional positions."""
|
| 241 |
+
backend = _backend(rt)
|
| 242 |
+
project_id = projects._text(project_id, 'project_id')
|
| 243 |
+
if matcher is not None:
|
| 244 |
+
_evaluate_project(rt, backend, project_id, matcher, today=today, by=by)
|
| 245 |
+
return backend.list_tasks(project_id)
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def get(rt, project_id, task_id):
|
| 249 |
+
"""Read one task within its tenant and project, or raise KeyError."""
|
| 250 |
+
task_id = projects._text(task_id, 'task_id')
|
| 251 |
+
for row in list_tasks(rt, project_id):
|
| 252 |
+
if row['id'] == task_id:
|
| 253 |
+
return row
|
| 254 |
+
raise KeyError('Task not found in this project.')
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def progress(rt, project_id):
|
| 258 |
+
"""Return completed leaf counts and their percentage, retaining manual progress without tasks."""
|
| 259 |
+
project_id = projects._text(project_id, 'project_id')
|
| 260 |
+
return _backend(rt).progress(project_id)
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def _mutate(rt, project_id, task_id, action, values, by):
|
| 264 |
+
project_id = projects._text(project_id, 'project_id')
|
| 265 |
+
task_id = projects._text(task_id, 'task_id')
|
| 266 |
+
by = projects._text(by, 'by')
|
| 267 |
+
before, after, result = _backend(rt).mutate(project_id, task_id, action, values, by)
|
| 268 |
+
projects._log(rt, project_id, by, 'tasks', before, after)
|
| 269 |
+
return result
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def create(rt, project_id, *, name, by, parent_id=None, position=None, assignee=None,
|
| 273 |
+
start_on=None, due_on=None, done=False, contingency_for=None, created_by=None,
|
| 274 |
+
targets=None, target_filter=None, completion_filter=None):
|
| 275 |
+
"""Create one task with at most one assignee and an optional dormant backup link."""
|
| 276 |
+
values = _fields(dict(name=name, assignee=assignee, start_on=start_on, due_on=due_on,
|
| 277 |
+
done=done, contingency_for=contingency_for))
|
| 278 |
+
if targets is not None or target_filter is not None or completion_filter is not None:
|
| 279 |
+
values.update(targets=targets, target_filter=target_filter, completion_filter=completion_filter)
|
| 280 |
+
_target_fields(values)
|
| 281 |
+
values.update(parent_id=projects._text(parent_id, 'parent_id', optional=True),
|
| 282 |
+
position=None if position is None else str(_position(position)),
|
| 283 |
+
created_by=projects._text(created_by or by, 'created_by'))
|
| 284 |
+
return _mutate(rt, project_id, uuid.uuid4().hex, 'create', values, by)
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def update(rt, project_id, task_id, changes, *, by):
|
| 288 |
+
"""Change task fields without allowing a second tree or activation write door."""
|
| 289 |
+
if not isinstance(changes, dict) or not changes or set(changes) - EDITABLE:
|
| 290 |
+
raise ValueError(f'Update only: {", ".join(sorted(EDITABLE))}.')
|
| 291 |
+
return _mutate(rt, project_id, task_id, 'update', dict(changes), by)
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def move(rt, project_id, task_id, *, by, parent_id=_UNCHANGED, position=None):
|
| 295 |
+
"""Reparent or reorder one task after checking the full ancestor chain."""
|
| 296 |
+
values = dict(position=None if position is None else str(_position(position)))
|
| 297 |
+
if parent_id is not _UNCHANGED:
|
| 298 |
+
values['parent_id'] = projects._text(parent_id, 'parent_id', optional=True)
|
| 299 |
+
return _mutate(rt, project_id, task_id, 'move', values, by)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def activate(rt, project_id, task_id, *, by):
|
| 303 |
+
"""Include a backup task and its descendants after an explicit human activation."""
|
| 304 |
+
return _mutate(rt, project_id, task_id, 'activate', {}, by)
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def delete(rt, project_id, task_id, *, by):
|
| 308 |
+
"""Delete one task and its full subtree in the project's write transaction."""
|
| 309 |
+
return _mutate(rt, project_id, task_id, 'delete', {}, by)
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def list_dependencies(rt, project_id):
|
| 313 |
+
"""Read all dependency edges within one tenant and project without creating storage."""
|
| 314 |
+
return _backend(rt).list_dependencies(projects._text(project_id, 'project_id'))
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def add_dependency(rt, project_id, from_task, to_task, *, type='FS', lag_days=0, by):
|
| 318 |
+
"""Link two tasks and shift the successor and its dependents to satisfy the new constraint."""
|
| 319 |
+
project_id = projects._text(project_id, 'project_id')
|
| 320 |
+
by = projects._text(by, 'by')
|
| 321 |
+
kind, lag_days = _dependency_fields(type, lag_days)
|
| 322 |
+
record = dict(id=uuid.uuid4().hex, project_id=project_id,
|
| 323 |
+
from_task=projects._text(from_task, 'from_task'),
|
| 324 |
+
to_task=projects._text(to_task, 'to_task'), type=kind, lag_days=lag_days)
|
| 325 |
+
result = _backend(rt).add_dependency(record)
|
| 326 |
+
projects._log(rt, project_id, by, 'dependencies', None, result)
|
| 327 |
+
return result
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def delete_dependency(rt, project_id, dependency_id, *, by):
|
| 331 |
+
"""Remove one project dependency without moving either task earlier."""
|
| 332 |
+
project_id = projects._text(project_id, 'project_id')
|
| 333 |
+
by = projects._text(by, 'by')
|
| 334 |
+
record = _backend(rt).delete_dependency(project_id, projects._text(dependency_id, 'dependency_id'))
|
| 335 |
+
projects._log(rt, project_id, by, 'dependencies', record, None)
|
| 336 |
+
return record
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def reschedule(rt, task_id, start_on, due_on, by):
|
| 340 |
+
"""Reschedule one tenant task and return every dependent shifted to satisfy its constraints."""
|
| 341 |
+
task_id, by = projects._text(task_id, 'task_id'), projects._text(by, 'by')
|
| 342 |
+
values = {field: projects._date(value, field) for field, value in
|
| 343 |
+
(('start_on', start_on), ('due_on', due_on))}
|
| 344 |
+
before, result = _backend(rt).reschedule(task_id, values)
|
| 345 |
+
projects._log(rt, result['task']['project_id'], by, 'tasks', before, result)
|
| 346 |
+
return result
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def validate(rt, project_id, candidate=None):
|
| 350 |
+
"""Reconcile progress against a separate authored-task read without the display calculator."""
|
| 351 |
+
backend = _backend(rt)
|
| 352 |
+
project_id = projects._text(project_id, 'project_id')
|
| 353 |
+
with backend.projects._connection() as con:
|
| 354 |
+
project = backend._project(con, project_id)
|
| 355 |
+
rows = (con.execute(f'SELECT id, parent_id, active, done_at FROM {backend.t(TASKS)} '
|
| 356 |
+
'WHERE tenant = %s AND project_id = %s',
|
| 357 |
+
(backend.tenant_slug, project_id)).fetchall()
|
| 358 |
+
if backend._exists(con) else [])
|
| 359 |
+
children, completed = defaultdict(list), {}
|
| 360 |
+
for task_id, parent_id, active, done_at in rows:
|
| 361 |
+
if active:
|
| 362 |
+
children[parent_id].append(task_id)
|
| 363 |
+
completed[task_id] = done_at is not None
|
| 364 |
+
pending, leaves = deque(children[None]), []
|
| 365 |
+
while pending:
|
| 366 |
+
task_id = pending.popleft()
|
| 367 |
+
if children[task_id]:
|
| 368 |
+
pending.extend(children[task_id])
|
| 369 |
+
else:
|
| 370 |
+
leaves.append(task_id)
|
| 371 |
+
done, total = sum(completed[task_id] for task_id in leaves), len(leaves)
|
| 372 |
+
expected = round(100 * done / total) if total else (0 if rows else project['progress'])
|
| 373 |
+
actual = project['progress'] if candidate is None else candidate
|
| 374 |
+
return {'check': 'project progress ties to active leaf tasks', 'ok': actual == expected,
|
| 375 |
+
'expected': expected, 'actual': actual, 'done': done, 'total': total}
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
def _evaluate_project(rt, backend, project_id, matcher, *, today, by):
|
| 379 |
+
day = projects._date(today or projects._today(), 'today')
|
| 380 |
+
by = projects._text(by, 'by')
|
| 381 |
+
rows = backend.list_tasks(project_id)
|
| 382 |
+
if not any(row.get('completion_filter') is not None for row in rows):
|
| 383 |
+
return []
|
| 384 |
+
events = []
|
| 385 |
+
with backend._write(project_id) as (con, project):
|
| 386 |
+
rows = backend._rows(con, project_id)
|
| 387 |
+
ancestors = {item['id']: item for item in rows}
|
| 388 |
+
for row in rows:
|
| 389 |
+
if row.get('completion_filter') is None:
|
| 390 |
+
continue
|
| 391 |
+
# A dormant backup and its subtree need human activation before evaluation.
|
| 392 |
+
current = row
|
| 393 |
+
while current['active'] and current['parent_id'] is not None:
|
| 394 |
+
current = ancestors[current['parent_id']]
|
| 395 |
+
if not current['active']:
|
| 396 |
+
continue
|
| 397 |
+
principal = (backend.tenant_slug, row.get('filter_by', row['created_by']))
|
| 398 |
+
if row.get('filter_closed_at'):
|
| 399 |
+
pairs = row['closed_targets']
|
| 400 |
+
elif row.get('target_filter') is not None:
|
| 401 |
+
saved = row['target_filter']
|
| 402 |
+
pairs = projects._pair_list((saved['grain'], key) for key in
|
| 403 |
+
matcher(rt, saved['grain'], saved['filter'], user=principal))
|
| 404 |
+
else:
|
| 405 |
+
pairs = row.get('targets') or []
|
| 406 |
+
met = set()
|
| 407 |
+
for grain in dict.fromkeys(pair['grain'] for pair in pairs):
|
| 408 |
+
keys = set(matcher(rt, grain, row['completion_filter'], user=principal))
|
| 409 |
+
met.update((grain, pair['key']) for pair in pairs if pair['grain'] == grain and pair['key'] in keys)
|
| 410 |
+
missing = [pair for pair in pairs if (pair['grain'], pair['key']) not in met]
|
| 411 |
+
after = dict(row, filter_status={'met': len(met), 'total': len(pairs),
|
| 412 |
+
'message': f'met {len(met)} of {len(pairs)}', 'evaluated_on': day})
|
| 413 |
+
if row.get('filter_closed_at'):
|
| 414 |
+
previous = {(pair['grain'], pair['key']) for pair in row.get('filter_missing', [])}
|
| 415 |
+
for pair in missing:
|
| 416 |
+
if (pair['grain'], pair['key']) not in previous:
|
| 417 |
+
events.append((row['id'], f'{projects.GRAIN_LABELS[pair["grain"]]} {pair["key"]} '
|
| 418 |
+
'no longer meets the filter', row, dict(after, target=pair)))
|
| 419 |
+
elif not row['done'] and pairs and not missing:
|
| 420 |
+
after.update(done=True, done_at=projects._now(), filter_closed_at=projects._now(),
|
| 421 |
+
closed_targets=pairs)
|
| 422 |
+
events.append((row['id'], 'Done by filter', row, after))
|
| 423 |
+
after['filter_missing'] = missing
|
| 424 |
+
backend._save(con, after)
|
| 425 |
+
backend._sync_progress(con, project_id, project['progress'])
|
| 426 |
+
for _task_id, action, before, after in events:
|
| 427 |
+
projects._log(rt, project_id, by, action, before, after)
|
| 428 |
+
return [dict(task_id=task_id, action=action) for task_id, action, _before, _after in events]
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def evaluate_filter_tasks(rt, matcher, *, today, by):
|
| 432 |
+
"""Evaluate every saved task criterion on refresh through the injected tenant matcher."""
|
| 433 |
+
backend = _backend(rt)
|
| 434 |
+
events = []
|
| 435 |
+
for project_id in backend.filter_projects():
|
| 436 |
+
events.extend(_evaluate_project(rt, backend, project_id, matcher, today=today, by=by))
|
| 437 |
+
return events
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
def validate_filter_tasks(rt, project_id, matcher):
|
| 441 |
+
"""Reconcile stored completion counts against a separate target and matcher read."""
|
| 442 |
+
backend = _backend(rt)
|
| 443 |
+
checks = []
|
| 444 |
+
for row in backend.list_tasks(projects._text(project_id, 'project_id')):
|
| 445 |
+
if row.get('filter_status') is None:
|
| 446 |
+
continue
|
| 447 |
+
user = (backend.tenant_slug, row.get('filter_by', row['created_by']))
|
| 448 |
+
saved = row.get('target_filter')
|
| 449 |
+
if row.get('filter_closed_at'):
|
| 450 |
+
targets = row['closed_targets']
|
| 451 |
+
elif saved is not None:
|
| 452 |
+
targets = [{'grain': saved['grain'], 'key': key}
|
| 453 |
+
for key in matcher(rt, saved['grain'], saved['filter'], user=user)]
|
| 454 |
+
else:
|
| 455 |
+
targets = row.get('targets') or []
|
| 456 |
+
matches = {grain: set(matcher(rt, grain, row['completion_filter'], user=user))
|
| 457 |
+
for grain in {pair['grain'] for pair in targets}}
|
| 458 |
+
met = sum(pair['key'] in matches[pair['grain']] for pair in targets)
|
| 459 |
+
actual = row['filter_status']
|
| 460 |
+
checks.append({'task': row['id'], 'expected': {'met': met, 'total': len(targets)},
|
| 461 |
+
'actual': {key: actual[key] for key in ('met', 'total')},
|
| 462 |
+
'ok': actual['met'] == met and actual['total'] == len(targets)})
|
| 463 |
+
return {'check': 'task filter counts tie to target rows', 'ok': all(row['ok'] for row in checks),
|
| 464 |
+
'tasks': checks}
|
| 465 |
+
|
| 466 |
+
|
| 467 |
+
class PgTaskStore:
|
| 468 |
+
"""Use the existing tenant backend, write guard and project row lock for task authorship."""
|
| 469 |
+
|
| 470 |
+
def __init__(self, backend):
|
| 471 |
+
self.projects = backend
|
| 472 |
+
self.tenant_slug = backend.tenant_slug
|
| 473 |
+
self.t = backend.t
|
| 474 |
+
|
| 475 |
+
def _exists(self, con, table=TASKS):
|
| 476 |
+
return con.execute('SELECT to_regclass(%s)', (self.t(table),)).fetchone()[0] is not None
|
| 477 |
+
|
| 478 |
+
def _project(self, con, project_id, lock=False):
|
| 479 |
+
if not self.projects._exists(con):
|
| 480 |
+
raise KeyError('Project not found.')
|
| 481 |
+
return self.projects._one(con, project_id, lock=lock)
|
| 482 |
+
|
| 483 |
+
def ensure_schema(self, con):
|
| 484 |
+
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(TASKS)} (
|
| 485 |
+
tenant TEXT NOT NULL, id TEXT NOT NULL, project_id TEXT NOT NULL,
|
| 486 |
+
parent_id TEXT, position NUMERIC NOT NULL, name TEXT NOT NULL,
|
| 487 |
+
assignee TEXT, start_on DATE, due_on DATE, done_at TEXT,
|
| 488 |
+
contingency_for TEXT, active BOOLEAN NOT NULL DEFAULT TRUE, created_by TEXT NOT NULL,
|
| 489 |
+
PRIMARY KEY (tenant, id), UNIQUE (tenant, project_id, id),
|
| 490 |
+
FOREIGN KEY (tenant, project_id) REFERENCES {self.t(projects.PROJECTS)} (tenant, id)
|
| 491 |
+
ON DELETE CASCADE,
|
| 492 |
+
FOREIGN KEY (tenant, project_id, parent_id) REFERENCES {self.t(TASKS)} (tenant, project_id, id)
|
| 493 |
+
DEFERRABLE INITIALLY DEFERRED,
|
| 494 |
+
CHECK (parent_id IS NULL OR parent_id <> id),
|
| 495 |
+
CHECK (contingency_for IS NULL OR contingency_for <> id),
|
| 496 |
+
CHECK (start_on IS NULL OR due_on IS NULL OR start_on <= due_on))''')
|
| 497 |
+
con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_tasks_siblings" '
|
| 498 |
+
f'ON {self.t(TASKS)} (tenant, project_id, parent_id, position, id)')
|
| 499 |
+
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(DEPENDENCIES)} (
|
| 500 |
+
tenant TEXT NOT NULL, id TEXT NOT NULL, project_id TEXT NOT NULL,
|
| 501 |
+
from_task TEXT NOT NULL, to_task TEXT NOT NULL, type TEXT NOT NULL,
|
| 502 |
+
lag_days INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (tenant, id),
|
| 503 |
+
UNIQUE (tenant, project_id, from_task, to_task, type),
|
| 504 |
+
FOREIGN KEY (tenant, project_id) REFERENCES {self.t(projects.PROJECTS)} (tenant, id)
|
| 505 |
+
ON DELETE CASCADE,
|
| 506 |
+
FOREIGN KEY (tenant, project_id, from_task) REFERENCES {self.t(TASKS)} (tenant, project_id, id)
|
| 507 |
+
ON DELETE CASCADE,
|
| 508 |
+
FOREIGN KEY (tenant, project_id, to_task) REFERENCES {self.t(TASKS)} (tenant, project_id, id)
|
| 509 |
+
ON DELETE CASCADE,
|
| 510 |
+
CHECK (from_task <> to_task), CHECK (type IN ('FS', 'SS', 'FF', 'SF')))''')
|
| 511 |
+
con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_dependencies_target" '
|
| 512 |
+
f'ON {self.t(DEPENDENCIES)} (tenant, project_id, to_task)')
|
| 513 |
+
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(FILTERS)} (
|
| 514 |
+
tenant TEXT NOT NULL, id TEXT NOT NULL, project_id TEXT NOT NULL, config TEXT NOT NULL,
|
| 515 |
+
PRIMARY KEY (tenant, id),
|
| 516 |
+
FOREIGN KEY (tenant, project_id, id) REFERENCES {self.t(TASKS)} (tenant, project_id, id)
|
| 517 |
+
ON DELETE CASCADE)''')
|
| 518 |
+
|
| 519 |
+
@contextmanager
|
| 520 |
+
def _write(self, project_id):
|
| 521 |
+
self.projects._write_guard()
|
| 522 |
+
with self.projects._connection() as con:
|
| 523 |
+
project = self._project(con, project_id, lock=True)
|
| 524 |
+
self.ensure_schema(con)
|
| 525 |
+
yield con, project
|
| 526 |
+
|
| 527 |
+
def _rows(self, con, project_id):
|
| 528 |
+
rows = con.execute(f'SELECT {", ".join(TASK_COLUMNS)} FROM {self.t(TASKS)} '
|
| 529 |
+
'WHERE tenant = %s AND project_id = %s '
|
| 530 |
+
'ORDER BY parent_id NULLS FIRST, position, id',
|
| 531 |
+
(self.tenant_slug, project_id)).fetchall()
|
| 532 |
+
records = [_record(row) for row in rows]
|
| 533 |
+
configs = {}
|
| 534 |
+
if self._exists(con, FILTERS):
|
| 535 |
+
configs = {task_id: json.loads(value) for task_id, value in con.execute(
|
| 536 |
+
f'SELECT id, config FROM {self.t(FILTERS)} WHERE tenant = %s AND project_id = %s',
|
| 537 |
+
(self.tenant_slug, project_id)).fetchall()}
|
| 538 |
+
return [dict(row, **configs.get(row['id'], {})) for row in records]
|
| 539 |
+
|
| 540 |
+
def filter_projects(self):
|
| 541 |
+
with self.projects._connection() as con:
|
| 542 |
+
if not self._exists(con, FILTERS):
|
| 543 |
+
return []
|
| 544 |
+
return [row[0] for row in con.execute(
|
| 545 |
+
f'SELECT DISTINCT project_id FROM {self.t(FILTERS)} WHERE tenant = %s ORDER BY project_id',
|
| 546 |
+
(self.tenant_slug,)).fetchall()]
|
| 547 |
+
|
| 548 |
+
def list_tasks(self, project_id):
|
| 549 |
+
with self.projects._connection() as con:
|
| 550 |
+
self._project(con, project_id)
|
| 551 |
+
return self._rows(con, project_id) if self._exists(con) else []
|
| 552 |
+
|
| 553 |
+
def _dependencies(self, con, project_id):
|
| 554 |
+
if not self._exists(con, DEPENDENCIES):
|
| 555 |
+
return []
|
| 556 |
+
rows = con.execute(f'SELECT {", ".join(DEPENDENCY_COLUMNS)} FROM {self.t(DEPENDENCIES)} '
|
| 557 |
+
'WHERE tenant = %s AND project_id = %s ORDER BY id',
|
| 558 |
+
(self.tenant_slug, project_id)).fetchall()
|
| 559 |
+
return [dict(zip(DEPENDENCY_COLUMNS, row)) for row in rows]
|
| 560 |
+
|
| 561 |
+
def list_dependencies(self, project_id):
|
| 562 |
+
with self.projects._connection() as con:
|
| 563 |
+
self._project(con, project_id)
|
| 564 |
+
return self._dependencies(con, project_id)
|
| 565 |
+
|
| 566 |
+
def add_dependency(self, record):
|
| 567 |
+
with self._write(record['project_id']) as (con, project):
|
| 568 |
+
rows = {row['id']: row for row in self._rows(con, record['project_id'])}
|
| 569 |
+
dependencies = self._dependencies(con, record['project_id'])
|
| 570 |
+
_check_dependency(rows, dependencies, record['from_task'], record['to_task'])
|
| 571 |
+
if any(all(edge[field] == record[field] for field in ('from_task', 'to_task', 'type'))
|
| 572 |
+
for edge in dependencies):
|
| 573 |
+
raise ValueError('This task dependency already exists.')
|
| 574 |
+
shifted = _propagate(rows, [*dependencies, record], record['to_task'])
|
| 575 |
+
columns = ('tenant', *DEPENDENCY_COLUMNS)
|
| 576 |
+
con.execute(f'INSERT INTO {self.t(DEPENDENCIES)} ({", ".join(columns)}) VALUES '
|
| 577 |
+
f'({", ".join("%s" for _ in columns)})',
|
| 578 |
+
(self.tenant_slug, *(record[field] for field in DEPENDENCY_COLUMNS)))
|
| 579 |
+
for row in shifted:
|
| 580 |
+
self._save(con, row)
|
| 581 |
+
self._sync_progress(con, record['project_id'], project['progress'])
|
| 582 |
+
return dict(record, shifted=shifted)
|
| 583 |
+
|
| 584 |
+
def delete_dependency(self, project_id, dependency_id):
|
| 585 |
+
with self._write(project_id) as (con, _project):
|
| 586 |
+
record = next((row for row in self._dependencies(con, project_id)
|
| 587 |
+
if row['id'] == dependency_id), None)
|
| 588 |
+
if record is None:
|
| 589 |
+
raise KeyError('Dependency not found in this project.')
|
| 590 |
+
con.execute(f'DELETE FROM {self.t(DEPENDENCIES)} '
|
| 591 |
+
'WHERE tenant = %s AND project_id = %s AND id = %s',
|
| 592 |
+
(self.tenant_slug, project_id, dependency_id))
|
| 593 |
+
return record
|
| 594 |
+
|
| 595 |
+
def reschedule(self, task_id, values):
|
| 596 |
+
with self.projects._connection() as con:
|
| 597 |
+
found = (con.execute(f'SELECT project_id FROM {self.t(TASKS)} WHERE tenant = %s AND id = %s',
|
| 598 |
+
(self.tenant_slug, task_id)).fetchone() if self._exists(con) else None)
|
| 599 |
+
if found is None:
|
| 600 |
+
raise KeyError('Task not found in this tenant.')
|
| 601 |
+
with self._write(found[0]) as (con, project):
|
| 602 |
+
rows = {row['id']: row for row in self._rows(con, found[0])}
|
| 603 |
+
if task_id not in rows:
|
| 604 |
+
raise KeyError('Task not found in this project.')
|
| 605 |
+
old = dict(rows)
|
| 606 |
+
rows[task_id] = _fields(values, rows[task_id])
|
| 607 |
+
rows[task_id]['done'] = rows[task_id]['done_at'] is not None
|
| 608 |
+
shifted = _propagate(rows, self._dependencies(con, found[0]), task_id)
|
| 609 |
+
shifted = [row for row in shifted if row['id'] != task_id]
|
| 610 |
+
for row in [rows[task_id], *shifted]:
|
| 611 |
+
self._save(con, row)
|
| 612 |
+
self._sync_progress(con, found[0], project['progress'])
|
| 613 |
+
before = {'task': old[task_id], 'shifted': [old[row['id']] for row in shifted]}
|
| 614 |
+
return before, {'task': rows[task_id], 'shifted': shifted}
|
| 615 |
+
|
| 616 |
+
def progress(self, project_id):
|
| 617 |
+
with self.projects._connection() as con:
|
| 618 |
+
project = self._project(con, project_id)
|
| 619 |
+
rows = self._rows(con, project_id) if self._exists(con) else []
|
| 620 |
+
return _leaf_progress(rows, project['progress'])
|
| 621 |
+
|
| 622 |
+
def _next_position(self, rows, parent_id, task_id):
|
| 623 |
+
positions = [_position(row['position']) for row in rows.values()
|
| 624 |
+
if row['parent_id'] == parent_id and row['id'] != task_id]
|
| 625 |
+
last = max(positions, default=Decimal(0))
|
| 626 |
+
with localcontext() as context:
|
| 627 |
+
context.prec = max(28, len(last.as_tuple().digits) + abs(last.as_tuple().exponent) + 2)
|
| 628 |
+
return str(last + 1)
|
| 629 |
+
|
| 630 |
+
def _save(self, con, record, *, insert=False):
|
| 631 |
+
if insert:
|
| 632 |
+
columns = ('tenant', *TASK_COLUMNS)
|
| 633 |
+
con.execute(f'INSERT INTO {self.t(TASKS)} ({", ".join(columns)}) VALUES '
|
| 634 |
+
f'({", ".join("%s" for _ in columns)})',
|
| 635 |
+
(self.tenant_slug, *(record[field] for field in TASK_COLUMNS)))
|
| 636 |
+
else:
|
| 637 |
+
fields = tuple(field for field in TASK_COLUMNS if field not in ('id', 'project_id', 'created_by'))
|
| 638 |
+
con.execute(f'UPDATE {self.t(TASKS)} SET '
|
| 639 |
+
+ ', '.join(f'{field} = %s' for field in fields)
|
| 640 |
+
+ ' WHERE tenant = %s AND project_id = %s AND id = %s',
|
| 641 |
+
(*(record[field] for field in fields), self.tenant_slug, record['project_id'], record['id']))
|
| 642 |
+
if any(field in record for field in TARGET_FIELDS):
|
| 643 |
+
config = {field: record[field] for field in (*TARGET_FIELDS, *FILTER_STATE, 'filter_by') if field in record}
|
| 644 |
+
con.execute(f'INSERT INTO {self.t(FILTERS)} (tenant, id, project_id, config) VALUES (%s, %s, %s, %s) '
|
| 645 |
+
'ON CONFLICT (tenant, id) DO UPDATE SET config = excluded.config',
|
| 646 |
+
(self.tenant_slug, record['id'], record['project_id'], json.dumps(config, allow_nan=False)))
|
| 647 |
+
|
| 648 |
+
def _delete(self, con, project_id, ids):
|
| 649 |
+
# Delete descendants first so database cascade triggers add no depth ceiling.
|
| 650 |
+
for part in projects._chunks(list(reversed(ids))):
|
| 651 |
+
marks = ', '.join('%s' for _ in part)
|
| 652 |
+
args = (self.tenant_slug, project_id, *part)
|
| 653 |
+
self._delete_dependencies(con, project_id, part)
|
| 654 |
+
con.execute(f'DELETE FROM {self.t(FILTERS)} '
|
| 655 |
+
f'WHERE tenant = %s AND project_id = %s AND id IN ({marks})', args)
|
| 656 |
+
con.execute(f'UPDATE {self.t(TASKS)} SET contingency_for = NULL '
|
| 657 |
+
f'WHERE tenant = %s AND project_id = %s AND contingency_for IN ({marks})', args)
|
| 658 |
+
con.execute(f'DELETE FROM {self.t(TASKS)} '
|
| 659 |
+
f'WHERE tenant = %s AND project_id = %s AND id IN ({marks})', args)
|
| 660 |
+
|
| 661 |
+
def _delete_dependencies(self, con, project_id, ids):
|
| 662 |
+
marks = ', '.join('%s' for _ in ids)
|
| 663 |
+
con.execute(f'DELETE FROM {self.t(DEPENDENCIES)} '
|
| 664 |
+
f'WHERE tenant = %s AND project_id = %s '
|
| 665 |
+
f'AND (from_task IN ({marks}) OR to_task IN ({marks}))',
|
| 666 |
+
(self.tenant_slug, project_id, *ids, *ids))
|
| 667 |
+
|
| 668 |
+
def _sync_progress(self, con, project_id, fallback):
|
| 669 |
+
result = _leaf_progress(self._rows(con, project_id), fallback)
|
| 670 |
+
con.execute(f'UPDATE {self.t(projects.PROJECTS)} SET progress = %s, updated_at = %s '
|
| 671 |
+
'WHERE tenant = %s AND id = %s',
|
| 672 |
+
(result['progress'], projects._now(), self.tenant_slug, project_id))
|
| 673 |
+
return result
|
| 674 |
+
|
| 675 |
+
def mutate(self, project_id, task_id, action, values, by):
|
| 676 |
+
with self._write(project_id) as (con, project):
|
| 677 |
+
rows = {row['id']: row for row in self._rows(con, project_id)}
|
| 678 |
+
before = rows.get(task_id)
|
| 679 |
+
if action != 'create' and before is None:
|
| 680 |
+
raise KeyError('Task not found in this project.')
|
| 681 |
+
if action == 'delete':
|
| 682 |
+
ids = _subtree(rows, task_id)
|
| 683 |
+
self._delete(con, project_id, ids)
|
| 684 |
+
counts = self._sync_progress(con, project_id, project['progress'])
|
| 685 |
+
return before, None, {'deleted': task_id, 'task_ids': ids, 'progress': counts['progress']}
|
| 686 |
+
if action == 'create':
|
| 687 |
+
after = dict(values, id=task_id, project_id=project_id,
|
| 688 |
+
active=values['contingency_for'] is None)
|
| 689 |
+
elif action == 'update':
|
| 690 |
+
after = _fields(values, before)
|
| 691 |
+
if after['contingency_for'] != before['contingency_for'] and after['contingency_for'] is not None:
|
| 692 |
+
after['active'] = False
|
| 693 |
+
elif action == 'move':
|
| 694 |
+
after = dict(before, **values)
|
| 695 |
+
elif action == 'activate':
|
| 696 |
+
after = dict(before, active=True)
|
| 697 |
+
else:
|
| 698 |
+
raise ValueError('Unknown task change.')
|
| 699 |
+
if set(TARGET_FIELDS) & values.keys():
|
| 700 |
+
if before and before.get('filter_closed_at') and values.get('done') is not False:
|
| 701 |
+
raise ValueError('Reopen and reset the task before changing its targets or filter.')
|
| 702 |
+
after['filter_by'] = by
|
| 703 |
+
for field in FILTER_STATE:
|
| 704 |
+
after.pop(field, None)
|
| 705 |
+
if values.get('done') is False:
|
| 706 |
+
for field in FILTER_STATE:
|
| 707 |
+
after.pop(field, None)
|
| 708 |
+
if after.get('completion_filter') is not None:
|
| 709 |
+
after['filter_by'] = by
|
| 710 |
+
_check_parent(rows, task_id, after['parent_id'])
|
| 711 |
+
_check_contingency(rows, task_id, after['contingency_for'])
|
| 712 |
+
if after['position'] is None:
|
| 713 |
+
after['position'] = self._next_position(rows, after['parent_id'], task_id)
|
| 714 |
+
shifted = []
|
| 715 |
+
if action == 'update' and {'start_on', 'due_on'} & values.keys():
|
| 716 |
+
rows[task_id] = after
|
| 717 |
+
shifted = _propagate(rows, self._dependencies(con, project_id), task_id)
|
| 718 |
+
after = rows[task_id]
|
| 719 |
+
self._save(con, after, insert=action == 'create')
|
| 720 |
+
for row in shifted:
|
| 721 |
+
if row['id'] != task_id:
|
| 722 |
+
self._save(con, row)
|
| 723 |
+
self._sync_progress(con, project_id, project['progress'])
|
| 724 |
+
saved = next(row for row in self._rows(con, project_id) if row['id'] == task_id)
|
| 725 |
+
return before, saved, saved
|
platform/core/amazon_projects.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
"""Per-tenant PostgreSQL Amazon projects, each assigned to many SKUs, ASINs and Parent ASINs.
|
| 2 |
|
| 3 |
No request reads the `user_tables` document for a project (C6). Core imports no harness module
|
| 4 |
-
(verify_layering) and no App module (rule 13).
|
| 5 |
A door called with `by` appends one `core.activity_log` event per create, field or target change.
|
| 6 |
"""
|
| 7 |
|
|
@@ -9,13 +9,16 @@ from contextlib import contextmanager
|
|
| 9 |
from datetime import date, datetime, timezone
|
| 10 |
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
| 11 |
import json
|
|
|
|
|
|
|
| 12 |
import re
|
| 13 |
import sys
|
| 14 |
import uuid
|
| 15 |
|
| 16 |
PROJECTS = 'amazon_projects'
|
| 17 |
TARGETS = 'amazon_project_targets'
|
| 18 |
-
|
|
|
|
| 19 |
|
| 20 |
GRAINS = ('sku', 'asin', 'parent_asin')
|
| 21 |
#: R18: the fixed status set of SDLC 82.
|
|
@@ -32,7 +35,7 @@ PROJECT_COLUMNS = ('id', 'tenant', 'name', 'status', 'priority', 'owner', 'start
|
|
| 32 |
'created_by')
|
| 33 |
TARGET_COLUMNS = ('project_id', 'grain', 'key', 'added_at')
|
| 34 |
EDITABLE = frozenset(('name', 'status', 'priority', 'owner', 'start_on', 'target_on', 'progress', 'notes',
|
| 35 |
-
'completed_on', 'result_configs'))
|
| 36 |
RESULT_CONFIGS_KEY = '_resultConfigs'
|
| 37 |
DATES = ('start_on', 'target_on', 'completed_on')
|
| 38 |
NOTES_BYTES = 65536
|
|
@@ -45,6 +48,11 @@ ACTIVITY_FIELDS = {'name': 'name', 'priority': 'priority', 'owner': 'owner', 'st
|
|
| 45 |
'target_on': 'targetOn', 'progress': 'progress', 'notes': 'notes',
|
| 46 |
'completed_on': 'completedOn', 'result_configs': 'resultConfigs', 'status': 'status'}
|
| 47 |
GRAIN_LABELS = {'sku': 'SKU', 'asin': 'ASIN', 'parent_asin': 'Parent ASIN'}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
|
| 49 |
#: R39: an old `ut_amazon_projects` status becomes (status, progress).
|
| 50 |
MOVE_STATUS = {'Open': ('In progress', 0), 'Done': ('Completed', 100)}
|
|
@@ -81,6 +89,19 @@ def owner_identity(value, tenant=None):
|
|
| 81 |
return None
|
| 82 |
|
| 83 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
def _in_sql(values):
|
| 85 |
return ', '.join(f"'{value}'" for value in values)
|
| 86 |
|
|
@@ -160,10 +181,11 @@ def target(grain, key):
|
|
| 160 |
|
| 161 |
def _targets(values):
|
| 162 |
"""Unique (grain, key) pairs in first-seen order, from dicts or pairs."""
|
| 163 |
-
out = []
|
| 164 |
for value in values or ():
|
| 165 |
pair = _target(value.get('grain'), value.get('key')) if isinstance(value, dict) else _target(*value)
|
| 166 |
-
if pair not in
|
|
|
|
| 167 |
out.append(pair)
|
| 168 |
return out
|
| 169 |
|
|
@@ -179,6 +201,8 @@ def _fields(values, *, previous=None):
|
|
| 179 |
result['progress'] = _progress(result.get('progress'))
|
| 180 |
for field in DATES:
|
| 181 |
result[field] = _date(result.get(field), field)
|
|
|
|
|
|
|
| 182 |
from core import amazon_project_results
|
| 183 |
result['result_configs'] = amazon_project_results.configurations(result.get('result_configs', ()))
|
| 184 |
if (values.get('completed_on') is not None and result['status'] != COMPLETED):
|
|
@@ -216,6 +240,9 @@ def _record(values, columns=PROJECT_COLUMNS):
|
|
| 216 |
value = values[offset] if len(values) > offset else None
|
| 217 |
result[field] = json.loads(value) if isinstance(value, str) else value
|
| 218 |
result['result_configs'] = (result.get('impact') or {}).get(RESULT_CONFIGS_KEY, [])
|
|
|
|
|
|
|
|
|
|
| 219 |
return result
|
| 220 |
|
| 221 |
|
|
@@ -262,12 +289,13 @@ def summary(projects):
|
|
| 262 |
|
| 263 |
def create(rt, *, name, status='Backlog', priority='No priority', owner=None, start_on=None,
|
| 264 |
target_on=None, progress=0, notes=None, targets=(), created_by=None,
|
| 265 |
-
completed_on=None, source_id=None, by=None, result_configs=()):
|
| 266 |
"""Insert one project and its targets in one transaction. `source_id` is unique per tenant."""
|
| 267 |
values = _fields(dict(name=name, status=status, priority=priority, owner=owner, start_on=start_on,
|
| 268 |
target_on=target_on, progress=progress, notes=notes, completed_on=completed_on,
|
| 269 |
result_configs=result_configs))
|
| 270 |
backend = _backend(rt)
|
|
|
|
| 271 |
stamp = _now()
|
| 272 |
values.update(id=uuid.uuid4().hex, tenant=backend.tenant_slug, created_at=stamp, updated_at=stamp,
|
| 273 |
created_by=_text(created_by or by, 'created_by', optional=True),
|
|
@@ -275,6 +303,7 @@ def create(rt, *, name, status='Backlog', priority='No priority', owner=None, st
|
|
| 275 |
record = backend.insert(values, _targets(targets), stamp)
|
| 276 |
if by is not None:
|
| 277 |
_log(rt, record['id'], by, 'project', None, record['name'])
|
|
|
|
| 278 |
return record
|
| 279 |
|
| 280 |
|
|
@@ -298,11 +327,12 @@ def update(rt, project_id, changes, by=None):
|
|
| 298 |
continue
|
| 299 |
if before[field] != after[field]:
|
| 300 |
_log(rt, project_id, by, name, before[field], after[field])
|
|
|
|
| 301 |
return after
|
| 302 |
|
| 303 |
|
| 304 |
def delete(rt, project_id, by=None):
|
| 305 |
-
"""Delete one project
|
| 306 |
project_id = _text(project_id, 'project_id')
|
| 307 |
backend = _backend(rt)
|
| 308 |
name = backend.get(project_id)['name'] if by is not None else None
|
|
@@ -558,6 +588,12 @@ def _log(rt, project_id, by, field, before, after):
|
|
| 558 |
"""Append one project event. A failed append is logged, never undoes the committed write."""
|
| 559 |
from core import activity_log
|
| 560 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 561 |
activity_log.append(rt, kind='app', object={'app': ACTIVITY_KEY, 'id': project_id},
|
| 562 |
by=_text(by, 'by'), changes={field: {'before': before, 'after': after}})
|
| 563 |
except Exception as exc: # noqa: BLE001
|
|
@@ -565,6 +601,16 @@ def _log(rt, project_id, by, field, before, after):
|
|
| 565 |
file=sys.stderr)
|
| 566 |
|
| 567 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 568 |
def _shown(value):
|
| 569 |
"""A target list as one line of text, else the value."""
|
| 570 |
if isinstance(value, list):
|
|
@@ -572,30 +618,254 @@ def _shown(value):
|
|
| 572 |
return ', '.join(v['metric'] for v in value)
|
| 573 |
return ', '.join(f'{GRAIN_LABELS.get(t.get("grain"), t.get("grain"))} {t.get("key")}'
|
| 574 |
for t in value if isinstance(t, dict))
|
|
|
|
|
|
|
| 575 |
identity = owner_identity(value)
|
| 576 |
return identity['name'] if identity else value
|
| 577 |
|
| 578 |
|
| 579 |
-
def
|
| 580 |
-
"""
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 584 |
out = []
|
| 585 |
-
for
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 596 |
return out
|
| 597 |
|
| 598 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 599 |
# --------------------------------------------------------------------------- the move (R39)
|
| 600 |
|
| 601 |
def _loose_date(value):
|
|
@@ -815,7 +1085,7 @@ class PgProjectStore:
|
|
| 815 |
return con.execute('SELECT to_regclass(%s)', (self.t(PROJECTS),)).fetchone()[0] is not None
|
| 816 |
|
| 817 |
def ensure_schema(self, con):
|
| 818 |
-
"""Create
|
| 819 |
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(PROJECTS)} (
|
| 820 |
id TEXT NOT NULL, tenant TEXT NOT NULL, name TEXT NOT NULL,
|
| 821 |
status TEXT NOT NULL {STATUS_CHECK},
|
|
@@ -834,6 +1104,11 @@ class PgProjectStore:
|
|
| 834 |
key TEXT NOT NULL, added_at TEXT NOT NULL,
|
| 835 |
PRIMARY KEY (tenant, project_id, grain, key),
|
| 836 |
FOREIGN KEY (tenant, project_id) REFERENCES {self.t(PROJECTS)} (tenant, id) ON DELETE CASCADE)''')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 837 |
# One row's lookup is one indexed read on this index.
|
| 838 |
con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" '
|
| 839 |
f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)')
|
|
@@ -847,19 +1122,26 @@ class PgProjectStore:
|
|
| 847 |
self.ensure_schema(con)
|
| 848 |
yield con
|
| 849 |
|
| 850 |
-
def _select_sql(self, where, prefix=''):
|
| 851 |
cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS)
|
| 852 |
count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
|
| 853 |
'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
|
| 854 |
-
return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()} '
|
| 855 |
f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}')
|
| 856 |
|
| 857 |
def _rollup_projection(self):
|
| 858 |
# Old schemas read blank figures until a guarded write adds the columns.
|
| 859 |
return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'"
|
| 860 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 861 |
def _one(self, con, project_id, lock=False):
|
| 862 |
-
row = con.execute(self._select_sql(' AND p.id = %s'
|
|
|
|
| 863 |
(self.tenant_slug, project_id)).fetchone()
|
| 864 |
return _record(row)
|
| 865 |
|
|
@@ -870,12 +1152,25 @@ class PgProjectStore:
|
|
| 870 |
|
| 871 |
def _insert_targets(self, con, project_id, pairs, stamp):
|
| 872 |
cols = ('tenant',) + TARGET_COLUMNS
|
| 873 |
-
for part in _chunks(pairs
|
| 874 |
con.execute(f'INSERT INTO {self.t(TARGETS)} ({", ".join(cols)}) VALUES '
|
| 875 |
+ ', '.join(f'({_marks(len(cols))})' for _ in part)
|
| 876 |
+ ' ON CONFLICT (tenant, project_id, grain, key) DO NOTHING',
|
| 877 |
tuple(v for grain, key in part for v in (self.tenant_slug, project_id, grain, key, stamp)))
|
| 878 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 879 |
def _touch(self, con, project_id, stamp):
|
| 880 |
current = self._one(con, project_id, lock=True)
|
| 881 |
impact = _with_result_configs(None, current['result_configs'])
|
|
@@ -891,6 +1186,7 @@ class PgProjectStore:
|
|
| 891 |
con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) '
|
| 892 |
f'VALUES ({_marks(len(columns) - 1)}, %s::jsonb)', values)
|
| 893 |
self._insert_targets(con, record['id'], pairs, stamp)
|
|
|
|
| 894 |
return self._one(con, record['id'])
|
| 895 |
|
| 896 |
def get(self, project_id):
|
|
@@ -903,7 +1199,9 @@ class PgProjectStore:
|
|
| 903 |
with self._write() as con:
|
| 904 |
current = self._one(con, project_id, lock=True)
|
| 905 |
candidate = _fields(changes, previous=current)
|
| 906 |
-
|
|
|
|
|
|
|
| 907 |
changed_anchor = any(candidate[k] != current[k] for k in ('status', 'completed_on', 'result_configs'))
|
| 908 |
impact = _with_result_configs(None if changed_anchor else current['impact'], candidate['result_configs'])
|
| 909 |
con.execute(f'UPDATE {self.t(PROJECTS)} SET '
|
|
@@ -911,10 +1209,14 @@ class PgProjectStore:
|
|
| 911 |
+ ' WHERE tenant = %s AND id = %s',
|
| 912 |
tuple(candidate[field] for field in fields)
|
| 913 |
+ (stamp, json.dumps(impact), self.tenant_slug, project_id))
|
|
|
|
|
|
|
| 914 |
return self._one(con, project_id)
|
| 915 |
|
| 916 |
def delete(self, project_id):
|
| 917 |
with self._write() as con:
|
|
|
|
|
|
|
| 918 |
con.execute(f'DELETE FROM {self.t(TARGETS)} WHERE tenant = %s AND project_id = %s',
|
| 919 |
(self.tenant_slug, project_id))
|
| 920 |
row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id',
|
|
@@ -927,7 +1229,9 @@ class PgProjectStore:
|
|
| 927 |
with self._connection() as con:
|
| 928 |
if not self._exists(con):
|
| 929 |
return []
|
| 930 |
-
rows = con.execute(self._select_sql(' ORDER BY p.created_at, p.id'
|
|
|
|
|
|
|
| 931 |
return [_record(row) for row in rows]
|
| 932 |
|
| 933 |
def targets(self, project_id):
|
|
@@ -942,13 +1246,14 @@ class PgProjectStore:
|
|
| 942 |
with self._connection() as con:
|
| 943 |
if not self._exists(con):
|
| 944 |
return out
|
|
|
|
| 945 |
for part in _chunks(pairs):
|
| 946 |
match = ' OR '.join(['(t.grain = %s AND t.key = %s)'] * len(part))
|
| 947 |
cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS)
|
| 948 |
count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
|
| 949 |
'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
|
| 950 |
rows = con.execute(
|
| 951 |
-
f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()} '
|
| 952 |
f'FROM {self.t(TARGETS)} AS t '
|
| 953 |
f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id '
|
| 954 |
f'WHERE t.tenant = %s AND ({match}) ORDER BY p.created_at, p.id',
|
|
|
|
| 1 |
"""Per-tenant PostgreSQL Amazon projects, each assigned to many SKUs, ASINs and Parent ASINs.
|
| 2 |
|
| 3 |
No request reads the `user_tables` document for a project (C6). Core imports no harness module
|
| 4 |
+
(verify_layering) and no App module (rule 13). Tables are created on the first write.
|
| 5 |
A door called with `by` appends one `core.activity_log` event per create, field or target change.
|
| 6 |
"""
|
| 7 |
|
|
|
|
| 9 |
from datetime import date, datetime, timezone
|
| 10 |
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
| 11 |
import json
|
| 12 |
+
import hashlib
|
| 13 |
+
import heapq
|
| 14 |
import re
|
| 15 |
import sys
|
| 16 |
import uuid
|
| 17 |
|
| 18 |
PROJECTS = 'amazon_projects'
|
| 19 |
TARGETS = 'amazon_project_targets'
|
| 20 |
+
MEMBERS = 'amazon_project_members'
|
| 21 |
+
TABLES = (PROJECTS, TARGETS, MEMBERS)
|
| 22 |
|
| 23 |
GRAINS = ('sku', 'asin', 'parent_asin')
|
| 24 |
#: R18: the fixed status set of SDLC 82.
|
|
|
|
| 35 |
'created_by')
|
| 36 |
TARGET_COLUMNS = ('project_id', 'grain', 'key', 'added_at')
|
| 37 |
EDITABLE = frozenset(('name', 'status', 'priority', 'owner', 'start_on', 'target_on', 'progress', 'notes',
|
| 38 |
+
'completed_on', 'result_configs', 'contributors'))
|
| 39 |
RESULT_CONFIGS_KEY = '_resultConfigs'
|
| 40 |
DATES = ('start_on', 'target_on', 'completed_on')
|
| 41 |
NOTES_BYTES = 65536
|
|
|
|
| 48 |
'target_on': 'targetOn', 'progress': 'progress', 'notes': 'notes',
|
| 49 |
'completed_on': 'completedOn', 'result_configs': 'resultConfigs', 'status': 'status'}
|
| 50 |
GRAIN_LABELS = {'sku': 'SKU', 'asin': 'ASIN', 'parent_asin': 'Parent ASIN'}
|
| 51 |
+
LISTING_ACTIVITY_FIELDS = {'title': 'Title', 'item_name': 'Title', 'image': 'Main image',
|
| 52 |
+
'main_image': 'Main image', 'main_image_url': 'Main image',
|
| 53 |
+
'main_product_image_locator': 'Main image',
|
| 54 |
+
'description': 'Description', 'product_description': 'Description',
|
| 55 |
+
'bullet_point': 'Bullet points'}
|
| 56 |
|
| 57 |
#: R39: an old `ut_amazon_projects` status becomes (status, progress).
|
| 58 |
MOVE_STATUS = {'Open': ('In progress', 0), 'Done': ('Completed', 100)}
|
|
|
|
| 89 |
return None
|
| 90 |
|
| 91 |
|
| 92 |
+
def _member_values(values, tenant):
|
| 93 |
+
"""Normalize unique contributor identities bound to the project's tenant."""
|
| 94 |
+
if not isinstance(values, (list, tuple)):
|
| 95 |
+
raise ValueError('Contributors must be a list of workspace users.')
|
| 96 |
+
members = {}
|
| 97 |
+
for identity in values:
|
| 98 |
+
if not isinstance(identity, dict) or identity.get('tenant') != tenant:
|
| 99 |
+
raise ValueError('Each contributor must belong to this workspace.')
|
| 100 |
+
canonical = owner_identity(owner_value(tenant, identity), tenant)
|
| 101 |
+
members[canonical['username']] = {key: canonical[key] for key in ('username', 'name')}
|
| 102 |
+
return sorted(members.values(), key=lambda member: member['username'])
|
| 103 |
+
|
| 104 |
+
|
| 105 |
def _in_sql(values):
|
| 106 |
return ', '.join(f"'{value}'" for value in values)
|
| 107 |
|
|
|
|
| 181 |
|
| 182 |
def _targets(values):
|
| 183 |
"""Unique (grain, key) pairs in first-seen order, from dicts or pairs."""
|
| 184 |
+
out, seen = [], set()
|
| 185 |
for value in values or ():
|
| 186 |
pair = _target(value.get('grain'), value.get('key')) if isinstance(value, dict) else _target(*value)
|
| 187 |
+
if pair not in seen:
|
| 188 |
+
seen.add(pair)
|
| 189 |
out.append(pair)
|
| 190 |
return out
|
| 191 |
|
|
|
|
| 201 |
result['progress'] = _progress(result.get('progress'))
|
| 202 |
for field in DATES:
|
| 203 |
result[field] = _date(result.get(field), field)
|
| 204 |
+
if result['start_on'] is not None and result['target_on'] is not None and result['target_on'] < result['start_on']:
|
| 205 |
+
raise ValueError('Target date cannot be before the start date.')
|
| 206 |
from core import amazon_project_results
|
| 207 |
result['result_configs'] = amazon_project_results.configurations(result.get('result_configs', ()))
|
| 208 |
if (values.get('completed_on') is not None and result['status'] != COMPLETED):
|
|
|
|
| 240 |
value = values[offset] if len(values) > offset else None
|
| 241 |
result[field] = json.loads(value) if isinstance(value, str) else value
|
| 242 |
result['result_configs'] = (result.get('impact') or {}).get(RESULT_CONFIGS_KEY, [])
|
| 243 |
+
value = values[len(columns) + 3] if len(values) > len(columns) + 3 else None
|
| 244 |
+
members = json.loads(value) if isinstance(value, str) else value
|
| 245 |
+
result['contributors'] = sorted(members or [], key=lambda member: member['username'])
|
| 246 |
return result
|
| 247 |
|
| 248 |
|
|
|
|
| 289 |
|
| 290 |
def create(rt, *, name, status='Backlog', priority='No priority', owner=None, start_on=None,
|
| 291 |
target_on=None, progress=0, notes=None, targets=(), created_by=None,
|
| 292 |
+
completed_on=None, source_id=None, by=None, result_configs=(), contributors=()):
|
| 293 |
"""Insert one project and its targets in one transaction. `source_id` is unique per tenant."""
|
| 294 |
values = _fields(dict(name=name, status=status, priority=priority, owner=owner, start_on=start_on,
|
| 295 |
target_on=target_on, progress=progress, notes=notes, completed_on=completed_on,
|
| 296 |
result_configs=result_configs))
|
| 297 |
backend = _backend(rt)
|
| 298 |
+
values['contributors'] = _member_values(contributors, backend.tenant_slug)
|
| 299 |
stamp = _now()
|
| 300 |
values.update(id=uuid.uuid4().hex, tenant=backend.tenant_slug, created_at=stamp, updated_at=stamp,
|
| 301 |
created_by=_text(created_by or by, 'created_by', optional=True),
|
|
|
|
| 303 |
record = backend.insert(values, _targets(targets), stamp)
|
| 304 |
if by is not None:
|
| 305 |
_log(rt, record['id'], by, 'project', None, record['name'])
|
| 306 |
+
_log_members(rt, record['id'], by, [], record['contributors'])
|
| 307 |
return record
|
| 308 |
|
| 309 |
|
|
|
|
| 327 |
continue
|
| 328 |
if before[field] != after[field]:
|
| 329 |
_log(rt, project_id, by, name, before[field], after[field])
|
| 330 |
+
_log_members(rt, project_id, by, before['contributors'], after['contributors'])
|
| 331 |
return after
|
| 332 |
|
| 333 |
|
| 334 |
def delete(rt, project_id, by=None):
|
| 335 |
+
"""Delete one project, its targets and its contributors in one transaction."""
|
| 336 |
project_id = _text(project_id, 'project_id')
|
| 337 |
backend = _backend(rt)
|
| 338 |
name = backend.get(project_id)['name'] if by is not None else None
|
|
|
|
| 588 |
"""Append one project event. A failed append is logged, never undoes the committed write."""
|
| 589 |
from core import activity_log
|
| 590 |
try:
|
| 591 |
+
if field == 'dependencies':
|
| 592 |
+
from core import amazon_project_tasks
|
| 593 |
+
names = {task['id']: task['name'] for task in amazon_project_tasks.list_tasks(rt, project_id)}
|
| 594 |
+
before, after = [dict(value, from_name=names.get(value['from_task'], value['from_task']),
|
| 595 |
+
to_name=names.get(value['to_task'], value['to_task']))
|
| 596 |
+
if isinstance(value, dict) else value for value in (before, after)]
|
| 597 |
activity_log.append(rt, kind='app', object={'app': ACTIVITY_KEY, 'id': project_id},
|
| 598 |
by=_text(by, 'by'), changes={field: {'before': before, 'after': after}})
|
| 599 |
except Exception as exc: # noqa: BLE001
|
|
|
|
| 601 |
file=sys.stderr)
|
| 602 |
|
| 603 |
|
| 604 |
+
def _log_members(rt, project_id, by, before, after):
|
| 605 |
+
"""Append one event for each contributor username added or removed."""
|
| 606 |
+
old = {member['username']: member for member in before}
|
| 607 |
+
new = {member['username']: member for member in after}
|
| 608 |
+
for username in sorted(old.keys() - new.keys()):
|
| 609 |
+
_log(rt, project_id, by, 'contributors', old[username], None)
|
| 610 |
+
for username in sorted(new.keys() - old.keys()):
|
| 611 |
+
_log(rt, project_id, by, 'contributors', None, new[username])
|
| 612 |
+
|
| 613 |
+
|
| 614 |
def _shown(value):
|
| 615 |
"""A target list as one line of text, else the value."""
|
| 616 |
if isinstance(value, list):
|
|
|
|
| 618 |
return ', '.join(v['metric'] for v in value)
|
| 619 |
return ', '.join(f'{GRAIN_LABELS.get(t.get("grain"), t.get("grain"))} {t.get("key")}'
|
| 620 |
for t in value if isinstance(t, dict))
|
| 621 |
+
if isinstance(value, dict) and 'username' in value:
|
| 622 |
+
return value['name']
|
| 623 |
identity = owner_identity(value)
|
| 624 |
return identity['name'] if identity else value
|
| 625 |
|
| 626 |
|
| 627 |
+
def _activity_value(value, task_names):
|
| 628 |
+
"""Show task and dependency names without exposing their stored record payloads."""
|
| 629 |
+
if isinstance(value, dict):
|
| 630 |
+
task = value.get('task', value)
|
| 631 |
+
if isinstance(task, dict) and 'name' in task and 'username' not in task:
|
| 632 |
+
return task['name']
|
| 633 |
+
if 'from_task' in value and 'to_task' in value:
|
| 634 |
+
return ' -> '.join(value.get(key.replace('_task', '_name')) or task_names.get(value[key], value[key])
|
| 635 |
+
for key in ('from_task', 'to_task'))
|
| 636 |
+
return _shown(value)
|
| 637 |
+
|
| 638 |
+
|
| 639 |
+
TASK_ACTIVITY_FIELDS = ('name', 'assignee', 'start_on', 'due_on', 'done')
|
| 640 |
+
CLIPPED = ' [truncated]'
|
| 641 |
+
|
| 642 |
+
|
| 643 |
+
def _skip_space(text, position):
|
| 644 |
+
while position < len(text) and text[position].isspace():
|
| 645 |
+
position += 1
|
| 646 |
+
return position
|
| 647 |
+
|
| 648 |
+
|
| 649 |
+
def _clipped_members(preview):
|
| 650 |
+
"""Read the members of a clipped JSON object that end before the cut, entering an open task."""
|
| 651 |
+
decoder, out, position = json.JSONDecoder(), {}, 1
|
| 652 |
+
while True:
|
| 653 |
+
try:
|
| 654 |
+
key, position = decoder.raw_decode(preview, _skip_space(preview, position))
|
| 655 |
+
except ValueError:
|
| 656 |
+
return out
|
| 657 |
+
position = _skip_space(preview, position)
|
| 658 |
+
if not isinstance(key, str) or preview[position:position + 1] != ':':
|
| 659 |
+
return out
|
| 660 |
+
position = _skip_space(preview, position + 1)
|
| 661 |
+
try:
|
| 662 |
+
value, position = decoder.raw_decode(preview, position)
|
| 663 |
+
except ValueError:
|
| 664 |
+
# A reschedule logs {shifted, task}, so its task is often the member the cut opens.
|
| 665 |
+
return {'task': _clipped_members(preview[position:])} if (
|
| 666 |
+
key == 'task' and preview[position:position + 1] == '{') else out
|
| 667 |
+
position = _skip_space(preview, position)
|
| 668 |
+
# A scalar that runs into the cut may itself be cut, so keep it only before a comma or brace.
|
| 669 |
+
if preview[position:position + 1] not in (',', '}'):
|
| 670 |
+
return out
|
| 671 |
+
out[key] = value
|
| 672 |
+
if preview[position] == '}':
|
| 673 |
+
return out
|
| 674 |
+
position += 1
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
def _task_fields(value, truncated, actor_names):
|
| 678 |
+
"""One task side as its person-readable fields, and whether the log clipped it."""
|
| 679 |
+
clipped = bool(truncated) and isinstance(value, str) and value.endswith(CLIPPED)
|
| 680 |
+
if clipped:
|
| 681 |
+
preview = value[:-len(CLIPPED)]
|
| 682 |
+
value = _clipped_members(preview) if preview.startswith('{') else {}
|
| 683 |
+
if not isinstance(value, dict):
|
| 684 |
+
return {}, True
|
| 685 |
+
if 'name' not in value and isinstance(value.get('task'), dict):
|
| 686 |
+
value = value['task']
|
| 687 |
+
fields = {key: value[key] for key in TASK_ACTIVITY_FIELDS
|
| 688 |
+
if key in value and (value[key] is None or isinstance(value[key], (str, bool)))}
|
| 689 |
+
if isinstance(fields.get('assignee'), str):
|
| 690 |
+
fields['assignee'] = actor_names.get(fields['assignee'], fields['assignee'])
|
| 691 |
+
return fields, clipped
|
| 692 |
+
|
| 693 |
+
|
| 694 |
+
def _task_change(before, after, truncated, actor_names):
|
| 695 |
+
"""A changed task as its name and each readable field that differs, never its stored record."""
|
| 696 |
+
(old, old_cut), (new, new_cut) = (_task_fields(side, truncated, actor_names) for side in (before, after))
|
| 697 |
+
partial = old_cut or new_cut
|
| 698 |
+
changed = {key for key in TASK_ACTIVITY_FIELDS if (key in old and key in new and old[key] != new[key])
|
| 699 |
+
or (not partial and (key in old) != (key in new))}
|
| 700 |
+
sides = []
|
| 701 |
+
for fields, cut in ((old, old_cut), (new, new_cut)):
|
| 702 |
+
side = {key: value for key, value in fields.items() if key == 'name' or key in changed}
|
| 703 |
+
# The marker tells the client a clipped side compares only the fields both sides kept.
|
| 704 |
+
sides.append(json.dumps(side, ensure_ascii=False) + CLIPPED if cut else side)
|
| 705 |
+
return sides
|
| 706 |
+
|
| 707 |
+
|
| 708 |
+
def _activity_target_pair(change, truncated):
|
| 709 |
+
"""Recover an absent side only from a bounded JSON target-list pair."""
|
| 710 |
+
before, after = change.get('before'), change.get('after')
|
| 711 |
+
marker = ' [truncated]'
|
| 712 |
+
if truncated and all(isinstance(value, str) and value.endswith(marker) for value in (before, after)):
|
| 713 |
+
old, new = before[:-len(marker)], after[:-len(marker)]
|
| 714 |
+
if old == 'null' and new.startswith('['):
|
| 715 |
+
before = None
|
| 716 |
+
elif new == 'null' and old.startswith('['):
|
| 717 |
+
after = None
|
| 718 |
+
# Retain the preview marker because clipped arrays cannot establish a complete target list.
|
| 719 |
+
return before, after
|
| 720 |
+
|
| 721 |
+
|
| 722 |
+
def _activity_target_preview(value, truncated):
|
| 723 |
+
"""Show complete target identities from a bounded JSON array without guessing its missing tail."""
|
| 724 |
+
marker = ' [truncated]'
|
| 725 |
+
if not truncated or not isinstance(value, str) or not value.endswith(marker):
|
| 726 |
+
return value
|
| 727 |
+
preview = value[:-len(marker)]
|
| 728 |
+
if preview and not preview.startswith('['):
|
| 729 |
+
return value
|
| 730 |
+
decoder, targets, position = json.JSONDecoder(), [], 1
|
| 731 |
+
while position < len(preview):
|
| 732 |
+
while position < len(preview) and preview[position].isspace():
|
| 733 |
+
position += 1
|
| 734 |
+
try:
|
| 735 |
+
target, position = decoder.raw_decode(preview, position)
|
| 736 |
+
except ValueError:
|
| 737 |
+
break
|
| 738 |
+
if (not isinstance(target, dict) or target.get('grain') not in GRAIN_LABELS
|
| 739 |
+
or not isinstance(target.get('key'), str) or not target['key']):
|
| 740 |
+
break
|
| 741 |
+
targets.append(target)
|
| 742 |
+
while position < len(preview) and preview[position].isspace():
|
| 743 |
+
position += 1
|
| 744 |
+
if position == len(preview) or preview[position] != ',':
|
| 745 |
+
break
|
| 746 |
+
position += 1
|
| 747 |
+
shown = _shown(targets)
|
| 748 |
+
return f'{shown}. Target list incomplete' if shown else 'Target list incomplete'
|
| 749 |
+
|
| 750 |
+
|
| 751 |
+
def _activity_entries(event, actor_names, task_names):
|
| 752 |
+
"""Project one log event into named activity entries."""
|
| 753 |
out = []
|
| 754 |
+
for index, (field, change) in enumerate((event.get('changes') or {}).items()):
|
| 755 |
+
before, after = change.get('before'), change.get('after')
|
| 756 |
+
if field == 'project':
|
| 757 |
+
action, field = ('created' if before is None else 'deleted'), None
|
| 758 |
+
elif field == 'targets':
|
| 759 |
+
before, after = _activity_target_pair(change, event.get('changes_truncated'))
|
| 760 |
+
action = ('added' if before is None else 'removed' if after is None else 'changed') + ' targets'
|
| 761 |
+
before = _activity_target_preview(before, event.get('changes_truncated'))
|
| 762 |
+
after = _activity_target_preview(after, event.get('changes_truncated'))
|
| 763 |
+
elif field == 'contributors':
|
| 764 |
+
action = 'added contributor' if before is None else 'removed contributor'
|
| 765 |
+
elif field in ('tasks', 'dependencies'):
|
| 766 |
+
action = ('added' if before is None else 'removed' if after is None else 'changed') + ' ' + (
|
| 767 |
+
'task' if field == 'tasks' else 'dependency')
|
| 768 |
+
else:
|
| 769 |
+
action = 'changed'
|
| 770 |
+
if action == 'changed task':
|
| 771 |
+
shown = _task_change(before, after, event.get('changes_truncated'), actor_names)
|
| 772 |
+
else:
|
| 773 |
+
shown = [_activity_value(value, task_names) for value in (before, after)]
|
| 774 |
+
out.append({'id': f'loopable:{event["id"]}:{index}', 'at': event['occurred_at'],
|
| 775 |
+
'by': event['by'], 'byUsername': event['by'],
|
| 776 |
+
'byName': actor_names.get(event['by'], event['by']),
|
| 777 |
+
'action': action, 'field': field, 'from': shown[0], 'to': shown[1]})
|
| 778 |
return out
|
| 779 |
|
| 780 |
|
| 781 |
+
def activity_page(rt, project_id, *, cursor=None, limit=ACTIVITY_LIMIT, read_changes=None,
|
| 782 |
+
actor_names=None, task_names=None):
|
| 783 |
+
"""Merge bounded log pages with an injected C4 reader under one snapshot cutoff."""
|
| 784 |
+
from core import activity_log
|
| 785 |
+
if type(limit) is not int or not 1 <= limit <= ACTIVITY_LIMIT:
|
| 786 |
+
raise ValueError(f'limit must be between 1 and {ACTIVITY_LIMIT}.')
|
| 787 |
+
state = cursor or {'until': _now(), 'app': None, 'amazon': None}
|
| 788 |
+
if (not isinstance(state, dict) or set(state) != {'until', 'app', 'amazon'}
|
| 789 |
+
or not isinstance(state['until'], str)):
|
| 790 |
+
raise ValueError('Reload activity to continue.')
|
| 791 |
+
until = datetime.fromisoformat(state['until']).astimezone(timezone.utc).isoformat()
|
| 792 |
+
actor_names, task_names = actor_names or {}, task_names or {}
|
| 793 |
+
wanted = {'app': ACTIVITY_KEY, 'id': _text(project_id, 'project_id')}
|
| 794 |
+
|
| 795 |
+
def app_entries():
|
| 796 |
+
position = state['app']
|
| 797 |
+
# Re-read the boundary event because one event can contain several fields.
|
| 798 |
+
before = [position[0], position[1] + 1] if position else None
|
| 799 |
+
while True:
|
| 800 |
+
page = activity_log.read(rt, can_read_object=lambda obj: obj == wanted, object=wanted,
|
| 801 |
+
cursor=before, until=until, limit=limit)
|
| 802 |
+
for event in page['records']:
|
| 803 |
+
for index, entry in enumerate(_activity_entries(event, actor_names, task_names)):
|
| 804 |
+
at = entry['at']
|
| 805 |
+
next_position = [at, event['id'], index]
|
| 806 |
+
if position and at == position[0] and event['id'] == position[1] and index <= position[2]:
|
| 807 |
+
continue
|
| 808 |
+
yield ((at, 1, event['id'], -index), entry, 'app', next_position)
|
| 809 |
+
before = page['next_cursor']
|
| 810 |
+
if before is None:
|
| 811 |
+
return
|
| 812 |
+
|
| 813 |
+
def amazon_entries():
|
| 814 |
+
position = state['amazon']
|
| 815 |
+
skipped = 0
|
| 816 |
+
|
| 817 |
+
def candidates():
|
| 818 |
+
nonlocal skipped
|
| 819 |
+
for change in read_changes(None) if read_changes is not None else ():
|
| 820 |
+
at = datetime.fromisoformat(change['detected_at']).astimezone(timezone.utc).isoformat()
|
| 821 |
+
if at > until:
|
| 822 |
+
continue
|
| 823 |
+
identity = hashlib.sha256(json.dumps(change, sort_keys=True, default=str).encode()).hexdigest()
|
| 824 |
+
key = (at, identity)
|
| 825 |
+
if position:
|
| 826 |
+
if key > tuple(position[:2]):
|
| 827 |
+
continue
|
| 828 |
+
if key == tuple(position[:2]) and skipped < position[2]:
|
| 829 |
+
skipped += 1
|
| 830 |
+
continue
|
| 831 |
+
yield key, change
|
| 832 |
+
|
| 833 |
+
# C4 has no page argument, so retain only this response's bounded candidate window.
|
| 834 |
+
selected = heapq.nlargest(limit + 1, candidates(), key=lambda item: item[0])
|
| 835 |
+
previous, count = None, 0
|
| 836 |
+
for (at, identity), change in selected:
|
| 837 |
+
key = (at, identity)
|
| 838 |
+
count = count + 1 if key == previous else (
|
| 839 |
+
position[2] + 1 if position and key == tuple(position[:2]) else 1)
|
| 840 |
+
previous = key
|
| 841 |
+
attribute = change['attribute']
|
| 842 |
+
field = LISTING_ACTIVITY_FIELDS.get(attribute, attribute.replace('_', ' ').capitalize())
|
| 843 |
+
entry = {'id': f'amazon:{identity}:{count}', 'at': at, 'by': 'Amazon',
|
| 844 |
+
'byUsername': 'amazon', 'byName': 'Amazon', 'source': 'amazon',
|
| 845 |
+
'action': 'listing changed', 'field': field,
|
| 846 |
+
'from': change['before_value'], 'to': change['after_value'],
|
| 847 |
+
'attribute': change['attribute'], 'before': change['before_value'],
|
| 848 |
+
'after': change['after_value'], 'detectedAt': change['detected_at'],
|
| 849 |
+
'detectionSource': change['source'], 'sku': change['sku'], 'asin': change['asin'],
|
| 850 |
+
'marketplaceId': change['marketplace_id']}
|
| 851 |
+
yield ((at, 0, identity, -count), entry, 'amazon', [at, identity, count])
|
| 852 |
+
|
| 853 |
+
merged = heapq.merge(app_entries(), amazon_entries(), key=lambda item: item[0], reverse=True)
|
| 854 |
+
rows = []
|
| 855 |
+
next_state = dict(state)
|
| 856 |
+
for _key, entry, source, position in merged:
|
| 857 |
+
if len(rows) == limit:
|
| 858 |
+
return {'activity': rows, 'next_cursor': next_state}
|
| 859 |
+
rows.append(entry)
|
| 860 |
+
next_state[source] = position
|
| 861 |
+
return {'activity': rows, 'next_cursor': None}
|
| 862 |
+
|
| 863 |
+
|
| 864 |
+
def activity(rt, project_id, limit=ACTIVITY_LIMIT, **kwargs):
|
| 865 |
+
"""Return the first merged activity page for legacy project detail consumers."""
|
| 866 |
+
return activity_page(rt, project_id, limit=limit, **kwargs)['activity']
|
| 867 |
+
|
| 868 |
+
|
| 869 |
# --------------------------------------------------------------------------- the move (R39)
|
| 870 |
|
| 871 |
def _loose_date(value):
|
|
|
|
| 1085 |
return con.execute('SELECT to_regclass(%s)', (self.t(PROJECTS),)).fetchone()[0] is not None
|
| 1086 |
|
| 1087 |
def ensure_schema(self, con):
|
| 1088 |
+
"""Create project tables and their indexes, only on a write."""
|
| 1089 |
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(PROJECTS)} (
|
| 1090 |
id TEXT NOT NULL, tenant TEXT NOT NULL, name TEXT NOT NULL,
|
| 1091 |
status TEXT NOT NULL {STATUS_CHECK},
|
|
|
|
| 1104 |
key TEXT NOT NULL, added_at TEXT NOT NULL,
|
| 1105 |
PRIMARY KEY (tenant, project_id, grain, key),
|
| 1106 |
FOREIGN KEY (tenant, project_id) REFERENCES {self.t(PROJECTS)} (tenant, id) ON DELETE CASCADE)''')
|
| 1107 |
+
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.t(MEMBERS)} (
|
| 1108 |
+
tenant TEXT NOT NULL, project_id TEXT NOT NULL,
|
| 1109 |
+
username TEXT NOT NULL, name TEXT NOT NULL, added_at TEXT NOT NULL,
|
| 1110 |
+
PRIMARY KEY (tenant, project_id, username),
|
| 1111 |
+
FOREIGN KEY (tenant, project_id) REFERENCES {self.t(PROJECTS)} (tenant, id) ON DELETE CASCADE)''')
|
| 1112 |
# One row's lookup is one indexed read on this index.
|
| 1113 |
con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" '
|
| 1114 |
f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)')
|
|
|
|
| 1122 |
self.ensure_schema(con)
|
| 1123 |
yield con
|
| 1124 |
|
| 1125 |
+
def _select_sql(self, where, prefix='', members='NULL'):
|
| 1126 |
cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS)
|
| 1127 |
count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
|
| 1128 |
'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
|
| 1129 |
+
return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members} '
|
| 1130 |
f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}')
|
| 1131 |
|
| 1132 |
def _rollup_projection(self):
|
| 1133 |
# Old schemas read blank figures until a guarded write adds the columns.
|
| 1134 |
return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'"
|
| 1135 |
|
| 1136 |
+
def _members_projection(self, con):
|
| 1137 |
+
if con.execute('SELECT to_regclass(%s)', (self.t(MEMBERS),)).fetchone()[0] is None:
|
| 1138 |
+
return 'NULL'
|
| 1139 |
+
return ("(SELECT jsonb_agg(jsonb_build_object('username', m.username, 'name', m.name)) "
|
| 1140 |
+
f'FROM {self.t(MEMBERS)} AS m WHERE m.tenant = p.tenant AND m.project_id = p.id)')
|
| 1141 |
+
|
| 1142 |
def _one(self, con, project_id, lock=False):
|
| 1143 |
+
row = con.execute(self._select_sql(' AND p.id = %s', members=self._members_projection(con))
|
| 1144 |
+
+ (' FOR UPDATE OF p' if lock else ''),
|
| 1145 |
(self.tenant_slug, project_id)).fetchone()
|
| 1146 |
return _record(row)
|
| 1147 |
|
|
|
|
| 1152 |
|
| 1153 |
def _insert_targets(self, con, project_id, pairs, stamp):
|
| 1154 |
cols = ('tenant',) + TARGET_COLUMNS
|
| 1155 |
+
for part in _chunks(pairs):
|
| 1156 |
con.execute(f'INSERT INTO {self.t(TARGETS)} ({", ".join(cols)}) VALUES '
|
| 1157 |
+ ', '.join(f'({_marks(len(cols))})' for _ in part)
|
| 1158 |
+ ' ON CONFLICT (tenant, project_id, grain, key) DO NOTHING',
|
| 1159 |
tuple(v for grain, key in part for v in (self.tenant_slug, project_id, grain, key, stamp)))
|
| 1160 |
|
| 1161 |
+
def _replace_members(self, con, project_id, members, previous, stamp):
|
| 1162 |
+
wanted = {member['username'] for member in members}
|
| 1163 |
+
removed = sorted(member['username'] for member in previous if member['username'] not in wanted)
|
| 1164 |
+
for part in _chunks(removed):
|
| 1165 |
+
con.execute(f'DELETE FROM {self.t(MEMBERS)} WHERE tenant = %s AND project_id = %s '
|
| 1166 |
+
f'AND username IN ({_marks(len(part))})', (self.tenant_slug, project_id, *part))
|
| 1167 |
+
for part in _chunks(members, 100):
|
| 1168 |
+
con.execute(f'INSERT INTO {self.t(MEMBERS)} (tenant, project_id, username, name, added_at) VALUES '
|
| 1169 |
+
+ ', '.join(f'({_marks(5)})' for _ in part)
|
| 1170 |
+
+ ' ON CONFLICT (tenant, project_id, username) DO UPDATE SET name = EXCLUDED.name',
|
| 1171 |
+
tuple(value for member in part for value in
|
| 1172 |
+
(self.tenant_slug, project_id, member['username'], member['name'], stamp)))
|
| 1173 |
+
|
| 1174 |
def _touch(self, con, project_id, stamp):
|
| 1175 |
current = self._one(con, project_id, lock=True)
|
| 1176 |
impact = _with_result_configs(None, current['result_configs'])
|
|
|
|
| 1186 |
con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) '
|
| 1187 |
f'VALUES ({_marks(len(columns) - 1)}, %s::jsonb)', values)
|
| 1188 |
self._insert_targets(con, record['id'], pairs, stamp)
|
| 1189 |
+
self._replace_members(con, record['id'], record['contributors'], [], stamp)
|
| 1190 |
return self._one(con, record['id'])
|
| 1191 |
|
| 1192 |
def get(self, project_id):
|
|
|
|
| 1199 |
with self._write() as con:
|
| 1200 |
current = self._one(con, project_id, lock=True)
|
| 1201 |
candidate = _fields(changes, previous=current)
|
| 1202 |
+
if 'contributors' in changes:
|
| 1203 |
+
candidate['contributors'] = _member_values(changes['contributors'], self.tenant_slug)
|
| 1204 |
+
fields = sorted(EDITABLE - {'result_configs', 'contributors'})
|
| 1205 |
changed_anchor = any(candidate[k] != current[k] for k in ('status', 'completed_on', 'result_configs'))
|
| 1206 |
impact = _with_result_configs(None if changed_anchor else current['impact'], candidate['result_configs'])
|
| 1207 |
con.execute(f'UPDATE {self.t(PROJECTS)} SET '
|
|
|
|
| 1209 |
+ ' WHERE tenant = %s AND id = %s',
|
| 1210 |
tuple(candidate[field] for field in fields)
|
| 1211 |
+ (stamp, json.dumps(impact), self.tenant_slug, project_id))
|
| 1212 |
+
if 'contributors' in changes:
|
| 1213 |
+
self._replace_members(con, project_id, candidate['contributors'], current['contributors'], stamp)
|
| 1214 |
return self._one(con, project_id)
|
| 1215 |
|
| 1216 |
def delete(self, project_id):
|
| 1217 |
with self._write() as con:
|
| 1218 |
+
con.execute(f'DELETE FROM {self.t(MEMBERS)} WHERE tenant = %s AND project_id = %s',
|
| 1219 |
+
(self.tenant_slug, project_id))
|
| 1220 |
con.execute(f'DELETE FROM {self.t(TARGETS)} WHERE tenant = %s AND project_id = %s',
|
| 1221 |
(self.tenant_slug, project_id))
|
| 1222 |
row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id',
|
|
|
|
| 1229 |
with self._connection() as con:
|
| 1230 |
if not self._exists(con):
|
| 1231 |
return []
|
| 1232 |
+
rows = con.execute(self._select_sql(' ORDER BY p.created_at, p.id',
|
| 1233 |
+
members=self._members_projection(con)),
|
| 1234 |
+
(self.tenant_slug,)).fetchall()
|
| 1235 |
return [_record(row) for row in rows]
|
| 1236 |
|
| 1237 |
def targets(self, project_id):
|
|
|
|
| 1246 |
with self._connection() as con:
|
| 1247 |
if not self._exists(con):
|
| 1248 |
return out
|
| 1249 |
+
members = self._members_projection(con)
|
| 1250 |
for part in _chunks(pairs):
|
| 1251 |
match = ' OR '.join(['(t.grain = %s AND t.key = %s)'] * len(part))
|
| 1252 |
cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS)
|
| 1253 |
count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
|
| 1254 |
'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
|
| 1255 |
rows = con.execute(
|
| 1256 |
+
f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members} '
|
| 1257 |
f'FROM {self.t(TARGETS)} AS t '
|
| 1258 |
f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id '
|
| 1259 |
f'WHERE t.tenant = %s AND ({match}) ORDER BY p.created_at, p.id',
|
platform/core/audit.py
CHANGED
|
@@ -13,7 +13,7 @@ from core import account_rows, freeze_notice, platform_admin
|
|
| 13 |
AUDIT_KINDS = frozenset({
|
| 14 |
"auth.login", "auth.login_failed", "auth.lockout", "auth.mfa_enrolled",
|
| 15 |
"auth.mfa_failed", "auth.logout", "admin.grant", "credential.added",
|
| 16 |
-
"credential.rotated", "credential.revoked", "amazon.sync", "amazon.throttled",
|
| 17 |
"amazon.authorization_lost", "amazon.deletion_scheduled", "amazon.personal_read", "data.purge", "data.restore",
|
| 18 |
"config.changed", "customer.personal_read",
|
| 19 |
"external.query",
|
|
|
|
| 13 |
AUDIT_KINDS = frozenset({
|
| 14 |
"auth.login", "auth.login_failed", "auth.lockout", "auth.mfa_enrolled",
|
| 15 |
"auth.mfa_failed", "auth.logout", "admin.grant", "credential.added",
|
| 16 |
+
"credential.rotated", "credential.revoked", "amazon.access", "amazon.sync", "amazon.throttled",
|
| 17 |
"amazon.authorization_lost", "amazon.deletion_scheduled", "amazon.personal_read", "data.purge", "data.restore",
|
| 18 |
"config.changed", "customer.personal_read",
|
| 19 |
"external.query",
|
platform/core/connector_spec.py
CHANGED
|
@@ -812,6 +812,18 @@ AMAZON_TABLES = (
|
|
| 812 |
identity=("metric_id",),
|
| 813 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 814 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 815 |
ConnectorTableSpec(
|
| 816 |
key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance",
|
| 817 |
source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku",
|
|
@@ -3238,6 +3250,53 @@ AMAZON_LABELS = {
|
|
| 3238 |
"parent_asin_source": "Parent ASIN source",
|
| 3239 |
"relationships": "Relationships",
|
| 3240 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3241 |
_AMZN + "replenishment": {
|
| 3242 |
"_data_origin": "Data origin",
|
| 3243 |
"_report_document_id": "Report document ID",
|
|
@@ -3404,6 +3463,12 @@ AMAZON_DEFAULT_COLUMNS = {
|
|
| 3404 |
_AMZN + "replenishment": ("asin", "time_interval_start_date", "time_interval_end_date",
|
| 3405 |
"active_subscriptions", "shipped_subscription_units", "total_subscriptions_revenue",
|
| 3406 |
"revenue_penetration", "subscriber_retention_for30_days", "program_type"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3407 |
_AMZN + "sku_keywords": ("sku", "asin", "month", "search_query", "impressions", "clicks",
|
| 3408 |
"cart_adds", "purchases", "backend_keywords", "source_basis"),
|
| 3409 |
_AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
|
|
@@ -3527,7 +3592,7 @@ AMAZON_GROUPS = {
|
|
| 3527 |
**{_AMZN + t: "Listings" for t in (
|
| 3528 |
"listings", "pricing", "inactive_listings", "cancelled_listings", "suppressed_listings",
|
| 3529 |
"listing_defects", "sold_listings", "listing_items", "catalog_items", "b2b_recommended",
|
| 3530 |
-
"b2b_not_yet_on_amazon", "geo_opportunities", "b2b_quotes")},
|
| 3531 |
**{_AMZN + t: "Advertising" for t in (
|
| 3532 |
"brand_analytics", "search_query_performance", "search_catalog_performance",
|
| 3533 |
"market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
|
|
|
|
| 812 |
identity=("metric_id",),
|
| 813 |
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 814 |
),
|
| 815 |
+
ConnectorTableSpec(
|
| 816 |
+
key=_AMZN + "review_topics", native="customerFeedback/2024-06-01/items/{asin}/reviews/topics",
|
| 817 |
+
source="amzn_review_topics", label="Amazon Listings: Review topics", pinned="asin",
|
| 818 |
+
identity=("asin", "marketplace_id", "sentiment", "topic"),
|
| 819 |
+
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 820 |
+
),
|
| 821 |
+
ConnectorTableSpec(
|
| 822 |
+
key=_AMZN + "review_trends", native="customerFeedback/2024-06-01/items/{asin}/reviews/trends",
|
| 823 |
+
source="amzn_review_trends", label="Amazon Listings: Review trends", pinned="asin",
|
| 824 |
+
identity=("asin", "marketplace_id", "sentiment", "topic", "month"),
|
| 825 |
+
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 826 |
+
),
|
| 827 |
ConnectorTableSpec(
|
| 828 |
key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance",
|
| 829 |
source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku",
|
|
|
|
| 3250 |
"parent_asin_source": "Parent ASIN source",
|
| 3251 |
"relationships": "Relationships",
|
| 3252 |
},
|
| 3253 |
+
_AMZN + "review_topics": {
|
| 3254 |
+
"_data_origin": "Data origin",
|
| 3255 |
+
"_report_document_id": "Report document ID",
|
| 3256 |
+
"_report_row_number": "Report row",
|
| 3257 |
+
"_report_type": "Report type",
|
| 3258 |
+
"_snapshot_date": "Synced on",
|
| 3259 |
+
"asin": "ASIN",
|
| 3260 |
+
"marketplace_id": "Marketplace ID",
|
| 3261 |
+
"sentiment": "Sentiment",
|
| 3262 |
+
"topic": "Topic",
|
| 3263 |
+
"item_name": "Product name",
|
| 3264 |
+
"country_code": "Country",
|
| 3265 |
+
"date_range_start_date": "Review period start",
|
| 3266 |
+
"date_range_end_date": "Review period end",
|
| 3267 |
+
"asin_metrics_number_of_mentions": "ASIN mentions",
|
| 3268 |
+
"asin_metrics_occurrence_percentage": "ASIN occurrence",
|
| 3269 |
+
"asin_metrics_star_rating_impact": "ASIN star rating impact",
|
| 3270 |
+
"parent_asin_metrics_number_of_mentions": "Parent ASIN mentions",
|
| 3271 |
+
"parent_asin_metrics_occurrence_percentage": "Parent ASIN occurrence",
|
| 3272 |
+
"parent_asin_metrics_star_rating_impact": "Parent ASIN star rating impact",
|
| 3273 |
+
"browse_node_metrics_occurrence_percentage_all_products": "Browse node occurrence",
|
| 3274 |
+
"browse_node_metrics_star_rating_impact_all_products": "Browse node star rating impact",
|
| 3275 |
+
"child_asin_metrics_most_mentions_asin": "Child ASIN with most mentions",
|
| 3276 |
+
"child_asin_metrics_most_mentions_number_of_mentions": "Most child ASIN mentions",
|
| 3277 |
+
"child_asin_metrics_least_mentions_asin": "Child ASIN with least mentions",
|
| 3278 |
+
"child_asin_metrics_least_mentions_number_of_mentions": "Least child ASIN mentions",
|
| 3279 |
+
"review_snippet_1": "Review snippet 1",
|
| 3280 |
+
"review_snippet_2": "Review snippet 2",
|
| 3281 |
+
"review_snippet_3": "Review snippet 3",
|
| 3282 |
+
"review_snippets": "Review snippets",
|
| 3283 |
+
"subtopics": "Subtopics",
|
| 3284 |
+
},
|
| 3285 |
+
_AMZN + "review_trends": {
|
| 3286 |
+
"_data_origin": "Data origin",
|
| 3287 |
+
"_report_document_id": "Report document ID",
|
| 3288 |
+
"_report_row_number": "Report row",
|
| 3289 |
+
"_report_type": "Report type",
|
| 3290 |
+
"_snapshot_date": "Synced on",
|
| 3291 |
+
"asin": "ASIN",
|
| 3292 |
+
"marketplace_id": "Marketplace ID",
|
| 3293 |
+
"sentiment": "Sentiment",
|
| 3294 |
+
"topic": "Topic",
|
| 3295 |
+
"month": "Month",
|
| 3296 |
+
"date_range_start_date": "Review period start",
|
| 3297 |
+
"date_range_end_date": "Review period end",
|
| 3298 |
+
"asin_metrics_occurrence_percentage": "ASIN occurrence",
|
| 3299 |
+
},
|
| 3300 |
_AMZN + "replenishment": {
|
| 3301 |
"_data_origin": "Data origin",
|
| 3302 |
"_report_document_id": "Report document ID",
|
|
|
|
| 3463 |
_AMZN + "replenishment": ("asin", "time_interval_start_date", "time_interval_end_date",
|
| 3464 |
"active_subscriptions", "shipped_subscription_units", "total_subscriptions_revenue",
|
| 3465 |
"revenue_penetration", "subscriber_retention_for30_days", "program_type"),
|
| 3466 |
+
_AMZN + "review_topics": ("sentiment", "topic", "asin_metrics_number_of_mentions",
|
| 3467 |
+
"asin_metrics_occurrence_percentage", "asin_metrics_star_rating_impact",
|
| 3468 |
+
"parent_asin_metrics_number_of_mentions", "browse_node_metrics_occurrence_percentage_all_products",
|
| 3469 |
+
"review_snippet_1", "review_snippet_2", "review_snippet_3"),
|
| 3470 |
+
_AMZN + "review_trends": ("sentiment", "topic", "month", "asin_metrics_occurrence_percentage",
|
| 3471 |
+
"date_range_start_date", "date_range_end_date"),
|
| 3472 |
_AMZN + "sku_keywords": ("sku", "asin", "month", "search_query", "impressions", "clicks",
|
| 3473 |
"cart_adds", "purchases", "backend_keywords", "source_basis"),
|
| 3474 |
_AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
|
|
|
|
| 3592 |
**{_AMZN + t: "Listings" for t in (
|
| 3593 |
"listings", "pricing", "inactive_listings", "cancelled_listings", "suppressed_listings",
|
| 3594 |
"listing_defects", "sold_listings", "listing_items", "catalog_items", "b2b_recommended",
|
| 3595 |
+
"b2b_not_yet_on_amazon", "geo_opportunities", "b2b_quotes", "review_topics", "review_trends")},
|
| 3596 |
**{_AMZN + t: "Advertising" for t in (
|
| 3597 |
"brand_analytics", "search_query_performance", "search_catalog_performance",
|
| 3598 |
"market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
|
platform/core/external_tools.py
CHANGED
|
@@ -140,6 +140,21 @@ _QUERY_FILTER = {"oneOf": [
|
|
| 140 |
_object({"link": _STRING, "where": _query_filter_node(allow_link=False)},
|
| 141 |
("link", "where"), additionalProperties=False),
|
| 142 |
]}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
FILTER_NODE_REF = "#/$defs/filter_node"
|
| 144 |
FILTER_NODE_EXAMPLE = {"conj": "and", "nodes": [
|
| 145 |
{"colId": "status", "op": "eq", "value": "open"},
|
|
@@ -606,7 +621,9 @@ TOOLS = {
|
|
| 606 |
("workbook_id", "key", "name", "description"), additionalProperties=False),
|
| 607 |
_save_template, write=True),
|
| 608 |
"list_databases": _tool("List the databases you can access or see in Loopable.",
|
| 609 |
-
_object(
|
|
|
|
|
|
|
| 610 |
"describe_database": _tool("Read the fields you have permission to use in a database.",
|
| 611 |
_object({"database": _STRING}, ("database",), additionalProperties=False),
|
| 612 |
lambda ctx, args: ctx.describe_database(args["database"]), grants=("ai_apps",)),
|
|
@@ -615,7 +632,7 @@ TOOLS = {
|
|
| 615 |
_object({
|
| 616 |
"database": _STRING,
|
| 617 |
"fields": {"type": "array", "items": _STRING, "maxItems": 200},
|
| 618 |
-
"filters":
|
| 619 |
"sorts": {"type": "array", "maxItems": 8, "items": _object({
|
| 620 |
"field": _STRING, "direction": {"enum": ["asc", "desc"]},
|
| 621 |
}, ("field", "direction"), additionalProperties=False)},
|
|
@@ -633,6 +650,8 @@ TOOLS = {
|
|
| 633 |
_object({"field": _STRING, "unit": {"enum": ["month"]}}, ("field",),
|
| 634 |
additionalProperties=False),
|
| 635 |
]}},
|
|
|
|
|
|
|
| 636 |
}, ("database",), additionalProperties=False),
|
| 637 |
lambda ctx, args: ctx.query_database(args["database"], {
|
| 638 |
key: value for key, value in args.items() if key != "database"
|
|
@@ -786,7 +805,7 @@ def validate_query_filter_operators(filters):
|
|
| 786 |
stack.extend(node[key] for key in ("nodes", "children", "where") if key in node)
|
| 787 |
|
| 788 |
|
| 789 |
-
def validate_arguments(arguments, schema):
|
| 790 |
"""Validate grouped arguments against the selected action's JSON schema."""
|
| 791 |
if isinstance(arguments, dict) and "filters" in schema.get("properties", {}):
|
| 792 |
validate_query_filter_operators(arguments.get("filters"))
|
|
@@ -850,13 +869,97 @@ def validate_arguments(arguments, schema):
|
|
| 850 |
return False
|
| 851 |
return True
|
| 852 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 853 |
if not isinstance(arguments, dict) or not matches(arguments, schema):
|
| 854 |
-
|
|
|
|
|
|
|
| 855 |
encoded = json.dumps(arguments, ensure_ascii=False, allow_nan=False)
|
| 856 |
if any(prefix in encoded for prefix in ("lpb1.", "lpo1.", "lpr1.", "lpc1.")):
|
| 857 |
raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.")
|
| 858 |
|
| 859 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 860 |
def dispatch(ctx, name, arguments):
|
| 861 |
"""Dispatch one declared tool. The API translates core and grant refusals to MCP content."""
|
| 862 |
tool = TOOLS.get(name) if isinstance(name, str) else None
|
|
@@ -872,6 +975,8 @@ def dispatch(ctx, name, arguments):
|
|
| 872 |
else:
|
| 873 |
ctx.session.require(grant)
|
| 874 |
schema = tool["inputSchema"]
|
|
|
|
|
|
|
| 875 |
if not isinstance(arguments, dict):
|
| 876 |
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.")
|
| 877 |
missing = set(schema["required"]) - arguments.keys()
|
|
|
|
| 140 |
_object({"link": _STRING, "where": _query_filter_node(allow_link=False)},
|
| 141 |
("link", "where"), additionalProperties=False),
|
| 142 |
]}
|
| 143 |
+
_CLIENT_QUERY_LEAF = {"anyOf": [
|
| 144 |
+
_QUERY_FILTER_NODE,
|
| 145 |
+
_object({"field": _STRING, "op": _FILTER_OP, "value": {}},
|
| 146 |
+
("field", "op"), additionalProperties=False),
|
| 147 |
+
_object({"field": _STRING, "operator": {"enum": ["="]}, "value": {}},
|
| 148 |
+
("field", "operator"), additionalProperties=False),
|
| 149 |
+
]}
|
| 150 |
+
_CLIENT_QUERY_NODES = {"type": "array", "minItems": 1, "maxItems": 50,
|
| 151 |
+
"items": _CLIENT_QUERY_LEAF}
|
| 152 |
+
_CLIENT_QUERY_FILTER = {"anyOf": [
|
| 153 |
+
_QUERY_FILTER, _CLIENT_QUERY_LEAF, _CLIENT_QUERY_NODES,
|
| 154 |
+
*[_object({conj: _CLIENT_QUERY_NODES}, (conj,), additionalProperties=False)
|
| 155 |
+
for conj in ("and", "or")],
|
| 156 |
+
_object(additionalProperties={"type": ["string", "number", "boolean", "null"]}),
|
| 157 |
+
]}
|
| 158 |
FILTER_NODE_REF = "#/$defs/filter_node"
|
| 159 |
FILTER_NODE_EXAMPLE = {"conj": "and", "nodes": [
|
| 160 |
{"colId": "status", "op": "eq", "value": "open"},
|
|
|
|
| 621 |
("workbook_id", "key", "name", "description"), additionalProperties=False),
|
| 622 |
_save_template, write=True),
|
| 623 |
"list_databases": _tool("List the databases you can access or see in Loopable.",
|
| 624 |
+
_object({"limit": {"type": "integer", "minimum": 1,
|
| 625 |
+
"description": "Optional client hint. The complete permitted database list is returned."}},
|
| 626 |
+
additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("ai_apps",)),
|
| 627 |
"describe_database": _tool("Read the fields you have permission to use in a database.",
|
| 628 |
_object({"database": _STRING}, ("database",), additionalProperties=False),
|
| 629 |
lambda ctx, args: ctx.describe_database(args["database"]), grants=("ai_apps",)),
|
|
|
|
| 632 |
_object({
|
| 633 |
"database": _STRING,
|
| 634 |
"fields": {"type": "array", "items": _STRING, "maxItems": 200},
|
| 635 |
+
"filters": _CLIENT_QUERY_FILTER,
|
| 636 |
"sorts": {"type": "array", "maxItems": 8, "items": _object({
|
| 637 |
"field": _STRING, "direction": {"enum": ["asc", "desc"]},
|
| 638 |
}, ("field", "direction"), additionalProperties=False)},
|
|
|
|
| 650 |
_object({"field": _STRING, "unit": {"enum": ["month"]}}, ("field",),
|
| 651 |
additionalProperties=False),
|
| 652 |
]}},
|
| 653 |
+
"group_by": {"type": "array", "minItems": 1, "maxItems": 4,
|
| 654 |
+
"items": _STRING, "description": "Client alias for groupBy. Send only one spelling."},
|
| 655 |
}, ("database",), additionalProperties=False),
|
| 656 |
lambda ctx, args: ctx.query_database(args["database"], {
|
| 657 |
key: value for key, value in args.items() if key != "database"
|
|
|
|
| 805 |
stack.extend(node[key] for key in ("nodes", "children", "where") if key in node)
|
| 806 |
|
| 807 |
|
| 808 |
+
def validate_arguments(arguments, schema, *, named=False):
|
| 809 |
"""Validate grouped arguments against the selected action's JSON schema."""
|
| 810 |
if isinstance(arguments, dict) and "filters" in schema.get("properties", {}):
|
| 811 |
validate_query_filter_operators(arguments.get("filters"))
|
|
|
|
| 869 |
return False
|
| 870 |
return True
|
| 871 |
|
| 872 |
+
def problems(value, rule, path="arguments"):
|
| 873 |
+
if matches(value, rule):
|
| 874 |
+
return []
|
| 875 |
+
if not isinstance(rule, dict):
|
| 876 |
+
return [path]
|
| 877 |
+
for branch in ("anyOf", "oneOf"):
|
| 878 |
+
if branch in rule:
|
| 879 |
+
candidates = [problems(value, child, path) for child in rule[branch]]
|
| 880 |
+
return min(candidates, key=lambda found: (
|
| 881 |
+
-max((item.count(".") + item.count("[") for item in found), default=0), len(found)))
|
| 882 |
+
if isinstance(value, dict) and rule.get("type") == "object":
|
| 883 |
+
properties = rule.get("properties", {})
|
| 884 |
+
found = [path + "." + key for key in rule.get("required", ()) if key not in value]
|
| 885 |
+
for key, child in value.items():
|
| 886 |
+
found.extend(problems(child, properties.get(key, rule.get("additionalProperties", True)),
|
| 887 |
+
path + "." + str(key)))
|
| 888 |
+
return found or [path]
|
| 889 |
+
if isinstance(value, list) and rule.get("type") == "array":
|
| 890 |
+
found = [item for index, child in enumerate(value)
|
| 891 |
+
for item in problems(child, rule.get("items", True), f"{path}[{index}]")]
|
| 892 |
+
return found or [path]
|
| 893 |
+
return [path]
|
| 894 |
+
|
| 895 |
if not isinstance(arguments, dict) or not matches(arguments, schema):
|
| 896 |
+
detail = (" Invalid arguments: " + ", ".join(dict.fromkeys(problems(arguments, schema))) + "."
|
| 897 |
+
if named else "")
|
| 898 |
+
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema." + detail)
|
| 899 |
encoded = json.dumps(arguments, ensure_ascii=False, allow_nan=False)
|
| 900 |
if any(prefix in encoded for prefix in ("lpb1.", "lpo1.", "lpr1.", "lpc1.")):
|
| 901 |
raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.")
|
| 902 |
|
| 903 |
|
| 904 |
+
def table_read_arguments(name, arguments, schema=None):
|
| 905 |
+
"""Admit supported client spellings before the existing scoped Table read."""
|
| 906 |
+
schema = schema if schema is not None else TOOLS[name]["inputSchema"]
|
| 907 |
+
try:
|
| 908 |
+
try:
|
| 909 |
+
validate_arguments(arguments, schema, named=True)
|
| 910 |
+
except sheets.SheetRefusal as exc:
|
| 911 |
+
if name != "query_database" or exc.code != "tool_arguments_invalid":
|
| 912 |
+
raise
|
| 913 |
+
aggregate = schema["properties"]["aggregates"]
|
| 914 |
+
item = aggregate["items"]
|
| 915 |
+
# Preserve the semantic code only when aggregate member values cause the schema failure.
|
| 916 |
+
transport = {**schema, "properties": {**schema["properties"], "aggregates": {
|
| 917 |
+
**aggregate, "items": {**item, "properties": {key: {} for key in item["properties"]}}}}}
|
| 918 |
+
validate_arguments(arguments, transport, named=True)
|
| 919 |
+
accepted = ", ".join(item["properties"]["fn"]["enum"])
|
| 920 |
+
raise sheets.SheetRefusal("query_aggregates_invalid", exc.message + " Use aggregates like "
|
| 921 |
+
'[{"fn": "count"}, {"fn": "sum", "field": "<readable field key>"}]. '
|
| 922 |
+
"fn must be one of: " + accepted + ". When supplied, field must be a readable field key string.") from exc
|
| 923 |
+
args = copy.deepcopy(arguments)
|
| 924 |
+
if name != "query_database":
|
| 925 |
+
return args
|
| 926 |
+
if "group_by" in args:
|
| 927 |
+
if "groupBy" in args:
|
| 928 |
+
raise sheets.SheetRefusal("tool_arguments_invalid", "Send group_by or groupBy, not both.")
|
| 929 |
+
args["groupBy"] = args.pop("group_by")
|
| 930 |
+
filters = args.get("filters")
|
| 931 |
+
|
| 932 |
+
def leaf(node):
|
| 933 |
+
validate_arguments({"filters": node}, _object({"filters": _CLIENT_QUERY_LEAF}), named=True)
|
| 934 |
+
if "field" not in node:
|
| 935 |
+
return node
|
| 936 |
+
return {"colId": node["field"], "op": "eq" if "operator" in node else node["op"],
|
| 937 |
+
**({"value": node["value"]} if "value" in node else {})}
|
| 938 |
+
|
| 939 |
+
if isinstance(filters, list):
|
| 940 |
+
args["filters"] = [leaf(node) for node in filters]
|
| 941 |
+
elif isinstance(filters, dict):
|
| 942 |
+
if set(filters) in ({"and"}, {"or"}):
|
| 943 |
+
conj = next(iter(filters))
|
| 944 |
+
args["filters"] = {"conj": conj, "nodes": [leaf(node) for node in filters[conj]]}
|
| 945 |
+
elif set(filters) & {"colId", "field", "op", "operator"}:
|
| 946 |
+
args["filters"] = {"conj": "and", "nodes": [leaf(filters)]}
|
| 947 |
+
elif not set(filters) & {"conj", "nodes", "link", "where", "children", "and", "or"}:
|
| 948 |
+
if not filters or len(filters) > 50:
|
| 949 |
+
raise sheets.SheetRefusal("tool_arguments_invalid", "filters must contain 1 to 50 field equality values.")
|
| 950 |
+
args["filters"] = {"conj": "and", "nodes": [
|
| 951 |
+
{"colId": field, "op": "eq", "value": value} for field, value in filters.items()]}
|
| 952 |
+
validate_arguments({"filters": args["filters"]}, _object({"filters": _QUERY_FILTER}), named=True)
|
| 953 |
+
return args
|
| 954 |
+
except sheets.SheetRefusal as exc:
|
| 955 |
+
if exc.code != "tool_arguments_invalid":
|
| 956 |
+
raise
|
| 957 |
+
example = {"database": "<database>", "filters": {"conj": "and", "nodes": [
|
| 958 |
+
{"colId": "<field>", "op": "eq", "value": "<value>"}]}} if name == "query_database" else {}
|
| 959 |
+
shape = {"action": name, "arguments": example}
|
| 960 |
+
raise sheets.SheetRefusal(exc.code, exc.message + " Call table with " + json.dumps(shape) + ".") from exc
|
| 961 |
+
|
| 962 |
+
|
| 963 |
def dispatch(ctx, name, arguments):
|
| 964 |
"""Dispatch one declared tool. The API translates core and grant refusals to MCP content."""
|
| 965 |
tool = TOOLS.get(name) if isinstance(name, str) else None
|
|
|
|
| 975 |
else:
|
| 976 |
ctx.session.require(grant)
|
| 977 |
schema = tool["inputSchema"]
|
| 978 |
+
if name in {"query_database", "list_databases"}:
|
| 979 |
+
arguments = table_read_arguments(name, arguments, schema)
|
| 980 |
if not isinstance(arguments, dict):
|
| 981 |
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.")
|
| 982 |
missing = set(schema["required"]) - arguments.keys()
|
platform/core/filter_eval.py
ADDED
|
@@ -0,0 +1,827 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/filter_eval.py — the filter tree evaluated against ROWS, in Python (wave 15, C-EVAL).
|
| 2 |
+
|
| 3 |
+
THE THIRD ENGINE, and the one with a different job. There are now three readers of the same
|
| 4 |
+
filter tree and it is worth being precise about why:
|
| 5 |
+
|
| 6 |
+
* `customer-grid/useVisibleRows.ts` — the CLIENT, over in-memory rows. The reference.
|
| 7 |
+
* `harness/filter_sql.py` — compiles the tree to SQL for the DuckDB semantic tables.
|
| 8 |
+
* **this module** — evaluates the tree against wire-shaped row dicts in the
|
| 9 |
+
API/host process, where there is no SQL and the rows are
|
| 10 |
+
already in hand.
|
| 11 |
+
|
| 12 |
+
It exists because a PERMANENT (permission) filter has to be applied server-side, to rows the
|
| 13 |
+
server is about to send, on a surface whose columns are not a DuckDB table. Neither existing
|
| 14 |
+
engine can do that: the client one is TypeScript, and the SQL one needs a table.
|
| 15 |
+
|
| 16 |
+
⛔ TWO ENTRY POINTS, OPPOSITE DEFAULTS, AND THE DIFFERENCE IS THE WHOLE POINT.
|
| 17 |
+
|
| 18 |
+
matches(...) DISPLAY semantics. A faithful mirror of the client, including the places
|
| 19 |
+
where the client IGNORES a leaf. Refereed against the client by
|
| 20 |
+
`aios-web/verify_filter_parity.py` — if this disagrees with the browser about
|
| 21 |
+
one row, a saved view means two different things in two places.
|
| 22 |
+
|
| 23 |
+
permits(...) PERMISSION semantics. Same evaluator, `strict=True`: anything the display
|
| 24 |
+
engine would IGNORE, this DENIES.
|
| 25 |
+
|
| 26 |
+
Why they cannot be the same function (C-EVAL amendment 1, and the reason this module has a
|
| 27 |
+
docstring this long). The client engine is TRI-STATE — `True` / `False` / `None`, where `None`
|
| 28 |
+
means "this leaf is INACTIVE, parent ignores it entirely". A leaf naming a column that does not
|
| 29 |
+
exist returns `None` (`useVisibleRows.ts:318`), and a group whose children are all inactive is
|
| 30 |
+
itself inactive (:273). For a DISPLAY filter that is exactly right: a half-built condition must
|
| 31 |
+
not zero the grid, and a stale column reference must not either.
|
| 32 |
+
|
| 33 |
+
For a PERMISSION filter it is a catastrophe. `agent is Tara` is the only thing standing between
|
| 34 |
+
a sales rep and the whole book; rename or delete the `agent` field and that leaf becomes
|
| 35 |
+
`None` — ignored — and the rep silently receives every customer. The wall would be one field
|
| 36 |
+
rename away from open, and nothing would go red. So `permits()` inverts every ignore into a
|
| 37 |
+
denial: an unanswerable permission filter is an OUTAGE, not a licence.
|
| 38 |
+
|
| 39 |
+
The asymmetry is a PARAMETER (`strict`) rather than a second implementation, deliberately: a
|
| 40 |
+
mirror must copy the original's asymmetries, and the only way to be sure it still does next
|
| 41 |
+
month is for both behaviours to come out of the same walk.
|
| 42 |
+
|
| 43 |
+
⚠ WHAT THIS MODULE DOES NOT RE-IMPLEMENT. Activeness (`is_rule_active`), the JS-`Number`
|
| 44 |
+
coercion (`to_num`), the op vocabularies, cohort-id parsing and the date-window/anchor resolvers
|
| 45 |
+
all already exist in `filter_sql.py` and `windows.py`, and are already held against the client by
|
| 46 |
+
`verify_filter_engine.py` (525 checks) and `verify_windows.py` (123). Importing them is not
|
| 47 |
+
laziness — a second copy of "what counts as an active rule" is precisely the drift those gates
|
| 48 |
+
exist to catch, and it would be a copy the gates do not cover.
|
| 49 |
+
|
| 50 |
+
IMPORT EDGE. This module imports `harness.filter_sql` at module level. That is the interim
|
| 51 |
+
layering edge `("filter_eval.py", "harness.filter_sql")`, retired when `filter_sql` moves into
|
| 52 |
+
core (lane G amend).
|
| 53 |
+
"""
|
| 54 |
+
import json
|
| 55 |
+
import math
|
| 56 |
+
from collections import namedtuple
|
| 57 |
+
|
| 58 |
+
from harness.filter_sql import ( # noqa: F401 (COHORT_OPS re-exported for callers)
|
| 59 |
+
COHORT_FIELD, COHORT_OPS, COHORT_OP_ALIASES, FILTER_OPS, MULTI_VALUE_OPS, RANK_OPS,
|
| 60 |
+
is_rule_active, parse_cohort_ids, parse_filter_values, to_num,
|
| 61 |
+
_resolve_anchor, _window_bounds,
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
#: The CLIENT's numeric family, which is WIDER than `filter_sql.NUMERIC_TYPES`. `rating` (an
|
| 65 |
+
#: integer 1..max) and `formula` (computed in the browser and injected at the field's key) compare
|
| 66 |
+
#: as numbers here because this engine sees the same assembled row the client does. They are
|
| 67 |
+
#: absent from the SQL port for a good reason — they are client-only strata that can never reach a
|
| 68 |
+
#: DuckDB table — so the two lists differing is correct, not drift. Mirrors `isNumericType`.
|
| 69 |
+
NUMERIC_TYPES = frozenset({'currency', 'int', 'pct', 'rating', 'formula', 'autonumber'})
|
| 70 |
+
|
| 71 |
+
#: `created_time` is a DATETIME rendered read-only, and it filters as a DATE (truncated to the
|
| 72 |
+
#: day below). Mirrors `isDateType`.
|
| 73 |
+
DATE_TYPES = frozenset({'date', 'created_time', 'last_modified_time'})
|
| 74 |
+
|
| 75 |
+
#: The date ops that actually compare against an ANCHOR. Everything else falls to the
|
| 76 |
+
#: active-and-always-true default WITHOUT resolving one — ordering matters, see `_match_date`.
|
| 77 |
+
DATE_CMP_OPS = frozenset({'eq', 'neq', 'lt', 'lte', 'gt', 'gte'})
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
class EvalCtx:
|
| 81 |
+
"""Everything the engine needs besides the row and the columns.
|
| 82 |
+
|
| 83 |
+
All four members share one property — each is an answer a single ROW cannot compute — and
|
| 84 |
+
therefore share one failure mode: absent, the condition matches NOTHING rather than
|
| 85 |
+
everything. That is the widening sin the cohort/measure/rank leaves are each written to
|
| 86 |
+
avoid, and it is why none of these defaults to a permissive value.
|
| 87 |
+
"""
|
| 88 |
+
|
| 89 |
+
__slots__ = ('cohort_sets', 'measure_sets', 'rank_sets', 'today')
|
| 90 |
+
|
| 91 |
+
def __init__(self, cohort_sets=None, measure_sets=None, rank_sets=None, today=None):
|
| 92 |
+
#: {cohort_id: set(pid)} — the cohorts THIS reader may see.
|
| 93 |
+
self.cohort_sets = cohort_sets or {}
|
| 94 |
+
#: {rule_id: set(pid)} — resolved measure conditions (CG-8).
|
| 95 |
+
self.measure_sets = measure_sets or {}
|
| 96 |
+
#: {id(leaf) or leaf-key: set(pid)} — resolved rank slices (C-OPS).
|
| 97 |
+
self.rank_sets = rank_sets or {}
|
| 98 |
+
#: The TENANT's today, ISO. Never the server clock at call time — a date filter that
|
| 99 |
+
#: means something different per process is not a filter.
|
| 100 |
+
self.today = today
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
_EMPTY_CTX = EvalCtx()
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _is_group(node):
|
| 107 |
+
return isinstance(node, dict) and isinstance(node.get('children'), list)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def _is_cohort(node):
|
| 111 |
+
return isinstance(node, dict) and node.get('colId') == COHORT_FIELD
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _is_measure(node):
|
| 115 |
+
"""Mirrors `isMeasureRule`: a leaf carrying a `window` IS a measure condition."""
|
| 116 |
+
return isinstance(node, dict) and node.get('window') is not None
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _rhs_col(rhs):
|
| 120 |
+
"""Mirrors `rhsColId`: a STATISTIC names no column."""
|
| 121 |
+
if not isinstance(rhs, dict) or rhs.get('kind') == 'stat':
|
| 122 |
+
return None
|
| 123 |
+
return rhs.get('colId')
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _as_text(v):
|
| 127 |
+
"""The client's `String(raw ?? "").toLowerCase()`. `None` becomes '' (never 'None')."""
|
| 128 |
+
return ('' if v is None else str(v)).lower()
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _as_day(v):
|
| 132 |
+
"""Day-granularity truncation. A `created_time` cell is a DATETIME and every comparand in
|
| 133 |
+
this vocabulary is a DATE — lexically the longer string sorts AFTER its own day, so without
|
| 134 |
+
this `on or before <today>` would silently exclude today's rows. Identity for a pure date."""
|
| 135 |
+
return ('' if v is None else str(v))[:10]
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def _cmp_num(op, a, b):
|
| 139 |
+
"""`default: True` is the documented ACTIVE-and-always-true case — an operator that exists
|
| 140 |
+
but does not apply to this type (`contains` on a number) narrows nothing and asks nothing.
|
| 141 |
+
An operator in NO vocabulary never reaches here; `_eval` refuses it first."""
|
| 142 |
+
if op == 'gt':
|
| 143 |
+
return a > b
|
| 144 |
+
if op == 'gte':
|
| 145 |
+
return a >= b
|
| 146 |
+
if op == 'lt':
|
| 147 |
+
return a < b
|
| 148 |
+
if op == 'lte':
|
| 149 |
+
return a <= b
|
| 150 |
+
if op == 'eq':
|
| 151 |
+
return a == b
|
| 152 |
+
if op == 'neq':
|
| 153 |
+
return a != b
|
| 154 |
+
return True
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _cmp_str(op, a, v):
|
| 158 |
+
if op == 'eq':
|
| 159 |
+
return a == v
|
| 160 |
+
if op == 'neq':
|
| 161 |
+
return a != v
|
| 162 |
+
if op == 'lt':
|
| 163 |
+
return a < v
|
| 164 |
+
if op == 'lte':
|
| 165 |
+
return a <= v
|
| 166 |
+
if op == 'gt':
|
| 167 |
+
return a > v
|
| 168 |
+
if op == 'gte':
|
| 169 |
+
return a >= v
|
| 170 |
+
return True
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _cmp_text(op, a, v):
|
| 174 |
+
if op == 'contains':
|
| 175 |
+
return v in a
|
| 176 |
+
if op == 'doesNotContain':
|
| 177 |
+
return v not in a
|
| 178 |
+
if op == 'eq':
|
| 179 |
+
return a == v
|
| 180 |
+
if op == 'neq':
|
| 181 |
+
return a != v
|
| 182 |
+
return True
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _as_choice_text(raw):
|
| 186 |
+
"""The client's ``String(raw ?? "")`` over a CHOICE-SET cell, lowercased.
|
| 187 |
+
|
| 188 |
+
⚠ NOT ``_as_text``. A multi-select value can still arrive from the wire as a LIST (see
|
| 189 |
+
``_is_blank``, which already carries that case), and Python's ``str(['a', 'b'])`` renders
|
| 190 |
+
``"['a', 'b']"`` where JS's ``String(["a","b"])`` renders ``"a,b"`` — the comma-joined form
|
| 191 |
+
the storage layer uses and the one the membership test below has to see.
|
| 192 |
+
"""
|
| 193 |
+
if isinstance(raw, (list, tuple)):
|
| 194 |
+
return ','.join('' if x is None else str(x) for x in raw).lower()
|
| 195 |
+
return _as_text(raw)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def _cell_members(raw):
|
| 199 |
+
"""The MEMBERS of one cell, mirroring `useVisibleRows.cellMembers`.
|
| 200 |
+
|
| 201 |
+
A multi-select cell is a comma-joined set; a single-valued cell is a one-member set, so
|
| 202 |
+
``hasAnyOf`` over a select column means exactly what ``isAnyOf`` does there. A BLANK cell has
|
| 203 |
+
NO members, which is the blank decision the three engines pin as a lock-step case.
|
| 204 |
+
"""
|
| 205 |
+
return [m for m in (part.strip() for part in _as_choice_text(raw).split(',')) if m]
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def _match_multi_value(op, raw, values):
|
| 209 |
+
"""⭐ W51-T07 — the five multi-value operators, COMPOSED from `_cmp_text`.
|
| 210 |
+
|
| 211 |
+
⛔ THIS FUNCTION IS THE POINT OF THE WHOLE AMENDMENT. This module imports ``FILTER_OPS`` from
|
| 212 |
+
``filter_sql``, and its only vocabulary guard is ``op not in FILTER_OPS -> False``. The moment
|
| 213 |
+
the five ops joined that set, the guard began PASSING them straight into ``_cmp_text``, whose
|
| 214 |
+
tail returns True for an operator it does not know — so every row would have matched, in the
|
| 215 |
+
module that is BOTH the server-side display mirror (``matches``) and the PERMISSION wall
|
| 216 |
+
(``permits``). A bare ``return False`` would have been no better: the server would then
|
| 217 |
+
disagree with the browser about real rows, which is the drift this module exists to prevent.
|
| 218 |
+
|
| 219 |
+
``isAnyOf`` is the OR of ``_cmp_text('eq', ...)`` over the values, so it is the union of the
|
| 220 |
+
equivalent single-value leaves by construction rather than by testing.
|
| 221 |
+
"""
|
| 222 |
+
want = [v.strip().lower() for v in values]
|
| 223 |
+
# ⛔ NO VALUES = UNANSWERABLE = MATCHES NOTHING, uniformly. A SECOND layer: `_eval` already
|
| 224 |
+
# calls such a leaf inactive via `is_rule_active`. Written anyway because the vacuous cases
|
| 225 |
+
# go the WIDENING way — `all(... for v in [])` is True, so `hasAllOf []` would match every
|
| 226 |
+
# row, on a permission wall. `filter_sql` answers `1=0` for the identical input.
|
| 227 |
+
if not want:
|
| 228 |
+
return False
|
| 229 |
+
if op in ('isAnyOf', 'isNoneOf'):
|
| 230 |
+
a = _as_choice_text(raw)
|
| 231 |
+
hit = any(_cmp_text('eq', a, v) for v in want)
|
| 232 |
+
return hit if op == 'isAnyOf' else not hit
|
| 233 |
+
members = _cell_members(raw)
|
| 234 |
+
if op == 'hasAllOf':
|
| 235 |
+
return all(v in members for v in want)
|
| 236 |
+
hit = any(v in members for v in want)
|
| 237 |
+
return hit if op == 'hasAnyOf' else not hit
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def _customer_tag_rules(nodes):
|
| 241 |
+
"""Find every Customer type operand, including operands inside OR groups."""
|
| 242 |
+
for node in nodes:
|
| 243 |
+
if _is_group(node):
|
| 244 |
+
yield from _customer_tag_rules(node['children'])
|
| 245 |
+
elif isinstance(node, dict) and node.get('colId') == 'customer_type':
|
| 246 |
+
yield node
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _match_customer_tags(rule, raw):
|
| 250 |
+
"""Test exact JSON string-list membership, or return None when the rule is uncomputable."""
|
| 251 |
+
if not isinstance(raw, str) or rule.get('rhs') is not None or _is_measure(rule):
|
| 252 |
+
return None
|
| 253 |
+
try:
|
| 254 |
+
tags = json.loads(raw)
|
| 255 |
+
except (ValueError, RecursionError):
|
| 256 |
+
return None
|
| 257 |
+
if not isinstance(tags, list) or any(not isinstance(tag, str) for tag in tags):
|
| 258 |
+
return None
|
| 259 |
+
op = rule.get('op')
|
| 260 |
+
if op in ('isEmpty', 'isNotEmpty'):
|
| 261 |
+
return (not tags) if op == 'isEmpty' else bool(tags)
|
| 262 |
+
if op not in ('eq', 'neq', 'contains', 'doesNotContain') and op not in MULTI_VALUE_OPS:
|
| 263 |
+
return None
|
| 264 |
+
values = rule.get('values') if op in MULTI_VALUE_OPS else [rule.get('value')]
|
| 265 |
+
if (not isinstance(values, list) or not values
|
| 266 |
+
or any(not isinstance(value, str) or not value for value in values)):
|
| 267 |
+
return None
|
| 268 |
+
hits = [value in tags for value in values]
|
| 269 |
+
hit = all(hits) if op == 'hasAllOf' else any(hits)
|
| 270 |
+
return not hit if op in ('neq', 'doesNotContain', 'isNoneOf', 'hasNoneOf') else hit
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _is_blank(raw):
|
| 274 |
+
"""The condition-builder blank contract: absent, empty choice set, or ``'(none)'``.
|
| 275 |
+
|
| 276 |
+
Connector grids intentionally render a missing attribute as the visible ``'(none)'`` sentinel,
|
| 277 |
+
while multi-select wire values can arrive as a list before their comma-joined storage form.
|
| 278 |
+
Both are absence for ``isEmpty``; numeric 0 and false remain real values.
|
| 279 |
+
"""
|
| 280 |
+
if raw is None:
|
| 281 |
+
return True
|
| 282 |
+
if isinstance(raw, (list, tuple, set, frozenset)):
|
| 283 |
+
return not raw or all(_is_blank(value) for value in raw)
|
| 284 |
+
return isinstance(raw, str) and raw.strip() in ('', '(none)')
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def _match_numeric(rule, raw, other, has_rhs):
|
| 288 |
+
a = to_num(raw)
|
| 289 |
+
op = rule.get('op')
|
| 290 |
+
if has_rhs:
|
| 291 |
+
return _cmp_num(op, a, to_num(other))
|
| 292 |
+
if op == 'between':
|
| 293 |
+
v, v2 = rule.get('value'), rule.get('value2')
|
| 294 |
+
if v == '' or v2 is None or v2 == '':
|
| 295 |
+
return True # both bounds required
|
| 296 |
+
lo, hi = to_num(v), to_num(v2)
|
| 297 |
+
if lo > hi:
|
| 298 |
+
lo, hi = hi, lo
|
| 299 |
+
return lo <= a <= hi
|
| 300 |
+
if rule.get('value') == '':
|
| 301 |
+
return True # inactive until a value is typed
|
| 302 |
+
return _cmp_num(op, a, to_num(rule.get('value')))
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def _match_date(rule, raw, other, has_rhs, today):
|
| 306 |
+
a = _as_day(raw)
|
| 307 |
+
op = rule.get('op')
|
| 308 |
+
|
| 309 |
+
# "is within [the past month]" — a RANGE test, and the only date op reading `dateWindow`.
|
| 310 |
+
if op == 'within':
|
| 311 |
+
if not today:
|
| 312 |
+
return False
|
| 313 |
+
bounds = _window_bounds(rule.get('dateWindow'), today)
|
| 314 |
+
if not bounds:
|
| 315 |
+
return False # unresolvable range -> matches nothing
|
| 316 |
+
if a == '':
|
| 317 |
+
# A blank date is not "within" any range. Without this, '' >= a null lower bound is
|
| 318 |
+
# vacuously true and every never-ordered customer lands inside every window.
|
| 319 |
+
return False
|
| 320 |
+
lo, hi = bounds
|
| 321 |
+
return (lo is None or a >= lo) and (hi is None or a <= hi)
|
| 322 |
+
|
| 323 |
+
if has_rhs:
|
| 324 |
+
return _cmp_str(op, a, _as_day(other))
|
| 325 |
+
if op == 'between':
|
| 326 |
+
v, v2 = rule.get('value'), rule.get('value2')
|
| 327 |
+
if v == '' or v2 is None or v2 == '':
|
| 328 |
+
return True
|
| 329 |
+
lo, hi = (v, v2) if v < v2 else (v2, v)
|
| 330 |
+
return lo <= a <= hi
|
| 331 |
+
|
| 332 |
+
# An op that does not apply to a date (`contains`) is ACTIVE-and-always-true, decided BEFORE
|
| 333 |
+
# any anchor is resolved — there is no date in that question to resolve. Ordering this after
|
| 334 |
+
# the anchor would turn `contains` on a date from always-true into matches-nothing, which is
|
| 335 |
+
# a different answer under `or`.
|
| 336 |
+
if op not in DATE_CMP_OPS:
|
| 337 |
+
return True
|
| 338 |
+
if not rule.get('dateMode') and rule.get('value') == '':
|
| 339 |
+
return True # inactive until a value is typed
|
| 340 |
+
mode = rule.get('dateMode') or 'exact'
|
| 341 |
+
# `today` is required by a RELATIVE anchor and by nothing else, which is what keeps every
|
| 342 |
+
# pre-anchor saved view working in a payload that carries no today.
|
| 343 |
+
if mode != 'exact' and not today:
|
| 344 |
+
return False
|
| 345 |
+
v = _resolve_anchor(mode, rule.get('value'), today or '1970-01-01')
|
| 346 |
+
if v is None:
|
| 347 |
+
# Unresolvable (junk date, missing N) -> matches nothing. The honest answer to a
|
| 348 |
+
# question with no date in it.
|
| 349 |
+
return False
|
| 350 |
+
return _cmp_str(op, a, v)
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def _match_filter(rule, row, columns, today):
|
| 354 |
+
"""One LEAF against one row, dispatched on the field's declared type.
|
| 355 |
+
|
| 356 |
+
`isEmpty`/`isNotEmpty` are handled FIRST and type-independently: they must never fall
|
| 357 |
+
through to the "blank value -> inactive" guards below, which would turn them into
|
| 358 |
+
always-true no-ops.
|
| 359 |
+
"""
|
| 360 |
+
field = columns.get(rule.get('colId')) or {}
|
| 361 |
+
ftype = field.get('type')
|
| 362 |
+
raw = row.get(rule.get('colId'))
|
| 363 |
+
op = rule.get('op')
|
| 364 |
+
|
| 365 |
+
if op == 'isEmpty':
|
| 366 |
+
return _is_blank(raw)
|
| 367 |
+
if op == 'isNotEmpty':
|
| 368 |
+
return not _is_blank(raw)
|
| 369 |
+
|
| 370 |
+
# A RANK op can never be answered from one row. `_eval` routes rank leaves to their resolved
|
| 371 |
+
# set and one should never arrive here; this is the SECOND layer, and it exists because of
|
| 372 |
+
# what the first layer's absence would do — `_cmp_num`'s `default: True` would make `topN`
|
| 373 |
+
# match EVERY row, a filter that widens under a count that agrees with it.
|
| 374 |
+
if op in RANK_OPS:
|
| 375 |
+
return False
|
| 376 |
+
|
| 377 |
+
# ⭐ W51-T07 — the MULTI-VALUE family, ON THE OP and before the type dispatch. Every branch
|
| 378 |
+
# below ends in a blank-value short-circuit and these leaves carry a blank `value`; one
|
| 379 |
+
# fall-through and a permission wall reading `Agent is any of [...]` admits every row.
|
| 380 |
+
if op in MULTI_VALUE_OPS:
|
| 381 |
+
if ftype in DATE_TYPES:
|
| 382 |
+
return False
|
| 383 |
+
values = parse_filter_values(rule.get('values'))
|
| 384 |
+
if ftype in NUMERIC_TYPES:
|
| 385 |
+
if op not in ('isAnyOf', 'isNoneOf') or not values:
|
| 386 |
+
return False
|
| 387 |
+
number = to_num(raw)
|
| 388 |
+
hit = any(_cmp_num('eq', number, to_num(value)) for value in values)
|
| 389 |
+
return hit if op == 'isAnyOf' else not hit
|
| 390 |
+
return _match_multi_value(op, raw, values)
|
| 391 |
+
|
| 392 |
+
rhs = rule.get('rhs')
|
| 393 |
+
has_rhs = rhs is not None
|
| 394 |
+
other = row.get(_rhs_col(rhs) or '') if has_rhs else None
|
| 395 |
+
|
| 396 |
+
if ftype in NUMERIC_TYPES:
|
| 397 |
+
return _match_numeric(rule, raw, other, has_rhs)
|
| 398 |
+
if ftype in DATE_TYPES:
|
| 399 |
+
return _match_date(rule, raw, other, has_rhs, today)
|
| 400 |
+
|
| 401 |
+
a = _as_text(raw)
|
| 402 |
+
if has_rhs:
|
| 403 |
+
return _cmp_text(op, a, _as_text(other))
|
| 404 |
+
if rule.get('value') == '':
|
| 405 |
+
return True
|
| 406 |
+
return _cmp_text(op, a, _as_text(rule.get('value')))
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
def _eval(node, row, columns, ctx, strict):
|
| 410 |
+
"""One node against one row. `True` / `False` / `None`, where `None` = INACTIVE.
|
| 411 |
+
|
| 412 |
+
`strict` is the permission mode: every `None` this function would return becomes `False`
|
| 413 |
+
instead. Read the two behaviours side by side rather than trusting either alone — the
|
| 414 |
+
difference between them IS the security property (see the module docstring).
|
| 415 |
+
"""
|
| 416 |
+
ignore = False if strict else None
|
| 417 |
+
|
| 418 |
+
if _is_group(node):
|
| 419 |
+
saw_active = False
|
| 420 |
+
all_true = True
|
| 421 |
+
any_true = False
|
| 422 |
+
for child in node['children']:
|
| 423 |
+
v = _eval(child, row, columns, ctx, strict)
|
| 424 |
+
if v is None:
|
| 425 |
+
continue # inactive -> ignored entirely
|
| 426 |
+
saw_active = True
|
| 427 |
+
all_true = all_true and v
|
| 428 |
+
any_true = any_true or v
|
| 429 |
+
if not saw_active:
|
| 430 |
+
# A group with no active children is itself inactive. In strict mode an empty or
|
| 431 |
+
# wholly-unanswerable permission group DENIES: a wall that evaluates to "nothing to
|
| 432 |
+
# say" must not mean "let them through".
|
| 433 |
+
return ignore
|
| 434 |
+
return any_true if node.get('conj') == 'or' else all_true
|
| 435 |
+
|
| 436 |
+
if not isinstance(node, dict):
|
| 437 |
+
return ignore
|
| 438 |
+
|
| 439 |
+
# A COHORT leaf, BEFORE the column lookup: `__cohort__` is not a column, so the lookup below
|
| 440 |
+
# would call it INACTIVE — and inactive means every row on screen under a count nobody doubts.
|
| 441 |
+
if _is_cohort(node):
|
| 442 |
+
if not is_rule_active(node, columns):
|
| 443 |
+
return ignore
|
| 444 |
+
ids = parse_cohort_ids(node.get('value'))
|
| 445 |
+
sets = [ctx.cohort_sets.get(cid) for cid in ids]
|
| 446 |
+
# ANY named cohort we were given no membership for makes the WHOLE condition
|
| 447 |
+
# unanswerable, and unanswerable matches nothing — for `is none of` as well. Answering
|
| 448 |
+
# "everything" to `is none of [a cohort we cannot see]` is the widening sin wearing a
|
| 449 |
+
# plausible face, and PARTIAL evaluation is worse: `is none of [A, B]` quietly becoming
|
| 450 |
+
# `is none of [A]` would show every row in B under a count nobody would doubt.
|
| 451 |
+
if any(s is None for s in sets):
|
| 452 |
+
return False
|
| 453 |
+
pid = row.get('pid')
|
| 454 |
+
op = node.get('op')
|
| 455 |
+
op = op if op in COHORT_OPS else COHORT_OP_ALIASES.get(op, 'anyOf')
|
| 456 |
+
if op == 'allOf':
|
| 457 |
+
return all(pid in s for s in sets)
|
| 458 |
+
in_any = any(pid in s for s in sets)
|
| 459 |
+
return (not in_any) if op == 'noneOf' else in_any
|
| 460 |
+
|
| 461 |
+
# A MEASURE leaf, also before the column lookup and for the same reason: a measure's colId
|
| 462 |
+
# ("revenue") is not a column of this table. An ACTIVE rule whose set has not been resolved
|
| 463 |
+
# is PENDING and matches NOTHING — never everything.
|
| 464 |
+
if _is_measure(node):
|
| 465 |
+
if not is_rule_active(node, columns):
|
| 466 |
+
return ignore
|
| 467 |
+
s = ctx.measure_sets.get(node.get('id')) if node.get('id') else None
|
| 468 |
+
if s is None:
|
| 469 |
+
return False
|
| 470 |
+
return row.get('pid') in s
|
| 471 |
+
|
| 472 |
+
if strict and node.get('colId') == 'customer_type':
|
| 473 |
+
return ('customer_type' in columns
|
| 474 |
+
and _match_customer_tags(node, row.get('customer_type')) is True)
|
| 475 |
+
|
| 476 |
+
# ── a column leaf ────────────────────────────────────────────────────────────────────────
|
| 477 |
+
if node.get('colId') not in columns:
|
| 478 |
+
# THE LEAF THIS WHOLE MODULE'S `strict` FLAG EXISTS FOR. Display: ignore it. Permission:
|
| 479 |
+
# deny — see the module docstring's field-rename walkthrough.
|
| 480 |
+
return ignore
|
| 481 |
+
if node.get('rhs') is not None and _rhs_col(node['rhs']) not in columns:
|
| 482 |
+
# An rhs naming a column this table does not have is ignored by the SAME rule as an
|
| 483 |
+
# unknown LEFT column. Treating one side as "ignore" and the other as "match nothing"
|
| 484 |
+
# would make `a > deleted` behave differently from `deleted > a`.
|
| 485 |
+
return ignore
|
| 486 |
+
if not is_rule_active(node, columns):
|
| 487 |
+
return ignore
|
| 488 |
+
|
| 489 |
+
# A RANK leaf is a MEMBERSHIP test, like a cohort or a measure: the set was computed in a
|
| 490 |
+
# prior pass over the other rows. No set = REFUSE, never "ignore" and never "true" — both of
|
| 491 |
+
# those SHOW rows.
|
| 492 |
+
if node.get('op') in RANK_OPS:
|
| 493 |
+
# ``_rank_key`` gained an explicit scope in Wave 43. Accept the old four-part key
|
| 494 |
+
# for callers which resolved their own set before that addition; a missing answer is
|
| 495 |
+
# still a refusal, never a permissive fallback.
|
| 496 |
+
s = ctx.rank_sets.get(_rank_key(node))
|
| 497 |
+
if s is None:
|
| 498 |
+
s = ctx.rank_sets.get(_legacy_rank_key(node))
|
| 499 |
+
return (row.get('pid') in s) if s is not None else False
|
| 500 |
+
|
| 501 |
+
# An operator in NO vocabulary at all. The comparators end in `default: True`, right for an
|
| 502 |
+
# operator that exists but does not apply to this type; NOT right for one that exists
|
| 503 |
+
# nowhere (`inTertile` out of a stale saved view), which would match every row.
|
| 504 |
+
if node.get('op') not in FILTER_OPS:
|
| 505 |
+
return False
|
| 506 |
+
|
| 507 |
+
return _match_filter(node, row, columns, ctx.today)
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
def _rank_key(node):
|
| 511 |
+
"""The client keys `rankSets` by leaf IDENTITY (a Map keyed on the node object). Python
|
| 512 |
+
dicts cannot key on a dict, so the caller that resolves rank sets and this evaluator agree
|
| 513 |
+
on a derived key. Kept in ONE place so the two cannot drift apart."""
|
| 514 |
+
return (_legacy_rank_key(node) + (_rank_scope_token(node),))
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
def _legacy_rank_key(node):
|
| 518 |
+
"""The Wave-42 four-part resolver key, retained for supplied membership sets."""
|
| 519 |
+
return (node.get('id') or '', node.get('colId') or '', node.get('op') or '',
|
| 520 |
+
str(node.get('value') if node.get('value') is not None else ''))
|
| 521 |
+
|
| 522 |
+
|
| 523 |
+
def _rank_scope_token(node):
|
| 524 |
+
"""A stable, serialisable identity for one rank leaf's scope configuration.
|
| 525 |
+
|
| 526 |
+
``rankScope`` is deliberately optional: old views keep their global meaning. A grouped
|
| 527 |
+
rank may name its group field explicitly (``{mode:'group', field:'agent'}``) or defer to
|
| 528 |
+
the view's ``groupBy`` through the concise ``'group'`` shape. The resolver below refuses
|
| 529 |
+
the latter when no group field was supplied, rather than silently widening it to global.
|
| 530 |
+
"""
|
| 531 |
+
raw = node.get('rankScope') if isinstance(node, dict) else None
|
| 532 |
+
if isinstance(raw, dict):
|
| 533 |
+
mode = raw.get('mode', raw.get('scope'))
|
| 534 |
+
field = raw.get('field', raw.get('groupBy'))
|
| 535 |
+
if mode == 'group':
|
| 536 |
+
return 'group:' + (str(field).strip()[:80] if field is not None else '')
|
| 537 |
+
return 'global'
|
| 538 |
+
return 'group:' if raw == 'group' else 'global'
|
| 539 |
+
|
| 540 |
+
|
| 541 |
+
def _rank_scope(node, group_by):
|
| 542 |
+
"""``('global' | 'group', field | None)`` for a cleaned or directly-posted rank leaf."""
|
| 543 |
+
raw = node.get('rankScope') if isinstance(node, dict) else None
|
| 544 |
+
if raw == 'group':
|
| 545 |
+
field = group_by
|
| 546 |
+
elif isinstance(raw, dict) and raw.get('mode', raw.get('scope')) == 'group':
|
| 547 |
+
field = raw.get('field', raw.get('groupBy', group_by))
|
| 548 |
+
else:
|
| 549 |
+
return 'global', None
|
| 550 |
+
field = str(field or '').strip()[:80]
|
| 551 |
+
return ('group', field or None)
|
| 552 |
+
|
| 553 |
+
|
| 554 |
+
def _rank_leaves(nodes, columns):
|
| 555 |
+
"""ACTIVE rank leaves in tree order; measure leaves are not rank questions."""
|
| 556 |
+
out = []
|
| 557 |
+
for node in nodes or ():
|
| 558 |
+
if _is_group(node):
|
| 559 |
+
out.extend(_rank_leaves(node.get('children'), columns))
|
| 560 |
+
elif (isinstance(node, dict) and node.get('op') in RANK_OPS
|
| 561 |
+
and not _is_measure(node) and is_rule_active(node, columns)):
|
| 562 |
+
out.append(node)
|
| 563 |
+
return out
|
| 564 |
+
|
| 565 |
+
|
| 566 |
+
def _strip_rank_leaves(nodes):
|
| 567 |
+
"""The residue that defines the ranking domain (the client performs the same pass)."""
|
| 568 |
+
out = []
|
| 569 |
+
for node in nodes or ():
|
| 570 |
+
if _is_group(node):
|
| 571 |
+
children = _strip_rank_leaves(node.get('children'))
|
| 572 |
+
if children:
|
| 573 |
+
out.append({'conj': 'or' if node.get('conj') == 'or' else 'and',
|
| 574 |
+
'children': children})
|
| 575 |
+
elif not (isinstance(node, dict) and node.get('op') in RANK_OPS
|
| 576 |
+
and not _is_measure(node)):
|
| 577 |
+
out.append(node)
|
| 578 |
+
return out
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
def _rank_number(raw):
|
| 582 |
+
"""A rank value, where blank/unparseable is ABSENT rather than numeric zero."""
|
| 583 |
+
if raw is None or raw == '':
|
| 584 |
+
return None
|
| 585 |
+
if isinstance(raw, bool):
|
| 586 |
+
return float(raw) # JavaScript ``Number(true)`` parity.
|
| 587 |
+
try:
|
| 588 |
+
value = float(raw)
|
| 589 |
+
except (TypeError, ValueError):
|
| 590 |
+
return None
|
| 591 |
+
return value if math.isfinite(value) else None
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
def _rank_bound(value, lo, hi):
|
| 595 |
+
"""The exact whole-number bounds the rank builder writes; junk remains unanswerable."""
|
| 596 |
+
try:
|
| 597 |
+
number = float(value)
|
| 598 |
+
except (TypeError, ValueError):
|
| 599 |
+
return None
|
| 600 |
+
if not math.isfinite(number) or int(number) != number:
|
| 601 |
+
return None
|
| 602 |
+
number = int(number)
|
| 603 |
+
return number if lo <= number <= hi else None
|
| 604 |
+
|
| 605 |
+
|
| 606 |
+
def _pid_order(pid):
|
| 607 |
+
"""Stable ascending pid order even for mixed legacy string/integer wire ids."""
|
| 608 |
+
try:
|
| 609 |
+
return 0, int(pid)
|
| 610 |
+
except (TypeError, ValueError):
|
| 611 |
+
return 1, str(pid)
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def _group_value(row, key):
|
| 615 |
+
"""A hashable, deterministic grouping value; list/dict cells remain one visible group."""
|
| 616 |
+
value = row.get(key)
|
| 617 |
+
if isinstance(value, (list, dict, set)):
|
| 618 |
+
return repr(value)
|
| 619 |
+
try:
|
| 620 |
+
hash(value)
|
| 621 |
+
except TypeError:
|
| 622 |
+
return repr(value)
|
| 623 |
+
return value
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
RankResolution = namedtuple('RankResolution', 'sets unanswerable')
|
| 627 |
+
|
| 628 |
+
|
| 629 |
+
def resolve_rank_sets(tree, rows, fields, ctx=None, group_by=None):
|
| 630 |
+
"""Resolve top/bottom/percentile/average slices over the server's filtered row domain.
|
| 631 |
+
|
| 632 |
+
Rank filters cannot be compiled as a row-level SQL ``WHERE`` clause, but they *can* and must
|
| 633 |
+
be answered before an in-memory API/host payload is filtered. The residue (all non-rank
|
| 634 |
+
conditions) defines the domain, ties are selected by ascending ``pid`` in both directions,
|
| 635 |
+
and ``rankScope: 'group'`` independently ranks every ``group_by`` bucket. A grouped request
|
| 636 |
+
without an actual group key is unanswerable and gets an empty membership set; it never falls
|
| 637 |
+
back to a global rank.
|
| 638 |
+
"""
|
| 639 |
+
columns = _columns_map(fields)
|
| 640 |
+
nodes, conj = tree_parts(tree)
|
| 641 |
+
leaves = _rank_leaves(nodes, columns)
|
| 642 |
+
if not leaves:
|
| 643 |
+
return RankResolution({}, 0)
|
| 644 |
+
|
| 645 |
+
base_ctx = ctx or _EMPTY_CTX
|
| 646 |
+
residue = _strip_rank_leaves(nodes)
|
| 647 |
+
residue_tree = {'nodes': residue, 'conj': conj}
|
| 648 |
+
domain = [row for row in (rows or ())
|
| 649 |
+
if matches(residue_tree, row, columns, base_ctx)]
|
| 650 |
+
sets, unanswerable = {}, 0
|
| 651 |
+
|
| 652 |
+
for leaf in leaves:
|
| 653 |
+
key = _rank_key(leaf)
|
| 654 |
+
field = columns.get(leaf.get('colId')) or {}
|
| 655 |
+
if field.get('type') not in NUMERIC_TYPES:
|
| 656 |
+
sets[key] = set()
|
| 657 |
+
unanswerable += 1
|
| 658 |
+
continue
|
| 659 |
+
scope, scoped_by = _rank_scope(leaf, group_by)
|
| 660 |
+
if scope == 'group' and scoped_by not in columns:
|
| 661 |
+
sets[key] = set()
|
| 662 |
+
unanswerable += 1
|
| 663 |
+
continue
|
| 664 |
+
groups = {}
|
| 665 |
+
for row in domain:
|
| 666 |
+
group_key = _group_value(row, scoped_by) if scope == 'group' else None
|
| 667 |
+
groups.setdefault(group_key, []).append(row)
|
| 668 |
+
|
| 669 |
+
selected, answered = set(), False
|
| 670 |
+
for group_rows in groups.values():
|
| 671 |
+
ranked = []
|
| 672 |
+
for row in group_rows:
|
| 673 |
+
value = _rank_number(row.get(leaf.get('colId')))
|
| 674 |
+
if value is not None:
|
| 675 |
+
ranked.append((value, row.get('pid')))
|
| 676 |
+
if not ranked:
|
| 677 |
+
continue
|
| 678 |
+
answered = True
|
| 679 |
+
asc = sorted(ranked, key=lambda item: (item[0], _pid_order(item[1])))
|
| 680 |
+
desc = sorted(ranked, key=lambda item: (-item[0], _pid_order(item[1])))
|
| 681 |
+
count = len(asc)
|
| 682 |
+
op = leaf.get('op')
|
| 683 |
+
keep = None
|
| 684 |
+
if op == 'topN':
|
| 685 |
+
bound = _rank_bound(leaf.get('value'), 1, 10000)
|
| 686 |
+
keep = desc[:bound] if bound is not None else None
|
| 687 |
+
elif op == 'bottomN':
|
| 688 |
+
bound = _rank_bound(leaf.get('value'), 1, 10000)
|
| 689 |
+
keep = asc[:bound] if bound is not None else None
|
| 690 |
+
elif op in ('inTopPct', 'inBottomPct'):
|
| 691 |
+
pct = _rank_bound(leaf.get('value'), 1, 100)
|
| 692 |
+
take = min(count, max(1, math.ceil(count * pct / 100))) if pct else None
|
| 693 |
+
order = desc if op == 'inTopPct' else asc
|
| 694 |
+
keep = order[:take] if take is not None else None
|
| 695 |
+
elif op in ('aboveAvg', 'belowAvg'):
|
| 696 |
+
mean = sum(value for value, _ in asc) / count
|
| 697 |
+
keep = [(value, pid) for value, pid in asc
|
| 698 |
+
if (value > mean if op == 'aboveAvg' else value < mean)]
|
| 699 |
+
elif op in ('inQuartile', 'inDecile'):
|
| 700 |
+
buckets = 4 if op == 'inQuartile' else 10
|
| 701 |
+
wanted = _rank_bound(leaf.get('value'), 1, buckets)
|
| 702 |
+
if wanted is not None and count >= buckets:
|
| 703 |
+
keep = [item for index, item in enumerate(asc)
|
| 704 |
+
if min(buckets, (index * buckets) // count + 1) == wanted]
|
| 705 |
+
if keep is None:
|
| 706 |
+
selected = set()
|
| 707 |
+
answered = False
|
| 708 |
+
break
|
| 709 |
+
selected.update(pid for _, pid in keep)
|
| 710 |
+
sets[key] = selected
|
| 711 |
+
if domain and not answered:
|
| 712 |
+
unanswerable += 1
|
| 713 |
+
return RankResolution(sets, unanswerable)
|
| 714 |
+
|
| 715 |
+
|
| 716 |
+
def _columns_map(fields):
|
| 717 |
+
"""`[{key,type,…}]` -> `{key: field}`. Accepts a mapping unchanged, so callers holding
|
| 718 |
+
either shape need no adapter."""
|
| 719 |
+
if isinstance(fields, dict):
|
| 720 |
+
return fields
|
| 721 |
+
out = {}
|
| 722 |
+
for f in fields or ():
|
| 723 |
+
if isinstance(f, dict) and f.get('key'):
|
| 724 |
+
out[f['key']] = f
|
| 725 |
+
return out
|
| 726 |
+
|
| 727 |
+
|
| 728 |
+
def evaluate(tree, row, fields, ctx=None):
|
| 729 |
+
"""The TRI-STATE the client actually has: `True` / `False` / `None`, where `None` means the
|
| 730 |
+
tree is INACTIVE — it asks nothing, so it narrows nothing.
|
| 731 |
+
|
| 732 |
+
Exposed because collapsing `None` into `True` (which `matches` does, correctly, for its own
|
| 733 |
+
purpose) destroys a distinction some callers must keep. The `memberPids` union is the one
|
| 734 |
+
that bit: the client's rule is `members.has(pid) || matchFilterTree(…)` and that union sits
|
| 735 |
+
INSIDE `if (filters.length)`, so an INACTIVE tree must show every row and ignore the member
|
| 736 |
+
list — whereas an ACTIVE tree that merely fails must still admit hand-picked members. A
|
| 737 |
+
caller holding only `matches` cannot tell those two apart and will narrow an inactive view
|
| 738 |
+
down to its hand-picked rows. (`filter_sql.compile_filter_tree` keeps the same distinction
|
| 739 |
+
by returning None rather than a predicate; this is the row-engine's equivalent.)
|
| 740 |
+
|
| 741 |
+
Activeness is ROW-INDEPENDENT — every `None` here comes from `is_rule_active` or an empty
|
| 742 |
+
group, none of which reads the row — so a caller may compute it once per tree if it wishes.
|
| 743 |
+
"""
|
| 744 |
+
columns = _columns_map(fields)
|
| 745 |
+
nodes, conj = tree_parts(tree)
|
| 746 |
+
return _eval({'conj': conj, 'children': nodes}, row, columns, ctx or _EMPTY_CTX, False)
|
| 747 |
+
|
| 748 |
+
|
| 749 |
+
def matches(tree, row, fields, ctx=None):
|
| 750 |
+
"""DISPLAY semantics — does `row` survive this filter tree? Mirrors the client exactly.
|
| 751 |
+
|
| 752 |
+
A wholly INACTIVE tree returns True: no narrowing, every row shows. That is what the browser
|
| 753 |
+
does with a half-built condition, and this function's whole job is to agree with the browser.
|
| 754 |
+
Use `evaluate()` instead when you need to tell "inactive" from "matched" (see its docstring).
|
| 755 |
+
"""
|
| 756 |
+
v = evaluate(tree, row, fields, ctx)
|
| 757 |
+
return True if v is None else bool(v)
|
| 758 |
+
|
| 759 |
+
|
| 760 |
+
def visible_pids(tree, rows, fields, ctx=None, member_pids=None, group_by=None):
|
| 761 |
+
"""The client's whole row pipeline for one tree, member union included — the shape a caller
|
| 762 |
+
should reach for instead of assembling `matches` in a loop.
|
| 763 |
+
|
| 764 |
+
Mirrors `useVisibleRows`: an INACTIVE tree yields every row (and IGNORES `member_pids`,
|
| 765 |
+
because the union lives inside `if (filters.length)`); an ACTIVE tree yields the rows it
|
| 766 |
+
matches UNION the hand-picked members.
|
| 767 |
+
"""
|
| 768 |
+
ctx = ctx or _EMPTY_CTX
|
| 769 |
+
# This is the whole-payload entry point, so it is the one place a server can answer a rank
|
| 770 |
+
# filter without trusting a browser to have done a prior pass. Preserve supplied cohort /
|
| 771 |
+
# measure answers, but replace rank memberships with this payload's authoritative answer.
|
| 772 |
+
# Direct ``evaluate`` intentionally stays a one-row primitive and therefore stays
|
| 773 |
+
# fail-closed when a caller has not supplied a membership set.
|
| 774 |
+
resolved = resolve_rank_sets(tree, rows, fields, ctx, group_by=group_by)
|
| 775 |
+
if resolved.sets:
|
| 776 |
+
rank_sets = dict(ctx.rank_sets)
|
| 777 |
+
rank_sets.update(resolved.sets)
|
| 778 |
+
ctx = EvalCtx(ctx.cohort_sets, ctx.measure_sets, rank_sets, ctx.today)
|
| 779 |
+
members = set(member_pids or ())
|
| 780 |
+
out, active = [], False
|
| 781 |
+
verdicts = []
|
| 782 |
+
for r in rows:
|
| 783 |
+
v = evaluate(tree, r, fields, ctx)
|
| 784 |
+
if v is not None:
|
| 785 |
+
active = True
|
| 786 |
+
verdicts.append((r, v))
|
| 787 |
+
for r, v in verdicts:
|
| 788 |
+
if not active:
|
| 789 |
+
out.append(r.get('pid'))
|
| 790 |
+
elif v or (members and r.get('pid') in members):
|
| 791 |
+
out.append(r.get('pid'))
|
| 792 |
+
return out
|
| 793 |
+
|
| 794 |
+
|
| 795 |
+
def permits(tree, row, fields, ctx=None):
|
| 796 |
+
"""PERMISSION semantics — may this reader SEE `row` under this permanent filter?
|
| 797 |
+
|
| 798 |
+
Same evaluator, every ignore inverted into a denial. An empty/absent tree means "no
|
| 799 |
+
restriction declared" and admits (the caller decides whether a module with no entry is
|
| 800 |
+
reachable at all — that is `perm_scope`'s question, not this one); anything PRESENT but
|
| 801 |
+
unanswerable DENIES.
|
| 802 |
+
"""
|
| 803 |
+
nodes, conj = tree_parts(tree)
|
| 804 |
+
if not nodes:
|
| 805 |
+
return True # nothing declared -> nothing to enforce here
|
| 806 |
+
columns = _columns_map(fields)
|
| 807 |
+
tag_rules = list(_customer_tag_rules(nodes))
|
| 808 |
+
# An uncomputable tag operand denies the whole wall, including an OR branch.
|
| 809 |
+
if tag_rules and ('customer_type' not in columns or any(
|
| 810 |
+
_match_customer_tags(rule, row.get('customer_type')) is None for rule in tag_rules)):
|
| 811 |
+
return False
|
| 812 |
+
v = _eval({'conj': conj, 'children': nodes}, row, columns, ctx or _EMPTY_CTX, True)
|
| 813 |
+
return bool(v)
|
| 814 |
+
|
| 815 |
+
|
| 816 |
+
def tree_parts(tree):
|
| 817 |
+
"""Read C-PERM amendment 2's `FilterTree` — `{conj?, nodes}` — into `(nodes, conj)`.
|
| 818 |
+
|
| 819 |
+
Accepts a BARE LIST too, because that is what `ViewConfig.filters` is and what
|
| 820 |
+
`clean_filter_tree` returns; an absent conj is `'and'`, mirroring `ViewConfig.filterConj`.
|
| 821 |
+
One reader, so the absent-conj default lives here instead of at every call site.
|
| 822 |
+
"""
|
| 823 |
+
if isinstance(tree, dict):
|
| 824 |
+
nodes = tree.get('nodes')
|
| 825 |
+
conj = 'or' if tree.get('conj') == 'or' else 'and'
|
| 826 |
+
return (nodes if isinstance(nodes, list) else []), conj
|
| 827 |
+
return (tree if isinstance(tree, list) else []), 'and'
|
platform/core/grid_events.py
CHANGED
|
@@ -39,7 +39,9 @@ down, and this note is the marker for that work.
|
|
| 39 |
"""
|
| 40 |
import datetime as dt
|
| 41 |
import json as _json_ev # wave 23 (C7): the json field's parse wall
|
|
|
|
| 42 |
from dataclasses import dataclass, field
|
|
|
|
| 43 |
from functools import wraps
|
| 44 |
# ⚠ ALIASED ON PURPOSE. `field` is imported above for the dataclasses that need it, and this file
|
| 45 |
# uses `field` as a LOCAL VARIABLE in a dozen handlers (`field = {'key': key, …}`). A
|
|
@@ -136,6 +138,87 @@ class EventResult:
|
|
| 136 |
write_id: str = ''
|
| 137 |
|
| 138 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
@dataclass
|
| 140 |
class EventCtx:
|
| 141 |
"""Everything the handler is allowed to know, and nothing about how it is being served.
|
|
@@ -176,6 +259,7 @@ class EventCtx:
|
|
| 176 |
# Core can normalise a recurrence's finite shape, but it cannot decide whether a named
|
| 177 |
# same/linked field is visible to this caller without reaching upward into the API layer.
|
| 178 |
recurrence_validator: Optional[Callable[[object], Optional[dict]]] = None
|
|
|
|
| 179 |
# Value-history restore reuses the ordinary cell wall. This stamp changes only the audit
|
| 180 |
# event kind and admits ``None`` as "remove the prior cell" after every normal field/row
|
| 181 |
# permission check has passed; browser-authored edits never set it.
|
|
@@ -207,6 +291,7 @@ class EventCtx:
|
|
| 207 |
durable_receipts: bool = False
|
| 208 |
#: The route's assembly workspace, lent to the first event only (`handle_one`).
|
| 209 |
workspace_lend: Optional[dict] = None
|
|
|
|
| 210 |
|
| 211 |
def __post_init__(self):
|
| 212 |
# A caller that forgets `seen_ids` gets per-call dedup rather than a crash — the SET is
|
|
@@ -238,6 +323,30 @@ def _store_of(ctx):
|
|
| 238 |
return inner if inner is not None else store
|
| 239 |
|
| 240 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 241 |
def _blobs(ctx):
|
| 242 |
"""The BYTE store for this ctx — `_store_of`'s sibling, split out because it is NOT yet
|
| 243 |
equivalent and pretending otherwise would hide a live residency hole.
|
|
@@ -279,48 +388,6 @@ def _ag_overview_id():
|
|
| 279 |
EVENT_ORIGIN = {'client': 'grid-events', 'ip': None}
|
| 280 |
|
| 281 |
|
| 282 |
-
def _event_rev(event):
|
| 283 |
-
"""THE STORE REVISION THIS BROWSER DECLARED IT WAS LOOKING AT, or None if it declared none.
|
| 284 |
-
|
| 285 |
-
⭐⭐ W51-T33 — THE ONE LINK THAT WAS MISSING, AND IT IS THE WHOLE TICKET.
|
| 286 |
-
`CustomerGrid.tsx::stampViewWrite` has put `rev` on every view write since W50-T28,
|
| 287 |
-
`table_store._audited_update` has enforced it on both backends since W50-T25, and
|
| 288 |
-
`main.py::_view_revision_stale` has answered `409 view_revision_stale {rev_current}` for any
|
| 289 |
-
door that raises — while THIS channel, the one the browser actually writes through, read the
|
| 290 |
-
stamp nowhere and passed no `rev=` at all. Every layer was green and the guard was inert
|
| 291 |
-
([[a-rev-guard-inert-on-the-events-channel]]): two people editing one view still produced two
|
| 292 |
-
200s and the first edit still vanished. Reading the value here is the only thing that was
|
| 293 |
-
missing, which is why this helper is four lines and not a subsystem.
|
| 294 |
-
|
| 295 |
-
⛔ THE EXCEPTION IS NOT CAUGHT ANYWHERE BELOW, DELIBERATELY. `ViewRevisionStale` is a
|
| 296 |
-
`RuntimeError` precisely so the `except Exception` fallbacks dotted through these handlers
|
| 297 |
-
cannot swallow it (`view_audit.py`'s own class docstring says so), and `main.py:232`'s
|
| 298 |
-
handler names THIS channel as the door it exists for: *"the grid events channel above all,
|
| 299 |
-
which raises straight out of `table_store` with nothing between it and the client"*. Catching
|
| 300 |
-
it here to convert it into a toast would make a shipped, documented handler unreachable and
|
| 301 |
-
answer 200 for a write that was refused.
|
| 302 |
-
|
| 303 |
-
⛔ ABSENT IS None, AND None IS ACCEPTED — the pre-C3 behaviour, unchanged. A write that
|
| 304 |
-
declares no revision is simply not asked the question, which is what keeps every unstamped
|
| 305 |
-
door (and every caller that predates the stamp) writing exactly as it does today.
|
| 306 |
-
|
| 307 |
-
⚠ ANYTHING UNREADABLE AS AN INTEGER IS TREATED AS ABSENT RATHER THAN AS ZERO, and that is
|
| 308 |
-
not tidiness. `store_pg.revision()` reports a MISSING key as rev 0, so coercing junk to 0
|
| 309 |
-
would turn a malformed stamp into a confident claim about an empty bucket — the one value
|
| 310 |
-
most likely to disagree with the store and refuse a write nobody was racing.
|
| 311 |
-
⚠ `bool` is an `int` in Python, so `True` would arrive as revision 1. Refused by name.
|
| 312 |
-
"""
|
| 313 |
-
if not isinstance(event, dict):
|
| 314 |
-
return None
|
| 315 |
-
raw = event.get('rev')
|
| 316 |
-
if raw is None or isinstance(raw, bool):
|
| 317 |
-
return None
|
| 318 |
-
try:
|
| 319 |
-
return int(raw)
|
| 320 |
-
except (TypeError, ValueError):
|
| 321 |
-
return None
|
| 322 |
-
|
| 323 |
-
|
| 324 |
def _tops(ctx):
|
| 325 |
"""The table-ops object this ctx operates on — the customer table unless the caller chose
|
| 326 |
another topic. ONE resolution point so 'which bucket does this write land in' has exactly
|
|
@@ -467,7 +534,7 @@ def _field_share_role(ctx, key, definition):
|
|
| 467 |
_storage_key, topic = _field_share_keys(ctx)
|
| 468 |
return _shares.role_for(
|
| 469 |
'field', _shares.field_oid(topic, key), ctx.uname,
|
| 470 |
-
is_admin=ctx.admin, st=
|
| 471 |
except Exception:
|
| 472 |
return None
|
| 473 |
|
|
@@ -734,7 +801,8 @@ def _drop_dead_application_refs(ctx, views):
|
|
| 734 |
|
| 735 |
def workspace_corrections(st, table_key, uname, assembly):
|
| 736 |
"""Only the workspace render wire spends pending acknowledgements, after the memo read."""
|
| 737 |
-
|
|
|
|
| 738 |
return assembly
|
| 739 |
from core.table_store import TableStore
|
| 740 |
fields = TableStore(table_key, st=st).workspace(uname, consume_corrections=True)['fields']
|
|
@@ -753,7 +821,7 @@ def workspace_corrections(st, table_key, uname, assembly):
|
|
| 753 |
|
| 754 |
|
| 755 |
def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False,
|
| 756 |
-
*, include_views=True):
|
| 757 |
"""Durable Airtable-style view/schema/overlay state, with an honest session fallback.
|
| 758 |
|
| 759 |
Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED
|
|
@@ -778,6 +846,8 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_do
|
|
| 778 |
site because app.py computes `admin = bool(is_admin())` once per render. `allowed_pids`
|
| 779 |
stays an EXPLICIT argument because the folder leg deliberately omits it.
|
| 780 |
"""
|
|
|
|
|
|
|
| 781 |
uname = ctx.uname
|
| 782 |
if _store_of(ctx).available():
|
| 783 |
# Wave 48 C3: this is the only runtime transition door. It either promotes every
|
|
@@ -788,6 +858,7 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_do
|
|
| 788 |
include_document=bool(capture_document))
|
| 789 |
if capture_document:
|
| 790 |
ws, ctx.workspace_document = read
|
|
|
|
| 791 |
else:
|
| 792 |
ws = read
|
| 793 |
if not include_views:
|
|
@@ -939,6 +1010,295 @@ def _clean_secondary_view(raw):
|
|
| 939 |
}
|
| 940 |
|
| 941 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 942 |
def _shared_folder_record(fid, folder):
|
| 943 |
"""C9's wire record for a folder granted to somebody else: `{id, name, order?, icon?}`.
|
| 944 |
|
|
@@ -980,6 +1340,17 @@ def _shared_view_role(tops, view_id, owner, uname, st=None):
|
|
| 980 |
return role if role in ('view', 'edit') else None
|
| 981 |
|
| 982 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 983 |
def _granted_views(ctx, uname):
|
| 984 |
"""{view_id: STAMPED view} for every view GRANTED to `uname` by name (wave 21, item 9, C1).
|
| 985 |
|
|
@@ -998,10 +1369,26 @@ def _granted_views(ctx, uname):
|
|
| 998 |
import core.shares as shares
|
| 999 |
tops = _tops(ctx)
|
| 1000 |
st = getattr(tops, 'st', None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1001 |
ids = (shares.shared_with(uname, kind='view', st=st) or {}).get('view') or []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1002 |
out = {}
|
| 1003 |
for vid in ids:
|
| 1004 |
-
found = tops.find_view(vid) if hasattr(tops, 'find_view') else None
|
| 1005 |
if not found:
|
| 1006 |
continue
|
| 1007 |
owner, view = found
|
|
@@ -1038,7 +1425,7 @@ def _granted_views(ctx, uname):
|
|
| 1038 |
except Exception:
|
| 1039 |
folder_ids = []
|
| 1040 |
for fid in folder_ids:
|
| 1041 |
-
found = tops.find_folder(fid) if hasattr(tops, 'find_folder') else None
|
| 1042 |
if not found:
|
| 1043 |
continue # a folder on another topic contributes nothing here
|
| 1044 |
owner, folder, inside = found
|
|
@@ -1209,7 +1596,7 @@ def _refuse(ctx, code, message, key='', *, event='field_upsert'):
|
|
| 1209 |
try:
|
| 1210 |
out = getattr(ctx, 'out', None)
|
| 1211 |
if out is not None and hasattr(out, 'refusals'):
|
| 1212 |
-
if
|
| 1213 |
out.refusals.append({
|
| 1214 |
'event': str(event or 'field_upsert')[:40],
|
| 1215 |
'key': str(key or '')[:80],
|
|
@@ -1403,14 +1790,309 @@ def _activity_action(fn):
|
|
| 1403 |
return wrapped
|
| 1404 |
|
| 1405 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1406 |
def _edit_receipt(fn):
|
| 1407 |
@wraps(fn)
|
| 1408 |
def wrapped(event, ctx):
|
| 1409 |
from copy import copy, deepcopy
|
| 1410 |
from dataclasses import replace
|
| 1411 |
from core import edit_receipts
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1412 |
if not isinstance(event, dict):
|
| 1413 |
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1414 |
identity = event.get('opId', event.get('id'))
|
| 1415 |
active_id = edit_receipts.current_op_id()
|
| 1416 |
# Trusted staged previews own their confirmation ledger and must not gain transport metadata.
|
|
@@ -1423,7 +2105,7 @@ def _edit_receipt(fn):
|
|
| 1423 |
if len(ctx.seen_ids) > 400:
|
| 1424 |
for stale in list(ctx.seen_ids)[:200]:
|
| 1425 |
ctx.seen_ids.pop(stale, None)
|
| 1426 |
-
return
|
| 1427 |
if not identity:
|
| 1428 |
return False
|
| 1429 |
try:
|
|
@@ -1442,12 +2124,16 @@ def _edit_receipt(fn):
|
|
| 1442 |
if table is not None:
|
| 1443 |
table._st = checked
|
| 1444 |
write_ctx = replace(ctx, st=checked, table=table, workspace_lend=lend)
|
| 1445 |
-
rerender =
|
| 1446 |
write = dict(ctx.out.writes.get(identity) or {})
|
| 1447 |
write['opId'] = identity
|
| 1448 |
result = deepcopy({'rerender': bool(rerender), 'write': write,
|
| 1449 |
'refusals': ctx.out.refusals[before:],
|
| 1450 |
'doc': ctx.out.doc, 'toast': ctx.out.toast})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1451 |
edit_receipts.observe_event(result)
|
| 1452 |
return result
|
| 1453 |
|
|
@@ -1457,6 +2143,8 @@ def _edit_receipt(fn):
|
|
| 1457 |
_store_of(ctx), ctx.uname,
|
| 1458 |
edit_receipts.operation(ctx.scope_key, event.get('type'), event.get('pid')),
|
| 1459 |
identity, apply)
|
|
|
|
|
|
|
| 1460 |
except edit_receipts.ReceiptUnavailable as exc:
|
| 1461 |
raise StoreUnavailable(str(exc)) from exc
|
| 1462 |
receipt = receipt.get('event') or receipt
|
|
@@ -1528,6 +2216,8 @@ def handle_one(event, ctx):
|
|
| 1528 |
event_id = event['id']
|
| 1529 |
ctx.out.write_id = event_id
|
| 1530 |
kind = event.get('type')
|
|
|
|
|
|
|
| 1531 |
if kind == 'overlay_patch' and str(scope_key) == 'ut_activity':
|
| 1532 |
return _refuse(ctx, 'activity_read_only', 'Activity records cannot be edited.',
|
| 1533 |
event=kind)
|
|
@@ -1631,19 +2321,39 @@ def handle_one(event, ctx):
|
|
| 1631 |
|
| 1632 |
if kind == 'view_upsert':
|
| 1633 |
raw = event.get('view')
|
| 1634 |
-
if not isinstance(raw, dict)
|
| 1635 |
return False
|
| 1636 |
view_id = str(raw.get('id') or '').strip()[:120]
|
| 1637 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1638 |
requested_name = name
|
| 1639 |
-
if not view_id or
|
| 1640 |
return False
|
| 1641 |
-
display = raw['config'].get('display')
|
| 1642 |
-
if
|
| 1643 |
-
return _folder_refuse(
|
| 1644 |
-
ctx, kind, 'view_mode_retired',
|
| 1645 |
-
'Time-series Views are retired. Choose an available View display mode.',
|
| 1646 |
-
key=view_id)
|
| 1647 |
if 'permissions' in raw:
|
| 1648 |
return _refuse(
|
| 1649 |
ctx, 'view_permissions_retired',
|
|
@@ -1654,7 +2364,7 @@ def handle_one(event, ctx):
|
|
| 1654 |
# here for the same reason the page does it: a missing embed build must
|
| 1655 |
# never break page import.
|
| 1656 |
import aios_grid as _ag
|
| 1657 |
-
cfg =
|
| 1658 |
# A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered,
|
| 1659 |
# shown, grouped or coloured by, because it is not a column of this table. Hence the
|
| 1660 |
# widened key set here and the plain `valid_keys` everywhere below.
|
|
@@ -1671,8 +2381,6 @@ def handle_one(event, ctx):
|
|
| 1671 |
# so this needs no new plumbing; it is the value nobody had handed to the validator.
|
| 1672 |
# ⚠ Edited by lane C, not this file's owner: `platform/core/grid_events.py` is in NO fence
|
| 1673 |
# and lane C was the last session open. See `mailbox/C.md` NOTE C-24.
|
| 1674 |
-
_view_ids = {str(v.get('id')) for v in (visible_views or ())
|
| 1675 |
-
if isinstance(v, dict) and v.get('id')}
|
| 1676 |
# ⛔⛔ THE CYCLE REFUSAL'S HOST HALF (C4), and it is the ticket's `done-when` rather than a
|
| 1677 |
# guard bolted on afterwards. View X filtered on "is in View Y" where Y is filtered on X
|
| 1678 |
# cannot be resolved by anything: X needs Y needs X. The browser refuses it too, with a
|
|
@@ -1691,81 +2399,11 @@ def handle_one(event, ctx):
|
|
| 1691 |
'. A view cannot filter on a view that filters on it, because neither can '
|
| 1692 |
'be worked out without the other.', view_id)
|
| 1693 |
return False
|
| 1694 |
-
filter_removals =
|
| 1695 |
-
|
| 1696 |
-
|
| 1697 |
-
|
| 1698 |
-
|
| 1699 |
-
sorts = []
|
| 1700 |
-
for rule in list(cfg.get('sorts') or [])[:20]:
|
| 1701 |
-
if (isinstance(rule, dict) and rule.get('colId') in valid_keys
|
| 1702 |
-
and rule.get('dir') in ('asc', 'desc')):
|
| 1703 |
-
clean_sort = {'colId': rule['colId'], 'dir': rule['dir']}
|
| 1704 |
-
if rule.get('locked') is True:
|
| 1705 |
-
clean_sort['locked'] = True
|
| 1706 |
-
sorts.append(clean_sort)
|
| 1707 |
-
order = list(dict.fromkeys(
|
| 1708 |
-
key for key in list(cfg.get('order') or []) if key in valid_keys))
|
| 1709 |
-
order += [field['key'] for field in fields if field['key'] not in order]
|
| 1710 |
-
visible = list(dict.fromkeys(
|
| 1711 |
-
key for key in list(cfg.get('visible') or []) if key in valid_keys))
|
| 1712 |
-
widths = {}
|
| 1713 |
-
raw_widths = cfg.get('widths') if isinstance(cfg.get('widths'), dict) else {}
|
| 1714 |
-
for key, value in raw_widths.items():
|
| 1715 |
-
if key in valid_keys:
|
| 1716 |
-
try:
|
| 1717 |
-
widths[key] = max(60, min(600, round(float(value))))
|
| 1718 |
-
except (TypeError, ValueError):
|
| 1719 |
-
pass
|
| 1720 |
-
# ⭐⭐ W50-T03 / R2 — THE SUMMARY CHOICE IS A VIEW SETTING, SO IT HAS TO SURVIVE THIS DOOR.
|
| 1721 |
-
#
|
| 1722 |
-
# ⛔ THE BUG THIS CLOSES IS SILENT AND LOOKS LIKE A CLIENT BUG. This whitelist REBUILDS the
|
| 1723 |
-
# stored config from named keys, so a key it does not name is not rejected loudly — it is
|
| 1724 |
-
# simply absent from what gets written. The reader picks Median, the cell shows Median, and
|
| 1725 |
-
# the choice disappears the next time the host copy wins a reconcile. Exactly the shape
|
| 1726 |
-
# `secondaryView` above records having paid for once already.
|
| 1727 |
-
#
|
| 1728 |
-
# ⚠ `'none'` IS A CHOICE, NOT AN ABSENCE (types.ts::SummaryChoice): R2 defaults numeric
|
| 1729 |
-
# kinds to Sum, so a reader turning a money column's summary OFF must be able to SAY so,
|
| 1730 |
-
# and an absent key means "nothing chosen" and re-defaults to Sum. Dropping `'none'` here
|
| 1731 |
-
# would make that switch un-turn-off-able.
|
| 1732 |
-
import aios_grid as _ag_summary
|
| 1733 |
-
summary = {}
|
| 1734 |
-
raw_summary = cfg.get('summary') if isinstance(cfg.get('summary'), dict) else {}
|
| 1735 |
-
for key, value in list(raw_summary.items())[:400]:
|
| 1736 |
-
if key not in valid_keys or not isinstance(value, dict):
|
| 1737 |
-
continue
|
| 1738 |
-
agg = value.get('agg')
|
| 1739 |
-
# The vocabulary is `aios_grid.SUMMARY_AGGS` plus `'none'` — NOT `FIELD_AGGS`, which is
|
| 1740 |
-
# `field.agg`'s six-name write vocabulary and rejects `count_distinct`/`percentile`.
|
| 1741 |
-
#
|
| 1742 |
-
# ⭐⭐ W51-T02 / OWNER RULING R3 — AND PLUS `'share'`, ALLOWLISTED BY NAME RIGHT HERE
|
| 1743 |
-
# RATHER THAN BY WIDENING `aios_grid.SUMMARY_AGGS`.
|
| 1744 |
-
#
|
| 1745 |
-
# R3 puts "% of total" on the bottom-bar picker, so `share` is a choice a reader can
|
| 1746 |
-
# store and it has to survive this door — this whitelist REBUILDS the config from
|
| 1747 |
-
# named keys, so an unlisted value is not rejected loudly, it simply never gets
|
| 1748 |
-
# written and the choice vanishes on the next reconcile (the bug the note above
|
| 1749 |
-
# closed for `count_distinct`).
|
| 1750 |
-
# ⛔ IT IS NOT A WIRE AGG AND MUST NOT JOIN `aios_grid.SUMMARY_AGGS`. That tuple is
|
| 1751 |
-
# C1's eight-name clause, `verify_grid_ux.py::summary_bar_shape` holds it byte-for-byte
|
| 1752 |
-
# against the client's `iconShapes.SUMMARY_AGGS`, and it is what the `/aggregates`
|
| 1753 |
-
# route compiles into SQL. No engine has a "share of the view" aggregate: a share is a
|
| 1754 |
-
# DIVISION of two ordinary ones (a numeric column's sum, anything else's count) and
|
| 1755 |
-
# the client does it. So the picker's extra name is admitted where choices are STORED
|
| 1756 |
-
# and nowhere a query is COMPILED — which is also why `aios_grid.py`, another lane's
|
| 1757 |
-
# file, is never opened for this.
|
| 1758 |
-
if agg != 'none' and agg != 'share' and agg not in _ag_summary.SUMMARY_AGGS:
|
| 1759 |
-
continue
|
| 1760 |
-
choice = {'agg': agg}
|
| 1761 |
-
if agg == 'percentile':
|
| 1762 |
-
try:
|
| 1763 |
-
choice['p'] = max(0, min(100, round(float(value.get('p')))))
|
| 1764 |
-
except (TypeError, ValueError):
|
| 1765 |
-
# A percentile with no readable point is not a percentile. Drop the whole
|
| 1766 |
-
# choice rather than store one that would answer at a point nobody picked.
|
| 1767 |
-
continue
|
| 1768 |
-
summary[key] = choice
|
| 1769 |
# C-LOCK (wave 2026-08-02, item 12's mechanism): a view may be LOCKED to one cohort.
|
| 1770 |
# `cohort_ids` is the same permission list the cohort filter leaf uses — a lock naming
|
| 1771 |
# a cohort this session cannot see is DROPPED at write, exactly like a cohort rule.
|
|
@@ -1782,84 +2420,10 @@ def handle_one(event, ctx):
|
|
| 1782 |
_is_locked_view = view_id in set(cohort_ids)
|
| 1783 |
if _is_locked_view:
|
| 1784 |
cohort_lock = view_id
|
| 1785 |
-
|
| 1786 |
-
'
|
| 1787 |
-
# conjunction joining the ROOT-level conditions; anything but an
|
| 1788 |
-
# explicit 'or' means 'and' (legacy views had an implicit AND).
|
| 1789 |
-
'filterConj': 'or' if cfg.get('filterConj') == 'or' else 'and',
|
| 1790 |
-
**({'cohortLock': cohort_lock}
|
| 1791 |
-
if cohort_lock and cohort_lock in set(cohort_ids) else {}),
|
| 1792 |
-
# ⭐⭐ WAVE 32 · R5 / CONTRACT C4 — "Mark important" (raised by SESSION C as ASK C-12).
|
| 1793 |
-
#
|
| 1794 |
-
# ⛔ WITHOUT THIS LINE THE MARK IS A LIE THAT SURVIVES UNTIL RELOAD. This dict is
|
| 1795 |
-
# REBUILT FROM AN ALLOWLIST on every autosave — a column resize is enough — so a key
|
| 1796 |
-
# the client sends and this literal does not name is silently dropped. The badge would
|
| 1797 |
-
# paint, the write would answer 200, and the mark would be gone on the next read: the
|
| 1798 |
-
# exact shape T24's done-when says "persists across a reload" to catch, and
|
| 1799 |
-
# [[a-migration-that-runs-on-the-next-write]] on a write path.
|
| 1800 |
-
#
|
| 1801 |
-
# ⚠ UNCONDITIONAL, NOT SPREAD-CONDITIONAL LIKE `cohortLock` ABOVE, and the asymmetry is
|
| 1802 |
-
# the whole point: R5's mark must be REMOVABLE. A key written only when present means
|
| 1803 |
-
# unmarking sends `important: false`, the allowlist drops it, and the stored `true`
|
| 1804 |
-
# survives — a mark you can set and never clear.
|
| 1805 |
-
# ⚠ `is True`, not truthy: the client sends a real boolean, and a string `"false"` must
|
| 1806 |
-
# not become a mark.
|
| 1807 |
-
# ⚠ NO PERMISSION GATE, deliberately. R5 makes this a personal legibility mark, not a
|
| 1808 |
-
# lock; the source/grant write wall below already decides who may write this view at
|
| 1809 |
-
# all, and adding a second wall here would be the two-normalizers shape this wave is
|
| 1810 |
-
# already fixing elsewhere.
|
| 1811 |
-
'important': cfg.get('important') is True,
|
| 1812 |
-
'sorts': sorts,
|
| 1813 |
-
'groupBy': cfg.get('groupBy') if cfg.get('groupBy') in valid_keys else None,
|
| 1814 |
-
'colorBy': cfg.get('colorBy') if cfg.get('colorBy') in valid_keys else None,
|
| 1815 |
-
'rowHeightMode': (cfg.get('rowHeightMode')
|
| 1816 |
-
if cfg.get('rowHeightMode') in ('compact', 'short', 'medium', 'tall')
|
| 1817 |
-
else 'short'),
|
| 1818 |
-
'order': order,
|
| 1819 |
-
'visible': visible,
|
| 1820 |
-
'widths': widths,
|
| 1821 |
-
'summary': summary,
|
| 1822 |
-
'memberPids': [pid for pid in list(cfg.get('memberPids') or [])
|
| 1823 |
-
if isinstance(pid, int) and pid in allowed_pids],
|
| 1824 |
-
# ⭐⭐ W41-T20 clause 5 / C6 -- see `_clean_secondary_view` above.
|
| 1825 |
-
'secondaryView': _clean_secondary_view(cfg.get('secondaryView')),
|
| 1826 |
-
}
|
| 1827 |
-
# W45 C3: a View either owns a direct display spec OR references one reusable
|
| 1828 |
-
# Application. It never stores both. The identity/revision cleaner is shared with the
|
| 1829 |
-
# template projection, and this is its production write path (not a verifier-only helper).
|
| 1830 |
-
from core import view_templates as _view_templates
|
| 1831 |
-
application = None
|
| 1832 |
-
if cfg.get('application') is not None:
|
| 1833 |
-
application = _view_templates.application_ref({'config': cfg})
|
| 1834 |
-
if application is None:
|
| 1835 |
-
_refuse(ctx, 'invalid_application_ref',
|
| 1836 |
-
'that Application reference is not a valid typed identity', view_id)
|
| 1837 |
-
return False
|
| 1838 |
-
# S67-T39 / R36: a reference to a DELETED App is dropped, not stored, so the View
|
| 1839 |
-
# saves as a plain view of its database. Unknown registry state keeps the ref.
|
| 1840 |
-
_live = _live_application_ids(ctx)
|
| 1841 |
-
if _live is not None and application['id'] not in _live:
|
| 1842 |
-
application = None
|
| 1843 |
-
else:
|
| 1844 |
-
config['application'] = application
|
| 1845 |
-
# Wave-6 item 10: a direct non-grid display mode rides the view config, validated
|
| 1846 |
-
# structurally (mode whitelist, refs must be this table's fields). A bound Application's
|
| 1847 |
-
# config remains in the registry and is resolved on read, so an offered clone is dropped.
|
| 1848 |
-
if application is None:
|
| 1849 |
-
display = _ag._clean_display(cfg.get('display'), valid_keys)
|
| 1850 |
-
if display:
|
| 1851 |
-
config['display'] = display
|
| 1852 |
-
# Wave-6 item 11 (pin): how many leading columns stay frozen while scrolling
|
| 1853 |
-
# sideways. Clamped 1..8 — 0 would unfreeze the identity column the whole
|
| 1854 |
-
# surface pivots on; absent = the legacy single frozen column.
|
| 1855 |
-
if cfg.get('frozenCount') is not None:
|
| 1856 |
-
try:
|
| 1857 |
-
config['frozenCount'] = max(1, min(8, int(cfg['frozenCount'])))
|
| 1858 |
-
except (TypeError, ValueError):
|
| 1859 |
-
pass
|
| 1860 |
# The prior stored View supplies lock state and its immutable filter records. Sharing
|
| 1861 |
# itself is exclusively the modern grant registry checked at the foreign-view wall below.
|
| 1862 |
-
prior = (ws.get('views') or {}).get(view_id) or {}
|
| 1863 |
# A visible foreign view is a grant projection. The modern foreign-view wall below
|
| 1864 |
# resolves the owner's personal source and checks the grant registry before deciding
|
| 1865 |
# whether this is a create; there is no legacy bucket to consult as a second authority.
|
|
@@ -1902,12 +2466,6 @@ def handle_one(event, ctx):
|
|
| 1902 |
config['display'] = dict(prior['config']['display'])
|
| 1903 |
else:
|
| 1904 |
config.pop('display', None)
|
| 1905 |
-
# A SYSTEM VIEW is per-user state and is never a share source. `kind` is
|
| 1906 |
-
# host-validated below; read the existing value when present so a browser cannot change
|
| 1907 |
-
# the residency/identity of a stored View.
|
| 1908 |
-
_kind = (prior.get('kind') if not is_new
|
| 1909 |
-
else (raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom')
|
| 1910 |
-
else 'custom'))
|
| 1911 |
# C3: only the creator or an admin may SET or CLEAR the lock; anyone else keeps whatever
|
| 1912 |
# is already stored. Enforced here rather than hidden in the client. This composes with
|
| 1913 |
# C4 and does not override it: C4 governs who may edit the CONTENTS, C3 who may freeze
|
|
@@ -1915,7 +2473,10 @@ def handle_one(event, ctx):
|
|
| 1915 |
# ⚠ Inert in practice TODAY — a user can only ever write their own per-user workspace,
|
| 1916 |
# so `uname == creator` always holds. Correct plumbing for shared views, not a live gate.
|
| 1917 |
may_lock = admin or uname == creator
|
| 1918 |
-
locked =
|
|
|
|
|
|
|
|
|
|
| 1919 |
# C3: the LOCK's actual job — a locked view's display mode is frozen, so a locked Kanban
|
| 1920 |
# stays Kanban. This half IS live per-user and is the part the owner asked for. Filters,
|
| 1921 |
# sorts and columns stay editable by design.
|
|
@@ -1947,7 +2508,7 @@ def handle_one(event, ctx):
|
|
| 1947 |
else 'custom',
|
| 1948 |
'locked': locked,
|
| 1949 |
'createdBy': creator,
|
| 1950 |
-
'note':
|
| 1951 |
'config': config,
|
| 1952 |
'filterRemovals': filter_removals}
|
| 1953 |
# System/list projections are not all persisted in this table store, but custom views
|
|
@@ -1972,7 +2533,7 @@ def handle_one(event, ctx):
|
|
| 1972 |
# win the merge, so the receiver sees their private copy forever and calls sharing
|
| 1973 |
# broken). Role `edit` (or admin) writes THROUGH to the owner's record — config only;
|
| 1974 |
# `createdBy` stays pinned from the stored source and grants remain the access wall.
|
| 1975 |
-
#
|
| 1976 |
#
|
| 1977 |
# ⛔⛔ WAVE 33 · T13 (owner item 3) — A PINNED PER-USER ID IS NOT A FOREIGN VIEW, AND
|
| 1978 |
# TREATING IT AS ONE IS A CROSS-USER WRITE THAT REPORTS SUCCESS.
|
|
@@ -1994,14 +2555,12 @@ def handle_one(event, ctx):
|
|
| 1994 |
# VIEW IS PER-USER STATE AND MUST NEVER BE SHARED", enforced there by pinning
|
| 1995 |
# `perms = {'edit': 'personal'}`. A per-user view can therefore never BE a grant, which
|
| 1996 |
# is the only thing this wall exists to protect — so exempting it narrows nothing.
|
| 1997 |
-
_pinned_per_user = (_kind == 'system' or view_id in ('all-customers', _ag_overview_id()))
|
| 1998 |
# W48-T20 / C4 — an assignment is neither the owner's view nor a generic foreign
|
| 1999 |
# share. Its filter lives in TableStore's ``__assigned__`` projection and this write
|
| 2000 |
# door accepts only the recipient's presentation. In particular, a forged ``edit``
|
| 2001 |
# grant or a client flip of ``assigned`` cannot route this through the generic
|
| 2002 |
# share-through-to-owner branch below.
|
| 2003 |
-
|
| 2004 |
-
if raw.get('assigned') or prior.get('assigned') or view_id in assigned:
|
| 2005 |
if view_id not in assigned:
|
| 2006 |
ctx.out.toast = 'That assigned view is no longer available to you.'
|
| 2007 |
return False
|
|
@@ -2014,34 +2573,14 @@ def handle_one(event, ctx):
|
|
| 2014 |
# load-bearing: it replaces any optimistic browser filter with the locked server
|
| 2015 |
# projection before another action can use it.
|
| 2016 |
return True
|
| 2017 |
-
_save_as = uname
|
| 2018 |
-
if _store_of(ctx).available() and not _pinned_per_user:
|
| 2019 |
-
_foreign = _tops(ctx).find_view(view_id)
|
| 2020 |
-
if _foreign and _foreign[0] != uname:
|
| 2021 |
-
_o_uname, _stored = _foreign
|
| 2022 |
-
if not admin:
|
| 2023 |
-
if _shared_view_role(_tops(ctx), view_id, _o_uname, uname,
|
| 2024 |
-
st=getattr(_tops(ctx), 'st', None)) != 'edit':
|
| 2025 |
-
ctx.out.toast = ('That view is shared with you read-only. Ask its '
|
| 2026 |
-
'owner for edit access.')
|
| 2027 |
-
return False
|
| 2028 |
-
view['createdBy'] = _stored.get('createdBy', _o_uname)
|
| 2029 |
-
_save_as = _o_uname
|
| 2030 |
if _store_of(ctx).available():
|
| 2031 |
# ⛔ W50-T25 / D8 — `_save_as` IS THE BUCKET THE WRITE LANDED IN and is therefore
|
| 2032 |
# the audit `owner`; `uname` is who DID it. On the foreign-owner branch above they
|
| 2033 |
# differ, and a row that conflated them would name the wrong person in the one
|
| 2034 |
# record whose entire job is naming the right one.
|
| 2035 |
-
# ⭐⭐ W51-T33 — `rev=` IS THE STAMP THE BROWSER ALREADY SENDS, finally read.
|
| 2036 |
-
# See `_event_rev`. The refusal is NOT caught: `main.py:232` turns
|
| 2037 |
-
# `ViewRevisionStale` into `409 view_revision_stale {rev_current}` for exactly this
|
| 2038 |
-
# door, and `RevisionUnverifiable` into a 503 beside it, so both reach the client in
|
| 2039 |
-
# the shape `routes_tables.py`'s direct view routes already answer.
|
| 2040 |
-
# S82-T156: a stale stamp that undoes no later change lands (D-634).
|
| 2041 |
view = (_tops(ctx).save_view(
|
| 2042 |
_save_as, view, reserved_names=_reserved_view_names,
|
| 2043 |
-
actor=uname, origin=EVENT_ORIGIN
|
| 2044 |
-
rebase=True) or view)
|
| 2045 |
else:
|
| 2046 |
_session_ready()
|
| 2047 |
_view_names = list(_reserved_view_names)
|
|
@@ -2307,8 +2846,9 @@ def handle_one(event, ctx):
|
|
| 2307 |
if ftype in ('link', 'rollup') or (ftype == 'if' and scope_key.startswith('ut_')):
|
| 2308 |
return _refuse(
|
| 2309 |
ctx, 'wrong_door',
|
| 2310 |
-
'
|
| 2311 |
-
'
|
|
|
|
| 2312 |
readonly = ftype in _ag2.READONLY_CUSTOM_TYPES
|
| 2313 |
# The editable family must SAY it is overlay (the client's own contract); the
|
| 2314 |
# read-only pair (formula / created_time) is stored under the cohort column's
|
|
@@ -2594,7 +3134,8 @@ def handle_one(event, ctx):
|
|
| 2594 |
fresh_fields = _shared_overlay.fields(shared_storage_key, st=ctx.table.st) or {}
|
| 2595 |
else:
|
| 2596 |
fresh_fields = (table_workspace(
|
| 2597 |
-
ctx, allowed_pids=allowed_pids, consume_corrections=False
|
|
|
|
| 2598 |
durable_field = fresh_fields.get(key)
|
| 2599 |
if source_backed:
|
| 2600 |
durable_field = _source_echo(field, durable_field)
|
|
@@ -3073,67 +3614,21 @@ def handle_one(event, ctx):
|
|
| 3073 |
placed, clean,
|
| 3074 |
{sfc: _own_ids(sfc, cur_ws) for sfc in _agf.FOLDER_SURFACES})
|
| 3075 |
|
| 3076 |
-
#
|
| 3077 |
-
|
| 3078 |
-
|
| 3079 |
-
|
| 3080 |
-
|
| 3081 |
-
|
| 3082 |
-
|
| 3083 |
-
|
| 3084 |
-
|
| 3085 |
-
and _store_of(ctx).available() else None)
|
| 3086 |
-
_res = _mutate(table_workspace(ctx, consume_corrections=False))
|
| 3087 |
-
if _res is None:
|
| 3088 |
-
# Refused: nothing is written — and now the caller is TOLD, with the cause `_mutate`
|
| 3089 |
-
# recorded at the line that decided it. The fallback exists so a future `return None`
|
| 3090 |
-
# added without a `_no(...)` degrades to a generic sentence rather than to silence.
|
| 3091 |
return _folder_refuse(
|
| 3092 |
ctx, kind,
|
| 3093 |
why.get('code') or 'folder_change_refused',
|
| 3094 |
why.get('message') or ('That folder change could not be saved, so nothing was '
|
| 3095 |
'altered. Reload the page and try again.'),
|
| 3096 |
key=why.get('key') or '')
|
| 3097 |
-
# ⭐⭐ W41-T12's HANDOVER, TAKEN. `save_folders` VALIDATES before it writes and RAISES
|
| 3098 |
-
# `FolderTreeError` on a cycle or an over-deep chain, having written nothing. Letting that
|
| 3099 |
-
# escape would 500 the whole event window; catching it and dropping the result would put
|
| 3100 |
-
# it straight back into the silent-200 path this ticket exists to close. `str(exc)` is
|
| 3101 |
-
# already the sentence a person reads and already C7-clean, and `folder_id` is an
|
| 3102 |
-
# attribute, so nothing is parsed out of the message.
|
| 3103 |
-
#
|
| 3104 |
-
# ⭐⭐ REACHABLE NOW, AND THIS COMMENT SAID THE OPPOSITE UNTIL /wave-validate CHANGED IT.
|
| 3105 |
-
# It read: 'UNREACHABLE FROM THIS DOOR TODAY ... `clean_folders` STRIPS `parent` entirely,
|
| 3106 |
-
# so no nesting can reach the validator through here and neither raise can fire. This is
|
| 3107 |
-
# the forward contract with T12: the day `clean_folders` learns `parent`, the refusal is
|
| 3108 |
-
# already wired and worded.'
|
| 3109 |
-
#
|
| 3110 |
-
# That day is 2026-08-25. `aios_grid.clean_folders` carries `parent` through, the
|
| 3111 |
-
# `folder_create` branch above forwards it, and `validate_folder_tree` now fires for real:
|
| 3112 |
-
# an 11-deep chain raises `too_deep` naming the folder, a cycle raises `cycle`, and a
|
| 3113 |
-
# dangling parent REPAIRS to root with a receipt this branch turns into a toast. Driven
|
| 3114 |
-
# both ways, including the negative control that a payload with no `parent` anywhere comes
|
| 3115 |
-
# back byte-identical with zero repairs.
|
| 3116 |
-
#
|
| 3117 |
-
# ⛔ LEFT AS A CORRECTION RATHER THAN DELETED. A comment asserting a guard is dead is
|
| 3118 |
-
# exactly what stops the next reader checking, and this one would have outlived the fix.
|
| 3119 |
-
try:
|
| 3120 |
-
# ⭐⭐ W51-T33 — THE SECOND STAMPED VIEW WRITE, and it is stamped by the client's
|
| 3121 |
-
# own decision: `CustomerGrid.tsx:10075` emits the views-rail `item_move` through
|
| 3122 |
-
# `stampViewWrite`, with the comment *"`move` is one of C3's seven audited actions
|
| 3123 |
-
# and it is a view WRITE like any other"*. `save_folders` has taken `rev=` since
|
| 3124 |
-
# W50-T25, so the value had a door on both sides and no line between them.
|
| 3125 |
-
# ⚠ THE OTHER FOUR KINDS THAT LAND HERE ARE UNAFFECTED. `folder_create`,
|
| 3126 |
-
# `folder_rename`, `folder_delete` and `folder_reorder` carry no stamp, and neither
|
| 3127 |
-
# does `ViewSidebar.tsx:2022`'s folder-nest `item_move`, so `_event_rev` answers None
|
| 3128 |
-
# for every one of them and the comparison is not asked — byte-identical behaviour.
|
| 3129 |
-
_receipt = _tops(ctx).save_folders(uname, _res[0], _res[1],
|
| 3130 |
-
actor=uname, origin=EVENT_ORIGIN,
|
| 3131 |
-
rev=_event_rev(event),
|
| 3132 |
-
rebase_at=_tree_rev)
|
| 3133 |
-
except _tstore.FolderTreeError as _fte:
|
| 3134 |
-
return _folder_refuse(ctx, kind,
|
| 3135 |
-
str(getattr(_fte, 'reason', '') or 'folder_tree_illegal'),
|
| 3136 |
-
str(_fte), key=str(getattr(_fte, 'folder_id', '') or ''))
|
| 3137 |
# ⭐⭐ THE DURABILITY HALF — the owner's actual complaint. See `_commit_folder_write`: the
|
| 3138 |
# write above lands in `Store._cache` and the hub commit is deferred by up to ~22s, so
|
| 3139 |
# a second process (or the same one after a restart) cannot read it back. Forced here,
|
|
@@ -3903,7 +4398,8 @@ def handle_one(event, ctx):
|
|
| 3903 |
_history_after.update(updates)
|
| 3904 |
_history_after.update(dict(row_events))
|
| 3905 |
_history_st = ctx.st
|
| 3906 |
-
if
|
|
|
|
| 3907 |
# One receipt carries every committed field delta, even when legacy hooks fan out.
|
| 3908 |
changes = {key: {'before': _history_before.get(key),
|
| 3909 |
'after': _history_after.get(key)}
|
|
@@ -3914,7 +4410,7 @@ def handle_one(event, ctx):
|
|
| 3914 |
'type': 'event_field', 'table': _history_table, 'rowId': str(pid),
|
| 3915 |
'field': _k, 'after': _v, 'st': ctx.table.st, 'user': uname,
|
| 3916 |
'receiptId': event_id, 'changes': changes})
|
| 3917 |
-
|
| 3918 |
from core import received_changes
|
| 3919 |
changes = {key: {'before': _history_before.get(key), 'after': _history_after.get(key)}
|
| 3920 |
for key in set(updates) | {key for key, _value in row_events}
|
|
|
|
| 39 |
"""
|
| 40 |
import datetime as dt
|
| 41 |
import json as _json_ev # wave 23 (C7): the json field's parse wall
|
| 42 |
+
import re
|
| 43 |
from dataclasses import dataclass, field
|
| 44 |
+
from contextlib import contextmanager, nullcontext
|
| 45 |
from functools import wraps
|
| 46 |
# ⚠ ALIASED ON PURPOSE. `field` is imported above for the dataclasses that need it, and this file
|
| 47 |
# uses `field` as a LOCAL VARIABLE in a dozen handlers (`field = {'key': key, …}`). A
|
|
|
|
| 138 |
write_id: str = ''
|
| 139 |
|
| 140 |
|
| 141 |
+
class AdmissionSnapshot:
|
| 142 |
+
"""Lend one request's workspace and grants to admission, never to a mutation callback."""
|
| 143 |
+
|
| 144 |
+
def __init__(self, runtime, uname, table_key, *, schema_only=False):
|
| 145 |
+
self._runtime, self.uname, self.table_key = runtime, uname, table_key
|
| 146 |
+
self.schema_only = bool(schema_only)
|
| 147 |
+
self._documents = {}
|
| 148 |
+
self._grid_admission = self
|
| 149 |
+
|
| 150 |
+
def __getattr__(self, name):
|
| 151 |
+
return getattr(self._runtime, name)
|
| 152 |
+
|
| 153 |
+
def _holds(self, name):
|
| 154 |
+
return name in (self.table_key, 'object_shares')
|
| 155 |
+
|
| 156 |
+
def invalidate(self, *names):
|
| 157 |
+
"""Discard admission documents after canonical writes or fresh reads."""
|
| 158 |
+
for name in names or tuple(self._documents):
|
| 159 |
+
self._documents.pop(name, None)
|
| 160 |
+
|
| 161 |
+
def _document(self, name):
|
| 162 |
+
from copy import deepcopy
|
| 163 |
+
if name not in self._documents:
|
| 164 |
+
if self.schema_only and name == self.table_key:
|
| 165 |
+
value = self._runtime.get_projection(name, drop=('overlays',))
|
| 166 |
+
else:
|
| 167 |
+
read = getattr(self._runtime, '_read_strict', self._runtime.get)
|
| 168 |
+
value = read(name)
|
| 169 |
+
self._documents[name] = value or {}
|
| 170 |
+
return deepcopy(self._documents[name])
|
| 171 |
+
|
| 172 |
+
def get(self, name, fresh=False):
|
| 173 |
+
if fresh:
|
| 174 |
+
self.invalidate(name)
|
| 175 |
+
if self._holds(name) and not fresh:
|
| 176 |
+
return self._document(name)
|
| 177 |
+
return self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)
|
| 178 |
+
|
| 179 |
+
def _read_strict(self, name):
|
| 180 |
+
if self._holds(name):
|
| 181 |
+
return self._document(name)
|
| 182 |
+
return getattr(self._runtime, '_read_strict', self._runtime.get)(name)
|
| 183 |
+
|
| 184 |
+
def get_entry(self, name, entry):
|
| 185 |
+
if self._holds(name):
|
| 186 |
+
return self._document(name).get(str(entry))
|
| 187 |
+
return self._runtime.get_entry(name, entry)
|
| 188 |
+
|
| 189 |
+
def get_projection(self, name, drop=()):
|
| 190 |
+
if self._holds(name):
|
| 191 |
+
from core.store import _project
|
| 192 |
+
return _project(self._document(name), frozenset(drop))
|
| 193 |
+
return self._runtime.get_projection(name, drop=drop)
|
| 194 |
+
|
| 195 |
+
def update(self, name, *args, **kwargs):
|
| 196 |
+
try:
|
| 197 |
+
return self._runtime.update(name, *args, **kwargs)
|
| 198 |
+
finally:
|
| 199 |
+
self.invalidate(name)
|
| 200 |
+
|
| 201 |
+
def put(self, name, *args, **kwargs):
|
| 202 |
+
try:
|
| 203 |
+
return self._runtime.put(name, *args, **kwargs)
|
| 204 |
+
finally:
|
| 205 |
+
self.invalidate(name)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def request_admission(session, table_key, *, schema_only=False):
|
| 209 |
+
"""Bind a detached host session to a fresh tenant, user and table admission snapshot."""
|
| 210 |
+
from copy import copy
|
| 211 |
+
actor = copy(session)
|
| 212 |
+
runtime = session.runtime
|
| 213 |
+
if not (isinstance(runtime, AdmissionSnapshot) and runtime.uname == session.uname
|
| 214 |
+
and runtime.table_key == table_key and runtime.schema_only == bool(schema_only)):
|
| 215 |
+
while isinstance(runtime, AdmissionSnapshot):
|
| 216 |
+
runtime = runtime._runtime
|
| 217 |
+
runtime = AdmissionSnapshot(runtime, session.uname, table_key, schema_only=schema_only)
|
| 218 |
+
actor.runtime = runtime
|
| 219 |
+
return actor
|
| 220 |
+
|
| 221 |
+
|
| 222 |
@dataclass
|
| 223 |
class EventCtx:
|
| 224 |
"""Everything the handler is allowed to know, and nothing about how it is being served.
|
|
|
|
| 259 |
# Core can normalise a recurrence's finite shape, but it cannot decide whether a named
|
| 260 |
# same/linked field is visible to this caller without reaching upward into the API layer.
|
| 261 |
recurrence_validator: Optional[Callable[[object], Optional[dict]]] = None
|
| 262 |
+
record_admitter: Optional[Callable[[int], bool]] = None
|
| 263 |
# Value-history restore reuses the ordinary cell wall. This stamp changes only the audit
|
| 264 |
# event kind and admits ``None`` as "remove the prior cell" after every normal field/row
|
| 265 |
# permission check has passed; browser-authored edits never set it.
|
|
|
|
| 291 |
durable_receipts: bool = False
|
| 292 |
#: The route's assembly workspace, lent to the first event only (`handle_one`).
|
| 293 |
workspace_lend: Optional[dict] = None
|
| 294 |
+
admission: Optional[AdmissionSnapshot] = None
|
| 295 |
|
| 296 |
def __post_init__(self):
|
| 297 |
# A caller that forgets `seen_ids` gets per-call dedup rather than a crash — the SET is
|
|
|
|
| 323 |
return inner if inner is not None else store
|
| 324 |
|
| 325 |
|
| 326 |
+
def _admission_ctx(ctx):
|
| 327 |
+
"""Copy only admission readers while keeping the mutation context on its canonical store."""
|
| 328 |
+
snapshot = getattr(ctx, 'admission', None)
|
| 329 |
+
if snapshot is None:
|
| 330 |
+
return ctx
|
| 331 |
+
from copy import copy
|
| 332 |
+
from dataclasses import replace
|
| 333 |
+
st = _store_of(ctx)
|
| 334 |
+
from core.edit_receipts import CheckedStore
|
| 335 |
+
if isinstance(st, CheckedStore):
|
| 336 |
+
st = st.st
|
| 337 |
+
while hasattr(st, '_runtime'):
|
| 338 |
+
st = st._runtime
|
| 339 |
+
source = snapshot._runtime
|
| 340 |
+
while hasattr(source, '_runtime'):
|
| 341 |
+
source = source._runtime
|
| 342 |
+
if (st is not source or snapshot.uname != ctx.uname
|
| 343 |
+
or snapshot.table_key != _tops(ctx).table_key):
|
| 344 |
+
raise ValueError('admission snapshot does not match this tenant, user and table')
|
| 345 |
+
table = copy(_tops(ctx))
|
| 346 |
+
table._st = snapshot
|
| 347 |
+
return replace(ctx, st=snapshot, table=table, admission=None)
|
| 348 |
+
|
| 349 |
+
|
| 350 |
def _blobs(ctx):
|
| 351 |
"""The BYTE store for this ctx — `_store_of`'s sibling, split out because it is NOT yet
|
| 352 |
equivalent and pretending otherwise would hide a live residency hole.
|
|
|
|
| 388 |
EVENT_ORIGIN = {'client': 'grid-events', 'ip': None}
|
| 389 |
|
| 390 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
def _tops(ctx):
|
| 392 |
"""The table-ops object this ctx operates on — the customer table unless the caller chose
|
| 393 |
another topic. ONE resolution point so 'which bucket does this write land in' has exactly
|
|
|
|
| 534 |
_storage_key, topic = _field_share_keys(ctx)
|
| 535 |
return _shares.role_for(
|
| 536 |
'field', _shares.field_oid(topic, key), ctx.uname,
|
| 537 |
+
is_admin=ctx.admin, st=_store_of(_admission_ctx(ctx)))
|
| 538 |
except Exception:
|
| 539 |
return None
|
| 540 |
|
|
|
|
| 801 |
|
| 802 |
def workspace_corrections(st, table_key, uname, assembly):
|
| 803 |
"""Only the workspace render wire spends pending acknowledgements, after the memo read."""
|
| 804 |
+
# Only the memoised path sets the flag. An absent flag is unknown, so read the workspace.
|
| 805 |
+
if assembly.get('_pending_corrections') is False:
|
| 806 |
return assembly
|
| 807 |
from core.table_store import TableStore
|
| 808 |
fields = TableStore(table_key, st=st).workspace(uname, consume_corrections=True)['fields']
|
|
|
|
| 821 |
|
| 822 |
|
| 823 |
def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False,
|
| 824 |
+
*, include_views=True, admission=True):
|
| 825 |
"""Durable Airtable-style view/schema/overlay state, with an honest session fallback.
|
| 826 |
|
| 827 |
Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED
|
|
|
|
| 846 |
site because app.py computes `admin = bool(is_admin())` once per render. `allowed_pids`
|
| 847 |
stays an EXPLICIT argument because the folder leg deliberately omits it.
|
| 848 |
"""
|
| 849 |
+
original_ctx = ctx
|
| 850 |
+
ctx = _admission_ctx(ctx) if admission else ctx
|
| 851 |
uname = ctx.uname
|
| 852 |
if _store_of(ctx).available():
|
| 853 |
# Wave 48 C3: this is the only runtime transition door. It either promotes every
|
|
|
|
| 858 |
include_document=bool(capture_document))
|
| 859 |
if capture_document:
|
| 860 |
ws, ctx.workspace_document = read
|
| 861 |
+
original_ctx.workspace_document = ctx.workspace_document
|
| 862 |
else:
|
| 863 |
ws = read
|
| 864 |
if not include_views:
|
|
|
|
| 1010 |
}
|
| 1011 |
|
| 1012 |
|
| 1013 |
+
_SERVER_VIEW_MEMBERS = frozenset({
|
| 1014 |
+
'cohortLock', 'createdBy', 'updatedAt', 'filterRemovals', 'kind',
|
| 1015 |
+
})
|
| 1016 |
+
_VIEW_POSITION = re.compile(r'[0-9A-Za-z]{1,64}\Z')
|
| 1017 |
+
|
| 1018 |
+
|
| 1019 |
+
@dataclass
|
| 1020 |
+
class _ViewMemberCtx:
|
| 1021 |
+
"""Lend create-time filter accounting without changing the request context."""
|
| 1022 |
+
source: EventCtx
|
| 1023 |
+
filter_removals: list = field(default_factory=list)
|
| 1024 |
+
filter_budget: Optional[list] = None
|
| 1025 |
+
|
| 1026 |
+
def __getattr__(self, name):
|
| 1027 |
+
return getattr(self.source, name)
|
| 1028 |
+
|
| 1029 |
+
|
| 1030 |
+
def _view_member_refusal(member, code, message):
|
| 1031 |
+
return None, {'member': member, 'code': code, 'message': message}
|
| 1032 |
+
|
| 1033 |
+
|
| 1034 |
+
def _view_field_key(key, valid_keys):
|
| 1035 |
+
return '__zone__' if key == 'zone' and '__zone__' in valid_keys else key
|
| 1036 |
+
|
| 1037 |
+
|
| 1038 |
+
def _clean_view_sort(value):
|
| 1039 |
+
if not isinstance(value, dict) or value.get('dir') not in ('asc', 'desc'):
|
| 1040 |
+
return None
|
| 1041 |
+
result = {'dir': value['dir']}
|
| 1042 |
+
# Legacy sort arrays have no position. Explicit positions follow the C1 alphabet.
|
| 1043 |
+
if 'pos' in value:
|
| 1044 |
+
if not isinstance(value['pos'], str) or not _VIEW_POSITION.fullmatch(value['pos']):
|
| 1045 |
+
return None
|
| 1046 |
+
result['pos'] = value['pos']
|
| 1047 |
+
if value.get('locked') is True:
|
| 1048 |
+
result['locked'] = True
|
| 1049 |
+
return result
|
| 1050 |
+
|
| 1051 |
+
|
| 1052 |
+
def _clean_view_summary(value):
|
| 1053 |
+
import aios_grid as _ag
|
| 1054 |
+
if not isinstance(value, dict):
|
| 1055 |
+
return None
|
| 1056 |
+
agg = value.get('agg')
|
| 1057 |
+
# Share is a stored picker choice. It is not a query aggregate.
|
| 1058 |
+
if agg not in (*_ag.SUMMARY_AGGS, 'none', 'share'):
|
| 1059 |
+
return None
|
| 1060 |
+
result = {'agg': agg}
|
| 1061 |
+
if agg == 'percentile':
|
| 1062 |
+
try:
|
| 1063 |
+
result['p'] = max(0, min(100, round(float(value.get('p')))))
|
| 1064 |
+
except (TypeError, ValueError, OverflowError):
|
| 1065 |
+
return None
|
| 1066 |
+
return result
|
| 1067 |
+
|
| 1068 |
+
|
| 1069 |
+
def _clean_view_filter(value, ctx, valid_keys):
|
| 1070 |
+
import aios_grid as _ag
|
| 1071 |
+
node = value['node']
|
| 1072 |
+
# C1 creates an empty group before separate operations attach its children.
|
| 1073 |
+
if node.get('children') == []:
|
| 1074 |
+
cleaned = {'conj': 'or' if node.get('conj') == 'or' else 'and', 'children': []}
|
| 1075 |
+
if isinstance(node.get('id'), str):
|
| 1076 |
+
cleaned['id'] = node['id'][:64]
|
| 1077 |
+
return {'node': cleaned, 'parent': value.get('parent')}
|
| 1078 |
+
migrated = _ag.migrate_zone_projection_config({'filters': [node]})['filters']
|
| 1079 |
+
filter_removals = getattr(ctx, 'filter_removals', None)
|
| 1080 |
+
if filter_removals is None:
|
| 1081 |
+
filter_removals = []
|
| 1082 |
+
# Measures belong to filters only. Every other member uses table fields.
|
| 1083 |
+
nodes = _ag.clean_filter_tree(
|
| 1084 |
+
migrated, valid_keys | set(ctx.measure_keys),
|
| 1085 |
+
budget=getattr(ctx, 'filter_budget', None), cohort_ids=set(ctx.cohort_ids),
|
| 1086 |
+
visible_view_ids={str(view['id']) for view in ctx.visible_views
|
| 1087 |
+
if isinstance(view, dict) and view.get('id')},
|
| 1088 |
+
report=filter_removals)
|
| 1089 |
+
if not nodes:
|
| 1090 |
+
return None
|
| 1091 |
+
cleaned = nodes[0]
|
| 1092 |
+
if isinstance(node.get('children'), list) and isinstance(node.get('id'), str):
|
| 1093 |
+
cleaned['id'] = node['id'][:64]
|
| 1094 |
+
return {'node': cleaned, 'parent': value.get('parent')}
|
| 1095 |
+
|
| 1096 |
+
|
| 1097 |
+
def clean_view_member(member, value, ctx, prior):
|
| 1098 |
+
"""Return one cleaned C1 member or a refusal before either write door applies it."""
|
| 1099 |
+
def refuse(code, message):
|
| 1100 |
+
return _view_member_refusal(member, code, message)
|
| 1101 |
+
|
| 1102 |
+
if not isinstance(member, str):
|
| 1103 |
+
return refuse('unknown_member', 'Choose a supported View setting.')
|
| 1104 |
+
family, separator, key = member.partition(':')
|
| 1105 |
+
if family in _SERVER_VIEW_MEMBERS:
|
| 1106 |
+
return refuse('server_member', 'The server sets this View setting.')
|
| 1107 |
+
field_families = ('hidden', 'sort', 'width', 'position', 'summary')
|
| 1108 |
+
scalar_families = ('group', 'display', 'name', 'note', 'locked', 'colorBy',
|
| 1109 |
+
'rowHeightMode', 'frozenCount', 'important', 'secondaryView',
|
| 1110 |
+
'application', 'fullscreen', 'filterConj')
|
| 1111 |
+
if (family not in (*field_families, *scalar_families, 'filter', 'memberPid')
|
| 1112 |
+
or (family in (*field_families, 'filter', 'memberPid') and not separator)
|
| 1113 |
+
or (family in scalar_families and separator and family != 'filterConj')):
|
| 1114 |
+
return refuse('unknown_member', 'Choose a supported View setting.')
|
| 1115 |
+
valid_keys = {item['key'] for item in ctx.fields}
|
| 1116 |
+
if family in field_families:
|
| 1117 |
+
key = _view_field_key(key, valid_keys)
|
| 1118 |
+
if key not in valid_keys:
|
| 1119 |
+
return refuse('unknown_field', 'Choose a field in this table.')
|
| 1120 |
+
if family == 'hidden' or family in ('locked', 'important', 'fullscreen'):
|
| 1121 |
+
if not isinstance(value, bool):
|
| 1122 |
+
return refuse('invalid_boolean', 'Choose true or false for this View setting.')
|
| 1123 |
+
return value, None
|
| 1124 |
+
if family == 'sort':
|
| 1125 |
+
cleaned = _clean_view_sort(value)
|
| 1126 |
+
return ((cleaned, None) if cleaned is not None else
|
| 1127 |
+
refuse('invalid_sort', 'Choose a sort direction and a valid position.'))
|
| 1128 |
+
if family == 'width':
|
| 1129 |
+
try:
|
| 1130 |
+
return max(60, min(600, round(float(value)))), None
|
| 1131 |
+
except (TypeError, ValueError, OverflowError):
|
| 1132 |
+
return refuse('invalid_width', 'Choose a numeric column width.')
|
| 1133 |
+
if family == 'position':
|
| 1134 |
+
if isinstance(value, str) and _VIEW_POSITION.fullmatch(value):
|
| 1135 |
+
return value, None
|
| 1136 |
+
return refuse('invalid_position', 'Use 1 to 64 letters or digits for the column position.')
|
| 1137 |
+
if family == 'summary':
|
| 1138 |
+
cleaned = _clean_view_summary(value)
|
| 1139 |
+
return ((cleaned, None) if cleaned is not None else
|
| 1140 |
+
refuse('invalid_summary', 'Choose a supported column summary.'))
|
| 1141 |
+
if family == 'memberPid':
|
| 1142 |
+
try:
|
| 1143 |
+
pid = int(key)
|
| 1144 |
+
except (TypeError, ValueError):
|
| 1145 |
+
pid = None
|
| 1146 |
+
admit_record = getattr(ctx, 'record_admitter', None)
|
| 1147 |
+
if pid not in (ctx.allowed_pids or ()) and not (
|
| 1148 |
+
isinstance(pid, int) and 0 < pid < 2**63 and callable(admit_record)
|
| 1149 |
+
and admit_record(pid)):
|
| 1150 |
+
return refuse('unknown_record', 'Choose a record available in this table.')
|
| 1151 |
+
if not isinstance(value, bool):
|
| 1152 |
+
return refuse('invalid_boolean', 'Choose true or false for record membership.')
|
| 1153 |
+
return value, None
|
| 1154 |
+
if family == 'filter':
|
| 1155 |
+
if (not key or len(key) > 64 or not isinstance(value, dict)
|
| 1156 |
+
or not isinstance(value.get('node'), dict)
|
| 1157 |
+
or (value.get('parent') is not None and
|
| 1158 |
+
(not isinstance(value['parent'], str) or not value['parent']
|
| 1159 |
+
or len(value['parent']) > 64))):
|
| 1160 |
+
return refuse('invalid_filter', 'Choose a valid filter condition and parent group.')
|
| 1161 |
+
try:
|
| 1162 |
+
cleaned = _clean_view_filter(value, ctx, valid_keys)
|
| 1163 |
+
except (TypeError, ValueError, OverflowError):
|
| 1164 |
+
cleaned = None
|
| 1165 |
+
return ((cleaned, None) if cleaned is not None else
|
| 1166 |
+
refuse('invalid_filter', 'Choose a filter available in this table.'))
|
| 1167 |
+
if family == 'filterConj':
|
| 1168 |
+
if (separator and (not key or len(key) > 64)) or value not in ('and', 'or'):
|
| 1169 |
+
return refuse('invalid_conjunction', 'Choose "and" or "or" for the filter group.')
|
| 1170 |
+
return value, None
|
| 1171 |
+
if family in ('group', 'colorBy'):
|
| 1172 |
+
value = _view_field_key(value, valid_keys)
|
| 1173 |
+
if value is None or (isinstance(value, str) and value in valid_keys):
|
| 1174 |
+
return value, None
|
| 1175 |
+
return refuse('unknown_field', 'Choose a field in this table.')
|
| 1176 |
+
if family == 'name':
|
| 1177 |
+
name = str(value or '').strip()[:120]
|
| 1178 |
+
return ((name, None) if name else
|
| 1179 |
+
refuse('invalid_name', 'Give this View a name.'))
|
| 1180 |
+
if family == 'note':
|
| 1181 |
+
return str(value or '')[:2000], None
|
| 1182 |
+
if family == 'rowHeightMode':
|
| 1183 |
+
if value in ('compact', 'short', 'medium', 'tall'):
|
| 1184 |
+
return value, None
|
| 1185 |
+
return refuse('invalid_row_height', 'Choose a supported row height.')
|
| 1186 |
+
if family == 'frozenCount':
|
| 1187 |
+
try:
|
| 1188 |
+
return max(1, min(8, int(value))), None
|
| 1189 |
+
except (TypeError, ValueError, OverflowError):
|
| 1190 |
+
return refuse('invalid_frozen_count', 'Choose a numeric frozen column count.')
|
| 1191 |
+
if family == 'secondaryView':
|
| 1192 |
+
try:
|
| 1193 |
+
cleaned = _clean_secondary_view(value)
|
| 1194 |
+
except (TypeError, ValueError, OverflowError):
|
| 1195 |
+
cleaned = None
|
| 1196 |
+
return ((cleaned, None) if value is None or cleaned is not None else
|
| 1197 |
+
refuse('invalid_secondary_view', 'Choose a related database and linking field.'))
|
| 1198 |
+
if family == 'application':
|
| 1199 |
+
if value is None:
|
| 1200 |
+
return None, None
|
| 1201 |
+
from core import view_templates
|
| 1202 |
+
try:
|
| 1203 |
+
cleaned = view_templates.application_ref({'config': {'application': value}})
|
| 1204 |
+
except (TypeError, ValueError, OverflowError):
|
| 1205 |
+
cleaned = None
|
| 1206 |
+
if cleaned is None:
|
| 1207 |
+
return refuse('invalid_application_ref',
|
| 1208 |
+
'That Application reference is not a valid typed identity.')
|
| 1209 |
+
live = _live_application_ids(ctx)
|
| 1210 |
+
return (None if live is not None and cleaned['id'] not in live else cleaned), None
|
| 1211 |
+
if family == 'display':
|
| 1212 |
+
import aios_grid as _ag
|
| 1213 |
+
if isinstance(value, dict) and value.get('mode') == 'timeseries':
|
| 1214 |
+
return refuse('view_mode_retired',
|
| 1215 |
+
'Time-series Views are retired. Choose an available View display mode.')
|
| 1216 |
+
try:
|
| 1217 |
+
cleaned = _ag._clean_display(value, valid_keys)
|
| 1218 |
+
except (TypeError, ValueError, OverflowError):
|
| 1219 |
+
cleaned = None
|
| 1220 |
+
return ((cleaned, None) if value is None or cleaned is not None
|
| 1221 |
+
or value in ({'mode': 'grid'}, {'mode': 'grid', 'groupSummary': False}) else
|
| 1222 |
+
refuse('invalid_display', 'Choose an available View display mode.'))
|
| 1223 |
+
|
| 1224 |
+
|
| 1225 |
+
def _clean_view_config(cfg, ctx, prior, display):
|
| 1226 |
+
"""Project legacy create arrays through the C1 member cleaner."""
|
| 1227 |
+
import aios_grid as _ag
|
| 1228 |
+
cleaner_ctx = _ViewMemberCtx(ctx, filter_budget=[_ag.MAX_FILTER_NODES])
|
| 1229 |
+
|
| 1230 |
+
def clean(member, value, default=None):
|
| 1231 |
+
cleaned, refusal = clean_view_member(member, value, cleaner_ctx, prior)
|
| 1232 |
+
return default if refusal else cleaned
|
| 1233 |
+
|
| 1234 |
+
valid_keys = {item['key'] for item in ctx.fields}
|
| 1235 |
+
config = {
|
| 1236 |
+
'filters': [], 'filterConj': clean('filterConj', cfg.get('filterConj', 'and'), 'and'),
|
| 1237 |
+
'important': clean('important', cfg.get('important', False), False),
|
| 1238 |
+
'sorts': [], 'groupBy': clean('group', cfg.get('groupBy')),
|
| 1239 |
+
'colorBy': clean('colorBy', cfg.get('colorBy')),
|
| 1240 |
+
'rowHeightMode': clean('rowHeightMode', cfg.get('rowHeightMode', 'short'), 'short'),
|
| 1241 |
+
'visible': [], 'widths': {}, 'summary': {}, 'memberPids': [],
|
| 1242 |
+
'secondaryView': clean('secondaryView', cfg.get('secondaryView')),
|
| 1243 |
+
}
|
| 1244 |
+
raw_filters = cfg.get('filters') if isinstance(cfg.get('filters'), list) else []
|
| 1245 |
+
for index, node in enumerate(raw_filters[:_ag.MAX_FILTER_SIBLINGS]):
|
| 1246 |
+
node_id = str(node.get('id') or f'legacy-{index}') if isinstance(node, dict) else f'legacy-{index}'
|
| 1247 |
+
cleaned = clean('filter:' + node_id, {'node': node, 'parent': None})
|
| 1248 |
+
if cleaned is not None:
|
| 1249 |
+
config['filters'].append(cleaned['node'])
|
| 1250 |
+
raw_sorts = cfg.get('sorts') if isinstance(cfg.get('sorts'), list) else []
|
| 1251 |
+
for rule in raw_sorts[:20]:
|
| 1252 |
+
if not isinstance(rule, dict) or not isinstance(rule.get('colId'), str):
|
| 1253 |
+
continue
|
| 1254 |
+
key = _view_field_key(rule['colId'], valid_keys)
|
| 1255 |
+
cleaned = clean('sort:' + key, rule)
|
| 1256 |
+
if cleaned is not None:
|
| 1257 |
+
config['sorts'].append({'colId': key, **cleaned})
|
| 1258 |
+
# Order remains an array. S85-T02 owns position seeding and order derivation.
|
| 1259 |
+
order = cfg.get('order') if isinstance(cfg.get('order'), list) else []
|
| 1260 |
+
config['order'] = list(dict.fromkeys(
|
| 1261 |
+
_view_field_key(key, valid_keys) for key in order
|
| 1262 |
+
if isinstance(key, str) and _view_field_key(key, valid_keys) in valid_keys))
|
| 1263 |
+
config['order'] += [item['key'] for item in ctx.fields if item['key'] not in config['order']]
|
| 1264 |
+
visible = cfg.get('visible') if isinstance(cfg.get('visible'), list) else []
|
| 1265 |
+
for key in dict.fromkeys(_view_field_key(key, valid_keys) for key in visible if isinstance(key, str)):
|
| 1266 |
+
hidden = clean('hidden:' + key, False)
|
| 1267 |
+
if hidden is False:
|
| 1268 |
+
config['visible'].append(key)
|
| 1269 |
+
for source, family in (('widths', 'width'), ('summary', 'summary')):
|
| 1270 |
+
values = cfg.get(source) if isinstance(cfg.get(source), dict) else {}
|
| 1271 |
+
entries = list(values.items())[:400] if family == 'summary' else values.items()
|
| 1272 |
+
for key, value in entries:
|
| 1273 |
+
if not isinstance(key, str):
|
| 1274 |
+
continue
|
| 1275 |
+
key = _view_field_key(key, valid_keys)
|
| 1276 |
+
cleaned = clean(family + ':' + key, value)
|
| 1277 |
+
if cleaned is not None:
|
| 1278 |
+
config[source][key] = cleaned
|
| 1279 |
+
pids = cfg.get('memberPids') if isinstance(cfg.get('memberPids'), list) else []
|
| 1280 |
+
for pid in pids:
|
| 1281 |
+
if type(pid) is int and clean('memberPid:' + str(pid), True) is True:
|
| 1282 |
+
config['memberPids'].append(pid)
|
| 1283 |
+
application, refusal = clean_view_member('application', cfg.get('application'), cleaner_ctx, prior)
|
| 1284 |
+
if refusal:
|
| 1285 |
+
return None, cleaner_ctx.filter_removals, refusal
|
| 1286 |
+
if cfg.get('groupSummary') is True:
|
| 1287 |
+
# A legacy root toggle lands in the display member, where viewOps.ts reads it.
|
| 1288 |
+
display = {**(display or {'mode': 'grid'}), 'groupSummary': True}
|
| 1289 |
+
if application is not None:
|
| 1290 |
+
config['application'] = application
|
| 1291 |
+
elif display is not None:
|
| 1292 |
+
config['display'] = display
|
| 1293 |
+
if cfg.get('frozenCount') is not None:
|
| 1294 |
+
frozen = clean('frozenCount', cfg['frozenCount'])
|
| 1295 |
+
if frozen is not None:
|
| 1296 |
+
config['frozenCount'] = frozen
|
| 1297 |
+
if 'fullscreen' in cfg:
|
| 1298 |
+
config['fullscreen'] = clean('fullscreen', cfg['fullscreen'], False)
|
| 1299 |
+
return config, cleaner_ctx.filter_removals, None
|
| 1300 |
+
|
| 1301 |
+
|
| 1302 |
def _shared_folder_record(fid, folder):
|
| 1303 |
"""C9's wire record for a folder granted to somebody else: `{id, name, order?, icon?}`.
|
| 1304 |
|
|
|
|
| 1340 |
return role if role in ('view', 'edit') else None
|
| 1341 |
|
| 1342 |
|
| 1343 |
+
_VIEW_READ_ONLY_MESSAGE = "Ask the view's owner for edit access."
|
| 1344 |
+
|
| 1345 |
+
|
| 1346 |
+
def _may_edit_view(tops, view_id, owner, view, uname, admin):
|
| 1347 |
+
"""Admit the stored creator, an administrator or an edit grantee on this tenant's View."""
|
| 1348 |
+
creator = str(view.get('createdBy') or owner).strip().lower()
|
| 1349 |
+
return (admin or creator == str(uname).strip().lower()
|
| 1350 |
+
or _shared_view_role(tops, view_id, owner, uname,
|
| 1351 |
+
st=getattr(tops, 'st', None)) == 'edit')
|
| 1352 |
+
|
| 1353 |
+
|
| 1354 |
def _granted_views(ctx, uname):
|
| 1355 |
"""{view_id: STAMPED view} for every view GRANTED to `uname` by name (wave 21, item 9, C1).
|
| 1356 |
|
|
|
|
| 1369 |
import core.shares as shares
|
| 1370 |
tops = _tops(ctx)
|
| 1371 |
st = getattr(tops, 'st', None)
|
| 1372 |
+
snapshot = getattr(st, '_grid_admission', None)
|
| 1373 |
+
lookup = {}
|
| 1374 |
+
if (isinstance(snapshot, AdmissionSnapshot) and isinstance(tops, _tstore.TableStore)
|
| 1375 |
+
and tops._settings_rows is None):
|
| 1376 |
+
tops.migrate_legacy_shared_views()
|
| 1377 |
+
lookup['document'] = snapshot.get(tops.table_key)
|
| 1378 |
ids = (shares.shared_with(uname, kind='view', st=st) or {}).get('view') or []
|
| 1379 |
+
views = dict(lookup, settings=None) if lookup else {}
|
| 1380 |
+
if ids and not lookup and isinstance(tops, _tstore.TableStore):
|
| 1381 |
+
# S85-T169: resolve the settings backend and the workspace document once for all ids.
|
| 1382 |
+
views['settings'] = rows = tops._settings_rows
|
| 1383 |
+
if rows is None:
|
| 1384 |
+
tops.migrate_legacy_shared_views()
|
| 1385 |
+
try:
|
| 1386 |
+
views['document'] = st.get(tops.table_key) or {}
|
| 1387 |
+
except Exception:
|
| 1388 |
+
views = {'settings': None} # each find_view keeps its own fail-closed read
|
| 1389 |
out = {}
|
| 1390 |
for vid in ids:
|
| 1391 |
+
found = tops.find_view(vid, **views) if hasattr(tops, 'find_view') else None
|
| 1392 |
if not found:
|
| 1393 |
continue
|
| 1394 |
owner, view = found
|
|
|
|
| 1425 |
except Exception:
|
| 1426 |
folder_ids = []
|
| 1427 |
for fid in folder_ids:
|
| 1428 |
+
found = tops.find_folder(fid, **lookup) if hasattr(tops, 'find_folder') else None
|
| 1429 |
if not found:
|
| 1430 |
continue # a folder on another topic contributes nothing here
|
| 1431 |
owner, folder, inside = found
|
|
|
|
| 1596 |
try:
|
| 1597 |
out = getattr(ctx, 'out', None)
|
| 1598 |
if out is not None and hasattr(out, 'refusals'):
|
| 1599 |
+
if sum(item.get('event') != 'view_op' for item in out.refusals) < 24:
|
| 1600 |
out.refusals.append({
|
| 1601 |
'event': str(event or 'field_upsert')[:40],
|
| 1602 |
'key': str(key or '')[:80],
|
|
|
|
| 1790 |
return wrapped
|
| 1791 |
|
| 1792 |
|
| 1793 |
+
def _view_op_reply(ctx, result, *, rerender=False):
|
| 1794 |
+
"""Expose one C1 result without receipt history or a shared toast."""
|
| 1795 |
+
row = {key: result[key] for key in ('opId', 'member', 'applied') if key in result}
|
| 1796 |
+
row['rerender'] = bool(rerender)
|
| 1797 |
+
if result.get('refused'):
|
| 1798 |
+
row['refused'] = {key: result['refused'][key] for key in ('reason', 'member', 'message')}
|
| 1799 |
+
ctx.out.refusals.append({'event': 'view_op', **row['refused']})
|
| 1800 |
+
if isinstance(row.get('opId'), str):
|
| 1801 |
+
ctx.out.writes[row['opId']] = row
|
| 1802 |
+
return row
|
| 1803 |
+
|
| 1804 |
+
|
| 1805 |
+
def _admit_view_op(view, candidate, member, *, owner, actor, admin):
|
| 1806 |
+
"""Check the resolved operation against the current locked View inside its update."""
|
| 1807 |
+
import aios_grid as _ag
|
| 1808 |
+
family = member.partition(':')[0]
|
| 1809 |
+
if family == 'locked' and not (admin or str(view.get('createdBy') or owner).lower()
|
| 1810 |
+
== str(actor).lower()):
|
| 1811 |
+
return {'reason': 'lock_owner_only',
|
| 1812 |
+
'message': 'Only the View creator or an administrator can change its lock.'}
|
| 1813 |
+
cfg, next_cfg = view.get('config') or {}, candidate.get('config') or {}
|
| 1814 |
+
if (family in ('display', 'application') and view.get('locked')
|
| 1815 |
+
and view.get('kind') != 'system' and view.get('id') != 'all-customers'
|
| 1816 |
+
and _tstore._view_member_value(view, member) != _tstore._view_member_value(candidate, member)):
|
| 1817 |
+
return {'reason': 'view_mode_locked', 'message': 'Unlock this View before changing its display.'}
|
| 1818 |
+
filters, sorts = next_cfg.get('filters') or [], next_cfg.get('sorts') or []
|
| 1819 |
+
if (_ag.preserve_locked_view_records(cfg, filters, sorts) != (filters, sorts)
|
| 1820 |
+
or (family == 'filterConj' and filters
|
| 1821 |
+
and cfg.get('filterConj', 'and') != next_cfg.get('filterConj', 'and')
|
| 1822 |
+
and _ag.preserve_locked_view_records(cfg, [], sorts)[0])):
|
| 1823 |
+
return {'reason': 'locked_view_record',
|
| 1824 |
+
'message': 'Copy this View to change its locked filter or sort records.'}
|
| 1825 |
+
return None
|
| 1826 |
+
|
| 1827 |
+
|
| 1828 |
+
@contextmanager
|
| 1829 |
+
def _cohort_name_transaction(st):
|
| 1830 |
+
"""Keep a cohort View write and its synchronization on one bounded PostgreSQL connection."""
|
| 1831 |
+
from core import edit_receipts, store_pg
|
| 1832 |
+
backend = edit_receipts._backend(st)
|
| 1833 |
+
if isinstance(backend, edit_receipts.PgReceipts):
|
| 1834 |
+
with backend._connection() as con, con.transaction():
|
| 1835 |
+
with store_pg.PgStore(backend.slug).transaction(con):
|
| 1836 |
+
yield con, backend.slug
|
| 1837 |
+
else:
|
| 1838 |
+
yield None, None
|
| 1839 |
+
|
| 1840 |
+
|
| 1841 |
+
def _sync_cohort_view_name(ctx, tops, owner, view_id):
|
| 1842 |
+
"""Copy the canonical View name while its source remains locked through the cohort write."""
|
| 1843 |
+
from core import store_pg
|
| 1844 |
+
sets = _cohorts(ctx)
|
| 1845 |
+
cohort_name = (sets.all_for(owner).get(view_id) or {}).get('name')
|
| 1846 |
+
|
| 1847 |
+
def synchronize(data):
|
| 1848 |
+
view = ((data.get(owner) or {}).get('views') or {}).get(view_id) or {}
|
| 1849 |
+
name = view.get('name')
|
| 1850 |
+
if name and cohort_name != name:
|
| 1851 |
+
sets.rename(owner, view_id, name)
|
| 1852 |
+
return True
|
| 1853 |
+
return False
|
| 1854 |
+
|
| 1855 |
+
with _cohort_name_transaction(tops.st) as (con, slug):
|
| 1856 |
+
if con is not None:
|
| 1857 |
+
rows = tops._settings_rows
|
| 1858 |
+
if rows is not None:
|
| 1859 |
+
with rows._reading():
|
| 1860 |
+
synchronize(rows._settings(con, (owner,)))
|
| 1861 |
+
else:
|
| 1862 |
+
row = con.execute(store_pg._q(
|
| 1863 |
+
'SELECT value FROM {tbl} WHERE key = %s FOR UPDATE',
|
| 1864 |
+
'store_kv', slug), (tops.table_key,)).fetchone()
|
| 1865 |
+
synchronize(row[0] if row else {})
|
| 1866 |
+
else:
|
| 1867 |
+
# Detached stores provide source exclusion through their update contract.
|
| 1868 |
+
class Unchanged(Exception):
|
| 1869 |
+
pass
|
| 1870 |
+
|
| 1871 |
+
def change(data, _audit):
|
| 1872 |
+
if not synchronize(data):
|
| 1873 |
+
raise Unchanged
|
| 1874 |
+
return data
|
| 1875 |
+
|
| 1876 |
+
try:
|
| 1877 |
+
tops._audited_update(change, flush='sync', owners=(owner,))
|
| 1878 |
+
except Unchanged:
|
| 1879 |
+
pass
|
| 1880 |
+
|
| 1881 |
+
|
| 1882 |
+
def handle_view_ops(events, ctx):
|
| 1883 |
+
"""Authorize one View batch and apply its cleaned members through the owner-row writer."""
|
| 1884 |
+
from copy import deepcopy
|
| 1885 |
+
from core import edit_receipts, view_settings_rows
|
| 1886 |
+
import aios_grid as _ag
|
| 1887 |
+
from harness import measure_filter as _mfc
|
| 1888 |
+
|
| 1889 |
+
events = [dict(event, client_op_id=event.get(
|
| 1890 |
+
'client_op_id', event.get('opId', event.get('id')))) for event in events]
|
| 1891 |
+
if not events:
|
| 1892 |
+
return []
|
| 1893 |
+
ctx.workspace_lend = None
|
| 1894 |
+
view_id = str(events[0].get('view_id') or '').strip()
|
| 1895 |
+
|
| 1896 |
+
def refusal(event, reason, message):
|
| 1897 |
+
return {'opId': event.get('client_op_id'), 'member': event.get('member'),
|
| 1898 |
+
'applied': False, 'refused': {
|
| 1899 |
+
'reason': reason, 'member': event.get('member'), 'message': message}}
|
| 1900 |
+
|
| 1901 |
+
def refuse_all(reason, message):
|
| 1902 |
+
return [_view_op_reply(ctx, refusal(event, reason, message)) for event in events]
|
| 1903 |
+
|
| 1904 |
+
if not _store_of(ctx).available():
|
| 1905 |
+
raise StoreUnavailable('The View store is unavailable.')
|
| 1906 |
+
tops = _tops(ctx)
|
| 1907 |
+
admission_tops = _tops(_admission_ctx(ctx))
|
| 1908 |
+
owner = ctx.uname
|
| 1909 |
+
workspace = admission_tops._read_owners(owner).get(owner) or {}
|
| 1910 |
+
prior = (workspace.get('views') or {}).get(view_id)
|
| 1911 |
+
projected = next((view for view in ctx.visible_views
|
| 1912 |
+
if isinstance(view, dict) and view.get('id') == view_id), {})
|
| 1913 |
+
_pinned_per_user = ((prior or projected).get('kind') == 'system'
|
| 1914 |
+
or view_id in ('all-customers', _ag_overview_id()))
|
| 1915 |
+
assigned = admission_tops.assigned_views(ctx.uname, ctx.allowed_pids)
|
| 1916 |
+
is_assigned = view_id in assigned
|
| 1917 |
+
if is_assigned:
|
| 1918 |
+
prior = assigned[view_id]
|
| 1919 |
+
workspace = admission_tops._read_owners(_tstore.ASSIGNED_KEY).get(_tstore.ASSIGNED_KEY) or {}
|
| 1920 |
+
elif view_id.startswith('assigned:') or projected.get('assigned'):
|
| 1921 |
+
return refuse_all('view_gone', 'That assigned View is no longer available to you.')
|
| 1922 |
+
elif not _pinned_per_user:
|
| 1923 |
+
found = admission_tops.find_view(view_id)
|
| 1924 |
+
if found and found[0] != ctx.uname:
|
| 1925 |
+
owner, prior = found
|
| 1926 |
+
workspace = admission_tops._read_owners(owner).get(owner) or {}
|
| 1927 |
+
prior = (workspace.get('views') or {}).get(view_id)
|
| 1928 |
+
seed_view = not prior and _pinned_per_user and bool(projected)
|
| 1929 |
+
if seed_view:
|
| 1930 |
+
# A pinned projection gets its own source, never another user's same-id source.
|
| 1931 |
+
prior = {key: deepcopy(value) for key, value in projected.items()
|
| 1932 |
+
if key in ('id', 'name', 'kind', 'locked', 'note', 'config')}
|
| 1933 |
+
prior['createdBy'] = ctx.uname
|
| 1934 |
+
prior['kind'] = 'system'
|
| 1935 |
+
if not isinstance(prior, dict):
|
| 1936 |
+
return refuse_all('view_gone', 'That View is no longer available.')
|
| 1937 |
+
if not is_assigned and not _pinned_per_user and not _may_edit_view(
|
| 1938 |
+
admission_tops, view_id, owner, prior, ctx.uname, ctx.admin):
|
| 1939 |
+
return refuse_all('view_read_only', _VIEW_READ_ONLY_MESSAGE)
|
| 1940 |
+
|
| 1941 |
+
def admit(view, candidate, member):
|
| 1942 |
+
if not is_assigned and not _pinned_per_user and not _may_edit_view(
|
| 1943 |
+
tops, view_id, owner, view, ctx.uname, ctx.admin):
|
| 1944 |
+
return {'reason': 'view_read_only', 'message': _VIEW_READ_ONLY_MESSAGE}
|
| 1945 |
+
return _admit_view_op(view, candidate, member, owner=owner,
|
| 1946 |
+
actor=ctx.uname, admin=ctx.admin)
|
| 1947 |
+
|
| 1948 |
+
receipts = view_settings_rows.ViewOpReceipts(workspace, getattr(tops.st, 'key', tops.table_key))
|
| 1949 |
+
action = edit_receipts.operation(tops.table_key, 'view_op', view_id)
|
| 1950 |
+
preview = deepcopy(prior)
|
| 1951 |
+
prepared, slots, answers, known = [], [], [None] * len(events), {}
|
| 1952 |
+
for index, event in enumerate(events):
|
| 1953 |
+
member, operation = _tstore._view_member_identity(event.get('member')), event.get('op')
|
| 1954 |
+
identity = event.get('client_op_id')
|
| 1955 |
+
|
| 1956 |
+
def refuse(reason, message):
|
| 1957 |
+
answers[index] = refusal(event, reason, message)
|
| 1958 |
+
|
| 1959 |
+
try:
|
| 1960 |
+
edit_receipts.op_id(identity)
|
| 1961 |
+
except ValueError:
|
| 1962 |
+
refuse('invalid_op_id', 'Give this View operation a client operation id.')
|
| 1963 |
+
continue
|
| 1964 |
+
if str(event.get('view_id') or '').strip() != view_id:
|
| 1965 |
+
refuse('view_gone', 'That View is no longer available.')
|
| 1966 |
+
continue
|
| 1967 |
+
replay = receipts.result(ctx.uname, action, identity)
|
| 1968 |
+
if replay:
|
| 1969 |
+
answers[index] = replay
|
| 1970 |
+
continue
|
| 1971 |
+
if identity in known:
|
| 1972 |
+
prepared.append(next(item for item in prepared if item['client_op_id'] == identity))
|
| 1973 |
+
slots.append(index)
|
| 1974 |
+
continue
|
| 1975 |
+
family, _, key = member.partition(':') if isinstance(member, str) else ('', '', '')
|
| 1976 |
+
if is_assigned and family in ('filter', 'filterConj', 'memberPid'):
|
| 1977 |
+
refuse('assigned_filter_fixed', 'The assigned View fixes its filters and record membership.')
|
| 1978 |
+
continue
|
| 1979 |
+
if operation not in ('set', 'remove', 'undo'):
|
| 1980 |
+
refuse('invalid_op', 'Choose set, remove or undo for this View setting.')
|
| 1981 |
+
continue
|
| 1982 |
+
present, value = operation != 'remove', event.get('value')
|
| 1983 |
+
if operation == 'undo':
|
| 1984 |
+
original = known.get(value) if isinstance(value, str) else None
|
| 1985 |
+
original = original or receipts.result(ctx.uname, action, value)
|
| 1986 |
+
if (not original or not original.get('applied')
|
| 1987 |
+
or _tstore._view_member_identity(original.get('member')) != member):
|
| 1988 |
+
refuse('changed_since', 'That change is not available to undo.')
|
| 1989 |
+
continue
|
| 1990 |
+
present, value = original['beforePresent'], original['before']
|
| 1991 |
+
if not present:
|
| 1992 |
+
if family not in ('sort', 'width', 'filter', 'summary', 'memberPid') and operation != 'undo':
|
| 1993 |
+
refuse('invalid_op', 'Set a value for this View setting.')
|
| 1994 |
+
continue
|
| 1995 |
+
sample = {'sort': {'dir': 'asc', 'pos': 'V'}, 'width': 100,
|
| 1996 |
+
'filter': {'node': {'children': []}, 'parent': None},
|
| 1997 |
+
'summary': {'agg': 'none'}, 'memberPid': False}.get(family)
|
| 1998 |
+
# Undo can remove an optional scalar that was absent before the user's edit.
|
| 1999 |
+
if operation == 'undo' and family not in ('sort', 'width', 'filter', 'summary', 'memberPid'):
|
| 2000 |
+
sample = _tstore._view_member_value(preview, member)[1]
|
| 2001 |
+
cleaned, denied = clean_view_member(member, sample, ctx, preview)
|
| 2002 |
+
else:
|
| 2003 |
+
cleaned, denied = clean_view_member(member, value, ctx, preview)
|
| 2004 |
+
if denied:
|
| 2005 |
+
code = 'target_gone' if denied['code'] == 'unknown_field' else denied['code']
|
| 2006 |
+
refuse(code, denied['message'])
|
| 2007 |
+
continue
|
| 2008 |
+
if operation == 'set' and family == 'sort' and 'pos' not in cleaned:
|
| 2009 |
+
refuse('invalid_position', 'Use 1 to 64 letters or digits for the position.')
|
| 2010 |
+
continue
|
| 2011 |
+
if family in ('hidden', 'sort', 'width', 'position', 'summary'):
|
| 2012 |
+
member = family + ':' + _view_field_key(key, {field['key'] for field in ctx.fields})
|
| 2013 |
+
cfg = preview.get('config') or {}
|
| 2014 |
+
candidate = deepcopy(preview)
|
| 2015 |
+
if family == 'position' and 'positions' not in candidate.setdefault('config', {}):
|
| 2016 |
+
candidate['config']['positions'] = _tstore._view_position_seed(cfg.get('order') or [])
|
| 2017 |
+
before_present, before = _tstore._view_member_value(candidate, member)
|
| 2018 |
+
try:
|
| 2019 |
+
_tstore._set_view_member(candidate, member, cleaned, present=present)
|
| 2020 |
+
except _tstore._ViewOpRefused as exc:
|
| 2021 |
+
refuse(exc.reason, str(exc))
|
| 2022 |
+
continue
|
| 2023 |
+
next_cfg = candidate.get('config') or {}
|
| 2024 |
+
filters, sorts = next_cfg.get('filters') or [], next_cfg.get('sorts') or []
|
| 2025 |
+
if family == 'filter' and present:
|
| 2026 |
+
cycle = _ag.view_filter_cycle(view_id, filters, lambda vid: (
|
| 2027 |
+
next(((view.get('config') or {}).get('filters') or [] for view in ctx.visible_views
|
| 2028 |
+
if isinstance(view, dict) and view.get('id') == vid), [])))
|
| 2029 |
+
if cycle:
|
| 2030 |
+
refuse('view_cycle', 'Choose a View that does not refer back to this View.')
|
| 2031 |
+
continue
|
| 2032 |
+
if is_assigned and family in ('name', 'note', 'locked'):
|
| 2033 |
+
refuse('unknown_member', 'Choose a presentation setting for this assigned View.')
|
| 2034 |
+
continue
|
| 2035 |
+
preview = candidate
|
| 2036 |
+
after_present, after = _tstore._view_member_value(preview, member)
|
| 2037 |
+
known[identity] = {'opId': identity, 'member': member, 'applied': True,
|
| 2038 |
+
'before': before, 'beforePresent': before_present,
|
| 2039 |
+
'after': after, 'afterPresent': after_present}
|
| 2040 |
+
prepared.append({**event, 'member': member,
|
| 2041 |
+
'value': cleaned if operation == 'set' else event.get('value')})
|
| 2042 |
+
slots.append(index)
|
| 2043 |
+
|
| 2044 |
+
cohort_name = view_id in ctx.cohort_ids and any(event.get('member') == 'name' for event in events)
|
| 2045 |
+
with _cohort_name_transaction(tops.st) if cohort_name else nullcontext():
|
| 2046 |
+
if prepared:
|
| 2047 |
+
try:
|
| 2048 |
+
landed = tops.apply_view_ops(
|
| 2049 |
+
owner, view_id, prepared, actor=ctx.uname, origin=EVENT_ORIGIN,
|
| 2050 |
+
assigned_recipient=ctx.uname if is_assigned else None,
|
| 2051 |
+
seed_view=prior if seed_view else None,
|
| 2052 |
+
admit=admit)
|
| 2053 |
+
except edit_receipts.ReceiptUnavailable as exc:
|
| 2054 |
+
raise StoreUnavailable(str(exc)) from exc
|
| 2055 |
+
finally:
|
| 2056 |
+
if ctx.admission is not None:
|
| 2057 |
+
ctx.admission.invalidate(tops.table_key)
|
| 2058 |
+
for index, result in zip(slots, landed):
|
| 2059 |
+
answers[index] = result
|
| 2060 |
+
preview = (tops.assigned_views(ctx.uname, ctx.allowed_pids).get(view_id, {}) if is_assigned else
|
| 2061 |
+
((tops._read_owners(owner).get(owner) or {}).get('views') or {}).get(view_id, {}))
|
| 2062 |
+
if view_id in ctx.cohort_ids and any(
|
| 2063 |
+
result.get('applied') and result.get('member') == 'name' for result in answers):
|
| 2064 |
+
_sync_cohort_view_name(ctx, tops, owner, view_id)
|
| 2065 |
+
unresolved = {str(rule.get('id') or '') for rule in measure_leaves(
|
| 2066 |
+
preview.get('config', {}).get('filters') or []) if _mfc.rule_complete(rule)}
|
| 2067 |
+
rerender = bool(unresolved - set(ctx.resolved_ids) - {''})
|
| 2068 |
+
return [_view_op_reply(ctx, answer, rerender=rerender and answer.get('applied', False))
|
| 2069 |
+
for answer in answers]
|
| 2070 |
+
|
| 2071 |
+
|
| 2072 |
+
class _UnwrittenViewUpsert(Exception):
|
| 2073 |
+
"""Abort the receipt transaction for an existing View's whole-save refusal."""
|
| 2074 |
+
|
| 2075 |
+
|
| 2076 |
def _edit_receipt(fn):
|
| 2077 |
@wraps(fn)
|
| 2078 |
def wrapped(event, ctx):
|
| 2079 |
from copy import copy, deepcopy
|
| 2080 |
from dataclasses import replace
|
| 2081 |
from core import edit_receipts
|
| 2082 |
+
def apply_event(event, write_ctx):
|
| 2083 |
+
try:
|
| 2084 |
+
return fn(event, write_ctx)
|
| 2085 |
+
finally:
|
| 2086 |
+
# Canonical writers bypass the admission handle, including row-backed settings.
|
| 2087 |
+
if write_ctx.admission is not None:
|
| 2088 |
+
write_ctx.admission.invalidate()
|
| 2089 |
+
|
| 2090 |
if not isinstance(event, dict):
|
| 2091 |
return False
|
| 2092 |
+
# C1 stores receipts with its owner-row update, rather than the event wrapper.
|
| 2093 |
+
if event.get('type') == 'view_op':
|
| 2094 |
+
identity = event.get('client_op_id', event.get('opId', event.get('id')))
|
| 2095 |
+
return fn(dict(event, id=identity, client_op_id=identity), ctx)
|
| 2096 |
identity = event.get('opId', event.get('id'))
|
| 2097 |
active_id = edit_receipts.current_op_id()
|
| 2098 |
# Trusted staged previews own their confirmation ledger and must not gain transport metadata.
|
|
|
|
| 2105 |
if len(ctx.seen_ids) > 400:
|
| 2106 |
for stale in list(ctx.seen_ids)[:200]:
|
| 2107 |
ctx.seen_ids.pop(stale, None)
|
| 2108 |
+
return apply_event(dict(event, id=identity), ctx)
|
| 2109 |
if not identity:
|
| 2110 |
return False
|
| 2111 |
try:
|
|
|
|
| 2124 |
if table is not None:
|
| 2125 |
table._st = checked
|
| 2126 |
write_ctx = replace(ctx, st=checked, table=table, workspace_lend=lend)
|
| 2127 |
+
rerender = apply_event(event, write_ctx)
|
| 2128 |
write = dict(ctx.out.writes.get(identity) or {})
|
| 2129 |
write['opId'] = identity
|
| 2130 |
result = deepcopy({'rerender': bool(rerender), 'write': write,
|
| 2131 |
'refusals': ctx.out.refusals[before:],
|
| 2132 |
'doc': ctx.out.doc, 'toast': ctx.out.toast})
|
| 2133 |
+
if event.get('type') == 'view_upsert' and any(
|
| 2134 |
+
row.get('code') in ('view_ops_required', 'view_read_only')
|
| 2135 |
+
for row in result['refusals']):
|
| 2136 |
+
raise _UnwrittenViewUpsert(result)
|
| 2137 |
edit_receipts.observe_event(result)
|
| 2138 |
return result
|
| 2139 |
|
|
|
|
| 2143 |
_store_of(ctx), ctx.uname,
|
| 2144 |
edit_receipts.operation(ctx.scope_key, event.get('type'), event.get('pid')),
|
| 2145 |
identity, apply)
|
| 2146 |
+
except _UnwrittenViewUpsert as exc:
|
| 2147 |
+
receipt = exc.args[0]
|
| 2148 |
except edit_receipts.ReceiptUnavailable as exc:
|
| 2149 |
raise StoreUnavailable(str(exc)) from exc
|
| 2150 |
receipt = receipt.get('event') or receipt
|
|
|
|
| 2216 |
event_id = event['id']
|
| 2217 |
ctx.out.write_id = event_id
|
| 2218 |
kind = event.get('type')
|
| 2219 |
+
if kind == 'view_op':
|
| 2220 |
+
return any(row['rerender'] for row in handle_view_ops([event], ctx))
|
| 2221 |
if kind == 'overlay_patch' and str(scope_key) == 'ut_activity':
|
| 2222 |
return _refuse(ctx, 'activity_read_only', 'Activity records cannot be edited.',
|
| 2223 |
event=kind)
|
|
|
|
| 2321 |
|
| 2322 |
if kind == 'view_upsert':
|
| 2323 |
raw = event.get('view')
|
| 2324 |
+
if not isinstance(raw, dict):
|
| 2325 |
return False
|
| 2326 |
view_id = str(raw.get('id') or '').strip()[:120]
|
| 2327 |
+
prior = (ws.get('views') or {}).get(view_id) or {}
|
| 2328 |
+
tops = _tops(ctx)
|
| 2329 |
+
assigned = tops.assigned_views(uname, allowed_pids) if _store_of(ctx).available() else {}
|
| 2330 |
+
is_assigned = raw.get('assigned') or prior.get('assigned') or view_id in assigned
|
| 2331 |
+
# Stored system Views and pinned ids keep personal presentation in the caller's stratum.
|
| 2332 |
+
_pinned_per_user = (prior.get('kind') == 'system'
|
| 2333 |
+
or view_id in ('all-customers', _ag_overview_id()))
|
| 2334 |
+
_save_as = uname
|
| 2335 |
+
if _store_of(ctx).available() and not _pinned_per_user and view_id not in assigned:
|
| 2336 |
+
found = tops.find_view(view_id, prefer_owner=uname)
|
| 2337 |
+
if found:
|
| 2338 |
+
_save_as, prior = found
|
| 2339 |
+
if prior.get('kind') == 'system':
|
| 2340 |
+
_save_as, prior = uname, (ws.get('views') or {}).get(view_id) or {}
|
| 2341 |
+
elif not _may_edit_view(tops, view_id, _save_as, prior, uname, admin):
|
| 2342 |
+
return _refuse(ctx, 'view_read_only', _VIEW_READ_ONLY_MESSAGE,
|
| 2343 |
+
view_id, event=kind)
|
| 2344 |
+
if ctx.durable_receipts and (prior or view_id in assigned):
|
| 2345 |
+
return _refuse(ctx, 'view_ops_required',
|
| 2346 |
+
'This page is out of date. Reload to keep editing this view.',
|
| 2347 |
+
view_id, event=kind)
|
| 2348 |
+
if not isinstance(raw.get('config'), dict):
|
| 2349 |
+
return False
|
| 2350 |
+
name, name_refusal = clean_view_member('name', raw.get('name'), ctx, prior)
|
| 2351 |
requested_name = name
|
| 2352 |
+
if not view_id or name_refusal:
|
| 2353 |
return False
|
| 2354 |
+
display, display_refusal = clean_view_member('display', raw['config'].get('display'), ctx, prior)
|
| 2355 |
+
if display_refusal and display_refusal['code'] == 'view_mode_retired':
|
| 2356 |
+
return _folder_refuse(ctx, kind, display_refusal['code'], display_refusal['message'], key=view_id)
|
|
|
|
|
|
|
|
|
|
| 2357 |
if 'permissions' in raw:
|
| 2358 |
return _refuse(
|
| 2359 |
ctx, 'view_permissions_retired',
|
|
|
|
| 2364 |
# here for the same reason the page does it: a missing embed build must
|
| 2365 |
# never break page import.
|
| 2366 |
import aios_grid as _ag
|
| 2367 |
+
cfg = raw['config']
|
| 2368 |
# A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered,
|
| 2369 |
# shown, grouped or coloured by, because it is not a column of this table. Hence the
|
| 2370 |
# widened key set here and the plain `valid_keys` everywhere below.
|
|
|
|
| 2381 |
# so this needs no new plumbing; it is the value nobody had handed to the validator.
|
| 2382 |
# ⚠ Edited by lane C, not this file's owner: `platform/core/grid_events.py` is in NO fence
|
| 2383 |
# and lane C was the last session open. See `mailbox/C.md` NOTE C-24.
|
|
|
|
|
|
|
| 2384 |
# ⛔⛔ THE CYCLE REFUSAL'S HOST HALF (C4), and it is the ticket's `done-when` rather than a
|
| 2385 |
# guard bolted on afterwards. View X filtered on "is in View Y" where Y is filtered on X
|
| 2386 |
# cannot be resolved by anything: X needs Y needs X. The browser refuses it too, with a
|
|
|
|
| 2399 |
'. A view cannot filter on a view that filters on it, because neither can '
|
| 2400 |
'be worked out without the other.', view_id)
|
| 2401 |
return False
|
| 2402 |
+
config, filter_removals, config_refusal = _clean_view_config(cfg, ctx, prior, display)
|
| 2403 |
+
if config_refusal:
|
| 2404 |
+
_refuse(ctx, config_refusal['code'], config_refusal['message'], view_id, event=kind)
|
| 2405 |
+
return False
|
| 2406 |
+
filters, sorts = config['filters'], config['sorts']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2407 |
# C-LOCK (wave 2026-08-02, item 12's mechanism): a view may be LOCKED to one cohort.
|
| 2408 |
# `cohort_ids` is the same permission list the cohort filter leaf uses — a lock naming
|
| 2409 |
# a cohort this session cannot see is DROPPED at write, exactly like a cohort rule.
|
|
|
|
| 2420 |
_is_locked_view = view_id in set(cohort_ids)
|
| 2421 |
if _is_locked_view:
|
| 2422 |
cohort_lock = view_id
|
| 2423 |
+
if cohort_lock and cohort_lock in set(cohort_ids):
|
| 2424 |
+
config['cohortLock'] = cohort_lock
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2425 |
# The prior stored View supplies lock state and its immutable filter records. Sharing
|
| 2426 |
# itself is exclusively the modern grant registry checked at the foreign-view wall below.
|
|
|
|
| 2427 |
# A visible foreign view is a grant projection. The modern foreign-view wall below
|
| 2428 |
# resolves the owner's personal source and checks the grant registry before deciding
|
| 2429 |
# whether this is a create; there is no legacy bucket to consult as a second authority.
|
|
|
|
| 2466 |
config['display'] = dict(prior['config']['display'])
|
| 2467 |
else:
|
| 2468 |
config.pop('display', None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2469 |
# C3: only the creator or an admin may SET or CLEAR the lock; anyone else keeps whatever
|
| 2470 |
# is already stored. Enforced here rather than hidden in the client. This composes with
|
| 2471 |
# C4 and does not override it: C4 governs who may edit the CONTENTS, C3 who may freeze
|
|
|
|
| 2473 |
# ⚠ Inert in practice TODAY — a user can only ever write their own per-user workspace,
|
| 2474 |
# so `uname == creator` always holds. Correct plumbing for shared views, not a live gate.
|
| 2475 |
may_lock = admin or uname == creator
|
| 2476 |
+
locked, lock_refusal = clean_view_member(
|
| 2477 |
+
'locked', raw.get('locked', False) if may_lock else prior.get('locked', False), ctx, prior)
|
| 2478 |
+
if lock_refusal:
|
| 2479 |
+
locked = bool(prior.get('locked')) if not is_new else False
|
| 2480 |
# C3: the LOCK's actual job — a locked view's display mode is frozen, so a locked Kanban
|
| 2481 |
# stays Kanban. This half IS live per-user and is the part the owner asked for. Filters,
|
| 2482 |
# sorts and columns stay editable by design.
|
|
|
|
| 2508 |
else 'custom',
|
| 2509 |
'locked': locked,
|
| 2510 |
'createdBy': creator,
|
| 2511 |
+
'note': clean_view_member('note', raw.get('note'), ctx, prior)[0],
|
| 2512 |
'config': config,
|
| 2513 |
'filterRemovals': filter_removals}
|
| 2514 |
# System/list projections are not all persisted in this table store, but custom views
|
|
|
|
| 2533 |
# win the merge, so the receiver sees their private copy forever and calls sharing
|
| 2534 |
# broken). Role `edit` (or admin) writes THROUGH to the owner's record — config only;
|
| 2535 |
# `createdBy` stays pinned from the stored source and grants remain the access wall.
|
| 2536 |
+
# A reader receives a result-row refusal before any View mutation.
|
| 2537 |
#
|
| 2538 |
# ⛔⛔ WAVE 33 · T13 (owner item 3) — A PINNED PER-USER ID IS NOT A FOREIGN VIEW, AND
|
| 2539 |
# TREATING IT AS ONE IS A CROSS-USER WRITE THAT REPORTS SUCCESS.
|
|
|
|
| 2555 |
# VIEW IS PER-USER STATE AND MUST NEVER BE SHARED", enforced there by pinning
|
| 2556 |
# `perms = {'edit': 'personal'}`. A per-user view can therefore never BE a grant, which
|
| 2557 |
# is the only thing this wall exists to protect — so exempting it narrows nothing.
|
|
|
|
| 2558 |
# W48-T20 / C4 — an assignment is neither the owner's view nor a generic foreign
|
| 2559 |
# share. Its filter lives in TableStore's ``__assigned__`` projection and this write
|
| 2560 |
# door accepts only the recipient's presentation. In particular, a forged ``edit``
|
| 2561 |
# grant or a client flip of ``assigned`` cannot route this through the generic
|
| 2562 |
# share-through-to-owner branch below.
|
| 2563 |
+
if is_assigned:
|
|
|
|
| 2564 |
if view_id not in assigned:
|
| 2565 |
ctx.out.toast = 'That assigned view is no longer available to you.'
|
| 2566 |
return False
|
|
|
|
| 2573 |
# load-bearing: it replaces any optimistic browser filter with the locked server
|
| 2574 |
# projection before another action can use it.
|
| 2575 |
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2576 |
if _store_of(ctx).available():
|
| 2577 |
# ⛔ W50-T25 / D8 — `_save_as` IS THE BUCKET THE WRITE LANDED IN and is therefore
|
| 2578 |
# the audit `owner`; `uname` is who DID it. On the foreign-owner branch above they
|
| 2579 |
# differ, and a row that conflated them would name the wrong person in the one
|
| 2580 |
# record whose entire job is naming the right one.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2581 |
view = (_tops(ctx).save_view(
|
| 2582 |
_save_as, view, reserved_names=_reserved_view_names,
|
| 2583 |
+
actor=uname, origin=EVENT_ORIGIN) or view)
|
|
|
|
| 2584 |
else:
|
| 2585 |
_session_ready()
|
| 2586 |
_view_names = list(_reserved_view_names)
|
|
|
|
| 2846 |
if ftype in ('link', 'rollup') or (ftype == 'if' and scope_key.startswith('ut_')):
|
| 2847 |
return _refuse(
|
| 2848 |
ctx, 'wrong_door',
|
| 2849 |
+
'Create this schema field from the Fields panel through '
|
| 2850 |
+
'POST /api/v1/tables/{key}/fields. On a connected registry database, '
|
| 2851 |
+
'only an administrator can create a Link field here.', key)
|
| 2852 |
readonly = ftype in _ag2.READONLY_CUSTOM_TYPES
|
| 2853 |
# The editable family must SAY it is overlay (the client's own contract); the
|
| 2854 |
# read-only pair (formula / created_time) is stored under the cohort column's
|
|
|
|
| 3134 |
fresh_fields = _shared_overlay.fields(shared_storage_key, st=ctx.table.st) or {}
|
| 3135 |
else:
|
| 3136 |
fresh_fields = (table_workspace(
|
| 3137 |
+
ctx, allowed_pids=allowed_pids, consume_corrections=False,
|
| 3138 |
+
admission=False).get('fields') or {})
|
| 3139 |
durable_field = fresh_fields.get(key)
|
| 3140 |
if source_backed:
|
| 3141 |
durable_field = _source_echo(field, durable_field)
|
|
|
|
| 3614 |
placed, clean,
|
| 3615 |
{sfc: _own_ids(sfc, cur_ws) for sfc in _agf.FOLDER_SURFACES})
|
| 3616 |
|
| 3617 |
+
# Folder gestures mutate the latest owner row inside the write, without a whole-tree stamp.
|
| 3618 |
+
try:
|
| 3619 |
+
_receipt = _tops(ctx).save_folders(uname, mutate=_mutate,
|
| 3620 |
+
actor=uname, origin=EVENT_ORIGIN)
|
| 3621 |
+
except _tstore.FolderTreeError as _fte:
|
| 3622 |
+
return _folder_refuse(ctx, kind,
|
| 3623 |
+
str(getattr(_fte, 'reason', '') or 'folder_tree_illegal'),
|
| 3624 |
+
str(_fte), key=str(getattr(_fte, 'folder_id', '') or ''))
|
| 3625 |
+
if _receipt is None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3626 |
return _folder_refuse(
|
| 3627 |
ctx, kind,
|
| 3628 |
why.get('code') or 'folder_change_refused',
|
| 3629 |
why.get('message') or ('That folder change could not be saved, so nothing was '
|
| 3630 |
'altered. Reload the page and try again.'),
|
| 3631 |
key=why.get('key') or '')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3632 |
# ⭐⭐ THE DURABILITY HALF — the owner's actual complaint. See `_commit_folder_write`: the
|
| 3633 |
# write above lands in `Store._cache` and the hub commit is deferred by up to ~22s, so
|
| 3634 |
# a second process (or the same one after a restart) cannot read it back. Forced here,
|
|
|
|
| 4398 |
_history_after.update(updates)
|
| 4399 |
_history_after.update(dict(row_events))
|
| 4400 |
_history_st = ctx.st
|
| 4401 |
+
if ctx.table is not None and (_history_table.startswith('ut_')
|
| 4402 |
+
or _history_table in ('customer', 'product')):
|
| 4403 |
# One receipt carries every committed field delta, even when legacy hooks fan out.
|
| 4404 |
changes = {key: {'before': _history_before.get(key),
|
| 4405 |
'after': _history_after.get(key)}
|
|
|
|
| 4410 |
'type': 'event_field', 'table': _history_table, 'rowId': str(pid),
|
| 4411 |
'field': _k, 'after': _v, 'st': ctx.table.st, 'user': uname,
|
| 4412 |
'receiptId': event_id, 'changes': changes})
|
| 4413 |
+
if _history_table in ('customer', 'product') and getattr(_history_st, 'key', None):
|
| 4414 |
from core import received_changes
|
| 4415 |
changes = {key: {'before': _history_before.get(key), 'after': _history_after.get(key)}
|
| 4416 |
for key in set(updates) | {key for key, _value in row_events}
|
platform/core/module_tables.py
CHANGED
|
@@ -42,12 +42,12 @@ def declared_sources(definition):
|
|
| 42 |
|
| 43 |
|
| 44 |
def ensure_module_table(st, username, specs=None, *, owner=None, table_key=None, label=None,
|
| 45 |
-
fields=None, owned_keys=None, sources=None, stale=None, repair=None,
|
| 46 |
error=ModuleTableRefused):
|
| 47 |
"""Create or adopt module tables; add missing columns, never retype one. Returns the spec keys.
|
| 48 |
|
| 49 |
`specs` holds `{key, label, fields}` dicts with stored field definitions, plus optional
|
| 50 |
-
`sources`. A stored label that differs is rewritten. `stale(spec, stored)` and
|
| 51 |
`repair(spec, table)` let the caller add its own repairs to the one store write.
|
| 52 |
"""
|
| 53 |
ut = _ut()
|
|
@@ -60,10 +60,15 @@ def ensure_module_table(st, username, specs=None, *, owner=None, table_key=None,
|
|
| 60 |
spec["ownedKeys"] = list(owned_keys)
|
| 61 |
if sources is not None:
|
| 62 |
spec[SOURCES_KEY] = list(sources)
|
|
|
|
|
|
|
| 63 |
specs = [spec]
|
| 64 |
elif table_key is not None:
|
| 65 |
raise TypeError("Pass either specs or table_key.")
|
| 66 |
specs = [dict(spec) for spec in specs]
|
|
|
|
|
|
|
|
|
|
| 67 |
keys = [spec["key"] for spec in specs]
|
| 68 |
ut.register_module_tables(*keys)
|
| 69 |
|
|
@@ -71,12 +76,16 @@ def ensure_module_table(st, username, specs=None, *, owner=None, table_key=None,
|
|
| 71 |
have = ut.get_one(spec["key"], st)
|
| 72 |
if have is None or have.get("label") != spec["label"]:
|
| 73 |
return True
|
|
|
|
|
|
|
| 74 |
if not {f["key"] for f in spec["fields"]} <= {f.get("key") for f in have.get("fields") or []}:
|
| 75 |
return True
|
| 76 |
if SOURCES_KEY in spec and declared_sources(have) != _clean_sources(spec[SOURCES_KEY]):
|
| 77 |
return True
|
| 78 |
if "ownedKeys" in spec and have.get("ownedKeys") != spec["ownedKeys"]:
|
| 79 |
return True
|
|
|
|
|
|
|
| 80 |
return bool(stale and stale(spec, have))
|
| 81 |
|
| 82 |
todo = [spec for spec in specs if _is_stale(spec)]
|
|
@@ -93,19 +102,25 @@ def ensure_module_table(st, username, specs=None, *, owner=None, table_key=None,
|
|
| 93 |
if table is None:
|
| 94 |
# A module table takes no cap slot (register_module_tables), so nothing refuses here.
|
| 95 |
table = cur[key] = {"key": key, "label": want_label, "source": ut.AUTOMATION_SOURCE,
|
|
|
|
| 96 |
"createdBy": username or owner, "created": created,
|
| 97 |
"fields": [dict(f) for f in wanted], "rows": {}}
|
| 98 |
if SOURCES_KEY in spec:
|
| 99 |
table[SOURCES_KEY] = list(_clean_sources(spec[SOURCES_KEY]))
|
| 100 |
if "ownedKeys" in spec:
|
| 101 |
table["ownedKeys"] = list(spec["ownedKeys"])
|
|
|
|
|
|
|
| 102 |
continue
|
|
|
|
| 103 |
if table.get("label") != want_label:
|
| 104 |
table["label"] = want_label
|
| 105 |
if SOURCES_KEY in spec:
|
| 106 |
table[SOURCES_KEY] = list(_clean_sources(spec[SOURCES_KEY]))
|
| 107 |
if "ownedKeys" in spec:
|
| 108 |
table["ownedKeys"] = list(spec["ownedKeys"])
|
|
|
|
|
|
|
| 109 |
present = {f.get("key") for f in table.get("fields") or []}
|
| 110 |
if repair is not None:
|
| 111 |
repair(spec, table)
|
|
|
|
| 42 |
|
| 43 |
|
| 44 |
def ensure_module_table(st, username, specs=None, *, owner=None, table_key=None, label=None,
|
| 45 |
+
fields=None, owned_keys=None, sources=None, field_limit=None, stale=None, repair=None,
|
| 46 |
error=ModuleTableRefused):
|
| 47 |
"""Create or adopt module tables; add missing columns, never retype one. Returns the spec keys.
|
| 48 |
|
| 49 |
`specs` holds `{key, label, fields}` dicts with stored field definitions, plus optional
|
| 50 |
+
`sources` and `fieldLimit`. A stored label that differs is rewritten. `stale(spec, stored)` and
|
| 51 |
`repair(spec, table)` let the caller add its own repairs to the one store write.
|
| 52 |
"""
|
| 53 |
ut = _ut()
|
|
|
|
| 60 |
spec["ownedKeys"] = list(owned_keys)
|
| 61 |
if sources is not None:
|
| 62 |
spec[SOURCES_KEY] = list(sources)
|
| 63 |
+
if field_limit is not None:
|
| 64 |
+
spec['fieldLimit'] = field_limit
|
| 65 |
specs = [spec]
|
| 66 |
elif table_key is not None:
|
| 67 |
raise TypeError("Pass either specs or table_key.")
|
| 68 |
specs = [dict(spec) for spec in specs]
|
| 69 |
+
for spec in specs:
|
| 70 |
+
if 'fieldLimit' in spec and (type(spec['fieldLimit']) is not int or spec['fieldLimit'] <= 0):
|
| 71 |
+
raise ValueError('A module fieldLimit must be a positive integer.')
|
| 72 |
keys = [spec["key"] for spec in specs]
|
| 73 |
ut.register_module_tables(*keys)
|
| 74 |
|
|
|
|
| 76 |
have = ut.get_one(spec["key"], st)
|
| 77 |
if have is None or have.get("label") != spec["label"]:
|
| 78 |
return True
|
| 79 |
+
if have.get(ut.MODULE_OWNED_KEY) is not True:
|
| 80 |
+
return True
|
| 81 |
if not {f["key"] for f in spec["fields"]} <= {f.get("key") for f in have.get("fields") or []}:
|
| 82 |
return True
|
| 83 |
if SOURCES_KEY in spec and declared_sources(have) != _clean_sources(spec[SOURCES_KEY]):
|
| 84 |
return True
|
| 85 |
if "ownedKeys" in spec and have.get("ownedKeys") != spec["ownedKeys"]:
|
| 86 |
return True
|
| 87 |
+
if 'fieldLimit' in spec and have.get('fieldLimit') != spec['fieldLimit']:
|
| 88 |
+
return True
|
| 89 |
return bool(stale and stale(spec, have))
|
| 90 |
|
| 91 |
todo = [spec for spec in specs if _is_stale(spec)]
|
|
|
|
| 102 |
if table is None:
|
| 103 |
# A module table takes no cap slot (register_module_tables), so nothing refuses here.
|
| 104 |
table = cur[key] = {"key": key, "label": want_label, "source": ut.AUTOMATION_SOURCE,
|
| 105 |
+
ut.MODULE_OWNED_KEY: True,
|
| 106 |
"createdBy": username or owner, "created": created,
|
| 107 |
"fields": [dict(f) for f in wanted], "rows": {}}
|
| 108 |
if SOURCES_KEY in spec:
|
| 109 |
table[SOURCES_KEY] = list(_clean_sources(spec[SOURCES_KEY]))
|
| 110 |
if "ownedKeys" in spec:
|
| 111 |
table["ownedKeys"] = list(spec["ownedKeys"])
|
| 112 |
+
if 'fieldLimit' in spec:
|
| 113 |
+
table['fieldLimit'] = spec['fieldLimit']
|
| 114 |
continue
|
| 115 |
+
table[ut.MODULE_OWNED_KEY] = True
|
| 116 |
if table.get("label") != want_label:
|
| 117 |
table["label"] = want_label
|
| 118 |
if SOURCES_KEY in spec:
|
| 119 |
table[SOURCES_KEY] = list(_clean_sources(spec[SOURCES_KEY]))
|
| 120 |
if "ownedKeys" in spec:
|
| 121 |
table["ownedKeys"] = list(spec["ownedKeys"])
|
| 122 |
+
if 'fieldLimit' in spec:
|
| 123 |
+
table['fieldLimit'] = spec['fieldLimit']
|
| 124 |
present = {f.get("key") for f in table.get("fields") or []}
|
| 125 |
if repair is not None:
|
| 126 |
repair(spec, table)
|
platform/core/perm_scope.py
CHANGED
|
@@ -908,7 +908,7 @@ def _enrich_stored_wall(leaves, rows, fields, module, st):
|
|
| 908 |
|
| 909 |
def _wall_source_revisions(st, table_key, tree):
|
| 910 |
"""Every derived dependency contributes its storage revision, including mirror syncs."""
|
| 911 |
-
from
|
| 912 |
leaves = _wall_leaf_keys(fe.tree_parts(tree)[0])
|
| 913 |
if not leaves:
|
| 914 |
return {}
|
|
@@ -1065,7 +1065,7 @@ def _enrich_for_wall(tree, rows, fields, module, st):
|
|
| 1065 |
set difference and reaches no store at all.
|
| 1066 |
"""
|
| 1067 |
try:
|
| 1068 |
-
from
|
| 1069 |
# ⛔ `tree_parts`, NEVER `tree['nodes']`. A `FilterTree` is a PAIR and a BARE LIST is also
|
| 1070 |
# a legal shape (C-PERM amendment 2); a second reader of it here would answer "no leaves"
|
| 1071 |
# for a wall the evaluator one line down reads perfectly well.
|
|
@@ -1369,7 +1369,7 @@ def apply_row_scope(rows, user, module, fields, ctx=None, st=None, defn=None, re
|
|
| 1369 |
caller's held table definition to a storeless door without changing the ordinary wall.
|
| 1370 |
|
| 1371 |
`permits`, never `matches` — an unanswerable permanent filter DENIES rather than being
|
| 1372 |
-
ignored. See `
|
| 1373 |
the difference a leak rather than a preference.
|
| 1374 |
|
| 1375 |
⭐⭐ `st` IS OWNER I16's SECOND HALF AND IT IS OPTIONAL SO THE FIRST HALF CANNOT MOVE.
|
|
@@ -1396,7 +1396,7 @@ def apply_row_scope(rows, user, module, fields, ctx=None, st=None, defn=None, re
|
|
| 1396 |
tree = (e or {}).get('filter')
|
| 1397 |
if not tree:
|
| 1398 |
return list(rows or ())
|
| 1399 |
-
from
|
| 1400 |
if tree:
|
| 1401 |
# ⛔⛔ THE ENRICHMENT ANSWERS THE WALL AND MUST NEVER REACH A CALLER. It merges a
|
| 1402 |
# column's value onto a COPY of each row so `permits` can evaluate a leaf naming it;
|
|
@@ -1446,7 +1446,7 @@ def assistant_apply_row_scope(rows, user, module, fields, ctx=None, st=None, def
|
|
| 1446 |
tree = e.get('filter')
|
| 1447 |
if not tree:
|
| 1448 |
return list(rows or ())
|
| 1449 |
-
from
|
| 1450 |
if tree:
|
| 1451 |
# Same rule as `apply_row_scope`: judge on the copy, return the ORIGINAL. See the block
|
| 1452 |
# there for the measured leak this prevents.
|
|
@@ -1477,7 +1477,7 @@ def validate_assistant_filter(tree, fields):
|
|
| 1477 |
"""
|
| 1478 |
if tree is None:
|
| 1479 |
return None
|
| 1480 |
-
from
|
| 1481 |
from core.filter_contract import MAX_FILTER_VALUES
|
| 1482 |
|
| 1483 |
by_key = {str(field.get('key')): field for field in (fields or ())
|
|
@@ -1988,7 +1988,7 @@ def _derive_from_filter(user, module):
|
|
| 1988 |
return perms.scope_team_id(user), perms.scope_agent(user)
|
| 1989 |
return None, None
|
| 1990 |
|
| 1991 |
-
from
|
| 1992 |
nodes, conj = fe.tree_parts(tree)
|
| 1993 |
if conj == 'or':
|
| 1994 |
return None, None
|
|
@@ -2265,7 +2265,7 @@ def describe_scope(user, module, fields, st=None):
|
|
| 2265 |
tree = (entry(user, module) or {}).get('filter')
|
| 2266 |
parts, rows = [], []
|
| 2267 |
if tree:
|
| 2268 |
-
from
|
| 2269 |
hidden = set(hidden_keys(user, module, list(by_key.values()), st=st)) if by_key else set()
|
| 2270 |
nodes, conj = fe.tree_parts(tree)
|
| 2271 |
nodes = [n for n in nodes if isinstance(n, dict)]
|
|
@@ -3012,7 +3012,8 @@ def _window_number_sql(expr, *, postgres=False, json_sql=None):
|
|
| 3012 |
|
| 3013 |
def _window_wall(user, key, fields, columns, ctx=None, *, postgres=False, id_sql=None, st=None):
|
| 3014 |
"""Compile each permanent leaf separately so an inactive OR branch stays false."""
|
| 3015 |
-
from
|
|
|
|
| 3016 |
tree = None if perms.is_admin(user) else (entry(user, key) or {}).get('filter')
|
| 3017 |
if not tree:
|
| 3018 |
return 'TRUE', []
|
|
@@ -3362,7 +3363,7 @@ class _UserTableWindow:
|
|
| 3362 |
'type': 'int' if f.get('type') in ('rating', 'formula', 'autonumber') else
|
| 3363 |
'date' if f.get('type') in ('created_time', 'last_modified_time') else f.get('type', 'text')}
|
| 3364 |
for f in self.fields if f.get('key')}
|
| 3365 |
-
from
|
| 3366 |
nodes, _conj = filter_eval.tree_parts((entry(user, self.key) or {}).get('filter'))
|
| 3367 |
|
| 3368 |
def rhs_keys(nodes):
|
|
|
|
| 908 |
|
| 909 |
def _wall_source_revisions(st, table_key, tree):
|
| 910 |
"""Every derived dependency contributes its storage revision, including mirror syncs."""
|
| 911 |
+
from core import filter_eval as fe
|
| 912 |
leaves = _wall_leaf_keys(fe.tree_parts(tree)[0])
|
| 913 |
if not leaves:
|
| 914 |
return {}
|
|
|
|
| 1065 |
set difference and reaches no store at all.
|
| 1066 |
"""
|
| 1067 |
try:
|
| 1068 |
+
from core import filter_eval as fe
|
| 1069 |
# ⛔ `tree_parts`, NEVER `tree['nodes']`. A `FilterTree` is a PAIR and a BARE LIST is also
|
| 1070 |
# a legal shape (C-PERM amendment 2); a second reader of it here would answer "no leaves"
|
| 1071 |
# for a wall the evaluator one line down reads perfectly well.
|
|
|
|
| 1369 |
caller's held table definition to a storeless door without changing the ordinary wall.
|
| 1370 |
|
| 1371 |
`permits`, never `matches` — an unanswerable permanent filter DENIES rather than being
|
| 1372 |
+
ignored. See `core/filter_eval`'s docstring for the field-rename walkthrough that makes
|
| 1373 |
the difference a leak rather than a preference.
|
| 1374 |
|
| 1375 |
⭐⭐ `st` IS OWNER I16's SECOND HALF AND IT IS OPTIONAL SO THE FIRST HALF CANNOT MOVE.
|
|
|
|
| 1396 |
tree = (e or {}).get('filter')
|
| 1397 |
if not tree:
|
| 1398 |
return list(rows or ())
|
| 1399 |
+
from core import filter_eval as fe
|
| 1400 |
if tree:
|
| 1401 |
# ⛔⛔ THE ENRICHMENT ANSWERS THE WALL AND MUST NEVER REACH A CALLER. It merges a
|
| 1402 |
# column's value onto a COPY of each row so `permits` can evaluate a leaf naming it;
|
|
|
|
| 1446 |
tree = e.get('filter')
|
| 1447 |
if not tree:
|
| 1448 |
return list(rows or ())
|
| 1449 |
+
from core import filter_eval as fe
|
| 1450 |
if tree:
|
| 1451 |
# Same rule as `apply_row_scope`: judge on the copy, return the ORIGINAL. See the block
|
| 1452 |
# there for the measured leak this prevents.
|
|
|
|
| 1477 |
"""
|
| 1478 |
if tree is None:
|
| 1479 |
return None
|
| 1480 |
+
from core import filter_eval as fe
|
| 1481 |
from core.filter_contract import MAX_FILTER_VALUES
|
| 1482 |
|
| 1483 |
by_key = {str(field.get('key')): field for field in (fields or ())
|
|
|
|
| 1988 |
return perms.scope_team_id(user), perms.scope_agent(user)
|
| 1989 |
return None, None
|
| 1990 |
|
| 1991 |
+
from core import filter_eval as fe
|
| 1992 |
nodes, conj = fe.tree_parts(tree)
|
| 1993 |
if conj == 'or':
|
| 1994 |
return None, None
|
|
|
|
| 2265 |
tree = (entry(user, module) or {}).get('filter')
|
| 2266 |
parts, rows = [], []
|
| 2267 |
if tree:
|
| 2268 |
+
from core import filter_eval as fe
|
| 2269 |
hidden = set(hidden_keys(user, module, list(by_key.values()), st=st)) if by_key else set()
|
| 2270 |
nodes, conj = fe.tree_parts(tree)
|
| 2271 |
nodes = [n for n in nodes if isinstance(n, dict)]
|
|
|
|
| 3012 |
|
| 3013 |
def _window_wall(user, key, fields, columns, ctx=None, *, postgres=False, id_sql=None, st=None):
|
| 3014 |
"""Compile each permanent leaf separately so an inactive OR branch stays false."""
|
| 3015 |
+
from core import filter_eval as fe
|
| 3016 |
+
from harness import filter_sql as fs
|
| 3017 |
tree = None if perms.is_admin(user) else (entry(user, key) or {}).get('filter')
|
| 3018 |
if not tree:
|
| 3019 |
return 'TRUE', []
|
|
|
|
| 3363 |
'type': 'int' if f.get('type') in ('rating', 'formula', 'autonumber') else
|
| 3364 |
'date' if f.get('type') in ('created_time', 'last_modified_time') else f.get('type', 'text')}
|
| 3365 |
for f in self.fields if f.get('key')}
|
| 3366 |
+
from core import filter_eval
|
| 3367 |
nodes, _conj = filter_eval.tree_parts((entry(user, self.key) or {}).get('filter'))
|
| 3368 |
|
| 3369 |
def rhs_keys(nodes):
|
platform/core/store_pg.py
CHANGED
|
@@ -621,9 +621,10 @@ def _pool():
|
|
| 621 |
# default. Thirty seconds still reuses a normal request burst without creating a resident
|
| 622 |
# database client.
|
| 623 |
limits = _pg_limits()
|
|
|
|
| 624 |
connection_kwargs = dict(connect_timeout=limits['connect'], keepalives=1,
|
| 625 |
keepalives_idle=30, keepalives_interval=10, keepalives_count=3,
|
| 626 |
-
tcp_user_timeout=limits['network_ms'])
|
| 627 |
if sslrootcert is not None:
|
| 628 |
connection_kwargs['sslrootcert'] = sslrootcert
|
| 629 |
# The pool applies transaction settings and lends connections with checked socket polls.
|
|
@@ -723,6 +724,9 @@ class SharedReadTimeout(TimeoutError):
|
|
| 723 |
def _bump_write_generation(name, tenant_slug):
|
| 724 |
_request_timing.clear_read_memo()
|
| 725 |
key = (_effective_slug(tenant_slug), str(name))
|
|
|
|
|
|
|
|
|
|
| 726 |
with _READ_FLIGHT_LOCK:
|
| 727 |
_WRITE_GENERATION[key] = _WRITE_GENERATION.get(key, 0) + 1
|
| 728 |
with _WHOLE_TEXT_LOCK:
|
|
@@ -885,15 +889,38 @@ def _validated_content(name, drops, tenant_slug):
|
|
| 885 |
return False, None
|
| 886 |
|
| 887 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 888 |
def _view_rows(name, tenant_slug=None):
|
| 889 |
-
"""Resolve an explicitly migrated workspace without creating schema or settings.
|
|
|
|
|
|
|
|
|
|
| 890 |
from core import view_settings_rows as settings
|
| 891 |
if not settings.supports(name):
|
| 892 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 893 |
with _pool().connection() as con:
|
| 894 |
row = con.execute(_q("SELECT value ->> '__layout__' FROM {tbl} WHERE key = %s",
|
| 895 |
'store_kv', tenant_slug), (str(name),)).fetchone()
|
| 896 |
-
|
|
|
|
|
|
|
|
|
|
| 897 |
|
| 898 |
|
| 899 |
def _view_rows_many(names, tenant_slug=None):
|
|
@@ -2358,7 +2385,8 @@ def _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row):
|
|
| 2358 |
raise settings.MigrationRequired('row-backed settings require explicit owner-row mutations')
|
| 2359 |
if _ut_marker(name, tenant_slug, data):
|
| 2360 |
raise _RowsLayoutFound(name)
|
| 2361 |
-
|
|
|
|
| 2362 |
data = result if result is not None else data
|
| 2363 |
payload = _payload(name, data, con, tenant_slug)
|
| 2364 |
# The write, the audit rows and COMMIT go in one round trip on this module's pool.
|
|
@@ -2394,6 +2422,22 @@ def _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row):
|
|
| 2394 |
return data
|
| 2395 |
|
| 2396 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2397 |
class _RowsLayoutFound(Exception):
|
| 2398 |
"""The locked `user_tables` value is the row-mode marker: roll back and write rows instead."""
|
| 2399 |
|
|
|
|
| 621 |
# default. Thirty seconds still reuses a normal request burst without creating a resident
|
| 622 |
# database client.
|
| 623 |
limits = _pg_limits()
|
| 624 |
+
# No server-side prepares: a failed pipeline or PgBouncer leaves psycopg naming an absent statement.
|
| 625 |
connection_kwargs = dict(connect_timeout=limits['connect'], keepalives=1,
|
| 626 |
keepalives_idle=30, keepalives_interval=10, keepalives_count=3,
|
| 627 |
+
tcp_user_timeout=limits['network_ms'], prepare_threshold=None)
|
| 628 |
if sslrootcert is not None:
|
| 629 |
connection_kwargs['sslrootcert'] = sslrootcert
|
| 630 |
# The pool applies transaction settings and lends connections with checked socket polls.
|
|
|
|
| 724 |
def _bump_write_generation(name, tenant_slug):
|
| 725 |
_request_timing.clear_read_memo()
|
| 726 |
key = (_effective_slug(tenant_slug), str(name))
|
| 727 |
+
layouts = _layout_memo()
|
| 728 |
+
if layouts is not None:
|
| 729 |
+
layouts.pop(key, None)
|
| 730 |
with _READ_FLIGHT_LOCK:
|
| 731 |
_WRITE_GENERATION[key] = _WRITE_GENERATION.get(key, 0) + 1
|
| 732 |
with _WHOLE_TEXT_LOCK:
|
|
|
|
| 889 |
return False, None
|
| 890 |
|
| 891 |
|
| 892 |
+
#: Accumulator attribute holding one request's settings-layout answers (S85-T169).
|
| 893 |
+
_LAYOUT_MEMO = '_store_pg_layout_probes'
|
| 894 |
+
|
| 895 |
+
|
| 896 |
+
def _layout_memo():
|
| 897 |
+
"""`{(slug, name): migrated}` for the current request, or None outside one or in a caller transaction."""
|
| 898 |
+
acc = _request_timing.CURRENT.get()
|
| 899 |
+
if acc is None or acc.closed or _CALLER_TRANSACTION.get() is not None:
|
| 900 |
+
return None
|
| 901 |
+
with acc.lock:
|
| 902 |
+
return acc.__dict__.setdefault(_LAYOUT_MEMO, {})
|
| 903 |
+
|
| 904 |
+
|
| 905 |
def _view_rows(name, tenant_slug=None):
|
| 906 |
+
"""Resolve an explicitly migrated workspace without creating schema or settings.
|
| 907 |
+
|
| 908 |
+
A request reads each workspace's layout marker once. Its own write to that key reads it again.
|
| 909 |
+
"""
|
| 910 |
from core import view_settings_rows as settings
|
| 911 |
if not settings.supports(name):
|
| 912 |
return None
|
| 913 |
+
layouts = _layout_memo()
|
| 914 |
+
key = (_effective_slug(tenant_slug), str(name))
|
| 915 |
+
if layouts is not None and key in layouts:
|
| 916 |
+
return settings.PgRows(name, key[0]) if layouts[key] else None
|
| 917 |
with _pool().connection() as con:
|
| 918 |
row = con.execute(_q("SELECT value ->> '__layout__' FROM {tbl} WHERE key = %s",
|
| 919 |
'store_kv', tenant_slug), (str(name),)).fetchone()
|
| 920 |
+
migrated = bool(row and row[0] == settings.LAYOUT)
|
| 921 |
+
if layouts is not None:
|
| 922 |
+
layouts[key] = migrated
|
| 923 |
+
return settings.PgRows(name, _effective_slug(tenant_slug)) if migrated else None
|
| 924 |
|
| 925 |
|
| 926 |
def _view_rows_many(names, tenant_slug=None):
|
|
|
|
| 2385 |
raise settings.MigrationRequired('row-backed settings require explicit owner-row mutations')
|
| 2386 |
if _ut_marker(name, tenant_slug, data):
|
| 2387 |
raise _RowsLayoutFound(name)
|
| 2388 |
+
with _lend_holding(con, tenant_slug):
|
| 2389 |
+
result = fn(data)
|
| 2390 |
data = result if result is not None else data
|
| 2391 |
payload = _payload(name, data, con, tenant_slug)
|
| 2392 |
# The write, the audit rows and COMMIT go in one round trip on this module's pool.
|
|
|
|
| 2422 |
return data
|
| 2423 |
|
| 2424 |
|
| 2425 |
+
@contextmanager
|
| 2426 |
+
def _lend_holding(con, tenant_slug):
|
| 2427 |
+
"""Serve the callback's nested store calls on the connection that holds the row lock.
|
| 2428 |
+
|
| 2429 |
+
A second pooled borrow under the lock deadlocks a full pool and starves sign-in (D-3480).
|
| 2430 |
+
"""
|
| 2431 |
+
if _CALLER_TRANSACTION.get() is not None:
|
| 2432 |
+
yield
|
| 2433 |
+
return
|
| 2434 |
+
token = _CALLER_TRANSACTION.set(_EnlistedPool(con, _effective_slug(tenant_slug)))
|
| 2435 |
+
try:
|
| 2436 |
+
yield
|
| 2437 |
+
finally:
|
| 2438 |
+
_CALLER_TRANSACTION.reset(token)
|
| 2439 |
+
|
| 2440 |
+
|
| 2441 |
class _RowsLayoutFound(Exception):
|
| 2442 |
"""The locked `user_tables` value is the row-mode marker: roll back and write rows instead."""
|
| 2443 |
|
platform/core/table_store.py
CHANGED
|
@@ -16,8 +16,8 @@ A LIST (membership/formula semantics) is deliberately a different store from a V
|
|
| 16 |
(presentation/query state) — see modules/customer_data.py's customer_lists key.
|
| 17 |
"""
|
| 18 |
import copy
|
| 19 |
-
import hashlib
|
| 20 |
import json
|
|
|
|
| 21 |
import time
|
| 22 |
|
| 23 |
import core.store as store
|
|
@@ -88,6 +88,9 @@ RESERVED_MEMBERS = (SHARED_KEY, ASSIGNED_KEY, AUDIT_KEY)
|
|
| 88 |
#: is NOT in `RESERVED_MEMBERS` (that tuple names TOP-LEVEL members that are not users).
|
| 89 |
TRASH_KEY = 'trash'
|
| 90 |
|
|
|
|
|
|
|
|
|
|
| 91 |
#: R9's retention, in days. ⛔ A CONSTANT RATHER THAN A LITERAL: it is read by `delete_view`,
|
| 92 |
#: by `trash()` and by the gate that proves a 31-day-old entry purges while a 29-day-old one
|
| 93 |
#: does not, and three copies of `30` is how two of them drift.
|
|
@@ -97,146 +100,180 @@ TRASH_RETENTION_DAYS = 30
|
|
| 97 |
#: matching `view_audit.row`'s `ts` and `store_pg.revision`'s `updated_at` on both backends).
|
| 98 |
TRASH_RETENTION_SECONDS = TRASH_RETENTION_DAYS * 24 * 60 * 60
|
| 99 |
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
# write, and the outbox then resends that refused write forever.
|
| 107 |
-
#
|
| 108 |
-
# Each view write records its author, landed revision and the member digests from before it.
|
| 109 |
-
# The next before-image isolates each write's changes. Known requester writes are skipped.
|
| 110 |
-
# A foreign or unknown writer's changed member stays protected when this write would replace it.
|
| 111 |
-
#
|
| 112 |
-
# ⛔ The record lives in the owner's own stratum beside `views`, so it rides the same update and
|
| 113 |
-
# the same settings row. `workspace()` never projects it.
|
| 114 |
-
# ⚠ An unstamped write tags itself with a revision read just before its update. A write that
|
| 115 |
-
# lands between that read and the update makes the tag one low, so a conflict in that window can
|
| 116 |
-
# be missed. The stamped path tags exactly.
|
| 117 |
-
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
| 118 |
|
| 119 |
-
#: The per-owner member holding each view's recent write record.
|
| 120 |
-
VIEW_WRITES_KEY = 'viewWrites'
|
| 121 |
-
# Folder ids are surface-scoped and keep history separate from view ids.
|
| 122 |
-
FOLDER_WRITES_KEY = 'folderWrites'
|
| 123 |
-
#: Writes kept per view. Older senders need a matching trimmed-author boundary.
|
| 124 |
-
VIEW_WRITES_KEEP = 8
|
| 125 |
-
#: Views kept per stratum. The least recently written record goes first.
|
| 126 |
-
VIEW_WRITES_VIEWS = 400
|
| 127 |
-
#: Members the server sets or derives, so two writers never disagree about them.
|
| 128 |
-
_DERIVED_VIEW_MEMBERS = frozenset({'updatedAt', 'filterRemovals', 'createdBy', 'hasShares',
|
| 129 |
-
'sharedOut'})
|
| 130 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
|
| 132 |
-
def _member_digest(value):
|
| 133 |
-
text = json.dumps(value, sort_keys=True, separators=(',', ':'), default=str)
|
| 134 |
-
return hashlib.sha1(text.encode('utf-8')).hexdigest()[:16]
|
| 135 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
|
| 137 |
-
def view_members(view):
|
| 138 |
-
"""`{member: digest}` for a saved view, one member per config key, or None for no view."""
|
| 139 |
-
if not isinstance(view, dict):
|
| 140 |
-
return None
|
| 141 |
-
out = {}
|
| 142 |
-
for key, value in view.items():
|
| 143 |
-
if key in _DERIVED_VIEW_MEMBERS:
|
| 144 |
-
continue
|
| 145 |
-
if key == 'config' and isinstance(value, dict):
|
| 146 |
-
out.update((f'config.{name}', _member_digest(item)) for name, item in value.items())
|
| 147 |
-
else:
|
| 148 |
-
out[str(key)] = _member_digest(value)
|
| 149 |
-
return out
|
| 150 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
|
|
|
|
|
|
| 155 |
return
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
return
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
def
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
if
|
| 220 |
-
return
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
return
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
def
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
if
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
|
| 241 |
|
| 242 |
class ViewNotInTrash(LookupError):
|
|
@@ -785,7 +822,7 @@ class TableStore:
|
|
| 785 |
return self._st.get(self.table_key) or {}
|
| 786 |
|
| 787 |
def revision(self):
|
| 788 |
-
"""The
|
| 789 |
rows = self._settings_rows
|
| 790 |
if rows is not None:
|
| 791 |
with rows._reading() as con:
|
|
@@ -810,30 +847,8 @@ class TableStore:
|
|
| 810 |
rev = self.revision()
|
| 811 |
return rev, self._st.get(self.table_key) or {}
|
| 812 |
|
| 813 |
-
def
|
| 814 |
-
|
| 815 |
-
if rev is not None:
|
| 816 |
-
return int(rev) + 1
|
| 817 |
-
current = self.revision()
|
| 818 |
-
return None if current is None else current + 1
|
| 819 |
-
|
| 820 |
-
def _land_stale(self, owner, sent, stale, conflict, write, at=None):
|
| 821 |
-
"""Land a stale stamped write that undoes no later change, else raise its refusal.
|
| 822 |
-
|
| 823 |
-
`write(rev)` repeats the write guarded at `rev`. `at` pins that revision for a write
|
| 824 |
-
whose payload was computed from the workspace as it stood then.
|
| 825 |
-
"""
|
| 826 |
-
for _attempt in range(1 if at is not None else 3):
|
| 827 |
-
current, data = self._read_owners_at(owner)
|
| 828 |
-
if current is None or conflict((data or {}).get(owner) or {}, int(sent)):
|
| 829 |
-
break
|
| 830 |
-
try:
|
| 831 |
-
return write(current if at is None else at)
|
| 832 |
-
except view_audit.ViewRevisionStale as again:
|
| 833 |
-
stale = again
|
| 834 |
-
raise view_audit.ViewRevisionStale(rev_current=stale.rev_current, rev_sent=int(sent))
|
| 835 |
-
|
| 836 |
-
def log_view_activity(self, view_id, change, *, by, before=None, after=None, via=None):
|
| 837 |
"""Append one View event through the exact workspace store after its write succeeds."""
|
| 838 |
from core.user_tables import _activity_writer
|
| 839 |
append = _activity_writer(self._st, by)
|
|
@@ -844,10 +859,17 @@ class TableStore:
|
|
| 844 |
if workspace == self.table_key), self.table_key.removesuffix('_table_workspace'))
|
| 845 |
if isinstance(via, dict):
|
| 846 |
via = via.get('client') or None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 847 |
append(kind='view', object={'database': database, 'id': str(view_id)}, by=by,
|
| 848 |
-
via=via, changes=
|
| 849 |
|
| 850 |
-
def find_view(self, view_id, prefer_owner=None, include_trashed=False
|
|
|
|
| 851 |
"""`(owner_username, view)` for a view living in ANY personal stratum, else None.
|
| 852 |
|
| 853 |
⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
|
|
@@ -883,7 +905,8 @@ class TableStore:
|
|
| 883 |
vid = str(view_id or '').strip()
|
| 884 |
if not vid:
|
| 885 |
return None
|
| 886 |
-
|
|
|
|
| 887 |
if rows is not None:
|
| 888 |
hit = rows.find_view(vid, prefer_owner)
|
| 889 |
if hit is not None or not include_trashed:
|
|
@@ -895,9 +918,10 @@ class TableStore:
|
|
| 895 |
if isinstance(record, dict):
|
| 896 |
return owner, dict(record, id=vid)
|
| 897 |
return None
|
| 898 |
-
|
|
|
|
| 899 |
try:
|
| 900 |
-
data = self._st.get(self.table_key) or {}
|
| 901 |
except Exception:
|
| 902 |
return None
|
| 903 |
mine = str(prefer_owner or '').strip()
|
|
@@ -932,7 +956,7 @@ class TableStore:
|
|
| 932 |
return str(who), dict(record, id=vid)
|
| 933 |
return None
|
| 934 |
|
| 935 |
-
def find_folder(self, folder_id):
|
| 936 |
"""`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
|
| 937 |
personal stratum, else None. `find_view`'s sibling, and here for the same reason.
|
| 938 |
|
|
@@ -954,7 +978,7 @@ class TableStore:
|
|
| 954 |
if rows is not None:
|
| 955 |
return rows.find_folder(fid)
|
| 956 |
try:
|
| 957 |
-
data = self._st.get(self.table_key) or {}
|
| 958 |
except Exception:
|
| 959 |
return None
|
| 960 |
for username, ws in data.items():
|
|
@@ -1015,7 +1039,7 @@ class TableStore:
|
|
| 1015 |
self.migrate_legacy_shared_views()
|
| 1016 |
rev = None
|
| 1017 |
try:
|
| 1018 |
-
#
|
| 1019 |
rev, data = self._read_owners_at(username, SHARED_KEY, ASSIGNED_KEY)
|
| 1020 |
ws = data.get(username, {}) or {}
|
| 1021 |
# A collision acknowledgement is protocol state, not part of a field definition.
|
|
@@ -1119,39 +1143,8 @@ class TableStore:
|
|
| 1119 |
return (out, data) if include_document else out
|
| 1120 |
|
| 1121 |
# ---------------------------------------------------------------- write
|
| 1122 |
-
def _audited_update(self, apply, *,
|
| 1123 |
-
"""
|
| 1124 |
-
|
| 1125 |
-
⭐⭐ W50-T25 (ruling R9 / contract C3) — THE DOOR EVERY VIEW MUTATION GOES THROUGH.
|
| 1126 |
-
`apply(data, record)` receives the whole workspace document as the store's locked read
|
| 1127 |
-
returned it, plus `record(...)` — the ONLY way to write an audit row. Calling `record`
|
| 1128 |
-
inside `apply` is what makes the row and the workspace change the SAME commit: two
|
| 1129 |
-
transactions would let the change land while the trail row failed, and the trail would
|
| 1130 |
-
then have a hole exactly where the bug was (owner instruction 19).
|
| 1131 |
-
|
| 1132 |
-
`rev` is the `store_kv.rev` the client last read, or None for no check.
|
| 1133 |
-
|
| 1134 |
-
TWO ENFORCEMENT PATHS, and the difference is stated rather than implied:
|
| 1135 |
-
|
| 1136 |
-
* IN-TRANSACTION (`view_audit.guards(st)`) — the handle's `update` takes `expect_rev`,
|
| 1137 |
-
so the comparison happens against the value its own `SELECT … FOR UPDATE` returned
|
| 1138 |
-
and NO writer can slip between the check and the write. `core/store_pg.py` alone
|
| 1139 |
-
offers this.
|
| 1140 |
-
* ADVISORY (everything else, which is every caller today) — read `revision()`, compare,
|
| 1141 |
-
then write. A real TOCTOU window: another writer can land in between, so this refuses
|
| 1142 |
-
every stale write it can SEE but cannot promise it saw all of them.
|
| 1143 |
-
|
| 1144 |
-
⛔ THE ADVISORY PATH IS NEVER SILENTLY SKIPPED. A handle that cannot answer `revision()`
|
| 1145 |
-
at all makes the guard unanswerable, and an unanswerable guard fails CLOSED: it raises
|
| 1146 |
-
`RevisionUnverifiable` rather than writing as though nothing had been asked.
|
| 1147 |
-
|
| 1148 |
-
⛔⛔ AND ADVISORY IS WHAT PRODUCTION GETS TODAY. `harness/runtime.TenantRuntime.update`
|
| 1149 |
-
is `(self, name, fn, flush="sync")` with no keyword forward, and every API caller hands
|
| 1150 |
-
this class `st=session.runtime`. One line in that file — outside W50-T25's fence — flips
|
| 1151 |
-
every view write onto the in-transaction path and moves the trail's home from the
|
| 1152 |
-
`__audit__` document ring to `control.audit`. Until then the ring IS the trail, and it is
|
| 1153 |
-
atomic on Postgres too, because `apply` runs inside that backend's own transaction.
|
| 1154 |
-
"""
|
| 1155 |
st = self._st
|
| 1156 |
row_backend = self._settings_rows
|
| 1157 |
# A LIST, because one folder move changes several views' placements and each is its own
|
|
@@ -1160,6 +1153,7 @@ class TableStore:
|
|
| 1160 |
rows = []
|
| 1161 |
|
| 1162 |
def _apply(data):
|
|
|
|
| 1163 |
data = data if isinstance(data, dict) else {}
|
| 1164 |
|
| 1165 |
def record(*, action, owner, view_id, actor=None, origin=None,
|
|
@@ -1170,7 +1164,7 @@ class TableStore:
|
|
| 1170 |
return None
|
| 1171 |
values = dict(table_key=self.table_key, owner=owner, view_id=view_id,
|
| 1172 |
action=action, actor=actor or owner, origin=origin,
|
| 1173 |
-
before=before, after=after
|
| 1174 |
if row_backend is not None:
|
| 1175 |
entry = view_audit.row(tenant=row_backend.tenant_slug, **values)
|
| 1176 |
else:
|
|
@@ -1184,27 +1178,12 @@ class TableStore:
|
|
| 1184 |
if row_backend is not None:
|
| 1185 |
if owners is None:
|
| 1186 |
raise ValueError('a row settings mutation needs explicit owner rows')
|
| 1187 |
-
return row_backend.update(_apply, owners=owners,
|
| 1188 |
-
if view_audit.
|
| 1189 |
-
return st.update(self.table_key, _apply, flush=flush,
|
| 1190 |
-
expect_rev=rev, audit_row=rows)
|
| 1191 |
-
if rev is not None:
|
| 1192 |
-
reader = getattr(st, 'revision', None)
|
| 1193 |
-
if not callable(reader):
|
| 1194 |
-
raise view_audit.RevisionUnverifiable(
|
| 1195 |
-
'this deployment cannot tell which revision of the view list you are '
|
| 1196 |
-
'looking at, so the change was not saved. Reload and try again.')
|
| 1197 |
-
try:
|
| 1198 |
-
current = int((reader(self.table_key) or {}).get('rev') or 0)
|
| 1199 |
-
except Exception as exc: # noqa: BLE001
|
| 1200 |
-
raise view_audit.RevisionUnverifiable(
|
| 1201 |
-
'the view list revision could not be read, so the change was not saved. '
|
| 1202 |
-
'Reload and try again.') from exc
|
| 1203 |
-
if current != int(rev):
|
| 1204 |
-
raise view_audit.ViewRevisionStale(rev_current=current, rev_sent=int(rev))
|
| 1205 |
return st.update(self.table_key, _apply, flush=flush)
|
| 1206 |
|
| 1207 |
-
def _update(self, username, change, shared=None, *,
|
| 1208 |
"""Apply `change(ws)` to this user's stratum, and `shared(shared_fields)` to the
|
| 1209 |
tenant-wide one, in ONE transaction.
|
| 1210 |
|
|
@@ -1251,7 +1230,7 @@ class TableStore:
|
|
| 1251 |
# the in-process cache (read-your-writes for every subsequent render); the hub write
|
| 1252 |
# coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
|
| 1253 |
owners = (username, SHARED_KEY) if shared is not None else (username,)
|
| 1254 |
-
return self._audited_update(_up,
|
| 1255 |
|
| 1256 |
def rename_choice_values(self, username, change):
|
| 1257 |
"""Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
|
|
@@ -1353,68 +1332,39 @@ class TableStore:
|
|
| 1353 |
|
| 1354 |
self._update(username, _set)
|
| 1355 |
|
| 1356 |
-
def save_folders(self, username, folders, item_folders, *,
|
| 1357 |
-
actor=None, origin=None
|
| 1358 |
-
"""
|
| 1359 |
-
|
| 1360 |
-
|
| 1361 |
-
|
| 1362 |
-
|
| 1363 |
-
|
| 1364 |
-
not there yet. One write, one consistent state.
|
| 1365 |
-
|
| 1366 |
-
⭐⭐ W41-T12 (R1 + C9) — AND IT VALIDATES NOW. That first paragraph used to say the
|
| 1367 |
-
caller had already VALIDATED, and until wave 41 the shape was flat enough for that to be
|
| 1368 |
-
harmless. `parent` changes it: a depth cap and a cycle refusal that live only in the
|
| 1369 |
-
caller are a convenience, the same two here are a wall, and nesting is the first folder
|
| 1370 |
-
feature where the difference can hang a reader instead of merely looking wrong.
|
| 1371 |
-
`validate_folder_tree` above carries the rules and the reasoning.
|
| 1372 |
-
|
| 1373 |
-
⛔ VALIDATION RUNS BEFORE `_update`, NEVER INSIDE ITS CALLBACK. `_update` pre-creates
|
| 1374 |
-
five strata on the cached document and commits with flush='async', so a raise from
|
| 1375 |
-
inside the callback would abandon a half-touched transaction whose commit is already in
|
| 1376 |
-
flight. Raising first means a refused tree leaves the store exactly as it was.
|
| 1377 |
-
|
| 1378 |
-
RETURNS the accepted picture and every repair made along the way:
|
| 1379 |
-
`{'folders': …, 'itemFolders': …, 'repairs': [{'reason': …, …}]}`. It returned None
|
| 1380 |
-
before; the return value is what lets a caller say WHY a folder came back at the root.
|
| 1381 |
-
RAISES `FolderTreeError` on a cycle or an over-deep chain, having written nothing.
|
| 1382 |
-
|
| 1383 |
-
`rebase_at` (S82-T156) is the revision the caller read the tree at. A stale `rev` then
|
| 1384 |
-
lands, guarded at `rebase_at`, when it undoes no later view placement or folder parent change.
|
| 1385 |
"""
|
| 1386 |
-
tree, placed, repairs = validate_folder_tree(folders, item_folders)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1387 |
# ⭐ W50-T25 — THIS IS C3's `move` DOOR, and the placement map is keyed by SURFACE
|
| 1388 |
# FIRST (`{'views': {viewId: folderId}}`). Reading item ids off the top level finds the
|
| 1389 |
# surface names instead and matches nothing, which is `find_folder`'s own recorded trap
|
| 1390 |
# one layer down; the audit hook would then silently record zero moves.
|
| 1391 |
was = {}
|
| 1392 |
-
landing = {}
|
| 1393 |
-
|
| 1394 |
-
def _parents(raw):
|
| 1395 |
-
return {json.dumps([surface, str(row['id'])]): row.get('parent')
|
| 1396 |
-
for surface, rows in (raw or {}).items() if isinstance(rows, list)
|
| 1397 |
-
for row in rows if isinstance(row, dict) and row.get('id')}
|
| 1398 |
-
|
| 1399 |
-
parents = _parents(tree)
|
| 1400 |
|
| 1401 |
def _set(ws):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1402 |
was.clear()
|
| 1403 |
was.update({str(k): v for k, v
|
| 1404 |
in ((ws.get('itemFolders') or {}).get('views') or {}).items()})
|
| 1405 |
-
before_parents = _parents(ws.get('folders'))
|
| 1406 |
-
for folder_id in sorted(set(before_parents) | set(parents)):
|
| 1407 |
-
if before_parents.get(folder_id) != parents.get(folder_id):
|
| 1408 |
-
_note_write(ws, folder_id, landing.get('at'), 'parent',
|
| 1409 |
-
{'parent': before_parents.get(folder_id)}, FOLDER_WRITES_KEY,
|
| 1410 |
-
author=actor or username)
|
| 1411 |
ws['folders'] = tree
|
| 1412 |
ws['itemFolders'] = placed
|
| 1413 |
-
now = placed.get('views') or {}
|
| 1414 |
-
for view_id in sorted(set(was) | {str(k) for k in now}):
|
| 1415 |
-
if was.get(view_id) != now.get(view_id):
|
| 1416 |
-
note_view_write(ws, view_id, landing.get('at'), 'place',
|
| 1417 |
-
{'folderId': was.get(view_id)}, author=actor or username)
|
| 1418 |
|
| 1419 |
def _audit(ws, record):
|
| 1420 |
now = (ws.get('itemFolders') or {}).get('views') or {}
|
|
@@ -1434,31 +1384,10 @@ class TableStore:
|
|
| 1434 |
before=dict(base, folderId=before_folder),
|
| 1435 |
after=dict(base, folderId=after_folder))
|
| 1436 |
|
| 1437 |
-
def _write(at_rev):
|
| 1438 |
-
landing['at'] = self._next_rev(at_rev)
|
| 1439 |
-
self._update(username, _set, rev=at_rev, audit=_audit)
|
| 1440 |
-
|
| 1441 |
-
def _conflict(ws, sent):
|
| 1442 |
-
filed = (placed.get('views') or {})
|
| 1443 |
-
held = ((ws.get('itemFolders') or {}).get('views') or {})
|
| 1444 |
-
held_parents = _parents(ws.get('folders'))
|
| 1445 |
-
for folder_id in set(parents) | set(held_parents):
|
| 1446 |
-
current = held_parents.get(folder_id)
|
| 1447 |
-
if parents.get(folder_id) == current:
|
| 1448 |
-
continue
|
| 1449 |
-
later, judgeable = _later_writes(ws, folder_id, sent, ('parent',),
|
| 1450 |
-
book_key=FOLDER_WRITES_KEY)
|
| 1451 |
-
if later and (not judgeable or current != (later[0].get('before') or {}).get('parent')):
|
| 1452 |
-
return True
|
| 1453 |
-
return any(stale_place_conflict(ws, vid, filed.get(vid), sent)
|
| 1454 |
-
for vid in set(filed) | set(held) if filed.get(vid) != held.get(vid))
|
| 1455 |
-
|
| 1456 |
try:
|
| 1457 |
-
|
| 1458 |
-
except
|
| 1459 |
-
|
| 1460 |
-
raise
|
| 1461 |
-
self._land_stale(username, rev, stale, _conflict, _write, at=rebase_at)
|
| 1462 |
# Fresh containers: the caller inspecting its receipt must not be able to reach into
|
| 1463 |
# the document the store is holding and edit it from underneath the cache.
|
| 1464 |
return {'folders': {s: list(rows) for s, rows in tree.items()},
|
|
@@ -1468,7 +1397,7 @@ class TableStore:
|
|
| 1468 |
def save_view_order(self, username, order):
|
| 1469 |
"""⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail.
|
| 1470 |
|
| 1471 |
-
Wholesale
|
| 1472 |
list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
|
| 1473 |
view went.
|
| 1474 |
|
|
@@ -1825,7 +1754,7 @@ class TableStore:
|
|
| 1825 |
return receipt
|
| 1826 |
|
| 1827 |
def assign_view(self, owner, view, recipient, allowed_pids=(), *,
|
| 1828 |
-
actor=None, origin=None
|
| 1829 |
"""Create one immutable assigned-view projection for ``recipient``.
|
| 1830 |
|
| 1831 |
The filter snapshot is deliberately separate from the presentation
|
|
@@ -1856,8 +1785,11 @@ class TableStore:
|
|
| 1856 |
}
|
| 1857 |
def _up(data, audit):
|
| 1858 |
record['updatedAt'] = write_execution.utcnow().isoformat()
|
| 1859 |
-
|
|
|
|
| 1860 |
bucket.setdefault(who, {})[assignment_id] = record
|
|
|
|
|
|
|
| 1861 |
# ⭐ W50-T25 — C3's `share` action. The audit `owner` is the SOURCE VIEW's owner,
|
| 1862 |
# not the recipient and not necessarily the actor: the trail answers "what happened
|
| 1863 |
# to whose view", and an assignment is something done TO the owner's record.
|
|
@@ -1866,7 +1798,7 @@ class TableStore:
|
|
| 1866 |
before=None,
|
| 1867 |
after=dict(view_audit.summary(source) or {}, recipient=who))
|
| 1868 |
return data
|
| 1869 |
-
self._audited_update(_up,
|
| 1870 |
self.log_view_activity(source_id, 'assigned', by=actor or owner, after=who, via=origin)
|
| 1871 |
return dict(record)
|
| 1872 |
|
|
@@ -1934,14 +1866,211 @@ class TableStore:
|
|
| 1934 |
if not rows:
|
| 1935 |
views.pop(who, None)
|
| 1936 |
if not views:
|
| 1937 |
-
data
|
|
|
|
|
|
|
|
|
|
| 1938 |
return data
|
| 1939 |
|
| 1940 |
self._audited_update(lambda data, _audit: _up(data), owners=(ASSIGNED_KEY,))
|
| 1941 |
return dismissed['value']
|
| 1942 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1943 |
def save_view(self, username, view, reserved_names=(), *,
|
| 1944 |
-
actor=None, origin=None,
|
| 1945 |
"""Upsert a SavedView into its creator's personal workspace.
|
| 1946 |
|
| 1947 |
Sharing is exclusively a grant-registry concern; this method never chooses a second home
|
|
@@ -1963,9 +2092,6 @@ class TableStore:
|
|
| 1963 |
`view_upsert` with a fresh id — `grid_events` L2408 says so in as many words), and
|
| 1964 |
`restore` belongs to W50-T26's trash door. Both are honest only when the caller names
|
| 1965 |
them; nothing is guessed.
|
| 1966 |
-
|
| 1967 |
-
`rebase=True` (S82-T156) lands a stale `rev` when the write undoes no change made after
|
| 1968 |
-
it, guarded again at the current revision. A real conflict still raises `ViewRevisionStale`.
|
| 1969 |
"""
|
| 1970 |
view_id = str((view or {}).get('id') or '').strip()
|
| 1971 |
if not view_id:
|
|
@@ -1982,7 +2108,6 @@ class TableStore:
|
|
| 1982 |
raise ValueError('Saved Views use modern grants; view-level permissions are retired')
|
| 1983 |
accepted = {}
|
| 1984 |
activity_before = {}
|
| 1985 |
-
landing = {}
|
| 1986 |
|
| 1987 |
def _up(data, audit):
|
| 1988 |
# View names are tenant-global across personal source records. A grant is only an
|
|
@@ -2010,32 +2135,18 @@ class TableStore:
|
|
| 2010 |
prior = (ws.get('views') or {}).get(view_id)
|
| 2011 |
activity_before['value'] = view_audit.summary(prior)
|
| 2012 |
ws.setdefault('views', {})[view_id] = payload
|
| 2013 |
-
note_view_write(ws, view_id, landing.get('at'), 'view', view_members(prior),
|
| 2014 |
-
author=actor or username)
|
| 2015 |
audit(action=action or view_audit.derive_action(prior, payload),
|
| 2016 |
owner=username, view_id=view_id, actor=actor, origin=origin,
|
| 2017 |
before=view_audit.summary(prior), after=view_audit.summary(payload))
|
| 2018 |
return data
|
| 2019 |
|
| 2020 |
-
|
| 2021 |
-
landing['at'] = self._next_rev(at_rev)
|
| 2022 |
-
self._audited_update(_up, rev=at_rev, flush='async', owners=(username,))
|
| 2023 |
-
|
| 2024 |
-
try:
|
| 2025 |
-
_write(rev)
|
| 2026 |
-
except view_audit.ViewRevisionStale as stale:
|
| 2027 |
-
if rev is None or not rebase:
|
| 2028 |
-
raise
|
| 2029 |
-
self._land_stale(username, rev, stale,
|
| 2030 |
-
lambda ws, sent: stale_view_conflict(
|
| 2031 |
-
ws, view_id, requested, sent, requester=actor or username),
|
| 2032 |
-
_write)
|
| 2033 |
self.log_view_activity(view_id, 'saved', by=actor or username,
|
| 2034 |
before=activity_before.get('value'),
|
| 2035 |
after=view_audit.summary(accepted), via=origin)
|
| 2036 |
return dict(accepted)
|
| 2037 |
|
| 2038 |
-
def delete_view(self, username, view_id, *, actor=None, origin=None
|
| 2039 |
"""Delete an owner source View and its dependent assignments, never another user's.
|
| 2040 |
|
| 2041 |
⭐⭐ W50-T25 — C3's `delete` action, and `actor` is a SEPARATE parameter from `username`
|
|
@@ -2065,7 +2176,6 @@ class TableStore:
|
|
| 2065 |
# a caller that named no actor must not produce a trash entry blaming nobody.
|
| 2066 |
who = str(actor or owner or '') or None
|
| 2067 |
removed = {}
|
| 2068 |
-
at = self._next_rev(rev)
|
| 2069 |
|
| 2070 |
def _up(data, audit):
|
| 2071 |
views = data.get(owner, {}).get('views') or {}
|
|
@@ -2073,8 +2183,6 @@ class TableStore:
|
|
| 2073 |
removed.clear()
|
| 2074 |
if isinstance(before, dict):
|
| 2075 |
removed['value'] = view_audit.summary(before)
|
| 2076 |
-
note_view_write(data.setdefault(owner, {}), vid, at, 'gone', view_members(before),
|
| 2077 |
-
author=who)
|
| 2078 |
views.pop(vid, None)
|
| 2079 |
audit(action='delete' if isinstance(before, dict) else None,
|
| 2080 |
owner=owner, view_id=vid, actor=actor, origin=origin,
|
|
@@ -2114,7 +2222,7 @@ class TableStore:
|
|
| 2114 |
purge_trash(bucket, now=stamp)
|
| 2115 |
return data
|
| 2116 |
|
| 2117 |
-
self._audited_update(_up,
|
| 2118 |
if removed:
|
| 2119 |
self.log_view_activity(vid, 'deleted', by=who, before=removed['value'], via=origin)
|
| 2120 |
if row_backed:
|
|
@@ -2207,7 +2315,7 @@ class TableStore:
|
|
| 2207 |
return False
|
| 2208 |
return True
|
| 2209 |
|
| 2210 |
-
def purge_view(self, username, view_id, *,
|
| 2211 |
"""R11's "Delete forever": destroy one trashed view NOW. `(name, adopted_cell_count)`.
|
| 2212 |
|
| 2213 |
⛔ THE ADOPTION RUNS FIRST AND A FAILURE REFUSES THE PURGE. Unlike the retention sweep on
|
|
@@ -2219,8 +2327,7 @@ class TableStore:
|
|
| 2219 |
inventing an eighth here would widen a vocabulary two gates pin. The `delete` row written
|
| 2220 |
when the view entered the trash is still the record of its removal. Booked as a finding.
|
| 2221 |
|
| 2222 |
-
|
| 2223 |
-
read-modify-write. `action=None` is how that door spells "no row" (see `delete_view`).
|
| 2224 |
"""
|
| 2225 |
vid = str(view_id or '').strip()
|
| 2226 |
owner = str(username or '').strip().lower()
|
|
@@ -2262,7 +2369,7 @@ class TableStore:
|
|
| 2262 |
audit(action=None, owner=owner, view_id=vid)
|
| 2263 |
return current
|
| 2264 |
|
| 2265 |
-
self._audited_update(_up,
|
| 2266 |
if removed:
|
| 2267 |
self.log_view_activity(vid, 'purged', by=actor or owner,
|
| 2268 |
before=removed['value'], via=origin)
|
|
@@ -2373,7 +2480,7 @@ class TableStore:
|
|
| 2373 |
return str(who), dict(entry, id=vid)
|
| 2374 |
return None
|
| 2375 |
|
| 2376 |
-
def restore_view(self, username, view_id, *, actor=None, origin=None
|
| 2377 |
"""Move one trashed view back into `views`, with every assignment the delete cascaded.
|
| 2378 |
|
| 2379 |
⭐⭐ R9's *"one-click Restore"*, and it is an AUDITED action: `restore` is one of C3's
|
|
@@ -2448,7 +2555,7 @@ class TableStore:
|
|
| 2448 |
before=None, after=view_audit.summary(payload))
|
| 2449 |
return data
|
| 2450 |
|
| 2451 |
-
self._audited_update(_up,
|
| 2452 |
self.log_view_activity(vid, 'restored', by=actor or owner,
|
| 2453 |
after=view_audit.summary(restored), via=origin)
|
| 2454 |
return dict(restored)
|
|
@@ -2597,8 +2704,8 @@ class TableStore:
|
|
| 2597 |
Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
|
| 2598 |
columns — the caller enforces the prefix). The stored overlay VALUES for the key are
|
| 2599 |
scrubbed with it: a deleted column's cells must not linger as orphan data that would
|
| 2600 |
-
silently resurface if the key were ever reused. Views
|
| 2601 |
-
|
| 2602 |
|
| 2603 |
⭐⭐ W36-T25 — AND THE TENANT-WIDE SUMMARY GOES WITH IT, for exactly the reason the
|
| 2604 |
paragraph above gives about cells: an orphan `agg` under a deleted key is state nobody can
|
|
@@ -2616,6 +2723,8 @@ class TableStore:
|
|
| 2616 |
for row in ws['overlays'].values():
|
| 2617 |
if isinstance(row, dict):
|
| 2618 |
row.pop(key, None)
|
|
|
|
|
|
|
| 2619 |
|
| 2620 |
self._update(username, _drop, shared=lambda shared_fields: shared_fields.pop(key, None))
|
| 2621 |
|
|
@@ -2756,7 +2865,12 @@ class TableStore:
|
|
| 2756 |
entry['options'] = merge_choice_options(entry.get('options'), values)
|
| 2757 |
fields[key] = entry
|
| 2758 |
|
| 2759 |
-
self._update(username, _patch)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2760 |
|
| 2761 |
|
| 2762 |
def make(table_key, st=None):
|
|
|
|
| 16 |
(presentation/query state) — see modules/customer_data.py's customer_lists key.
|
| 17 |
"""
|
| 18 |
import copy
|
|
|
|
| 19 |
import json
|
| 20 |
+
import re
|
| 21 |
import time
|
| 22 |
|
| 23 |
import core.store as store
|
|
|
|
| 88 |
#: is NOT in `RESERVED_MEMBERS` (that tuple names TOP-LEVEL members that are not users).
|
| 89 |
TRASH_KEY = 'trash'
|
| 90 |
|
| 91 |
+
#: `find_view(settings=...)` default: resolve the settings-rows backend on this call.
|
| 92 |
+
SETTINGS_UNRESOLVED = object()
|
| 93 |
+
|
| 94 |
#: R9's retention, in days. ⛔ A CONSTANT RATHER THAN A LITERAL: it is read by `delete_view`,
|
| 95 |
#: by `trash()` and by the gate that proves a 31-day-old entry purges while a 29-day-old one
|
| 96 |
#: does not, and three copies of `30` is how two of them drift.
|
|
|
|
| 100 |
#: matching `view_audit.row`'s `ts` and `store_pg.revision`'s `updated_at` on both backends).
|
| 101 |
TRASH_RETENTION_SECONDS = TRASH_RETENTION_DAYS * 24 * 60 * 60
|
| 102 |
|
| 103 |
+
_VIEW_OP_MEMBERS = 'viewOpMembers'
|
| 104 |
+
_VIEW_OP_TOP = frozenset(('name', 'note', 'locked'))
|
| 105 |
+
_VIEW_OP_CONFIG = frozenset(('group', 'display', 'filterConj', 'colorBy', 'rowHeightMode',
|
| 106 |
+
'frozenCount', 'important', 'secondaryView', 'application', 'fullscreen'))
|
| 107 |
+
_VIEW_OP_FAMILIES = frozenset(('hidden', 'sort', 'filter', 'width', 'position', 'summary', 'memberPid'))
|
| 108 |
+
_VIEW_OP_POSITION = re.compile(r'[0-9A-Za-z]{1,64}\Z')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
|
| 111 |
+
def _view_member_identity(member):
|
| 112 |
+
"""Use one receipt and last-writer identity for equivalent record ids."""
|
| 113 |
+
if isinstance(member, str) and member.startswith('memberPid:'):
|
| 114 |
+
try:
|
| 115 |
+
return 'memberPid:' + str(int(member.partition(':')[2]))
|
| 116 |
+
except ValueError:
|
| 117 |
+
pass
|
| 118 |
+
return member
|
| 119 |
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
+
class _ViewOpRefused(ValueError):
|
| 122 |
+
def __init__(self, reason, message):
|
| 123 |
+
super().__init__(message)
|
| 124 |
+
self.reason = reason
|
| 125 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
|
| 127 |
+
def _view_filter_nodes(nodes, parent=None):
|
| 128 |
+
for node in nodes:
|
| 129 |
+
yield node, parent, nodes
|
| 130 |
+
if isinstance(node.get('children'), list):
|
| 131 |
+
yield from _view_filter_nodes(node['children'], node.get('id'))
|
| 132 |
|
| 133 |
+
|
| 134 |
+
def prune_view_field_members(view, *, live_keys=None, removed_keys=(), prune_filters=True):
|
| 135 |
+
"""Drop dead field references from a mutable View without changing live member values."""
|
| 136 |
+
cfg = view.get('config') if isinstance(view, dict) else None
|
| 137 |
+
if not isinstance(cfg, dict):
|
| 138 |
return
|
| 139 |
+
removed = set(removed_keys)
|
| 140 |
+
|
| 141 |
+
def keep(key):
|
| 142 |
+
return key not in removed and (live_keys is None or key in live_keys)
|
| 143 |
+
|
| 144 |
+
for name in ('visible', 'order'):
|
| 145 |
+
if isinstance(cfg.get(name), list):
|
| 146 |
+
cfg[name] = [key for key in cfg[name] if keep(key)]
|
| 147 |
+
for name in ('widths', 'positions', 'summary'):
|
| 148 |
+
if isinstance(cfg.get(name), dict):
|
| 149 |
+
cfg[name] = {key: value for key, value in cfg[name].items() if keep(key)}
|
| 150 |
+
if isinstance(cfg.get('sorts'), list):
|
| 151 |
+
cfg['sorts'] = [rule for rule in cfg['sorts']
|
| 152 |
+
if isinstance(rule, dict) and keep(rule.get('colId'))]
|
| 153 |
+
for name in ('groupBy', 'colorBy'):
|
| 154 |
+
if cfg.get(name) is not None and not keep(cfg[name]):
|
| 155 |
+
cfg.pop(name, None)
|
| 156 |
+
|
| 157 |
+
def filters(nodes):
|
| 158 |
+
result = []
|
| 159 |
+
for node in nodes:
|
| 160 |
+
if not isinstance(node, dict):
|
| 161 |
+
continue
|
| 162 |
+
if isinstance(node.get('children'), list):
|
| 163 |
+
node = {**node, 'children': filters(node['children'])}
|
| 164 |
+
elif not keep(node.get('colId')) or (node.get('rhsColId') and not keep(node['rhsColId'])):
|
| 165 |
+
continue
|
| 166 |
+
result.append(node)
|
| 167 |
+
return result
|
| 168 |
+
|
| 169 |
+
if prune_filters and isinstance(cfg.get('filters'), list):
|
| 170 |
+
cfg['filters'] = filters(cfg['filters'])
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _view_position_seed(keys):
|
| 174 |
+
# Match the browser's initial column and legacy sort positions.
|
| 175 |
+
alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
| 176 |
+
result = {}
|
| 177 |
+
for index, key in enumerate(dict.fromkeys(keys), 1):
|
| 178 |
+
number, digits = index, ''
|
| 179 |
+
while number:
|
| 180 |
+
number, remainder = divmod(number, len(alphabet))
|
| 181 |
+
digits = alphabet[remainder] + digits
|
| 182 |
+
result[key] = digits.rjust(8, '0') + 'V'
|
| 183 |
+
return result
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _view_member_value(view, member):
|
| 187 |
+
family, _, key = member.partition(':')
|
| 188 |
+
cfg = view.get('config') or {}
|
| 189 |
+
if family in _VIEW_OP_TOP:
|
| 190 |
+
return member in view, copy.deepcopy(view.get(member))
|
| 191 |
+
if family == 'hidden':
|
| 192 |
+
return True, key not in cfg.get('visible', [])
|
| 193 |
+
if family == 'memberPid':
|
| 194 |
+
return True, int(key) in cfg.get('memberPids', [])
|
| 195 |
+
if family == 'sort':
|
| 196 |
+
hit = next((rule for rule in cfg.get('sorts', []) if rule.get('colId') == key), None)
|
| 197 |
+
return hit is not None, ({k: copy.deepcopy(v) for k, v in hit.items() if k != 'colId'}
|
| 198 |
+
if hit is not None else None)
|
| 199 |
+
if family in ('filter', 'filterConj') and key:
|
| 200 |
+
hit = next(((node, parent) for node, parent, _ in
|
| 201 |
+
_view_filter_nodes(cfg.get('filters', [])) if node.get('id') == key), None)
|
| 202 |
+
if family == 'filter':
|
| 203 |
+
return hit is not None, {'node': copy.deepcopy(hit[0]), 'parent': hit[1]} if hit else None
|
| 204 |
+
return bool(hit and 'children' in hit[0]), hit[0].get('conj', 'and') if hit else None
|
| 205 |
+
maps = {'width': 'widths', 'summary': 'summary', 'position': 'positions'}
|
| 206 |
+
if family in maps:
|
| 207 |
+
values = cfg.get(maps[family]) or {}
|
| 208 |
+
return key in values, copy.deepcopy(values.get(key))
|
| 209 |
+
name = 'groupBy' if member == 'group' else member
|
| 210 |
+
return name in cfg, copy.deepcopy(cfg.get(name))
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def _set_view_member(view, member, value, *, present=True):
|
| 214 |
+
family, _, key = member.partition(':')
|
| 215 |
+
cfg = view.setdefault('config', {})
|
| 216 |
+
if family == 'hidden':
|
| 217 |
+
visible = list(cfg.get('visible') or [])
|
| 218 |
+
cfg['visible'] = ([item for item in visible if item != key] if value else
|
| 219 |
+
list(dict.fromkeys([*visible, key])))
|
| 220 |
+
elif family == 'memberPid':
|
| 221 |
+
pids = list(cfg.get('memberPids') or [])
|
| 222 |
+
cfg['memberPids'] = ([pid for pid in pids if pid != int(key)] if not present or not value
|
| 223 |
+
else list(dict.fromkeys([*pids, int(key)])))
|
| 224 |
+
elif family == 'sort':
|
| 225 |
+
rules = list(cfg.get('sorts') or [])
|
| 226 |
+
if any('pos' not in rule for rule in rules):
|
| 227 |
+
seeds = _view_position_seed(rule['colId'] for rule in rules)
|
| 228 |
+
rules = [{**rule, 'pos': rule.get('pos', seeds[rule['colId']])} for rule in rules]
|
| 229 |
+
rules = [rule for rule in rules if rule.get('colId') != key]
|
| 230 |
+
if present:
|
| 231 |
+
rules.append({'colId': key, **copy.deepcopy(value)})
|
| 232 |
+
cfg['sorts'] = sorted(rules, key=lambda rule: (rule['pos'], rule['colId']))
|
| 233 |
+
elif family == 'filter':
|
| 234 |
+
nodes = cfg.setdefault('filters', [])
|
| 235 |
+
found = list(_view_filter_nodes(nodes))
|
| 236 |
+
target = next((node for node, _, _ in found if node.get('id') == key), None)
|
| 237 |
+
parent = value.get('parent') if present else None
|
| 238 |
+
destination = nodes
|
| 239 |
+
if parent is not None:
|
| 240 |
+
destination = next((node.get('children') for node, _, _ in found
|
| 241 |
+
if node.get('id') == parent and isinstance(node.get('children'), list)), None)
|
| 242 |
+
descendants = ({node.get('id') for node, _, _ in
|
| 243 |
+
_view_filter_nodes([target])} if target else set())
|
| 244 |
+
if destination is None or parent in descendants:
|
| 245 |
+
raise _ViewOpRefused('target_gone', 'That filter group is no longer available.')
|
| 246 |
+
for node, _, siblings in found:
|
| 247 |
+
if node.get('id') == key:
|
| 248 |
+
siblings.remove(node)
|
| 249 |
+
if present:
|
| 250 |
+
node = copy.deepcopy(value['node'])
|
| 251 |
+
if target and isinstance(target.get('children'), list) and 'children' in node:
|
| 252 |
+
node['children'] = target['children']
|
| 253 |
+
node['id'] = key
|
| 254 |
+
destination.append(node)
|
| 255 |
+
elif family == 'filterConj' and key:
|
| 256 |
+
target = next((node for node, _, _ in _view_filter_nodes(cfg.get('filters', []))
|
| 257 |
+
if node.get('id') == key and isinstance(node.get('children'), list)), None)
|
| 258 |
+
if target is None:
|
| 259 |
+
raise _ViewOpRefused('target_gone', 'That filter group is no longer available.')
|
| 260 |
+
target['conj'] = value
|
| 261 |
+
elif family in ('width', 'summary', 'position'):
|
| 262 |
+
name = {'width': 'widths', 'summary': 'summary', 'position': 'positions'}[family]
|
| 263 |
+
values = cfg.setdefault(name, {})
|
| 264 |
+
if present:
|
| 265 |
+
values[key] = copy.deepcopy(value)
|
| 266 |
+
else:
|
| 267 |
+
values.pop(key, None)
|
| 268 |
+
if family == 'position':
|
| 269 |
+
cfg['order'] = sorted(values, key=lambda item: (values[item], item))
|
| 270 |
+
else:
|
| 271 |
+
target = view if family in _VIEW_OP_TOP else cfg
|
| 272 |
+
name = 'groupBy' if member == 'group' else member
|
| 273 |
+
if present:
|
| 274 |
+
target[name] = copy.deepcopy(value)
|
| 275 |
+
else:
|
| 276 |
+
target.pop(name, None)
|
| 277 |
|
| 278 |
|
| 279 |
class ViewNotInTrash(LookupError):
|
|
|
|
| 822 |
return self._st.get(self.table_key) or {}
|
| 823 |
|
| 824 |
def revision(self):
|
| 825 |
+
"""The workspace change revision, or None if unreadable."""
|
| 826 |
rows = self._settings_rows
|
| 827 |
if rows is not None:
|
| 828 |
with rows._reading() as con:
|
|
|
|
| 847 |
rev = self.revision()
|
| 848 |
return rev, self._st.get(self.table_key) or {}
|
| 849 |
|
| 850 |
+
def log_view_activity(self, view_id, change, *, by, before=None, after=None, via=None,
|
| 851 |
+
member=None, overwrote=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 852 |
"""Append one View event through the exact workspace store after its write succeeds."""
|
| 853 |
from core.user_tables import _activity_writer
|
| 854 |
append = _activity_writer(self._st, by)
|
|
|
|
| 859 |
if workspace == self.table_key), self.table_key.removesuffix('_table_workspace'))
|
| 860 |
if isinstance(via, dict):
|
| 861 |
via = via.get('client') or None
|
| 862 |
+
changes = {}
|
| 863 |
+
if member is not None:
|
| 864 |
+
changes.update(member={'before': None, 'after': member},
|
| 865 |
+
by={'before': None, 'after': by},
|
| 866 |
+
overwrote={'before': None, 'after': overwrote})
|
| 867 |
+
changes[change] = {'before': before, 'after': after}
|
| 868 |
append(kind='view', object={'database': database, 'id': str(view_id)}, by=by,
|
| 869 |
+
via=via, changes=changes)
|
| 870 |
|
| 871 |
+
def find_view(self, view_id, prefer_owner=None, include_trashed=False, *, document=None,
|
| 872 |
+
settings=SETTINGS_UNRESOLVED):
|
| 873 |
"""`(owner_username, view)` for a view living in ANY personal stratum, else None.
|
| 874 |
|
| 875 |
⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
|
|
|
|
| 905 |
vid = str(view_id or '').strip()
|
| 906 |
if not vid:
|
| 907 |
return None
|
| 908 |
+
# A caller looping over ids passes the backend it resolved once for this read (S85-T169).
|
| 909 |
+
rows = self._settings_rows if settings is SETTINGS_UNRESOLVED else settings
|
| 910 |
if rows is not None:
|
| 911 |
hit = rows.find_view(vid, prefer_owner)
|
| 912 |
if hit is not None or not include_trashed:
|
|
|
|
| 918 |
if isinstance(record, dict):
|
| 919 |
return owner, dict(record, id=vid)
|
| 920 |
return None
|
| 921 |
+
if document is None:
|
| 922 |
+
self.migrate_legacy_shared_views()
|
| 923 |
try:
|
| 924 |
+
data = document if document is not None else self._st.get(self.table_key) or {}
|
| 925 |
except Exception:
|
| 926 |
return None
|
| 927 |
mine = str(prefer_owner or '').strip()
|
|
|
|
| 956 |
return str(who), dict(record, id=vid)
|
| 957 |
return None
|
| 958 |
|
| 959 |
+
def find_folder(self, folder_id, *, document=None):
|
| 960 |
"""`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
|
| 961 |
personal stratum, else None. `find_view`'s sibling, and here for the same reason.
|
| 962 |
|
|
|
|
| 978 |
if rows is not None:
|
| 979 |
return rows.find_folder(fid)
|
| 980 |
try:
|
| 981 |
+
data = document if document is not None else self._st.get(self.table_key) or {}
|
| 982 |
except Exception:
|
| 983 |
return None
|
| 984 |
for username, ws in data.items():
|
|
|
|
| 1039 |
self.migrate_legacy_shared_views()
|
| 1040 |
rev = None
|
| 1041 |
try:
|
| 1042 |
+
# Publish the store change token independently of view write admission.
|
| 1043 |
rev, data = self._read_owners_at(username, SHARED_KEY, ASSIGNED_KEY)
|
| 1044 |
ws = data.get(username, {}) or {}
|
| 1045 |
# A collision acknowledgement is protocol state, not part of a field definition.
|
|
|
|
| 1143 |
return (out, data) if include_document else out
|
| 1144 |
|
| 1145 |
# ---------------------------------------------------------------- write
|
| 1146 |
+
def _audited_update(self, apply, *, flush='async', owners=None):
|
| 1147 |
+
"""Commit a workspace mutation and its audit rows in one store transaction."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1148 |
st = self._st
|
| 1149 |
row_backend = self._settings_rows
|
| 1150 |
# A LIST, because one folder move changes several views' placements and each is its own
|
|
|
|
| 1153 |
rows = []
|
| 1154 |
|
| 1155 |
def _apply(data):
|
| 1156 |
+
rows.clear()
|
| 1157 |
data = data if isinstance(data, dict) else {}
|
| 1158 |
|
| 1159 |
def record(*, action, owner, view_id, actor=None, origin=None,
|
|
|
|
| 1164 |
return None
|
| 1165 |
values = dict(table_key=self.table_key, owner=owner, view_id=view_id,
|
| 1166 |
action=action, actor=actor or owner, origin=origin,
|
| 1167 |
+
before=before, after=after)
|
| 1168 |
if row_backend is not None:
|
| 1169 |
entry = view_audit.row(tenant=row_backend.tenant_slug, **values)
|
| 1170 |
else:
|
|
|
|
| 1178 |
if row_backend is not None:
|
| 1179 |
if owners is None:
|
| 1180 |
raise ValueError('a row settings mutation needs explicit owner rows')
|
| 1181 |
+
return row_backend.update(_apply, owners=owners, audit_rows=rows)
|
| 1182 |
+
if view_audit.home(st) == view_audit.CONTROL_AUDIT:
|
| 1183 |
+
return st.update(self.table_key, _apply, flush=flush, audit_row=rows)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1184 |
return st.update(self.table_key, _apply, flush=flush)
|
| 1185 |
|
| 1186 |
+
def _update(self, username, change, shared=None, *, audit=None):
|
| 1187 |
"""Apply `change(ws)` to this user's stratum, and `shared(shared_fields)` to the
|
| 1188 |
tenant-wide one, in ONE transaction.
|
| 1189 |
|
|
|
|
| 1230 |
# the in-process cache (read-your-writes for every subsequent render); the hub write
|
| 1231 |
# coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
|
| 1232 |
owners = (username, SHARED_KEY) if shared is not None else (username,)
|
| 1233 |
+
return self._audited_update(_up, flush='async', owners=owners)
|
| 1234 |
|
| 1235 |
def rename_choice_values(self, username, change):
|
| 1236 |
"""Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
|
|
|
|
| 1332 |
|
| 1333 |
self._update(username, _set)
|
| 1334 |
|
| 1335 |
+
def save_folders(self, username, folders=None, item_folders=None, *, mutate=None,
|
| 1336 |
+
actor=None, origin=None):
|
| 1337 |
+
"""Save a validated folder picture or apply `mutate(ws)` to the latest stored workspace.
|
| 1338 |
+
|
| 1339 |
+
A mutation returns `(folders, item_folders)` or None to refuse without writing.
|
| 1340 |
+
Validation precedes folder, placement and audit changes inside the update closure.
|
| 1341 |
+
Events use this callback without a whole-tree revision guard, so unrelated gestures survive.
|
| 1342 |
+
The receipt contains `folders`, `itemFolders` and `repairs`; illegal trees raise `FolderTreeError`.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1343 |
"""
|
| 1344 |
+
tree, placed, repairs = (validate_folder_tree(folders, item_folders) if mutate is None
|
| 1345 |
+
else ({}, {}, []))
|
| 1346 |
+
|
| 1347 |
+
class _FolderMutationRefused(Exception):
|
| 1348 |
+
pass
|
| 1349 |
+
|
| 1350 |
# ⭐ W50-T25 — THIS IS C3's `move` DOOR, and the placement map is keyed by SURFACE
|
| 1351 |
# FIRST (`{'views': {viewId: folderId}}`). Reading item ids off the top level finds the
|
| 1352 |
# surface names instead and matches nothing, which is `find_folder`'s own recorded trap
|
| 1353 |
# one layer down; the audit hook would then silently record zero moves.
|
| 1354 |
was = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1355 |
|
| 1356 |
def _set(ws):
|
| 1357 |
+
nonlocal tree, placed, repairs
|
| 1358 |
+
if mutate is not None:
|
| 1359 |
+
picture = mutate(ws)
|
| 1360 |
+
if picture is None:
|
| 1361 |
+
raise _FolderMutationRefused()
|
| 1362 |
+
tree, placed, repairs = validate_folder_tree(*picture)
|
| 1363 |
was.clear()
|
| 1364 |
was.update({str(k): v for k, v
|
| 1365 |
in ((ws.get('itemFolders') or {}).get('views') or {}).items()})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1366 |
ws['folders'] = tree
|
| 1367 |
ws['itemFolders'] = placed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1368 |
|
| 1369 |
def _audit(ws, record):
|
| 1370 |
now = (ws.get('itemFolders') or {}).get('views') or {}
|
|
|
|
| 1384 |
before=dict(base, folderId=before_folder),
|
| 1385 |
after=dict(base, folderId=after_folder))
|
| 1386 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1387 |
try:
|
| 1388 |
+
self._update(username, _set, audit=_audit)
|
| 1389 |
+
except _FolderMutationRefused:
|
| 1390 |
+
return None
|
|
|
|
|
|
|
| 1391 |
# Fresh containers: the caller inspecting its receipt must not be able to reach into
|
| 1392 |
# the document the store is holding and edit it from underneath the cache.
|
| 1393 |
return {'folders': {s: list(rows) for s, rows in tree.items()},
|
|
|
|
| 1397 |
def save_view_order(self, username, order):
|
| 1398 |
"""⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail.
|
| 1399 |
|
| 1400 |
+
Wholesale: the client sends the full
|
| 1401 |
list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
|
| 1402 |
view went.
|
| 1403 |
|
|
|
|
| 1754 |
return receipt
|
| 1755 |
|
| 1756 |
def assign_view(self, owner, view, recipient, allowed_pids=(), *,
|
| 1757 |
+
actor=None, origin=None):
|
| 1758 |
"""Create one immutable assigned-view projection for ``recipient``.
|
| 1759 |
|
| 1760 |
The filter snapshot is deliberately separate from the presentation
|
|
|
|
| 1785 |
}
|
| 1786 |
def _up(data, audit):
|
| 1787 |
record['updatedAt'] = write_execution.utcnow().isoformat()
|
| 1788 |
+
assigned = data.setdefault(ASSIGNED_KEY, {})
|
| 1789 |
+
bucket = assigned.setdefault('views', {})
|
| 1790 |
bucket.setdefault(who, {})[assignment_id] = record
|
| 1791 |
+
for member in (assigned.get(_VIEW_OP_MEMBERS) or {}).get(assignment_id, {}).values():
|
| 1792 |
+
member.update(by=actor or owner, client_op_id=None)
|
| 1793 |
# ⭐ W50-T25 — C3's `share` action. The audit `owner` is the SOURCE VIEW's owner,
|
| 1794 |
# not the recipient and not necessarily the actor: the trail answers "what happened
|
| 1795 |
# to whose view", and an assignment is something done TO the owner's record.
|
|
|
|
| 1798 |
before=None,
|
| 1799 |
after=dict(view_audit.summary(source) or {}, recipient=who))
|
| 1800 |
return data
|
| 1801 |
+
self._audited_update(_up, flush='async', owners=(ASSIGNED_KEY,))
|
| 1802 |
self.log_view_activity(source_id, 'assigned', by=actor or owner, after=who, via=origin)
|
| 1803 |
return dict(record)
|
| 1804 |
|
|
|
|
| 1866 |
if not rows:
|
| 1867 |
views.pop(who, None)
|
| 1868 |
if not views:
|
| 1869 |
+
assigned = data[ASSIGNED_KEY]
|
| 1870 |
+
assigned.pop('views', None)
|
| 1871 |
+
if not assigned:
|
| 1872 |
+
data.pop(ASSIGNED_KEY, None)
|
| 1873 |
return data
|
| 1874 |
|
| 1875 |
self._audited_update(lambda data, _audit: _up(data), owners=(ASSIGNED_KEY,))
|
| 1876 |
return dismissed['value']
|
| 1877 |
|
| 1878 |
+
def _view_op_fields(self, workspace):
|
| 1879 |
+
"""Resolve current user-table fields without reading source records."""
|
| 1880 |
+
own = workspace.get('fields') or {}
|
| 1881 |
+
if view_settings_rows.supports(self.table_key):
|
| 1882 |
+
from core import user_tables
|
| 1883 |
+
definition = user_tables.get_definition(self._shared_stratum_key(), st=self._st) or {}
|
| 1884 |
+
return ({field['key'] for field in definition.get('fields', [])}
|
| 1885 |
+
| {key for key in own if key.startswith(('custom_', 'measure_'))}
|
| 1886 |
+
| self._tenant_wide_keys())
|
| 1887 |
+
# The event adapter checks built-in fields against its scoped field contract.
|
| 1888 |
+
return None
|
| 1889 |
+
|
| 1890 |
+
def apply_view_ops(self, owner, view_id, ops, *, actor, origin,
|
| 1891 |
+
assigned_recipient=None, admit=None, seed_view=None):
|
| 1892 |
+
"""Apply one cleaned, authorized batch to the locked View and commit receipts with its member changes."""
|
| 1893 |
+
from core import edit_receipts
|
| 1894 |
+
if owner in RESERVED_MEMBERS:
|
| 1895 |
+
raise ValueError('reserved username')
|
| 1896 |
+
view_id, actor = str(view_id), str(actor)
|
| 1897 |
+
recipient = (str(assigned_recipient).strip().lower()
|
| 1898 |
+
if assigned_recipient is not None else None)
|
| 1899 |
+
if recipient is not None and (not recipient or recipient != actor.strip().lower()
|
| 1900 |
+
or recipient != str(owner).strip().lower()):
|
| 1901 |
+
raise ValueError('assigned presentation belongs to its recipient')
|
| 1902 |
+
if seed_view is not None and (recipient is not None or owner != actor
|
| 1903 |
+
or seed_view.get('id') != view_id
|
| 1904 |
+
or seed_view.get('kind') != 'system'):
|
| 1905 |
+
raise ValueError('only a personal system View can be seeded')
|
| 1906 |
+
row_owner = ASSIGNED_KEY if recipient is not None else owner
|
| 1907 |
+
requested = copy.deepcopy(list(ops))
|
| 1908 |
+
action = edit_receipts.operation(self.table_key, 'view_op', view_id)
|
| 1909 |
+
results, activity = [], []
|
| 1910 |
+
|
| 1911 |
+
def apply(data, audit):
|
| 1912 |
+
results.clear()
|
| 1913 |
+
activity.clear()
|
| 1914 |
+
ws = data.setdefault(row_owner, {})
|
| 1915 |
+
assignment = (((ws.get('views') or {}).get(recipient) or {}).get(view_id)
|
| 1916 |
+
if recipient is not None else None)
|
| 1917 |
+
receipts = view_settings_rows.ViewOpReceipts(ws, getattr(self._st, 'key', self.table_key))
|
| 1918 |
+
target_fields = {}
|
| 1919 |
+
for operation in requested:
|
| 1920 |
+
member = _view_member_identity(operation.get('member'))
|
| 1921 |
+
identity = operation.get('client_op_id')
|
| 1922 |
+
base = {'opId': identity, 'member': member, 'applied': False}
|
| 1923 |
+
|
| 1924 |
+
def refuse(reason, message, **extra):
|
| 1925 |
+
return {**base, 'refused': {'reason': reason, 'member': member,
|
| 1926 |
+
'message': message, **extra}}
|
| 1927 |
+
|
| 1928 |
+
def land():
|
| 1929 |
+
view = (ws.get('views') or {}).get(view_id)
|
| 1930 |
+
if (recipient is not None and isinstance(assignment, dict)
|
| 1931 |
+
and isinstance(assignment.get('filter'), dict)):
|
| 1932 |
+
view = {'id': view_id, 'kind': 'assigned', 'name': assignment.get('name'),
|
| 1933 |
+
'config': {**copy.deepcopy(assignment.get('presentation') or {}),
|
| 1934 |
+
**copy.deepcopy(assignment['filter'])}}
|
| 1935 |
+
if view is None and seed_view is not None:
|
| 1936 |
+
view = copy.deepcopy(seed_view)
|
| 1937 |
+
if not isinstance(view, dict) or view_id in (ws.get(TRASH_KEY) or {}):
|
| 1938 |
+
return refuse('view_gone', 'That View is no longer available.')
|
| 1939 |
+
if operation.get('view_id', view_id) != view_id:
|
| 1940 |
+
return refuse('view_gone', 'That View is no longer available.')
|
| 1941 |
+
if not isinstance(member, str):
|
| 1942 |
+
return refuse('unknown_member', 'Choose a supported View setting.')
|
| 1943 |
+
family, separator, key = member.partition(':')
|
| 1944 |
+
if recipient is not None:
|
| 1945 |
+
if family in ('filter', 'filterConj', 'memberPid'):
|
| 1946 |
+
return refuse('assigned_filter_fixed',
|
| 1947 |
+
'The assigned View fixes its filters and record membership.')
|
| 1948 |
+
if family in _VIEW_OP_TOP:
|
| 1949 |
+
return refuse('unknown_member', 'Choose a presentation setting for this assigned View.')
|
| 1950 |
+
if (family not in _VIEW_OP_TOP | _VIEW_OP_CONFIG | _VIEW_OP_FAMILIES
|
| 1951 |
+
or (family in _VIEW_OP_FAMILIES and not key)
|
| 1952 |
+
or (family == 'filterConj' and separator and not key)
|
| 1953 |
+
or (separator and family in _VIEW_OP_TOP | _VIEW_OP_CONFIG
|
| 1954 |
+
and family != 'filterConj')):
|
| 1955 |
+
return refuse('unknown_member', 'Choose a supported View setting.')
|
| 1956 |
+
kind, value = operation.get('op'), operation.get('value')
|
| 1957 |
+
if kind not in ('set', 'remove', 'undo'):
|
| 1958 |
+
return refuse('invalid_op', 'Choose set, remove or undo for this View setting.')
|
| 1959 |
+
if 'keys' not in target_fields:
|
| 1960 |
+
field_ws = (self._read_owners(owner).get(owner) or {}
|
| 1961 |
+
if recipient is not None else ws)
|
| 1962 |
+
target_fields['keys'] = self._view_op_fields(field_ws)
|
| 1963 |
+
fields = target_fields['keys']
|
| 1964 |
+
target = key if family in ('hidden', 'sort', 'width', 'position', 'summary') else None
|
| 1965 |
+
if target is not None and fields is not None and target not in fields:
|
| 1966 |
+
return refuse('target_gone', 'That field is no longer available in this table.')
|
| 1967 |
+
candidate = copy.deepcopy(view)
|
| 1968 |
+
cfg = candidate.setdefault('config', {})
|
| 1969 |
+
if family == 'position' and 'positions' not in cfg:
|
| 1970 |
+
cfg['positions'] = _view_position_seed(cfg.get('order') or [])
|
| 1971 |
+
if family == 'sort':
|
| 1972 |
+
rules = cfg.get('sorts') or []
|
| 1973 |
+
seeds = _view_position_seed(rule['colId'] for rule in rules)
|
| 1974 |
+
cfg['sorts'] = [{**rule, 'pos': rule.get('pos', seeds[rule['colId']])}
|
| 1975 |
+
for rule in rules]
|
| 1976 |
+
before_present, before = _view_member_value(candidate, member)
|
| 1977 |
+
writers = (ws.get(_VIEW_OP_MEMBERS) or {}).get(view_id) or {}
|
| 1978 |
+
matching = ([writer for name, writer in writers.items()
|
| 1979 |
+
if _view_member_identity(name) == member] if family == 'memberPid' else
|
| 1980 |
+
[writers[member]] if member in writers else [])
|
| 1981 |
+
previous = writers.get(member) or next(iter(matching), None)
|
| 1982 |
+
present = kind != 'remove'
|
| 1983 |
+
if kind == 'undo':
|
| 1984 |
+
original = receipts.result(actor, action, value)
|
| 1985 |
+
if (not original or not original.get('applied')
|
| 1986 |
+
or _view_member_identity(original.get('member')) != member):
|
| 1987 |
+
return refuse('changed_since', 'That change is not available to undo.', by=actor)
|
| 1988 |
+
if ((before_present, before) != (original['afterPresent'], original['after'])
|
| 1989 |
+
or any(writer.get('client_op_id') != value or writer.get('by') != actor
|
| 1990 |
+
for writer in matching)):
|
| 1991 |
+
by = previous.get('by') if previous else 'another editor'
|
| 1992 |
+
return refuse('changed_since', f'{by} changed this View setting after your edit.', by=by)
|
| 1993 |
+
present, value = original['beforePresent'], original['before']
|
| 1994 |
+
elif kind == 'remove' and family not in ('sort', 'width', 'filter', 'summary', 'memberPid'):
|
| 1995 |
+
return refuse('invalid_op', 'Set a value for this View setting.')
|
| 1996 |
+
if (family in ('group', 'colorBy') and present and value is not None
|
| 1997 |
+
and fields is not None and value not in fields):
|
| 1998 |
+
return refuse('target_gone', 'That field is no longer available in this table.')
|
| 1999 |
+
if family == 'filter' and present:
|
| 2000 |
+
node = value.get('node') if isinstance(value, dict) else None
|
| 2001 |
+
if not isinstance(node, dict):
|
| 2002 |
+
return refuse('invalid_filter', 'Choose a valid filter condition and parent group.')
|
| 2003 |
+
columns = [item.get('colId') for item, _, _ in _view_filter_nodes([node])
|
| 2004 |
+
if item.get('colId')]
|
| 2005 |
+
if fields is not None and any(
|
| 2006 |
+
column not in fields and not column.startswith(('__', 'measure_'))
|
| 2007 |
+
for column in columns):
|
| 2008 |
+
return refuse('target_gone', 'That field is no longer available in this table.')
|
| 2009 |
+
if kind == 'set' and family in ('sort', 'position'):
|
| 2010 |
+
position = value.get('pos') if isinstance(value, dict) else value
|
| 2011 |
+
if not isinstance(position, str) or not _VIEW_OP_POSITION.fullmatch(position):
|
| 2012 |
+
return refuse('invalid_position', 'Use 1 to 64 letters or digits for the position.')
|
| 2013 |
+
if family == 'sort' and (not isinstance(value, dict) or value.get('dir') not in ('asc', 'desc')):
|
| 2014 |
+
return refuse('invalid_sort', 'Choose a sort direction and a valid position.')
|
| 2015 |
+
try:
|
| 2016 |
+
_set_view_member(candidate, member, value, present=present)
|
| 2017 |
+
except _ViewOpRefused as exc:
|
| 2018 |
+
return refuse(exc.reason, str(exc))
|
| 2019 |
+
if admit is not None:
|
| 2020 |
+
denied = admit(view, candidate, member)
|
| 2021 |
+
if denied:
|
| 2022 |
+
return refuse(denied['reason'], denied['message'])
|
| 2023 |
+
if family == 'name':
|
| 2024 |
+
names = ([name for who, vid, name in data.view_index() if vid != view_id]
|
| 2025 |
+
if isinstance(data, view_settings_rows.ScopedDocument) else
|
| 2026 |
+
[other.get('name') for who, workspace in data.items()
|
| 2027 |
+
if who not in RESERVED_MEMBERS and isinstance(workspace, dict)
|
| 2028 |
+
for vid, other in (workspace.get('views') or {}).items()
|
| 2029 |
+
if vid != view_id and isinstance(other, dict)])
|
| 2030 |
+
candidate['name'] = _unique_name(candidate.get('name'), names)
|
| 2031 |
+
after_present, after = _view_member_value(candidate, member)
|
| 2032 |
+
candidate['updatedAt'] = write_execution.utcnow().isoformat()
|
| 2033 |
+
if recipient is not None:
|
| 2034 |
+
assignment['presentation'] = {
|
| 2035 |
+
name: value for name, value in candidate['config'].items()
|
| 2036 |
+
if name not in ('filters', 'filterConj', 'memberPids')}
|
| 2037 |
+
assignment['updatedAt'] = candidate['updatedAt']
|
| 2038 |
+
else:
|
| 2039 |
+
ws.setdefault('views', {})[view_id] = candidate
|
| 2040 |
+
result = {**base, 'applied': True, 'before': before, 'after': after,
|
| 2041 |
+
'beforePresent': before_present, 'afterPresent': after_present,
|
| 2042 |
+
'by': actor, 'overwrote': previous.get('by') if previous else None}
|
| 2043 |
+
writers = ws.setdefault(_VIEW_OP_MEMBERS, {}).setdefault(view_id, {})
|
| 2044 |
+
if family == 'memberPid':
|
| 2045 |
+
for name in tuple(writers):
|
| 2046 |
+
if name != member and _view_member_identity(name) == member:
|
| 2047 |
+
writers.pop(name)
|
| 2048 |
+
writers[member] = {'by': actor, 'client_op_id': identity}
|
| 2049 |
+
audit(action='rename' if family == 'name' and before != after else None,
|
| 2050 |
+
owner=owner, view_id=view_id, actor=actor, origin=origin,
|
| 2051 |
+
before=view_audit.summary(view), after=view_audit.summary(candidate))
|
| 2052 |
+
activity.append(copy.deepcopy(result))
|
| 2053 |
+
return result
|
| 2054 |
+
|
| 2055 |
+
try:
|
| 2056 |
+
edit_receipts.op_id(identity)
|
| 2057 |
+
except ValueError:
|
| 2058 |
+
results.append(refuse('invalid_op_id', 'Give this View operation a client operation id.'))
|
| 2059 |
+
continue
|
| 2060 |
+
results.append(edit_receipts.run_once(receipts, actor, action, identity, land))
|
| 2061 |
+
return data
|
| 2062 |
+
|
| 2063 |
+
stored = self._audited_update(apply, flush='sync', owners=(row_owner,))
|
| 2064 |
+
if stored is None or stored is False:
|
| 2065 |
+
raise edit_receipts.ReceiptUnavailable('The View operation was not saved. Retry this client operation id.')
|
| 2066 |
+
for result in activity:
|
| 2067 |
+
self.log_view_activity(view_id, 'changed', by=actor, member=result['member'],
|
| 2068 |
+
before=result['before'], after=result['after'],
|
| 2069 |
+
overwrote=result['overwrote'], via=origin)
|
| 2070 |
+
return copy.deepcopy(results)
|
| 2071 |
+
|
| 2072 |
def save_view(self, username, view, reserved_names=(), *,
|
| 2073 |
+
actor=None, origin=None, action=None):
|
| 2074 |
"""Upsert a SavedView into its creator's personal workspace.
|
| 2075 |
|
| 2076 |
Sharing is exclusively a grant-registry concern; this method never chooses a second home
|
|
|
|
| 2092 |
`view_upsert` with a fresh id — `grid_events` L2408 says so in as many words), and
|
| 2093 |
`restore` belongs to W50-T26's trash door. Both are honest only when the caller names
|
| 2094 |
them; nothing is guessed.
|
|
|
|
|
|
|
|
|
|
| 2095 |
"""
|
| 2096 |
view_id = str((view or {}).get('id') or '').strip()
|
| 2097 |
if not view_id:
|
|
|
|
| 2108 |
raise ValueError('Saved Views use modern grants; view-level permissions are retired')
|
| 2109 |
accepted = {}
|
| 2110 |
activity_before = {}
|
|
|
|
| 2111 |
|
| 2112 |
def _up(data, audit):
|
| 2113 |
# View names are tenant-global across personal source records. A grant is only an
|
|
|
|
| 2135 |
prior = (ws.get('views') or {}).get(view_id)
|
| 2136 |
activity_before['value'] = view_audit.summary(prior)
|
| 2137 |
ws.setdefault('views', {})[view_id] = payload
|
|
|
|
|
|
|
| 2138 |
audit(action=action or view_audit.derive_action(prior, payload),
|
| 2139 |
owner=username, view_id=view_id, actor=actor, origin=origin,
|
| 2140 |
before=view_audit.summary(prior), after=view_audit.summary(payload))
|
| 2141 |
return data
|
| 2142 |
|
| 2143 |
+
self._audited_update(_up, flush='async', owners=(username,))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2144 |
self.log_view_activity(view_id, 'saved', by=actor or username,
|
| 2145 |
before=activity_before.get('value'),
|
| 2146 |
after=view_audit.summary(accepted), via=origin)
|
| 2147 |
return dict(accepted)
|
| 2148 |
|
| 2149 |
+
def delete_view(self, username, view_id, *, actor=None, origin=None):
|
| 2150 |
"""Delete an owner source View and its dependent assignments, never another user's.
|
| 2151 |
|
| 2152 |
⭐⭐ W50-T25 — C3's `delete` action, and `actor` is a SEPARATE parameter from `username`
|
|
|
|
| 2176 |
# a caller that named no actor must not produce a trash entry blaming nobody.
|
| 2177 |
who = str(actor or owner or '') or None
|
| 2178 |
removed = {}
|
|
|
|
| 2179 |
|
| 2180 |
def _up(data, audit):
|
| 2181 |
views = data.get(owner, {}).get('views') or {}
|
|
|
|
| 2183 |
removed.clear()
|
| 2184 |
if isinstance(before, dict):
|
| 2185 |
removed['value'] = view_audit.summary(before)
|
|
|
|
|
|
|
| 2186 |
views.pop(vid, None)
|
| 2187 |
audit(action='delete' if isinstance(before, dict) else None,
|
| 2188 |
owner=owner, view_id=vid, actor=actor, origin=origin,
|
|
|
|
| 2222 |
purge_trash(bucket, now=stamp)
|
| 2223 |
return data
|
| 2224 |
|
| 2225 |
+
self._audited_update(_up, flush='async', owners=(owner, ASSIGNED_KEY))
|
| 2226 |
if removed:
|
| 2227 |
self.log_view_activity(vid, 'deleted', by=who, before=removed['value'], via=origin)
|
| 2228 |
if row_backed:
|
|
|
|
| 2315 |
return False
|
| 2316 |
return True
|
| 2317 |
|
| 2318 |
+
def purge_view(self, username, view_id, *, actor=None, origin=None):
|
| 2319 |
"""R11's "Delete forever": destroy one trashed view NOW. `(name, adopted_cell_count)`.
|
| 2320 |
|
| 2321 |
⛔ THE ADOPTION RUNS FIRST AND A FAILURE REFUSES THE PURGE. Unlike the retention sweep on
|
|
|
|
| 2327 |
inventing an eighth here would widen a vocabulary two gates pin. The `delete` row written
|
| 2328 |
when the view entered the trash is still the record of its removal. Booked as a finding.
|
| 2329 |
|
| 2330 |
+
The locked update keeps trash removal atomic. `action=None` requests no audit row.
|
|
|
|
| 2331 |
"""
|
| 2332 |
vid = str(view_id or '').strip()
|
| 2333 |
owner = str(username or '').strip().lower()
|
|
|
|
| 2369 |
audit(action=None, owner=owner, view_id=vid)
|
| 2370 |
return current
|
| 2371 |
|
| 2372 |
+
self._audited_update(_up, flush='async', owners=(owner,))
|
| 2373 |
if removed:
|
| 2374 |
self.log_view_activity(vid, 'purged', by=actor or owner,
|
| 2375 |
before=removed['value'], via=origin)
|
|
|
|
| 2480 |
return str(who), dict(entry, id=vid)
|
| 2481 |
return None
|
| 2482 |
|
| 2483 |
+
def restore_view(self, username, view_id, *, actor=None, origin=None):
|
| 2484 |
"""Move one trashed view back into `views`, with every assignment the delete cascaded.
|
| 2485 |
|
| 2486 |
⭐⭐ R9's *"one-click Restore"*, and it is an AUDITED action: `restore` is one of C3's
|
|
|
|
| 2555 |
before=None, after=view_audit.summary(payload))
|
| 2556 |
return data
|
| 2557 |
|
| 2558 |
+
self._audited_update(_up, flush='async', owners=(owner, ASSIGNED_KEY))
|
| 2559 |
self.log_view_activity(vid, 'restored', by=actor or owner,
|
| 2560 |
after=view_audit.summary(restored), via=origin)
|
| 2561 |
return dict(restored)
|
|
|
|
| 2704 |
Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
|
| 2705 |
columns — the caller enforces the prefix). The stored overlay VALUES for the key are
|
| 2706 |
scrubbed with it: a deleted column's cells must not linger as orphan data that would
|
| 2707 |
+
silently resurface if the key were ever reused. This user's Views lose the field's
|
| 2708 |
+
members in this transaction. The workspace reader cleans other users' projections.
|
| 2709 |
|
| 2710 |
⭐⭐ W36-T25 — AND THE TENANT-WIDE SUMMARY GOES WITH IT, for exactly the reason the
|
| 2711 |
paragraph above gives about cells: an orphan `agg` under a deleted key is state nobody can
|
|
|
|
| 2723 |
for row in ws['overlays'].values():
|
| 2724 |
if isinstance(row, dict):
|
| 2725 |
row.pop(key, None)
|
| 2726 |
+
for view in ws['views'].values():
|
| 2727 |
+
prune_view_field_members(view, removed_keys=(key,))
|
| 2728 |
|
| 2729 |
self._update(username, _drop, shared=lambda shared_fields: shared_fields.pop(key, None))
|
| 2730 |
|
|
|
|
| 2865 |
entry['options'] = merge_choice_options(entry.get('options'), values)
|
| 2866 |
fields[key] = entry
|
| 2867 |
|
| 2868 |
+
committed = self._update(username, _patch)
|
| 2869 |
+
readback = getattr(self, 'overlay_readback', None)
|
| 2870 |
+
if isinstance(readback, dict):
|
| 2871 |
+
from copy import deepcopy
|
| 2872 |
+
readback[str(int(pid))] = deepcopy(
|
| 2873 |
+
(((committed or {}).get(username) or {}).get('overlays') or {}).get(str(int(pid))) or {})
|
| 2874 |
|
| 2875 |
|
| 2876 |
def make(table_key, st=None):
|
platform/core/user_tables.py
CHANGED
|
@@ -50,6 +50,7 @@ import inspect
|
|
| 50 |
import json as _json
|
| 51 |
import math
|
| 52 |
import re
|
|
|
|
| 53 |
import uuid as _uuid
|
| 54 |
from copy import deepcopy
|
| 55 |
from contextlib import contextmanager
|
|
@@ -315,6 +316,8 @@ def counts_toward_table_cap(table_key, definition=None):
|
|
| 315 |
|
| 316 |
#: A module's own databases (Command Center's four) are not what people create: no cap slot either.
|
| 317 |
_MODULE_OWNED = set()
|
|
|
|
|
|
|
| 318 |
|
| 319 |
|
| 320 |
def register_module_tables(*keys):
|
|
@@ -438,6 +441,7 @@ def strip_materialised(st=None):
|
|
| 438 |
return doc
|
| 439 |
|
| 440 |
s.update(STORE_KEY, _strip, flush='sync')
|
|
|
|
| 441 |
try:
|
| 442 |
out['after'] = len(_json.dumps(s.get(STORE_KEY) or {}, default=str))
|
| 443 |
except Exception: # noqa: BLE001
|
|
@@ -740,6 +744,7 @@ def record_connector_health(table_key, state, *, st=None, last_success=None,
|
|
| 740 |
return document
|
| 741 |
|
| 742 |
landed = _st(st).update(STORE_KEY, _set, flush='sync')
|
|
|
|
| 743 |
return dict((landed.get(key) or {}).get(CONNECTOR_HEALTH_FIELD) or receipt)
|
| 744 |
|
| 745 |
|
|
@@ -829,8 +834,7 @@ def purge_credential_tables(entry_type, connection_id, rt=None, st=None):
|
|
| 829 |
return out
|
| 830 |
|
| 831 |
|
| 832 |
-
|
| 833 |
-
MAX_FIELDS = 100
|
| 834 |
#: the badge the nav paints for a table with no connector behind it
|
| 835 |
BLANK_SOURCE = 'Blank'
|
| 836 |
#: automation-created tables carry their maker instead (wave 18 C4-AUTO)
|
|
@@ -930,11 +934,26 @@ def records_mutable(table_key, st=None):
|
|
| 930 |
this human-door predicate.
|
| 931 |
"""
|
| 932 |
table = get_definition(table_key, st, fallback=get_one) or {}
|
| 933 |
-
|
| 934 |
-
|
| 935 |
-
|
| 936 |
-
|
| 937 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 938 |
|
| 939 |
#: The field types a user table may declare. ⚠ Kept a SUBSET of
|
| 940 |
#: `aios_grid.CUSTOM_FIELD_TYPES` (gated in verify_api's W18-UT section). `created_time` stays
|
|
@@ -1327,15 +1346,52 @@ def _log_field_list_change(target, table_key, by, via, change):
|
|
| 1327 |
f'removed={removed} added={added} rev={rev} updated_at={stamp}', flush=True)
|
| 1328 |
|
| 1329 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1330 |
def update_with_activity(table_key, mutate, *, st=None, by=None, via=None,
|
| 1331 |
row_ids=(), field_keys=(), flush='sync', audit_restore_ids=(), **update_kwargs):
|
| 1332 |
"""Commit a table mutation and log each changed selected record or field once."""
|
| 1333 |
# None selects all identities. Empty tuples select none. Guards stay inside mutate/update.
|
| 1334 |
target = _st(st)
|
|
|
|
| 1335 |
original_mutate = mutate
|
| 1336 |
row_ids = None if row_ids is None else tuple(str(r) for r in row_ids)
|
| 1337 |
# D-118: the field keys this write removed or added, from the final store attempt only.
|
| 1338 |
field_list_change = {}
|
|
|
|
|
|
|
| 1339 |
|
| 1340 |
def public_result(result):
|
| 1341 |
if (isinstance(result, dict) and isinstance(result.get('ut_activity'), dict)
|
|
@@ -1368,6 +1424,7 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None,
|
|
| 1368 |
selected = {str(r): selected[str(r)] for r in row_ids if str(r) in selected}
|
| 1369 |
before_rows = deepcopy(selected)
|
| 1370 |
prior_field_keys = _field_list_keys(prior_table)
|
|
|
|
| 1371 |
outbox = ((current or {}).get('ut_activity') or {}).get('_activity_outbox')
|
| 1372 |
# A record-mode outbox writes per entry; a replaced document escalates to table mode.
|
| 1373 |
queued = {} if getattr(outbox, 'lazy_records', False) else deepcopy(outbox or {})
|
|
@@ -1390,13 +1447,17 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None,
|
|
| 1390 |
field_list_change.update(
|
| 1391 |
removed=[k for k in prior_field_keys if k not in after_field_keys],
|
| 1392 |
added=[k for k in after_field_keys if k not in prior_field_keys])
|
|
|
|
|
|
|
| 1393 |
return document
|
| 1394 |
|
| 1395 |
append = _activity_writer(target, by)
|
| 1396 |
if append is None:
|
| 1397 |
result = target.update(STORE_KEY, mutate, flush=flush, **update_kwargs)
|
| 1398 |
if result is not False:
|
|
|
|
| 1399 |
_log_field_list_change(target, table_key, by, via, field_list_change)
|
|
|
|
| 1400 |
return public_result(result)
|
| 1401 |
row_ids = None if row_ids is None else tuple(str(r) for r in row_ids)
|
| 1402 |
field_keys = None if field_keys is None else frozenset(str(k) for k in field_keys)
|
|
@@ -1468,7 +1529,9 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None,
|
|
| 1468 |
|
| 1469 |
result = target.update(STORE_KEY, apply, flush=flush, **update_kwargs)
|
| 1470 |
if result is not False:
|
|
|
|
| 1471 |
_log_field_list_change(target, table_key, by, via, field_list_change)
|
|
|
|
| 1472 |
from core import store_pg
|
| 1473 |
enlisted = store_pg._CALLER_TRANSACTION.get() is not None
|
| 1474 |
if batch is not None and not enlisted:
|
|
@@ -1478,6 +1541,100 @@ def update_with_activity(table_key, mutate, *, st=None, by=None, via=None,
|
|
| 1478 |
return public_result(result)
|
| 1479 |
|
| 1480 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1481 |
#: Wave 22 (contract C3) — the ROW-EVENT seam. The API layer APPENDS listeners (the automation
|
| 1482 |
#: engine's trigger hook, registered by `routes_automation` — the one module that imports both
|
| 1483 |
#: sides); platform code only ever EMITS. This is what keeps the Notion loop-prevention law
|
|
@@ -1899,13 +2056,17 @@ def allocate_autonumbers(table, values):
|
|
| 1899 |
return values
|
| 1900 |
|
| 1901 |
|
| 1902 |
-
def clean_fields(raw, recurrence_validator=None):
|
| 1903 |
"""Validate a caller-supplied field list into the stored shape — refuse junk, never store
|
| 1904 |
it. Returns a list of `{key,label,type,source}` dicts (source is always 'overlay': every
|
| 1905 |
base column of a user table is user-editable by construction), or None when nothing
|
| 1906 |
survives. Unknown types are DROPPED, not coerced — a column silently retyped is a lie."""
|
| 1907 |
-
|
| 1908 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1909 |
if not isinstance(f, dict):
|
| 1910 |
continue
|
| 1911 |
label = ' '.join(str(f.get('label') or f.get('key') or '').split())[:80]
|
|
@@ -1914,6 +2075,9 @@ def clean_fields(raw, recurrence_validator=None):
|
|
| 1914 |
ftype = str(f.get('type') or 'text').strip().lower()
|
| 1915 |
if not key or key in seen or ftype not in UT_FIELD_TYPES or not label:
|
| 1916 |
continue
|
|
|
|
|
|
|
|
|
|
| 1917 |
seen.add(key)
|
| 1918 |
origin = str(f.get('origin') or 'user')
|
| 1919 |
authority = str(f.get('valueAuthority') or 'user-entered')
|
|
@@ -2072,6 +2236,8 @@ def clean_fields(raw, recurrence_validator=None):
|
|
| 2072 |
continue
|
| 2073 |
entry['defaultValue'] = f['defaultValue']
|
| 2074 |
out.append(entry)
|
|
|
|
|
|
|
| 2075 |
return out or None
|
| 2076 |
|
| 2077 |
|
|
@@ -2509,13 +2675,27 @@ def is_managed_system_table(table_key, st=None, definition=None):
|
|
| 2509 |
else (get_one(table_key, st) or {})))
|
| 2510 |
|
| 2511 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2512 |
def create(label, username, fields=None, source=None, st=None, *, managed_by=None, capability=None,
|
| 2513 |
-
recurrence_validator=None):
|
| 2514 |
"""Create a blank table. Returns its key, or None if refused.
|
| 2515 |
|
| 2516 |
Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) —
|
| 2517 |
-
the caller is a UI form and a refusal is a message, not an exception.
|
|
|
|
| 2518 |
"""
|
|
|
|
|
|
|
|
|
|
| 2519 |
if managed_by is not None:
|
| 2520 |
from core import route_system
|
| 2521 |
if managed_by != route_system.MANAGED_BY or not route_system.is_internal_writer(capability):
|
|
@@ -2571,6 +2751,7 @@ def create(label, username, fields=None, source=None, st=None, *, managed_by=Non
|
|
| 2571 |
return cur
|
| 2572 |
|
| 2573 |
_st(st).update(STORE_KEY, _add, flush='sync') # a creation is not a hot path; commit it
|
|
|
|
| 2574 |
from core import shares
|
| 2575 |
shares.repair_published_database_grants(_st(st), table_keys=(key,))
|
| 2576 |
return key
|
|
@@ -2602,6 +2783,7 @@ def delete(table_key, st=None):
|
|
| 2602 |
cur.pop(key, None)
|
| 2603 |
return cur
|
| 2604 |
s.update(STORE_KEY, _drop, flush='sync')
|
|
|
|
| 2605 |
|
| 2606 |
# A zone is tenant data attached to this table's row-id namespace. Leaving it behind would
|
| 2607 |
# let a newly-created table with the same key inherit somebody else's geographic boundary.
|
|
@@ -2694,18 +2876,46 @@ def delete(table_key, st=None):
|
|
| 2694 |
pass
|
| 2695 |
|
| 2696 |
|
| 2697 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2698 |
"""Replace a table's base-field contract (wave 18 — the automation engine grows tables it
|
| 2699 |
-
creates). Cleaned with the same validator as `create`; refuses to leave a table fieldless.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2700 |
_validate_link_selection_fields(fields, st)
|
| 2701 |
-
clean = clean_fields(fields)
|
| 2702 |
if not clean or not is_user_table(table_key, st):
|
| 2703 |
return False
|
| 2704 |
|
| 2705 |
def _set(cur):
|
| 2706 |
t = cur.get(str(table_key))
|
| 2707 |
if t is not None:
|
| 2708 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2709 |
previous = {f['key']: f for f in t.get('fields') or []}
|
| 2710 |
for field in replacement:
|
| 2711 |
if field.get('type') == 'autonumber':
|
|
@@ -2927,6 +3137,7 @@ def set_label(table_key, label, st=None):
|
|
| 2927 |
return cur
|
| 2928 |
|
| 2929 |
_st(st).update(STORE_KEY, _set, flush='sync')
|
|
|
|
| 2930 |
return label
|
| 2931 |
|
| 2932 |
|
|
@@ -3081,7 +3292,7 @@ class LinkSelectionError(ValueError):
|
|
| 3081 |
def clean_link_selection_filter(raw, fields=None):
|
| 3082 |
"""Clean a complete filter-kit restriction without dropping any submitted condition."""
|
| 3083 |
import aios_grid
|
| 3084 |
-
from
|
| 3085 |
|
| 3086 |
if (not isinstance(raw, dict) or set(raw) - {'conj', 'nodes'}
|
| 3087 |
or raw.get('conj', 'and') not in ('and', 'or')
|
|
@@ -3147,7 +3358,7 @@ def clean_link_selection_filter(raw, fields=None):
|
|
| 3147 |
|
| 3148 |
def link_selection_permits(field, row, fields):
|
| 3149 |
"""Apply a Link restriction to an already permission-scoped target row and its real schema."""
|
| 3150 |
-
from
|
| 3151 |
bag = field.get('link') if isinstance(field, dict) else None
|
| 3152 |
if not isinstance(bag, dict) or 'selectionFilter' not in bag:
|
| 3153 |
return True
|
|
@@ -3157,7 +3368,7 @@ def link_selection_permits(field, row, fields):
|
|
| 3157 |
|
| 3158 |
def filter_link_selection(field, rows, fields):
|
| 3159 |
"""Narrow permission-scoped picker candidates using the Link's canonical restriction."""
|
| 3160 |
-
from
|
| 3161 |
bag = field.get('link') if isinstance(field, dict) else None
|
| 3162 |
if not isinstance(bag, dict) or 'selectionFilter' not in bag:
|
| 3163 |
return list(rows)
|
|
@@ -3180,7 +3391,7 @@ def _validate_link_selection_fields(fields, st=None):
|
|
| 3180 |
|
| 3181 |
def _validate_link_selection_values(fields, values, tables):
|
| 3182 |
"""Refuse restricted Link writes against the target rows in the same store snapshot."""
|
| 3183 |
-
from
|
| 3184 |
for field in fields or ():
|
| 3185 |
bag = field.get('link') if isinstance(field, dict) else None
|
| 3186 |
key = field.get('key')
|
|
@@ -4004,6 +4215,7 @@ def stamp_ai_enrich(table_key, col_id, marks, st=None):
|
|
| 4004 |
return cur
|
| 4005 |
|
| 4006 |
_st(st).update(STORE_KEY, _set, flush='async')
|
|
|
|
| 4007 |
return len(clean)
|
| 4008 |
|
| 4009 |
|
|
@@ -5046,7 +5258,8 @@ def add_field(table_key, raw, st=None, recurrence_validator=None, *, by=None, vi
|
|
| 5046 |
return None
|
| 5047 |
defn = get_definition(table_key, st) or {}
|
| 5048 |
have = [f for f in (defn.get('fields') or [])]
|
| 5049 |
-
|
|
|
|
| 5050 |
return None
|
| 5051 |
field = _clean_field(raw, recurrence_validator=recurrence_validator,
|
| 5052 |
valid_keys={str(f.get('key')) for f in have if f.get('key')})
|
|
@@ -5072,7 +5285,8 @@ def add_field(table_key, raw, st=None, recurrence_validator=None, *, by=None, vi
|
|
| 5072 |
if str(table_key) == 'ut_activity':
|
| 5073 |
t = cur['ut_activity'] = _activity_definition(t)
|
| 5074 |
if t is not None:
|
| 5075 |
-
|
|
|
|
| 5076 |
or any(f.get('key') == field['key'] for f in t.get('fields') or [])):
|
| 5077 |
return cur
|
| 5078 |
if field.get('type') == 'autonumber':
|
|
@@ -5202,7 +5416,8 @@ def sync_reciprocal_link(table_key, field_key, st=None):
|
|
| 5202 |
# link with no reciprocal is exactly what every link was before this function existed, and
|
| 5203 |
# it still resolves its own cell. `reciprocal` is simply not stamped on the bag, so the
|
| 5204 |
# next `sync` on a table with room creates it — this is a skip, not a tombstone.
|
| 5205 |
-
|
|
|
|
| 5206 |
# ⭐ W41-T15 — the skip the paragraph above describes, now REPORTED. It was already
|
| 5207 |
# deliberate; it was never legible from outside the function.
|
| 5208 |
_refuse('target_full',
|
|
@@ -5234,6 +5449,7 @@ def sync_reciprocal_link(table_key, field_key, st=None):
|
|
| 5234 |
return cur
|
| 5235 |
|
| 5236 |
_st(st).update(STORE_KEY, _sync, flush='sync')
|
|
|
|
| 5237 |
return result
|
| 5238 |
|
| 5239 |
|
|
@@ -5593,6 +5809,7 @@ def patch_cell_notes(table_key, row_id, notes, ctx, *, op_id, source_row_exists=
|
|
| 5593 |
result = edit_receipts.CheckedStore(ctx.st).update(STORE_KEY, merge, flush='sync')
|
| 5594 |
if not saved or result is None or result is False:
|
| 5595 |
raise edit_receipts.ReceiptUnavailable('The cell note was not saved. Retry this opId.')
|
|
|
|
| 5596 |
return {'pid': row_id, 'opId': identity, 'value': {CELL_NOTES_KEY: deepcopy(delta)}}
|
| 5597 |
|
| 5598 |
# Match the row PATCH receipt identity and envelope so its active transaction is reused.
|
|
|
|
| 50 |
import json as _json
|
| 51 |
import math
|
| 52 |
import re
|
| 53 |
+
import sys
|
| 54 |
import uuid as _uuid
|
| 55 |
from copy import deepcopy
|
| 56 |
from contextlib import contextmanager
|
|
|
|
| 316 |
|
| 317 |
#: A module's own databases (Command Center's four) are not what people create: no cap slot either.
|
| 318 |
_MODULE_OWNED = set()
|
| 319 |
+
# Only ensure_module_table stamps this stored declaration; public create builds its own metadata.
|
| 320 |
+
MODULE_OWNED_KEY = 'moduleOwned'
|
| 321 |
|
| 322 |
|
| 323 |
def register_module_tables(*keys):
|
|
|
|
| 441 |
return doc
|
| 442 |
|
| 443 |
s.update(STORE_KEY, _strip, flush='sync')
|
| 444 |
+
_log_store_write(s, '*', 'strip_materialised')
|
| 445 |
try:
|
| 446 |
out['after'] = len(_json.dumps(s.get(STORE_KEY) or {}, default=str))
|
| 447 |
except Exception: # noqa: BLE001
|
|
|
|
| 744 |
return document
|
| 745 |
|
| 746 |
landed = _st(st).update(STORE_KEY, _set, flush='sync')
|
| 747 |
+
_log_store_write(_st(st), key, 'record_connector_health')
|
| 748 |
return dict((landed.get(key) or {}).get(CONNECTOR_HEALTH_FIELD) or receipt)
|
| 749 |
|
| 750 |
|
|
|
|
| 834 |
return out
|
| 835 |
|
| 836 |
|
| 837 |
+
MAX_FIELDS = 60
|
|
|
|
| 838 |
#: the badge the nav paints for a table with no connector behind it
|
| 839 |
BLANK_SOURCE = 'Blank'
|
| 840 |
#: automation-created tables carry their maker instead (wave 18 C4-AUTO)
|
|
|
|
| 934 |
this human-door predicate.
|
| 935 |
"""
|
| 936 |
table = get_definition(table_key, st, fallback=get_one) or {}
|
| 937 |
+
return _records_mutable_definition(table_key, table)
|
| 938 |
+
|
| 939 |
+
|
| 940 |
+
def _records_mutable_definition(table_key, definition):
|
| 941 |
+
"""Evaluate the record lock from an already admitted stored definition."""
|
| 942 |
+
return bool(definition) and str(table_key) not in _LOCKED_RECORD_KEYS \
|
| 943 |
+
and definition.get('recordMode') != AUTOMATION_RECORD_MODE
|
| 944 |
+
|
| 945 |
+
|
| 946 |
+
def field_limit(table_key=None, st=None, *, definition=None):
|
| 947 |
+
"""Return the module or user field ceiling, or None for a records-locked connector."""
|
| 948 |
+
if definition is None:
|
| 949 |
+
definition = get_definition(table_key, st) if table_key else {}
|
| 950 |
+
module_owned = (str(table_key or (definition or {}).get('key')) in _MODULE_OWNED
|
| 951 |
+
or (definition or {}).get(MODULE_OWNED_KEY) is True)
|
| 952 |
+
if (definition and not module_owned
|
| 953 |
+
and not _records_mutable_definition(table_key, definition)):
|
| 954 |
+
return None
|
| 955 |
+
limit = (definition or {}).get('fieldLimit')
|
| 956 |
+
return limit if type(limit) is int and limit > 0 else MAX_FIELDS
|
| 957 |
|
| 958 |
#: The field types a user table may declare. ⚠ Kept a SUBSET of
|
| 959 |
#: `aios_grid.CUSTOM_FIELD_TYPES` (gated in verify_api's W18-UT section). `created_time` stays
|
|
|
|
| 1346 |
f'removed={removed} added={added} rev={rev} updated_at={stamp}', flush=True)
|
| 1347 |
|
| 1348 |
|
| 1349 |
+
def _store_write_state(target, name=STORE_KEY):
|
| 1350 |
+
"""D-118: `(dirty, owned, sha)` of one bucket in the store behind `target`, None where the backend keeps none.
|
| 1351 |
+
|
| 1352 |
+
Reads the three per-name maps without the store lock: `get()` holds that lock across a download.
|
| 1353 |
+
"""
|
| 1354 |
+
st, key = target, str(name)
|
| 1355 |
+
try:
|
| 1356 |
+
for _ in range(8):
|
| 1357 |
+
if isinstance(st, _Lent):
|
| 1358 |
+
st = st._runtime
|
| 1359 |
+
elif callable(getattr(type(st), 'store_key', None)) and callable(getattr(type(st), '_store', None)):
|
| 1360 |
+
key, st = st.store_key(key), st._store() # a TenantRuntime namespaces the key
|
| 1361 |
+
else:
|
| 1362 |
+
break
|
| 1363 |
+
if st is store:
|
| 1364 |
+
st = store._d()
|
| 1365 |
+
return tuple(m.get(key) if isinstance(m, dict) else None
|
| 1366 |
+
for m in (vars(st).get('_dirty'), vars(st).get('_owned'), vars(st).get('_base')))
|
| 1367 |
+
except Exception: # noqa: BLE001
|
| 1368 |
+
return None, None, None
|
| 1369 |
+
|
| 1370 |
+
|
| 1371 |
+
def _log_store_write(target, table_key, op):
|
| 1372 |
+
"""D-118: one stderr line per committed `user_tables` write: table, op, dirty, owned, commit sha.
|
| 1373 |
+
|
| 1374 |
+
It carries no cell value and never fails the write it describes.
|
| 1375 |
+
"""
|
| 1376 |
+
if _is_preview_store(target):
|
| 1377 |
+
return
|
| 1378 |
+
dirty, owned, sha = _store_write_state(target)
|
| 1379 |
+
print(f'[user-tables] D-118 write: table={table_key} op={op} dirty={dirty} owned={owned} '
|
| 1380 |
+
f'sha={sha}', file=sys.stderr, flush=True)
|
| 1381 |
+
|
| 1382 |
+
|
| 1383 |
def update_with_activity(table_key, mutate, *, st=None, by=None, via=None,
|
| 1384 |
row_ids=(), field_keys=(), flush='sync', audit_restore_ids=(), **update_kwargs):
|
| 1385 |
"""Commit a table mutation and log each changed selected record or field once."""
|
| 1386 |
# None selects all identities. Empty tuples select none. Guards stay inside mutate/update.
|
| 1387 |
target = _st(st)
|
| 1388 |
+
op = sys._getframe(1).f_code.co_name # the door, for the D-118 write line
|
| 1389 |
original_mutate = mutate
|
| 1390 |
row_ids = None if row_ids is None else tuple(str(r) for r in row_ids)
|
| 1391 |
# D-118: the field keys this write removed or added, from the final store attempt only.
|
| 1392 |
field_list_change = {}
|
| 1393 |
+
# D-179: flows newly tagged on this table, from the final attempt; the flow index learns them.
|
| 1394 |
+
new_flows = set()
|
| 1395 |
|
| 1396 |
def public_result(result):
|
| 1397 |
if (isinstance(result, dict) and isinstance(result.get('ut_activity'), dict)
|
|
|
|
| 1424 |
selected = {str(r): selected[str(r)] for r in row_ids if str(r) in selected}
|
| 1425 |
before_rows = deepcopy(selected)
|
| 1426 |
prior_field_keys = _field_list_keys(prior_table)
|
| 1427 |
+
prior_flows = _flow_tags(prior_table)
|
| 1428 |
outbox = ((current or {}).get('ut_activity') or {}).get('_activity_outbox')
|
| 1429 |
# A record-mode outbox writes per entry; a replaced document escalates to table mode.
|
| 1430 |
queued = {} if getattr(outbox, 'lazy_records', False) else deepcopy(outbox or {})
|
|
|
|
| 1447 |
field_list_change.update(
|
| 1448 |
removed=[k for k in prior_field_keys if k not in after_field_keys],
|
| 1449 |
added=[k for k in after_field_keys if k not in prior_field_keys])
|
| 1450 |
+
new_flows.clear()
|
| 1451 |
+
new_flows.update(_flow_tags(table if isinstance(table, dict) else {}) - prior_flows)
|
| 1452 |
return document
|
| 1453 |
|
| 1454 |
append = _activity_writer(target, by)
|
| 1455 |
if append is None:
|
| 1456 |
result = target.update(STORE_KEY, mutate, flush=flush, **update_kwargs)
|
| 1457 |
if result is not False:
|
| 1458 |
+
_log_store_write(target, table_key, op)
|
| 1459 |
_log_field_list_change(target, table_key, by, via, field_list_change)
|
| 1460 |
+
_index_new_flows(target, table_key, new_flows)
|
| 1461 |
return public_result(result)
|
| 1462 |
row_ids = None if row_ids is None else tuple(str(r) for r in row_ids)
|
| 1463 |
field_keys = None if field_keys is None else frozenset(str(k) for k in field_keys)
|
|
|
|
| 1529 |
|
| 1530 |
result = target.update(STORE_KEY, apply, flush=flush, **update_kwargs)
|
| 1531 |
if result is not False:
|
| 1532 |
+
_log_store_write(target, table_key, op)
|
| 1533 |
_log_field_list_change(target, table_key, by, via, field_list_change)
|
| 1534 |
+
_index_new_flows(target, table_key, new_flows)
|
| 1535 |
from core import store_pg
|
| 1536 |
enlisted = store_pg._CALLER_TRANSACTION.get() is not None
|
| 1537 |
if batch is not None and not enlisted:
|
|
|
|
| 1541 |
return public_result(result)
|
| 1542 |
|
| 1543 |
|
| 1544 |
+
#: D-179: `{flowId: [table_key, ...]}`, the tables each automation tagged. An entry is COMPLETE
|
| 1545 |
+
#: or ABSENT: only a new flow starts one (`create=True`), every tagging door extends it, and a
|
| 1546 |
+
#: flow with no entry (one tagged before the index existed) keeps the whole-scan path.
|
| 1547 |
+
FLOW_INDEX_KEY = 'user_tables_flow_index'
|
| 1548 |
+
|
| 1549 |
+
|
| 1550 |
+
def _flow_tags(table):
|
| 1551 |
+
"""The `automation.flowId` values on one table's fields."""
|
| 1552 |
+
return {str((f.get('automation') or {}).get('flowId') or '')
|
| 1553 |
+
for f in ((table or {}).get('fields') or [])
|
| 1554 |
+
if isinstance(f, dict) and isinstance(f.get('automation'), dict)} - {''}
|
| 1555 |
+
|
| 1556 |
+
|
| 1557 |
+
def _index_new_flows(target, table_key, flows):
|
| 1558 |
+
"""Extend each indexed flow in `flows` with `table_key`; a failure is logged, never raised."""
|
| 1559 |
+
for fid in sorted(flows or ()):
|
| 1560 |
+
try:
|
| 1561 |
+
index_flow_tables(fid, (table_key,), target)
|
| 1562 |
+
except Exception as exc: # noqa: BLE001
|
| 1563 |
+
print(f'[user-tables] D-179 flow index not extended: flow={fid} table={table_key} '
|
| 1564 |
+
f'({type(exc).__name__})', file=sys.stderr, flush=True)
|
| 1565 |
+
|
| 1566 |
+
|
| 1567 |
+
def flow_index_tables(flow_id, st=None):
|
| 1568 |
+
"""The table keys `flow_id` tagged, or None when the index has no entry for it."""
|
| 1569 |
+
try:
|
| 1570 |
+
index = _st(st).get(FLOW_INDEX_KEY) or {}
|
| 1571 |
+
except Exception: # noqa: BLE001
|
| 1572 |
+
return None
|
| 1573 |
+
entry = index.get(str(flow_id)) if isinstance(index, dict) else None
|
| 1574 |
+
return [str(k) for k in entry] if isinstance(entry, list) else None
|
| 1575 |
+
|
| 1576 |
+
|
| 1577 |
+
def index_flow_tables(flow_id, table_keys=(), st=None, *, create=False):
|
| 1578 |
+
"""Name `table_keys` under `flow_id`. Only `create` starts an entry: a new flow tagged nothing yet.
|
| 1579 |
+
|
| 1580 |
+
A failed extend DROPS the entry, so a flow is never served by an index that misses a table.
|
| 1581 |
+
The start is async: a lost start leaves no entry, which is the safe answer. An extend is sync.
|
| 1582 |
+
"""
|
| 1583 |
+
fid = str(flow_id or '').strip()
|
| 1584 |
+
keys = sorted({str(k) for k in (table_keys or ()) if str(k or '').strip()})
|
| 1585 |
+
if not fid:
|
| 1586 |
+
return
|
| 1587 |
+
if not create:
|
| 1588 |
+
have = flow_index_tables(fid, st)
|
| 1589 |
+
if have is None or set(keys) <= set(have):
|
| 1590 |
+
return
|
| 1591 |
+
|
| 1592 |
+
def _up(cur):
|
| 1593 |
+
cur = cur if isinstance(cur, dict) else {}
|
| 1594 |
+
entry = cur.get(fid)
|
| 1595 |
+
if create:
|
| 1596 |
+
cur[fid] = keys
|
| 1597 |
+
elif isinstance(entry, list):
|
| 1598 |
+
cur[fid] = sorted(set(entry) | set(keys))
|
| 1599 |
+
return cur
|
| 1600 |
+
try:
|
| 1601 |
+
_st(st).update(FLOW_INDEX_KEY, _up, flush='async' if create else 'sync')
|
| 1602 |
+
except Exception: # noqa: BLE001
|
| 1603 |
+
if create:
|
| 1604 |
+
raise
|
| 1605 |
+
_st(st).update(FLOW_INDEX_KEY, lambda cur: {k: v for k, v in (cur or {}).items()
|
| 1606 |
+
if k != fid}, flush='sync')
|
| 1607 |
+
|
| 1608 |
+
|
| 1609 |
+
def update_tables(table_keys, mutate, st=None, *, flush='sync'):
|
| 1610 |
+
"""Run `mutate(key, table)` on each named table in ONE write that declares those tables first.
|
| 1611 |
+
|
| 1612 |
+
On the row layout `lock_records` loads only the named definitions, so no whole document is read.
|
| 1613 |
+
A named table that no longer exists is skipped.
|
| 1614 |
+
"""
|
| 1615 |
+
keys = sorted({str(k) for k in (table_keys or ()) if str(k or '').strip()})
|
| 1616 |
+
target = _st(st)
|
| 1617 |
+
op = sys._getframe(1).f_code.co_name
|
| 1618 |
+
|
| 1619 |
+
def _apply(cur):
|
| 1620 |
+
cur = cur if isinstance(cur, dict) else {}
|
| 1621 |
+
lock = getattr(cur, 'lock_records', None)
|
| 1622 |
+
if callable(lock):
|
| 1623 |
+
lock({k: [] for k in keys})
|
| 1624 |
+
for k in keys:
|
| 1625 |
+
table = cur.get(k)
|
| 1626 |
+
if isinstance(table, dict):
|
| 1627 |
+
mutate(k, table)
|
| 1628 |
+
return cur
|
| 1629 |
+
if not keys:
|
| 1630 |
+
return None
|
| 1631 |
+
result = target.update(STORE_KEY, _apply, flush=flush)
|
| 1632 |
+
if result is not False:
|
| 1633 |
+
for k in keys:
|
| 1634 |
+
_log_store_write(target, k, op)
|
| 1635 |
+
return result
|
| 1636 |
+
|
| 1637 |
+
|
| 1638 |
#: Wave 22 (contract C3) — the ROW-EVENT seam. The API layer APPENDS listeners (the automation
|
| 1639 |
#: engine's trigger hook, registered by `routes_automation` — the one module that imports both
|
| 1640 |
#: sides); platform code only ever EMITS. This is what keeps the Notion loop-prevention law
|
|
|
|
| 2056 |
return values
|
| 2057 |
|
| 2058 |
|
| 2059 |
+
def clean_fields(raw, recurrence_validator=None, *, table_key=None, st=None, definition=None):
|
| 2060 |
"""Validate a caller-supplied field list into the stored shape — refuse junk, never store
|
| 2061 |
it. Returns a list of `{key,label,type,source}` dicts (source is always 'overlay': every
|
| 2062 |
base column of a user table is user-editable by construction), or None when nothing
|
| 2063 |
survives. Unknown types are DROPPED, not coerced — a column silently retyped is a lie."""
|
| 2064 |
+
if definition is None:
|
| 2065 |
+
definition = get_definition(table_key, st) if table_key else {}
|
| 2066 |
+
limit = field_limit(table_key, st, definition=definition)
|
| 2067 |
+
admitted = {f.get('key') for f in (definition or {}).get('fields') or [] if isinstance(f, dict)}
|
| 2068 |
+
out, seen, added = [], set(), 0
|
| 2069 |
+
for f in raw or []:
|
| 2070 |
if not isinstance(f, dict):
|
| 2071 |
continue
|
| 2072 |
label = ' '.join(str(f.get('label') or f.get('key') or '').split())[:80]
|
|
|
|
| 2075 |
ftype = str(f.get('type') or 'text').strip().lower()
|
| 2076 |
if not key or key in seen or ftype not in UT_FIELD_TYPES or not label:
|
| 2077 |
continue
|
| 2078 |
+
# A lower ceiling refuses new fields but never slices off an admitted definition.
|
| 2079 |
+
if limit is not None and key not in admitted and len(admitted) + added >= limit:
|
| 2080 |
+
continue
|
| 2081 |
seen.add(key)
|
| 2082 |
origin = str(f.get('origin') or 'user')
|
| 2083 |
authority = str(f.get('valueAuthority') or 'user-entered')
|
|
|
|
| 2236 |
continue
|
| 2237 |
entry['defaultValue'] = f['defaultValue']
|
| 2238 |
out.append(entry)
|
| 2239 |
+
if key not in admitted:
|
| 2240 |
+
added += 1
|
| 2241 |
return out or None
|
| 2242 |
|
| 2243 |
|
|
|
|
| 2675 |
else (get_one(table_key, st) or {})))
|
| 2676 |
|
| 2677 |
|
| 2678 |
+
class SubjectClaimRefused(ValueError):
|
| 2679 |
+
"""A create door refused a source `subject` claim (D-250)."""
|
| 2680 |
+
code = 'subject_claim_refused'
|
| 2681 |
+
|
| 2682 |
+
|
| 2683 |
+
#: The sentence a refused `subject` claim returns to the client.
|
| 2684 |
+
SUBJECT_CLAIM_REFUSAL = ('Only a connector creates a database for a source subject. '
|
| 2685 |
+
'A database made in Table claims no subject. Remove "subject" and try again.')
|
| 2686 |
+
|
| 2687 |
+
|
| 2688 |
def create(label, username, fields=None, source=None, st=None, *, managed_by=None, capability=None,
|
| 2689 |
+
recurrence_validator=None, subject=None):
|
| 2690 |
"""Create a blank table. Returns its key, or None if refused.
|
| 2691 |
|
| 2692 |
Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) —
|
| 2693 |
+
the caller is a UI form and a refusal is a message, not an exception. A `subject` claim
|
| 2694 |
+
raises `SubjectClaimRefused`: one subject, one database binds connector tables only.
|
| 2695 |
"""
|
| 2696 |
+
# D-250: only `odoo_relational._ensure_table_inplace` stamps `subject`, under its conflict check.
|
| 2697 |
+
if subject is not None or (isinstance(source, dict) and 'subject' in source):
|
| 2698 |
+
raise SubjectClaimRefused(SUBJECT_CLAIM_REFUSAL)
|
| 2699 |
if managed_by is not None:
|
| 2700 |
from core import route_system
|
| 2701 |
if managed_by != route_system.MANAGED_BY or not route_system.is_internal_writer(capability):
|
|
|
|
| 2751 |
return cur
|
| 2752 |
|
| 2753 |
_st(st).update(STORE_KEY, _add, flush='sync') # a creation is not a hot path; commit it
|
| 2754 |
+
_log_store_write(_st(st), key, 'create')
|
| 2755 |
from core import shares
|
| 2756 |
shares.repair_published_database_grants(_st(st), table_keys=(key,))
|
| 2757 |
return key
|
|
|
|
| 2783 |
cur.pop(key, None)
|
| 2784 |
return cur
|
| 2785 |
s.update(STORE_KEY, _drop, flush='sync')
|
| 2786 |
+
_log_store_write(s, table_key, 'delete')
|
| 2787 |
|
| 2788 |
# A zone is tenant data attached to this table's row-id namespace. Leaving it behind would
|
| 2789 |
# let a newly-created table with the same key inherit somebody else's geographic boundary.
|
|
|
|
| 2876 |
pass
|
| 2877 |
|
| 2878 |
|
| 2879 |
+
def _rebase_field_list(edited, base, locked):
|
| 2880 |
+
"""D-118: the locked field list with only the caller's edits of `base` applied, by key.
|
| 2881 |
+
|
| 2882 |
+
A key the caller left as it read it keeps the locked definition. A key absent from `locked`
|
| 2883 |
+
is written only when `base` never held it, so a field deleted after the read stays deleted.
|
| 2884 |
+
"""
|
| 2885 |
+
def keyed(fields):
|
| 2886 |
+
return {str(f['key']): f for f in fields or [] if isinstance(f, dict) and f.get('key')}
|
| 2887 |
+
|
| 2888 |
+
before, after = keyed(base), keyed(edited)
|
| 2889 |
+
out = []
|
| 2890 |
+
for field in locked or []:
|
| 2891 |
+
key = str(field.get('key'))
|
| 2892 |
+
if key in before and key not in after:
|
| 2893 |
+
continue # the caller removed it
|
| 2894 |
+
out.append(after[key] if key in after and after[key] != before.get(key) else field)
|
| 2895 |
+
held = {str(f.get('key')) for f in locked or []}
|
| 2896 |
+
return out + [f for key, f in after.items() if key not in before and key not in held]
|
| 2897 |
+
|
| 2898 |
+
|
| 2899 |
+
def set_fields(table_key, fields, st=None, *, by=None, via=None, base=None):
|
| 2900 |
"""Replace a table's base-field contract (wave 18 — the automation engine grows tables it
|
| 2901 |
+
creates). Cleaned with the same validator as `create`; refuses to leave a table fieldless.
|
| 2902 |
+
|
| 2903 |
+
`base` is the field list the caller read before it edited `fields`. With it, the write
|
| 2904 |
+
applies only the caller's changes to the locked list (`_rebase_field_list`).
|
| 2905 |
+
"""
|
| 2906 |
_validate_link_selection_fields(fields, st)
|
| 2907 |
+
clean = clean_fields(fields, table_key=table_key, st=st)
|
| 2908 |
if not clean or not is_user_table(table_key, st):
|
| 2909 |
return False
|
| 2910 |
|
| 2911 |
def _set(cur):
|
| 2912 |
t = cur.get(str(table_key))
|
| 2913 |
if t is not None:
|
| 2914 |
+
wanted = fields if base is None else _rebase_field_list(fields, base, t.get('fields'))
|
| 2915 |
+
# Re-evaluate against the locked definition, including fields admitted by a concurrent writer.
|
| 2916 |
+
replacement = clean_fields(wanted, table_key=table_key, definition=t)
|
| 2917 |
+
if not replacement:
|
| 2918 |
+
return cur
|
| 2919 |
previous = {f['key']: f for f in t.get('fields') or []}
|
| 2920 |
for field in replacement:
|
| 2921 |
if field.get('type') == 'autonumber':
|
|
|
|
| 3137 |
return cur
|
| 3138 |
|
| 3139 |
_st(st).update(STORE_KEY, _set, flush='sync')
|
| 3140 |
+
_log_store_write(_st(st), table_key, 'set_label')
|
| 3141 |
return label
|
| 3142 |
|
| 3143 |
|
|
|
|
| 3292 |
def clean_link_selection_filter(raw, fields=None):
|
| 3293 |
"""Clean a complete filter-kit restriction without dropping any submitted condition."""
|
| 3294 |
import aios_grid
|
| 3295 |
+
from core.filter_eval import is_rule_active
|
| 3296 |
|
| 3297 |
if (not isinstance(raw, dict) or set(raw) - {'conj', 'nodes'}
|
| 3298 |
or raw.get('conj', 'and') not in ('and', 'or')
|
|
|
|
| 3358 |
|
| 3359 |
def link_selection_permits(field, row, fields):
|
| 3360 |
"""Apply a Link restriction to an already permission-scoped target row and its real schema."""
|
| 3361 |
+
from core.filter_eval import permits
|
| 3362 |
bag = field.get('link') if isinstance(field, dict) else None
|
| 3363 |
if not isinstance(bag, dict) or 'selectionFilter' not in bag:
|
| 3364 |
return True
|
|
|
|
| 3368 |
|
| 3369 |
def filter_link_selection(field, rows, fields):
|
| 3370 |
"""Narrow permission-scoped picker candidates using the Link's canonical restriction."""
|
| 3371 |
+
from core.filter_eval import permits
|
| 3372 |
bag = field.get('link') if isinstance(field, dict) else None
|
| 3373 |
if not isinstance(bag, dict) or 'selectionFilter' not in bag:
|
| 3374 |
return list(rows)
|
|
|
|
| 3391 |
|
| 3392 |
def _validate_link_selection_values(fields, values, tables):
|
| 3393 |
"""Refuse restricted Link writes against the target rows in the same store snapshot."""
|
| 3394 |
+
from core.filter_eval import permits
|
| 3395 |
for field in fields or ():
|
| 3396 |
bag = field.get('link') if isinstance(field, dict) else None
|
| 3397 |
key = field.get('key')
|
|
|
|
| 4215 |
return cur
|
| 4216 |
|
| 4217 |
_st(st).update(STORE_KEY, _set, flush='async')
|
| 4218 |
+
_log_store_write(_st(st), table_key, 'stamp_ai_enrich')
|
| 4219 |
return len(clean)
|
| 4220 |
|
| 4221 |
|
|
|
|
| 5258 |
return None
|
| 5259 |
defn = get_definition(table_key, st) or {}
|
| 5260 |
have = [f for f in (defn.get('fields') or [])]
|
| 5261 |
+
limit = field_limit(table_key, st, definition=defn)
|
| 5262 |
+
if limit is not None and len(have) >= limit:
|
| 5263 |
return None
|
| 5264 |
field = _clean_field(raw, recurrence_validator=recurrence_validator,
|
| 5265 |
valid_keys={str(f.get('key')) for f in have if f.get('key')})
|
|
|
|
| 5285 |
if str(table_key) == 'ut_activity':
|
| 5286 |
t = cur['ut_activity'] = _activity_definition(t)
|
| 5287 |
if t is not None:
|
| 5288 |
+
limit = field_limit(table_key, definition=t)
|
| 5289 |
+
if ((limit is not None and len(t.get('fields') or []) >= limit)
|
| 5290 |
or any(f.get('key') == field['key'] for f in t.get('fields') or [])):
|
| 5291 |
return cur
|
| 5292 |
if field.get('type') == 'autonumber':
|
|
|
|
| 5416 |
# link with no reciprocal is exactly what every link was before this function existed, and
|
| 5417 |
# it still resolves its own cell. `reciprocal` is simply not stamped on the bag, so the
|
| 5418 |
# next `sync` on a table with room creates it — this is a skip, not a tombstone.
|
| 5419 |
+
limit = field_limit(target_key, definition=target)
|
| 5420 |
+
if limit is not None and len(target.get('fields') or []) >= limit:
|
| 5421 |
# ⭐ W41-T15 — the skip the paragraph above describes, now REPORTED. It was already
|
| 5422 |
# deliberate; it was never legible from outside the function.
|
| 5423 |
_refuse('target_full',
|
|
|
|
| 5449 |
return cur
|
| 5450 |
|
| 5451 |
_st(st).update(STORE_KEY, _sync, flush='sync')
|
| 5452 |
+
_log_store_write(_st(st), table_key, 'sync_reciprocal_link')
|
| 5453 |
return result
|
| 5454 |
|
| 5455 |
|
|
|
|
| 5809 |
result = edit_receipts.CheckedStore(ctx.st).update(STORE_KEY, merge, flush='sync')
|
| 5810 |
if not saved or result is None or result is False:
|
| 5811 |
raise edit_receipts.ReceiptUnavailable('The cell note was not saved. Retry this opId.')
|
| 5812 |
+
_log_store_write(ctx.st, table_key, 'patch_cell_notes')
|
| 5813 |
return {'pid': row_id, 'opId': identity, 'value': {CELL_NOTES_KEY: deepcopy(delta)}}
|
| 5814 |
|
| 5815 |
# Match the row PATCH receipt identity and envelope so its active transaction is reused.
|