Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- Dockerfile +9 -1
- RELEASES.json +1 -1
- VERSION +1 -1
- api/agent_runtime.py +14 -2
- api/amazon_relational.py +34 -9
- api/app_routes.py +58 -0
- api/app_rules.py +1 -2
- api/app_templates.py +195 -0
- api/application_registry.py +329 -109
- api/assistant_stream.py +19 -24
- api/catalog_store.py +33 -5
- api/connectors_amazon_ads.py +130 -0
- api/deps.py +57 -0
- api/error_sink.py +151 -0
- api/external_freshness.py +17 -4
- api/external_openapi.py +2 -2
- api/loopy_catalog_tools.py +24 -4
- api/main.py +138 -26
- api/odoo_relational.py +212 -73
- api/process_role.py +357 -11
- api/providers.py +98 -2
- api/request_class.py +456 -53
- api/request_timing.py +1 -0
- api/routes_account_health.py +142 -0
- api/routes_admin.py +174 -1
- api/routes_aged_stock.py +167 -0
- api/routes_amazon_operator.py +133 -0
- api/routes_assets.py +24 -9
- api/routes_automation.py +5 -3
- api/routes_catalogs.py +17 -10
- api/routes_connectors.py +49 -10
- api/routes_customers.py +16 -0
- api/routes_external.py +123 -7
- api/routes_keychain.py +1 -1
- api/routes_nav.py +24 -10
- api/routes_odoo_attachments.py +38 -4
- api/routes_opportunity_finder.py +159 -0
- api/routes_payouts.py +228 -0
- api/routes_query.py +2 -0
- api/routes_ranking_diagnostic.py +158 -0
- api/routes_reimbursements.py +118 -0
- api/routes_restock_planner.py +156 -0
- api/routes_returns_repeat.py +159 -0
- api/routes_shares.py +106 -0
- api/routes_sheet.py +168 -33
- api/routes_tables.py +1 -2
- api/routes_templates.py +70 -15
- api/sheet_warm.py +131 -31
- api/sync_worker.py +8 -0
- api/test_odoo_rank_refresh.py +3 -3
Dockerfile
CHANGED
|
@@ -120,4 +120,12 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
|
|
| 120 |
# nginx's own keepalive_timeout default and comfortably exceeds the AWS ALB default (60s) other
|
| 121 |
# proxies in this class use, so it is the safest available floor without over-fitting to a number
|
| 122 |
# we cannot read. Re-aim this value again, never delete the flag, if a measurement finds otherwise.
|
| 123 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
# nginx's own keepalive_timeout default and comfortably exceeds the AWS ALB default (60s) other
|
| 121 |
# proxies in this class use, so it is the safest available floor without over-fitting to a number
|
| 122 |
# we cannot read. Re-aim this value again, never delete the flag, if a measurement finds otherwise.
|
| 123 |
+
#
|
| 124 |
+
# S79-T37 (R25): the launcher sizes the web tier to the container. `request_class.py --env` reads
|
| 125 |
+
# the cgroup CPU and memory limits (never os.cpu_count(), which reports the host) and exports the
|
| 126 |
+
# uvicorn worker count, heavy concurrency and DuckDB caps. An env value set on the Space wins.
|
| 127 |
+
# Several workers start only with AIOS_MIRROR_VERSIONS=1, which opens the mirror read-only in each
|
| 128 |
+
# of them, and the shell holds W at 1 without it. 2 vCPU keeps one worker and today's caps.
|
| 129 |
+
# Several workers load the app through a factory: uvicorn imports a plain app string in its
|
| 130 |
+
# supervisor too, and that process would run the boot threads without serving.
|
| 131 |
+
CMD ["sh", "-c", "eval \"$(python api/request_class.py --env)\"; W=1; if [ \"${AIOS_MIRROR_VERSIONS:-0}\" = 1 ]; then W=\"${AIOS_UVICORN_WORKERS:-1}\"; fi; export AIOS_UVICORN_WORKERS=\"$W\"; APP=api.main:app; if [ \"$W\" -gt 1 ]; then APP=\"--factory api.request_class:web_app\"; fi; exec uvicorn $APP --host 0.0.0.0 --port ${PORT} --workers \"$W\" --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"]
|
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "fb1493505",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
fb1493505
|
api/agent_runtime.py
CHANGED
|
@@ -742,7 +742,18 @@ def _hold(session, name, args, effect, held, turn):
|
|
| 742 |
resolved = _fill(args, refs, len(held) + 1)
|
| 743 |
if _PLACEHOLDER.search(json.dumps(resolved)):
|
| 744 |
raise ValueError("A change can only use an earlier step's created object.")
|
| 745 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 746 |
_observe_sources(session, turn, name, resolved, preview["result"])
|
| 747 |
held.append({"tool": name, "args": copy.deepcopy(args), "effect": effect, "ref": ref,
|
| 748 |
"resolvedArgs": resolved, "preview": preview,
|
|
@@ -826,7 +837,8 @@ def _apply_change(session, entry):
|
|
| 826 |
import assistant_write_preview
|
| 827 |
if args != step.get("resolvedArgs"):
|
| 828 |
raise ValueError("An earlier step no longer matches this preview. Ask again.")
|
| 829 |
-
result =
|
|
|
|
| 830 |
_observe_sources(session, turn, step["tool"], args, result)
|
| 831 |
finally:
|
| 832 |
_PRODUCING_TURN.reset(token)
|
|
|
|
| 742 |
resolved = _fill(args, refs, len(held) + 1)
|
| 743 |
if _PLACEHOLDER.search(json.dumps(resolved)):
|
| 744 |
raise ValueError("A change can only use an earlier step's created object.")
|
| 745 |
+
if name == "apps_confirm":
|
| 746 |
+
import routes_external as external
|
| 747 |
+
payload = external._read_confirmation_token(resolved.get("confirmation_token"), session=session)
|
| 748 |
+
if payload.get("tool") != "apps":
|
| 749 |
+
raise ValueError("Choose an App preview to confirm.")
|
| 750 |
+
external._load_operation(session, payload["operation_id"], confirming=True)
|
| 751 |
+
result = external._preview_external_write(
|
| 752 |
+
session, "apps", payload["arguments"], issue_token=False, execution=payload["execution"],
|
| 753 |
+
purpose=payload.get("purpose"), about=payload.get("about"))
|
| 754 |
+
preview = {"documents": {}, "result": result, "diff": result.get("diff") or []}
|
| 755 |
+
else:
|
| 756 |
+
preview = assistant_write_preview.prepare(session, name, resolved, prior=held)
|
| 757 |
_observe_sources(session, turn, name, resolved, preview["result"])
|
| 758 |
held.append({"tool": name, "args": copy.deepcopy(args), "effect": effect, "ref": ref,
|
| 759 |
"resolvedArgs": resolved, "preview": preview,
|
|
|
|
| 837 |
import assistant_write_preview
|
| 838 |
if args != step.get("resolvedArgs"):
|
| 839 |
raise ValueError("An earlier step no longer matches this preview. Ask again.")
|
| 840 |
+
result = (_dispatch(session, "apps_confirm", args) if step["tool"] == "apps_confirm"
|
| 841 |
+
else assistant_write_preview.apply(session, step))
|
| 842 |
_observe_sources(session, turn, step["tool"], args, result)
|
| 843 |
finally:
|
| 844 |
_PRODUCING_TURN.reset(token)
|
api/amazon_relational.py
CHANGED
|
@@ -47,6 +47,7 @@ if str(_API.parent.parent / "platform") not in sys.path:
|
|
| 47 |
import odoo_relational as _odoo # noqa: E402
|
| 48 |
from core import connector_spec # noqa: E402
|
| 49 |
from harness import amazon_client, amazon_schema, amazon_store, datastore # noqa: E402
|
|
|
|
| 50 |
|
| 51 |
Refused = _odoo.Refused
|
| 52 |
|
|
@@ -57,7 +58,10 @@ PREFIX = "ut_amzn_"
|
|
| 57 |
|
| 58 |
#: `ut_*` key -> the mirror table `amazon_store` writes. Derived from ONE list so a table cannot
|
| 59 |
#: exist on one side and not the other.
|
| 60 |
-
|
|
|
|
|
|
|
|
|
|
| 61 |
|
| 62 |
#: ⭐⭐ **THE TABLE SHAPE IS NOW READ FROM `platform/core/connector_spec.py`, NOT TYPED HERE**
|
| 63 |
#: (W53-T37, contract C2). `LABELS`, `PINNED`, `IDENTITY` and `LINKS` keep their module-level
|
|
@@ -66,7 +70,20 @@ SOURCE = {PREFIX + t: amazon_store.table_name(t) for t in amazon_store.DATABASE_
|
|
| 66 |
#: longer be stated twice and say two things. Every comment that used to justify a literal here
|
| 67 |
#: (the two spines, R12, the report-vs-API spellings, the cardinalities) travelled with the values
|
| 68 |
#: into `connector_spec.AMAZON_TABLES`; read it there.
|
| 69 |
-
_SPEC = {s.key: s for s in connector_spec.tables_for("amazon")}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
|
| 71 |
#: ⛔ A `ut_*` KEY THAT `amazon_store` SPAWNS BUT THE SPEC DOES NOT DECLARE IS ABSENT FROM ALL FOUR
|
| 72 |
#: DICTS BELOW, ON PURPOSE. `verify_amazon.section_relational` asserts the four key sets equal
|
|
@@ -106,8 +123,10 @@ def source_attribution(key):
|
|
| 106 |
if key not in SOURCE or key not in _SPEC:
|
| 107 |
raise KeyError(f"unknown Amazon database {key!r}")
|
| 108 |
return {
|
| 109 |
-
"key": "amazon_sp",
|
| 110 |
-
"origin": "
|
|
|
|
|
|
|
| 111 |
"rawTable": SOURCE[key],
|
| 112 |
"native": _SPEC[key].native,
|
| 113 |
"readOnly": True,
|
|
@@ -621,8 +640,8 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True):
|
|
| 621 |
return missing_source(key)
|
| 622 |
names = {name for name, _dtype in columns}
|
| 623 |
table = key.removeprefix(PREFIX)
|
| 624 |
-
identity =
|
| 625 |
-
fallback = sorted(set((*identity, *
|
| 626 |
source = _quoted(SOURCE[key])
|
| 627 |
|
| 628 |
def value(name, owner=source):
|
|
@@ -713,14 +732,14 @@ def grid_sources(cur=None):
|
|
| 713 |
def _row_id(row, key, occurrence=0):
|
| 714 |
"""Return a legacy row key from declared identity, lineage and equal-key occurrence."""
|
| 715 |
table = key.removeprefix(PREFIX)
|
| 716 |
-
columns =
|
| 717 |
identity = tuple(row.get(name) for name in columns)
|
| 718 |
if all(value not in (None, "") for value in identity):
|
| 719 |
if table in _LEGACY_IDENTITY_TABLES and len(identity) == 1:
|
| 720 |
return str(identity[0])
|
| 721 |
payload = json.dumps(identity, ensure_ascii=True, default=str)
|
| 722 |
else:
|
| 723 |
-
fallback = sorted(set((*columns, *
|
| 724 |
values = tuple(None if row.get(name) in (None, "") else str(row[name])
|
| 725 |
for name in fallback)
|
| 726 |
payload = json.dumps(("__amazon_missing_identity__", *values, str(occurrence)),
|
|
@@ -1100,6 +1119,9 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1100 |
for key, _label, _fields, _rows in plans}
|
| 1101 |
listing_ids = (_listing_identity_rows(cur_mirror)
|
| 1102 |
if PREFIX + "listing_items" in built else ())
|
|
|
|
|
|
|
|
|
|
| 1103 |
for key, _label, _fields, rows in plans:
|
| 1104 |
if isinstance(rows, dict):
|
| 1105 |
continue
|
|
@@ -1116,7 +1138,8 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1116 |
candidates = (
|
| 1117 |
_row_id(raw, key, occurrence),
|
| 1118 |
hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32])
|
| 1119 |
-
declared =
|
|
|
|
| 1120 |
if all(raw.get(name) not in (None, "") for name in declared):
|
| 1121 |
old_composite = json.dumps(tuple(raw[name] for name in declared),
|
| 1122 |
ensure_ascii=True, default=str)
|
|
@@ -1235,6 +1258,8 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1235 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 1236 |
# mutable display label or from the process having imported this module.
|
| 1237 |
doc[key]["connector"] = source_attribution(key)
|
|
|
|
|
|
|
| 1238 |
if key == PREFIX + "listing_items":
|
| 1239 |
doc[key]["amazonListingIdentity"] = "sku"
|
| 1240 |
return doc
|
|
|
|
| 47 |
import odoo_relational as _odoo # noqa: E402
|
| 48 |
from core import connector_spec # noqa: E402
|
| 49 |
from harness import amazon_client, amazon_schema, amazon_store, datastore # noqa: E402
|
| 50 |
+
from harness import amazon_ads_store # noqa: E402
|
| 51 |
|
| 52 |
Refused = _odoo.Refused
|
| 53 |
|
|
|
|
| 58 |
|
| 59 |
#: `ut_*` key -> the mirror table `amazon_store` writes. Derived from ONE list so a table cannot
|
| 60 |
#: exist on one side and not the other.
|
| 61 |
+
#: S79-T13: the Ads tables (`ut_amzn_ads_*`, connector `amazon_ads`) live in the same mirror.
|
| 62 |
+
_ADS_SPEC = {s.key: s for s in connector_spec.tables_for("amazon_ads")}
|
| 63 |
+
SOURCE = {**{PREFIX + t: amazon_store.table_name(t) for t in amazon_store.DATABASE_TABLES},
|
| 64 |
+
**{k: amazon_ads_store.table_name(s.source) for k, s in _ADS_SPEC.items()}}
|
| 65 |
|
| 66 |
#: ⭐⭐ **THE TABLE SHAPE IS NOW READ FROM `platform/core/connector_spec.py`, NOT TYPED HERE**
|
| 67 |
#: (W53-T37, contract C2). `LABELS`, `PINNED`, `IDENTITY` and `LINKS` keep their module-level
|
|
|
|
| 70 |
#: longer be stated twice and say two things. Every comment that used to justify a literal here
|
| 71 |
#: (the two spines, R12, the report-vs-API spellings, the cardinalities) travelled with the values
|
| 72 |
#: into `connector_spec.AMAZON_TABLES`; read it there.
|
| 73 |
+
_SPEC = {**{s.key: s for s in connector_spec.tables_for("amazon")}, **_ADS_SPEC}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _identity_columns(table, columns=()):
|
| 77 |
+
"""An Ads row is identified by its stored `_row_key`; every other table by amazon_schema."""
|
| 78 |
+
if PREFIX + table in _ADS_SPEC:
|
| 79 |
+
return ("_row_key",)
|
| 80 |
+
return amazon_schema.row_key_columns_for(table, columns)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def _fallback_columns(table):
|
| 84 |
+
if PREFIX + table in _ADS_SPEC:
|
| 85 |
+
return ("_report_id",)
|
| 86 |
+
return amazon_schema.row_fallback_columns_for(table)
|
| 87 |
|
| 88 |
#: ⛔ A `ut_*` KEY THAT `amazon_store` SPAWNS BUT THE SPEC DOES NOT DECLARE IS ABSENT FROM ALL FOUR
|
| 89 |
#: DICTS BELOW, ON PURPOSE. `verify_amazon.section_relational` asserts the four key sets equal
|
|
|
|
| 123 |
if key not in SOURCE or key not in _SPEC:
|
| 124 |
raise KeyError(f"unknown Amazon database {key!r}")
|
| 125 |
return {
|
| 126 |
+
"key": "amazon_ads" if key in _ADS_SPEC else "amazon_sp",
|
| 127 |
+
"origin": ("Amazon Ads API" if key in _ADS_SPEC
|
| 128 |
+
else "Apify Amazon search dataset" if key == PREFIX + "keyword_ranks"
|
| 129 |
+
else "Amazon Services API"),
|
| 130 |
"rawTable": SOURCE[key],
|
| 131 |
"native": _SPEC[key].native,
|
| 132 |
"readOnly": True,
|
|
|
|
| 640 |
return missing_source(key)
|
| 641 |
names = {name for name, _dtype in columns}
|
| 642 |
table = key.removeprefix(PREFIX)
|
| 643 |
+
identity = _identity_columns(table, names)
|
| 644 |
+
fallback = sorted(set((*identity, *_fallback_columns(table))))
|
| 645 |
source = _quoted(SOURCE[key])
|
| 646 |
|
| 647 |
def value(name, owner=source):
|
|
|
|
| 732 |
def _row_id(row, key, occurrence=0):
|
| 733 |
"""Return a legacy row key from declared identity, lineage and equal-key occurrence."""
|
| 734 |
table = key.removeprefix(PREFIX)
|
| 735 |
+
columns = _identity_columns(table, row)
|
| 736 |
identity = tuple(row.get(name) for name in columns)
|
| 737 |
if all(value not in (None, "") for value in identity):
|
| 738 |
if table in _LEGACY_IDENTITY_TABLES and len(identity) == 1:
|
| 739 |
return str(identity[0])
|
| 740 |
payload = json.dumps(identity, ensure_ascii=True, default=str)
|
| 741 |
else:
|
| 742 |
+
fallback = sorted(set((*columns, *_fallback_columns(table))))
|
| 743 |
values = tuple(None if row.get(name) in (None, "") else str(row[name])
|
| 744 |
for name in fallback)
|
| 745 |
payload = json.dumps(("__amazon_missing_identity__", *values, str(occurrence)),
|
|
|
|
| 1119 |
for key, _label, _fields, _rows in plans}
|
| 1120 |
listing_ids = (_listing_identity_rows(cur_mirror)
|
| 1121 |
if PREFIX + "listing_items" in built else ())
|
| 1122 |
+
# S79-T13: each Ads table's description states its history window and any recorded limit.
|
| 1123 |
+
ads_descriptions = {key: amazon_ads_store.table_description(SOURCE[key], cur_mirror)
|
| 1124 |
+
for key, *_rest in plans if key in _ADS_SPEC}
|
| 1125 |
for key, _label, _fields, rows in plans:
|
| 1126 |
if isinstance(rows, dict):
|
| 1127 |
continue
|
|
|
|
| 1138 |
candidates = (
|
| 1139 |
_row_id(raw, key, occurrence),
|
| 1140 |
hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32])
|
| 1141 |
+
declared = (("_row_key",) if key in _ADS_SPEC
|
| 1142 |
+
else amazon_schema.TABLE_SPECS[key.removeprefix(PREFIX)]["row_identity"])
|
| 1143 |
if all(raw.get(name) not in (None, "") for name in declared):
|
| 1144 |
old_composite = json.dumps(tuple(raw[name] for name in declared),
|
| 1145 |
ensure_ascii=True, default=str)
|
|
|
|
| 1258 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 1259 |
# mutable display label or from the process having imported this module.
|
| 1260 |
doc[key]["connector"] = source_attribution(key)
|
| 1261 |
+
if ads_descriptions.get(key):
|
| 1262 |
+
doc[key]["description"] = ads_descriptions[key]
|
| 1263 |
if key == PREFIX + "listing_items":
|
| 1264 |
doc[key]["amazonListingIdentity"] = "sku"
|
| 1265 |
return doc
|
api/app_routes.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""C3: mount every C1 App router by name, recording a failed import instead of raising it."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import sys
|
| 5 |
+
from importlib import import_module
|
| 6 |
+
|
| 7 |
+
import app_templates
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def _failures_record():
|
| 11 |
+
main = sys.modules.get("main")
|
| 12 |
+
record = getattr(main, "ROUTER_MOUNT_FAILURES", None) if main is not None else None
|
| 13 |
+
return record if isinstance(record, dict) else app_templates.ROUTER_MOUNT_FAILURES
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _flat(routes):
|
| 17 |
+
"""Every route, descending into a router FastAPI keeps as one included entry."""
|
| 18 |
+
for route in routes:
|
| 19 |
+
yield route
|
| 20 |
+
inner = getattr(route, "original_router", None)
|
| 21 |
+
if inner is not None:
|
| 22 |
+
yield from _flat(inner.routes)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _included(app, router):
|
| 26 |
+
endpoints = {getattr(route, "endpoint", None) for route in _flat(router.routes)}
|
| 27 |
+
endpoints.discard(None)
|
| 28 |
+
return any(getattr(route, "original_router", None) is router
|
| 29 |
+
or getattr(route, "endpoint", None) in endpoints for route in _flat(app.routes))
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def mount_app_routers(app, failures=None):
|
| 33 |
+
"""Include each C1 `router` once, with main.py `_mount_router` semantics; returns those mounted."""
|
| 34 |
+
record = failures if isinstance(failures, dict) else _failures_record()
|
| 35 |
+
mounted = []
|
| 36 |
+
for row in app_templates.TEMPLATES:
|
| 37 |
+
name = row["router"]
|
| 38 |
+
if not name:
|
| 39 |
+
continue
|
| 40 |
+
try:
|
| 41 |
+
router = import_module(name).router
|
| 42 |
+
# An already included router (Competitor watch, mounted by main.py) is not added twice.
|
| 43 |
+
if not _included(app, router):
|
| 44 |
+
app.include_router(router)
|
| 45 |
+
except Exception as exc: # Boot must answer even when a router cannot load.
|
| 46 |
+
failure = {"error": type(exc).__name__,
|
| 47 |
+
"missing_module": exc.name if isinstance(exc, ModuleNotFoundError) else ""}
|
| 48 |
+
record.setdefault(name, []).append(failure)
|
| 49 |
+
if record is not _failures_record():
|
| 50 |
+
# The offer predicate reads main.py's record and this one, never a caller's own dict.
|
| 51 |
+
app_templates.ROUTER_MOUNT_FAILURES.setdefault(name, []).append(failure)
|
| 52 |
+
# Exception messages can contain configuration values; report the class/module only.
|
| 53 |
+
print(f"[aios-api] ROUTER MOUNT FAILED: {name}: {failure}; "
|
| 54 |
+
"its App template is withheld until the router mounts")
|
| 55 |
+
continue
|
| 56 |
+
app_templates.MOUNTED_ROUTERS.add(name)
|
| 57 |
+
mounted.append(name)
|
| 58 |
+
return mounted
|
api/app_rules.py
CHANGED
|
@@ -441,8 +441,7 @@ def _catalog_selftest():
|
|
| 441 |
source = rt.get("product_table_workspace__shared")
|
| 442 |
contracts = registry.chart_field_contracts(session, ["product_data"])
|
| 443 |
app = registry.create(rt, tenant, "member", {
|
| 444 |
-
"name": "Template fixture", "templateKind": "catalog",
|
| 445 |
-
"fieldMap": {"key": "code", "title": "product", "image": "catalog_photo"},
|
| 446 |
"navPlacement": {"section": "app", "order": 0}},
|
| 447 |
allowed_databases={"product_data"}, field_contracts=contracts)
|
| 448 |
api = FastAPI()
|
|
|
|
| 441 |
source = rt.get("product_table_workspace__shared")
|
| 442 |
contracts = registry.chart_field_contracts(session, ["product_data"])
|
| 443 |
app = registry.create(rt, tenant, "member", {
|
| 444 |
+
"name": "Template fixture", "templateKind": "catalog",
|
|
|
|
| 445 |
"navPlacement": {"section": "app", "order": 0}},
|
| 446 |
allowed_databases={"product_data"}, field_contracts=contracts)
|
| 447 |
api = FastAPI()
|
api/app_templates.py
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""C1: the shipped App template catalogue, one row per App template, each bundled with its connector.
|
| 2 |
+
|
| 3 |
+
The registry's template contracts and `core.external_tools`' templateKind enum derive from or
|
| 4 |
+
mirror `TEMPLATES`; `verify_app_templates.py` holds all three name for name.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import sys
|
| 9 |
+
from importlib import import_module
|
| 10 |
+
from types import MappingProxyType
|
| 11 |
+
|
| 12 |
+
PROCESSES = ("Create value", "Acquire", "Convert", "Deliver", "Retain", "Finance and cash",
|
| 13 |
+
"Risk and compliance", "None")
|
| 14 |
+
|
| 15 |
+
# A C1 connector key is not a keychain entry type: Amazon SP-API is stored as `amazon_sp` (F25).
|
| 16 |
+
CONNECTOR_ENTRY_TYPES = MappingProxyType({"odoo": "odoo", "amazon": "amazon_sp", "amazon_ads": "amazon_ads"})
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _amazon(kind, label, purpose, process, sources, *, connector="amazon", proposes=True, tools=()):
|
| 20 |
+
return {"kind": kind, "label": label, "purpose": purpose, "connector": connector,
|
| 21 |
+
"sources": tuple(sources), "process": process, "mark": kind,
|
| 22 |
+
"module": "modules." + kind, "router": "routes_" + kind,
|
| 23 |
+
"view": kind.replace("_", "-"), "tools": tuple(tools), "proposes": proposes}
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
TEMPLATES = (
|
| 27 |
+
# Routing and Catalog render through Map mode; their routes are mounted by main.py.
|
| 28 |
+
# They read FIELD_BINDINGS over their source, so an App of either kind stores no field map.
|
| 29 |
+
{"kind": "route", "label": "Routing", "purpose": "Plan delivery routes for your customers.",
|
| 30 |
+
"connector": "odoo", "sources": ("customer_data",), "process": "None", "mark": "route",
|
| 31 |
+
"module": None, "router": None, "view": None, "tools": (), "proposes": False},
|
| 32 |
+
{"kind": "catalog", "label": "Catalog", "purpose": "Build and print product catalogs.",
|
| 33 |
+
"connector": "odoo", "sources": ("product_data",), "process": "None", "mark": "catalog",
|
| 34 |
+
"module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"),
|
| 35 |
+
"proposes": False},
|
| 36 |
+
_amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.",
|
| 37 |
+
"Acquire", ("ut_amzn_brand_analytics",), proposes=False),
|
| 38 |
+
_amazon("ranking_diagnostic", "Ranking diagnostic", "Find why a listing lost its rank.",
|
| 39 |
+
"Acquire", ("ut_amzn_pricing", "ut_amzn_catalog_items", "ut_amzn_brand_analytics",
|
| 40 |
+
"ut_amzn_economics", "ut_amzn_fba_inventory", "ut_amzn_suppressed_listings",
|
| 41 |
+
"ut_amzn_stranded_inventory", "ut_amzn_inventory_planning"), proposes=False),
|
| 42 |
+
_amazon("amazon_operator", "Operator inbox", "Approve ad changes in one batch.",
|
| 43 |
+
"Acquire", (), connector="amazon_ads", proposes=False),
|
| 44 |
+
_amazon("restock_planner", "Restock planner", "Send stock to Amazon before it runs out.",
|
| 45 |
+
"Deliver", ("ut_amzn_restock_recommendations", "ut_amzn_inventory_planning",
|
| 46 |
+
"ut_amzn_fba_inventory")),
|
| 47 |
+
_amazon("reimbursements", "Reimbursements", "Find FBA money Amazon owes you.",
|
| 48 |
+
"Create value", ("ut_amzn_reimbursements", "ut_amzn_inventory_ledger_detail",
|
| 49 |
+
"ut_amzn_fba_customer_returns")),
|
| 50 |
+
_amazon("aged_stock", "Aged stock", "Move old stock before storage fees grow.",
|
| 51 |
+
"Create value", ("ut_amzn_inventory_planning", "ut_amzn_long_term_storage_fees",
|
| 52 |
+
"ut_amzn_storage_fees", "ut_amzn_recommended_removals",
|
| 53 |
+
"ut_amzn_stranded_inventory")),
|
| 54 |
+
_amazon("opportunity_finder", "Opportunity finder", "Find rising searches you do not serve yet.",
|
| 55 |
+
"Create value", ("ut_amzn_brand_analytics", "ut_amzn_search_query_performance", "ut_amzn_listings")),
|
| 56 |
+
_amazon("returns_repeat", "Returns and repeat buyers", "See why buyers return and who buys again.",
|
| 57 |
+
"Retain", ("ut_amzn_fba_customer_returns", "ut_amzn_repeat_purchase",
|
| 58 |
+
"ut_amzn_fba_shipped_sales", "ut_amzn_market_basket")),
|
| 59 |
+
_amazon("payouts", "Payouts", "Tie each payout to its orders and fees.",
|
| 60 |
+
"Finance and cash", ("ut_amzn_settlements", "ut_amzn_economics", "ut_amzn_transactions")),
|
| 61 |
+
_amazon("account_health", "Account health", "Catch account risks before Amazon acts.",
|
| 62 |
+
"Risk and compliance", ("ut_amzn_account_health",)),
|
| 63 |
+
)
|
| 64 |
+
BY_KIND = MappingProxyType({row["kind"]: row for row in TEMPLATES})
|
| 65 |
+
KINDS = tuple(BY_KIND)
|
| 66 |
+
|
| 67 |
+
#: R33: the fixed role to field binding each kind reads from its first C1 source.
|
| 68 |
+
#: Catalog price is Royal 1 tier 1: the pooled `price_1` is not in the typed product_data schema.
|
| 69 |
+
FIELD_BINDINGS = MappingProxyType({
|
| 70 |
+
"route": MappingProxyType({"title": "customer", "address": "street",
|
| 71 |
+
"latitude": "lat", "longitude": "lon"}),
|
| 72 |
+
"catalog": MappingProxyType({"key": "code", "title": "product", "image": "image",
|
| 73 |
+
"sku": "code", "price": "price_royal1_t1", "category": "category"}),
|
| 74 |
+
"competitor_watch": MappingProxyType({"term": "search_term"}),
|
| 75 |
+
})
|
| 76 |
+
#: Kinds whose App record stores no source or field map: the registry projects C1 on read.
|
| 77 |
+
BOUND_KINDS = ("route", "catalog")
|
| 78 |
+
_BINDING_KEYS = ("sourceDatabase", "fieldMap")
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def source(kind):
|
| 82 |
+
"""The C1 source table a kind reads, or None."""
|
| 83 |
+
row = BY_KIND.get(kind)
|
| 84 |
+
return row["sources"][0] if row and row["sources"] else None
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def field_binding(kind):
|
| 88 |
+
"""A copy of the kind's fixed role to field map; empty when it names none."""
|
| 89 |
+
return dict(FIELD_BINDINGS.get(kind) or {})
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def bound(app):
|
| 93 |
+
"""Read projection: a copy of a bound kind's App with the C1 source and field map filled in.
|
| 94 |
+
|
| 95 |
+
A stored App that still carries its own map keeps it until the S79-T04 migration.
|
| 96 |
+
"""
|
| 97 |
+
if not isinstance(app, dict) or app.get("templateKind") not in BOUND_KINDS:
|
| 98 |
+
return app
|
| 99 |
+
row = dict(app)
|
| 100 |
+
projection = {"sourceDatabase": source(row["templateKind"]),
|
| 101 |
+
"fieldMap": field_binding(row["templateKind"])}
|
| 102 |
+
if row.get("fieldMap"):
|
| 103 |
+
projection = {"sourceDatabase": row.get("sourceDatabase") or projection["sourceDatabase"]}
|
| 104 |
+
row.update(projection)
|
| 105 |
+
return row
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def own_binding(app):
|
| 109 |
+
"""The binding keys a stored App carries that differ from what `bound` projects."""
|
| 110 |
+
if not isinstance(app, dict) or app.get("templateKind") not in BOUND_KINDS:
|
| 111 |
+
return {key: app[key] for key in _BINDING_KEYS if isinstance(app, dict) and key in app}
|
| 112 |
+
kind = app["templateKind"]
|
| 113 |
+
fixed = field_binding(kind)
|
| 114 |
+
held = app.get("fieldMap") or {}
|
| 115 |
+
if (app.get("sourceDatabase") in (None, source(kind))
|
| 116 |
+
and all(fixed.get(role) == field for role, field in held.items())):
|
| 117 |
+
return {}
|
| 118 |
+
return {key: app[key] for key in _BINDING_KEYS if key in app}
|
| 119 |
+
|
| 120 |
+
#: Routers `app_routes.mount_app_routers` failed to mount when main.py's record is absent.
|
| 121 |
+
ROUTER_MOUNT_FAILURES = {}
|
| 122 |
+
#: Routers `app_routes.mount_app_routers` mounted or found already included.
|
| 123 |
+
MOUNTED_ROUTERS = set()
|
| 124 |
+
# A failed seam import is printed once per process and withholds its kind.
|
| 125 |
+
_SEAM_FAILURES = {}
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def entry_type(connector):
|
| 129 |
+
"""The keychain entry type that proves a C1 connector key is connected."""
|
| 130 |
+
return CONNECTOR_ENTRY_TYPES.get(connector, connector)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def connectors_present(session):
|
| 134 |
+
"""C1 connector keys this reader has. Any read failure counts as nothing connected."""
|
| 135 |
+
try:
|
| 136 |
+
from routes_connectors import _entry_types
|
| 137 |
+
types, _scopes, failure = _entry_types(session.runtime, session.uname)
|
| 138 |
+
have = set() if failure else set(types or ())
|
| 139 |
+
except Exception as exc: # noqa: BLE001
|
| 140 |
+
from core.request_timing import raise_if_resource_error
|
| 141 |
+
raise_if_resource_error(exc)
|
| 142 |
+
have = set()
|
| 143 |
+
if "odoo" not in have:
|
| 144 |
+
try:
|
| 145 |
+
from routes_keychain import env_odoo_available
|
| 146 |
+
if env_odoo_available(session.runtime):
|
| 147 |
+
have.add("odoo")
|
| 148 |
+
except Exception: # noqa: BLE001
|
| 149 |
+
pass
|
| 150 |
+
return {key for key in CONNECTOR_ENTRY_TYPES if entry_type(key) in have}
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def connector_present(connector, session):
|
| 154 |
+
return connector in connectors_present(session)
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def router_failures():
|
| 158 |
+
"""Every recorded router mount failure: main.py's record joined with this module's own."""
|
| 159 |
+
merged = {name: list(rows) for name, rows in ROUTER_MOUNT_FAILURES.items()}
|
| 160 |
+
main = sys.modules.get("main")
|
| 161 |
+
record = getattr(main, "ROUTER_MOUNT_FAILURES", None) if main is not None else None
|
| 162 |
+
if isinstance(record, dict) and record is not ROUTER_MOUNT_FAILURES:
|
| 163 |
+
for name, rows in record.items():
|
| 164 |
+
merged.setdefault(name, []).extend(rows)
|
| 165 |
+
return merged
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def _importable(name, attribute=None):
|
| 169 |
+
if name in _SEAM_FAILURES:
|
| 170 |
+
return False
|
| 171 |
+
try:
|
| 172 |
+
module = import_module(name)
|
| 173 |
+
if attribute and not hasattr(module, attribute):
|
| 174 |
+
raise ImportError(f"{name} has no {attribute}")
|
| 175 |
+
return True
|
| 176 |
+
except Exception as exc: # noqa: BLE001
|
| 177 |
+
_SEAM_FAILURES[name] = type(exc).__name__
|
| 178 |
+
print(f"[aios-api] APP TEMPLATE WITHHELD: {name}: {type(exc).__name__}")
|
| 179 |
+
return False
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def seams_ready(kind):
|
| 183 |
+
"""True when the kind's module imports and its router imports and never failed to mount."""
|
| 184 |
+
row = BY_KIND.get(kind)
|
| 185 |
+
if row is None:
|
| 186 |
+
return False
|
| 187 |
+
if row["module"] and not _importable(row["module"]):
|
| 188 |
+
return False
|
| 189 |
+
router = row["router"]
|
| 190 |
+
return not router or (_importable(router, "router") and router not in router_failures())
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def reset_seam_cache():
|
| 194 |
+
"""Forget recorded seam import failures (test fixtures only)."""
|
| 195 |
+
_SEAM_FAILURES.clear()
|
api/application_registry.py
CHANGED
|
@@ -8,6 +8,7 @@ import json
|
|
| 8 |
from core import shares, write_execution
|
| 9 |
from core.request_timing import raise_if_resource_error
|
| 10 |
import app_rules
|
|
|
|
| 11 |
|
| 12 |
from modules.finance_apps import (finance_application as _finance_application,
|
| 13 |
finance_applications as _finance_applications,
|
|
@@ -19,6 +20,7 @@ APPLICATION_TYPES = ("chart", "route", "map", "code")
|
|
| 19 |
# Finance descriptors remain available to the governed fact routes.
|
| 20 |
FINANCE_TEMPLATE_KIND = "finance"
|
| 21 |
_KEY = "applications_v1"
|
|
|
|
| 22 |
_MAX = 500
|
| 23 |
_MAX_DISPLAY_BYTES = 512 * 1024
|
| 24 |
|
|
@@ -66,28 +68,58 @@ _TEMPLATE_CONTRACTS = {
|
|
| 66 |
},
|
| 67 |
}
|
| 68 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
# Keep stored copy identities readable after a template stops accepting new Apps.
|
| 70 |
APPLICATION_WIZARD_TEMPLATE_KINDS = tuple(_TEMPLATE_CONTRACTS)
|
| 71 |
APPLICATION_TEMPLATE_KINDS = tuple(_TEMPLATE_CONTRACTS)
|
| 72 |
|
| 73 |
#: S78-T17 (C4, R20, R21): templates offered only to a tenant that passes the kind's gate.
|
| 74 |
#: They stay out of the ungated tuples above, which other gates pin to Routing and Catalog.
|
|
|
|
| 75 |
BRAND_ANALYTICS_DATABASE = "ut_amzn_brand_analytics"
|
| 76 |
-
|
|
|
|
| 77 |
"competitor_watch": {
|
| 78 |
-
"kind": "competitor_watch",
|
| 79 |
-
"label": "Competitor watch",
|
| 80 |
-
# The screen reads the competitor watch routes; the source names the Amazon database.
|
| 81 |
-
"applicationType": "map",
|
| 82 |
-
"displayMode": "map",
|
| 83 |
"fieldRoles": [
|
| 84 |
{"key": "term", "label": "Search term", "required": True, "types": []},
|
| 85 |
],
|
| 86 |
-
"defaultPlacement": {"section": "app", "order": 0},
|
| 87 |
-
"sourceDatabase": BRAND_ANALYTICS_DATABASE,
|
| 88 |
"sourceLabel": "Amazon brand analytics",
|
| 89 |
},
|
| 90 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
GATED_TEMPLATE_KINDS = tuple(_GATED_TEMPLATE_CONTRACTS)
|
| 92 |
|
| 93 |
|
|
@@ -140,13 +172,56 @@ def brand_analytics_offered(session):
|
|
| 140 |
return False
|
| 141 |
|
| 142 |
|
| 143 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
|
| 145 |
|
| 146 |
class ApplicationError(ValueError):
|
| 147 |
pass
|
| 148 |
|
| 149 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
class ApplicationRuleError(ApplicationError):
|
| 151 |
"""Expose an App rule refusal through every writer's existing error handler."""
|
| 152 |
code = "app_rule"
|
|
@@ -176,7 +251,8 @@ def template_identity(app):
|
|
| 176 |
if kind == FINANCE_TEMPLATE_KIND:
|
| 177 |
key = str(app.get("financeKey") or "")
|
| 178 |
return f"{FINANCE_TEMPLATE_KIND}:{key}" if key else None
|
| 179 |
-
|
|
|
|
| 180 |
|
| 181 |
|
| 182 |
def _own_template_app(applications, user, identity):
|
|
@@ -208,12 +284,15 @@ def own_template_apps(runtime, user):
|
|
| 208 |
def template_contracts(session=None):
|
| 209 |
"""The trusted renderer contracts offered by the Application creation flow.
|
| 210 |
|
| 211 |
-
|
| 212 |
"""
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
|
|
|
|
|
|
|
|
|
| 217 |
return rows
|
| 218 |
|
| 219 |
|
|
@@ -366,8 +445,12 @@ def _clean_doc(raw):
|
|
| 366 |
|
| 367 |
|
| 368 |
def load(runtime):
|
| 369 |
-
|
|
|
|
| 370 |
except Exception: return blank_doc()
|
|
|
|
|
|
|
|
|
|
| 371 |
|
| 372 |
|
| 373 |
def _server_origin(origin_door=None, via=None):
|
|
@@ -436,6 +519,14 @@ def _database_free_code(app):
|
|
| 436 |
and not app.get("compatibleDatabases"))
|
| 437 |
|
| 438 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 439 |
def _finance_permission(app):
|
| 440 |
try:
|
| 441 |
return _finance_application(str(app.get("financeKey") or ""))["permissionKey"]
|
|
@@ -711,66 +802,45 @@ def _refuse_legacy_catalog_changes(app, config):
|
|
| 711 |
|
| 712 |
def _template_definition(kind, payload, *, semantic_records=None, field_contracts=None,
|
| 713 |
allowed_databases=None):
|
|
|
|
| 714 |
contract = _TEMPLATE_CONTRACTS.get(kind) or _GATED_TEMPLATE_CONTRACTS.get(kind)
|
| 715 |
if contract is None:
|
| 716 |
raise ApplicationError("application template kind is not recognized")
|
| 717 |
template_kind = str(payload.get("templateKind") or "").strip().lower()
|
| 718 |
if template_kind != kind:
|
| 719 |
raise ApplicationError(f"{kind} applications require templateKind {kind!r}")
|
| 720 |
-
|
| 721 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 722 |
raise ApplicationError("sourceDatabase is required")
|
| 723 |
-
|
| 724 |
raise ApplicationError(f"{contract['label']} reads "
|
| 725 |
-
f"{contract.get('sourceLabel') or
|
| 726 |
"Choose it as the source database.")
|
| 727 |
-
|
| 728 |
raise PermissionError("an Application cannot bind a database you cannot open")
|
| 729 |
-
if "compatibleDatabases" in payload:
|
| 730 |
compatible = _compat(payload.get("compatibleDatabases"))
|
| 731 |
if compatible != [source]:
|
| 732 |
raise ApplicationError("a template Application has exactly one sourceDatabase")
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
if
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
available = _contract_fields(source, semantic_records, field_contracts)
|
| 747 |
-
if not available:
|
| 748 |
-
raise ApplicationError("the source database has no readable field contract")
|
| 749 |
-
field_map = {}
|
| 750 |
-
for role, raw_field in raw_map.items():
|
| 751 |
-
if raw_field is not None and not isinstance(raw_field, str):
|
| 752 |
-
raise ApplicationError(f"fieldMap role {role!r} must name a source field with a text key")
|
| 753 |
-
field = str(raw_field or "").strip()
|
| 754 |
-
if not field:
|
| 755 |
-
continue
|
| 756 |
-
if len(field) > 160 or field not in available:
|
| 757 |
-
raise ApplicationError(f"fieldMap role {role!r} does not name a source field")
|
| 758 |
-
allowed_types = set(roles[role].get("types") or ())
|
| 759 |
-
if allowed_types and available[field] not in allowed_types:
|
| 760 |
-
raise ApplicationError(
|
| 761 |
-
f"fieldMap role {role!r} cannot use field type {available[field]!r}")
|
| 762 |
-
field_map[role] = field
|
| 763 |
-
latitude, longitude = field_map.get("latitude"), field_map.get("longitude")
|
| 764 |
-
if bool(latitude) != bool(longitude):
|
| 765 |
-
raise ApplicationError("Routing needs both coordinate fields or neither")
|
| 766 |
-
if latitude and latitude == longitude:
|
| 767 |
-
raise ApplicationError("Routing latitude and longitude must use different fields")
|
| 768 |
-
# A new Route must name at least one placement source. Stored definitions are not revalidated
|
| 769 |
-
# on read, so this create-time refusal does not retire older Applications with an empty map.
|
| 770 |
-
if kind == "route" and not (latitude and longitude) and not field_map.get("address"):
|
| 771 |
-
raise ApplicationError("Routing needs both coordinate fields or an address field")
|
| 772 |
-
|
| 773 |
-
raw_placement = payload.get("navPlacement")
|
| 774 |
if not isinstance(raw_placement, dict) or set(raw_placement) != {"section", "order"}:
|
| 775 |
raise ApplicationError("navPlacement must contain section and order")
|
| 776 |
section = str(raw_placement.get("section") or "").strip().lower()
|
|
@@ -779,8 +849,33 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract
|
|
| 779 |
raise ApplicationError("template Applications must be placed in App")
|
| 780 |
if isinstance(order, bool) or not isinstance(order, int) or not 0 <= order <= 999:
|
| 781 |
raise ApplicationError("navPlacement order must be an integer from 0 to 999")
|
| 782 |
-
|
| 783 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 784 |
|
| 785 |
|
| 786 |
def validate_application(payload, tenant, *, semantic_records=None, field_contracts=None,
|
|
@@ -830,7 +925,7 @@ def validate_application(payload, tenant, *, semantic_records=None, field_contra
|
|
| 830 |
|
| 831 |
def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
|
| 832 |
is_default=False):
|
| 833 |
-
row=copy.deepcopy(app)
|
| 834 |
row.setdefault("origin", None)
|
| 835 |
if row.get("type") == "code":
|
| 836 |
# Only the authenticated code door returns the body.
|
|
@@ -844,6 +939,11 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
|
|
| 844 |
"default":bool(is_admin)}
|
| 845 |
row["starred"] = user in (app.get("starredBy") or ())
|
| 846 |
row["isDefault"] = bool(is_default)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 847 |
if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
|
| 848 |
# A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
|
| 849 |
row["capabilities"]["default"] = False
|
|
@@ -864,6 +964,8 @@ def dependent_view_counts(runtime, database):
|
|
| 864 |
"""S67-T39 / R36: {app_id: number of saved Views on `database` bound to it}, over EVERY
|
| 865 |
user's Views in that database's workspace. The Delete App dialog names this count before
|
| 866 |
confirm. None when the workspace cannot be read, so the dialog never states a guessed 0.
|
|
|
|
|
|
|
| 867 |
"""
|
| 868 |
from core.view_templates import workspace_key
|
| 869 |
key = workspace_key(database)
|
|
@@ -874,16 +976,11 @@ def dependent_view_counts(runtime, database):
|
|
| 874 |
except Exception:
|
| 875 |
return None
|
| 876 |
counts = {}
|
| 877 |
-
for member, bucket in (doc.items() if isinstance(doc, dict) else ())
|
| 878 |
-
|
| 879 |
-
|
| 880 |
-
|
| 881 |
-
|
| 882 |
-
config = view.get("config") if isinstance(view, dict) else None
|
| 883 |
-
ref = config.get("application") if isinstance(config, dict) else None
|
| 884 |
-
app_id = str(ref.get("id") or "") if isinstance(ref, dict) else ""
|
| 885 |
-
if app_id:
|
| 886 |
-
counts[app_id] = counts.get(app_id, 0) + 1
|
| 887 |
return counts
|
| 888 |
|
| 889 |
|
|
@@ -900,7 +997,7 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 900 |
if not may_read(app,user,is_admin): continue
|
| 901 |
if database not in (app.get("compatibleDatabases") or ()): continue
|
| 902 |
elif not _listed(app, user): continue
|
| 903 |
-
if (not
|
| 904 |
and not _allowed(app.get("compatibleDatabases"), allowed_databases)): continue
|
| 905 |
if kind and app.get("type") != kind: continue
|
| 906 |
if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
|
|
@@ -947,7 +1044,7 @@ def _build_application(tenant, user, payload, *, semantic_records=None, field_co
|
|
| 947 |
name=" ".join(str(payload.get("name") or "").split())[:120]
|
| 948 |
if not name: raise ApplicationError("application name is required")
|
| 949 |
app_id=f"app_{write_execution.new_id()[:16]}"; now=write_execution.utcnow().isoformat()
|
| 950 |
-
compatible = (
|
| 951 |
else _compat(payload.get("compatibleDatabases"), allow_empty=kind == "code"))
|
| 952 |
if _allowed(compatible, allowed_databases) != set(compatible):
|
| 953 |
raise PermissionError("an Application cannot bind a database you cannot open")
|
|
@@ -1035,7 +1132,8 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, field_contr
|
|
| 1035 |
identity = template_identity(app)
|
| 1036 |
def apply(raw):
|
| 1037 |
doc=_clean_doc(raw)
|
| 1038 |
-
|
|
|
|
| 1039 |
# R216 and R247 allow every template to have several independent copies.
|
| 1040 |
doc["applications"][app_id]=app; return doc
|
| 1041 |
_update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of)
|
|
@@ -1066,10 +1164,14 @@ def ensure_route_application(runtime, tenant, *, accounts=None, wall=None):
|
|
| 1066 |
"""
|
| 1067 |
result = _ensure_route_seed(runtime, tenant)
|
| 1068 |
copies = ensure_user_app_copies(runtime, tenant, accounts=accounts, wall=wall)
|
|
|
|
| 1069 |
folded = dict(result)
|
| 1070 |
folded["userCopies"] = copies
|
|
|
|
| 1071 |
note = "user copies " + copies["outcome"] + (
|
| 1072 |
": " + copies["reason"] if copies.get("reason") else "")
|
|
|
|
|
|
|
| 1073 |
folded["reason"] = result["reason"] + "; " + note if result.get("reason") else note
|
| 1074 |
return folded
|
| 1075 |
|
|
@@ -1106,13 +1208,12 @@ def _ensure_route_seed(runtime, tenant):
|
|
| 1106 |
app_id = existing.get("id") if existing is not None else None
|
| 1107 |
outcome = "existing" if existing is not None else "no_routes"
|
| 1108 |
if existing is None and route_count:
|
| 1109 |
-
|
| 1110 |
-
|
|
|
|
| 1111 |
app = _build_application(
|
| 1112 |
tenant, "loopable",
|
| 1113 |
{"name": "Routing", "templateKind": "route", "type": "route",
|
| 1114 |
-
"sourceDatabase": "customer_data",
|
| 1115 |
-
"fieldMap": {"title": "customer", "latitude": "lat", "longitude": "lon"},
|
| 1116 |
"navPlacement": {"section": "app", "order": 0}},
|
| 1117 |
field_contracts={"customer_data": [
|
| 1118 |
{"key": "customer", "type": "text"}, {"key": "lat", "type": "text"},
|
|
@@ -1130,6 +1231,96 @@ def _ensure_route_seed(runtime, tenant):
|
|
| 1130 |
return result
|
| 1131 |
|
| 1132 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1133 |
_COPIES = "userAppCopies"
|
| 1134 |
_APP_INDEX_KEY = "module_index_app"
|
| 1135 |
|
|
@@ -1331,7 +1522,7 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, fie
|
|
| 1331 |
doc=_clean_doc(raw); app=doc["applications"].get(app_id)
|
| 1332 |
if not app: raise ApplicationError("application does not exist")
|
| 1333 |
if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
|
| 1334 |
-
if not
|
| 1335 |
raise ApplicationError("application does not exist")
|
| 1336 |
if (app.get("templateKind") in APPLICATION_TEMPLATE_KINDS + GATED_TEMPLATE_KINDS
|
| 1337 |
and "compatibleDatabases" in patch):
|
|
@@ -1366,14 +1557,9 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, fie
|
|
| 1366 |
_refuse_legacy_catalog_changes(app, config)
|
| 1367 |
app["config"] = config
|
| 1368 |
if "fieldMap" in patch:
|
| 1369 |
-
|
| 1370 |
-
|
| 1371 |
-
|
| 1372 |
-
definition = _template_definition(
|
| 1373 |
-
kind, {**app, "fieldMap": patch["fieldMap"]},
|
| 1374 |
-
semantic_records=semantic_records, field_contracts=field_contracts,
|
| 1375 |
-
allowed_databases=allowed_databases)
|
| 1376 |
-
app["fieldMap"] = definition["fieldMap"]
|
| 1377 |
compatible = (_compat(patch["compatibleDatabases"], allow_empty=app["type"] == "code")
|
| 1378 |
if "compatibleDatabases" in patch else app.get("compatibleDatabases") or [])
|
| 1379 |
if _allowed(compatible, allowed_databases) != set(compatible):
|
|
@@ -1419,13 +1605,16 @@ def duplicate(runtime, app_id, tenant, user, is_admin=False, *, semantic_records
|
|
| 1419 |
app=load(runtime)["applications"].get(app_id)
|
| 1420 |
if not may_read(app,user,is_admin): raise ApplicationError("application does not exist")
|
| 1421 |
if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
|
| 1422 |
-
if not
|
| 1423 |
raise ApplicationError("application does not exist")
|
| 1424 |
payload={"type":app["type"],"name":f"{app['name']} copy",
|
| 1425 |
"compatibleDatabases":app.get("compatibleDatabases"),
|
| 1426 |
"semanticRoleBindings":app.get("semanticRoleBindings"),"config":app.get("config")}
|
| 1427 |
-
|
|
|
|
| 1428 |
if key in app: payload[key]=copy.deepcopy(app[key])
|
|
|
|
|
|
|
| 1429 |
return create(runtime,tenant,user,payload,
|
| 1430 |
semantic_records=semantic_records, field_contracts=field_contracts,
|
| 1431 |
allowed_databases=allowed_databases, via=via, on_behalf_of=on_behalf_of,
|
|
@@ -1435,11 +1624,10 @@ def duplicate(runtime, app_id, tenant, user, is_admin=False, *, semantic_records
|
|
| 1435 |
_ANY_NAME = object()
|
| 1436 |
|
| 1437 |
|
| 1438 |
-
def
|
| 1439 |
-
"""
|
| 1440 |
|
| 1441 |
-
|
| 1442 |
-
Returns the number of Views changed. Mutates `doc` in place.
|
| 1443 |
"""
|
| 1444 |
def nested_views(rows):
|
| 1445 |
for row in (rows.values() if isinstance(rows, dict) else rows if isinstance(rows, list) else ()):
|
|
@@ -1448,20 +1636,38 @@ def _strip_view_refs(doc, app_id):
|
|
| 1448 |
elif isinstance(row, (dict, list)):
|
| 1449 |
yield from nested_views(row)
|
| 1450 |
|
| 1451 |
-
|
| 1452 |
-
|
| 1453 |
-
|
| 1454 |
-
|
| 1455 |
-
|
| 1456 |
-
|
|
|
|
| 1457 |
if isinstance(entry, dict) and isinstance(entry.get("view"), dict):
|
| 1458 |
yield entry["view"]
|
| 1459 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1460 |
cleared = 0
|
| 1461 |
-
|
| 1462 |
-
|
| 1463 |
-
continue
|
| 1464 |
-
for view in views_of(bucket):
|
| 1465 |
changed = False
|
| 1466 |
for key in ("config", "presentation"):
|
| 1467 |
config = view.get(key) if isinstance(view, dict) else None
|
|
@@ -1711,7 +1917,7 @@ def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None,
|
|
| 1711 |
doc=_clean_doc(raw); app=doc["applications"].get(app_id)
|
| 1712 |
if not app: raise ApplicationError("application does not exist")
|
| 1713 |
if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
|
| 1714 |
-
if not
|
| 1715 |
raise ApplicationError("application does not exist")
|
| 1716 |
if not may_edit(app,user,is_admin): raise PermissionError("application belongs to another user")
|
| 1717 |
if confirm_name is not _ANY_NAME and (
|
|
@@ -1732,6 +1938,20 @@ def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None,
|
|
| 1732 |
result["viewsCleared"] = cleared
|
| 1733 |
if failed:
|
| 1734 |
result["viewsNotCleared"] = "; ".join(failed)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1735 |
return result
|
| 1736 |
|
| 1737 |
|
|
|
|
| 8 |
from core import shares, write_execution
|
| 9 |
from core.request_timing import raise_if_resource_error
|
| 10 |
import app_rules
|
| 11 |
+
import app_templates as _app_templates
|
| 12 |
|
| 13 |
from modules.finance_apps import (finance_application as _finance_application,
|
| 14 |
finance_applications as _finance_applications,
|
|
|
|
| 20 |
# Finance descriptors remain available to the governed fact routes.
|
| 21 |
FINANCE_TEMPLATE_KIND = "finance"
|
| 22 |
_KEY = "applications_v1"
|
| 23 |
+
#: D-8392 (R58): the most live Apps one account may create. The cap counts per account, never per tenant.
|
| 24 |
_MAX = 500
|
| 25 |
_MAX_DISPLAY_BYTES = 512 * 1024
|
| 26 |
|
|
|
|
| 68 |
},
|
| 69 |
}
|
| 70 |
|
| 71 |
+
# C1 (S79-T01): every contract carries its catalogue row's wire metadata.
|
| 72 |
+
_C1_WIRE_KEYS = ("connector", "purpose", "process", "mark", "view")
|
| 73 |
+
# R33: Routing and Catalog read their C1 source themselves, so the wizard asks for no database.
|
| 74 |
+
_BUNDLED_SOURCE_LABELS = {"customer_data": "Odoo customers", "product_data": "Odoo products"}
|
| 75 |
+
for _kind, _contract in _TEMPLATE_CONTRACTS.items():
|
| 76 |
+
_contract.update({key: _app_templates.BY_KIND[_kind][key] for key in _C1_WIRE_KEYS})
|
| 77 |
+
_contract.update(sourceDatabase=_app_templates.source(_kind),
|
| 78 |
+
sourceLabel=_BUNDLED_SOURCE_LABELS[_app_templates.source(_kind)])
|
| 79 |
+
|
| 80 |
# Keep stored copy identities readable after a template stops accepting new Apps.
|
| 81 |
APPLICATION_WIZARD_TEMPLATE_KINDS = tuple(_TEMPLATE_CONTRACTS)
|
| 82 |
APPLICATION_TEMPLATE_KINDS = tuple(_TEMPLATE_CONTRACTS)
|
| 83 |
|
| 84 |
#: S78-T17 (C4, R20, R21): templates offered only to a tenant that passes the kind's gate.
|
| 85 |
#: They stay out of the ungated tuples above, which other gates pin to Routing and Catalog.
|
| 86 |
+
#: S79-T01 (C1): every connector-gated kind in `app_templates.TEMPLATES` is one of them.
|
| 87 |
BRAND_ANALYTICS_DATABASE = "ut_amzn_brand_analytics"
|
| 88 |
+
# Wire fields a C1 row does not carry, per gated kind.
|
| 89 |
+
_GATED_WIRE = {
|
| 90 |
"competitor_watch": {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
"fieldRoles": [
|
| 92 |
{"key": "term", "label": "Search term", "required": True, "types": []},
|
| 93 |
],
|
|
|
|
|
|
|
| 94 |
"sourceLabel": "Amazon brand analytics",
|
| 95 |
},
|
| 96 |
}
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _source_label(database):
|
| 100 |
+
"""The connector table's label, so a refusal never names a `ut_` key."""
|
| 101 |
+
try:
|
| 102 |
+
from core import connector_spec
|
| 103 |
+
return connector_spec.spec_for(database).label
|
| 104 |
+
except Exception: # noqa: BLE001
|
| 105 |
+
return None
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _gated_contract(row):
|
| 109 |
+
# The screen reads the kind's own routes; the source names its connector database.
|
| 110 |
+
contract = {"kind": row["kind"], "label": row["label"],
|
| 111 |
+
"applicationType": "map", "displayMode": "map", "fieldRoles": [],
|
| 112 |
+
"defaultPlacement": {"section": "app", "order": 0},
|
| 113 |
+
**{key: row[key] for key in _C1_WIRE_KEYS}}
|
| 114 |
+
if row["sources"]:
|
| 115 |
+
contract["sourceDatabase"] = row["sources"][0]
|
| 116 |
+
contract["sourceLabel"] = _source_label(row["sources"][0])
|
| 117 |
+
contract.update(copy.deepcopy(_GATED_WIRE.get(row["kind"], {})))
|
| 118 |
+
return contract
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
_GATED_TEMPLATE_CONTRACTS = {row["kind"]: _gated_contract(row) for row in _app_templates.TEMPLATES
|
| 122 |
+
if row["kind"] not in _TEMPLATE_CONTRACTS}
|
| 123 |
GATED_TEMPLATE_KINDS = tuple(_GATED_TEMPLATE_CONTRACTS)
|
| 124 |
|
| 125 |
|
|
|
|
| 172 |
return False
|
| 173 |
|
| 174 |
|
| 175 |
+
def _connector_gate(connector):
|
| 176 |
+
"""C1 gate: the reader has the kind's connector (F25: the connector key, never a `ut_` prefix)."""
|
| 177 |
+
def gate(session):
|
| 178 |
+
return _app_templates.connector_present(connector, session)
|
| 179 |
+
gate.connector = connector
|
| 180 |
+
return gate
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
# Competitor watch's gate already requires the Amazon connection; it adds brand analytics (R21).
|
| 184 |
+
_TEMPLATE_GATES = {kind: brand_analytics_offered if kind == "competitor_watch"
|
| 185 |
+
else _connector_gate(contract["connector"])
|
| 186 |
+
for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()}
|
| 187 |
+
#: S79-T02 (instr 20): Routing and Catalog are offered only to a reader with Odoo.
|
| 188 |
+
_BUNDLED_GATES = {kind: _connector_gate(contract["connector"])
|
| 189 |
+
for kind, contract in _TEMPLATE_CONTRACTS.items()}
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def template_offered(kind, session):
|
| 193 |
+
"""C1 offer predicate: the module imports, the router mounted and the kind's gate holds."""
|
| 194 |
+
gate = _TEMPLATE_GATES.get(kind) or _BUNDLED_GATES.get(kind)
|
| 195 |
+
if gate is None or not _app_templates.seams_ready(kind):
|
| 196 |
+
return False
|
| 197 |
+
try:
|
| 198 |
+
return bool(gate(session))
|
| 199 |
+
except Exception as exc: # noqa: BLE001
|
| 200 |
+
raise_if_resource_error(exc)
|
| 201 |
+
return False
|
| 202 |
|
| 203 |
|
| 204 |
class ApplicationError(ValueError):
|
| 205 |
pass
|
| 206 |
|
| 207 |
|
| 208 |
+
def account_capacity():
|
| 209 |
+
"""The per-account App cap. `AIOS_APP_CAPACITY` lowers it for one server, as a lane proof does."""
|
| 210 |
+
import os
|
| 211 |
+
raw = str(os.environ.get("AIOS_APP_CAPACITY") or "").strip()
|
| 212 |
+
return min(int(raw), _MAX) if raw.isdigit() else _MAX
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def capacity_refusal(doc, user):
|
| 216 |
+
"""D-8392: the sentence naming the cap when `user` already holds it, else None."""
|
| 217 |
+
cap = account_capacity()
|
| 218 |
+
held = sum(1 for app in (doc.get("applications") or {}).values()
|
| 219 |
+
if isinstance(app, dict) and app.get("createdBy") == user and app.get("retired") is not True)
|
| 220 |
+
if held < cap:
|
| 221 |
+
return None
|
| 222 |
+
return f"Your account can hold up to {cap} App{'' if cap == 1 else 's'}. Delete an App to add another."
|
| 223 |
+
|
| 224 |
+
|
| 225 |
class ApplicationRuleError(ApplicationError):
|
| 226 |
"""Expose an App rule refusal through every writer's existing error handler."""
|
| 227 |
code = "app_rule"
|
|
|
|
| 251 |
if kind == FINANCE_TEMPLATE_KIND:
|
| 252 |
key = str(app.get("financeKey") or "")
|
| 253 |
return f"{FINANCE_TEMPLATE_KIND}:{key}" if key else None
|
| 254 |
+
# S79-T05: a connector-gated copy is grouped too, so the App directory can show Added.
|
| 255 |
+
return kind if kind in APPLICATION_WIZARD_TEMPLATE_KINDS + GATED_TEMPLATE_KINDS else None
|
| 256 |
|
| 257 |
|
| 258 |
def _own_template_app(applications, user, identity):
|
|
|
|
| 284 |
def template_contracts(session=None):
|
| 285 |
"""The trusted renderer contracts offered by the Application creation flow.
|
| 286 |
|
| 287 |
+
With a session, every template is offered only when the reader has its connector (instr 20).
|
| 288 |
"""
|
| 289 |
+
if session is None:
|
| 290 |
+
return _ungated_template_contracts()
|
| 291 |
+
have = _app_templates.connectors_present(session)
|
| 292 |
+
rows = [copy.deepcopy(contract) for contract in _TEMPLATE_CONTRACTS.values()
|
| 293 |
+
if contract["connector"] in have]
|
| 294 |
+
rows.extend(copy.deepcopy(contract) for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()
|
| 295 |
+
if template_offered(kind, session))
|
| 296 |
return rows
|
| 297 |
|
| 298 |
|
|
|
|
| 445 |
|
| 446 |
|
| 447 |
def load(runtime):
|
| 448 |
+
"""The read door: a Routing or Catalog App carries its C1 binding (`app_templates.bound`)."""
|
| 449 |
+
try: doc = _clean_doc(runtime.get(_KEY))
|
| 450 |
except Exception: return blank_doc()
|
| 451 |
+
doc["applications"] = {key: _app_templates.bound(app)
|
| 452 |
+
for key, app in doc["applications"].items()}
|
| 453 |
+
return doc
|
| 454 |
|
| 455 |
|
| 456 |
def _server_origin(origin_door=None, via=None):
|
|
|
|
| 519 |
and not app.get("compatibleDatabases"))
|
| 520 |
|
| 521 |
|
| 522 |
+
def _database_free(app):
|
| 523 |
+
"""A code App, or an App whose C1 kind names no source, opens without a database binding."""
|
| 524 |
+
if _database_free_code(app):
|
| 525 |
+
return True
|
| 526 |
+
row = _app_templates.BY_KIND.get(str(app.get("templateKind") or "")) if isinstance(app, dict) else None
|
| 527 |
+
return bool(row and not row["sources"] and not app.get("compatibleDatabases"))
|
| 528 |
+
|
| 529 |
+
|
| 530 |
def _finance_permission(app):
|
| 531 |
try:
|
| 532 |
return _finance_application(str(app.get("financeKey") or ""))["permissionKey"]
|
|
|
|
| 802 |
|
| 803 |
def _template_definition(kind, payload, *, semantic_records=None, field_contracts=None,
|
| 804 |
allowed_databases=None):
|
| 805 |
+
"""R33: a template App binds its C1 source and fixed field map; the caller names neither."""
|
| 806 |
contract = _TEMPLATE_CONTRACTS.get(kind) or _GATED_TEMPLATE_CONTRACTS.get(kind)
|
| 807 |
if contract is None:
|
| 808 |
raise ApplicationError("application template kind is not recognized")
|
| 809 |
template_kind = str(payload.get("templateKind") or "").strip().lower()
|
| 810 |
if template_kind != kind:
|
| 811 |
raise ApplicationError(f"{kind} applications require templateKind {kind!r}")
|
| 812 |
+
named = str(payload.get("sourceDatabase") or "").strip()[:120]
|
| 813 |
+
source = _app_templates.source(kind) or named
|
| 814 |
+
# S79-T05: a C1 kind whose `sources` is empty (amazon_operator) reads its own routes, no database.
|
| 815 |
+
sourceless = kind in _app_templates.BY_KIND and not _app_templates.BY_KIND[kind]["sources"]
|
| 816 |
+
if sourceless:
|
| 817 |
+
if named or payload.get("compatibleDatabases"):
|
| 818 |
+
raise ApplicationError(f"{contract['label']} reads no database. Remove the source database.")
|
| 819 |
+
elif not source:
|
| 820 |
raise ApplicationError("sourceDatabase is required")
|
| 821 |
+
elif named and named != source:
|
| 822 |
raise ApplicationError(f"{contract['label']} reads "
|
| 823 |
+
f"{contract.get('sourceLabel') or source} only. "
|
| 824 |
"Choose it as the source database.")
|
| 825 |
+
elif source not in set(allowed_databases or ()):
|
| 826 |
raise PermissionError("an Application cannot bind a database you cannot open")
|
| 827 |
+
if not sourceless and "compatibleDatabases" in payload:
|
| 828 |
compatible = _compat(payload.get("compatibleDatabases"))
|
| 829 |
if compatible != [source]:
|
| 830 |
raise ApplicationError("a template Application has exactly one sourceDatabase")
|
| 831 |
+
# A legacy caller may still send a field map: the fixed binding replaces it.
|
| 832 |
+
binding = {} if sourceless else _app_templates.field_binding(kind)
|
| 833 |
+
if binding:
|
| 834 |
+
available = _contract_fields(source, semantic_records, field_contracts)
|
| 835 |
+
if not available:
|
| 836 |
+
raise ApplicationError("the source database has no readable field contract")
|
| 837 |
+
unreadable = sorted(role["label"] for role in contract["fieldRoles"]
|
| 838 |
+
if role["required"] and binding.get(role["key"]) not in available)
|
| 839 |
+
if unreadable:
|
| 840 |
+
raise ApplicationError(f"{contract['label']} needs fields your account cannot read: "
|
| 841 |
+
+ ", ".join(unreadable) + ". Ask an administrator for access.")
|
| 842 |
+
|
| 843 |
+
raw_placement = payload.get("navPlacement") or contract["defaultPlacement"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 844 |
if not isinstance(raw_placement, dict) or set(raw_placement) != {"section", "order"}:
|
| 845 |
raise ApplicationError("navPlacement must contain section and order")
|
| 846 |
section = str(raw_placement.get("section") or "").strip().lower()
|
|
|
|
| 849 |
raise ApplicationError("template Applications must be placed in App")
|
| 850 |
if isinstance(order, bool) or not isinstance(order, int) or not 0 <= order <= 999:
|
| 851 |
raise ApplicationError("navPlacement order must be an integer from 0 to 999")
|
| 852 |
+
definition = {"templateKind": kind, "compatibleDatabases": [] if sourceless else [source],
|
| 853 |
+
"navPlacement": {"section": "app", "order": order}}
|
| 854 |
+
if kind not in _app_templates.BOUND_KINDS and not sourceless:
|
| 855 |
+
definition.update(sourceDatabase=source, fieldMap=binding)
|
| 856 |
+
return definition
|
| 857 |
+
|
| 858 |
+
|
| 859 |
+
def bound_application(app, available):
|
| 860 |
+
"""The App with its field map read against the caller's `{field: type}` schema.
|
| 861 |
+
|
| 862 |
+
A C1 binding drops an optional role the caller cannot read. A stored map stays strict.
|
| 863 |
+
"""
|
| 864 |
+
stored = bool(isinstance(app, dict) and app.get("fieldMap"))
|
| 865 |
+
row = _app_templates.bound(app)
|
| 866 |
+
contract = _template_contract(str(row.get("templateKind") or "")) or {}
|
| 867 |
+
roles = {role["key"]: role for role in contract.get("fieldRoles") or ()}
|
| 868 |
+
resolved = {}
|
| 869 |
+
for role, field in (row.get("fieldMap") or {}).items():
|
| 870 |
+
spec = roles.get(role)
|
| 871 |
+
if spec is None:
|
| 872 |
+
raise ApplicationError(f"fieldMap role {role!r} is outside the template contract")
|
| 873 |
+
types = set(spec.get("types") or ())
|
| 874 |
+
if field in available and (not types or available[field] in types):
|
| 875 |
+
resolved[role] = field
|
| 876 |
+
elif stored or spec["required"]:
|
| 877 |
+
raise ApplicationError(f"fieldMap role {role!r} does not name a readable source field")
|
| 878 |
+
return {**row, "fieldMap": resolved}
|
| 879 |
|
| 880 |
|
| 881 |
def validate_application(payload, tenant, *, semantic_records=None, field_contracts=None,
|
|
|
|
| 925 |
|
| 926 |
def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
|
| 927 |
is_default=False):
|
| 928 |
+
row=copy.deepcopy(_app_templates.bound(app))
|
| 929 |
row.setdefault("origin", None)
|
| 930 |
if row.get("type") == "code":
|
| 931 |
# Only the authenticated code door returns the body.
|
|
|
|
| 939 |
"default":bool(is_admin)}
|
| 940 |
row["starred"] = user in (app.get("starredBy") or ())
|
| 941 |
row["isDefault"] = bool(is_default)
|
| 942 |
+
catalogue = _app_templates.BY_KIND.get(str(row.get("templateKind") or ""))
|
| 943 |
+
if catalogue and catalogue["view"]:
|
| 944 |
+
# C2: the client mounts apps/<templateView>/AppView.tsx and fills the two marks.
|
| 945 |
+
row.update(templateView=catalogue["view"], templateMark=catalogue["mark"],
|
| 946 |
+
templateConnector=catalogue["connector"])
|
| 947 |
if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
|
| 948 |
# A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
|
| 949 |
row["capabilities"]["default"] = False
|
|
|
|
| 964 |
"""S67-T39 / R36: {app_id: number of saved Views on `database` bound to it}, over EVERY
|
| 965 |
user's Views in that database's workspace. The Delete App dialog names this count before
|
| 966 |
confirm. None when the workspace cannot be read, so the dialog never states a guessed 0.
|
| 967 |
+
|
| 968 |
+
D-2233: it walks the Views `delete` clears, nested and `presentation` ones too, trash excepted.
|
| 969 |
"""
|
| 970 |
from core.view_templates import workspace_key
|
| 971 |
key = workspace_key(database)
|
|
|
|
| 976 |
except Exception:
|
| 977 |
return None
|
| 978 |
counts = {}
|
| 979 |
+
users = {member: bucket for member, bucket in (doc.items() if isinstance(doc, dict) else ())
|
| 980 |
+
if member not in _RESERVED_WORKSPACE_MEMBERS}
|
| 981 |
+
for view in _workspace_views(users, trash=False):
|
| 982 |
+
for app_id in _view_app_ids(view):
|
| 983 |
+
counts[app_id] = counts.get(app_id, 0) + 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 984 |
return counts
|
| 985 |
|
| 986 |
|
|
|
|
| 997 |
if not may_read(app,user,is_admin): continue
|
| 998 |
if database not in (app.get("compatibleDatabases") or ()): continue
|
| 999 |
elif not _listed(app, user): continue
|
| 1000 |
+
if (not _database_free(app)
|
| 1001 |
and not _allowed(app.get("compatibleDatabases"), allowed_databases)): continue
|
| 1002 |
if kind and app.get("type") != kind: continue
|
| 1003 |
if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
|
|
|
|
| 1044 |
name=" ".join(str(payload.get("name") or "").split())[:120]
|
| 1045 |
if not name: raise ApplicationError("application name is required")
|
| 1046 |
app_id=f"app_{write_execution.new_id()[:16]}"; now=write_execution.utcnow().isoformat()
|
| 1047 |
+
compatible = (list(definition["compatibleDatabases"]) if definition
|
| 1048 |
else _compat(payload.get("compatibleDatabases"), allow_empty=kind == "code"))
|
| 1049 |
if _allowed(compatible, allowed_databases) != set(compatible):
|
| 1050 |
raise PermissionError("an Application cannot bind a database you cannot open")
|
|
|
|
| 1132 |
identity = template_identity(app)
|
| 1133 |
def apply(raw):
|
| 1134 |
doc=_clean_doc(raw)
|
| 1135 |
+
refusal = capacity_refusal(doc, user)
|
| 1136 |
+
if refusal: raise ApplicationError(refusal)
|
| 1137 |
# R216 and R247 allow every template to have several independent copies.
|
| 1138 |
doc["applications"][app_id]=app; return doc
|
| 1139 |
_update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of)
|
|
|
|
| 1164 |
"""
|
| 1165 |
result = _ensure_route_seed(runtime, tenant)
|
| 1166 |
copies = ensure_user_app_copies(runtime, tenant, accounts=accounts, wall=wall)
|
| 1167 |
+
bindings = migrate_c1_bindings(runtime, tenant)
|
| 1168 |
folded = dict(result)
|
| 1169 |
folded["userCopies"] = copies
|
| 1170 |
+
folded["c1Bindings"] = bindings
|
| 1171 |
note = "user copies " + copies["outcome"] + (
|
| 1172 |
": " + copies["reason"] if copies.get("reason") else "")
|
| 1173 |
+
note += "; C1 bindings " + bindings["outcome"] + (
|
| 1174 |
+
": " + bindings["reason"] if bindings.get("reason") else "")
|
| 1175 |
folded["reason"] = result["reason"] + "; " + note if result.get("reason") else note
|
| 1176 |
return folded
|
| 1177 |
|
|
|
|
| 1208 |
app_id = existing.get("id") if existing is not None else None
|
| 1209 |
outcome = "existing" if existing is not None else "no_routes"
|
| 1210 |
if existing is None and route_count:
|
| 1211 |
+
refusal = capacity_refusal(doc, "loopable")
|
| 1212 |
+
if refusal:
|
| 1213 |
+
raise ApplicationError(refusal)
|
| 1214 |
app = _build_application(
|
| 1215 |
tenant, "loopable",
|
| 1216 |
{"name": "Routing", "templateKind": "route", "type": "route",
|
|
|
|
|
|
|
| 1217 |
"navPlacement": {"section": "app", "order": 0}},
|
| 1218 |
field_contracts={"customer_data": [
|
| 1219 |
{"key": "customer", "type": "text"}, {"key": "lat", "type": "text"},
|
|
|
|
| 1231 |
return result
|
| 1232 |
|
| 1233 |
|
| 1234 |
+
#: Marker of the R33 migration: the binding each rewritten App held before, and the Apps it kept.
|
| 1235 |
+
C1_BINDINGS_MARKER = "c1Bindings"
|
| 1236 |
+
|
| 1237 |
+
|
| 1238 |
+
def _c1_rewrite(app):
|
| 1239 |
+
"""A Routing or Catalog App rewritten to its C1 binding, or None when it must stay as stored.
|
| 1240 |
+
|
| 1241 |
+
An App whose source or field map differs from C1 (`own_binding` non-empty) is never rewritten.
|
| 1242 |
+
"""
|
| 1243 |
+
if not isinstance(app, dict) or app.get("templateKind") not in _app_templates.BOUND_KINDS:
|
| 1244 |
+
return None
|
| 1245 |
+
if "sourceDatabase" not in app and "fieldMap" not in app:
|
| 1246 |
+
return None
|
| 1247 |
+
if _app_templates.own_binding(app):
|
| 1248 |
+
return None
|
| 1249 |
+
source = _app_templates.source(app["templateKind"])
|
| 1250 |
+
row = copy.deepcopy(app)
|
| 1251 |
+
row.pop("sourceDatabase", None)
|
| 1252 |
+
row.pop("fieldMap", None)
|
| 1253 |
+
row["compatibleDatabases"] = [source, *(database for database in app.get("compatibleDatabases") or ()
|
| 1254 |
+
if database != source)]
|
| 1255 |
+
return row
|
| 1256 |
+
|
| 1257 |
+
|
| 1258 |
+
def _c1_plan(doc, app_ids):
|
| 1259 |
+
"""`{id: rewritten App}` plus the stored Apps that keep a binding of their own, for one document."""
|
| 1260 |
+
rewrites, kept = {}, []
|
| 1261 |
+
for app_id, app in sorted(doc["applications"].items()):
|
| 1262 |
+
if app_ids is not None and app_id not in app_ids:
|
| 1263 |
+
continue
|
| 1264 |
+
if not isinstance(app, dict) or app.get("templateKind") not in _app_templates.BOUND_KINDS:
|
| 1265 |
+
continue
|
| 1266 |
+
rewritten = _c1_rewrite(app)
|
| 1267 |
+
if rewritten is not None:
|
| 1268 |
+
rewrites[app_id] = rewritten
|
| 1269 |
+
elif _app_templates.own_binding(app):
|
| 1270 |
+
kept.append({"id": app_id, "templateKind": app["templateKind"], "name": app.get("name"),
|
| 1271 |
+
"sourceDatabase": app.get("sourceDatabase"),
|
| 1272 |
+
"fieldMap": copy.deepcopy(app.get("fieldMap"))})
|
| 1273 |
+
return rewrites, kept
|
| 1274 |
+
|
| 1275 |
+
|
| 1276 |
+
def migrate_c1_bindings(runtime, tenant, *, app_ids=None):
|
| 1277 |
+
"""R33: rewrite every stored Routing and Catalog App to its C1 binding.
|
| 1278 |
+
|
| 1279 |
+
Writes only when an App still stores a binding equal to C1, so a second run changes nothing.
|
| 1280 |
+
An App with its own source or field map is kept and listed. A proof run
|
| 1281 |
+
passes `app_ids` to touch only the named Apps; the boot call never passes it.
|
| 1282 |
+
"""
|
| 1283 |
+
scope = None if app_ids is None else set(app_ids)
|
| 1284 |
+
try:
|
| 1285 |
+
doc = _clean_doc(_read_strict(runtime, _KEY))
|
| 1286 |
+
except Exception as exc:
|
| 1287 |
+
return {"outcome": "skipped", "reason": f"registry read failed: {type(exc).__name__}: {exc}",
|
| 1288 |
+
"rewritten": [], "kept": []}
|
| 1289 |
+
rewrites, kept = _c1_plan(doc, scope)
|
| 1290 |
+
result = {"outcome": "unchanged", "rewritten": [], "kept": kept}
|
| 1291 |
+
if rewrites:
|
| 1292 |
+
def apply(raw):
|
| 1293 |
+
current = _clean_doc(raw)
|
| 1294 |
+
planned, held = _c1_plan(current, scope)
|
| 1295 |
+
result["kept"] = held
|
| 1296 |
+
if not planned:
|
| 1297 |
+
return raw
|
| 1298 |
+
marker = current.setdefault("migrations", {}).setdefault(
|
| 1299 |
+
C1_BINDINGS_MARKER, {"rewritten": {}, "kept": []})
|
| 1300 |
+
for app_id, app in planned.items():
|
| 1301 |
+
before = current["applications"][app_id]
|
| 1302 |
+
marker["rewritten"][app_id] = {key: copy.deepcopy(before[key]) for key in
|
| 1303 |
+
("sourceDatabase", "fieldMap", "compatibleDatabases")
|
| 1304 |
+
if key in before}
|
| 1305 |
+
current["applications"][app_id] = app
|
| 1306 |
+
marker["kept"] = sorted({row["id"] for row in held} | set(marker.get("kept") or ()))
|
| 1307 |
+
marker["at"] = write_execution.utcnow().isoformat()
|
| 1308 |
+
result.update(outcome="migrated", rewritten=sorted(planned))
|
| 1309 |
+
return current
|
| 1310 |
+
_update_document(runtime, apply)
|
| 1311 |
+
if result["rewritten"]:
|
| 1312 |
+
try:
|
| 1313 |
+
from catalog_store import migrate_snapshots
|
| 1314 |
+
result["catalogSnapshots"] = migrate_snapshots(runtime, result["rewritten"])
|
| 1315 |
+
except Exception as exc:
|
| 1316 |
+
result["catalogSnapshots"] = f"skipped: {type(exc).__name__}"
|
| 1317 |
+
parts = [f"{len(result['rewritten'])} rewritten"] if result["rewritten"] else []
|
| 1318 |
+
if result["kept"]:
|
| 1319 |
+
parts.append("kept with their own binding: " + ", ".join(row["id"] for row in result["kept"]))
|
| 1320 |
+
result["reason"] = "; ".join(parts)
|
| 1321 |
+
return result
|
| 1322 |
+
|
| 1323 |
+
|
| 1324 |
_COPIES = "userAppCopies"
|
| 1325 |
_APP_INDEX_KEY = "module_index_app"
|
| 1326 |
|
|
|
|
| 1522 |
doc=_clean_doc(raw); app=doc["applications"].get(app_id)
|
| 1523 |
if not app: raise ApplicationError("application does not exist")
|
| 1524 |
if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
|
| 1525 |
+
if not _database_free(app):
|
| 1526 |
raise ApplicationError("application does not exist")
|
| 1527 |
if (app.get("templateKind") in APPLICATION_TEMPLATE_KINDS + GATED_TEMPLATE_KINDS
|
| 1528 |
and "compatibleDatabases" in patch):
|
|
|
|
| 1557 |
_refuse_legacy_catalog_changes(app, config)
|
| 1558 |
app["config"] = config
|
| 1559 |
if "fieldMap" in patch:
|
| 1560 |
+
# R33: a template App reads the fields its template names.
|
| 1561 |
+
raise ApplicationError("This App reads the fields its template names. "
|
| 1562 |
+
"Its field map cannot change.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1563 |
compatible = (_compat(patch["compatibleDatabases"], allow_empty=app["type"] == "code")
|
| 1564 |
if "compatibleDatabases" in patch else app.get("compatibleDatabases") or [])
|
| 1565 |
if _allowed(compatible, allowed_databases) != set(compatible):
|
|
|
|
| 1605 |
app=load(runtime)["applications"].get(app_id)
|
| 1606 |
if not may_read(app,user,is_admin): raise ApplicationError("application does not exist")
|
| 1607 |
if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
|
| 1608 |
+
if not _database_free(app):
|
| 1609 |
raise ApplicationError("application does not exist")
|
| 1610 |
payload={"type":app["type"],"name":f"{app['name']} copy",
|
| 1611 |
"compatibleDatabases":app.get("compatibleDatabases"),
|
| 1612 |
"semanticRoleBindings":app.get("semanticRoleBindings"),"config":app.get("config")}
|
| 1613 |
+
# R33: a template copy binds C1 itself, so neither the source nor the field map is copied.
|
| 1614 |
+
for key in ("templateKind", "navPlacement", "financeKey"):
|
| 1615 |
if key in app: payload[key]=copy.deepcopy(app[key])
|
| 1616 |
+
if app.get("templateKind") not in _app_templates.BOUND_KINDS and "sourceDatabase" in app:
|
| 1617 |
+
payload["sourceDatabase"] = app["sourceDatabase"]
|
| 1618 |
return create(runtime,tenant,user,payload,
|
| 1619 |
semantic_records=semantic_records, field_contracts=field_contracts,
|
| 1620 |
allowed_databases=allowed_databases, via=via, on_behalf_of=on_behalf_of,
|
|
|
|
| 1624 |
_ANY_NAME = object()
|
| 1625 |
|
| 1626 |
|
| 1627 |
+
def _workspace_views(doc, *, trash=True):
|
| 1628 |
+
"""Every View in one workspace document: each member's Views, reserved members included.
|
| 1629 |
|
| 1630 |
+
Descends into nested folders. With `trash`, each owner's trash entries are walked too.
|
|
|
|
| 1631 |
"""
|
| 1632 |
def nested_views(rows):
|
| 1633 |
for row in (rows.values() if isinstance(rows, dict) else rows if isinstance(rows, list) else ()):
|
|
|
|
| 1636 |
elif isinstance(row, (dict, list)):
|
| 1637 |
yield from nested_views(row)
|
| 1638 |
|
| 1639 |
+
for bucket in (doc.values() if isinstance(doc, dict) else ()):
|
| 1640 |
+
if not isinstance(bucket, dict):
|
| 1641 |
+
continue
|
| 1642 |
+
yield from nested_views(bucket.get("views"))
|
| 1643 |
+
held = bucket.get("trash") if trash else None
|
| 1644 |
+
for entry in (held.values() if isinstance(held, dict)
|
| 1645 |
+
else held if isinstance(held, list) else ()):
|
| 1646 |
if isinstance(entry, dict) and isinstance(entry.get("view"), dict):
|
| 1647 |
yield entry["view"]
|
| 1648 |
|
| 1649 |
+
|
| 1650 |
+
def _view_app_ids(view):
|
| 1651 |
+
"""The distinct App ids a View's `config` or `presentation` names."""
|
| 1652 |
+
ids = []
|
| 1653 |
+
for key in ("config", "presentation"):
|
| 1654 |
+
config = view.get(key) if isinstance(view, dict) else None
|
| 1655 |
+
ref = config.get("application") if isinstance(config, dict) else None
|
| 1656 |
+
app_id = str(ref.get("id") or "") if isinstance(ref, dict) else ""
|
| 1657 |
+
if app_id and app_id not in ids:
|
| 1658 |
+
ids.append(app_id)
|
| 1659 |
+
return ids
|
| 1660 |
+
|
| 1661 |
+
|
| 1662 |
+
def _strip_view_refs(doc, app_id):
|
| 1663 |
+
"""Remove `config.application` naming `app_id` from every View in one workspace document.
|
| 1664 |
+
|
| 1665 |
+
Walks every top-level member, reserved ones included, and each owner's trash entries.
|
| 1666 |
+
Returns the number of Views changed. Mutates `doc` in place.
|
| 1667 |
+
"""
|
| 1668 |
cleared = 0
|
| 1669 |
+
if isinstance(doc, dict):
|
| 1670 |
+
for view in _workspace_views(doc):
|
|
|
|
|
|
|
| 1671 |
changed = False
|
| 1672 |
for key in ("config", "presentation"):
|
| 1673 |
config = view.get(key) if isinstance(view, dict) else None
|
|
|
|
| 1917 |
doc=_clean_doc(raw); app=doc["applications"].get(app_id)
|
| 1918 |
if not app: raise ApplicationError("application does not exist")
|
| 1919 |
if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
|
| 1920 |
+
if not _database_free(app):
|
| 1921 |
raise ApplicationError("application does not exist")
|
| 1922 |
if not may_edit(app,user,is_admin): raise PermissionError("application belongs to another user")
|
| 1923 |
if confirm_name is not _ANY_NAME and (
|
|
|
|
| 1938 |
result["viewsCleared"] = cleared
|
| 1939 |
if failed:
|
| 1940 |
result["viewsNotCleared"] = "; ".join(failed)
|
| 1941 |
+
kind = removed.get("templateKind")
|
| 1942 |
+
contract = _app_templates.BY_KIND.get(kind)
|
| 1943 |
+
if contract and contract.get("module"):
|
| 1944 |
+
try:
|
| 1945 |
+
if not any(row.get("templateKind") == kind for row in load(runtime)["applications"].values()):
|
| 1946 |
+
from importlib import import_module
|
| 1947 |
+
module = import_module(contract["module"])
|
| 1948 |
+
cleanup = next((getattr(module, name) for name in
|
| 1949 |
+
("remove_app_data", "on_app_removed", "remove")
|
| 1950 |
+
if callable(getattr(module, name, None))), None)
|
| 1951 |
+
if cleanup is not None:
|
| 1952 |
+
cleanup(removed.get("tenant") or runtime.key, st=runtime)
|
| 1953 |
+
except Exception as exc:
|
| 1954 |
+
result["appDataNotCleared"] = type(exc).__name__
|
| 1955 |
return result
|
| 1956 |
|
| 1957 |
|
api/assistant_stream.py
CHANGED
|
@@ -1,10 +1,10 @@
|
|
| 1 |
"""Expose one Query turn as server-sent events with same-turn error recovery."""
|
| 2 |
import asyncio
|
| 3 |
import contextvars
|
| 4 |
-
import
|
| 5 |
-
import io
|
| 6 |
import json
|
| 7 |
import threading
|
|
|
|
| 8 |
|
| 9 |
import anyio
|
| 10 |
|
|
@@ -39,34 +39,24 @@ class StreamGZipMiddleware:
|
|
| 39 |
await responder(scope, receive, send)
|
| 40 |
|
| 41 |
|
| 42 |
-
from starlette.middleware.gzip import
|
| 43 |
|
| 44 |
|
| 45 |
-
class _OffLoopGZipResponder(
|
| 46 |
"""Retain Starlette's header and chunk behavior with compression outside the event loop."""
|
| 47 |
|
|
|
|
|
|
|
| 48 |
def __init__(self, app, minimum_size, compresslevel):
|
| 49 |
-
|
| 50 |
self.compresslevel = compresslevel
|
| 51 |
-
self.
|
| 52 |
-
|
| 53 |
-
async def __call__(self, scope, receive, send):
|
| 54 |
-
try:
|
| 55 |
-
await IdentityResponder.__call__(self, scope, receive, send)
|
| 56 |
-
finally:
|
| 57 |
-
if self.gzip_file is not None:
|
| 58 |
-
def close():
|
| 59 |
-
self.gzip_file.close()
|
| 60 |
-
self.gzip_buffer.close()
|
| 61 |
-
with anyio.CancelScope(shield=True):
|
| 62 |
-
await anyio.to_thread.run_sync(close)
|
| 63 |
|
| 64 |
def _compress(self, body, more):
|
| 65 |
-
if self.
|
| 66 |
-
self.
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
return GZipResponder.apply_compression(self, body, more_body=more)
|
| 70 |
|
| 71 |
async def send_with_compression(self, message):
|
| 72 |
if (message["type"] == "http.response.body"
|
|
@@ -77,8 +67,13 @@ class _OffLoopGZipResponder(GZipResponder):
|
|
| 77 |
lambda: self._compress(body, more))
|
| 78 |
await super().send_with_compression(message)
|
| 79 |
|
| 80 |
-
|
| 81 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
|
| 83 |
|
| 84 |
def _event(name, data):
|
|
|
|
| 1 |
"""Expose one Query turn as server-sent events with same-turn error recovery."""
|
| 2 |
import asyncio
|
| 3 |
import contextvars
|
| 4 |
+
import inspect
|
|
|
|
| 5 |
import json
|
| 6 |
import threading
|
| 7 |
+
import zlib
|
| 8 |
|
| 9 |
import anyio
|
| 10 |
|
|
|
|
| 39 |
await responder(scope, receive, send)
|
| 40 |
|
| 41 |
|
| 42 |
+
from starlette.middleware.gzip import IdentityResponder
|
| 43 |
|
| 44 |
|
| 45 |
+
class _OffLoopGZipResponder(IdentityResponder):
|
| 46 |
"""Retain Starlette's header and chunk behavior with compression outside the event loop."""
|
| 47 |
|
| 48 |
+
content_encoding = "gzip"
|
| 49 |
+
|
| 50 |
def __init__(self, app, minimum_size, compresslevel):
|
| 51 |
+
super().__init__(app, minimum_size)
|
| 52 |
self.compresslevel = compresslevel
|
| 53 |
+
self._compressor = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
def _compress(self, body, more):
|
| 56 |
+
if self._compressor is None:
|
| 57 |
+
self._compressor = zlib.compressobj(self.compresslevel, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
|
| 58 |
+
out = self._compressor.compress(body)
|
| 59 |
+
return out if more else out + self._compressor.flush()
|
|
|
|
| 60 |
|
| 61 |
async def send_with_compression(self, message):
|
| 62 |
if (message["type"] == "http.response.body"
|
|
|
|
| 67 |
lambda: self._compress(body, more))
|
| 68 |
await super().send_with_compression(message)
|
| 69 |
|
| 70 |
+
# Starlette 1.4 awaits apply_compression and 1.3 calls it, so match the installed base.
|
| 71 |
+
if inspect.iscoroutinefunction(IdentityResponder.apply_compression):
|
| 72 |
+
async def apply_compression(self, body, *, more_body):
|
| 73 |
+
return self._compressed_body
|
| 74 |
+
else:
|
| 75 |
+
def apply_compression(self, body, *, more_body):
|
| 76 |
+
return self._compressed_body
|
| 77 |
|
| 78 |
|
| 79 |
def _event(name, data):
|
api/catalog_store.py
CHANGED
|
@@ -7,6 +7,7 @@ import hashlib
|
|
| 7 |
import json
|
| 8 |
|
| 9 |
import app_rules
|
|
|
|
| 10 |
import application_registry as applications
|
| 11 |
from core import write_execution
|
| 12 |
from deps import err
|
|
@@ -116,7 +117,33 @@ def normalize_pages(pages, *, legacy=False, identity=""):
|
|
| 116 |
|
| 117 |
|
| 118 |
def _snapshot(app):
|
| 119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
|
| 122 |
def _insert(doc, record):
|
|
@@ -194,7 +221,8 @@ def require_app(session, app_id, *, write=False, allow_orphan=False):
|
|
| 194 |
may = applications.may_edit if write else applications.may_read
|
| 195 |
if not may(app, session.uname, session.admin):
|
| 196 |
raise err(403, "catalog_forbidden", "Your account cannot access this Catalog App. Ask its owner for access.")
|
| 197 |
-
|
|
|
|
| 198 |
_assert_database_scope(session, source)
|
| 199 |
session.require(source)
|
| 200 |
if not perm_scope.may_read(session.user, source, st=runtime):
|
|
@@ -202,12 +230,12 @@ def require_app(session, app_id, *, write=False, allow_orphan=False):
|
|
| 202 |
if write:
|
| 203 |
_assert_write_scope(session, "catalogs")
|
| 204 |
contracts = applications.chart_field_contracts(session, [source])
|
|
|
|
| 205 |
try:
|
| 206 |
-
applications.
|
| 207 |
-
allowed_databases={source})
|
| 208 |
except applications.ApplicationError:
|
| 209 |
raise err(403, "catalog_fields_forbidden", "The Catalog App binds unavailable fields. Ask its owner to rebind readable fields.") from None
|
| 210 |
-
return app,
|
| 211 |
|
| 212 |
|
| 213 |
def _binding_wall(record, app, fields):
|
|
|
|
| 7 |
import json
|
| 8 |
|
| 9 |
import app_rules
|
| 10 |
+
import app_templates
|
| 11 |
import application_registry as applications
|
| 12 |
from core import write_execution
|
| 13 |
from deps import err
|
|
|
|
| 117 |
|
| 118 |
|
| 119 |
def _snapshot(app):
|
| 120 |
+
# R33: a snapshot keeps a source or field map only when it differs from the C1 binding.
|
| 121 |
+
kept = _SNAPSHOT_KEYS - {"sourceDatabase", "fieldMap"}
|
| 122 |
+
row = {key: copy.deepcopy(value) for key, value in app.items() if key in kept}
|
| 123 |
+
row.update(copy.deepcopy(app_templates.own_binding(app)))
|
| 124 |
+
return row
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def migrate_snapshots(runtime, app_ids):
|
| 128 |
+
"""R33: drop a C1 binding from the named Apps' catalog snapshots. Writes only on a change."""
|
| 129 |
+
wanted = set(app_ids)
|
| 130 |
+
|
| 131 |
+
def stale(doc):
|
| 132 |
+
return sorted(app_id for app_id, row in doc["appSnapshots"].items()
|
| 133 |
+
if app_id in wanted and isinstance(row, dict)
|
| 134 |
+
and applications._c1_rewrite(row) is not None)
|
| 135 |
+
|
| 136 |
+
if not stale(_load(runtime)):
|
| 137 |
+
return "unchanged"
|
| 138 |
+
|
| 139 |
+
def apply(doc):
|
| 140 |
+
changed = stale(doc)
|
| 141 |
+
for app_id in changed:
|
| 142 |
+
row = applications._c1_rewrite(doc["appSnapshots"][app_id])
|
| 143 |
+
doc["appSnapshots"][app_id] = {key: value for key, value in row.items() if key in _SNAPSHOT_KEYS}
|
| 144 |
+
return changed
|
| 145 |
+
|
| 146 |
+
return f"{len(_update(runtime, apply))} rewritten"
|
| 147 |
|
| 148 |
|
| 149 |
def _insert(doc, record):
|
|
|
|
| 221 |
may = applications.may_edit if write else applications.may_read
|
| 222 |
if not may(app, session.uname, session.admin):
|
| 223 |
raise err(403, "catalog_forbidden", "Your account cannot access this Catalog App. Ask its owner for access.")
|
| 224 |
+
# R33: Catalog reads Odoo products through its C1 binding unless the record holds its own.
|
| 225 |
+
source = app_templates.bound(app).get("sourceDatabase")
|
| 226 |
_assert_database_scope(session, source)
|
| 227 |
session.require(source)
|
| 228 |
if not perm_scope.may_read(session.user, source, st=runtime):
|
|
|
|
| 230 |
if write:
|
| 231 |
_assert_write_scope(session, "catalogs")
|
| 232 |
contracts = applications.chart_field_contracts(session, [source])
|
| 233 |
+
fields = applications._contract_fields(source, field_contracts=contracts)
|
| 234 |
try:
|
| 235 |
+
app = applications.bound_application(app, fields)
|
|
|
|
| 236 |
except applications.ApplicationError:
|
| 237 |
raise err(403, "catalog_fields_forbidden", "The Catalog App binds unavailable fields. Ask its owner to rebind readable fields.") from None
|
| 238 |
+
return app, fields
|
| 239 |
|
| 240 |
|
| 241 |
def _binding_wall(record, app, fields):
|
api/connectors_amazon_ads.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""connectors_amazon_ads.py: the Amazon Ads connector's API layer (S79-T13): the probe and the card state.
|
| 2 |
+
|
| 3 |
+
The transport is `platform/harness/amazon_ads_client.py`. This module adds the liveness probe the
|
| 4 |
+
keychain calls through `register_prober`, and the card state the Connectors directory reads.
|
| 5 |
+
|
| 6 |
+
⛔ READ ONLY. No route here writes. The probe makes one Login with Amazon token exchange and one
|
| 7 |
+
profile listing GET, and only when an administrator tests a stored credential.
|
| 8 |
+
⛔ `AMAZON_ADS_FIXTURE=1` answers from the fixture tokens below and opens no socket.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import json
|
| 13 |
+
import sys
|
| 14 |
+
import urllib.request
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
_PLATFORM = Path(__file__).resolve().parents[2] / "platform"
|
| 18 |
+
if str(_PLATFORM) not in sys.path:
|
| 19 |
+
sys.path.insert(0, str(_PLATFORM))
|
| 20 |
+
|
| 21 |
+
from harness import amazon_ads_client as client # noqa: E402
|
| 22 |
+
|
| 23 |
+
KEY = client.KEY
|
| 24 |
+
LABEL = client.LABEL
|
| 25 |
+
#: Must be a `core.keychain.ENTRY_TYPES` member, or storing the credential 400s.
|
| 26 |
+
ENTRY_TYPE = client.ENTRY_TYPE
|
| 27 |
+
CREDENTIAL_FIELDS = client.CREDENTIAL_FIELDS
|
| 28 |
+
|
| 29 |
+
#: The one read the probe makes after the token exchange. GET only, one path.
|
| 30 |
+
PROFILES_PATH = "v2/profiles"
|
| 31 |
+
|
| 32 |
+
#: Fixture refresh tokens. A token with the prefix passes; the expired one fails.
|
| 33 |
+
FIXTURE_TOKEN_PREFIX = "Atzr|fixture-ads"
|
| 34 |
+
FIXTURE_REFRESH_TOKEN = FIXTURE_TOKEN_PREFIX + "-ok"
|
| 35 |
+
FIXTURE_EXPIRED_TOKEN = FIXTURE_TOKEN_PREFIX + "-expired"
|
| 36 |
+
FIXTURE_PROFILE_ID = "1234567890"
|
| 37 |
+
|
| 38 |
+
#: Messages a person reads. No dash, no provider text.
|
| 39 |
+
OK_MESSAGE = "Amazon Ads confirmed profile access."
|
| 40 |
+
EXPIRED_MESSAGE = "The Amazon Ads token expired. Connect again."
|
| 41 |
+
NO_PROFILE_MESSAGE = "The token does not reach this profile. Check the profile ID."
|
| 42 |
+
NOT_FIXTURE_MESSAGE = "Fixture mode accepts only the fixture token."
|
| 43 |
+
SIGN_IN_MESSAGE = "Amazon Ads did not accept the sign in. Check the fields and connect again."
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def fixture_fields(*, expired=False):
|
| 47 |
+
"""The fixture credential an administrator pastes on staging. -> the four fields."""
|
| 48 |
+
return {"client_id": "amzn1.application-oa2-client.fixture",
|
| 49 |
+
"client_secret": "fixture-ads-secret",
|
| 50 |
+
"refresh_token": FIXTURE_EXPIRED_TOKEN if expired else FIXTURE_REFRESH_TOKEN,
|
| 51 |
+
"profile_id": FIXTURE_PROFILE_ID}
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def missing_fields(fields):
|
| 55 |
+
"""The credential fields that are absent or blank, in declaration order."""
|
| 56 |
+
f = fields if isinstance(fields, dict) else {}
|
| 57 |
+
return [key for key in CREDENTIAL_FIELDS if not isinstance(f.get(key), str) or not f[key].strip()]
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _fixture_probe(fields):
|
| 61 |
+
token = fields["refresh_token"].strip()
|
| 62 |
+
if token == FIXTURE_EXPIRED_TOKEN:
|
| 63 |
+
return {"ok": False, "message": EXPIRED_MESSAGE}
|
| 64 |
+
if not token.startswith(FIXTURE_TOKEN_PREFIX):
|
| 65 |
+
return {"ok": False, "message": NOT_FIXTURE_MESSAGE}
|
| 66 |
+
return {"ok": True, "message": OK_MESSAGE}
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _profiles(access_token, fields, timeout):
|
| 70 |
+
"""The advertiser's profile ids. One GET; any failure raises `client.AdsError`."""
|
| 71 |
+
host = client.HOSTS.get(str(fields.get("region") or client.DEFAULT_REGION).lower(),
|
| 72 |
+
client.HOSTS[client.DEFAULT_REGION])
|
| 73 |
+
req = urllib.request.Request(f"{host}/{PROFILES_PATH}", method="GET", headers={
|
| 74 |
+
"Authorization": f"Bearer {access_token}",
|
| 75 |
+
"Amazon-Advertising-API-ClientId": fields["client_id"], "Accept": "application/json"})
|
| 76 |
+
try:
|
| 77 |
+
with urllib.request.urlopen(req, timeout=timeout) as r:
|
| 78 |
+
body = json.loads(r.read().decode("utf-8", "replace") or "[]")
|
| 79 |
+
except Exception as exc: # noqa: BLE001
|
| 80 |
+
raise client.AdsError(f"Amazon Ads profiles could not be read ({type(exc).__name__})") from None
|
| 81 |
+
return {str(p.get("profileId")) for p in body if isinstance(p, dict)} if isinstance(body, list) else set()
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def probe_credential(fields, timeout=8):
|
| 85 |
+
"""`{ok, message}` for a stored `amazon_ads` credential: the keychain's liveness probe."""
|
| 86 |
+
f = fields if isinstance(fields, dict) else {}
|
| 87 |
+
missing = missing_fields(f)
|
| 88 |
+
if missing:
|
| 89 |
+
return {"ok": False, "message": "Missing " + ", ".join(missing) + ". Enter every field."}
|
| 90 |
+
if client.fixture_mode():
|
| 91 |
+
return _fixture_probe(f)
|
| 92 |
+
try:
|
| 93 |
+
profiles = _profiles(client._token(f), f, timeout)
|
| 94 |
+
except client.AdsError:
|
| 95 |
+
return {"ok": False, "message": SIGN_IN_MESSAGE}
|
| 96 |
+
if f["profile_id"].strip() not in profiles:
|
| 97 |
+
return {"ok": False, "message": NO_PROFILE_MESSAGE}
|
| 98 |
+
return {"ok": True, "message": OK_MESSAGE}
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def register():
|
| 102 |
+
"""Fill the keychain's probe slot for `amazon_ads`. Never raises."""
|
| 103 |
+
try:
|
| 104 |
+
import core.keychain as kc
|
| 105 |
+
return kc.register_prober(ENTRY_TYPE, probe_credential)
|
| 106 |
+
except Exception: # noqa: BLE001
|
| 107 |
+
return False
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def held(rt, entry_id):
|
| 111 |
+
"""True when this entry's current ciphertext carries a passing probe proof. No decrypt, no I/O.
|
| 112 |
+
|
| 113 |
+
A failed test, an expired token or a replaced secret clears the proof, so the card reads Connect.
|
| 114 |
+
"""
|
| 115 |
+
try:
|
| 116 |
+
from core import keychain as kc
|
| 117 |
+
return bool(kc.entry_verified(rt, entry_id))
|
| 118 |
+
except Exception: # noqa: BLE001
|
| 119 |
+
return False
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def card_state(verified):
|
| 123 |
+
"""The directory state: `connected` (Settings) only with a verified credential, else `available`."""
|
| 124 |
+
return "connected" if verified else "available"
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
__all__ = ["KEY", "LABEL", "ENTRY_TYPE", "CREDENTIAL_FIELDS", "PROFILES_PATH", "FIXTURE_TOKEN_PREFIX",
|
| 128 |
+
"FIXTURE_REFRESH_TOKEN", "FIXTURE_EXPIRED_TOKEN", "FIXTURE_PROFILE_ID", "OK_MESSAGE",
|
| 129 |
+
"EXPIRED_MESSAGE", "fixture_fields", "missing_fields", "probe_credential", "register",
|
| 130 |
+
"held", "card_state"]
|
api/deps.py
CHANGED
|
@@ -1661,3 +1661,60 @@ def module_gate(module_key):
|
|
| 1661 |
session.require(module_key)
|
| 1662 |
return session
|
| 1663 |
return _dep
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1661 |
session.require(module_key)
|
| 1662 |
return session
|
| 1663 |
return _dep
|
| 1664 |
+
|
| 1665 |
+
|
| 1666 |
+
def _bearer_header(request: Request):
|
| 1667 |
+
value = request.headers.get("authorization")
|
| 1668 |
+
return value if isinstance(value, str) and value.strip() else None
|
| 1669 |
+
|
| 1670 |
+
|
| 1671 |
+
def _cookie_session(request: Request, response: Response) -> Session:
|
| 1672 |
+
"""`require_session`, or the app's override of it, so a gate's override still applies here."""
|
| 1673 |
+
app = request.scope.get("app")
|
| 1674 |
+
override = (getattr(app, "dependency_overrides", None) or {}).get(require_session)
|
| 1675 |
+
if override is None:
|
| 1676 |
+
return require_session(request, response)
|
| 1677 |
+
import inspect
|
| 1678 |
+
names = inspect.signature(override).parameters
|
| 1679 |
+
return override(**{key: value for key, value in (("request", request), ("response", response))
|
| 1680 |
+
if key in names})
|
| 1681 |
+
|
| 1682 |
+
|
| 1683 |
+
def require_session_or_bearer(request: Request, response: Response) -> Session:
|
| 1684 |
+
"""The cookie session when a cookie is sent, else an external bearer under its own key's grants.
|
| 1685 |
+
|
| 1686 |
+
`require_session` stays cookie only. A bearer session never reissues the cookie.
|
| 1687 |
+
"""
|
| 1688 |
+
bearer = _bearer_header(request)
|
| 1689 |
+
if bearer is None or request.cookies.get(aios_session.COOKIE_NAME):
|
| 1690 |
+
try:
|
| 1691 |
+
return _cookie_session(request, response)
|
| 1692 |
+
except HTTPException as exc:
|
| 1693 |
+
if bearer is None or exc.status_code != 401:
|
| 1694 |
+
raise
|
| 1695 |
+
import routes_external # routes_external imports deps, so this import stays lazy.
|
| 1696 |
+
return routes_external.require_external_analyst(
|
| 1697 |
+
routes_external.require_external_session(bearer, request))
|
| 1698 |
+
|
| 1699 |
+
|
| 1700 |
+
def is_bearer(session) -> bool:
|
| 1701 |
+
"""True for a session an external API key or OAuth token resolved."""
|
| 1702 |
+
return (getattr(session, "claims", None) or {}).get("external") is True
|
| 1703 |
+
|
| 1704 |
+
|
| 1705 |
+
def require_bearer_write(session, tool_name):
|
| 1706 |
+
"""Refuse a read-only or refresh-only credential on a write route. A cookie session passes."""
|
| 1707 |
+
if is_bearer(session):
|
| 1708 |
+
import routes_external
|
| 1709 |
+
routes_external._assert_write_scope(session, tool_name)
|
| 1710 |
+
|
| 1711 |
+
|
| 1712 |
+
def module_gate_or_bearer(module_key, database=None):
|
| 1713 |
+
"""`module_gate` that also admits a bearer, walled to its key's database scope."""
|
| 1714 |
+
def _dep(session: Session = Depends(require_session_or_bearer)) -> Session:
|
| 1715 |
+
session.require(module_key)
|
| 1716 |
+
if is_bearer(session):
|
| 1717 |
+
import routes_external
|
| 1718 |
+
routes_external._assert_database_scope(session, database or module_key)
|
| 1719 |
+
return session
|
| 1720 |
+
return _dep
|
api/error_sink.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Store secret-free exception groups and send one daily operations digest."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from datetime import date, datetime, timedelta, timezone
|
| 5 |
+
from functools import partial
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
import logging
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import re
|
| 11 |
+
import sys
|
| 12 |
+
from uuid import uuid4
|
| 13 |
+
|
| 14 |
+
LOG = logging.getLogger(__name__)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def utc_now():
|
| 18 |
+
return datetime.now(timezone.utc)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def store_backend():
|
| 22 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "platform"))
|
| 23 |
+
import core.store as store
|
| 24 |
+
return store
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def day_key(day):
|
| 28 |
+
return "ops_error_sink_" + date.fromisoformat(day).isoformat()
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def stack_hash(error):
|
| 32 |
+
frames = []
|
| 33 |
+
trace = error.__traceback__
|
| 34 |
+
while trace:
|
| 35 |
+
code = trace.tb_frame.f_code
|
| 36 |
+
frames.append((code.co_filename, code.co_name, trace.tb_lineno))
|
| 37 |
+
trace = trace.tb_next
|
| 38 |
+
signature = (type(error).__module__, type(error).__qualname__, frames)
|
| 39 |
+
return hashlib.sha256(json.dumps(signature).encode()).hexdigest()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def verified_tenant(scope):
|
| 43 |
+
from starlette.requests import Request
|
| 44 |
+
request = Request(scope)
|
| 45 |
+
authorization = request.headers.get("authorization")
|
| 46 |
+
if authorization:
|
| 47 |
+
external = sys.modules.get("api.routes_external") or sys.modules.get("routes_external")
|
| 48 |
+
if external is None:
|
| 49 |
+
return "unknown"
|
| 50 |
+
try:
|
| 51 |
+
value = external.require_external_session(authorization=authorization, request=request).tenant
|
| 52 |
+
except Exception:
|
| 53 |
+
return "unknown"
|
| 54 |
+
else:
|
| 55 |
+
import aios_session
|
| 56 |
+
claims = aios_session.read(request.cookies.get(aios_session.COOKIE_NAME, ""))
|
| 57 |
+
value = (claims or {}).get("t", "unknown")
|
| 58 |
+
return value if isinstance(value, str) and re.fullmatch(r"[a-z0-9][a-z0-9-]{0,63}", value) else "unknown"
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def record_error(scope, error, *, status=500, backend=None, now=None, tenant_resolver=None):
|
| 62 |
+
backend = backend if backend is not None else store_backend()
|
| 63 |
+
now = now or utc_now()
|
| 64 |
+
if now.utcoffset() is None:
|
| 65 |
+
raise ValueError("error timestamps must include a timezone")
|
| 66 |
+
now = now.astimezone(timezone.utc)
|
| 67 |
+
route = getattr(scope.get("route"), "path", None) or scope.get("_timing_route") or "<unmatched>"
|
| 68 |
+
tenant = (tenant_resolver or verified_tenant)(scope)
|
| 69 |
+
event = {"route": route, "status": int(status), "tenant": tenant, "stack_hash": stack_hash(error)}
|
| 70 |
+
group = hashlib.sha256(json.dumps(event, sort_keys=True).encode()).hexdigest()
|
| 71 |
+
stamp = now.isoformat()
|
| 72 |
+
|
| 73 |
+
def update(document):
|
| 74 |
+
document = document or {"events": {}}
|
| 75 |
+
events = document.setdefault("events", {})
|
| 76 |
+
previous = events.get(group, {})
|
| 77 |
+
events[group] = {**event, "count": previous.get("count", 0) + 1,
|
| 78 |
+
"first": previous.get("first", stamp), "last": stamp}
|
| 79 |
+
return document
|
| 80 |
+
|
| 81 |
+
backend.update(day_key(now.date().isoformat()), update)
|
| 82 |
+
return group
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class ErrorSinkMiddleware:
|
| 86 |
+
def __init__(self, app, *, backend=None, tenant_resolver=None, clock=None):
|
| 87 |
+
self.app, self.backend = app, backend
|
| 88 |
+
self.tenant_resolver, self.clock = tenant_resolver, clock or utc_now
|
| 89 |
+
|
| 90 |
+
async def __call__(self, scope, receive, send):
|
| 91 |
+
if scope["type"] != "http":
|
| 92 |
+
return await self.app(scope, receive, send)
|
| 93 |
+
try:
|
| 94 |
+
return await self.app(scope, receive, send)
|
| 95 |
+
except Exception as error:
|
| 96 |
+
if not scope.get("_error_sink_recorded"):
|
| 97 |
+
scope["_error_sink_recorded"] = True
|
| 98 |
+
try:
|
| 99 |
+
from starlette.concurrency import run_in_threadpool
|
| 100 |
+
await run_in_threadpool(partial(record_error, scope, error, backend=self.backend,
|
| 101 |
+
now=self.clock(), tenant_resolver=self.tenant_resolver))
|
| 102 |
+
except Exception:
|
| 103 |
+
LOG.error("Error sink persistence failed; original request exception preserved")
|
| 104 |
+
raise
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def daily_digest(*, sender, day=None, backend=None, now=None):
|
| 108 |
+
backend = backend if backend is not None else store_backend()
|
| 109 |
+
now = (now or utc_now()).astimezone(timezone.utc)
|
| 110 |
+
day = day or (now.date() - timedelta(days=1)).isoformat()
|
| 111 |
+
key, claim = day_key(day), uuid4().hex
|
| 112 |
+
|
| 113 |
+
def acquire(document):
|
| 114 |
+
document = document or {"events": {}}
|
| 115 |
+
previous = document.get("digest") or {}
|
| 116 |
+
if not document.get("events") or previous.get("state") in {"sent", "sending", "uncertain"}:
|
| 117 |
+
return document
|
| 118 |
+
document["digest"] = {"state": "sending", "claim": claim, "at": now.isoformat()}
|
| 119 |
+
return document
|
| 120 |
+
|
| 121 |
+
document = backend.update(key, acquire)
|
| 122 |
+
receipt = document.get("digest") or {}
|
| 123 |
+
if receipt.get("claim") != claim:
|
| 124 |
+
pending = receipt.get("state") in {"sending", "uncertain"}
|
| 125 |
+
return {"ok": not pending, "sent": False, "day": day,
|
| 126 |
+
"reason": "delivery_needs_reconciliation" if pending else "empty_or_already_sent"}
|
| 127 |
+
events = sorted(document["events"].values(), key=lambda event: (-event["count"], event["stack_hash"]))
|
| 128 |
+
total = sum(event["count"] for event in events)
|
| 129 |
+
lines = [f"Loopable errors for {day}: {total} events in {len(events)} groups"]
|
| 130 |
+
for event in events:
|
| 131 |
+
line = f"{event['count']} x {event['status']} {event['route']} [{event['tenant']}] {event['stack_hash'][:12]}"
|
| 132 |
+
if len(("\n".join(lines) + "\n" + line).encode()) > 3500:
|
| 133 |
+
lines.append(f"Remaining groups: {len(events) - len(lines) + 1}. Full groups are in the error sink.")
|
| 134 |
+
break
|
| 135 |
+
lines.append(line)
|
| 136 |
+
|
| 137 |
+
def finish(current, state):
|
| 138 |
+
if (current.get("digest") or {}).get("claim") == claim:
|
| 139 |
+
current["digest"] = {"state": state, "claim": claim, "at": now.isoformat()}
|
| 140 |
+
return current
|
| 141 |
+
|
| 142 |
+
try:
|
| 143 |
+
delivered = sender("\n".join(lines))
|
| 144 |
+
except Exception:
|
| 145 |
+
backend.update(key, lambda current: finish(current, "uncertain"))
|
| 146 |
+
raise
|
| 147 |
+
if delivered is False:
|
| 148 |
+
backend.update(key, lambda current: finish(current, "failed"))
|
| 149 |
+
raise RuntimeError("digest sender refused delivery")
|
| 150 |
+
backend.update(key, lambda current: finish(current, "sent"))
|
| 151 |
+
return {"sent": True, "day": day, "groups": len(events), "events": total}
|
api/external_freshness.py
CHANGED
|
@@ -67,6 +67,21 @@ _AMAZON_DERIVED_INPUTS = {"sku_keywords": ("listing_items", "search_query_perfor
|
|
| 67 |
_AMAZON_DERIVED_RECEIPTS = {"keyword_ranks": "amzn_apify_keyword_ranks"}
|
| 68 |
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
def amazon_sync_state(cur):
|
| 71 |
"""`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
|
| 72 |
return {str(entity): at for entity, at in cur.execute(
|
|
@@ -94,9 +109,7 @@ def amazon_sync_stamp(cur, table_key, state=None):
|
|
| 94 |
R11: a refusal overwrites the raw receipt and a pass with no window writes no mark, so the
|
| 95 |
reconciliation proofs also count. The time is the receipt's own, never a restore or read time.
|
| 96 |
"""
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
raw = amazon_relational.SOURCE.get(table_key)
|
| 100 |
if not raw or (cur is None and state is None):
|
| 101 |
return None
|
| 102 |
if state is None:
|
|
@@ -161,7 +174,7 @@ def amazon_nav(session, pages):
|
|
| 161 |
bases = {str(name) for (name,) in cur.execute(
|
| 162 |
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' "
|
| 163 |
"AND table_type = 'BASE TABLE' AND starts_with(table_name, 'amzn_')").fetchall()}
|
| 164 |
-
raws = {page["key"]:
|
| 165 |
probed = sorted({raw for raw in raws.values() if raw in bases})
|
| 166 |
held = {}
|
| 167 |
if probed:
|
|
|
|
| 67 |
_AMAZON_DERIVED_RECEIPTS = {"keyword_ranks": "amzn_apify_keyword_ranks"}
|
| 68 |
|
| 69 |
|
| 70 |
+
def _amazon_raw(table_key, source=None):
|
| 71 |
+
"""The mirror table behind one `ut_amzn_*` key: the relational map, else an Ads spec (S79-T13)."""
|
| 72 |
+
if source is None:
|
| 73 |
+
import amazon_relational
|
| 74 |
+
source = amazon_relational.SOURCE
|
| 75 |
+
raw = source.get(table_key)
|
| 76 |
+
if raw:
|
| 77 |
+
return raw
|
| 78 |
+
from core import connector_spec
|
| 79 |
+
spec = connector_spec.spec_for(table_key)
|
| 80 |
+
if spec is not None and connector_spec.connector_of(table_key) == connector_spec.AMAZON_ADS_CONNECTOR:
|
| 81 |
+
return spec.source
|
| 82 |
+
return None
|
| 83 |
+
|
| 84 |
+
|
| 85 |
def amazon_sync_state(cur):
|
| 86 |
"""`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
|
| 87 |
return {str(entity): at for entity, at in cur.execute(
|
|
|
|
| 109 |
R11: a refusal overwrites the raw receipt and a pass with no window writes no mark, so the
|
| 110 |
reconciliation proofs also count. The time is the receipt's own, never a restore or read time.
|
| 111 |
"""
|
| 112 |
+
raw = _amazon_raw(table_key)
|
|
|
|
|
|
|
| 113 |
if not raw or (cur is None and state is None):
|
| 114 |
return None
|
| 115 |
if state is None:
|
|
|
|
| 174 |
bases = {str(name) for (name,) in cur.execute(
|
| 175 |
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' "
|
| 176 |
"AND table_type = 'BASE TABLE' AND starts_with(table_name, 'amzn_')").fetchall()}
|
| 177 |
+
raws = {page["key"]: _amazon_raw(page["key"], amazon_relational.SOURCE) for page in rows}
|
| 178 |
probed = sorted({raw for raw in raws.values() if raw in bases})
|
| 179 |
held = {}
|
| 180 |
if probed:
|
api/external_openapi.py
CHANGED
|
@@ -43,8 +43,8 @@ def _field_schema(field, source_kind):
|
|
| 43 |
kind = _JSON_TYPE.get(field.get("type"), "string")
|
| 44 |
types = [kind]
|
| 45 |
# Materialised table writes retain numeric cells as strings; reads preserve those bytes.
|
| 46 |
-
if
|
| 47 |
-
types
|
| 48 |
return {"type": [*types, "null"]}
|
| 49 |
|
| 50 |
|
|
|
|
| 43 |
kind = _JSON_TYPE.get(field.get("type"), "string")
|
| 44 |
types = [kind]
|
| 45 |
# Materialised table writes retain numeric cells as strings; reads preserve those bytes.
|
| 46 |
+
if kind in {"number", "integer"}:
|
| 47 |
+
types = ["number", "string"]
|
| 48 |
return {"type": [*types, "null"]}
|
| 49 |
|
| 50 |
|
api/loopy_catalog_tools.py
CHANGED
|
@@ -11,6 +11,7 @@ def _build_refusal(code, message):
|
|
| 11 |
def _build_app(ctx, args):
|
| 12 |
import application_registry as applications
|
| 13 |
import catalog_store
|
|
|
|
| 14 |
from fastapi import HTTPException
|
| 15 |
|
| 16 |
source = args["sourceDatabase"]
|
|
@@ -23,8 +24,8 @@ def _build_app(ctx, args):
|
|
| 23 |
if app["sourceDatabase"] != source:
|
| 24 |
_build_refusal("catalog_source_mismatch", "Choose a Catalog App over the requested source database.")
|
| 25 |
else:
|
| 26 |
-
candidates = [app for app in applications.load(ctx.session.runtime)["applications"].values()
|
| 27 |
-
if app.get("templateKind") == "catalog" and app.get("sourceDatabase") == source
|
| 28 |
and app.get("retired") is not True
|
| 29 |
and applications.may_read(app, ctx.session.uname, ctx.session.admin)
|
| 30 |
and applications.may_edit(app, ctx.session.uname, ctx.session.admin)]
|
|
@@ -40,8 +41,8 @@ def _build_app(ctx, args):
|
|
| 40 |
if app is None:
|
| 41 |
_build_refusal("catalog_app_unavailable", (
|
| 42 |
"No editable Catalog App is available for this source. Create one with apps.apply_template "
|
| 43 |
-
"using templateKind catalog
|
| 44 |
-
"
|
| 45 |
fields = {field["key"] for field in schema["fields"]}
|
| 46 |
if any(field not in fields for field in app["fieldMap"].values()):
|
| 47 |
_build_refusal("catalog_fields_forbidden", "The Catalog App uses unavailable fields. Rebind it to readable source fields.")
|
|
@@ -116,6 +117,7 @@ def _build_rows(ctx, source, fields, filters, sorts):
|
|
| 116 |
def catalog_build(ctx, args):
|
| 117 |
"""Build a C1 record through the caller's staged runtime and permissioned query callbacks."""
|
| 118 |
import catalog_store
|
|
|
|
| 119 |
from core import external_tools, write_execution
|
| 120 |
from loopy_file_tools import _secure_link
|
| 121 |
from loopy_links import application_link
|
|
@@ -220,6 +222,22 @@ def build_schema():
|
|
| 220 |
"required": ["sourceDatabase"], "additionalProperties": False}
|
| 221 |
|
| 222 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
def catalog_pdf(session, args):
|
| 224 |
import assistant_files
|
| 225 |
import catalog_render
|
|
@@ -234,6 +252,7 @@ def catalog_pdf(session, args):
|
|
| 234 |
_secure_link(links.app_link("#/assistant"))
|
| 235 |
_secure_link(links.artifact_link("catalog-preview"))
|
| 236 |
data, record, app = catalog_render.render_catalog(session, args["catalog_id"], ai=True)
|
|
|
|
| 237 |
title = _no_dashes(record["name"]).strip()[:200]
|
| 238 |
saved = assistant_files.save(session.runtime, session.uname, title=title, data=data,
|
| 239 |
mime="application/pdf", door=door,
|
|
@@ -287,6 +306,7 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None):
|
|
| 287 |
import sys
|
| 288 |
import threading
|
| 289 |
import catalog_store
|
|
|
|
| 290 |
|
| 291 |
render = render or sys.modules[__name__].catalog_pdf
|
| 292 |
budget = export_budget() if budget is None else budget
|
|
|
|
| 11 |
def _build_app(ctx, args):
|
| 12 |
import application_registry as applications
|
| 13 |
import catalog_store
|
| 14 |
+
import app_templates
|
| 15 |
from fastapi import HTTPException
|
| 16 |
|
| 17 |
source = args["sourceDatabase"]
|
|
|
|
| 24 |
if app["sourceDatabase"] != source:
|
| 25 |
_build_refusal("catalog_source_mismatch", "Choose a Catalog App over the requested source database.")
|
| 26 |
else:
|
| 27 |
+
candidates = [app_templates.bound(app) for app in applications.load(ctx.session.runtime)["applications"].values()
|
| 28 |
+
if app.get("templateKind") == "catalog" and app_templates.bound(app).get("sourceDatabase") == source
|
| 29 |
and app.get("retired") is not True
|
| 30 |
and applications.may_read(app, ctx.session.uname, ctx.session.admin)
|
| 31 |
and applications.may_edit(app, ctx.session.uname, ctx.session.admin)]
|
|
|
|
| 41 |
if app is None:
|
| 42 |
_build_refusal("catalog_app_unavailable", (
|
| 43 |
"No editable Catalog App is available for this source. Create one with apps.apply_template "
|
| 44 |
+
"using templateKind catalog. "
|
| 45 |
+
"The App binds its Odoo products automatically."))
|
| 46 |
fields = {field["key"] for field in schema["fields"]}
|
| 47 |
if any(field not in fields for field in app["fieldMap"].values()):
|
| 48 |
_build_refusal("catalog_fields_forbidden", "The Catalog App uses unavailable fields. Rebind it to readable source fields.")
|
|
|
|
| 117 |
def catalog_build(ctx, args):
|
| 118 |
"""Build a C1 record through the caller's staged runtime and permissioned query callbacks."""
|
| 119 |
import catalog_store
|
| 120 |
+
import app_templates
|
| 121 |
from core import external_tools, write_execution
|
| 122 |
from loopy_file_tools import _secure_link
|
| 123 |
from loopy_links import application_link
|
|
|
|
| 222 |
"required": ["sourceDatabase"], "additionalProperties": False}
|
| 223 |
|
| 224 |
|
| 225 |
+
def _register_render_read(session, source):
|
| 226 |
+
"""Record the render's product read in the caller's producing turn, so the PDF keeps its source (D-9854).
|
| 227 |
+
|
| 228 |
+
`assistant_files.save` keeps only sources the turn proved it read. The render reads the source
|
| 229 |
+
under this caller even when the producing turn holds another session object for the same user.
|
| 230 |
+
"""
|
| 231 |
+
import agent_runtime
|
| 232 |
+
|
| 233 |
+
producing = agent_runtime._PRODUCING_TURN.get()
|
| 234 |
+
if producing is None:
|
| 235 |
+
return
|
| 236 |
+
caller, turn = producing
|
| 237 |
+
if caller.runtime is session.runtime and caller.uname == session.uname:
|
| 238 |
+
agent_runtime._merge_sources(caller, turn, [source])
|
| 239 |
+
|
| 240 |
+
|
| 241 |
def catalog_pdf(session, args):
|
| 242 |
import assistant_files
|
| 243 |
import catalog_render
|
|
|
|
| 252 |
_secure_link(links.app_link("#/assistant"))
|
| 253 |
_secure_link(links.artifact_link("catalog-preview"))
|
| 254 |
data, record, app = catalog_render.render_catalog(session, args["catalog_id"], ai=True)
|
| 255 |
+
_register_render_read(session, app["sourceDatabase"])
|
| 256 |
title = _no_dashes(record["name"]).strip()[:200]
|
| 257 |
saved = assistant_files.save(session.runtime, session.uname, title=title, data=data,
|
| 258 |
mime="application/pdf", door=door,
|
|
|
|
| 306 |
import sys
|
| 307 |
import threading
|
| 308 |
import catalog_store
|
| 309 |
+
import app_templates
|
| 310 |
|
| 311 |
render = render or sys.modules[__name__].catalog_pdf
|
| 312 |
budget = export_budget() if budget is None else budget
|
api/main.py
CHANGED
|
@@ -146,7 +146,8 @@ from request_timing import RequestTiming # noqa: E402
|
|
| 146 |
class _TimedFastAPI(FastAPI):
|
| 147 |
def build_middleware_stack(self):
|
| 148 |
# Include error responses, freeze refusals, admission and compressed body sends.
|
| 149 |
-
|
|
|
|
| 150 |
|
| 151 |
|
| 152 |
app = _TimedFastAPI(title="AIOS web API")
|
|
@@ -208,7 +209,23 @@ def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
|
|
| 208 |
|
| 209 |
async def _resource_unavailable(request: Request, exc: Exception):
|
| 210 |
from request_timing import resource_response
|
| 211 |
-
return resource_response(exc)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
|
| 213 |
|
| 214 |
@app.exception_handler(Exception)
|
|
@@ -216,28 +233,31 @@ async def _unexpected_error(request: Request, exc: Exception):
|
|
| 216 |
from request_timing import resource_response
|
| 217 |
busy = resource_response(exc)
|
| 218 |
if busy is not None:
|
| 219 |
-
return busy
|
| 220 |
return JSONResponse({"error": {"code": "server_error", "message": "The request could not finish."}},
|
| 221 |
status_code=500)
|
| 222 |
|
| 223 |
|
| 224 |
-
from core.odoo import OdooPoolTimeout
|
| 225 |
-
from core.store_pg import SharedReadTimeout
|
| 226 |
from core.request_timing import RequestDeadlineExceeded
|
| 227 |
from request_timing import DeadlineCapacityExceeded
|
| 228 |
from duckdb import OutOfMemoryException
|
| 229 |
-
|
| 230 |
-
|
|
|
|
| 231 |
app.add_exception_handler(_resource_error, _resource_unavailable)
|
| 232 |
try:
|
| 233 |
from psycopg_pool import PoolTimeout, TooManyRequests
|
| 234 |
from psycopg import OperationalError, InterfaceError
|
| 235 |
from psycopg.errors import QueryCanceled, LockNotAvailable, IdleInTransactionSessionTimeout
|
|
|
|
| 236 |
except ImportError:
|
| 237 |
pass
|
| 238 |
else:
|
| 239 |
for _resource_error in (PoolTimeout, TooManyRequests, QueryCanceled, LockNotAvailable,
|
| 240 |
-
OperationalError, InterfaceError, IdleInTransactionSessionTimeout
|
|
|
|
| 241 |
app.add_exception_handler(_resource_error, _resource_unavailable)
|
| 242 |
|
| 243 |
|
|
@@ -381,6 +401,35 @@ async def health(challenge: str = ""):
|
|
| 381 |
return {"ok": True}
|
| 382 |
|
| 383 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 384 |
# W59-T43: boot stays available while a cross-lane module is absent, but never silently.
|
| 385 |
# verify_wiring.W59_PENDING_CROSS_LANE_HALVES owns the shrink-only declarations and rejects
|
| 386 |
# any failure outside them, including a missing dependency INSIDE a declared router.
|
|
@@ -609,6 +658,8 @@ app.include_router(routes_competitor_watch.router) # S78-T16 / S78-T17 competit
|
|
| 609 |
_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
|
| 610 |
_mount_router("routes_amazon_profit") # S78-T03: the Amazon SKU profit refresh and settings doors.
|
| 611 |
_mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
|
|
|
|
|
|
|
| 612 |
|
| 613 |
|
| 614 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
|
@@ -1041,7 +1092,7 @@ def _restore_amazon_mirror_once(why, tenant_key):
|
|
| 1041 |
# Publish measured definitions before any Odoo read can block the resident writer.
|
| 1042 |
import process_role as _process_role
|
| 1043 |
# The relational rebuild writes the store, so only the sync worker runs it.
|
| 1044 |
-
if (result.get("state") != "absent" and _process_role.
|
| 1045 |
rt.datastore_path.resolve() == _ds.DB_PATH.resolve()):
|
| 1046 |
state = _rebuild_amazon_relational(why, rt)
|
| 1047 |
if state.get("changed") and not state.get("applied") and not state.get("known"):
|
|
@@ -2914,10 +2965,11 @@ def _prewarm():
|
|
| 2914 |
if _process_role.role() == "worker":
|
| 2915 |
# The warm below fills per-process memos, which the sync worker never serves.
|
| 2916 |
return
|
|
|
|
|
|
|
| 2917 |
try:
|
| 2918 |
from harness import runtime as _runtime
|
| 2919 |
rt = _runtime.get_runtime("royal-imports")
|
| 2920 |
-
routes_customers.warm_default(rt)
|
| 2921 |
import pages as _pages
|
| 2922 |
_pages.warm_default(rt)
|
| 2923 |
# ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent
|
|
@@ -3098,8 +3150,8 @@ def _restore_sync_request_desks():
|
|
| 3098 |
from core import connector_sync
|
| 3099 |
from harness import runtime
|
| 3100 |
import process_role as _process_role
|
| 3101 |
-
#
|
| 3102 |
-
requeue = _process_role.
|
| 3103 |
for tenant_key in runtime.known_tenants():
|
| 3104 |
try:
|
| 3105 |
connector_sync.restore_sync_requests(runtime.get_runtime(tenant_key), requeue=requeue)
|
|
@@ -3110,30 +3162,90 @@ def _restore_sync_request_desks():
|
|
| 3110 |
def _boot_mirrors():
|
| 3111 |
import process_role as _role_boot
|
| 3112 |
_restore_sync_request_desks()
|
| 3113 |
-
|
|
|
|
|
|
|
| 3114 |
if _role_boot.role() == "web":
|
|
|
|
|
|
|
| 3115 |
# Every connector but Odoo reaches the web process through the worker's hand-off.
|
| 3116 |
print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
|
| 3117 |
# The biggest read caches are this process's own, so they warm here, before the first reader pays.
|
| 3118 |
_role_boot.start("read-warm", "light", _warm_read_pools)
|
| 3119 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3120 |
_restore_amazon_mirror("boot")
|
| 3121 |
_restore_declared_mirrors("boot")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3122 |
|
| 3123 |
|
| 3124 |
-
|
| 3125 |
-
|
| 3126 |
-
|
| 3127 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3128 |
try:
|
| 3129 |
-
|
| 3130 |
-
|
| 3131 |
-
|
| 3132 |
-
|
| 3133 |
-
|
| 3134 |
-
|
| 3135 |
-
|
| 3136 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3137 |
|
| 3138 |
|
| 3139 |
def _restore_declared_mirrors(why):
|
|
@@ -3183,7 +3295,7 @@ if (not data_binding.gate_process()
|
|
| 3183 |
|
| 3184 |
if (not data_binding.gate_process()
|
| 3185 |
and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
|
| 3186 |
-
and (
|
| 3187 |
and not _freeze_holds("app migrations")):
|
| 3188 |
import process_role as _role_apps
|
| 3189 |
_role_apps.thread(target=_boot_app_migrations, args=("boot",), daemon=True, kind="heavy",
|
|
|
|
| 146 |
class _TimedFastAPI(FastAPI):
|
| 147 |
def build_middleware_stack(self):
|
| 148 |
# Include error responses, freeze refusals, admission and compressed body sends.
|
| 149 |
+
from error_sink import ErrorSinkMiddleware
|
| 150 |
+
return ErrorSinkMiddleware(RequestTiming(super().build_middleware_stack()))
|
| 151 |
|
| 152 |
|
| 153 |
app = _TimedFastAPI(title="AIOS web API")
|
|
|
|
| 209 |
|
| 210 |
async def _resource_unavailable(request: Request, exc: Exception):
|
| 211 |
from request_timing import resource_response
|
| 212 |
+
return _with_holder(resource_response(exc), exc)
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _with_holder(response, exc):
|
| 216 |
+
seen = set()
|
| 217 |
+
while exc is not None and id(exc) not in seen:
|
| 218 |
+
seen.add(id(exc))
|
| 219 |
+
holder = getattr(exc, "aios_holder", None)
|
| 220 |
+
if isinstance(holder, dict) and response is not None:
|
| 221 |
+
import json
|
| 222 |
+
body = json.loads(response.body)
|
| 223 |
+
body["error"]["holder"] = holder
|
| 224 |
+
body["error"]["message"] = (
|
| 225 |
+
f"Another change is still saving ({holder.get('label', 'another request')}). Try again shortly.")
|
| 226 |
+
return JSONResponse(body, status_code=503, headers={"Retry-After": "1"})
|
| 227 |
+
exc = exc.__cause__ or exc.__context__
|
| 228 |
+
return response
|
| 229 |
|
| 230 |
|
| 231 |
@app.exception_handler(Exception)
|
|
|
|
| 233 |
from request_timing import resource_response
|
| 234 |
busy = resource_response(exc)
|
| 235 |
if busy is not None:
|
| 236 |
+
return _with_holder(busy, exc)
|
| 237 |
return JSONResponse({"error": {"code": "server_error", "message": "The request could not finish."}},
|
| 238 |
status_code=500)
|
| 239 |
|
| 240 |
|
| 241 |
+
from core.odoo import OdooPoolTimeout, OdooRequestTimeout
|
| 242 |
+
from core.store_pg import SharedReadTimeout, PostgresOperationTimeout
|
| 243 |
from core.request_timing import RequestDeadlineExceeded
|
| 244 |
from request_timing import DeadlineCapacityExceeded
|
| 245 |
from duckdb import OutOfMemoryException
|
| 246 |
+
# Each busy cause needs its own handler: the catch-all `Exception` one answers, then re-raises to uvicorn.
|
| 247 |
+
for _resource_error in (OdooPoolTimeout, OdooRequestTimeout, SharedReadTimeout, PostgresOperationTimeout,
|
| 248 |
+
RequestDeadlineExceeded, DeadlineCapacityExceeded, OutOfMemoryException):
|
| 249 |
app.add_exception_handler(_resource_error, _resource_unavailable)
|
| 250 |
try:
|
| 251 |
from psycopg_pool import PoolTimeout, TooManyRequests
|
| 252 |
from psycopg import OperationalError, InterfaceError
|
| 253 |
from psycopg.errors import QueryCanceled, LockNotAvailable, IdleInTransactionSessionTimeout
|
| 254 |
+
from psycopg.errors import ConnectionTimeout
|
| 255 |
except ImportError:
|
| 256 |
pass
|
| 257 |
else:
|
| 258 |
for _resource_error in (PoolTimeout, TooManyRequests, QueryCanceled, LockNotAvailable,
|
| 259 |
+
OperationalError, InterfaceError, IdleInTransactionSessionTimeout,
|
| 260 |
+
ConnectionTimeout):
|
| 261 |
app.add_exception_handler(_resource_error, _resource_unavailable)
|
| 262 |
|
| 263 |
|
|
|
|
| 401 |
return {"ok": True}
|
| 402 |
|
| 403 |
|
| 404 |
+
def _mirror_answers():
|
| 405 |
+
"""The resident Odoo mirror's live state for readiness. Never raises."""
|
| 406 |
+
try:
|
| 407 |
+
from harness import datastore as _ready_ds
|
| 408 |
+
return bool(_ready_ds.ready())
|
| 409 |
+
except Exception: # noqa: BLE001
|
| 410 |
+
return False
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
@app.get("/api/v1/ready")
|
| 414 |
+
def readiness_check():
|
| 415 |
+
"""C9: 200 only when the mirror is installed, the pools are open and the first reads answered.
|
| 416 |
+
|
| 417 |
+
Unauthenticated, because the deploy switch polls it before anyone signs in. It carries the
|
| 418 |
+
version on purpose (C9), unlike /api/health.
|
| 419 |
+
"""
|
| 420 |
+
report = process_role.readiness(mirror_live=_mirror_answers())
|
| 421 |
+
body = {"ready": report["ready"], "version": os.environ.get("AIOS_VERSION") or "unknown",
|
| 422 |
+
"checks": report["checks"]}
|
| 423 |
+
headers = {"Cache-Control": "no-store"}
|
| 424 |
+
if report["ready"]:
|
| 425 |
+
return JSONResponse(body, headers=headers)
|
| 426 |
+
body["failing"] = report["failing"]
|
| 427 |
+
body["error"] = {"code": "not_ready",
|
| 428 |
+
"message": "Not ready: " + ", ".join(report["failing"]) + ". "
|
| 429 |
+
+ " ".join(report["detail"].values())}
|
| 430 |
+
return JSONResponse(body, status_code=503, headers=headers)
|
| 431 |
+
|
| 432 |
+
|
| 433 |
# W59-T43: boot stays available while a cross-lane module is absent, but never silently.
|
| 434 |
# verify_wiring.W59_PENDING_CROSS_LANE_HALVES owns the shrink-only declarations and rejects
|
| 435 |
# any failure outside them, including a missing dependency INSIDE a declared router.
|
|
|
|
| 658 |
_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
|
| 659 |
_mount_router("routes_amazon_profit") # S78-T03: the Amazon SKU profit refresh and settings doors.
|
| 660 |
_mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
|
| 661 |
+
import app_routes
|
| 662 |
+
app_routes.mount_app_routers(app)
|
| 663 |
|
| 664 |
|
| 665 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
|
|
|
| 1092 |
# Publish measured definitions before any Odoo read can block the resident writer.
|
| 1093 |
import process_role as _process_role
|
| 1094 |
# The relational rebuild writes the store, so only the sync worker runs it.
|
| 1095 |
+
if (result.get("state") != "absent" and _process_role.may_write() and
|
| 1096 |
rt.datastore_path.resolve() == _ds.DB_PATH.resolve()):
|
| 1097 |
state = _rebuild_amazon_relational(why, rt)
|
| 1098 |
if state.get("changed") and not state.get("applied") and not state.get("known"):
|
|
|
|
| 2965 |
if _process_role.role() == "worker":
|
| 2966 |
# The warm below fills per-process memos, which the sync worker never serves.
|
| 2967 |
return
|
| 2968 |
+
# D-273: the shared read warm fills both default pools in runtime.pool_cache.
|
| 2969 |
+
_warm_read_pools(attempts=1)
|
| 2970 |
try:
|
| 2971 |
from harness import runtime as _runtime
|
| 2972 |
rt = _runtime.get_runtime("royal-imports")
|
|
|
|
| 2973 |
import pages as _pages
|
| 2974 |
_pages.warm_default(rt)
|
| 2975 |
# ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent
|
|
|
|
| 3150 |
from core import connector_sync
|
| 3151 |
from harness import runtime
|
| 3152 |
import process_role as _process_role
|
| 3153 |
+
# Only the sync writer requeues in-flight claims: never the web role, never a worker without the lease.
|
| 3154 |
+
requeue = _process_role.may_write()
|
| 3155 |
for tenant_key in runtime.known_tenants():
|
| 3156 |
try:
|
| 3157 |
connector_sync.restore_sync_requests(runtime.get_runtime(tenant_key), requeue=requeue)
|
|
|
|
| 3162 |
def _boot_mirrors():
|
| 3163 |
import process_role as _role_boot
|
| 3164 |
_restore_sync_request_desks()
|
| 3165 |
+
if _role_boot.role() != "web" or _role_boot.claim_primary():
|
| 3166 |
+
# With several web workers only the primary restores the mirror. The others read it.
|
| 3167 |
+
_seed_mirror_if_absent()
|
| 3168 |
if _role_boot.role() == "web":
|
| 3169 |
+
# Readiness also waits for the worker's boot hand-offs and the mirror's own completeness.
|
| 3170 |
+
_role_boot.mark_ready("mirror", True)
|
| 3171 |
# Every connector but Odoo reaches the web process through the worker's hand-off.
|
| 3172 |
print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
|
| 3173 |
# The biggest read caches are this process's own, so they warm here, before the first reader pays.
|
| 3174 |
_role_boot.start("read-warm", "light", _warm_read_pools)
|
| 3175 |
return
|
| 3176 |
+
if _role_boot.role() == "worker":
|
| 3177 |
+
try:
|
| 3178 |
+
_restore_amazon_mirror("boot")
|
| 3179 |
+
_restore_declared_mirrors("boot")
|
| 3180 |
+
finally:
|
| 3181 |
+
# Every boot hand-off is on disk now; the web process's readiness waits for this line.
|
| 3182 |
+
_role_boot.note_boot_restore()
|
| 3183 |
+
return
|
| 3184 |
_restore_amazon_mirror("boot")
|
| 3185 |
_restore_declared_mirrors("boot")
|
| 3186 |
+
# all and reader have no worker: their own restores finish before readiness and the warm.
|
| 3187 |
+
_role_boot.mark_ready("mirror", True)
|
| 3188 |
+
if _role_boot.role() == "reader" or data_binding.is_space():
|
| 3189 |
+
# A local `all` run keeps its old boot: AIOS_PREWARM=1 warms it.
|
| 3190 |
+
_warm_read_pools()
|
| 3191 |
|
| 3192 |
|
| 3193 |
+
#: A failed boot warm retries, so one slow first read cannot leave readiness red until a restart.
|
| 3194 |
+
_READ_WARM_ATTEMPTS = 40
|
| 3195 |
+
_READ_WARM_RETRY_SECONDS = 15
|
| 3196 |
+
|
| 3197 |
+
|
| 3198 |
+
def _probe_read_pools(rt):
|
| 3199 |
+
"""Open the mirror read cursor and the tenant store once. '' when both answer, else the cause."""
|
| 3200 |
+
from harness import datastore as _probe_ds
|
| 3201 |
+
if not _probe_ds.DB_PATH.exists():
|
| 3202 |
+
return "The mirror file is missing."
|
| 3203 |
+
cur = _probe_ds.ro_cursor()
|
| 3204 |
try:
|
| 3205 |
+
cur.execute("SELECT 1").fetchone()
|
| 3206 |
+
finally:
|
| 3207 |
+
cur.close()
|
| 3208 |
+
if rt.available() is False:
|
| 3209 |
+
return "The tenant store did not answer."
|
| 3210 |
+
return ""
|
| 3211 |
+
|
| 3212 |
+
|
| 3213 |
+
def _warm_read_pools(attempts=None, retry_seconds=None):
|
| 3214 |
+
"""Open the read pools, then build the consolidated customer and product pools, off the request path.
|
| 3215 |
+
|
| 3216 |
+
Marks the `pools` and `first_reads` readiness checks. A failure never clears a check that passed.
|
| 3217 |
+
"""
|
| 3218 |
+
import time as _t
|
| 3219 |
+
import process_role as _warm_role
|
| 3220 |
+
attempts = _READ_WARM_ATTEMPTS if attempts is None else max(1, int(attempts))
|
| 3221 |
+
retry_seconds = _READ_WARM_RETRY_SECONDS if retry_seconds is None else retry_seconds
|
| 3222 |
+
for attempt in range(1, attempts + 1):
|
| 3223 |
+
t0 = _t.time()
|
| 3224 |
+
stage = "pools"
|
| 3225 |
+
try:
|
| 3226 |
+
from harness import runtime as _runtime
|
| 3227 |
+
import routes_products
|
| 3228 |
+
rt = _runtime.get_runtime("royal-imports")
|
| 3229 |
+
problem = _probe_read_pools(rt)
|
| 3230 |
+
if problem:
|
| 3231 |
+
raise RuntimeError(problem)
|
| 3232 |
+
_warm_role.mark_ready("pools", True)
|
| 3233 |
+
stage = "first_reads"
|
| 3234 |
+
routes_customers.warm_default(rt)
|
| 3235 |
+
routes_products._pool_for(rt, None)
|
| 3236 |
+
_warm_role.mark_ready("first_reads", True)
|
| 3237 |
+
routes_customers.warm_bu_pools(rt)
|
| 3238 |
+
print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
|
| 3239 |
+
return True
|
| 3240 |
+
except Exception as exc: # noqa: BLE001
|
| 3241 |
+
if not _warm_role._READINESS.get(stage, {}).get("ok"):
|
| 3242 |
+
_warm_role.mark_ready(stage, False, f"The {'read pools' if stage == 'pools' else 'first reads'} "
|
| 3243 |
+
f"failed ({type(exc).__name__}); attempt {attempt} of {attempts}.")
|
| 3244 |
+
print(f"[aios-api] read warm failed at {stage} ({type(exc).__name__}: {exc}); "
|
| 3245 |
+
f"attempt {attempt} of {attempts}")
|
| 3246 |
+
if attempt < attempts:
|
| 3247 |
+
_t.sleep(retry_seconds)
|
| 3248 |
+
return False
|
| 3249 |
|
| 3250 |
|
| 3251 |
def _restore_declared_mirrors(why):
|
|
|
|
| 3295 |
|
| 3296 |
if (not data_binding.gate_process()
|
| 3297 |
and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
|
| 3298 |
+
and (bool(os.environ.get("SPACE_ID")) or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
|
| 3299 |
and not _freeze_holds("app migrations")):
|
| 3300 |
import process_role as _role_apps
|
| 3301 |
_role_apps.thread(target=_boot_app_migrations, args=("boot",), daemon=True, kind="heavy",
|
api/odoo_relational.py
CHANGED
|
@@ -2572,26 +2572,33 @@ class Refused(Exception):
|
|
| 2572 |
#: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole
|
| 2573 |
#: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code.
|
| 2574 |
#: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2575 |
def payment_fields():
|
| 2576 |
"""Declare company-currency payment facts without materializing source rows."""
|
| 2577 |
-
|
| 2578 |
-
|
| 2579 |
-
|
| 2580 |
-
|
| 2581 |
-
("
|
| 2582 |
-
|
| 2583 |
-
("partner_id", "Customer ID", "int"))]
|
| 2584 |
|
| 2585 |
|
| 2586 |
def payment_source_specs():
|
| 2587 |
"""Bind the payment database to its native payment identity and gross receipt amount."""
|
|
|
|
|
|
|
| 2588 |
return {PAYMENTS_KEY: {
|
| 2589 |
"table": "account_payment", "from_sql": "account_payment pay", "tables": {"pay": "account_payment"},
|
| 2590 |
-
"alias": "pay", "id": "pay.id", "where": "",
|
| 2591 |
-
"cols": {field["key"]: ("pay." + field["key"], converter)
|
| 2592 |
-
for field, converter in zip(payment_fields(),
|
| 2593 |
-
(lambda value: float(value or 0), _as_date, str, str, str, bool,
|
| 2594 |
-
lambda value: int(value or 0)))}}}
|
| 2595 |
|
| 2596 |
|
| 2597 |
TABLES = (
|
|
@@ -2716,12 +2723,19 @@ def _selection_field_contracts(fields, catalogue_row):
|
|
| 2716 |
return out
|
| 2717 |
|
| 2718 |
|
| 2719 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2720 |
"""Read-only introspection over an injected Odoo client.
|
| 2721 |
|
| 2722 |
``ir.model`` supplies candidates, ``check_access_rights('read')`` narrows them to this API
|
| 2723 |
account, and ``fields_get`` supplies only fields that account may inspect. No method in this
|
| 2724 |
path can mutate Odoo; the production client independently blocks write-shaped method names.
|
|
|
|
| 2725 |
"""
|
| 2726 |
errors = []
|
| 2727 |
try:
|
|
@@ -2734,13 +2748,16 @@ def discover_readable_models(client):
|
|
| 2734 |
model = str((item or {}).get("model") or "").strip().lower()
|
| 2735 |
if not model:
|
| 2736 |
continue
|
|
|
|
|
|
|
| 2737 |
try:
|
| 2738 |
-
readable =
|
| 2739 |
-
model, "check_access_rights", ["read"], {"raise_exception": False})
|
| 2740 |
except Exception as exc: # noqa: BLE001
|
| 2741 |
errors.append(f"{model}: access: {type(exc).__name__}: {exc}")
|
| 2742 |
continue
|
| 2743 |
-
if not
|
|
|
|
|
|
|
| 2744 |
continue
|
| 2745 |
try:
|
| 2746 |
fields = client.fields_get(
|
|
@@ -2764,28 +2781,125 @@ def _catalog_slot(connection_id):
|
|
| 2764 |
return str(connection_id or DEFAULT_CONNECTION)
|
| 2765 |
|
| 2766 |
|
| 2767 |
-
def
|
| 2768 |
-
"""
|
| 2769 |
try:
|
| 2770 |
cached = rt.get(MODEL_CATALOG_KEY) if rt is not None else None
|
| 2771 |
except Exception: # noqa: BLE001
|
| 2772 |
cached = None
|
| 2773 |
if not isinstance(cached, dict) or cached.get("version") != MODEL_CATALOG_VERSION:
|
| 2774 |
-
return {
|
| 2775 |
-
|
| 2776 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2777 |
return {"version": MODEL_CATALOG_VERSION,
|
| 2778 |
-
"complete":
|
| 2779 |
-
"capturedAt": str(
|
| 2780 |
-
"models": [dict(row) for row in (
|
| 2781 |
if isinstance(row, dict) and row.get("model")],
|
| 2782 |
-
"errors": [str(value) for value in (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2783 |
|
| 2784 |
|
| 2785 |
def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=False):
|
| 2786 |
"""Populate one tenant's catalogue once, or explicitly refresh it; never widen on failure."""
|
| 2787 |
-
|
| 2788 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2789 |
and not any(_missing_selection_metadata(row) for row in current["models"])):
|
| 2790 |
try:
|
| 2791 |
captured = _dt.datetime.fromisoformat(current["capturedAt"])
|
|
@@ -2802,46 +2916,42 @@ def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=Fa
|
|
| 2802 |
source = None
|
| 2803 |
if source is None:
|
| 2804 |
return current
|
|
|
|
| 2805 |
try:
|
| 2806 |
if client is None:
|
| 2807 |
from core.odoo import get_odoo
|
| 2808 |
with source._bound(): # tenant credential + scope
|
| 2809 |
-
rows, errors = discover_readable_models(get_odoo())
|
| 2810 |
else:
|
| 2811 |
-
rows, errors = discover_readable_models(client)
|
| 2812 |
except Exception as exc: # noqa: BLE001
|
| 2813 |
rows, errors = [], [f"catalogue: {type(exc).__name__}: {exc}"]
|
|
|
|
| 2814 |
# A partial retry is not evidence that a previously readable model disappeared. Preserve a
|
| 2815 |
# complete, stamped catalogue in full whenever the new pass reports any error; otherwise one
|
| 2816 |
# refused model could silently shrink the enablement roster. A first-ever partial pass may be
|
| 2817 |
# retained for diagnostics, but an entirely failed pass still has no catalogue to publish.
|
| 2818 |
-
|
| 2819 |
-
|
| 2820 |
-
|
|
|
|
| 2821 |
return current
|
| 2822 |
captured = _dt.datetime.now(_dt.timezone.utc).isoformat()
|
| 2823 |
result = {"version": MODEL_CATALOG_VERSION, "complete": not errors,
|
| 2824 |
-
"capturedAt": captured, "models": rows, "errors": errors[:50]}
|
| 2825 |
-
|
| 2826 |
-
|
| 2827 |
-
def _cache(current_cache):
|
| 2828 |
-
current_cache = dict(current_cache) if isinstance(current_cache, dict) else {}
|
| 2829 |
-
connections = dict(current_cache.get("connections") or {})
|
| 2830 |
-
connections[slot] = result
|
| 2831 |
-
return {"version": MODEL_CATALOG_VERSION, "connections": connections}
|
| 2832 |
-
|
| 2833 |
-
rt.update(MODEL_CATALOG_KEY, _cache, flush="sync")
|
| 2834 |
-
return result
|
| 2835 |
|
| 2836 |
|
| 2837 |
def refresh_model_catalog_entry(rt, model, connection_id=None, client=None):
|
| 2838 |
"""Refresh one model's fields with two read-only calls, without enumerating other models."""
|
| 2839 |
-
|
|
|
|
|
|
|
|
|
|
| 2840 |
|
| 2841 |
def _read_fields(bound_client):
|
| 2842 |
-
|
| 2843 |
-
model, "check_access_rights", ["read"], {"raise_exception": False})
|
| 2844 |
-
if not bool(readable):
|
| 2845 |
return None
|
| 2846 |
fields = bound_client.fields_get(
|
| 2847 |
model, attributes=_ODOO_FIELD_ATTRIBUTES)
|
|
@@ -2867,28 +2977,40 @@ def refresh_model_catalog_entry(rt, model, connection_id=None, client=None):
|
|
| 2867 |
# No answer (including unavailable credentials) is not evidence of denied model access.
|
| 2868 |
return current
|
| 2869 |
if fields is None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2870 |
raise Refused(f"This Odoo account cannot read the model {model}. "
|
| 2871 |
"Ask your Odoo administrator to grant read access in Odoo, not here.")
|
| 2872 |
-
slot = _catalog_slot(connection_id)
|
| 2873 |
|
| 2874 |
-
def
|
| 2875 |
-
|
| 2876 |
-
connections = dict(current_cache.get("connections") or {})
|
| 2877 |
-
result = dict(connections.get(slot) or current)
|
| 2878 |
rows = list(result.get("models") or ())
|
| 2879 |
-
|
| 2880 |
row = {"model": model,
|
| 2881 |
-
"name": " ".join(str((
|
| 2882 |
"fields": fields}
|
| 2883 |
result["models"] = [row if item.get("model") == model else item for item in rows]
|
| 2884 |
-
if
|
| 2885 |
result["models"].append(row)
|
| 2886 |
# One model neither captures a whole catalogue nor changes its completeness. Preserve
|
| 2887 |
# complete and capturedAt exactly, along with every unrelated row and connection slot.
|
| 2888 |
-
|
| 2889 |
-
|
| 2890 |
-
|
| 2891 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2892 |
return readable_model_catalog(rt, connection_id)
|
| 2893 |
|
| 2894 |
|
|
@@ -3255,6 +3377,10 @@ ODOO_SPEC = (
|
|
| 3255 |
"pinned": "invoice_no", "identity": (_ROW_ID_COL, "odoo_id"),
|
| 3256 |
"links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O),)},
|
| 3257 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3258 |
# ⚠ RETIRED FROM `TABLES` BY W33-T44, DECLARED HERE ANYWAY, AND THAT IS THE POINT OF A SPEC.
|
| 3259 |
# `RETIRED_KEYS` pops this key and `PRODUCTS_KEY` from a tenant's document on every sync, but
|
| 3260 |
# the key constants, the field builders and the readers in `_READERS` all still exist, the
|
|
@@ -5341,27 +5467,37 @@ def grid_choices(rt, connection_id=None):
|
|
| 5341 |
by a curated table is covered there; every remaining readable model receives a stable raw-model
|
| 5342 |
key and defaults off. Stored choices survive catalogue refreshes because config is keyed by the
|
| 5343 |
stable database key, never by its position.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5344 |
"""
|
| 5345 |
chosen = read_config(rt)["grids"]
|
| 5346 |
namespace = connection_namespace(rt, connection_id)[1] if connection_id else None
|
| 5347 |
-
|
|
|
|
|
|
|
|
|
|
| 5348 |
fixed = [{"key": key, "label": label, "model": ODOO_TABLE_MODELS.get(key, ""),
|
| 5349 |
"relevant": True, "defaultEnabled": True,
|
| 5350 |
"enabled": bool(chosen.get(key, True))}
|
| 5351 |
for _bucket, key, label, _fields in TABLES]
|
|
|
|
|
|
|
| 5352 |
# Before discovery, keep the established curated defaults. A complete stamped catalogue is
|
| 5353 |
# authoritative for this API account, so a curated database whose model is absent must not be
|
| 5354 |
# offered: relevance never widens access.
|
| 5355 |
-
if catalogue["complete"] and catalogue["capturedAt"]:
|
| 5356 |
readable = {str(row.get("model") or "") for row in catalogue["models"]}
|
| 5357 |
-
fixed = [row for row in fixed if
|
| 5358 |
covered = {row["model"] for row in fixed if row["model"]}
|
| 5359 |
dynamic = []
|
| 5360 |
try:
|
| 5361 |
from core import connector_spec
|
| 5362 |
-
for item in catalogue["models"]:
|
| 5363 |
model = str(item.get("model") or "")
|
| 5364 |
-
if not model or model in covered:
|
| 5365 |
continue
|
| 5366 |
key = connector_spec.odoo_model_table_key(model)
|
| 5367 |
dynamic.append({"key": key,
|
|
@@ -5795,7 +5931,7 @@ def plan(cur, rt=None, connection_id=None, buckets=None):
|
|
| 5795 |
existing = {}
|
| 5796 |
if rt is not None:
|
| 5797 |
try:
|
| 5798 |
-
existing =
|
| 5799 |
except Exception: # noqa: BLE001
|
| 5800 |
existing = {}
|
| 5801 |
# ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and
|
|
@@ -5814,12 +5950,10 @@ def plan(cur, rt=None, connection_id=None, buckets=None):
|
|
| 5814 |
# be eligible FOR), but the law is written without an exception, so this lends the evaluator
|
| 5815 |
# the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so
|
| 5816 |
# what is lent is the document as it stands the moment this plan applies.
|
| 5817 |
-
|
| 5818 |
-
|
| 5819 |
-
|
| 5820 |
-
|
| 5821 |
-
for k in READ_THROUGH_KEYS}},
|
| 5822 |
-
ut.STORE_KEY)
|
| 5823 |
|
| 5824 |
built, limits = {}, {}
|
| 5825 |
caps = {}
|
|
@@ -6192,9 +6326,14 @@ def apply_plan(rt, built, username="automation", today=None, report=None,
|
|
| 6192 |
generated = connector_spec.system_link_fields("odoo", live_columns)
|
| 6193 |
for key, label, fields, rows in plans:
|
| 6194 |
existing = {field["key"] for field in fields}
|
|
|
|
|
|
|
|
|
|
| 6195 |
fields = [*fields, *namespaced_fields(
|
| 6196 |
[_preset(field) for field in generated.get(base_table_key(key), ())
|
| 6197 |
-
if field["key"] not in existing
|
|
|
|
|
|
|
| 6198 |
written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp)
|
| 6199 |
from core import connector_sync
|
| 6200 |
cur[key]["connector"] = {
|
|
@@ -6590,7 +6729,7 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None,
|
|
| 6590 |
if not selected:
|
| 6591 |
return {}
|
| 6592 |
ut = _ut()
|
| 6593 |
-
existing =
|
| 6594 |
# ⛔ HOTFIX 2026-09-24: a model the store's byte budget already refused is NOT pulled again.
|
| 6595 |
# Without this, every resync re-read all of it from Odoo (account.invoice.line.agent: 250,000
|
| 6596 |
# rows) only for the store door to discard the rows. The refusal and its fix are on the wire
|
|
|
|
| 2572 |
#: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole
|
| 2573 |
#: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code.
|
| 2574 |
#: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits.
|
| 2575 |
+
#: (key, label, type, converter) per payment fact, in column order.
|
| 2576 |
+
_PAYMENT_FACTS = (
|
| 2577 |
+
("amount_company_currency_signed", "Company currency amount", "currency",
|
| 2578 |
+
lambda value: float(value or 0)),
|
| 2579 |
+
("date", "Date", "date", _as_date), ("state", "State", "text", str),
|
| 2580 |
+
("payment_type", "Payment type", "text", str), ("partner_type", "Partner type", "text", str),
|
| 2581 |
+
("is_internal_transfer", "Internal transfer", "checkbox", bool),
|
| 2582 |
+
("partner_id", "Customer ID", "int", lambda value: int(value or 0)))
|
| 2583 |
+
|
| 2584 |
+
|
| 2585 |
def payment_fields():
|
| 2586 |
"""Declare company-currency payment facts without materializing source rows."""
|
| 2587 |
+
# The pinned identity column is what ODOO_SPEC's payments row names (pinned and identity).
|
| 2588 |
+
return [_preset({"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
|
| 2589 |
+
"default": True, "pinned": True,
|
| 2590 |
+
"description": "The `account.payment` id. Also this row's id."})] + [
|
| 2591 |
+
_preset({"key": key, "label": label, "type": kind, "source": "overlay", "default": True})
|
| 2592 |
+
for key, label, kind, _convert in _PAYMENT_FACTS]
|
|
|
|
| 2593 |
|
| 2594 |
|
| 2595 |
def payment_source_specs():
|
| 2596 |
"""Bind the payment database to its native payment identity and gross receipt amount."""
|
| 2597 |
+
cols = {"odoo_id": ("pay.id", lambda value: int(value or 0))}
|
| 2598 |
+
cols.update({key: ("pay." + key, convert) for key, _label, _kind, convert in _PAYMENT_FACTS})
|
| 2599 |
return {PAYMENTS_KEY: {
|
| 2600 |
"table": "account_payment", "from_sql": "account_payment pay", "tables": {"pay": "account_payment"},
|
| 2601 |
+
"alias": "pay", "id": "pay.id", "where": "", "cols": cols}}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2602 |
|
| 2603 |
|
| 2604 |
TABLES = (
|
|
|
|
| 2723 |
return out
|
| 2724 |
|
| 2725 |
|
| 2726 |
+
def _read_access(client, model):
|
| 2727 |
+
"""Odoo's own read verdict for this credential; raises when Odoo gives no answer."""
|
| 2728 |
+
return bool(client.execute_kw(
|
| 2729 |
+
model, "check_access_rights", ["read"], {"raise_exception": False}))
|
| 2730 |
+
|
| 2731 |
+
|
| 2732 |
+
def discover_readable_models(client, verdicts=None):
|
| 2733 |
"""Read-only introspection over an injected Odoo client.
|
| 2734 |
|
| 2735 |
``ir.model`` supplies candidates, ``check_access_rights('read')`` narrows them to this API
|
| 2736 |
account, and ``fields_get`` supplies only fields that account may inspect. No method in this
|
| 2737 |
path can mutate Odoo; the production client independently blocks write-shaped method names.
|
| 2738 |
+
A ``verdicts`` dict receives every candidate: True, False (an explicit refusal) or None.
|
| 2739 |
"""
|
| 2740 |
errors = []
|
| 2741 |
try:
|
|
|
|
| 2748 |
model = str((item or {}).get("model") or "").strip().lower()
|
| 2749 |
if not model:
|
| 2750 |
continue
|
| 2751 |
+
if verdicts is not None:
|
| 2752 |
+
verdicts[model] = None
|
| 2753 |
try:
|
| 2754 |
+
readable = _read_access(client, model)
|
|
|
|
| 2755 |
except Exception as exc: # noqa: BLE001
|
| 2756 |
errors.append(f"{model}: access: {type(exc).__name__}: {exc}")
|
| 2757 |
continue
|
| 2758 |
+
if verdicts is not None:
|
| 2759 |
+
verdicts[model] = readable
|
| 2760 |
+
if not readable:
|
| 2761 |
continue
|
| 2762 |
try:
|
| 2763 |
fields = client.fields_get(
|
|
|
|
| 2781 |
return str(connection_id or DEFAULT_CONNECTION)
|
| 2782 |
|
| 2783 |
|
| 2784 |
+
def _read_catalog_slot(rt, connection_id=None):
|
| 2785 |
+
"""One read of this connection's raw catalogue slot, or {} before any discovery."""
|
| 2786 |
try:
|
| 2787 |
cached = rt.get(MODEL_CATALOG_KEY) if rt is not None else None
|
| 2788 |
except Exception: # noqa: BLE001
|
| 2789 |
cached = None
|
| 2790 |
if not isinstance(cached, dict) or cached.get("version") != MODEL_CATALOG_VERSION:
|
| 2791 |
+
return {}
|
| 2792 |
+
slot = (cached.get("connections") or {}).get(_catalog_slot(connection_id))
|
| 2793 |
+
return slot if isinstance(slot, dict) else {}
|
| 2794 |
+
|
| 2795 |
+
|
| 2796 |
+
def _catalog_view(slot):
|
| 2797 |
+
slot = slot if isinstance(slot, dict) else {}
|
| 2798 |
return {"version": MODEL_CATALOG_VERSION,
|
| 2799 |
+
"complete": slot.get("complete") is True,
|
| 2800 |
+
"capturedAt": str(slot.get("capturedAt") or ""),
|
| 2801 |
+
"models": [dict(row) for row in (slot.get("models") or ())
|
| 2802 |
if isinstance(row, dict) and row.get("model")],
|
| 2803 |
+
"errors": [str(value) for value in (slot.get("errors") or ())][:50]}
|
| 2804 |
+
|
| 2805 |
+
|
| 2806 |
+
def _model_names(values):
|
| 2807 |
+
return sorted({str(value).strip().lower() for value in (values or ())
|
| 2808 |
+
if str(value or "").strip()})
|
| 2809 |
+
|
| 2810 |
+
|
| 2811 |
+
def _access_view(slot):
|
| 2812 |
+
"""Who answered this slot, and which models Odoo refused or does not expose."""
|
| 2813 |
+
slot = slot if isinstance(slot, dict) else {}
|
| 2814 |
+
return {"source": str(slot.get("source") or ""),
|
| 2815 |
+
"denied": _model_names(slot.get("denied")),
|
| 2816 |
+
"absent": _model_names(slot.get("absent"))}
|
| 2817 |
+
|
| 2818 |
+
|
| 2819 |
+
def readable_model_catalog(rt, connection_id=None):
|
| 2820 |
+
"""The last complete tenant-local catalogue; reading it never contacts Odoo."""
|
| 2821 |
+
return _catalog_view(_read_catalog_slot(rt, connection_id))
|
| 2822 |
+
|
| 2823 |
+
|
| 2824 |
+
def model_access(rt, connection_id=None):
|
| 2825 |
+
"""This slot's explicit access facts (D-748, R70); reading them never contacts Odoo."""
|
| 2826 |
+
return _access_view(_read_catalog_slot(rt, connection_id))
|
| 2827 |
+
|
| 2828 |
+
|
| 2829 |
+
def _credential_identity(rt, connection_id=None):
|
| 2830 |
+
"""The stored credential that answers for this slot, or "" when unknown. Decrypts nothing."""
|
| 2831 |
+
slot = _catalog_slot(connection_id)
|
| 2832 |
+
if slot != DEFAULT_CONNECTION:
|
| 2833 |
+
return "keychain:" + slot
|
| 2834 |
+
default = default_connection_id(rt)
|
| 2835 |
+
if default:
|
| 2836 |
+
return "keychain:" + str(default)
|
| 2837 |
+
return "env" if getattr(rt, "key", None) == "royal-imports" else ""
|
| 2838 |
+
|
| 2839 |
+
|
| 2840 |
+
def _same_credential(stamped, current):
|
| 2841 |
+
"""Evidence stamped by another credential is not this credential's; an unknown side matches."""
|
| 2842 |
+
return not stamped or not current or stamped == current
|
| 2843 |
+
|
| 2844 |
+
|
| 2845 |
+
def _merged_access(previous, verdicts, identity):
|
| 2846 |
+
"""Explicit verdicts replace earlier ones; a model Odoo did not answer keeps its last verdict."""
|
| 2847 |
+
previous = previous or {"source": "", "denied": [], "absent": []}
|
| 2848 |
+
denied = set(previous["denied"])
|
| 2849 |
+
for model, verdict in verdicts.items():
|
| 2850 |
+
if verdict is True:
|
| 2851 |
+
denied.discard(model)
|
| 2852 |
+
elif verdict is False:
|
| 2853 |
+
denied.add(model)
|
| 2854 |
+
# An answered ir.model listing is the exposure fact; without one the last fact stands.
|
| 2855 |
+
absent = ({model for model in ODOO_TABLE_MODELS.values() if model and model not in verdicts}
|
| 2856 |
+
if verdicts else set(previous["absent"]))
|
| 2857 |
+
return {"source": identity or previous["source"], "denied": sorted(denied),
|
| 2858 |
+
"absent": sorted(absent)}
|
| 2859 |
+
|
| 2860 |
+
|
| 2861 |
+
def _write_catalog_slot(rt, connection_id, fn):
|
| 2862 |
+
"""Rewrite one connection's slot through `fn`; every other slot is carried unchanged."""
|
| 2863 |
+
slot_key = _catalog_slot(connection_id)
|
| 2864 |
+
|
| 2865 |
+
def _cache(current_cache):
|
| 2866 |
+
current_cache = dict(current_cache) if isinstance(current_cache, dict) else {}
|
| 2867 |
+
connections = dict(current_cache.get("connections") or {})
|
| 2868 |
+
connections[slot_key] = fn(dict(connections.get(slot_key) or {}))
|
| 2869 |
+
return {"version": MODEL_CATALOG_VERSION, "connections": connections}
|
| 2870 |
+
|
| 2871 |
+
rt.update(MODEL_CATALOG_KEY, _cache, flush="sync")
|
| 2872 |
+
|
| 2873 |
+
|
| 2874 |
+
def _narrow_catalog(rt, connection_id, current, previous, access):
|
| 2875 |
+
"""Withdraw refused and unexposed models from a retained slot; nothing here widens it."""
|
| 2876 |
+
withdrawn = set(access["denied"]) | set(access["absent"])
|
| 2877 |
+
models = [row for row in current["models"] if row.get("model") not in withdrawn]
|
| 2878 |
+
if (models == current["models"] and access["denied"] == previous["denied"]
|
| 2879 |
+
and access["absent"] == previous["absent"]):
|
| 2880 |
+
return current
|
| 2881 |
+
|
| 2882 |
+
def _narrowed(stored):
|
| 2883 |
+
# A narrowing is not a capture: complete, capturedAt, errors and the stamp stay as stored.
|
| 2884 |
+
if not stored:
|
| 2885 |
+
stored = {"source": access["source"]}
|
| 2886 |
+
stored["models"] = [row for row in (stored.get("models") or ())
|
| 2887 |
+
if not (isinstance(row, dict) and row.get("model") in withdrawn)]
|
| 2888 |
+
stored["denied"], stored["absent"] = access["denied"], access["absent"]
|
| 2889 |
+
return stored
|
| 2890 |
+
|
| 2891 |
+
_write_catalog_slot(rt, connection_id, _narrowed)
|
| 2892 |
+
return {**current, "models": models}
|
| 2893 |
|
| 2894 |
|
| 2895 |
def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=False):
|
| 2896 |
"""Populate one tenant's catalogue once, or explicitly refresh it; never widen on failure."""
|
| 2897 |
+
slot = _read_catalog_slot(rt, connection_id)
|
| 2898 |
+
current, previous = _catalog_view(slot), _access_view(slot)
|
| 2899 |
+
identity = _credential_identity(rt, connection_id)
|
| 2900 |
+
same = _same_credential(previous["source"], identity)
|
| 2901 |
+
# A slot another credential answered is never fresh for this one, whatever its age.
|
| 2902 |
+
if (current["capturedAt"] and not force and same
|
| 2903 |
and not any(_missing_selection_metadata(row) for row in current["models"])):
|
| 2904 |
try:
|
| 2905 |
captured = _dt.datetime.fromisoformat(current["capturedAt"])
|
|
|
|
| 2916 |
source = None
|
| 2917 |
if source is None:
|
| 2918 |
return current
|
| 2919 |
+
verdicts = {}
|
| 2920 |
try:
|
| 2921 |
if client is None:
|
| 2922 |
from core.odoo import get_odoo
|
| 2923 |
with source._bound(): # tenant credential + scope
|
| 2924 |
+
rows, errors = discover_readable_models(get_odoo(), verdicts)
|
| 2925 |
else:
|
| 2926 |
+
rows, errors = discover_readable_models(client, verdicts)
|
| 2927 |
except Exception as exc: # noqa: BLE001
|
| 2928 |
rows, errors = [], [f"catalogue: {type(exc).__name__}: {exc}"]
|
| 2929 |
+
access = _merged_access(previous if same else None, verdicts, identity)
|
| 2930 |
# A partial retry is not evidence that a previously readable model disappeared. Preserve a
|
| 2931 |
# complete, stamped catalogue in full whenever the new pass reports any error; otherwise one
|
| 2932 |
# refused model could silently shrink the enablement roster. A first-ever partial pass may be
|
| 2933 |
# retained for diagnostics, but an entirely failed pass still has no catalogue to publish.
|
| 2934 |
+
# R70: Odoo's explicit refusal, or a model it does not expose, still withdraws that model.
|
| 2935 |
+
if errors and same and (not rows or (current["complete"] and current["capturedAt"])):
|
| 2936 |
+
return _narrow_catalog(rt, connection_id, current, previous, access)
|
| 2937 |
+
if errors and not rows:
|
| 2938 |
return current
|
| 2939 |
captured = _dt.datetime.now(_dt.timezone.utc).isoformat()
|
| 2940 |
result = {"version": MODEL_CATALOG_VERSION, "complete": not errors,
|
| 2941 |
+
"capturedAt": captured, "models": rows, "errors": errors[:50], **access}
|
| 2942 |
+
_write_catalog_slot(rt, connection_id, lambda _stored: result)
|
| 2943 |
+
return _catalog_view(result)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2944 |
|
| 2945 |
|
| 2946 |
def refresh_model_catalog_entry(rt, model, connection_id=None, client=None):
|
| 2947 |
"""Refresh one model's fields with two read-only calls, without enumerating other models."""
|
| 2948 |
+
slot = _read_catalog_slot(rt, connection_id)
|
| 2949 |
+
current, previous = _catalog_view(slot), _access_view(slot)
|
| 2950 |
+
identity = _credential_identity(rt, connection_id)
|
| 2951 |
+
same = _same_credential(previous["source"], identity)
|
| 2952 |
|
| 2953 |
def _read_fields(bound_client):
|
| 2954 |
+
if not _read_access(bound_client, model):
|
|
|
|
|
|
|
| 2955 |
return None
|
| 2956 |
fields = bound_client.fields_get(
|
| 2957 |
model, attributes=_ODOO_FIELD_ATTRIBUTES)
|
|
|
|
| 2977 |
# No answer (including unavailable credentials) is not evidence of denied model access.
|
| 2978 |
return current
|
| 2979 |
if fields is None:
|
| 2980 |
+
# R70: this credential's explicit refusal withdraws the model from its own offer.
|
| 2981 |
+
if same and (model not in previous["denied"]
|
| 2982 |
+
or any(row.get("model") == model for row in current["models"])):
|
| 2983 |
+
def _deny(stored):
|
| 2984 |
+
stored = stored or {**current, "source": identity}
|
| 2985 |
+
stored["models"] = [row for row in (stored.get("models") or ())
|
| 2986 |
+
if not (isinstance(row, dict) and row.get("model") == model)]
|
| 2987 |
+
stored["denied"] = _model_names([*(stored.get("denied") or ()), model])
|
| 2988 |
+
return stored
|
| 2989 |
+
|
| 2990 |
+
_write_catalog_slot(rt, connection_id, _deny)
|
| 2991 |
raise Refused(f"This Odoo account cannot read the model {model}. "
|
| 2992 |
"Ask your Odoo administrator to grant read access in Odoo, not here.")
|
|
|
|
| 2993 |
|
| 2994 |
+
def _merge(stored):
|
| 2995 |
+
result = stored or {**current, "source": identity}
|
|
|
|
|
|
|
| 2996 |
rows = list(result.get("models") or ())
|
| 2997 |
+
prior = next((row for row in rows if row.get("model") == model), None)
|
| 2998 |
row = {"model": model,
|
| 2999 |
+
"name": " ".join(str((prior or {}).get("name") or model).split()),
|
| 3000 |
"fields": fields}
|
| 3001 |
result["models"] = [row if item.get("model") == model else item for item in rows]
|
| 3002 |
+
if prior is None:
|
| 3003 |
result["models"].append(row)
|
| 3004 |
# One model neither captures a whole catalogue nor changes its completeness. Preserve
|
| 3005 |
# complete and capturedAt exactly, along with every unrelated row and connection slot.
|
| 3006 |
+
# Its explicit read verdict does clear an earlier refusal for this credential.
|
| 3007 |
+
if same:
|
| 3008 |
+
for name in ("denied", "absent"):
|
| 3009 |
+
if name in result:
|
| 3010 |
+
result[name] = [value for value in _model_names(result[name]) if value != model]
|
| 3011 |
+
return result
|
| 3012 |
+
|
| 3013 |
+
_write_catalog_slot(rt, connection_id, _merge)
|
| 3014 |
return readable_model_catalog(rt, connection_id)
|
| 3015 |
|
| 3016 |
|
|
|
|
| 3377 |
"pinned": "invoice_no", "identity": (_ROW_ID_COL, "odoo_id"),
|
| 3378 |
"links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O),)},
|
| 3379 |
|
| 3380 |
+
# Read-through like the line grains; payments name customers and vendors alike, so no link.
|
| 3381 |
+
{"key": PAYMENTS_KEY, "source": "account_payment", "label": "Odoo payments",
|
| 3382 |
+
"pinned": "odoo_id", "identity": (_ROW_ID_COL, "odoo_id"), "links": ()},
|
| 3383 |
+
|
| 3384 |
# ⚠ RETIRED FROM `TABLES` BY W33-T44, DECLARED HERE ANYWAY, AND THAT IS THE POINT OF A SPEC.
|
| 3385 |
# `RETIRED_KEYS` pops this key and `PRODUCTS_KEY` from a tenant's document on every sync, but
|
| 3386 |
# the key constants, the field builders and the readers in `_READERS` all still exist, the
|
|
|
|
| 5467 |
by a curated table is covered there; every remaining readable model receives a stable raw-model
|
| 5468 |
key and defaults off. Stored choices survive catalogue refreshes because config is keyed by the
|
| 5469 |
stable database key, never by its position.
|
| 5470 |
+
|
| 5471 |
+
S79-T66 / R70 (D-748): a model is withdrawn only on this credential's own evidence. Odoo's
|
| 5472 |
+
explicit read refusal, or a curated model it does not expose, withdraws it at any catalogue
|
| 5473 |
+
age or completeness; no answer keeps the curated default. A slot another credential answered
|
| 5474 |
+
offers no discovered model at all.
|
| 5475 |
"""
|
| 5476 |
chosen = read_config(rt)["grids"]
|
| 5477 |
namespace = connection_namespace(rt, connection_id)[1] if connection_id else None
|
| 5478 |
+
slot = _read_catalog_slot(rt, namespace)
|
| 5479 |
+
catalogue, access = _catalog_view(slot), _access_view(slot)
|
| 5480 |
+
own = _same_credential(access["source"], _credential_identity(rt, namespace))
|
| 5481 |
+
withdrawn = set(access["denied"]) | set(access["absent"]) if own else set()
|
| 5482 |
fixed = [{"key": key, "label": label, "model": ODOO_TABLE_MODELS.get(key, ""),
|
| 5483 |
"relevant": True, "defaultEnabled": True,
|
| 5484 |
"enabled": bool(chosen.get(key, True))}
|
| 5485 |
for _bucket, key, label, _fields in TABLES]
|
| 5486 |
+
# A row with no model names nothing the connector exposes, so it is never offered.
|
| 5487 |
+
fixed = [row for row in fixed if row["model"] and row["model"] not in withdrawn]
|
| 5488 |
# Before discovery, keep the established curated defaults. A complete stamped catalogue is
|
| 5489 |
# authoritative for this API account, so a curated database whose model is absent must not be
|
| 5490 |
# offered: relevance never widens access.
|
| 5491 |
+
if own and catalogue["complete"] and catalogue["capturedAt"]:
|
| 5492 |
readable = {str(row.get("model") or "") for row in catalogue["models"]}
|
| 5493 |
+
fixed = [row for row in fixed if row["model"] in readable]
|
| 5494 |
covered = {row["model"] for row in fixed if row["model"]}
|
| 5495 |
dynamic = []
|
| 5496 |
try:
|
| 5497 |
from core import connector_spec
|
| 5498 |
+
for item in (catalogue["models"] if own else ()):
|
| 5499 |
model = str(item.get("model") or "")
|
| 5500 |
+
if not model or model in covered or model in withdrawn:
|
| 5501 |
continue
|
| 5502 |
key = connector_spec.odoo_model_table_key(model)
|
| 5503 |
dynamic.append({"key": key,
|
|
|
|
| 5931 |
existing = {}
|
| 5932 |
if rt is not None:
|
| 5933 |
try:
|
| 5934 |
+
existing = rt.get(ut.STORE_KEY) or {}
|
| 5935 |
except Exception: # noqa: BLE001
|
| 5936 |
existing = {}
|
| 5937 |
# ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and
|
|
|
|
| 5950 |
# be eligible FOR), but the law is written without an exception, so this lends the evaluator
|
| 5951 |
# the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so
|
| 5952 |
# what is lent is the document as it stands the moment this plan applies.
|
| 5953 |
+
for k in READ_THROUGH_KEYS:
|
| 5954 |
+
tk = table_key_for(k, connection_id)
|
| 5955 |
+
existing[tk] = {**(existing.get(tk) or {}), "readThrough": True}
|
| 5956 |
+
lent = _LentDoc(existing, ut.STORE_KEY)
|
|
|
|
|
|
|
| 5957 |
|
| 5958 |
built, limits = {}, {}
|
| 5959 |
caps = {}
|
|
|
|
| 6326 |
generated = connector_spec.system_link_fields("odoo", live_columns)
|
| 6327 |
for key, label, fields, rows in plans:
|
| 6328 |
existing = {field["key"] for field in fields}
|
| 6329 |
+
# A declared link on the same column to the same target already serves that edge.
|
| 6330 |
+
declared = {((field.get("link") or {}).get("table"), (field.get("link") or {}).get("from"))
|
| 6331 |
+
for field in fields if field.get("type") == "link"}
|
| 6332 |
fields = [*fields, *namespaced_fields(
|
| 6333 |
[_preset(field) for field in generated.get(base_table_key(key), ())
|
| 6334 |
+
if field["key"] not in existing
|
| 6335 |
+
and (field["link"]["table"], field["link"]["from"]) not in declared],
|
| 6336 |
+
connection_id)]
|
| 6337 |
written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp)
|
| 6338 |
from core import connector_sync
|
| 6339 |
cur[key]["connector"] = {
|
|
|
|
| 6729 |
if not selected:
|
| 6730 |
return {}
|
| 6731 |
ut = _ut()
|
| 6732 |
+
existing = rt.get(ut.STORE_KEY) or {}
|
| 6733 |
# ⛔ HOTFIX 2026-09-24: a model the store's byte budget already refused is NOT pulled again.
|
| 6734 |
# Without this, every resync re-read all of it from Odoo (account.invoice.line.agent: 250,000
|
| 6735 |
# rows) only for the store door to discard the rows. The refusal and its fix are on the wire
|
api/process_role.py
CHANGED
|
@@ -1,8 +1,15 @@
|
|
| 1 |
"""One process rule for every database: heavy background work runs only in the sync worker.
|
| 2 |
|
| 3 |
Roles: `web` serves reads and installs finished mirror snapshots, `worker` runs every heavy job,
|
| 4 |
-
`all` does both in one process (local runs and gates)
|
| 5 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
"""
|
| 7 |
from __future__ import annotations
|
| 8 |
|
|
@@ -10,6 +17,7 @@ import atexit
|
|
| 10 |
import json
|
| 11 |
import os
|
| 12 |
import shutil
|
|
|
|
| 13 |
import subprocess
|
| 14 |
import sys
|
| 15 |
import threading
|
|
@@ -17,13 +25,16 @@ import time
|
|
| 17 |
import uuid
|
| 18 |
from pathlib import Path
|
| 19 |
|
| 20 |
-
ROLES = ("web", "worker", "all")
|
| 21 |
KINDS = ("heavy", "light")
|
|
|
|
| 22 |
API_DIR = Path(__file__).resolve().parent
|
| 23 |
POLL_SECONDS = 15
|
| 24 |
HEARTBEAT_SECONDS = 30
|
| 25 |
HEARTBEAT_FILE = "worker.heartbeat.json"
|
| 26 |
READY_FILE = "READY.json"
|
|
|
|
|
|
|
| 27 |
INSTALL_ATTEMPTS = 3
|
| 28 |
INCOMPLETE_GRACE_SECONDS = 3600
|
| 29 |
|
|
@@ -45,7 +56,7 @@ ROLE, ROLE_SOURCE = resolve_role()
|
|
| 45 |
os.environ["AIOS_PROCESS_ROLE"] = ROLE
|
| 46 |
if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web":
|
| 47 |
print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True)
|
| 48 |
-
if ROLE
|
| 49 |
os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB")
|
| 50 |
os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2")
|
| 51 |
|
|
@@ -95,9 +106,14 @@ class _Refused:
|
|
| 95 |
|
| 96 |
|
| 97 |
def thread(*, target, name, kind, daemon=True, args=(), kwargs=None, delay=None, role_name=None):
|
| 98 |
-
"""An unstarted Thread, or a Timer when delay is given. A refused job gets an inert stand-in.
|
|
|
|
|
|
|
|
|
|
| 99 |
if not admit(name, kind, role_name):
|
| 100 |
return _Refused(name)
|
|
|
|
|
|
|
| 101 |
if delay is None:
|
| 102 |
job = threading.Thread(target=target, args=tuple(args), kwargs=dict(kwargs or {}),
|
| 103 |
daemon=daemon, name=name)
|
|
@@ -130,6 +146,305 @@ def started_jobs():
|
|
| 130 |
return {name: dict(row) for name, row in _STARTED.items()}
|
| 131 |
|
| 132 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
# ----------------------------------------------------------------------------- the hand-off dir
|
| 134 |
def handoff_dir():
|
| 135 |
return Path(os.environ.get("AIOS_MIRROR_HANDOFF_DIR") or "/tmp/aios-mirror-handoff")
|
|
@@ -200,9 +515,15 @@ def ready_handoffs(root=None):
|
|
| 200 |
def _install_default(connector, tenant, ready):
|
| 201 |
from harness import mirror_store, runtime
|
| 202 |
rt = runtime.get_runtime(tenant)
|
|
|
|
| 203 |
paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
|
| 204 |
-
|
| 205 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
try:
|
| 207 |
import routes_odoo_tables
|
| 208 |
routes_odoo_tables.invalidate_sources()
|
|
@@ -211,6 +532,14 @@ def _install_default(connector, tenant, ready):
|
|
| 211 |
return installed
|
| 212 |
|
| 213 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 214 |
def _remove_dir(path):
|
| 215 |
shutil.rmtree(path, ignore_errors=True)
|
| 216 |
|
|
@@ -336,6 +665,7 @@ def _web_watch_loop():
|
|
| 336 |
while not _STOP.wait(POLL_SECONDS):
|
| 337 |
try:
|
| 338 |
install_ready()
|
|
|
|
| 339 |
except Exception as exc: # noqa: BLE001
|
| 340 |
print(f"[process-role] hand-off scan failed ({type(exc).__name__}: {exc})", flush=True)
|
| 341 |
try:
|
|
@@ -359,7 +689,7 @@ def write_heartbeat(now=None):
|
|
| 359 |
resync = dict(getattr(app_main, "RESYNC_STATE", None) or {})
|
| 360 |
_write_json(handoff_dir() / HEARTBEAT_FILE,
|
| 361 |
{"pid": os.getpid(), "at": time.time() if now is None else now,
|
| 362 |
-
"catchUp": catch_up, "resync": resync})
|
| 363 |
|
| 364 |
|
| 365 |
def _heartbeat_loop(parent_pid):
|
|
@@ -382,6 +712,8 @@ def start_worker_services():
|
|
| 382 |
mirror_store.register_publish_listener(write_handoff)
|
| 383 |
parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
|
| 384 |
start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
|
|
|
|
|
|
|
| 385 |
start("sync-desk-poll", "heavy", _worker_desk_loop)
|
| 386 |
|
| 387 |
|
|
@@ -391,6 +723,8 @@ def _worker_env():
|
|
| 391 |
AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
|
| 392 |
for name in ("AIOS_DUCKDB_MEMORY", "AIOS_DUCKDB_THREADS"):
|
| 393 |
env.pop(name, None) # the worker sets its own caps
|
|
|
|
|
|
|
| 394 |
return env
|
| 395 |
|
| 396 |
|
|
@@ -406,6 +740,7 @@ def _supervise():
|
|
| 406 |
if proc is not None:
|
| 407 |
_WORKER.update(proc=proc, pid=proc.pid, startedAt=started)
|
| 408 |
print(f"[process-role] sync worker started pid={proc.pid}", flush=True)
|
|
|
|
| 409 |
code = proc.wait()
|
| 410 |
_WORKER["lastExit"] = code
|
| 411 |
else:
|
|
@@ -435,11 +770,21 @@ def stop_worker():
|
|
| 435 |
|
| 436 |
|
| 437 |
def start_web_services():
|
| 438 |
-
"""Web role only: forward Sync now, supervise the worker, install its snapshots.
|
|
|
|
|
|
|
|
|
|
| 439 |
if ROLE != "web" or _WORKER.get("supervisor") is not None:
|
| 440 |
return False
|
| 441 |
from core import connector_sync
|
|
|
|
| 442 |
connector_sync.register_request_listener(_forward_request)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 443 |
_WORKER["supervisor"] = start("sync-worker-supervisor", "light", _supervise)
|
| 444 |
start("mirror-handoff-watcher", "light", _web_watch_loop)
|
| 445 |
atexit.register(stop_worker)
|
|
@@ -460,7 +805,8 @@ def status():
|
|
| 460 |
age = None
|
| 461 |
if isinstance(beat, dict) and beat.get("pid") == pid and beat.get("at"):
|
| 462 |
age = round(max(0.0, time.time() - float(beat["at"])), 1)
|
| 463 |
-
return {"role": ROLE, "roleSource": ROLE_SOURCE,
|
| 464 |
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
| 465 |
"restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
|
| 466 |
-
"lastInstallError": dict(LAST_INSTALL_ERROR) or None}
|
|
|
|
|
|
| 1 |
"""One process rule for every database: heavy background work runs only in the sync worker.
|
| 2 |
|
| 3 |
Roles: `web` serves reads and installs finished mirror snapshots, `worker` runs every heavy job,
|
| 4 |
+
`all` does both in one process (local runs and gates), `reader` serves reads, spawns no worker and
|
| 5 |
+
never syncs (a second Space on the same store). Kinds: `heavy` moves source data or writes the
|
| 6 |
+
store in the background; `light` serves this process's own reads.
|
| 7 |
+
|
| 8 |
+
One sync writer per store: a worker runs its heavy jobs only while it holds the store's writer
|
| 9 |
+
lease, so a second Space's worker waits instead of becoming a second writer.
|
| 10 |
+
|
| 11 |
+
Several uvicorn workers (AIOS_UVICORN_WORKERS > 1): one web worker holds the primary lock and
|
| 12 |
+
supervises the sync worker and installs hand-offs. The others only read the mirror read-only.
|
| 13 |
"""
|
| 14 |
from __future__ import annotations
|
| 15 |
|
|
|
|
| 17 |
import json
|
| 18 |
import os
|
| 19 |
import shutil
|
| 20 |
+
import socket
|
| 21 |
import subprocess
|
| 22 |
import sys
|
| 23 |
import threading
|
|
|
|
| 25 |
import uuid
|
| 26 |
from pathlib import Path
|
| 27 |
|
| 28 |
+
ROLES = ("web", "worker", "all", "reader")
|
| 29 |
KINDS = ("heavy", "light")
|
| 30 |
+
SERVING_ROLES = ("web", "all", "reader")
|
| 31 |
API_DIR = Path(__file__).resolve().parent
|
| 32 |
POLL_SECONDS = 15
|
| 33 |
HEARTBEAT_SECONDS = 30
|
| 34 |
HEARTBEAT_FILE = "worker.heartbeat.json"
|
| 35 |
READY_FILE = "READY.json"
|
| 36 |
+
PRIMARY_LOCK = "web-primary.lock"
|
| 37 |
+
PRIMARY_FILE = "web-primary.json"
|
| 38 |
INSTALL_ATTEMPTS = 3
|
| 39 |
INCOMPLETE_GRACE_SECONDS = 3600
|
| 40 |
|
|
|
|
| 56 |
os.environ["AIOS_PROCESS_ROLE"] = ROLE
|
| 57 |
if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web":
|
| 58 |
print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True)
|
| 59 |
+
if ROLE in ("web", "reader"):
|
| 60 |
os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB")
|
| 61 |
os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2")
|
| 62 |
|
|
|
|
| 106 |
|
| 107 |
|
| 108 |
def thread(*, target, name, kind, daemon=True, args=(), kwargs=None, delay=None, role_name=None):
|
| 109 |
+
"""An unstarted Thread, or a Timer when delay is given. A refused job gets an inert stand-in.
|
| 110 |
+
|
| 111 |
+
While the writer lease is armed, a heavy job's target first waits until this process holds it.
|
| 112 |
+
"""
|
| 113 |
if not admit(name, kind, role_name):
|
| 114 |
return _Refused(name)
|
| 115 |
+
if kind == "heavy" and _WRITER["armed"]:
|
| 116 |
+
target = _after_writer_lease(target, name)
|
| 117 |
if delay is None:
|
| 118 |
job = threading.Thread(target=target, args=tuple(args), kwargs=dict(kwargs or {}),
|
| 119 |
daemon=daemon, name=name)
|
|
|
|
| 146 |
return {name: dict(row) for name, row in _STARTED.items()}
|
| 147 |
|
| 148 |
|
| 149 |
+
# ----------------------------------------------------------------------------- the writer lease
|
| 150 |
+
WRITER_LEASE_DOC = "sync_writer_lease"
|
| 151 |
+
WRITER_LEASE_SECONDS = 180
|
| 152 |
+
WRITER_POLL_SECONDS = 15
|
| 153 |
+
_WRITER = {"armed": False, "holder": None, "renewedAt": None, "lostAt": None, "waitingSince": None,
|
| 154 |
+
"error": None}
|
| 155 |
+
_WRITER_HELD = threading.Event()
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def writer_holder(env=None):
|
| 159 |
+
"""This container's lease identity, so a restarted worker in it takes its predecessor's lease."""
|
| 160 |
+
env = os.environ if env is None else env
|
| 161 |
+
return f"{env.get('SPACE_ID') or 'local'}@{socket.gethostname()}"
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def acquire_writer(rt, holder=None, *, now=None, ttl=None):
|
| 165 |
+
"""Take or renew the store's writer lease. True when `holder` holds it after the call.
|
| 166 |
+
|
| 167 |
+
The store's update is the compare-and-set: Postgres locks the row for the read-modify-write.
|
| 168 |
+
"""
|
| 169 |
+
holder = holder or writer_holder()
|
| 170 |
+
stamp = time.time() if now is None else float(now)
|
| 171 |
+
ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl)
|
| 172 |
+
|
| 173 |
+
def take(doc):
|
| 174 |
+
doc = dict(doc or {})
|
| 175 |
+
current = doc.get("holder")
|
| 176 |
+
expired = stamp - float(doc.get("renewedAt") or 0) > ttl
|
| 177 |
+
if current in (None, "", holder) or expired:
|
| 178 |
+
if current != holder:
|
| 179 |
+
doc.update(acquiredAt=stamp, previous=current)
|
| 180 |
+
doc.update(holder=holder, renewedAt=stamp, pid=os.getpid())
|
| 181 |
+
return doc
|
| 182 |
+
|
| 183 |
+
doc = rt.update(WRITER_LEASE_DOC, take)
|
| 184 |
+
if not isinstance(doc, dict):
|
| 185 |
+
doc = rt.get(WRITER_LEASE_DOC, fresh=True)
|
| 186 |
+
return isinstance(doc, dict) and doc.get("holder") == holder
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def release_writer(rt, holder=None):
|
| 190 |
+
"""Give the lease up at a clean exit, so the next writer need not wait for it to expire."""
|
| 191 |
+
holder = holder or writer_holder()
|
| 192 |
+
|
| 193 |
+
def drop(doc):
|
| 194 |
+
doc = dict(doc or {})
|
| 195 |
+
if doc.get("holder") == holder:
|
| 196 |
+
doc.update(holder=None, renewedAt=0, previous=holder)
|
| 197 |
+
return doc
|
| 198 |
+
|
| 199 |
+
rt.update(WRITER_LEASE_DOC, drop)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def holds_writer():
|
| 203 |
+
return _WRITER_HELD.is_set()
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def may_write(role_name=None):
|
| 207 |
+
"""allows('heavy'), plus the writer lease when armed. For a store write outside a heavy thread."""
|
| 208 |
+
return allows("heavy", role_name) and (not _WRITER["armed"] or _WRITER_HELD.is_set())
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _after_writer_lease(target, name):
|
| 212 |
+
"""Wrap a heavy job so it starts only after this process holds the writer lease."""
|
| 213 |
+
def gated(*args, **kwargs):
|
| 214 |
+
if not _WRITER_HELD.is_set():
|
| 215 |
+
print(f"[process-role] {name} waits for the sync writer lease", flush=True)
|
| 216 |
+
_WRITER_HELD.wait()
|
| 217 |
+
return target(*args, **kwargs)
|
| 218 |
+
gated.__name__ = getattr(target, "__name__", name)
|
| 219 |
+
return gated
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def writer_lease_tick(rt_factory, holder, *, now=None, ttl=None):
|
| 223 |
+
"""One acquire or renew. Returns 'held', 'waiting' or 'lost'. 'lost' means stop writing now."""
|
| 224 |
+
stamp = time.time() if now is None else float(now)
|
| 225 |
+
ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl)
|
| 226 |
+
try:
|
| 227 |
+
held = acquire_writer(rt_factory(), holder, now=stamp, ttl=ttl)
|
| 228 |
+
except Exception as exc: # noqa: BLE001
|
| 229 |
+
held = None
|
| 230 |
+
if _WRITER.get("error") != type(exc).__name__:
|
| 231 |
+
# Fail closed: no lease, no heavy job. Set AIOS_SYNC_WRITER_TENANT to a writable tenant.
|
| 232 |
+
print(f"[process-role] sync writer lease unreadable ({type(exc).__name__}); heavy jobs wait",
|
| 233 |
+
flush=True)
|
| 234 |
+
_WRITER["error"] = type(exc).__name__
|
| 235 |
+
else:
|
| 236 |
+
_WRITER["error"] = None
|
| 237 |
+
if held:
|
| 238 |
+
if not _WRITER_HELD.is_set():
|
| 239 |
+
print(f"[process-role] sync writer lease held by {holder}", flush=True)
|
| 240 |
+
_WRITER.update(renewedAt=stamp, waitingSince=None)
|
| 241 |
+
_WRITER_HELD.set()
|
| 242 |
+
return "held"
|
| 243 |
+
if _WRITER_HELD.is_set():
|
| 244 |
+
# Another holder took it, or renewals failed long enough that one may take it next.
|
| 245 |
+
if held is False or stamp - float(_WRITER["renewedAt"] or 0) > ttl - 2 * WRITER_POLL_SECONDS:
|
| 246 |
+
_WRITER_HELD.clear()
|
| 247 |
+
_WRITER["lostAt"] = stamp
|
| 248 |
+
print("[process-role] sync writer lease lost; this worker stops writing", flush=True)
|
| 249 |
+
return "lost"
|
| 250 |
+
return "held"
|
| 251 |
+
if _WRITER["waitingSince"] is None:
|
| 252 |
+
_WRITER["waitingSince"] = stamp
|
| 253 |
+
print("[process-role] another process holds the sync writer lease; heavy jobs wait", flush=True)
|
| 254 |
+
return "waiting"
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def _writer_lease_loop(rt_factory, holder, exit_fn):
|
| 258 |
+
while True:
|
| 259 |
+
if writer_lease_tick(rt_factory, holder) == "lost":
|
| 260 |
+
exit_fn(3)
|
| 261 |
+
return
|
| 262 |
+
if _STOP.wait(WRITER_POLL_SECONDS):
|
| 263 |
+
return
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def _writer_runtime():
|
| 267 |
+
from harness import runtime
|
| 268 |
+
return runtime.get_runtime(os.environ.get("AIOS_SYNC_WRITER_TENANT") or "royal-imports")
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def _release_at_exit():
|
| 272 |
+
if _WRITER_HELD.is_set():
|
| 273 |
+
try:
|
| 274 |
+
release_writer(_writer_runtime(), _WRITER["holder"])
|
| 275 |
+
except Exception: # noqa: BLE001
|
| 276 |
+
pass
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def arm_writer_lease(rt_factory=None, holder=None, exit_fn=None):
|
| 280 |
+
"""Worker only: every later heavy job waits for the lease. Call before any heavy job starts."""
|
| 281 |
+
if _WRITER["armed"]:
|
| 282 |
+
return False
|
| 283 |
+
_WRITER.update(armed=True, holder=holder or writer_holder())
|
| 284 |
+
# Light: it must run before the lease is held, and it writes only the lease document.
|
| 285 |
+
start("sync-writer-lease", "light", _writer_lease_loop,
|
| 286 |
+
args=(rt_factory or _writer_runtime, _WRITER["holder"], exit_fn or os._exit))
|
| 287 |
+
if rt_factory is None:
|
| 288 |
+
atexit.register(_release_at_exit)
|
| 289 |
+
return True
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
# ----------------------------------------------------------------------------- readiness (C9)
|
| 293 |
+
READY_CHECKS = ("mirror", "pools", "first_reads")
|
| 294 |
+
_READY_DEFAULT = {"mirror": "The mirror is not installed yet.",
|
| 295 |
+
"pools": "The read pools are not open yet.",
|
| 296 |
+
"first_reads": "The first reads have not answered yet."}
|
| 297 |
+
_READINESS = {}
|
| 298 |
+
_BOOT_RESTORE = {}
|
| 299 |
+
# The first full pass latches the boot conditions: a later worker restart or a fresh hand-off
|
| 300 |
+
# must not pull a warm web process out of rotation.
|
| 301 |
+
_READY_LATCH = {"at": None}
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def reset_readiness():
|
| 305 |
+
_READY_LATCH["at"] = None
|
| 306 |
+
_READINESS.clear()
|
| 307 |
+
_READINESS.update({name: {"ok": False, "detail": text, "at": None} for name, text in _READY_DEFAULT.items()})
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
reset_readiness()
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def mark_ready(check, ok, detail=""):
|
| 314 |
+
"""Record one readiness check. A check passes only after its own step finished in this process."""
|
| 315 |
+
if check not in _READY_DEFAULT:
|
| 316 |
+
raise ValueError(f"unknown readiness check {check!r}")
|
| 317 |
+
_READINESS[check] = {"ok": bool(ok), "detail": "" if ok else (detail or _READY_DEFAULT[check]),
|
| 318 |
+
"at": time.time()}
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def note_boot_restore(state="done"):
|
| 322 |
+
"""Worker side: its boot restore finished, so every hand-off it made is already on disk."""
|
| 323 |
+
_BOOT_RESTORE.update(state=state, at=time.time(), pid=os.getpid())
|
| 324 |
+
try:
|
| 325 |
+
write_heartbeat()
|
| 326 |
+
except Exception as exc: # noqa: BLE001
|
| 327 |
+
print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True)
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def worker_boot_restored(root=None):
|
| 331 |
+
"""Web side: the current worker's heartbeat says its boot restore finished."""
|
| 332 |
+
beat = _read_json(Path(root or handoff_dir()) / HEARTBEAT_FILE) or {}
|
| 333 |
+
restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
|
| 334 |
+
# A secondary web worker supervises nothing: the primary names the current sync worker.
|
| 335 |
+
pid = _WORKER.get("pid") if is_primary() else primary_state(root).get("workerPid")
|
| 336 |
+
return bool(restore.get("state")) and pid is not None and beat.get("pid") == pid
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def _installed_map(root=None):
|
| 340 |
+
if is_primary():
|
| 341 |
+
return dict(_INSTALLED)
|
| 342 |
+
installed = primary_state(root).get("installed") or {}
|
| 343 |
+
return {tuple(key.split("/", 1)): snap for key, snap in installed.items() if "/" in key}
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
def pending_handoffs(root=None):
|
| 347 |
+
"""`tenant/connector` for each newest READY hand-off not installed yet (by the primary)."""
|
| 348 |
+
installed = _installed_map(root)
|
| 349 |
+
return sorted(f"{key[0]}/{key[1]}" for key, rows in ready_handoffs(root).items()
|
| 350 |
+
if installed.get(key) != (rows[-1][2].get("snapshotId") or rows[-1][1].name))
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
# ----------------------------------------------------------------------------- several web workers
|
| 354 |
+
_PRIMARY = {"fd": None}
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
def web_workers(env=None):
|
| 358 |
+
"""How many uvicorn workers serve this container, from AIOS_UVICORN_WORKERS. 1 when unset."""
|
| 359 |
+
env = os.environ if env is None else env
|
| 360 |
+
try:
|
| 361 |
+
return max(1, int(env.get("AIOS_UVICORN_WORKERS") or 1))
|
| 362 |
+
except ValueError:
|
| 363 |
+
return 1
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
def is_primary():
|
| 367 |
+
"""True when this web process supervises the sync worker and installs. Always for one worker."""
|
| 368 |
+
return web_workers() == 1 or _PRIMARY["fd"] is not None
|
| 369 |
+
|
| 370 |
+
|
| 371 |
+
def claim_primary(root=None):
|
| 372 |
+
"""Take the container's primary lock without waiting. True when this process holds it.
|
| 373 |
+
|
| 374 |
+
The lock is a kernel flock: it drops when its holder dies, so a standby can take over.
|
| 375 |
+
"""
|
| 376 |
+
if is_primary():
|
| 377 |
+
return True
|
| 378 |
+
try:
|
| 379 |
+
import fcntl
|
| 380 |
+
except ImportError:
|
| 381 |
+
return False
|
| 382 |
+
path = Path(root or handoff_dir()) / PRIMARY_LOCK
|
| 383 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 384 |
+
fd = os.open(str(path), os.O_RDWR | os.O_CREAT, 0o600)
|
| 385 |
+
try:
|
| 386 |
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
| 387 |
+
except OSError:
|
| 388 |
+
os.close(fd)
|
| 389 |
+
return False
|
| 390 |
+
_PRIMARY["fd"] = fd
|
| 391 |
+
print(f"[process-role] web worker pid={os.getpid()} is the primary: it supervises the sync worker "
|
| 392 |
+
f"and installs hand-offs", flush=True)
|
| 393 |
+
write_primary_state(root)
|
| 394 |
+
return True
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
def write_primary_state(root=None):
|
| 398 |
+
"""Primary only: the sync worker pid and every installed snapshot, for the secondaries."""
|
| 399 |
+
if web_workers() == 1 or _PRIMARY["fd"] is None:
|
| 400 |
+
return False
|
| 401 |
+
try:
|
| 402 |
+
from harness import datastore
|
| 403 |
+
version = datastore.serving_file()[1]
|
| 404 |
+
except Exception: # noqa: BLE001
|
| 405 |
+
version = None
|
| 406 |
+
_write_json(Path(root or handoff_dir()) / PRIMARY_FILE, {
|
| 407 |
+
"pid": os.getpid(), "workerPid": _WORKER.get("pid"), "at": time.time(),
|
| 408 |
+
"installed": {f"{key[0]}/{key[1]}": snap for key, snap in _INSTALLED.items()},
|
| 409 |
+
"mirrorVersion": version})
|
| 410 |
+
return True
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
def primary_state(root=None):
|
| 414 |
+
state = _read_json(Path(root or handoff_dir()) / PRIMARY_FILE)
|
| 415 |
+
return state if isinstance(state, dict) else {}
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
def _standby_loop():
|
| 419 |
+
while not _STOP.wait(POLL_SECONDS):
|
| 420 |
+
if claim_primary():
|
| 421 |
+
start_web_services()
|
| 422 |
+
return
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
def readiness(mirror_live=None, role_name=None, root=None):
|
| 426 |
+
"""{ready, checks: {name: bool}, failing: [names], detail: {name: text}} for GET /api/v1/ready."""
|
| 427 |
+
role_name = role_name or ROLE
|
| 428 |
+
checks, detail = {}, {}
|
| 429 |
+
for name in READY_CHECKS:
|
| 430 |
+
row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")}
|
| 431 |
+
checks[name], detail[name] = bool(row["ok"]), row["detail"]
|
| 432 |
+
boot = role_name == "web" and _READY_LATCH["at"] is None
|
| 433 |
+
if "mirror" in checks and checks["mirror"]:
|
| 434 |
+
if mirror_live is False:
|
| 435 |
+
checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading."
|
| 436 |
+
elif boot and not worker_boot_restored(root):
|
| 437 |
+
checks["mirror"], detail["mirror"] = False, "The sync worker has not finished its boot restore."
|
| 438 |
+
elif boot and pending_handoffs(root):
|
| 439 |
+
checks["mirror"] = False
|
| 440 |
+
detail["mirror"] = "Mirror hand-offs are not installed yet: " + ", ".join(pending_handoffs(root)) + "."
|
| 441 |
+
failing = [name for name in checks if not checks[name]]
|
| 442 |
+
if not failing and _READY_LATCH["at"] is None:
|
| 443 |
+
_READY_LATCH["at"] = time.time()
|
| 444 |
+
return {"ready": not failing, "checks": checks, "failing": failing,
|
| 445 |
+
"detail": {name: detail[name] for name in failing}}
|
| 446 |
+
|
| 447 |
+
|
| 448 |
# ----------------------------------------------------------------------------- the hand-off dir
|
| 449 |
def handoff_dir():
|
| 450 |
return Path(os.environ.get("AIOS_MIRROR_HANDOFF_DIR") or "/tmp/aios-mirror-handoff")
|
|
|
|
| 515 |
def _install_default(connector, tenant, ready):
|
| 516 |
from harness import mirror_store, runtime
|
| 517 |
rt = runtime.get_runtime(tenant)
|
| 518 |
+
from harness import datastore
|
| 519 |
paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
|
| 520 |
+
resident = Path(rt.datastore_path).resolve() == datastore.DB_PATH.resolve()
|
| 521 |
+
with (datastore.versioned_write(tenant) if resident else _no_version()) as version:
|
| 522 |
+
installed = mirror_store.install_handoff(connector, tenant, rt, ready["manifest"],
|
| 523 |
+
ready.get("stateRows") or [], paths)
|
| 524 |
+
if version is not None:
|
| 525 |
+
# Already resident: keep serving the current version, publish nothing.
|
| 526 |
+
version["publish"] = installed is not False
|
| 527 |
try:
|
| 528 |
import routes_odoo_tables
|
| 529 |
routes_odoo_tables.invalidate_sources()
|
|
|
|
| 532 |
return installed
|
| 533 |
|
| 534 |
|
| 535 |
+
class _no_version:
|
| 536 |
+
def __enter__(self):
|
| 537 |
+
return None
|
| 538 |
+
|
| 539 |
+
def __exit__(self, *exc):
|
| 540 |
+
return False
|
| 541 |
+
|
| 542 |
+
|
| 543 |
def _remove_dir(path):
|
| 544 |
shutil.rmtree(path, ignore_errors=True)
|
| 545 |
|
|
|
|
| 665 |
while not _STOP.wait(POLL_SECONDS):
|
| 666 |
try:
|
| 667 |
install_ready()
|
| 668 |
+
write_primary_state()
|
| 669 |
except Exception as exc: # noqa: BLE001
|
| 670 |
print(f"[process-role] hand-off scan failed ({type(exc).__name__}: {exc})", flush=True)
|
| 671 |
try:
|
|
|
|
| 689 |
resync = dict(getattr(app_main, "RESYNC_STATE", None) or {})
|
| 690 |
_write_json(handoff_dir() / HEARTBEAT_FILE,
|
| 691 |
{"pid": os.getpid(), "at": time.time() if now is None else now,
|
| 692 |
+
"catchUp": catch_up, "resync": resync, "bootRestore": dict(_BOOT_RESTORE)})
|
| 693 |
|
| 694 |
|
| 695 |
def _heartbeat_loop(parent_pid):
|
|
|
|
| 712 |
mirror_store.register_publish_listener(write_handoff)
|
| 713 |
parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
|
| 714 |
start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
|
| 715 |
+
# Before the first heavy job: each one now waits until this worker holds the writer lease.
|
| 716 |
+
arm_writer_lease()
|
| 717 |
start("sync-desk-poll", "heavy", _worker_desk_loop)
|
| 718 |
|
| 719 |
|
|
|
|
| 723 |
AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
|
| 724 |
for name in ("AIOS_DUCKDB_MEMORY", "AIOS_DUCKDB_THREADS"):
|
| 725 |
env.pop(name, None) # the worker sets its own caps
|
| 726 |
+
# The worker writes its own file read-write and is never a web worker.
|
| 727 |
+
env.update(AIOS_MIRROR_VERSIONS="0", AIOS_UVICORN_WORKERS="1")
|
| 728 |
return env
|
| 729 |
|
| 730 |
|
|
|
|
| 740 |
if proc is not None:
|
| 741 |
_WORKER.update(proc=proc, pid=proc.pid, startedAt=started)
|
| 742 |
print(f"[process-role] sync worker started pid={proc.pid}", flush=True)
|
| 743 |
+
write_primary_state()
|
| 744 |
code = proc.wait()
|
| 745 |
_WORKER["lastExit"] = code
|
| 746 |
else:
|
|
|
|
| 770 |
|
| 771 |
|
| 772 |
def start_web_services():
|
| 773 |
+
"""Web role only: forward Sync now, supervise the worker, install its snapshots.
|
| 774 |
+
|
| 775 |
+
With several uvicorn workers only the primary does this. The others stand by and read.
|
| 776 |
+
"""
|
| 777 |
if ROLE != "web" or _WORKER.get("supervisor") is not None:
|
| 778 |
return False
|
| 779 |
from core import connector_sync
|
| 780 |
+
# Every web worker forwards the Sync now it accepts: the desk lives in the store.
|
| 781 |
connector_sync.register_request_listener(_forward_request)
|
| 782 |
+
if not claim_primary():
|
| 783 |
+
if _WORKER.get("standby") is None:
|
| 784 |
+
_WORKER["standby"] = start("web-primary-standby", "light", _standby_loop)
|
| 785 |
+
print(f"[process-role] web worker pid={os.getpid()} serves reads only; "
|
| 786 |
+
f"the primary supervises the sync worker", flush=True)
|
| 787 |
+
return False
|
| 788 |
_WORKER["supervisor"] = start("sync-worker-supervisor", "light", _supervise)
|
| 789 |
start("mirror-handoff-watcher", "light", _web_watch_loop)
|
| 790 |
atexit.register(stop_worker)
|
|
|
|
| 805 |
age = None
|
| 806 |
if isinstance(beat, dict) and beat.get("pid") == pid and beat.get("at"):
|
| 807 |
age = round(max(0.0, time.time() - float(beat["at"])), 1)
|
| 808 |
+
return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
|
| 809 |
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
| 810 |
"restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
|
| 811 |
+
"lastInstallError": dict(LAST_INSTALL_ERROR) or None},
|
| 812 |
+
"writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
|
api/providers.py
CHANGED
|
@@ -34,7 +34,9 @@ lets a tenant be moved off a vendor without a release.
|
|
| 34 |
from __future__ import annotations
|
| 35 |
|
| 36 |
import json
|
|
|
|
| 37 |
import os
|
|
|
|
| 38 |
import time
|
| 39 |
from dataclasses import dataclass, field
|
| 40 |
from datetime import datetime, timezone
|
|
@@ -573,7 +575,86 @@ def llm_chain(capability="llm_tool_calling"):
|
|
| 573 |
raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip()
|
| 574 |
names = [x.strip() for x in raw.split(",") if x.strip() in LLM_DEFAULT_ORDER] or list(LLM_DEFAULT_ORDER)
|
| 575 |
return [LLM_PROVIDERS[n] for n in names
|
| 576 |
-
if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 577 |
|
| 578 |
|
| 579 |
#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND
|
|
@@ -743,6 +824,20 @@ def anthropic_send(req, timeout=None, use_sdk=None):
|
|
| 743 |
later check in that process running on the wrong path and passing for the wrong reason.
|
| 744 |
"""
|
| 745 |
sdk = None if use_sdk is False else anthropic_sdk()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 746 |
payload = dict(req.get("json") or {})
|
| 747 |
if sdk is None:
|
| 748 |
r = requests.post(req["url"], headers=req["headers"], json=payload,
|
|
@@ -754,7 +849,8 @@ def anthropic_send(req, timeout=None, use_sdk=None):
|
|
| 754 |
|
| 755 |
key = (req.get("headers") or {}).get("x-api-key") or ""
|
| 756 |
try:
|
| 757 |
-
client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)
|
|
|
|
| 758 |
msg = client.messages.create(**payload)
|
| 759 |
# ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same
|
| 760 |
# wire-shaped dict the raw POST parses out of the response body.
|
|
|
|
| 34 |
from __future__ import annotations
|
| 35 |
|
| 36 |
import json
|
| 37 |
+
import math
|
| 38 |
import os
|
| 39 |
+
import threading
|
| 40 |
import time
|
| 41 |
from dataclasses import dataclass, field
|
| 42 |
from datetime import datetime, timezone
|
|
|
|
| 575 |
raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip()
|
| 576 |
names = [x.strip() for x in raw.split(",") if x.strip() in LLM_DEFAULT_ORDER] or list(LLM_DEFAULT_ORDER)
|
| 577 |
return [LLM_PROVIDERS[n] for n in names
|
| 578 |
+
if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)
|
| 579 |
+
and not llm_breaker_open(n)]
|
| 580 |
+
|
| 581 |
+
|
| 582 |
+
# ═══════════ LLM CIRCUIT BREAKER, ONE PER PROVIDER NAME (S79-T43) ═══════════════════════════════
|
| 583 |
+
# After N consecutive failures (transport, 408, 5xx) a provider opens for a cooldown and is skipped.
|
| 584 |
+
# After the cooldown it half-opens: the next recorded outcome closes it or re-opens it.
|
| 585 |
+
# ⛔ 400, 401, 402, 404, 413 and 429 are ANSWERS, so they never count. Credit has `_NO_CREDIT`.
|
| 586 |
+
# ⚠ Process-local like `_NO_CREDIT`: a latency guard, not a record.
|
| 587 |
+
_monotonic = time.monotonic # Gates patch this clock so no leg sleeps.
|
| 588 |
+
_LLM_BREAKERS: dict[str, dict] = {} # name -> {"failures": int, "opened_at": float | None, "trial": bool}
|
| 589 |
+
_LLM_BREAKER_LOCK = threading.Lock()
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
def _llm_breaker_setting(name, default, minimum):
|
| 593 |
+
try:
|
| 594 |
+
value = float(os.environ.get(name) or default)
|
| 595 |
+
return max(minimum, value) if math.isfinite(value) else float(default)
|
| 596 |
+
except ValueError:
|
| 597 |
+
return float(default)
|
| 598 |
+
|
| 599 |
+
|
| 600 |
+
def _llm_breaker_cooling(state):
|
| 601 |
+
"""Is an open breaker still inside its cooldown?"""
|
| 602 |
+
cooldown = _llm_breaker_setting("AIOS_LLM_BREAKER_COOLDOWN_S", 30, 0)
|
| 603 |
+
return _monotonic() - state["opened_at"] < cooldown
|
| 604 |
+
|
| 605 |
+
|
| 606 |
+
def llm_breaker_record(name, status):
|
| 607 |
+
"""Record one call outcome. 0 is a transport exception, 2xx closes, 0, 408 and 5xx count."""
|
| 608 |
+
code = int(status or 0)
|
| 609 |
+
with _LLM_BREAKER_LOCK:
|
| 610 |
+
if 200 <= code <= 299:
|
| 611 |
+
_LLM_BREAKERS.pop(str(name), None)
|
| 612 |
+
return
|
| 613 |
+
if not (code == 0 or code == 408 or 500 <= code <= 599):
|
| 614 |
+
if str(name) in _LLM_BREAKERS:
|
| 615 |
+
_LLM_BREAKERS[str(name)]["trial"] = False # An answer ends a trial, count unchanged.
|
| 616 |
+
return
|
| 617 |
+
state = _LLM_BREAKERS.setdefault(str(name), {"failures": 0, "opened_at": None, "trial": False})
|
| 618 |
+
state["failures"] += 1
|
| 619 |
+
limit = _llm_breaker_setting("AIOS_LLM_BREAKER_FAILURES", 5, 1)
|
| 620 |
+
if state["opened_at"] is not None or state["failures"] >= limit:
|
| 621 |
+
state["opened_at"] = _monotonic()
|
| 622 |
+
state["trial"] = False
|
| 623 |
+
|
| 624 |
+
|
| 625 |
+
def llm_breaker_open(name):
|
| 626 |
+
"""Is this provider refused right now? False while closed or while a half-open trial is allowed."""
|
| 627 |
+
with _LLM_BREAKER_LOCK:
|
| 628 |
+
state = _LLM_BREAKERS.get(str(name))
|
| 629 |
+
if state is None or state["opened_at"] is None:
|
| 630 |
+
return False
|
| 631 |
+
return bool(state["trial"] or _llm_breaker_cooling(state))
|
| 632 |
+
|
| 633 |
+
|
| 634 |
+
def _llm_breaker_admit(name):
|
| 635 |
+
"""Admit one call: True for the half-open trial, False when closed, None while open."""
|
| 636 |
+
with _LLM_BREAKER_LOCK:
|
| 637 |
+
state = _LLM_BREAKERS.get(str(name))
|
| 638 |
+
if state is None or state["opened_at"] is None:
|
| 639 |
+
return False
|
| 640 |
+
if state["trial"] or _llm_breaker_cooling(state):
|
| 641 |
+
return None
|
| 642 |
+
state["trial"] = True
|
| 643 |
+
return True
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
def llm_breaker_reset(name=None):
|
| 647 |
+
"""Close one breaker, or all of them. For a gate."""
|
| 648 |
+
with _LLM_BREAKER_LOCK:
|
| 649 |
+
if name is None:
|
| 650 |
+
_LLM_BREAKERS.clear()
|
| 651 |
+
else:
|
| 652 |
+
_LLM_BREAKERS.pop(str(name), None)
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
def _llm_max_retries():
|
| 656 |
+
"""SDK retries per call. 0 bounds one call by its own timeout."""
|
| 657 |
+
return int(_llm_breaker_setting("AIOS_LLM_MAX_RETRIES", 0, 0))
|
| 658 |
|
| 659 |
|
| 660 |
#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND
|
|
|
|
| 824 |
later check in that process running on the wrong path and passing for the wrong reason.
|
| 825 |
"""
|
| 826 |
sdk = None if use_sdk is False else anthropic_sdk()
|
| 827 |
+
transport = "http" if sdk is None else "sdk"
|
| 828 |
+
if _llm_breaker_admit("anthropic") is None:
|
| 829 |
+
return 503, {"error": {"type": "llm_breaker_open", "message": (
|
| 830 |
+
"Anthropic failed several times in a row, so calls pause briefly before the next try.")}}, transport
|
| 831 |
+
status = 0
|
| 832 |
+
try:
|
| 833 |
+
status, body, transport = _anthropic_post(sdk, req, timeout)
|
| 834 |
+
return status, body, transport
|
| 835 |
+
finally:
|
| 836 |
+
llm_breaker_record("anthropic", status)
|
| 837 |
+
|
| 838 |
+
|
| 839 |
+
def _anthropic_post(sdk, req, timeout):
|
| 840 |
+
"""One Anthropic call on the chosen transport. A raised exception reaches the breaker as 0."""
|
| 841 |
payload = dict(req.get("json") or {})
|
| 842 |
if sdk is None:
|
| 843 |
r = requests.post(req["url"], headers=req["headers"], json=payload,
|
|
|
|
| 849 |
|
| 850 |
key = (req.get("headers") or {}).get("x-api-key") or ""
|
| 851 |
try:
|
| 852 |
+
client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60),
|
| 853 |
+
max_retries=_llm_max_retries())
|
| 854 |
msg = client.messages.create(**payload)
|
| 855 |
# ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same
|
| 856 |
# wire-shaped dict the raw POST parses out of the response body.
|
api/request_class.py
CHANGED
|
@@ -1,18 +1,27 @@
|
|
| 1 |
-
"""Heavy request admission:
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
"""
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
import asyncio
|
|
|
|
| 10 |
import json
|
| 11 |
import os
|
| 12 |
import re
|
| 13 |
import sys
|
| 14 |
import threading
|
| 15 |
import time
|
|
|
|
| 16 |
from urllib.parse import parse_qs
|
| 17 |
|
| 18 |
BUSY_CODE = "server_busy"
|
|
@@ -36,23 +45,32 @@ def _sheet_read_heavy(query):
|
|
| 36 |
return not (tabs == "meta" and not _flag(query, "opening"))
|
| 37 |
|
| 38 |
|
| 39 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
HEAVY = (
|
| 41 |
-
({"GET"}, r"/api/v1/workspace", lambda q: not _flag(q, "fields_only")),
|
| 42 |
-
({"GET"}, r"/api/v1/products", None),
|
| 43 |
-
({"GET"}, r"/api/v1/customers", None),
|
| 44 |
-
({"GET", "POST"}, r"/api/v1/odoo-tables/[^/]+/rows", None),
|
| 45 |
-
({"GET", "POST"}, r"/api/v1/connected-tables/[^/]+/rows", None),
|
| 46 |
-
({"GET"}, r"/api/v1/tables/[^/]+/rows", None),
|
| 47 |
-
({"GET"}, r"/api/v1/sheets/(?!index$|import$|automation$)[^/]+", _sheet_read_heavy),
|
| 48 |
-
({"GET"}, r"/api/v1/sheets/[^/]+/tabs/[^/]+", None),
|
| 49 |
# In the web role a refresh only queues for the sync worker; it computes only in worker or all.
|
| 50 |
-
({"POST"}, r"/api/v1/sheets/[^/]+/refresh", lambda q: os.environ.get("AIOS_PROCESS_ROLE") != "web"),
|
| 51 |
-
({"GET", "POST"}, r"/api/v1/.*(?:/export|-export)(?:/.*)?", None),
|
| 52 |
-
({"POST"}, r"/api/v1/(?:external|typed)/databases/[^/]+/query", None),
|
| 53 |
-
({"POST"}, r"/api/v1/external/explorer/run", None),
|
| 54 |
)
|
| 55 |
-
_HEAVY = tuple((methods, re.compile(rx), pred) for methods, rx, pred in HEAVY)
|
|
|
|
|
|
|
| 56 |
|
| 57 |
# MCP tools/call names that read sheets or query databases.
|
| 58 |
MCP_HEAVY_TOOLS = {"sheet", "table", "data"}
|
|
@@ -88,18 +106,50 @@ def _mcp_heavy(body):
|
|
| 88 |
return False
|
| 89 |
|
| 90 |
|
| 91 |
-
def
|
| 92 |
-
"""
|
| 93 |
method = (method or "").upper()
|
| 94 |
if isinstance(query, bytes):
|
| 95 |
query = query.decode("latin-1")
|
| 96 |
q = parse_qs(query, keep_blank_values=True) if isinstance(query, str) else (query or {})
|
| 97 |
-
for methods, rx, pred in _HEAVY:
|
| 98 |
if method in methods and rx.fullmatch(path or "") and (pred is None or pred(q)):
|
| 99 |
-
return
|
| 100 |
if method == "POST" and path == MCP_PATH and _mcp_heavy(body):
|
| 101 |
-
return
|
| 102 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
|
| 104 |
|
| 105 |
class _Counters:
|
|
@@ -120,17 +170,40 @@ class _Counters:
|
|
| 120 |
|
| 121 |
|
| 122 |
_COUNTERS = _Counters()
|
|
|
|
|
|
|
| 123 |
|
| 124 |
|
| 125 |
def status():
|
| 126 |
"""Snapshot of heavy request admission for a settings or health surface."""
|
| 127 |
c = _COUNTERS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
with c.lock:
|
| 129 |
-
|
|
|
|
|
|
|
| 130 |
"queueSeconds": (c.queue_seconds if c.queue_seconds is not None
|
| 131 |
else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0)),
|
| 132 |
"inFlight": c.in_flight, "queued": c.queued,
|
| 133 |
-
"busyRefusals": c.busy_refusals, "admitted": c.admitted
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
|
| 135 |
|
| 136 |
def _env_int(name, default):
|
|
@@ -147,8 +220,32 @@ def _env_float(name, default):
|
|
| 147 |
return default
|
| 148 |
|
| 149 |
|
| 150 |
-
|
| 151 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
await send({"type": "http.response.start", "status": 503,
|
| 153 |
"headers": [(b"content-type", b"application/json"),
|
| 154 |
(b"content-length", str(len(body)).encode()),
|
|
@@ -157,25 +254,143 @@ async def _send_busy(send, cause="heavy_queue_timeout"):
|
|
| 157 |
await send({"type": "http.response.body", "body": body, "more_body": False})
|
| 158 |
|
| 159 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
class HeavyGate:
|
| 161 |
-
"""Pure ASGI middleware. Heavy requests take a permit; light ones pass
|
| 162 |
|
| 163 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
self.app = app
|
| 165 |
self.concurrency = concurrency or _env_int("AIOS_HEAVY_CONCURRENCY", 2)
|
| 166 |
self.queue_seconds = (queue_seconds if queue_seconds is not None
|
| 167 |
else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0))
|
| 168 |
self.counters = counters or _COUNTERS
|
| 169 |
self.counters.limit, self.counters.queue_seconds = self.concurrency, self.queue_seconds
|
| 170 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
self._loop = None
|
| 172 |
|
| 173 |
-
def _semaphore(self):
|
| 174 |
-
#
|
| 175 |
loop = asyncio.get_running_loop()
|
| 176 |
-
if self.
|
| 177 |
-
self.
|
| 178 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
|
| 180 |
async def __call__(self, scope, receive, send):
|
| 181 |
if scope.get("type") != "http":
|
|
@@ -189,23 +404,34 @@ class HeavyGate:
|
|
| 189 |
return
|
| 190 |
if classify(method, path, scope.get("query_string", b""), peek) != "heavy":
|
| 191 |
return await self.app(scope, _replay(body_msgs, receive), send)
|
|
|
|
| 192 |
if body_msgs is None:
|
| 193 |
body_msgs, _ = await _read_body(receive, None)
|
| 194 |
if body_msgs is None:
|
| 195 |
return
|
| 196 |
-
await self._run_heavy(scope, body_msgs, receive, send)
|
| 197 |
|
| 198 |
-
async def _run_heavy(self, scope, body_msgs, receive, send):
|
| 199 |
queued_at = time.perf_counter()
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
timing = sys.modules.get("core.request_timing")
|
| 204 |
budget_error = getattr(timing, "RequestDeadlineExceeded", TimeoutError)
|
| 205 |
acquire = asyncio.ensure_future(sem.acquire())
|
| 206 |
gone = asyncio.ensure_future(_wait_disconnect(receive))
|
| 207 |
try:
|
| 208 |
-
timeout = timing.remaining_budget(
|
| 209 |
done, _ = await asyncio.wait({acquire, gone}, timeout=timeout,
|
| 210 |
return_when=asyncio.FIRST_COMPLETED)
|
| 211 |
if timing:
|
|
@@ -213,8 +439,8 @@ class HeavyGate:
|
|
| 213 |
except budget_error:
|
| 214 |
if acquire.done() and not acquire.cancelled() and acquire.exception() is None:
|
| 215 |
sem.release()
|
| 216 |
-
|
| 217 |
-
await _send_busy(send, "request_deadline")
|
| 218 |
return
|
| 219 |
except BaseException:
|
| 220 |
# Cancellation can race a successful acquire before the handler owns the permit.
|
|
@@ -226,7 +452,7 @@ class HeavyGate:
|
|
| 226 |
timing = sys.modules.get("core.request_timing")
|
| 227 |
if timing is not None:
|
| 228 |
timing.record_queue((time.perf_counter() - queued_at) * 1000)
|
| 229 |
-
|
| 230 |
if not acquire.done():
|
| 231 |
acquire.cancel()
|
| 232 |
if not gone.done():
|
|
@@ -240,15 +466,15 @@ class HeavyGate:
|
|
| 240 |
except budget_error:
|
| 241 |
if got:
|
| 242 |
sem.release()
|
| 243 |
-
|
| 244 |
-
await _send_busy(send, "request_deadline")
|
| 245 |
return
|
| 246 |
if acquire not in done or gone in done:
|
| 247 |
if got:
|
| 248 |
sem.release()
|
| 249 |
if gone not in done:
|
| 250 |
-
|
| 251 |
-
await _send_busy(send)
|
| 252 |
return
|
| 253 |
released = False
|
| 254 |
|
|
@@ -256,7 +482,7 @@ class HeavyGate:
|
|
| 256 |
nonlocal released
|
| 257 |
if not released:
|
| 258 |
released = True
|
| 259 |
-
|
| 260 |
sem.release()
|
| 261 |
|
| 262 |
async def send_wrapped(message):
|
|
@@ -264,8 +490,8 @@ class HeavyGate:
|
|
| 264 |
if message.get("type") == "http.response.body" and not message.get("more_body"):
|
| 265 |
release()
|
| 266 |
|
| 267 |
-
|
| 268 |
-
|
| 269 |
try:
|
| 270 |
await self.app(scope, _replay(body_msgs, receive), send_wrapped)
|
| 271 |
finally:
|
|
@@ -307,3 +533,180 @@ def _replay(msgs, receive):
|
|
| 307 |
return pending.pop(0)
|
| 308 |
return await receive()
|
| 309 |
return replay
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Heavy request admission: bounded queues so CPU-heavy reads never starve light requests.
|
| 2 |
+
|
| 3 |
+
`classify` names the heavy doors in ONE registry, each in a named request class (grid rows,
|
| 4 |
+
workspace, sheet, export, query). `HeavyGate` gives every class its own limit and queue bound in
|
| 5 |
+
each web process: no permit is shared between classes, so a full class answers a 503
|
| 6 |
+
`server_busy` naming itself while another class keeps admitting. Inside a class, waiting requests
|
| 7 |
+
are admitted round robin by tenant, so one tenant's burst queues behind itself and another
|
| 8 |
+
tenant's read is admitted within its queue bound. Light requests never wait.
|
| 9 |
+
|
| 10 |
+
Box sizing lives here too: `python request_class.py --env` reads the container's CPU and memory
|
| 11 |
+
limits and prints the launch values (uvicorn workers, heavy concurrency, DuckDB caps).
|
| 12 |
+
Run as a script it imports only the standard library.
|
| 13 |
"""
|
| 14 |
from __future__ import annotations
|
| 15 |
|
| 16 |
import asyncio
|
| 17 |
+
import hashlib
|
| 18 |
import json
|
| 19 |
import os
|
| 20 |
import re
|
| 21 |
import sys
|
| 22 |
import threading
|
| 23 |
import time
|
| 24 |
+
from collections import deque
|
| 25 |
from urllib.parse import parse_qs
|
| 26 |
|
| 27 |
BUSY_CODE = "server_busy"
|
|
|
|
| 45 |
return not (tabs == "meta" and not _flag(query, "opening"))
|
| 46 |
|
| 47 |
|
| 48 |
+
#: The heavy request classes. Each has its own limit (`AIOS_HEAVY_CONCURRENCY_<CLASS>`, default
|
| 49 |
+
#: `AIOS_HEAVY_CONCURRENCY` for grid_rows and half of it for the others, see `class_limit`) and
|
| 50 |
+
#: queue bound (`AIOS_HEAVY_QUEUE_SECONDS_<CLASS>`, default `AIOS_HEAVY_QUEUE_SECONDS`).
|
| 51 |
+
CLASSES = ("grid_rows", "workspace", "sheet", "export", "query")
|
| 52 |
+
#: The class a heavy request with no registry row lands in (a caller that names no class).
|
| 53 |
+
DEFAULT_CLASS = "grid_rows"
|
| 54 |
+
|
| 55 |
+
# (class, methods, path regex, query predicate or None). A request is heavy when one row matches.
|
| 56 |
HEAVY = (
|
| 57 |
+
("workspace", {"GET"}, r"/api/v1/workspace", lambda q: not _flag(q, "fields_only")),
|
| 58 |
+
("grid_rows", {"GET"}, r"/api/v1/products", None),
|
| 59 |
+
("grid_rows", {"GET"}, r"/api/v1/customers", None),
|
| 60 |
+
("grid_rows", {"GET", "POST"}, r"/api/v1/odoo-tables/[^/]+/rows", None),
|
| 61 |
+
("grid_rows", {"GET", "POST"}, r"/api/v1/connected-tables/[^/]+/rows", None),
|
| 62 |
+
("grid_rows", {"GET"}, r"/api/v1/tables/[^/]+/rows", None),
|
| 63 |
+
("sheet", {"GET"}, r"/api/v1/sheets/(?!index$|import$|automation$)[^/]+", _sheet_read_heavy),
|
| 64 |
+
("sheet", {"GET"}, r"/api/v1/sheets/[^/]+/tabs/[^/]+", None),
|
| 65 |
# In the web role a refresh only queues for the sync worker; it computes only in worker or all.
|
| 66 |
+
("sheet", {"POST"}, r"/api/v1/sheets/[^/]+/refresh", lambda q: os.environ.get("AIOS_PROCESS_ROLE") != "web"),
|
| 67 |
+
("export", {"GET", "POST"}, r"/api/v1/.*(?:/export|-export)(?:/.*)?", None),
|
| 68 |
+
("query", {"POST"}, r"/api/v1/(?:external|typed)/databases/[^/]+/query", None),
|
| 69 |
+
("query", {"POST"}, r"/api/v1/external/explorer/run", None),
|
| 70 |
)
|
| 71 |
+
_HEAVY = tuple((cls, methods, re.compile(rx), pred) for cls, methods, rx, pred in HEAVY)
|
| 72 |
+
#: The class of a heavy MCP tools/call (sheet reads and database queries).
|
| 73 |
+
MCP_CLASS = "query"
|
| 74 |
|
| 75 |
# MCP tools/call names that read sheets or query databases.
|
| 76 |
MCP_HEAVY_TOOLS = {"sheet", "table", "data"}
|
|
|
|
| 106 |
return False
|
| 107 |
|
| 108 |
|
| 109 |
+
def request_class(method, path, query="", body=None):
|
| 110 |
+
"""The heavy class of one request, or None when it is light."""
|
| 111 |
method = (method or "").upper()
|
| 112 |
if isinstance(query, bytes):
|
| 113 |
query = query.decode("latin-1")
|
| 114 |
q = parse_qs(query, keep_blank_values=True) if isinstance(query, str) else (query or {})
|
| 115 |
+
for cls, methods, rx, pred in _HEAVY:
|
| 116 |
if method in methods and rx.fullmatch(path or "") and (pred is None or pred(q)):
|
| 117 |
+
return cls
|
| 118 |
if method == "POST" and path == MCP_PATH and _mcp_heavy(body):
|
| 119 |
+
return MCP_CLASS
|
| 120 |
+
return None
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def classify(method, path, query="", body=None):
|
| 124 |
+
"""Return "heavy" or "light" for one request. `query` is a raw query string or a parsed dict."""
|
| 125 |
+
return "heavy" if request_class(method, path, query, body) else "light"
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def request_tenant(scope):
|
| 129 |
+
"""The fairness key of one request: the signed session's tenant, else a hash of its API key.
|
| 130 |
+
|
| 131 |
+
No store read: the cookie is verified with its HMAC only. An unsigned request shares one key.
|
| 132 |
+
"""
|
| 133 |
+
cookie, auth = "", ""
|
| 134 |
+
for key, value in scope.get("headers") or ():
|
| 135 |
+
if key == b"cookie":
|
| 136 |
+
cookie = value.decode("latin-1")
|
| 137 |
+
elif key == b"authorization":
|
| 138 |
+
auth = value.decode("latin-1")
|
| 139 |
+
if cookie:
|
| 140 |
+
try:
|
| 141 |
+
import aios_session
|
| 142 |
+
for part in cookie.split(";"):
|
| 143 |
+
name, _, value = part.strip().partition("=")
|
| 144 |
+
if name == aios_session.COOKIE_NAME and value:
|
| 145 |
+
claims = aios_session.read(value)
|
| 146 |
+
if claims:
|
| 147 |
+
return "tenant:" + str(claims["t"]).strip().lower()
|
| 148 |
+
except Exception: # noqa: BLE001
|
| 149 |
+
pass
|
| 150 |
+
if auth:
|
| 151 |
+
return "key:" + hashlib.sha256(auth.encode("utf-8", "replace")).hexdigest()[:16]
|
| 152 |
+
return "anonymous"
|
| 153 |
|
| 154 |
|
| 155 |
class _Counters:
|
|
|
|
| 170 |
|
| 171 |
|
| 172 |
_COUNTERS = _Counters()
|
| 173 |
+
#: The same counters for each request class.
|
| 174 |
+
_CLASS_COUNTERS = {name: _Counters() for name in CLASSES}
|
| 175 |
|
| 176 |
|
| 177 |
def status():
|
| 178 |
"""Snapshot of heavy request admission for a settings or health surface."""
|
| 179 |
c = _COUNTERS
|
| 180 |
+
workers = web_workers()
|
| 181 |
+
classes = {}
|
| 182 |
+
for name, cc in _CLASS_COUNTERS.items():
|
| 183 |
+
with cc.lock:
|
| 184 |
+
limit, queue = class_limit(name)
|
| 185 |
+
classes[name] = {"limit": cc.limit if cc.limit is not None else limit,
|
| 186 |
+
"queueSeconds": cc.queue_seconds if cc.queue_seconds is not None else queue,
|
| 187 |
+
"inFlight": cc.in_flight, "queued": cc.queued,
|
| 188 |
+
"busyRefusals": cc.busy_refusals, "admitted": cc.admitted}
|
| 189 |
with c.lock:
|
| 190 |
+
limit = c.limit if c.limit is not None else _env_int("AIOS_HEAVY_CONCURRENCY", 2)
|
| 191 |
+
# Counters are per process: N uvicorn workers admit N times `limit` across the box.
|
| 192 |
+
return {"limit": limit, "workers": workers, "boxLimit": limit * workers,
|
| 193 |
"queueSeconds": (c.queue_seconds if c.queue_seconds is not None
|
| 194 |
else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0)),
|
| 195 |
"inFlight": c.in_flight, "queued": c.queued,
|
| 196 |
+
"busyRefusals": c.busy_refusals, "admitted": c.admitted,
|
| 197 |
+
"tenantCap": tenant_cap_env() or None, "classes": classes}
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def web_workers(env=None):
|
| 201 |
+
"""How many uvicorn workers share this box, from AIOS_UVICORN_WORKERS. 1 when unset."""
|
| 202 |
+
env = os.environ if env is None else env
|
| 203 |
+
try:
|
| 204 |
+
return max(1, int(env.get("AIOS_UVICORN_WORKERS") or 1))
|
| 205 |
+
except ValueError:
|
| 206 |
+
return 1
|
| 207 |
|
| 208 |
|
| 209 |
def _env_int(name, default):
|
|
|
|
| 220 |
return default
|
| 221 |
|
| 222 |
|
| 223 |
+
def tenant_cap_env():
|
| 224 |
+
"""AIOS_HEAVY_TENANT_CAP: most permits of one class one tenant holds. 0 or unset: the class limit."""
|
| 225 |
+
try:
|
| 226 |
+
return max(0, int(os.environ.get("AIOS_HEAVY_TENANT_CAP") or 0))
|
| 227 |
+
except ValueError:
|
| 228 |
+
return 0
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def class_limit(name):
|
| 232 |
+
"""(limit, queue seconds) of one request class from its env, else from the shared heavy env.
|
| 233 |
+
|
| 234 |
+
Unset, grid_rows takes the shared heavy limit and every other class half of it (at least 1),
|
| 235 |
+
so five independent classes do not multiply the heavy work one process admits.
|
| 236 |
+
"""
|
| 237 |
+
shared = _env_int("AIOS_HEAVY_CONCURRENCY", 2)
|
| 238 |
+
limit = shared if name == DEFAULT_CLASS else max(1, shared // 2)
|
| 239 |
+
queue = _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0)
|
| 240 |
+
return (_env_int(f"AIOS_HEAVY_CONCURRENCY_{name.upper()}", limit),
|
| 241 |
+
_env_float(f"AIOS_HEAVY_QUEUE_SECONDS_{name.upper()}", queue))
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
async def _send_busy(send, cause="heavy_queue_timeout", request_class=None):
|
| 245 |
+
error = {"code": BUSY_CODE, "cause": cause, "message": BUSY_MESSAGE}
|
| 246 |
+
if request_class:
|
| 247 |
+
error["requestClass"] = request_class
|
| 248 |
+
body = json.dumps({"error": error}).encode()
|
| 249 |
await send({"type": "http.response.start", "status": 503,
|
| 250 |
"headers": [(b"content-type", b"application/json"),
|
| 251 |
(b"content-length", str(len(body)).encode()),
|
|
|
|
| 254 |
await send({"type": "http.response.body", "body": body, "more_body": False})
|
| 255 |
|
| 256 |
|
| 257 |
+
class _FairSlots:
|
| 258 |
+
"""One class's permits in one event loop. A free permit goes to the waiting tenant served
|
| 259 |
+
least recently, so tenants take turns. A tenant holds at most `tenant_cap` permits.
|
| 260 |
+
|
| 261 |
+
FIFO would let one tenant's burst hold every other tenant's read past the queue bound.
|
| 262 |
+
"""
|
| 263 |
+
|
| 264 |
+
TURNS_KEPT = 1024
|
| 265 |
+
|
| 266 |
+
def __init__(self, limit, tenant_cap=None):
|
| 267 |
+
self.limit = max(1, int(limit))
|
| 268 |
+
self.tenant_cap = max(1, min(self.limit, int(tenant_cap or self.limit)))
|
| 269 |
+
self.running = 0
|
| 270 |
+
self.by_tenant = {}
|
| 271 |
+
self.waiting = {} # tenant -> deque of futures, oldest first
|
| 272 |
+
self.turn = {} # tenant -> the grant count at its last grant
|
| 273 |
+
self.grants = 0
|
| 274 |
+
|
| 275 |
+
@property
|
| 276 |
+
def _value(self):
|
| 277 |
+
"""Free permits, named as asyncio.Semaphore names them."""
|
| 278 |
+
return self.limit - self.running
|
| 279 |
+
|
| 280 |
+
def _dispatch(self):
|
| 281 |
+
while self.running < self.limit:
|
| 282 |
+
best = None
|
| 283 |
+
for tenant in list(self.waiting):
|
| 284 |
+
queue = self.waiting[tenant]
|
| 285 |
+
while queue and queue[0].done(): # a waiter that left (timeout, disconnect)
|
| 286 |
+
queue.popleft()
|
| 287 |
+
if not queue:
|
| 288 |
+
del self.waiting[tenant]
|
| 289 |
+
continue
|
| 290 |
+
if self.by_tenant.get(tenant, 0) >= self.tenant_cap:
|
| 291 |
+
continue
|
| 292 |
+
if best is None or self.turn.get(tenant, -1) < self.turn.get(best, -1):
|
| 293 |
+
best = tenant
|
| 294 |
+
if best is None:
|
| 295 |
+
return
|
| 296 |
+
queue = self.waiting[best]
|
| 297 |
+
self.running += 1
|
| 298 |
+
self.by_tenant[best] = self.by_tenant.get(best, 0) + 1
|
| 299 |
+
self.grants += 1
|
| 300 |
+
self.turn[best] = self.grants
|
| 301 |
+
queue.popleft().set_result(True)
|
| 302 |
+
if not queue:
|
| 303 |
+
del self.waiting[best]
|
| 304 |
+
if len(self.turn) > self.TURNS_KEPT:
|
| 305 |
+
for tenant in [t for t in self.turn if t not in self.by_tenant and t not in self.waiting]:
|
| 306 |
+
del self.turn[tenant]
|
| 307 |
+
|
| 308 |
+
async def acquire(self, tenant):
|
| 309 |
+
fut = asyncio.get_running_loop().create_future()
|
| 310 |
+
self.waiting.setdefault(tenant, deque()).append(fut)
|
| 311 |
+
self._dispatch()
|
| 312 |
+
try:
|
| 313 |
+
await fut
|
| 314 |
+
except asyncio.CancelledError:
|
| 315 |
+
if fut.done() and not fut.cancelled():
|
| 316 |
+
self.release(tenant) # granted in the same tick as the cancel
|
| 317 |
+
else:
|
| 318 |
+
fut.cancel()
|
| 319 |
+
raise
|
| 320 |
+
return True
|
| 321 |
+
|
| 322 |
+
def release(self, tenant):
|
| 323 |
+
self.running -= 1
|
| 324 |
+
held = self.by_tenant.get(tenant, 0) - 1
|
| 325 |
+
if held > 0:
|
| 326 |
+
self.by_tenant[tenant] = held
|
| 327 |
+
else:
|
| 328 |
+
self.by_tenant.pop(tenant, None)
|
| 329 |
+
self._dispatch()
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
class _Permit:
|
| 333 |
+
"""One tenant's handle on a class's permits: acquire() and release() as a Semaphore has them."""
|
| 334 |
+
|
| 335 |
+
def __init__(self, slots, tenant):
|
| 336 |
+
self.slots, self.tenant = slots, tenant
|
| 337 |
+
|
| 338 |
+
async def acquire(self):
|
| 339 |
+
return await self.slots.acquire(self.tenant)
|
| 340 |
+
|
| 341 |
+
def release(self):
|
| 342 |
+
self.slots.release(self.tenant)
|
| 343 |
+
|
| 344 |
+
@property
|
| 345 |
+
def _value(self):
|
| 346 |
+
return self.slots._value
|
| 347 |
+
|
| 348 |
+
|
| 349 |
class HeavyGate:
|
| 350 |
+
"""Pure ASGI middleware. Heavy requests take a permit of their class; light ones pass through.
|
| 351 |
|
| 352 |
+
`concurrency` and `queue_seconds` set every class at once (tests). Unset, each class reads
|
| 353 |
+
its own env, then the shared heavy env. `classes` overrides one class: {name: {limit, queueSeconds}}.
|
| 354 |
+
"""
|
| 355 |
+
|
| 356 |
+
def __init__(self, app, concurrency=None, queue_seconds=None, counters=None, classes=None,
|
| 357 |
+
tenant_cap=None):
|
| 358 |
self.app = app
|
| 359 |
self.concurrency = concurrency or _env_int("AIOS_HEAVY_CONCURRENCY", 2)
|
| 360 |
self.queue_seconds = (queue_seconds if queue_seconds is not None
|
| 361 |
else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0))
|
| 362 |
self.counters = counters or _COUNTERS
|
| 363 |
self.counters.limit, self.counters.queue_seconds = self.concurrency, self.queue_seconds
|
| 364 |
+
# A gate given its own counters (a test) keeps its own class counters too.
|
| 365 |
+
self.class_counters = ({name: _Counters() for name in CLASSES} if counters is not None
|
| 366 |
+
else _CLASS_COUNTERS)
|
| 367 |
+
self.tenant_cap = tenant_cap if tenant_cap is not None else tenant_cap_env()
|
| 368 |
+
self.limits = {}
|
| 369 |
+
for name in CLASSES:
|
| 370 |
+
limit, queue = class_limit(name)
|
| 371 |
+
if concurrency:
|
| 372 |
+
limit = concurrency
|
| 373 |
+
if queue_seconds is not None:
|
| 374 |
+
queue = queue_seconds
|
| 375 |
+
override = (classes or {}).get(name) or {}
|
| 376 |
+
limit = max(1, int(override.get("limit") or limit))
|
| 377 |
+
queue = float(override["queueSeconds"]) if override.get("queueSeconds") is not None else queue
|
| 378 |
+
self.limits[name] = (limit, queue)
|
| 379 |
+
cc = self.class_counters[name]
|
| 380 |
+
cc.limit, cc.queue_seconds = limit, queue
|
| 381 |
+
self._slots = {}
|
| 382 |
self._loop = None
|
| 383 |
|
| 384 |
+
def _semaphore(self, cls=DEFAULT_CLASS, tenant="anonymous"):
|
| 385 |
+
# Futures bind to the loop they wait on. One set of permits per loop.
|
| 386 |
loop = asyncio.get_running_loop()
|
| 387 |
+
if self._loop is not loop:
|
| 388 |
+
self._slots, self._loop = {}, loop
|
| 389 |
+
slots = self._slots.get(cls)
|
| 390 |
+
if slots is None:
|
| 391 |
+
limit = self.limits.get(cls, (self.concurrency, self.queue_seconds))[0]
|
| 392 |
+
slots = self._slots[cls] = _FairSlots(limit, self.tenant_cap or None)
|
| 393 |
+
return _Permit(slots, tenant)
|
| 394 |
|
| 395 |
async def __call__(self, scope, receive, send):
|
| 396 |
if scope.get("type") != "http":
|
|
|
|
| 404 |
return
|
| 405 |
if classify(method, path, scope.get("query_string", b""), peek) != "heavy":
|
| 406 |
return await self.app(scope, _replay(body_msgs, receive), send)
|
| 407 |
+
cls = request_class(method, path, scope.get("query_string", b""), peek) or DEFAULT_CLASS
|
| 408 |
if body_msgs is None:
|
| 409 |
body_msgs, _ = await _read_body(receive, None)
|
| 410 |
if body_msgs is None:
|
| 411 |
return
|
| 412 |
+
await self._run_heavy(scope, body_msgs, receive, send, cls=cls)
|
| 413 |
|
| 414 |
+
async def _run_heavy(self, scope, body_msgs, receive, send, cls=None, tenant=None):
|
| 415 |
queued_at = time.perf_counter()
|
| 416 |
+
if cls is None:
|
| 417 |
+
cls = request_class(scope.get("method", ""), scope.get("path", ""),
|
| 418 |
+
scope.get("query_string", b"")) or DEFAULT_CLASS
|
| 419 |
+
tenant = tenant if tenant is not None else request_tenant(scope)
|
| 420 |
+
queue_seconds = self.limits.get(cls, (self.concurrency, self.queue_seconds))[1]
|
| 421 |
+
sem = self._semaphore(cls, tenant)
|
| 422 |
+
cc = self.class_counters.get(cls) or _Counters()
|
| 423 |
+
|
| 424 |
+
def add(name, n):
|
| 425 |
+
self.counters.add(name, n)
|
| 426 |
+
cc.add(name, n)
|
| 427 |
+
|
| 428 |
+
add("queued", 1)
|
| 429 |
timing = sys.modules.get("core.request_timing")
|
| 430 |
budget_error = getattr(timing, "RequestDeadlineExceeded", TimeoutError)
|
| 431 |
acquire = asyncio.ensure_future(sem.acquire())
|
| 432 |
gone = asyncio.ensure_future(_wait_disconnect(receive))
|
| 433 |
try:
|
| 434 |
+
timeout = timing.remaining_budget(queue_seconds) if timing else queue_seconds
|
| 435 |
done, _ = await asyncio.wait({acquire, gone}, timeout=timeout,
|
| 436 |
return_when=asyncio.FIRST_COMPLETED)
|
| 437 |
if timing:
|
|
|
|
| 439 |
except budget_error:
|
| 440 |
if acquire.done() and not acquire.cancelled() and acquire.exception() is None:
|
| 441 |
sem.release()
|
| 442 |
+
add("busy_refusals", 1)
|
| 443 |
+
await _send_busy(send, "request_deadline", cls)
|
| 444 |
return
|
| 445 |
except BaseException:
|
| 446 |
# Cancellation can race a successful acquire before the handler owns the permit.
|
|
|
|
| 452 |
timing = sys.modules.get("core.request_timing")
|
| 453 |
if timing is not None:
|
| 454 |
timing.record_queue((time.perf_counter() - queued_at) * 1000)
|
| 455 |
+
add("queued", -1)
|
| 456 |
if not acquire.done():
|
| 457 |
acquire.cancel()
|
| 458 |
if not gone.done():
|
|
|
|
| 466 |
except budget_error:
|
| 467 |
if got:
|
| 468 |
sem.release()
|
| 469 |
+
add("busy_refusals", 1)
|
| 470 |
+
await _send_busy(send, "request_deadline", cls)
|
| 471 |
return
|
| 472 |
if acquire not in done or gone in done:
|
| 473 |
if got:
|
| 474 |
sem.release()
|
| 475 |
if gone not in done:
|
| 476 |
+
add("busy_refusals", 1)
|
| 477 |
+
await _send_busy(send, f"{cls}_queue_timeout", cls)
|
| 478 |
return
|
| 479 |
released = False
|
| 480 |
|
|
|
|
| 482 |
nonlocal released
|
| 483 |
if not released:
|
| 484 |
released = True
|
| 485 |
+
add("in_flight", -1)
|
| 486 |
sem.release()
|
| 487 |
|
| 488 |
async def send_wrapped(message):
|
|
|
|
| 490 |
if message.get("type") == "http.response.body" and not message.get("more_body"):
|
| 491 |
release()
|
| 492 |
|
| 493 |
+
add("in_flight", 1)
|
| 494 |
+
add("admitted", 1)
|
| 495 |
try:
|
| 496 |
await self.app(scope, _replay(body_msgs, receive), send_wrapped)
|
| 497 |
finally:
|
|
|
|
| 533 |
return pending.pop(0)
|
| 534 |
return await receive()
|
| 535 |
return replay
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
# ----------------------------------------------------------------------------- box sizing (R25)
|
| 539 |
+
#: The 2 vCPU tier: today's values, exactly. Staging (cpu-basic, 2 vCPU, 16 GB) runs these.
|
| 540 |
+
TODAY = {"workers": 1, "heavyConcurrency": 2, "webDuckdbMemory": "6GB", "webDuckdbThreads": 2,
|
| 541 |
+
"workerDuckdbMemory": "4GB", "workerDuckdbThreads": 1}
|
| 542 |
+
#: Launch env name for each sizing value. An explicit env value always wins over the derived one.
|
| 543 |
+
SIZING_ENV = {"workers": "AIOS_UVICORN_WORKERS", "heavyConcurrency": "AIOS_HEAVY_CONCURRENCY",
|
| 544 |
+
"webDuckdbMemory": "AIOS_WEB_DUCKDB_MEMORY", "webDuckdbThreads": "AIOS_WEB_DUCKDB_THREADS",
|
| 545 |
+
"workerDuckdbMemory": "AIOS_WORKER_DUCKDB_MEMORY",
|
| 546 |
+
"workerDuckdbThreads": "AIOS_WORKER_DUCKDB_THREADS"}
|
| 547 |
+
VERSIONS_ENV = "AIOS_MIRROR_VERSIONS"
|
| 548 |
+
MAX_WEB_WORKERS = 4
|
| 549 |
+
#: Share of box memory all DuckDB instances may take together (10 of 16 GB today).
|
| 550 |
+
DUCKDB_MEMORY_SHARE = 0.625
|
| 551 |
+
|
| 552 |
+
|
| 553 |
+
def _read(path):
|
| 554 |
+
try:
|
| 555 |
+
with open(path, encoding="ascii") as handle:
|
| 556 |
+
return handle.read().strip()
|
| 557 |
+
except (OSError, UnicodeError):
|
| 558 |
+
return None
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
def _cgroup_cpus(root):
|
| 562 |
+
raw = _read(os.path.join(root, "cpu.max")) # cgroup v2
|
| 563 |
+
if raw:
|
| 564 |
+
parts = raw.split()
|
| 565 |
+
if len(parts) == 2 and parts[0] != "max":
|
| 566 |
+
try:
|
| 567 |
+
return max(1, int(int(parts[0]) / int(parts[1]))), "cgroup v2 cpu.max"
|
| 568 |
+
except (ValueError, ZeroDivisionError):
|
| 569 |
+
pass
|
| 570 |
+
for sub in ("cpu", "cpu,cpuacct", "cpuacct,cpu"): # cgroup v1
|
| 571 |
+
quota = _read(os.path.join(root, sub, "cpu.cfs_quota_us"))
|
| 572 |
+
period = _read(os.path.join(root, sub, "cpu.cfs_period_us"))
|
| 573 |
+
try:
|
| 574 |
+
if quota and period and int(quota) > 0:
|
| 575 |
+
return max(1, int(int(quota) / int(period))), "cgroup v1 cpu.cfs_quota_us"
|
| 576 |
+
except (ValueError, ZeroDivisionError):
|
| 577 |
+
pass
|
| 578 |
+
return None, None
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
def _cgroup_memory(root):
|
| 582 |
+
raw = _read(os.path.join(root, "memory.max")) # cgroup v2
|
| 583 |
+
if raw and raw != "max" and raw.isdigit():
|
| 584 |
+
return int(raw), "cgroup v2 memory.max"
|
| 585 |
+
raw = _read(os.path.join(root, "memory", "memory.limit_in_bytes")) # cgroup v1
|
| 586 |
+
# v1 reports "no limit" as a huge page-aligned number.
|
| 587 |
+
if raw and raw.isdigit() and int(raw) < 1 << 60:
|
| 588 |
+
return int(raw), "cgroup v1 memory.limit_in_bytes"
|
| 589 |
+
return None, None
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
def _meminfo_bytes(path):
|
| 593 |
+
raw = _read(path) or ""
|
| 594 |
+
for line in raw.splitlines():
|
| 595 |
+
if line.startswith("MemTotal:"):
|
| 596 |
+
try:
|
| 597 |
+
return int(line.split()[1]) * 1024
|
| 598 |
+
except (IndexError, ValueError):
|
| 599 |
+
return None
|
| 600 |
+
return None
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
def detect_box(cgroup_root="/sys/fs/cgroup", meminfo="/proc/meminfo", affinity=None):
|
| 604 |
+
"""{cpus, memoryBytes, cpuSource, memorySource} for this container.
|
| 605 |
+
|
| 606 |
+
`os.cpu_count()` reports the HOST inside a container, so the cgroup limit is read first.
|
| 607 |
+
"""
|
| 608 |
+
if affinity is None:
|
| 609 |
+
try:
|
| 610 |
+
affinity = len(os.sched_getaffinity(0))
|
| 611 |
+
except (AttributeError, OSError):
|
| 612 |
+
affinity = os.cpu_count() or 1
|
| 613 |
+
cpus, cpu_source = _cgroup_cpus(cgroup_root)
|
| 614 |
+
if cpus is None or cpus > affinity:
|
| 615 |
+
cpus, cpu_source = affinity, "sched_getaffinity"
|
| 616 |
+
physical = _meminfo_bytes(meminfo)
|
| 617 |
+
memory, memory_source = _cgroup_memory(cgroup_root)
|
| 618 |
+
if memory is None or (physical and memory > physical):
|
| 619 |
+
memory, memory_source = physical, "/proc/meminfo"
|
| 620 |
+
return {"cpus": int(cpus), "memoryBytes": int(memory or 0), "cpuSource": cpu_source,
|
| 621 |
+
"memorySource": memory_source}
|
| 622 |
+
|
| 623 |
+
|
| 624 |
+
def sizing(cpus, memory_bytes, workers=None):
|
| 625 |
+
"""Launch values for a box. Under 4 CPUs with no forced worker count: TODAY exactly.
|
| 626 |
+
|
| 627 |
+
`workers` forces the uvicorn worker count, and the per-worker values follow it.
|
| 628 |
+
"""
|
| 629 |
+
cpus = max(1, int(cpus))
|
| 630 |
+
memory_gb = max(0.0, float(memory_bytes or 0) / 1e9)
|
| 631 |
+
if workers is None and cpus < 4:
|
| 632 |
+
return {"tier": "2 vCPU", **TODAY}
|
| 633 |
+
if workers is None:
|
| 634 |
+
workers = max(1, min(MAX_WEB_WORKERS, cpus // 2, int(memory_gb // 8)))
|
| 635 |
+
workers = max(1, int(workers))
|
| 636 |
+
worker_memory = max(4, int(memory_gb // 8))
|
| 637 |
+
duckdb_total = int(memory_gb * DUCKDB_MEMORY_SHARE)
|
| 638 |
+
return {"tier": "derived", "workers": workers,
|
| 639 |
+
# One heavy request per CPU across the box, split over the workers.
|
| 640 |
+
"heavyConcurrency": max(1, cpus // workers),
|
| 641 |
+
"webDuckdbMemory": f"{max(2, (duckdb_total - worker_memory) // workers)}GB",
|
| 642 |
+
"webDuckdbThreads": max(2, cpus // workers),
|
| 643 |
+
"workerDuckdbMemory": f"{worker_memory}GB",
|
| 644 |
+
"workerDuckdbThreads": max(1, cpus // 4)}
|
| 645 |
+
|
| 646 |
+
|
| 647 |
+
def launch_role(environ=None):
|
| 648 |
+
"""The role the app will resolve, as `process_role.resolve_role` does: a valid
|
| 649 |
+
AIOS_PROCESS_ROLE, else web on a Space (SPACE_ID set), else all."""
|
| 650 |
+
environ = os.environ if environ is None else environ
|
| 651 |
+
raw = str(environ.get("AIOS_PROCESS_ROLE") or "").strip().lower()
|
| 652 |
+
if raw in ("web", "worker", "all", "reader"):
|
| 653 |
+
return raw
|
| 654 |
+
return "web" if str(environ.get("SPACE_ID") or "").strip() else "all"
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
def launch_env(environ=None, box=None):
|
| 658 |
+
"""{env name: value} the launcher exports. Explicit env wins, except it never buys workers
|
| 659 |
+
without the read-only mirror: AIOS_MIRROR_VERSIONS=0 or a role other than web holds one worker.
|
| 660 |
+
"""
|
| 661 |
+
environ = os.environ if environ is None else environ
|
| 662 |
+
box = box or detect_box()
|
| 663 |
+
explicit = {key for key, name in SIZING_ENV.items() if str(environ.get(name) or "").strip()}
|
| 664 |
+
forced = web_workers(environ) if "workers" in explicit else None
|
| 665 |
+
role = launch_role(environ)
|
| 666 |
+
if str(environ.get(VERSIONS_ENV) or "").strip() == "0" or role != "web":
|
| 667 |
+
# Only the web role versions its mirror writes. Every other role restores in place.
|
| 668 |
+
forced = 1
|
| 669 |
+
values = sizing(box["cpus"], box["memoryBytes"], workers=forced)
|
| 670 |
+
out = {SIZING_ENV[key]: str(values[key]) for key in SIZING_ENV
|
| 671 |
+
if key not in explicit and key != "workers"}
|
| 672 |
+
workers = int(values["workers"])
|
| 673 |
+
out["AIOS_UVICORN_WORKERS"] = str(workers)
|
| 674 |
+
# Several workers each open the mirror read-only through its version pointer.
|
| 675 |
+
out[VERSIONS_ENV] = "1" if workers > 1 else "0"
|
| 676 |
+
return out
|
| 677 |
+
|
| 678 |
+
|
| 679 |
+
def web_app():
|
| 680 |
+
"""`uvicorn --factory` target for several workers. The supervisor imports only this module.
|
| 681 |
+
|
| 682 |
+
uvicorn imports a plain `api.main:app` in its supervisor process too, which would run the
|
| 683 |
+
app's boot threads in a process that never serves.
|
| 684 |
+
"""
|
| 685 |
+
import importlib
|
| 686 |
+
return importlib.import_module("api.main").app
|
| 687 |
+
|
| 688 |
+
|
| 689 |
+
def main(argv=None):
|
| 690 |
+
import argparse
|
| 691 |
+
import shlex
|
| 692 |
+
parser = argparse.ArgumentParser(description="Print launch values sized to this box.")
|
| 693 |
+
parser.add_argument("--env", action="store_true", help="print shell export lines")
|
| 694 |
+
# The env defaults let a gate point the launcher at a fixture box.
|
| 695 |
+
parser.add_argument("--cgroup-root", default=os.environ.get("AIOS_CGROUP_ROOT") or "/sys/fs/cgroup")
|
| 696 |
+
parser.add_argument("--meminfo", default=os.environ.get("AIOS_MEMINFO") or "/proc/meminfo")
|
| 697 |
+
args = parser.parse_args(argv)
|
| 698 |
+
box = detect_box(args.cgroup_root, args.meminfo)
|
| 699 |
+
env = launch_env(os.environ, box)
|
| 700 |
+
print(f"[box-sizing] cpus={box['cpus']} ({box['cpuSource']}) "
|
| 701 |
+
f"memory={box['memoryBytes'] / 1e9:.1f}GB ({box['memorySource']}) "
|
| 702 |
+
+ " ".join(f"{k}={v}" for k, v in env.items()), file=sys.stderr, flush=True)
|
| 703 |
+
if args.env:
|
| 704 |
+
for name, value in env.items():
|
| 705 |
+
print(f"export {name}={shlex.quote(value)}")
|
| 706 |
+
else:
|
| 707 |
+
print(json.dumps({"box": box, "env": env}, sort_keys=True))
|
| 708 |
+
return 0
|
| 709 |
+
|
| 710 |
+
|
| 711 |
+
if __name__ == "__main__":
|
| 712 |
+
sys.exit(main())
|
api/request_timing.py
CHANGED
|
@@ -29,6 +29,7 @@ def request_timeout_s():
|
|
| 29 |
|
| 30 |
|
| 31 |
_BUSY_CAUSES = {
|
|
|
|
| 32 |
"PoolTimeout": "postgres_pool_timeout", "TooManyRequests": "postgres_pool_full",
|
| 33 |
"QueryCanceled": "postgres_statement_timeout", "LockNotAvailable": "postgres_lock_timeout",
|
| 34 |
"SharedReadTimeout": "postgres_shared_read_timeout", "OdooPoolTimeout": "odoo_pool_timeout",
|
|
|
|
| 29 |
|
| 30 |
|
| 31 |
_BUSY_CAUSES = {
|
| 32 |
+
"OdooBreakerOpen": "odoo_breaker_open",
|
| 33 |
"PoolTimeout": "postgres_pool_timeout", "TooManyRequests": "postgres_pool_full",
|
| 34 |
"QueryCanceled": "postgres_statement_timeout", "LockNotAvailable": "postgres_lock_timeout",
|
| 35 |
"SharedReadTimeout": "postgres_shared_read_timeout", "OdooPoolTimeout": "odoo_pool_timeout",
|
api/routes_account_health.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Account health App routes: summary, metric history, owned alerts and settings, each App scoped (C3)."""
|
| 2 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 3 |
+
|
| 4 |
+
from deps import Session, err, require_session
|
| 5 |
+
|
| 6 |
+
TEMPLATE_KIND = "account_health"
|
| 7 |
+
router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["account-health"])
|
| 8 |
+
SOURCE = "ut_amzn_account_health"
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _module():
|
| 12 |
+
from modules import account_health
|
| 13 |
+
return account_health
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 17 |
+
"""Reads need read access to the account health database."""
|
| 18 |
+
from core import perm_scope
|
| 19 |
+
if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
|
| 20 |
+
raise err(403, "account_health_forbidden", "Your account cannot read Amazon account health.")
|
| 21 |
+
return session
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _writer(session: Session = Depends(_reader)) -> Session:
|
| 25 |
+
"""Writes also need a role that may edit."""
|
| 26 |
+
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 27 |
+
raise err(403, "account_health_read_only", "Your account can view account health but not change it.")
|
| 28 |
+
return session
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _tenant(session):
|
| 32 |
+
return str(getattr(session.runtime, "key", "") or session.tenant)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _call(fn, *args, **kwargs):
|
| 36 |
+
try:
|
| 37 |
+
return fn(*args, **kwargs)
|
| 38 |
+
except PermissionError as exc:
|
| 39 |
+
raise err(403, "account_health_read_only", str(exc)) from exc
|
| 40 |
+
except ValueError as exc:
|
| 41 |
+
raise err(400, "account_health_invalid", str(exc)) from exc
|
| 42 |
+
except KeyError as exc:
|
| 43 |
+
raise err(404, "account_health_not_found", "That alert or metric does not exist.") from exc
|
| 44 |
+
except Exception as exc: # noqa: BLE001
|
| 45 |
+
raise err(503, "account_health_unavailable", "Account health could not be read. Try again shortly.") from exc
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _app(session, app_id):
|
| 49 |
+
"""The Account health App record this request names, readable by the caller (404 otherwise)."""
|
| 50 |
+
import application_registry
|
| 51 |
+
try:
|
| 52 |
+
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 53 |
+
except Exception as exc: # noqa: BLE001
|
| 54 |
+
raise err(503, "account_health_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 55 |
+
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 56 |
+
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 57 |
+
raise err(404, "account_health_app_not_found", "That Account health App does not exist.")
|
| 58 |
+
return app
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _may_edit(session, app):
|
| 62 |
+
import application_registry
|
| 63 |
+
role = str(session.user.get("role") or "user").strip().lower()
|
| 64 |
+
return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
|
| 68 |
+
"""A read names the App, and the caller must be able to read that App."""
|
| 69 |
+
_app(session, app)
|
| 70 |
+
return session
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
|
| 74 |
+
"""A write names the App, and only an editor of that App may change it."""
|
| 75 |
+
if not _may_edit(session, _app(session, app)):
|
| 76 |
+
raise err(403, "account_health_read_only", "You can view this App but not change it.")
|
| 77 |
+
return session
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def owners(session):
|
| 81 |
+
"""People who may own an alert: active, non-service, non-viewer accounts of this tenant, by display name."""
|
| 82 |
+
ah = _module()
|
| 83 |
+
try:
|
| 84 |
+
users = session.runtime.get("users")
|
| 85 |
+
except Exception: # noqa: BLE001
|
| 86 |
+
users = None
|
| 87 |
+
want = _tenant(session).strip().lower()
|
| 88 |
+
names = set()
|
| 89 |
+
for name, rec in (users.items() if isinstance(users, dict) else ()):
|
| 90 |
+
if (not isinstance(rec, dict) or not rec.get("active", True) or ah._is_service(name, rec)
|
| 91 |
+
or str(rec.get("role") or "").strip().lower() == "viewer"
|
| 92 |
+
or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)):
|
| 93 |
+
continue
|
| 94 |
+
names.add(str(rec.get("name") or name).strip())
|
| 95 |
+
return sorted((n for n in names if n), key=str.lower)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
@router.get("/summary")
|
| 99 |
+
def read_summary(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
|
| 100 |
+
ah = _module()
|
| 101 |
+
out = _call(ah.summary, _tenant(session), st=session.runtime)
|
| 102 |
+
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
@router.get("/history")
|
| 106 |
+
def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 107 |
+
limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
|
| 108 |
+
ah = _module()
|
| 109 |
+
return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
@router.get("/items")
|
| 113 |
+
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 114 |
+
cursor: str | None = Query(None), limit: int = Query(200, ge=1),
|
| 115 |
+
session: Session = Depends(_app_reader)):
|
| 116 |
+
ah = _module()
|
| 117 |
+
listed = _call(ah.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit,
|
| 118 |
+
st=session.runtime)
|
| 119 |
+
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
@router.patch("/items/{item_id}")
|
| 123 |
+
def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 124 |
+
due: str | None = Body(None), note: str | None = Body(None),
|
| 125 |
+
session: Session = Depends(_app_writer)):
|
| 126 |
+
ah = _module()
|
| 127 |
+
return _call(ah.update_item, _tenant(session), session.user, item_id, owner=owner, status=status,
|
| 128 |
+
due=due, note=note, st=session.runtime)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
@router.get("/settings")
|
| 132 |
+
def read_settings(session: Session = Depends(_app_reader)):
|
| 133 |
+
ah = _module()
|
| 134 |
+
return _call(ah.settings, _tenant(session), st=session.runtime)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
@router.put("/settings")
|
| 138 |
+
def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: str | None = Body(None),
|
| 139 |
+
session: Session = Depends(_app_writer)):
|
| 140 |
+
ah = _module()
|
| 141 |
+
return _call(ah.set_settings, _tenant(session), alert_distance_pct=alertDistancePct,
|
| 142 |
+
default_owner=defaultOwner, st=session.runtime)
|
api/routes_admin.py
CHANGED
|
@@ -143,7 +143,17 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None):
|
|
| 143 |
# editor will open, so the two cannot disagree.
|
| 144 |
"accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st,
|
| 145 |
surface_keys=surface_keys),
|
| 146 |
-
"perms_v": int(rec.get("perms_v") or 0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
|
| 148 |
|
| 149 |
def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None):
|
|
@@ -454,6 +464,9 @@ def update_user(username: str, body: dict = Body(default=None),
|
|
| 454 |
if result == "last_admin":
|
| 455 |
raise err(400, "last_admin",
|
| 456 |
"an active tenant must retain at least one administrator")
|
|
|
|
|
|
|
|
|
|
| 457 |
if result != "updated":
|
| 458 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 459 |
|
|
@@ -539,6 +552,166 @@ def expire_password(username: str, session: Session = Depends(admin_gate)):
|
|
| 539 |
return Response(status_code=204)
|
| 540 |
|
| 541 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 542 |
# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ──────────────────
|
| 543 |
#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema
|
| 544 |
#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list
|
|
|
|
| 143 |
# editor will open, so the two cannot disagree.
|
| 144 |
"accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st,
|
| 145 |
surface_keys=surface_keys),
|
| 146 |
+
"perms_v": int(rec.get("perms_v") or 0),
|
| 147 |
+
# S79-T63: only a copied test account carries `test`, so a real account's keys stay exact.
|
| 148 |
+
**({"test": _test_marker(rec)} if _test_marker(rec) else {})}
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def _test_marker(rec):
|
| 152 |
+
mark = (rec or {}).get(users.TEST_ACCOUNT_KEY)
|
| 153 |
+
if not isinstance(mark, dict):
|
| 154 |
+
return None
|
| 155 |
+
return {"copiedFrom": mark.get("copied_from"), "createdBy": mark.get("created_by"),
|
| 156 |
+
"createdAt": mark.get("created_at")}
|
| 157 |
|
| 158 |
|
| 159 |
def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None):
|
|
|
|
| 464 |
if result == "last_admin":
|
| 465 |
raise err(400, "last_admin",
|
| 466 |
"an active tenant must retain at least one administrator")
|
| 467 |
+
if result in _TEST_ACCOUNT_REFUSALS:
|
| 468 |
+
# R42: a test account gets no email and no admin role, refused in the core writer.
|
| 469 |
+
raise err(400, result, _TEST_ACCOUNT_REFUSALS[result])
|
| 470 |
if result != "updated":
|
| 471 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 472 |
|
|
|
|
| 552 |
return Response(status_code=204)
|
| 553 |
|
| 554 |
|
| 555 |
+
# ── S79-T63 (R28, R42): test accounts that copy one member's permissions exactly ─────────────────
|
| 556 |
+
_TEST_ACCOUNT_REFUSALS = {
|
| 557 |
+
"test_account_no_email": "A test account cannot have an email address.",
|
| 558 |
+
"test_account_no_whatsapp": "A test account cannot connect WhatsApp.",
|
| 559 |
+
"test_account_not_admin": "A test account cannot be an administrator.",
|
| 560 |
+
}
|
| 561 |
+
|
| 562 |
+
|
| 563 |
+
def _copy_refusal(exc):
|
| 564 |
+
return err(exc.status, exc.code, str(exc))
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
#: The en and em dash, built from code points so this file carries no dash character (rule 2).
|
| 568 |
+
_DASHES = re.compile("[" + chr(0x2013) + chr(0x2014) + "]")
|
| 569 |
+
|
| 570 |
+
|
| 571 |
+
def _copy_failure(exc, verb):
|
| 572 |
+
# A store error can carry any text; rule 2 keeps dashes off the screen.
|
| 573 |
+
cause = _DASHES.sub(",", str(exc.cause or "")).strip().rstrip(".")
|
| 574 |
+
message = f"The test account was not {verb}. {cause}."
|
| 575 |
+
if exc.leftovers:
|
| 576 |
+
message += (" These parts could not be removed: " + ", ".join(exc.leftovers)
|
| 577 |
+
+ ". Remove the test account again to finish.")
|
| 578 |
+
elif verb == "created":
|
| 579 |
+
message += " Nothing was kept."
|
| 580 |
+
return err(503, "test_account_failed", message)
|
| 581 |
+
|
| 582 |
+
|
| 583 |
+
def _test_account_steps(session):
|
| 584 |
+
"""Copy steps for grant stores outside `core`: Connector grants, through their own route."""
|
| 585 |
+
import core.perm_scope as perm_scope
|
| 586 |
+
import routes_keychain as keychain
|
| 587 |
+
|
| 588 |
+
def held():
|
| 589 |
+
doc = session.runtime.get(perm_scope.CONNECTOR_GRANT_KEY, fresh=True)
|
| 590 |
+
if doc is not None and not isinstance(doc, dict):
|
| 591 |
+
raise RuntimeError("connector access could not be read")
|
| 592 |
+
return doc or {}
|
| 593 |
+
|
| 594 |
+
def copy_grants(username, source):
|
| 595 |
+
known = {kind["key"] for kind in keychain.connector_grant_kinds()}
|
| 596 |
+
kinds = sorted(k for k in (held().get(source) or ()) if k in known)
|
| 597 |
+
if kinds:
|
| 598 |
+
keychain.put_connector_grants(username, {"granted": kinds}, session)
|
| 599 |
+
|
| 600 |
+
def clear_grants(username):
|
| 601 |
+
if username in held():
|
| 602 |
+
keychain.put_connector_grants(username, {"granted": []}, session)
|
| 603 |
+
|
| 604 |
+
return [("connector access", copy_grants, clear_grants)]
|
| 605 |
+
|
| 606 |
+
|
| 607 |
+
@router.get("/admin/users/{username}/test-account/preview")
|
| 608 |
+
def preview_test_account(username: str, session: Session = Depends(admin_gate)):
|
| 609 |
+
"""What a test account copied from this member would carry, before anything is written."""
|
| 610 |
+
from core import test_accounts, user_tables
|
| 611 |
+
reg = _registry()
|
| 612 |
+
try:
|
| 613 |
+
plan = test_accounts.plan(username, reg, tenant=_tenant_of(session.user),
|
| 614 |
+
st=session.runtime)
|
| 615 |
+
lent = user_tables.lend_defs(session.runtime, schema_only=True)
|
| 616 |
+
except test_accounts.CopyRefused as exc:
|
| 617 |
+
raise _copy_refusal(exc)
|
| 618 |
+
except Exception as exc: # noqa: BLE001
|
| 619 |
+
raise err(503, "store_unavailable", "Access unavailable. Try again.") from exc
|
| 620 |
+
limits = _source_limits(plan["source"], reg.get(plan["source"]), session)
|
| 621 |
+
body = test_accounts.preview(plan, st=lent, limits=limits)
|
| 622 |
+
_label_grants(session, body["copy"]["grants"])
|
| 623 |
+
return body
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
def _source_limits(uname, rec, session):
|
| 627 |
+
"""The source's row limits, phrased for the administrator; None if they cannot be read.
|
| 628 |
+
|
| 629 |
+
Field walls are lifted for the phrasing only, so a condition on a hidden field is named
|
| 630 |
+
instead of reading "Limited by your access". The filters themselves are the source's.
|
| 631 |
+
"""
|
| 632 |
+
import core.perm_scope as perm_scope
|
| 633 |
+
try:
|
| 634 |
+
reader = deepcopy(users._public(uname, rec))
|
| 635 |
+
for entry in (reader.get("perms") or {}).values():
|
| 636 |
+
if isinstance(entry, dict):
|
| 637 |
+
entry.pop("hiddenFields", None)
|
| 638 |
+
return perm_scope.scope_limits(reader, st=session.runtime)
|
| 639 |
+
except Exception: # noqa: BLE001
|
| 640 |
+
return None
|
| 641 |
+
|
| 642 |
+
|
| 643 |
+
#: What a preview row says when a shared item's name cannot be read. Never a raw id.
|
| 644 |
+
_GRANT_FALLBACK = {"view": "A shared view", "folder": "A shared folder", "database": "A database",
|
| 645 |
+
"field": "A shared field", "cohort": "A locked view", "workbook": "A shared sheet"}
|
| 646 |
+
#: Naming stops after this many seconds; every grant is still listed, the rest by kind.
|
| 647 |
+
_LABEL_BUDGET_S = 3.0
|
| 648 |
+
|
| 649 |
+
|
| 650 |
+
def _label_grants(session, grants):
|
| 651 |
+
"""Name each grant through the share door's own resolver (`routes_shares._object_ref`)."""
|
| 652 |
+
import time
|
| 653 |
+
try:
|
| 654 |
+
from routes_shares import _object_ref
|
| 655 |
+
except Exception: # noqa: BLE001
|
| 656 |
+
_object_ref = None
|
| 657 |
+
deadline = time.monotonic() + _LABEL_BUDGET_S
|
| 658 |
+
for grant in grants:
|
| 659 |
+
if grant.get("label"):
|
| 660 |
+
continue
|
| 661 |
+
label = None
|
| 662 |
+
if time.monotonic() < deadline:
|
| 663 |
+
try:
|
| 664 |
+
if grant["kind"] == "workbook":
|
| 665 |
+
# `_object_ref` has no workbook branch; the workbook's own name is the label.
|
| 666 |
+
import core.sheets as sheets
|
| 667 |
+
label = str(sheets.get_workbook(session.runtime, grant["oid"]).get("name") or "")
|
| 668 |
+
elif _object_ref is not None:
|
| 669 |
+
label = str(_object_ref(session, grant["kind"], grant["oid"])[0] or "")
|
| 670 |
+
except Exception: # noqa: BLE001
|
| 671 |
+
label = None
|
| 672 |
+
label = (label or "").strip()
|
| 673 |
+
grant["label"] = label or _GRANT_FALLBACK.get(grant["kind"], "A shared item")
|
| 674 |
+
|
| 675 |
+
|
| 676 |
+
@router.post("/admin/test-accounts", status_code=201)
|
| 677 |
+
def create_test_account(body: dict = Body(default=None), session: Session = Depends(admin_gate)):
|
| 678 |
+
"""Create a test account whose permissions copy `source` exactly. The password shows once."""
|
| 679 |
+
from core import test_accounts
|
| 680 |
+
body = body if isinstance(body, dict) else {}
|
| 681 |
+
_require_store()
|
| 682 |
+
try:
|
| 683 |
+
made = test_accounts.create(body.get("source"), admin=session.uname,
|
| 684 |
+
tenant=_tenant_of(session.user), st=session.runtime,
|
| 685 |
+
steps=_test_account_steps(session))
|
| 686 |
+
except test_accounts.CopyRefused as exc:
|
| 687 |
+
raise _copy_refusal(exc)
|
| 688 |
+
except test_accounts.CopyFailed as exc:
|
| 689 |
+
raise _copy_failure(exc, "created")
|
| 690 |
+
except Exception as exc: # noqa: BLE001
|
| 691 |
+
raise err(503, "store_unavailable", "The test account was not created. Try again.") from exc
|
| 692 |
+
rec = _registry().get(made["username"])
|
| 693 |
+
if not isinstance(rec, dict):
|
| 694 |
+
raise err(503, "store_unavailable", "The test account was not saved. Try again.")
|
| 695 |
+
return {"user": _view(made["username"], rec), "password": made["password"]}
|
| 696 |
+
|
| 697 |
+
|
| 698 |
+
@router.delete("/admin/test-accounts/{username}", status_code=204)
|
| 699 |
+
def delete_test_account(username: str, session: Session = Depends(admin_gate)):
|
| 700 |
+
"""Remove a test account, its sign in and every grant naming it, in one call."""
|
| 701 |
+
from core import test_accounts
|
| 702 |
+
_require_store()
|
| 703 |
+
try:
|
| 704 |
+
test_accounts.remove(username, tenant=_tenant_of(session.user), st=session.runtime,
|
| 705 |
+
granter=session.uname, steps=_test_account_steps(session))
|
| 706 |
+
except test_accounts.CopyRefused as exc:
|
| 707 |
+
raise _copy_refusal(exc)
|
| 708 |
+
except test_accounts.CopyFailed as exc:
|
| 709 |
+
raise _copy_failure(exc, "fully removed")
|
| 710 |
+
except Exception as exc: # noqa: BLE001
|
| 711 |
+
raise err(503, "store_unavailable", "The test account was not removed. Try again.") from exc
|
| 712 |
+
return Response(status_code=204)
|
| 713 |
+
|
| 714 |
+
|
| 715 |
# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ──────────────────
|
| 716 |
#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema
|
| 717 |
#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list
|
api/routes_aged_stock.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Aged stock App routes (C3): summary, items, one SKU, settings, refresh and the rule 8 tie."""
|
| 2 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 3 |
+
|
| 4 |
+
from deps import Session, err, require_session
|
| 5 |
+
|
| 6 |
+
TEMPLATE_KIND = "aged_stock"
|
| 7 |
+
router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["aged-stock"])
|
| 8 |
+
SOURCE = "ut_amzn_storage_fees"
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _module():
|
| 12 |
+
from modules import aged_stock
|
| 13 |
+
return aged_stock
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 17 |
+
"""Reads need read access to the Amazon storage fees database."""
|
| 18 |
+
from core import perm_scope
|
| 19 |
+
if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
|
| 20 |
+
raise err(403, "aged_stock_forbidden", "Your account cannot read Amazon storage fees.")
|
| 21 |
+
return session
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _writer(session: Session = Depends(_reader)) -> Session:
|
| 25 |
+
"""Writes also need a role that may edit."""
|
| 26 |
+
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 27 |
+
raise err(403, "aged_stock_read_only", "You can view this App but not change it.")
|
| 28 |
+
return session
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _tenant(session):
|
| 32 |
+
return str(getattr(session.runtime, "key", "") or session.tenant)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _call(fn, *args, **kwargs):
|
| 36 |
+
try:
|
| 37 |
+
return fn(*args, **kwargs)
|
| 38 |
+
except PermissionError as exc:
|
| 39 |
+
raise err(403, "aged_stock_read_only", "You can view this App but not change it.") from exc
|
| 40 |
+
except ValueError as exc:
|
| 41 |
+
raise err(400, "aged_stock_invalid", str(exc)) from exc
|
| 42 |
+
except KeyError as exc:
|
| 43 |
+
raise err(404, "aged_stock_item_not_found", "That aged stock item does not exist.") from exc
|
| 44 |
+
except Exception as exc: # noqa: BLE001
|
| 45 |
+
raise err(503, "aged_stock_unavailable", "The aged stock view could not be read. Try again shortly.") from exc
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _app(session, app_id):
|
| 49 |
+
"""The Aged stock App record this request names, readable by the caller (404 otherwise)."""
|
| 50 |
+
import application_registry
|
| 51 |
+
try:
|
| 52 |
+
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 53 |
+
except Exception as exc: # noqa: BLE001
|
| 54 |
+
raise err(503, "aged_stock_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 55 |
+
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 56 |
+
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 57 |
+
raise err(404, "aged_stock_app_not_found", "That Aged stock App does not exist.")
|
| 58 |
+
return app
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _may_edit(session, app):
|
| 62 |
+
import application_registry
|
| 63 |
+
role = str(session.user.get("role") or "user").strip().lower()
|
| 64 |
+
return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
|
| 68 |
+
"""A read names the App, and the caller must be able to read that App."""
|
| 69 |
+
_app(session, app)
|
| 70 |
+
return session
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
|
| 74 |
+
"""A write names the App, and only an editor of that App may change it."""
|
| 75 |
+
if not _may_edit(session, _app(session, app)):
|
| 76 |
+
raise err(403, "aged_stock_read_only", "You can view this App but not change it.")
|
| 77 |
+
return session
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def owners(session):
|
| 81 |
+
"""People who may own an item: active, non-service, non-viewer accounts of this tenant, by display name."""
|
| 82 |
+
mod = _module()
|
| 83 |
+
try:
|
| 84 |
+
users = session.runtime.get("users")
|
| 85 |
+
except Exception: # noqa: BLE001
|
| 86 |
+
users = None
|
| 87 |
+
want = _tenant(session).strip().lower()
|
| 88 |
+
names = set()
|
| 89 |
+
for name, rec in (users.items() if isinstance(users, dict) else ()):
|
| 90 |
+
if (not isinstance(rec, dict) or not rec.get("active", True) or mod._is_service(name, rec)
|
| 91 |
+
or str(rec.get("role") or "").strip().lower() == "viewer"
|
| 92 |
+
or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)):
|
| 93 |
+
continue
|
| 94 |
+
names.add(str(rec.get("name") or name).strip())
|
| 95 |
+
return sorted((n for n in names if n), key=str.lower)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
#: The item fields a Next tier row carries, so the screen lists it with no second read.
|
| 99 |
+
ROW_FIELDS = ("action", "recommended_action", "owner", "status", "due", "units")
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _summary_rows(tenant_key, st):
|
| 103 |
+
"""The module summary with each Next tier row joined to its item's action, owner, status and due."""
|
| 104 |
+
mod = _module()
|
| 105 |
+
out = mod.summary(tenant_key, st=st)
|
| 106 |
+
stored = mod._doc(st, mod.KEY_ITEMS).get("items") or {}
|
| 107 |
+
for row in out.get("nextTier") or ():
|
| 108 |
+
item = stored.get(row.get("id")) or {}
|
| 109 |
+
row["item"] = {key: item.get(key) for key in ROW_FIELDS}
|
| 110 |
+
return out
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
@router.get("/summary")
|
| 114 |
+
def read_summary(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
|
| 115 |
+
out = _call(_summary_rows, _tenant(session), session.runtime)
|
| 116 |
+
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
@router.get("/items")
|
| 120 |
+
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 121 |
+
action: str | None = Query(None), cursor: str | None = Query(None),
|
| 122 |
+
limit: int = Query(200, ge=1), q: str | None = Query(None),
|
| 123 |
+
session: Session = Depends(_app_reader)):
|
| 124 |
+
listed = _call(_module().items, _tenant(session), session.uname, status=status, action=action,
|
| 125 |
+
cursor=cursor, limit=limit, q=q, st=session.runtime)
|
| 126 |
+
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
@router.get("/items/{item_id}")
|
| 130 |
+
def read_item(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
|
| 131 |
+
detail = _call(_module().item_detail, _tenant(session), item_id, st=session.runtime)
|
| 132 |
+
return {**detail, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
@router.patch("/items/{item_id}")
|
| 136 |
+
def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 137 |
+
due: str | None = Body(None), action: str | None = Body(None), note: str | None = Body(None),
|
| 138 |
+
session: Session = Depends(_app_writer)):
|
| 139 |
+
return _call(_module().update_item, _tenant(session), item_id, owner=owner, status=status, due=due,
|
| 140 |
+
action=action, note=note, user=session.uname or "unknown", st=session.runtime)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
@router.get("/settings")
|
| 144 |
+
def read_settings(session: Session = Depends(_app_reader)):
|
| 145 |
+
return _call(_module().settings, _tenant(session), st=session.runtime)
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
@router.patch("/settings")
|
| 149 |
+
def write_settings(adsFromDays: int | None = Body(None), priceDownFromDays: int | None = Body(None),
|
| 150 |
+
removeFromDays: int | None = Body(None), feeHorizonDays: int | None = Body(None),
|
| 151 |
+
dueDays: int | None = Body(None), defaultOwner: str | None = Body(None),
|
| 152 |
+
session: Session = Depends(_app_writer)):
|
| 153 |
+
return _call(_module().set_settings, _tenant(session), ads_from_days=adsFromDays,
|
| 154 |
+
price_down_from_days=priceDownFromDays, remove_from_days=removeFromDays,
|
| 155 |
+
fee_horizon_days=feeHorizonDays, due_days=dueDays, default_owner=defaultOwner,
|
| 156 |
+
st=session.runtime)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
@router.post("/refresh")
|
| 160 |
+
def refresh(session: Session = Depends(_app_writer)):
|
| 161 |
+
return _call(_module().refresh, _tenant(session), st=session.runtime)
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
@router.get("/validate")
|
| 165 |
+
def validate(start: str | None = Query(None), end: str | None = Query(None),
|
| 166 |
+
session: Session = Depends(_app_reader)):
|
| 167 |
+
return _call(_module().validate, _tenant(session), start=start, end=end, st=session.runtime)
|
api/routes_amazon_operator.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Amazon operator App routes (C3): today's batch, history, settings, and the batch approve and dismiss.
|
| 2 |
+
|
| 3 |
+
Every route names its App with `?app=<id>`. Approve and dismiss call `amazon_operator.set_status`, the
|
| 4 |
+
same path an MCP approval takes. The router never calls the Ads client.
|
| 5 |
+
"""
|
| 6 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 7 |
+
|
| 8 |
+
from deps import Session, err, require_session
|
| 9 |
+
|
| 10 |
+
router = APIRouter(prefix="/api/v1/app-kinds/amazon_operator", tags=["amazon-operator"])
|
| 11 |
+
TEMPLATE_KIND = "amazon_operator"
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _module():
|
| 15 |
+
from modules import amazon_operator
|
| 16 |
+
return amazon_operator
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 20 |
+
"""Reads need a signed in session. The App grant is checked per request."""
|
| 21 |
+
return session
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _writer(session: Session = Depends(_reader)) -> Session:
|
| 25 |
+
"""Writes also need a role that may edit."""
|
| 26 |
+
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 27 |
+
raise err(403, "amazon_operator_read_only", "Your account can view these proposals but not change them.")
|
| 28 |
+
return session
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _tenant(session):
|
| 32 |
+
return str(getattr(session.runtime, "key", "") or session.tenant)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _call(fn, *args, **kwargs):
|
| 36 |
+
try:
|
| 37 |
+
return fn(*args, **kwargs)
|
| 38 |
+
except PermissionError as exc:
|
| 39 |
+
raise err(403, "amazon_operator_read_only", str(exc)) from exc
|
| 40 |
+
except ValueError as exc:
|
| 41 |
+
raise err(400, "amazon_operator_invalid", str(exc)) from exc
|
| 42 |
+
except KeyError as exc:
|
| 43 |
+
raise err(404, "amazon_operator_not_found", "That proposal does not exist.") from exc
|
| 44 |
+
except Exception as exc: # noqa: BLE001
|
| 45 |
+
raise err(503, "amazon_operator_unavailable", "The Amazon operator could not be read. Try again shortly.") from exc
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _app(session, app_id):
|
| 49 |
+
"""The Amazon operator App record this request names, readable by the caller (404 otherwise)."""
|
| 50 |
+
import application_registry
|
| 51 |
+
try:
|
| 52 |
+
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 53 |
+
except Exception as exc: # noqa: BLE001
|
| 54 |
+
raise err(503, "amazon_operator_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 55 |
+
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 56 |
+
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 57 |
+
raise err(404, "amazon_operator_app_not_found", "That Amazon operator App does not exist.")
|
| 58 |
+
return app
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _may_edit(session, app):
|
| 62 |
+
import application_registry
|
| 63 |
+
role = str(session.user.get("role") or "user").strip().lower()
|
| 64 |
+
return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
|
| 68 |
+
"""C3: a read names the App, and the caller must be able to read that App."""
|
| 69 |
+
_app(session, app)
|
| 70 |
+
return session
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
|
| 74 |
+
"""C3: a write names the App, and only an editor of that App may change it."""
|
| 75 |
+
if not _may_edit(session, _app(session, app)):
|
| 76 |
+
raise err(403, "amazon_operator_read_only", "You can view this App but not change it.")
|
| 77 |
+
return session
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@router.get("/today")
|
| 81 |
+
def read_today(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
|
| 82 |
+
op = _module()
|
| 83 |
+
out = _call(op.today, _tenant(session), st=session.runtime)
|
| 84 |
+
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
@router.get("/history")
|
| 88 |
+
def read_history(status: str | None = Query(None), cursor: str | None = Query(None),
|
| 89 |
+
limit: int = Query(200, ge=1), session: Session = Depends(_app_reader)):
|
| 90 |
+
op = _module()
|
| 91 |
+
page = _call(op.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit,
|
| 92 |
+
st=session.runtime)
|
| 93 |
+
if cursor:
|
| 94 |
+
return page
|
| 95 |
+
# The first page carries the full count for the History tab. Uncapped: every row is counted (rule 1).
|
| 96 |
+
every = _call(op.items, _tenant(session), session.user, status=status, cursor=None, limit=None,
|
| 97 |
+
st=session.runtime)
|
| 98 |
+
return {**page, "total": len(every["items"])}
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
@router.get("/settings")
|
| 102 |
+
def read_settings(session: Session = Depends(_app_reader)):
|
| 103 |
+
op = _module()
|
| 104 |
+
return _call(op.settings, _tenant(session), st=session.runtime)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
@router.put("/settings")
|
| 108 |
+
def write_settings(stop_cover_days: float | None = Body(None), raise_cover_days: float | None = Body(None),
|
| 109 |
+
pause_below_profit: float | None = Body(None), bid_step_pct: float | None = Body(None),
|
| 110 |
+
budget_step_pct: float | None = Body(None), lookback_days: float | None = Body(None),
|
| 111 |
+
negative_min_clicks: float | None = Body(None), negative_min_cost: float | None = Body(None),
|
| 112 |
+
defaultOwner: str | None = Body(None), session: Session = Depends(_app_writer)):
|
| 113 |
+
op = _module()
|
| 114 |
+
return _call(op.set_settings, _tenant(session), st=session.runtime, default_owner=defaultOwner,
|
| 115 |
+
stop_cover_days=stop_cover_days, raise_cover_days=raise_cover_days,
|
| 116 |
+
pause_below_profit=pause_below_profit, bid_step_pct=bid_step_pct,
|
| 117 |
+
budget_step_pct=budget_step_pct, lookback_days=lookback_days,
|
| 118 |
+
negative_min_clicks=negative_min_clicks, negative_min_cost=negative_min_cost)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
@router.post("/approve")
|
| 122 |
+
def approve(item_ids: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 123 |
+
"""Approve the ticked proposals as one batch. Returns each row's outcome."""
|
| 124 |
+
op = _module()
|
| 125 |
+
return _call(op.set_status, _tenant(session), session.user, item_ids, "approved", st=session.runtime)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
@router.post("/dismiss")
|
| 129 |
+
def dismiss(item_ids: list[str] = Body(...), reason: str = Body(..., min_length=1),
|
| 130 |
+
session: Session = Depends(_app_writer)):
|
| 131 |
+
op = _module()
|
| 132 |
+
return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason,
|
| 133 |
+
st=session.runtime)
|
api/routes_assets.py
CHANGED
|
@@ -36,7 +36,7 @@ from fastapi import Body, Depends
|
|
| 36 |
from fastapi import APIRouter
|
| 37 |
from fastapi.responses import FileResponse, Response
|
| 38 |
|
| 39 |
-
from deps import Session, err,
|
| 40 |
from core import platform_admin
|
| 41 |
from routes_admin import admin_gate
|
| 42 |
import application_registry as applications
|
|
@@ -467,6 +467,13 @@ _EXT_MEDIA = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"}
|
|
| 467 |
#: closed charset — no slashes, no backslashes, no colons, so a code can never be a path.
|
| 468 |
_CODE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._+&()\-]{0,63}$")
|
| 469 |
_IMMUTABLE = "public, max-age=31536000, immutable"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 470 |
|
| 471 |
#: per-process index of the local master dir: rebuilt when the dir path or its mtime changes.
|
| 472 |
_DIR_INDEX = {"path": None, "mtime": None, "map": {}}
|
|
@@ -548,8 +555,16 @@ def _assets():
|
|
| 548 |
return assets
|
| 549 |
|
| 550 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 551 |
@router.get("/assets/products")
|
| 552 |
-
def asset_manifest(session: Session = Depends(
|
| 553 |
"""The codes that HAVE an image — so the catalog designer can mark gaps instead of
|
| 554 |
rendering broken tiles. `source` names which backend answered."""
|
| 555 |
d = _local_dir()
|
|
@@ -563,7 +578,7 @@ def asset_manifest(session: Session = Depends(module_gate(MODULE))):
|
|
| 563 |
|
| 564 |
@router.get("/assets/products/{code}")
|
| 565 |
def product_asset(code: str, q: str = "web",
|
| 566 |
-
session: Session = Depends(
|
| 567 |
if q not in _QUALITIES:
|
| 568 |
raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
|
| 569 |
norm = _norm(code)
|
|
@@ -583,7 +598,7 @@ def product_asset(code: str, q: str = "web",
|
|
| 583 |
|
| 584 |
media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
|
| 585 |
return FileResponse(str(path), media_type=media,
|
| 586 |
-
headers={"Cache-Control":
|
| 587 |
|
| 588 |
|
| 589 |
# ------------------------------------------------------------------ editorial (DEBT-5)
|
|
@@ -626,7 +641,7 @@ def _ed_dir_index(ed):
|
|
| 626 |
|
| 627 |
|
| 628 |
@router.get("/assets/editorial")
|
| 629 |
-
def editorial_manifest(session: Session = Depends(
|
| 630 |
"""The slugs that HAVE an editorial image — the designer's picker reads this."""
|
| 631 |
if _local_dir() is not None:
|
| 632 |
ed = _ed_dir()
|
|
@@ -637,7 +652,7 @@ def editorial_manifest(session: Session = Depends(module_gate(MODULE))):
|
|
| 637 |
|
| 638 |
@router.get("/assets/editorial/{slug}")
|
| 639 |
def editorial_asset(slug: str, q: str = "web",
|
| 640 |
-
session: Session = Depends(
|
| 641 |
if q not in _QUALITIES:
|
| 642 |
raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
|
| 643 |
slug = str(slug or "").strip().lower()
|
|
@@ -656,7 +671,7 @@ def editorial_asset(slug: str, q: str = "web",
|
|
| 656 |
raise err(404, "asset_not_found", f"no editorial image named {slug}")
|
| 657 |
media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
|
| 658 |
return FileResponse(str(path), media_type=media,
|
| 659 |
-
headers={"Cache-Control":
|
| 660 |
|
| 661 |
|
| 662 |
@router.post("/assets/editorial", status_code=201)
|
|
@@ -825,7 +840,7 @@ def record_upload(body: dict = Body(default=None),
|
|
| 825 |
|
| 826 |
@router.get("/assets/records/{asset_id}")
|
| 827 |
def record_asset(asset_id: str, q: str = "web",
|
| 828 |
-
session: Session = Depends(
|
| 829 |
"""Serve one record image for THIS session's tenant. Any signed-in user of the tenant may
|
| 830 |
read it — a picture on a row is visible to whoever can open the row, and the row walls
|
| 831 |
already decided that."""
|
|
@@ -853,4 +868,4 @@ def record_asset(asset_id: str, q: str = "web",
|
|
| 853 |
raise err(404, "asset_not_found", "no image for that reference")
|
| 854 |
media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
|
| 855 |
return FileResponse(str(path), media_type=media,
|
| 856 |
-
headers={"Cache-Control":
|
|
|
|
| 36 |
from fastapi import APIRouter
|
| 37 |
from fastapi.responses import FileResponse, Response
|
| 38 |
|
| 39 |
+
from deps import Session, err, is_bearer, module_gate_or_bearer, require_session, require_session_or_bearer
|
| 40 |
from core import platform_admin
|
| 41 |
from routes_admin import admin_gate
|
| 42 |
import application_registry as applications
|
|
|
|
| 467 |
#: closed charset — no slashes, no backslashes, no colons, so a code can never be a path.
|
| 468 |
_CODE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._+&()\-]{0,63}$")
|
| 469 |
_IMMUTABLE = "public, max-age=31536000, immutable"
|
| 470 |
+
#: A bearer answer stays out of shared caches: a CDN must never serve a key-fetched image to anyone.
|
| 471 |
+
_BEARER_CACHE = "private, max-age=300"
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
def _cache_control(session):
|
| 475 |
+
"""The cookie answer keeps its immutable header. A bearer answer is private."""
|
| 476 |
+
return _BEARER_CACHE if is_bearer(session) else _IMMUTABLE
|
| 477 |
|
| 478 |
#: per-process index of the local master dir: rebuilt when the dir path or its mtime changes.
|
| 479 |
_DIR_INDEX = {"path": None, "mtime": None, "map": {}}
|
|
|
|
| 555 |
return assets
|
| 556 |
|
| 557 |
|
| 558 |
+
def product_image_codes():
|
| 559 |
+
"""The upper-cased codes that have a master image, from the dir or the repo manifest."""
|
| 560 |
+
d = _local_dir()
|
| 561 |
+
if d is not None:
|
| 562 |
+
return set(_dir_index(d))
|
| 563 |
+
return {_norm(code) for code in (_assets().manifest() or {})}
|
| 564 |
+
|
| 565 |
+
|
| 566 |
@router.get("/assets/products")
|
| 567 |
+
def asset_manifest(session: Session = Depends(module_gate_or_bearer(MODULE))):
|
| 568 |
"""The codes that HAVE an image — so the catalog designer can mark gaps instead of
|
| 569 |
rendering broken tiles. `source` names which backend answered."""
|
| 570 |
d = _local_dir()
|
|
|
|
| 578 |
|
| 579 |
@router.get("/assets/products/{code}")
|
| 580 |
def product_asset(code: str, q: str = "web",
|
| 581 |
+
session: Session = Depends(module_gate_or_bearer(MODULE))):
|
| 582 |
if q not in _QUALITIES:
|
| 583 |
raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
|
| 584 |
norm = _norm(code)
|
|
|
|
| 598 |
|
| 599 |
media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
|
| 600 |
return FileResponse(str(path), media_type=media,
|
| 601 |
+
headers={"Cache-Control": _cache_control(session)})
|
| 602 |
|
| 603 |
|
| 604 |
# ------------------------------------------------------------------ editorial (DEBT-5)
|
|
|
|
| 641 |
|
| 642 |
|
| 643 |
@router.get("/assets/editorial")
|
| 644 |
+
def editorial_manifest(session: Session = Depends(module_gate_or_bearer(MODULE))):
|
| 645 |
"""The slugs that HAVE an editorial image — the designer's picker reads this."""
|
| 646 |
if _local_dir() is not None:
|
| 647 |
ed = _ed_dir()
|
|
|
|
| 652 |
|
| 653 |
@router.get("/assets/editorial/{slug}")
|
| 654 |
def editorial_asset(slug: str, q: str = "web",
|
| 655 |
+
session: Session = Depends(module_gate_or_bearer(MODULE))):
|
| 656 |
if q not in _QUALITIES:
|
| 657 |
raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
|
| 658 |
slug = str(slug or "").strip().lower()
|
|
|
|
| 671 |
raise err(404, "asset_not_found", f"no editorial image named {slug}")
|
| 672 |
media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
|
| 673 |
return FileResponse(str(path), media_type=media,
|
| 674 |
+
headers={"Cache-Control": _cache_control(session)})
|
| 675 |
|
| 676 |
|
| 677 |
@router.post("/assets/editorial", status_code=201)
|
|
|
|
| 840 |
|
| 841 |
@router.get("/assets/records/{asset_id}")
|
| 842 |
def record_asset(asset_id: str, q: str = "web",
|
| 843 |
+
session: Session = Depends(require_session_or_bearer)):
|
| 844 |
"""Serve one record image for THIS session's tenant. Any signed-in user of the tenant may
|
| 845 |
read it — a picture on a row is visible to whoever can open the row, and the row walls
|
| 846 |
already decided that."""
|
|
|
|
| 868 |
raise err(404, "asset_not_found", "no image for that reference")
|
| 869 |
media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
|
| 870 |
return FileResponse(str(path), media_type=media,
|
| 871 |
+
headers={"Cache-Control": _cache_control(session)})
|
api/routes_automation.py
CHANGED
|
@@ -1350,12 +1350,14 @@ def list_automations(session: Session = Depends(_GATE)):
|
|
| 1350 |
retired = []
|
| 1351 |
tables = None
|
| 1352 |
if session.runtime.available() and session.tenant not in _BOARD_RETIRED:
|
| 1353 |
-
tables = engine.ut_all(session.runtime)
|
| 1354 |
_BOARD_RETIRED.add(session.tenant)
|
| 1355 |
# Idempotent Board retirement removes only engine-marked stage fields and legacy Board
|
| 1356 |
# state. User-created Status/Stage columns remain intact.
|
| 1357 |
-
|
| 1358 |
-
|
|
|
|
|
|
|
|
|
|
| 1359 |
# ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY
|
| 1360 |
# ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds
|
| 1361 |
# it present and returns immediately. See `engine.seed_statements_agent` for why this one is
|
|
|
|
| 1350 |
retired = []
|
| 1351 |
tables = None
|
| 1352 |
if session.runtime.available() and session.tenant not in _BOARD_RETIRED:
|
|
|
|
| 1353 |
_BOARD_RETIRED.add(session.tenant)
|
| 1354 |
# Idempotent Board retirement removes only engine-marked stage fields and legacy Board
|
| 1355 |
# state. User-created Status/Stage columns remain intact.
|
| 1356 |
+
if not (session.runtime.get("automation_migrations") or {}).get("board_retired_v1"):
|
| 1357 |
+
tables = engine.ut_all(session.runtime)
|
| 1358 |
+
engine.retire_automation_board_state(session.runtime, tables=tables)
|
| 1359 |
+
retired = engine.retire_preset_automations(session.runtime)["removed"]
|
| 1360 |
+
session.runtime.update("automation_migrations", lambda d: {**(d or {}), "board_retired_v1": True})
|
| 1361 |
# ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY
|
| 1362 |
# ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds
|
| 1363 |
# it present and returns immediately. See `engine.seed_statements_agent` for why this one is
|
api/routes_catalogs.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
| 1 |
-
"""Authenticated catalog record routes for Catalog Apps.
|
|
|
|
|
|
|
|
|
|
| 2 |
from fastapi import APIRouter, Body, Depends, Query
|
| 3 |
from fastapi.responses import Response
|
| 4 |
|
| 5 |
import app_rules
|
| 6 |
import catalog_store
|
| 7 |
-
from deps import Session, err,
|
| 8 |
|
| 9 |
|
| 10 |
router = APIRouter(prefix="/api/v1")
|
|
@@ -20,29 +23,30 @@ def _call(operation, *args, **kwargs):
|
|
| 20 |
@router.get("/apps/{app_id}/catalogs")
|
| 21 |
def catalogs(app_id: str, cursor: str | None = None,
|
| 22 |
limit: int = Query(catalog_store.DEFAULT_LIMIT, ge=1, le=catalog_store.MAX_LIMIT),
|
| 23 |
-
session: Session = Depends(
|
| 24 |
return _call(catalog_store.list_catalogs, session, app_id, cursor=cursor, limit=limit)
|
| 25 |
|
| 26 |
|
| 27 |
@router.post("/apps/{app_id}/catalogs", status_code=201)
|
| 28 |
def create_catalog(app_id: str, body: dict = Body(default=None),
|
| 29 |
-
session: Session = Depends(
|
|
|
|
| 30 |
return {"catalog": _call(catalog_store.create_catalog, session, app_id, body)}
|
| 31 |
|
| 32 |
|
| 33 |
# Static paths must precede the catalog id route.
|
| 34 |
@router.get("/catalogs/templates")
|
| 35 |
-
def catalog_templates(session: Session = Depends(
|
| 36 |
return app_rules.catalog_templates()
|
| 37 |
|
| 38 |
|
| 39 |
@router.get("/catalogs/{catalog_id}")
|
| 40 |
-
def read_catalog(catalog_id: str, session: Session = Depends(
|
| 41 |
return {"catalog": _call(catalog_store.read_catalog, session, catalog_id)}
|
| 42 |
|
| 43 |
|
| 44 |
@router.get("/catalogs/{catalog_id}/pdf")
|
| 45 |
-
def catalog_pdf(catalog_id: str, session: Session = Depends(
|
| 46 |
import catalog_render
|
| 47 |
|
| 48 |
data, _record, _app = _call(catalog_render.render_catalog, session, catalog_id)
|
|
@@ -53,16 +57,19 @@ def catalog_pdf(catalog_id: str, session: Session = Depends(require_session)):
|
|
| 53 |
|
| 54 |
@router.patch("/catalogs/{catalog_id}")
|
| 55 |
def update_catalog(catalog_id: str, body: dict = Body(default=None),
|
| 56 |
-
session: Session = Depends(
|
|
|
|
| 57 |
return {"catalog": _call(catalog_store.update_catalog, session, catalog_id, body)}
|
| 58 |
|
| 59 |
|
| 60 |
@router.post("/catalogs/{catalog_id}/duplicate", status_code=201)
|
| 61 |
def duplicate_catalog(catalog_id: str, body: dict = Body(default=None),
|
| 62 |
-
session: Session = Depends(
|
|
|
|
| 63 |
return {"catalog": _call(catalog_store.duplicate_catalog, session, catalog_id, body)}
|
| 64 |
|
| 65 |
|
| 66 |
@router.delete("/catalogs/{catalog_id}")
|
| 67 |
-
def delete_catalog(catalog_id: str, session: Session = Depends(
|
|
|
|
| 68 |
return _call(catalog_store.delete_catalog, session, catalog_id)
|
|
|
|
| 1 |
+
"""Authenticated catalog record routes for Catalog Apps.
|
| 2 |
+
|
| 3 |
+
Every route admits the cookie or an external bearer under its own key's grants (S79-T09).
|
| 4 |
+
"""
|
| 5 |
from fastapi import APIRouter, Body, Depends, Query
|
| 6 |
from fastapi.responses import Response
|
| 7 |
|
| 8 |
import app_rules
|
| 9 |
import catalog_store
|
| 10 |
+
from deps import Session, err, require_bearer_write, require_session_or_bearer
|
| 11 |
|
| 12 |
|
| 13 |
router = APIRouter(prefix="/api/v1")
|
|
|
|
| 23 |
@router.get("/apps/{app_id}/catalogs")
|
| 24 |
def catalogs(app_id: str, cursor: str | None = None,
|
| 25 |
limit: int = Query(catalog_store.DEFAULT_LIMIT, ge=1, le=catalog_store.MAX_LIMIT),
|
| 26 |
+
session: Session = Depends(require_session_or_bearer)):
|
| 27 |
return _call(catalog_store.list_catalogs, session, app_id, cursor=cursor, limit=limit)
|
| 28 |
|
| 29 |
|
| 30 |
@router.post("/apps/{app_id}/catalogs", status_code=201)
|
| 31 |
def create_catalog(app_id: str, body: dict = Body(default=None),
|
| 32 |
+
session: Session = Depends(require_session_or_bearer)):
|
| 33 |
+
require_bearer_write(session, "catalogs")
|
| 34 |
return {"catalog": _call(catalog_store.create_catalog, session, app_id, body)}
|
| 35 |
|
| 36 |
|
| 37 |
# Static paths must precede the catalog id route.
|
| 38 |
@router.get("/catalogs/templates")
|
| 39 |
+
def catalog_templates(session: Session = Depends(require_session_or_bearer)):
|
| 40 |
return app_rules.catalog_templates()
|
| 41 |
|
| 42 |
|
| 43 |
@router.get("/catalogs/{catalog_id}")
|
| 44 |
+
def read_catalog(catalog_id: str, session: Session = Depends(require_session_or_bearer)):
|
| 45 |
return {"catalog": _call(catalog_store.read_catalog, session, catalog_id)}
|
| 46 |
|
| 47 |
|
| 48 |
@router.get("/catalogs/{catalog_id}/pdf")
|
| 49 |
+
def catalog_pdf(catalog_id: str, session: Session = Depends(require_session_or_bearer)):
|
| 50 |
import catalog_render
|
| 51 |
|
| 52 |
data, _record, _app = _call(catalog_render.render_catalog, session, catalog_id)
|
|
|
|
| 57 |
|
| 58 |
@router.patch("/catalogs/{catalog_id}")
|
| 59 |
def update_catalog(catalog_id: str, body: dict = Body(default=None),
|
| 60 |
+
session: Session = Depends(require_session_or_bearer)):
|
| 61 |
+
require_bearer_write(session, "catalogs")
|
| 62 |
return {"catalog": _call(catalog_store.update_catalog, session, catalog_id, body)}
|
| 63 |
|
| 64 |
|
| 65 |
@router.post("/catalogs/{catalog_id}/duplicate", status_code=201)
|
| 66 |
def duplicate_catalog(catalog_id: str, body: dict = Body(default=None),
|
| 67 |
+
session: Session = Depends(require_session_or_bearer)):
|
| 68 |
+
require_bearer_write(session, "catalogs")
|
| 69 |
return {"catalog": _call(catalog_store.duplicate_catalog, session, catalog_id, body)}
|
| 70 |
|
| 71 |
|
| 72 |
@router.delete("/catalogs/{catalog_id}")
|
| 73 |
+
def delete_catalog(catalog_id: str, session: Session = Depends(require_session_or_bearer)):
|
| 74 |
+
require_bearer_write(session, "catalogs")
|
| 75 |
return _call(catalog_store.delete_catalog, session, catalog_id)
|
api/routes_connectors.py
CHANGED
|
@@ -54,6 +54,7 @@ from core import connector_sync
|
|
| 54 |
from core.request_timing import raise_if_resource_error
|
| 55 |
from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD
|
| 56 |
from harness import amazon_client
|
|
|
|
| 57 |
#: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default,
|
| 58 |
#: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already
|
| 59 |
#: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module
|
|
@@ -499,6 +500,12 @@ TOKEN_CONNECTORS = [
|
|
| 499 |
"marketplace_id.",
|
| 500 |
# R22: status comes from the stored verification state, without mechanism copy.
|
| 501 |
"note": ""},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 502 |
]
|
| 503 |
|
| 504 |
#: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
|
|
@@ -602,7 +609,7 @@ def _register_probers():
|
|
| 602 |
store a credential and never the API's ability to boot.
|
| 603 |
"""
|
| 604 |
ok = False
|
| 605 |
-
for mod in ("connectors_meta", "connectors_amazon"):
|
| 606 |
try:
|
| 607 |
ok = __import__(mod).register() or ok
|
| 608 |
except Exception: # noqa: BLE001
|
|
@@ -986,7 +993,8 @@ def _entry_types(rt, uname=""):
|
|
| 986 |
types, scopes = set(), {}
|
| 987 |
for e in rows:
|
| 988 |
t = str(e.get("type") or "")
|
| 989 |
-
|
|
|
|
| 990 |
from core import keychain as kc
|
| 991 |
if kc.entry_verified(rt, e.get("id")):
|
| 992 |
types.add(t)
|
|
@@ -1238,7 +1246,9 @@ def directory(rt, uname, is_admin=False, incomplete=None):
|
|
| 1238 |
rows.append({
|
| 1239 |
"key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"],
|
| 1240 |
"hint": t["hint"],
|
| 1241 |
-
"state": (
|
|
|
|
|
|
|
| 1242 |
else "connected" if t["entry"] in have else "reconnect"
|
| 1243 |
if t["entry"] == "amazon_sp" and t["entry"] in scopes
|
| 1244 |
else "available"),
|
|
@@ -1268,8 +1278,10 @@ def directory(rt, uname, is_admin=False, incomplete=None):
|
|
| 1268 |
# 400s — the exact failure R6 removed from the Google card. Today that means
|
| 1269 |
# Stripe and Shopify open for a member and Meta Ads does not
|
| 1270 |
# ([[one-question-two-normalizers]]).
|
|
|
|
| 1271 |
"manage": ("keychain" if is_admin
|
| 1272 |
else "" if t["entry"] in TENANT_WIDE_TYPES
|
|
|
|
| 1273 |
else "keychain_personal"),
|
| 1274 |
# Said once, on the row, rather than in a paragraph somewhere (R13): storing the
|
| 1275 |
# key is real and useful, and it is not the same as the data being on screen.
|
|
@@ -3519,6 +3531,14 @@ def _entry_type_for(connector_key):
|
|
| 3519 |
_AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret",
|
| 3520 |
"refresh_token": "Refresh token", "seller_id": "Seller ID",
|
| 3521 |
"marketplace_id": "Marketplace ID"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3522 |
_CONNECT_FIELDS = {
|
| 3523 |
"odoo": [{"key": "url", "label": "Server URL", "secret": False,
|
| 3524 |
"hint": "Your Odoo address, starting with https://."},
|
|
@@ -3545,6 +3565,14 @@ _CONNECT_FIELDS = {
|
|
| 3545 |
**({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})}
|
| 3546 |
for key in amazon_client.CREDENTIAL_FIELDS
|
| 3547 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3548 |
"stripe": [{"key": "api_key", "label": "API key", "secret": True,
|
| 3549 |
"hint": "A restricted key with read access."}],
|
| 3550 |
"shopify": [{"key": "api_key", "label": "Admin API token", "secret": True,
|
|
@@ -3669,6 +3697,16 @@ _CONNECT_TUTORIALS = {
|
|
| 3669 |
"and check its marketplace access. Correct the saved key in Settings.",
|
| 3670 |
"testNote": "Checks marketplace access. Saves the key before testing.",
|
| 3671 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3672 |
"stripe": {
|
| 3673 |
"need": "Access to your Stripe dashboard's API keys.",
|
| 3674 |
"steps": ["Open the Stripe dashboard, then Developers and API keys.",
|
|
@@ -3695,12 +3733,12 @@ _CONNECT_TUTORIALS = {
|
|
| 3695 |
},
|
| 3696 |
"webhooks": {
|
| 3697 |
"need": "An automation and a system that can send an HTTP POST.",
|
| 3698 |
-
|
| 3699 |
-
|
| 3700 |
-
"
|
| 3701 |
-
"
|
| 3702 |
-
"
|
| 3703 |
-
"failureAction": "
|
| 3704 |
"testNote": "Sending to the hook runs the automation. Check its run history after an intentional send.",
|
| 3705 |
},
|
| 3706 |
}
|
|
@@ -3905,7 +3943,8 @@ def _detail_connect(entry_type, is_admin):
|
|
| 3905 |
if not entry_type:
|
| 3906 |
return None
|
| 3907 |
fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD)
|
| 3908 |
-
tenant_wide = entry_type in TENANT_WIDE_TYPES
|
|
|
|
| 3909 |
hint = ""
|
| 3910 |
try:
|
| 3911 |
hint = next((str(token.get("hint") or "") for token in TOKEN_CONNECTORS
|
|
|
|
| 54 |
from core.request_timing import raise_if_resource_error
|
| 55 |
from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD
|
| 56 |
from harness import amazon_client
|
| 57 |
+
import connectors_amazon_ads
|
| 58 |
#: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default,
|
| 59 |
#: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already
|
| 60 |
#: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module
|
|
|
|
| 500 |
"marketplace_id.",
|
| 501 |
# R22: status comes from the stored verification state, without mechanism copy.
|
| 502 |
"note": ""},
|
| 503 |
+
# S79-T13 (R7, R9): Amazon Ads reads SP, SB and SD reports. Settings only with a verified
|
| 504 |
+
# credential; an expired or failed token reads Connect again (`connectors_amazon_ads.card_state`).
|
| 505 |
+
{"key": "amazon_ads", "label": "Amazon Ads", "entry": "amazon_ads",
|
| 506 |
+
"desc": "Sponsored ads reports",
|
| 507 |
+
"hint": "Paste the four values from your Amazon Ads app",
|
| 508 |
+
"note": ""},
|
| 509 |
]
|
| 510 |
|
| 511 |
#: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
|
|
|
|
| 609 |
store a credential and never the API's ability to boot.
|
| 610 |
"""
|
| 611 |
ok = False
|
| 612 |
+
for mod in ("connectors_meta", "connectors_amazon", "connectors_amazon_ads"):
|
| 613 |
try:
|
| 614 |
ok = __import__(mod).register() or ok
|
| 615 |
except Exception: # noqa: BLE001
|
|
|
|
| 993 |
types, scopes = set(), {}
|
| 994 |
for e in rows:
|
| 995 |
t = str(e.get("type") or "")
|
| 996 |
+
# Amazon SP and Amazon Ads count only with a passing probe proof for the current secret.
|
| 997 |
+
if t in ("amazon_sp", "amazon_ads"):
|
| 998 |
from core import keychain as kc
|
| 999 |
if kc.entry_verified(rt, e.get("id")):
|
| 1000 |
types.add(t)
|
|
|
|
| 1246 |
rows.append({
|
| 1247 |
"key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"],
|
| 1248 |
"hint": t["hint"],
|
| 1249 |
+
"state": (connectors_amazon_ads.card_state(t["entry"] in have)
|
| 1250 |
+
if t["entry"] == connectors_amazon_ads.ENTRY_TYPE
|
| 1251 |
+
else "expired" if t["entry"] == "amazon_sp" and amazon_expired
|
| 1252 |
else "connected" if t["entry"] in have else "reconnect"
|
| 1253 |
if t["entry"] == "amazon_sp" and t["entry"] in scopes
|
| 1254 |
else "available"),
|
|
|
|
| 1278 |
# 400s — the exact failure R6 removed from the Google card. Today that means
|
| 1279 |
# Stripe and Shopify open for a member and Meta Ads does not
|
| 1280 |
# ([[one-question-two-normalizers]]).
|
| 1281 |
+
# S79-T13: Amazon Ads is business wide, before and after the keychain learns it.
|
| 1282 |
"manage": ("keychain" if is_admin
|
| 1283 |
else "" if t["entry"] in TENANT_WIDE_TYPES
|
| 1284 |
+
or t["entry"] == connectors_amazon_ads.ENTRY_TYPE
|
| 1285 |
else "keychain_personal"),
|
| 1286 |
# Said once, on the row, rather than in a paragraph somewhere (R13): storing the
|
| 1287 |
# key is real and useful, and it is not the same as the data being on screen.
|
|
|
|
| 3531 |
_AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret",
|
| 3532 |
"refresh_token": "Refresh token", "seller_id": "Seller ID",
|
| 3533 |
"marketplace_id": "Marketplace ID"}
|
| 3534 |
+
_AMAZON_ADS_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret",
|
| 3535 |
+
"refresh_token": "Refresh token", "profile_id": "Profile ID"}
|
| 3536 |
+
_AMAZON_ADS_FIELD_HINTS = {
|
| 3537 |
+
"client_id": "Your Amazon Ads app's Login with Amazon client ID.",
|
| 3538 |
+
"client_secret": "The same app's client secret.",
|
| 3539 |
+
"refresh_token": "The refresh token returned when you authorize the app.",
|
| 3540 |
+
"profile_id": "The advertising profile for your marketplace.",
|
| 3541 |
+
}
|
| 3542 |
_CONNECT_FIELDS = {
|
| 3543 |
"odoo": [{"key": "url", "label": "Server URL", "secret": False,
|
| 3544 |
"hint": "Your Odoo address, starting with https://."},
|
|
|
|
| 3565 |
**({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})}
|
| 3566 |
for key in amazon_client.CREDENTIAL_FIELDS
|
| 3567 |
],
|
| 3568 |
+
# S79-T13: keys and order derived from `amazon_ads_client.CREDENTIAL_FIELDS`, like Amazon SP.
|
| 3569 |
+
"amazon_ads": [
|
| 3570 |
+
{"key": key,
|
| 3571 |
+
"label": _AMAZON_ADS_FIELD_LABELS.get(key, key.replace("_", " ").capitalize()),
|
| 3572 |
+
"secret": key in ("client_secret", "refresh_token"),
|
| 3573 |
+
"hint": _AMAZON_ADS_FIELD_HINTS.get(key, "")}
|
| 3574 |
+
for key in connectors_amazon_ads.CREDENTIAL_FIELDS
|
| 3575 |
+
],
|
| 3576 |
"stripe": [{"key": "api_key", "label": "API key", "secret": True,
|
| 3577 |
"hint": "A restricted key with read access."}],
|
| 3578 |
"shopify": [{"key": "api_key", "label": "Admin API token", "secret": True,
|
|
|
|
| 3697 |
"and check its marketplace access. Correct the saved key in Settings.",
|
| 3698 |
"testNote": "Checks marketplace access. Saves the key before testing.",
|
| 3699 |
},
|
| 3700 |
+
"amazon_ads": {
|
| 3701 |
+
"need": "An Amazon Ads API app and access to your advertising account.",
|
| 3702 |
+
"steps": ["Open your Amazon Ads API app's Login with Amazon security profile.",
|
| 3703 |
+
"Copy its client ID and client secret.",
|
| 3704 |
+
"Authorize the app for your advertising account and copy the refresh token.",
|
| 3705 |
+
"Copy the profile ID of the marketplace you advertise in."],
|
| 3706 |
+
"failureAction": "Authorize the app again for your advertising account, then check "
|
| 3707 |
+
"that the profile ID belongs to it.",
|
| 3708 |
+
"testNote": "Checks access to your advertising profile. Saves the key before testing.",
|
| 3709 |
+
},
|
| 3710 |
"stripe": {
|
| 3711 |
"need": "Access to your Stripe dashboard's API keys.",
|
| 3712 |
"steps": ["Open the Stripe dashboard, then Developers and API keys.",
|
|
|
|
| 3733 |
},
|
| 3734 |
"webhooks": {
|
| 3735 |
"need": "An automation and a system that can send an HTTP POST.",
|
| 3736 |
+
# The client lists each automation's generated URL. No step names a credential or a keychain Test.
|
| 3737 |
+
"steps": ["Open Automations and create or open an automation.",
|
| 3738 |
+
"Choose the Webhooks trigger and save it. Saving creates its own URL.",
|
| 3739 |
+
"Copy its URL below. Keep it private, because anyone with it can run the automation.",
|
| 3740 |
+
"Send an HTTP POST to that URL from your other system."],
|
| 3741 |
+
"failureAction": "Copy the URL again from this guide. Check that the trigger is on.",
|
| 3742 |
"testNote": "Sending to the hook runs the automation. Check its run history after an intentional send.",
|
| 3743 |
},
|
| 3744 |
}
|
|
|
|
| 3943 |
if not entry_type:
|
| 3944 |
return None
|
| 3945 |
fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD)
|
| 3946 |
+
tenant_wide = (entry_type in TENANT_WIDE_TYPES
|
| 3947 |
+
or entry_type == connectors_amazon_ads.ENTRY_TYPE)
|
| 3948 |
hint = ""
|
| 3949 |
try:
|
| 3950 |
hint = next((str(token.get("hint") or "") for token in TOKEN_CONNECTORS
|
api/routes_customers.py
CHANGED
|
@@ -229,6 +229,22 @@ def warm_default(rt):
|
|
| 229 |
_pool_for(rt, None, None)
|
| 230 |
|
| 231 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
def _team_agent(session: Session):
|
| 233 |
"""The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by
|
| 234 |
`_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree.
|
|
|
|
| 229 |
_pool_for(rt, None, None)
|
| 230 |
|
| 231 |
|
| 232 |
+
def warm_bu_pools(rt):
|
| 233 |
+
"""Warm each permitted business-unit pool outside the request path."""
|
| 234 |
+
from core import context, perms
|
| 235 |
+
for label in perms.allowed_bu_labels({"bus": "all", "role": "admin"}):
|
| 236 |
+
if label == "All" or label not in context.BRANDS:
|
| 237 |
+
continue
|
| 238 |
+
team = perms.scope_team_id({"bus": [context.BRANDS[label]], "role": "user"})
|
| 239 |
+
if team is None:
|
| 240 |
+
continue
|
| 241 |
+
try:
|
| 242 |
+
_pool_for(rt, team, None)
|
| 243 |
+
print(f"[aios-api] customer read pool warm: {label}")
|
| 244 |
+
except Exception as exc:
|
| 245 |
+
print(f"[aios-api] customer read pool warm refused: {label} ({type(exc).__name__})")
|
| 246 |
+
|
| 247 |
+
|
| 248 |
def _team_agent(session: Session):
|
| 249 |
"""The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by
|
| 250 |
`_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree.
|
api/routes_external.py
CHANGED
|
@@ -27,6 +27,9 @@ import aios_session
|
|
| 27 |
from deps import Session, err, require_session
|
| 28 |
from core import audit, data_binding, external_access, external_oauth, external_tools, perm_scope, sheets
|
| 29 |
from harness import runtime
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
EXTERNAL_PREFIX = "/api/v1/external"
|
|
@@ -228,6 +231,9 @@ class _PreviewRuntime:
|
|
| 228 |
key = str(name)
|
| 229 |
self._load(key)
|
| 230 |
base = self._base_revisions.get(key)
|
|
|
|
|
|
|
|
|
|
| 231 |
rev = int((base or {}).get("rev") or 0) + self._local_revisions.get(key, 0)
|
| 232 |
return {"rev": rev, "token": _json_digest({"key": key, "rev": rev,
|
| 233 |
"base": base})}
|
|
@@ -851,6 +857,12 @@ def _assert_query_filter_bounds(raw) -> None:
|
|
| 851 |
stack.extend((child, depth + 1) for child in children)
|
| 852 |
|
| 853 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 854 |
def _assert_tool_query_filter_bounds(name, arguments):
|
| 855 |
"""Name query limits before the finite tool schema reports a generic shape error."""
|
| 856 |
if name in ("table", "data") and isinstance(arguments, dict) and arguments.get("action") == "query_database":
|
|
@@ -1616,6 +1628,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str:
|
|
| 1616 |
"",
|
| 1617 |
])
|
| 1618 |
|
|
|
|
| 1619 |
if public:
|
| 1620 |
lines.extend([
|
| 1621 |
"## Your read scope",
|
|
@@ -2381,7 +2394,7 @@ def external_setup(request: Request):
|
|
| 2381 |
|
| 2382 |
|
| 2383 |
@router.get("/skill.md")
|
| 2384 |
-
def portable_skill(request: Request, session: Session = Depends(
|
| 2385 |
"""Download current governed external read and build instructions."""
|
| 2386 |
require_external_analyst(session)
|
| 2387 |
document = _portable_skill_document(session, str(request.base_url))
|
|
@@ -2483,11 +2496,13 @@ def query_database(
|
|
| 2483 |
database: str,
|
| 2484 |
body: dict | None = Body(default=None),
|
| 2485 |
session: Session = Depends(require_external_session),
|
|
|
|
| 2486 |
):
|
| 2487 |
"""Read one freshly permissioned, bounded and cursor-stable Assistant snapshot."""
|
| 2488 |
try:
|
| 2489 |
require_external_analyst(session)
|
| 2490 |
-
|
|
|
|
| 2491 |
except HTTPException as exc:
|
| 2492 |
external_activity.record_refused(session, database, exc)
|
| 2493 |
raise
|
|
@@ -2557,6 +2572,11 @@ def _query_database(session: Session, database: str, payload: dict):
|
|
| 2557 |
"assistant_store_not_ready", "store_not_ready", "amazon_mirror_not_loaded"}:
|
| 2558 |
raise err(503, "query_source_unavailable",
|
| 2559 |
"This database is temporarily unavailable. Try the same query again shortly.") from exc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2560 |
if info.get("code") == "query_cursor_stale":
|
| 2561 |
fix = _MCP_FIXES["query_cursor_stale"]
|
| 2562 |
if fix not in info["message"]:
|
|
@@ -2612,6 +2632,37 @@ def _query_activity_page(session, payload, scoped, selected_keys, sorts, aggrega
|
|
| 2612 |
return result
|
| 2613 |
|
| 2614 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2615 |
def _query_database_page(session: Session, database: str, payload: dict):
|
| 2616 |
if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS:
|
| 2617 |
raise err(400, "query_arguments_invalid",
|
|
@@ -2784,6 +2835,7 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 2784 |
records = [{"pid": copy.deepcopy(row.get("pid")),
|
| 2785 |
**{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
|
| 2786 |
for row in page_rows]
|
|
|
|
| 2787 |
next_offset = offset + limit
|
| 2788 |
check_publication()
|
| 2789 |
next_cursor = (_encode_query_cursor(session, page_digest, snap, next_offset, publication)
|
|
@@ -2849,6 +2901,7 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 2849 |
**{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
|
| 2850 |
for row in page_rows
|
| 2851 |
]
|
|
|
|
| 2852 |
next_offset = offset + len(page_rows)
|
| 2853 |
check_publication()
|
| 2854 |
next_cursor = (_encode_query_cursor(session, query_digest, snapshot_digest, next_offset, publication)
|
|
@@ -3259,6 +3312,11 @@ def _application_upsert(session, args):
|
|
| 3259 |
allowed, _ = _application_access(session, current)
|
| 3260 |
if not current or not allowed:
|
| 3261 |
raise applications.ApplicationError("application does not exist")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3262 |
proposed = payload.get("compatibleDatabases") if "compatibleDatabases" in payload else current
|
| 3263 |
proposed_allowed, records = _application_access(session, proposed, strict=True)
|
| 3264 |
return {"application": applications.mutate(
|
|
@@ -4052,6 +4110,11 @@ def _affected_for(name, args, diff, result=None):
|
|
| 4052 |
ranges.append({"workbook_id": arguments.get("workbook_id"),
|
| 4053 |
"tab": arguments.get("tab") or arguments.get("worksheet"), "range": address})
|
| 4054 |
collect(args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4055 |
return {"ids": values[:100], "count": max(len(values), len(diff)), "ranges": ranges}
|
| 4056 |
|
| 4057 |
|
|
@@ -4617,7 +4680,7 @@ def _confirmation_claim(session, operation_id):
|
|
| 4617 |
if item["status"] != "preview":
|
| 4618 |
raise _OperationAlreadyClaimed()
|
| 4619 |
if item["payload"]["expiry"] <= time.time():
|
| 4620 |
-
raise err(409, "confirmation_expired", "
|
| 4621 |
item["status"] = "claimed"
|
| 4622 |
item["retain_until"] = time.time() + _OPERATION_RETENTION_SECONDS
|
| 4623 |
return raw
|
|
@@ -4746,6 +4809,24 @@ def _finish_website_note(session, payload, plan, result):
|
|
| 4746 |
_STALE_FIX = "so repeat the original call for a new preview and confirm with its new confirmation_token."
|
| 4747 |
|
| 4748 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4749 |
def _confirm_fix(args):
|
| 4750 |
"""Name what a confirm_change call got wrong and the shape that succeeds."""
|
| 4751 |
allowed = ("confirmation_token", "confirmation_phrase", "intent")
|
|
@@ -4778,8 +4859,7 @@ def _confirm_change(session, args):
|
|
| 4778 |
if payload.get("environment") != _write_environment():
|
| 4779 |
raise err(409, "confirmation_stale", "The environment changed after the preview, " + _STALE_FIX)
|
| 4780 |
if int(payload.get("expiry") or 0) <= int(time.time()):
|
| 4781 |
-
raise err(409, "confirmation_expired",
|
| 4782 |
-
"This confirmation token expired. Preview the change again.")
|
| 4783 |
if payload.get("strong") and args.get("confirmation_phrase") != _CONFIRMATION_PHRASE:
|
| 4784 |
raise err(400, "confirmation_phrase_required",
|
| 4785 |
"Enter the exact stronger confirmation phrase shown in the preview.")
|
|
@@ -4905,7 +4985,7 @@ def _confirm_change(session, args):
|
|
| 4905 |
_MCP_CODE_GROUPS = (
|
| 4906 |
("The arguments do not match this action. Read its guide and correct them.", """
|
| 4907 |
tool_arguments_invalid query_arguments_invalid query_aggregates_invalid query_aggregate_type
|
| 4908 |
-
query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid
|
| 4909 |
bad_application bad_entries bad_value bad_values bad_request bad_recurrence invalid_field_contract
|
| 4910 |
external_edit_empty external_edit_invalid oauth_edit_empty oauth_edit_invalid
|
| 4911 |
"""),
|
|
@@ -5000,6 +5080,12 @@ MCP_RESULT_CODES.update({
|
|
| 5000 |
"confirmation_session_required": "Read this confirmation through its authenticated connection.",
|
| 5001 |
"link_selection_refused": "Choose a readable linked record that matches the field's selection condition.",
|
| 5002 |
"app_rule": "Correct the App rule named in the refusal before saving it.",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5003 |
"audience_not_allowed": "A person must choose sharing with everyone on screen.",
|
| 5004 |
"not_owner": "You cannot administer this item's audience.",
|
| 5005 |
"unknown_people": "Choose active people in this workspace.",
|
|
@@ -5148,6 +5234,8 @@ _MCP_RESULT_OBJECTS = frozenset({
|
|
| 5148 |
"applications", "view", "views", "from", "to", "source", "affected", "aggregates", "tools",
|
| 5149 |
"annotations", "serverInfo", "environment",
|
| 5150 |
})
|
|
|
|
|
|
|
| 5151 |
_MCP_AUTHORED_DESCRIPTIONS = frozenset({
|
| 5152 |
"fields", "file", "files", "template", "templates", "application", "applications", "view", "views",
|
| 5153 |
})
|
|
@@ -5174,6 +5262,8 @@ def _mcp_result_copy(value, labels=None, *, field=None, metadata=False):
|
|
| 5174 |
result[key] = copy.deepcopy(child)
|
| 5175 |
elif key == "about":
|
| 5176 |
result[key] = copy.deepcopy(child)
|
|
|
|
|
|
|
| 5177 |
elif key in _MCP_RESULT_OPAQUE and not metadata:
|
| 5178 |
result[key] = copy.deepcopy(child)
|
| 5179 |
elif key in {"schema", "inputSchema"}:
|
|
@@ -5944,7 +6034,10 @@ def _record_mcp_method(session, req, reply):
|
|
| 5944 |
or audit.redact_intent(method) != method):
|
| 5945 |
method = "[redacted]"
|
| 5946 |
error = reply.get("error", {}).get("code")
|
| 5947 |
-
|
|
|
|
|
|
|
|
|
|
| 5948 |
external_activity.record_protocol(session, method, code=code)
|
| 5949 |
except Exception:
|
| 5950 |
pass
|
|
@@ -6175,6 +6268,20 @@ def _dispatch_mcp(req, session: Session, base_url: str = ""):
|
|
| 6175 |
return reply
|
| 6176 |
|
| 6177 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6178 |
def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
| 6179 |
"""One HTTP request object; the stdio precedent's JSON-RPC and content envelopes."""
|
| 6180 |
protocol_error = _mcp_protocol_error
|
|
@@ -6326,6 +6433,9 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 6326 |
wiki_log.record_build(session, kind=kind, object_id=arguments["arguments"]["path"],
|
| 6327 |
event_id="mcp-wiki:" + _json_digest({"path": arguments["arguments"]["path"], "version": data["version"]}),
|
| 6328 |
occurred_at=data["updated"], purpose=purpose, about=about, build_kind=kind)
|
|
|
|
|
|
|
|
|
|
| 6329 |
else:
|
| 6330 |
data = _dispatch_external_tool(_tool_context(session), name, arguments)
|
| 6331 |
if isinstance(data, dict) and data.get("refused"):
|
|
@@ -6353,6 +6463,10 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 6353 |
import routes_sheet
|
| 6354 |
if exc.code == "tool_arguments_invalid" and name == "confirm_change":
|
| 6355 |
exc = sheets.SheetRefusal(exc.code, _confirm_fix(arguments))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6356 |
translated = routes_sheet._refuse(exc)
|
| 6357 |
info = translated.detail["error"]
|
| 6358 |
out = _tool_refusal(name, info["code"], info["message"], translated.status_code,
|
|
@@ -6377,6 +6491,8 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 6377 |
message = info.get("message", "The app refused this operation.")
|
| 6378 |
if info.get("code") == "build_about_required" and isinstance(message, str):
|
| 6379 |
message += _mcp_about_fix(called)
|
|
|
|
|
|
|
| 6380 |
out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry)
|
| 6381 |
except (TypeError, ValueError, KeyError, AttributeError):
|
| 6382 |
out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
|
|
|
|
| 27 |
from deps import Session, err, require_session
|
| 28 |
from core import audit, data_binding, external_access, external_oauth, external_tools, perm_scope, sheets
|
| 29 |
from harness import runtime
|
| 30 |
+
from tool_groups.data import (QUERY_SYNTAX_CODES, UNKNOWN_DATABASE_FIX, bad_filter_message,
|
| 31 |
+
query_syntax_section, with_query_syntax)
|
| 32 |
+
from tool_groups.sheet import WORKBOOK_MISSING_FIX
|
| 33 |
|
| 34 |
|
| 35 |
EXTERNAL_PREFIX = "/api/v1/external"
|
|
|
|
| 231 |
key = str(name)
|
| 232 |
self._load(key)
|
| 233 |
base = self._base_revisions.get(key)
|
| 234 |
+
if isinstance(base, dict) and not self._local_revisions.get(key, 0):
|
| 235 |
+
# D-8920: an unchanged copy answers its source revision, so a preview staged inside a preview pins it.
|
| 236 |
+
return copy.deepcopy(base)
|
| 237 |
rev = int((base or {}).get("rev") or 0) + self._local_revisions.get(key, 0)
|
| 238 |
return {"rev": rev, "token": _json_digest({"key": key, "rev": rev,
|
| 239 |
"base": base})}
|
|
|
|
| 857 |
stack.extend((child, depth + 1) for child in children)
|
| 858 |
|
| 859 |
|
| 860 |
+
def _is_query_call(name, arguments):
|
| 861 |
+
"""Tell a query_database call at its bare, table or data address."""
|
| 862 |
+
return name == "query_database" or (
|
| 863 |
+
name in ("table", "data") and isinstance(arguments, dict) and arguments.get("action") == "query_database")
|
| 864 |
+
|
| 865 |
+
|
| 866 |
def _assert_tool_query_filter_bounds(name, arguments):
|
| 867 |
"""Name query limits before the finite tool schema reports a generic shape error."""
|
| 868 |
if name in ("table", "data") and isinstance(arguments, dict) and arguments.get("action") == "query_database":
|
|
|
|
| 1628 |
"",
|
| 1629 |
])
|
| 1630 |
|
| 1631 |
+
lines.extend(query_syntax_section())
|
| 1632 |
if public:
|
| 1633 |
lines.extend([
|
| 1634 |
"## Your read scope",
|
|
|
|
| 2394 |
|
| 2395 |
|
| 2396 |
@router.get("/skill.md")
|
| 2397 |
+
def portable_skill(request: Request, session: Session = Depends(deps.require_session_or_bearer)):
|
| 2398 |
"""Download current governed external read and build instructions."""
|
| 2399 |
require_external_analyst(session)
|
| 2400 |
document = _portable_skill_document(session, str(request.base_url))
|
|
|
|
| 2496 |
database: str,
|
| 2497 |
body: dict | None = Body(default=None),
|
| 2498 |
session: Session = Depends(require_external_session),
|
| 2499 |
+
request: Request = None,
|
| 2500 |
):
|
| 2501 |
"""Read one freshly permissioned, bounded and cursor-stable Assistant snapshot."""
|
| 2502 |
try:
|
| 2503 |
require_external_analyst(session)
|
| 2504 |
+
with _external_image_base(_mcp_public_origin(request) if request is not None else ""):
|
| 2505 |
+
return _query_database(session, database, body if body is not None else {})
|
| 2506 |
except HTTPException as exc:
|
| 2507 |
external_activity.record_refused(session, database, exc)
|
| 2508 |
raise
|
|
|
|
| 2572 |
"assistant_store_not_ready", "store_not_ready", "amazon_mirror_not_loaded"}:
|
| 2573 |
raise err(503, "query_source_unavailable",
|
| 2574 |
"This database is temporarily unavailable. Try the same query again shortly.") from exc
|
| 2575 |
+
if info.get("code") == "assistant_bad_filter":
|
| 2576 |
+
# S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules.
|
| 2577 |
+
info["message"] = bad_filter_message(info.get("message"))
|
| 2578 |
+
if info.get("code") in QUERY_SYNTAX_CODES and isinstance(info.get("message"), str):
|
| 2579 |
+
info["message"] = with_query_syntax(info["message"])
|
| 2580 |
if info.get("code") == "query_cursor_stale":
|
| 2581 |
fix = _MCP_FIXES["query_cursor_stale"]
|
| 2582 |
if fix not in info["message"]:
|
|
|
|
| 2632 |
return result
|
| 2633 |
|
| 2634 |
|
| 2635 |
+
@contextmanager
|
| 2636 |
+
def _external_image_base(base_url):
|
| 2637 |
+
"""Give product rows image URLs on this origin for one external query, and only that query."""
|
| 2638 |
+
from modules import product_data
|
| 2639 |
+
token = product_data.EXTERNAL_IMAGE_BASE.set(str(base_url or "").rstrip("/"))
|
| 2640 |
+
try:
|
| 2641 |
+
yield
|
| 2642 |
+
finally:
|
| 2643 |
+
product_data.EXTERNAL_IMAGE_BASE.reset(token)
|
| 2644 |
+
|
| 2645 |
+
|
| 2646 |
+
def _external_image_urls(session, database, rows, records):
|
| 2647 |
+
"""Replace each product row's Image cell with a URL the caller's own credential fetches."""
|
| 2648 |
+
if database != "product_data" or (getattr(session, "claims", None) or {}).get("external") is not True:
|
| 2649 |
+
return records
|
| 2650 |
+
from modules import product_data
|
| 2651 |
+
base = product_data.EXTERNAL_IMAGE_BASE.get()
|
| 2652 |
+
key = product_data.IMAGE_FIELD_KEY
|
| 2653 |
+
if base is None or not records or key not in records[0]:
|
| 2654 |
+
return records
|
| 2655 |
+
import routes_assets
|
| 2656 |
+
try:
|
| 2657 |
+
masters = routes_assets.product_image_codes()
|
| 2658 |
+
except Exception: # noqa: BLE001 - an unreadable manifest leaves empty cells empty, never fails the read
|
| 2659 |
+
masters = set()
|
| 2660 |
+
for row, record in zip(rows, records):
|
| 2661 |
+
path = product_data.image_path(record.get(key), row.get("code"), masters)
|
| 2662 |
+
record[key] = base + path if path.startswith("/") else path
|
| 2663 |
+
return records
|
| 2664 |
+
|
| 2665 |
+
|
| 2666 |
def _query_database_page(session: Session, database: str, payload: dict):
|
| 2667 |
if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS:
|
| 2668 |
raise err(400, "query_arguments_invalid",
|
|
|
|
| 2835 |
records = [{"pid": copy.deepcopy(row.get("pid")),
|
| 2836 |
**{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
|
| 2837 |
for row in page_rows]
|
| 2838 |
+
records = _external_image_urls(session, database, page_rows, records)
|
| 2839 |
next_offset = offset + limit
|
| 2840 |
check_publication()
|
| 2841 |
next_cursor = (_encode_query_cursor(session, page_digest, snap, next_offset, publication)
|
|
|
|
| 2901 |
**{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
|
| 2902 |
for row in page_rows
|
| 2903 |
]
|
| 2904 |
+
records = _external_image_urls(session, database, page_rows, records)
|
| 2905 |
next_offset = offset + len(page_rows)
|
| 2906 |
check_publication()
|
| 2907 |
next_cursor = (_encode_query_cursor(session, query_digest, snapshot_digest, next_offset, publication)
|
|
|
|
| 3312 |
allowed, _ = _application_access(session, current)
|
| 3313 |
if not current or not allowed:
|
| 3314 |
raise applications.ApplicationError("application does not exist")
|
| 3315 |
+
stored = applications.load(session.runtime)["applications"].get(app_id)
|
| 3316 |
+
from tool_groups.apps import is_template_app, template_sealed
|
| 3317 |
+
if is_template_app(stored):
|
| 3318 |
+
# R35: a template App's definition is sealed over tools. Admins rename and share it in web.
|
| 3319 |
+
raise template_sealed(stored)
|
| 3320 |
proposed = payload.get("compatibleDatabases") if "compatibleDatabases" in payload else current
|
| 3321 |
proposed_allowed, records = _application_access(session, proposed, strict=True)
|
| 3322 |
return {"application": applications.mutate(
|
|
|
|
| 4110 |
ranges.append({"workbook_id": arguments.get("workbook_id"),
|
| 4111 |
"tab": arguments.get("tab") or arguments.get("worksheet"), "range": address})
|
| 4112 |
collect(args)
|
| 4113 |
+
inner = args.get("arguments") if name == "apps" and isinstance(args.get("arguments"), dict) else {}
|
| 4114 |
+
# C8: an App item change names its App and every item id it changes.
|
| 4115 |
+
for value in [inner.get("application"), *(inner.get("item_ids") or [])]:
|
| 4116 |
+
if value is not None and str(value) not in values:
|
| 4117 |
+
values.append(str(value))
|
| 4118 |
return {"ids": values[:100], "count": max(len(values), len(diff)), "ranges": ranges}
|
| 4119 |
|
| 4120 |
|
|
|
|
| 4680 |
if item["status"] != "preview":
|
| 4681 |
raise _OperationAlreadyClaimed()
|
| 4682 |
if item["payload"]["expiry"] <= time.time():
|
| 4683 |
+
raise err(409, "confirmation_expired", _expired_fix(item["payload"]))
|
| 4684 |
item["status"] = "claimed"
|
| 4685 |
item["retain_until"] = time.time() + _OPERATION_RETENTION_SECONDS
|
| 4686 |
return raw
|
|
|
|
| 4809 |
_STALE_FIX = "so repeat the original call for a new preview and confirm with its new confirmation_token."
|
| 4810 |
|
| 4811 |
|
| 4812 |
+
def _expired_fix(payload):
|
| 4813 |
+
"""Name the call that previews an expired change again and the confirm that follows it."""
|
| 4814 |
+
import tool_groups
|
| 4815 |
+
payload = payload if isinstance(payload, dict) else {}
|
| 4816 |
+
tool, arguments = payload.get("tool"), payload.get("arguments")
|
| 4817 |
+
address = "the original tool"
|
| 4818 |
+
if isinstance(tool, str) and isinstance(arguments, dict):
|
| 4819 |
+
try:
|
| 4820 |
+
group, grouped = tool_groups.legacy_target(tool, arguments)
|
| 4821 |
+
address = group if group == tool else group + " with action " + grouped["action"]
|
| 4822 |
+
except (KeyError, TypeError, ValueError):
|
| 4823 |
+
if tool in tool_groups.groups() and isinstance(arguments.get("action"), str):
|
| 4824 |
+
address = tool + " with action " + arguments["action"]
|
| 4825 |
+
return ("This confirmation token expired. Call " + address + " again with the same arguments for a new "
|
| 4826 |
+
"preview. Then call confirm_change with its new confirmation_token within "
|
| 4827 |
+
+ str(_CONFIRMATION_TTL_SECONDS // 60) + " minutes.")
|
| 4828 |
+
|
| 4829 |
+
|
| 4830 |
def _confirm_fix(args):
|
| 4831 |
"""Name what a confirm_change call got wrong and the shape that succeeds."""
|
| 4832 |
allowed = ("confirmation_token", "confirmation_phrase", "intent")
|
|
|
|
| 4859 |
if payload.get("environment") != _write_environment():
|
| 4860 |
raise err(409, "confirmation_stale", "The environment changed after the preview, " + _STALE_FIX)
|
| 4861 |
if int(payload.get("expiry") or 0) <= int(time.time()):
|
| 4862 |
+
raise err(409, "confirmation_expired", _expired_fix(payload))
|
|
|
|
| 4863 |
if payload.get("strong") and args.get("confirmation_phrase") != _CONFIRMATION_PHRASE:
|
| 4864 |
raise err(400, "confirmation_phrase_required",
|
| 4865 |
"Enter the exact stronger confirmation phrase shown in the preview.")
|
|
|
|
| 4985 |
_MCP_CODE_GROUPS = (
|
| 4986 |
("The arguments do not match this action. Read its guide and correct them.", """
|
| 4987 |
tool_arguments_invalid query_arguments_invalid query_aggregates_invalid query_aggregate_type
|
| 4988 |
+
assistant_bad_filter query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid
|
| 4989 |
bad_application bad_entries bad_value bad_values bad_request bad_recurrence invalid_field_contract
|
| 4990 |
external_edit_empty external_edit_invalid oauth_edit_empty oauth_edit_invalid
|
| 4991 |
"""),
|
|
|
|
| 5080 |
"confirmation_session_required": "Read this confirmation through its authenticated connection.",
|
| 5081 |
"link_selection_refused": "Choose a readable linked record that matches the field's selection condition.",
|
| 5082 |
"app_rule": "Correct the App rule named in the refusal before saving it.",
|
| 5083 |
+
"template_app_sealed": "A Loopable template App cannot be changed over tools. Use its own tools. An Administrator can rename or share it in Loopable.",
|
| 5084 |
+
"app_unavailable": "No App with this id is shared with this account. Call apps.list for the Apps it can open.",
|
| 5085 |
+
"app_items_unavailable": "This App has no items to list or change over tools. Open it in Loopable.",
|
| 5086 |
+
"app_items_forbidden": "This account cannot change this App's items. Ask its owner or an Administrator.",
|
| 5087 |
+
"app_item_status_invalid": "Choose a status this App allows. See allowed.",
|
| 5088 |
+
"app_item_unknown": "Choose item ids that apps.items returned for this App.",
|
| 5089 |
"audience_not_allowed": "A person must choose sharing with everyone on screen.",
|
| 5090 |
"not_owner": "You cannot administer this item's audience.",
|
| 5091 |
"unknown_people": "Choose active people in this workspace.",
|
|
|
|
| 5234 |
"applications", "view", "views", "from", "to", "source", "affected", "aggregates", "tools",
|
| 5235 |
"annotations", "serverInfo", "environment",
|
| 5236 |
})
|
| 5237 |
+
# A link descriptor names its target database key. It is an identifier, never prose (D-9307).
|
| 5238 |
+
_MCP_RESULT_DESCRIPTORS = frozenset({"link"})
|
| 5239 |
_MCP_AUTHORED_DESCRIPTIONS = frozenset({
|
| 5240 |
"fields", "file", "files", "template", "templates", "application", "applications", "view", "views",
|
| 5241 |
})
|
|
|
|
| 5262 |
result[key] = copy.deepcopy(child)
|
| 5263 |
elif key == "about":
|
| 5264 |
result[key] = copy.deepcopy(child)
|
| 5265 |
+
elif key in _MCP_RESULT_DESCRIPTORS and isinstance(child, dict):
|
| 5266 |
+
result[key] = copy.deepcopy(child)
|
| 5267 |
elif key in _MCP_RESULT_OPAQUE and not metadata:
|
| 5268 |
result[key] = copy.deepcopy(child)
|
| 5269 |
elif key in {"schema", "inputSchema"}:
|
|
|
|
| 6034 |
or audit.redact_intent(method) != method):
|
| 6035 |
method = "[redacted]"
|
| 6036 |
error = reply.get("error", {}).get("code")
|
| 6037 |
+
if error == -32601:
|
| 6038 |
+
# D-2667: a probe for a method the server does not offer is no refused query and crowds out history.
|
| 6039 |
+
return
|
| 6040 |
+
code = "request_invalid" if error else ""
|
| 6041 |
external_activity.record_protocol(session, method, code=code)
|
| 6042 |
except Exception:
|
| 6043 |
pass
|
|
|
|
| 6268 |
return reply
|
| 6269 |
|
| 6270 |
|
| 6271 |
+
#: Raw refusal codes whose MCP answer names the call that succeeds next (S79-T11).
|
| 6272 |
+
_MCP_NEXT_CALL = {"workbook_missing": WORKBOOK_MISSING_FIX, "assistant_unknown_database": UNKNOWN_DATABASE_FIX}
|
| 6273 |
+
|
| 6274 |
+
|
| 6275 |
+
def _mcp_next_call(code, message):
|
| 6276 |
+
"""Append the call that succeeds next for a refusal that names a missing item (S79-T11)."""
|
| 6277 |
+
text = str(message or "").rstrip()
|
| 6278 |
+
fix = _MCP_NEXT_CALL.get(code, "")
|
| 6279 |
+
if code == "assistant_unknown_database":
|
| 6280 |
+
# The read path's lowercase reason is replaced: the fix sentence names the missing key itself.
|
| 6281 |
+
return fix
|
| 6282 |
+
return text if not fix or fix in text else (text + " " + fix).strip()
|
| 6283 |
+
|
| 6284 |
+
|
| 6285 |
def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
| 6286 |
"""One HTTP request object; the stdio precedent's JSON-RPC and content envelopes."""
|
| 6287 |
protocol_error = _mcp_protocol_error
|
|
|
|
| 6433 |
wiki_log.record_build(session, kind=kind, object_id=arguments["arguments"]["path"],
|
| 6434 |
event_id="mcp-wiki:" + _json_digest({"path": arguments["arguments"]["path"], "version": data["version"]}),
|
| 6435 |
occurred_at=data["updated"], purpose=purpose, about=about, build_kind=kind)
|
| 6436 |
+
elif name == "query_database":
|
| 6437 |
+
with _external_image_base(base_url):
|
| 6438 |
+
data = _dispatch_external_tool(_tool_context(session), name, arguments)
|
| 6439 |
else:
|
| 6440 |
data = _dispatch_external_tool(_tool_context(session), name, arguments)
|
| 6441 |
if isinstance(data, dict) and data.get("refused"):
|
|
|
|
| 6463 |
import routes_sheet
|
| 6464 |
if exc.code == "tool_arguments_invalid" and name == "confirm_change":
|
| 6465 |
exc = sheets.SheetRefusal(exc.code, _confirm_fix(arguments))
|
| 6466 |
+
if exc.code == "tool_arguments_invalid" and _is_query_call(name, arguments):
|
| 6467 |
+
exc = sheets.SheetRefusal(exc.code, with_query_syntax(exc.message))
|
| 6468 |
+
if exc.code in _MCP_NEXT_CALL:
|
| 6469 |
+
exc = sheets.SheetRefusal(exc.code, _mcp_next_call(exc.code, exc.message))
|
| 6470 |
translated = routes_sheet._refuse(exc)
|
| 6471 |
info = translated.detail["error"]
|
| 6472 |
out = _tool_refusal(name, info["code"], info["message"], translated.status_code,
|
|
|
|
| 6491 |
message = info.get("message", "The app refused this operation.")
|
| 6492 |
if info.get("code") == "build_about_required" and isinstance(message, str):
|
| 6493 |
message += _mcp_about_fix(called)
|
| 6494 |
+
if info.get("code") in _MCP_NEXT_CALL:
|
| 6495 |
+
message = _mcp_next_call(info["code"], message)
|
| 6496 |
out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry)
|
| 6497 |
except (TypeError, ValueError, KeyError, AttributeError):
|
| 6498 |
out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
|
api/routes_keychain.py
CHANGED
|
@@ -122,7 +122,7 @@ SCOPE_KEY = "keychain_scopes"
|
|
| 122 |
#: (the resolver-side guard held), but a door that stores a credential the product then ignores,
|
| 123 |
#: which is its own kind of wrong. `routes_connectors.py::_tokens` imports this tuple rather than
|
| 124 |
#: keeping its own, so its `manage` field for the Amazon card corrects itself in the same change.
|
| 125 |
-
TENANT_WIDE_TYPES = ("odoo", "meta_ads", "amazon_sp")
|
| 126 |
|
| 127 |
|
| 128 |
#: ⭐⭐ W53-T29 / OWNER RULING R6 — WHICH SCOPES ONE SOURCE MAY EVER CARRY, SAID AS DATA.
|
|
|
|
| 122 |
#: (the resolver-side guard held), but a door that stores a credential the product then ignores,
|
| 123 |
#: which is its own kind of wrong. `routes_connectors.py::_tokens` imports this tuple rather than
|
| 124 |
#: keeping its own, so its `manage` field for the Amazon card corrects itself in the same change.
|
| 125 |
+
TENANT_WIDE_TYPES = ("odoo", "meta_ads", "amazon_sp", "amazon_ads")
|
| 126 |
|
| 127 |
|
| 128 |
#: ⭐⭐ W53-T29 / OWNER RULING R6 — WHICH SCOPES ONE SOURCE MAY EVER CARRY, SAID AS DATA.
|
api/routes_nav.py
CHANGED
|
@@ -531,12 +531,24 @@ def _granted_view_ids(session):
|
|
| 531 |
#: of the reserved words below is still a second connection, and its sibling must still be named.
|
| 532 |
#:
|
| 533 |
#: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
|
| 534 |
-
#: `dataPath.ts::
|
| 535 |
-
#:
|
| 536 |
#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
|
| 537 |
#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
|
| 538 |
_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
|
| 539 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 540 |
#: The registry modules that ARE the designated default connection's data rather than a namespaced
|
| 541 |
#: copy of it: `odoo_relational.namespaced_fields` says so in as many words ("`customer_link`
|
| 542 |
#: targets the compiled registry module `customer_data`, which holds the DEFAULT connection's
|
|
@@ -712,7 +724,7 @@ def nav(session: Session = Depends(require_session)):
|
|
| 712 |
continue
|
| 713 |
pages.append({"key": e["key"], "label": e["label"],
|
| 714 |
"source": e.get("source") or "Blank",
|
| 715 |
-
"sourceGroup": e.get("sourceGroup")
|
| 716 |
except Exception:
|
| 717 |
# ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT.
|
| 718 |
#
|
|
@@ -771,8 +783,8 @@ def nav(session: Session = Depends(require_session)):
|
|
| 771 |
_folder = _odoo_folder_label(key, _conn_labels, _conn_default)
|
| 772 |
if _folder:
|
| 773 |
page["sourceGroup"] = _folder
|
| 774 |
-
elif page.get("user_table") or source in
|
| 775 |
-
page.setdefault("sourceGroup",
|
| 776 |
# ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
|
| 777 |
#
|
| 778 |
# W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS
|
|
@@ -1498,17 +1510,19 @@ def create_nav_application(body: dict = Body(default=None),
|
|
| 1498 |
wizard_kinds = {row["kind"] for row in application_registry.template_contracts(session=session)}
|
| 1499 |
if template_kind not in wizard_kinds:
|
| 1500 |
raise err(400, "bad_application", "application template kind is not recognized")
|
| 1501 |
-
|
| 1502 |
-
|
|
|
|
|
|
|
| 1503 |
raise err(400, "bad_application", "sourceDatabase is required")
|
| 1504 |
-
fields = application_registry.chart_field_contracts(session, [source])[source]
|
| 1505 |
if not session.runtime.available():
|
| 1506 |
raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.")
|
| 1507 |
try:
|
| 1508 |
application = application_registry.create(
|
| 1509 |
session.runtime, session.tenant, session.uname, payload,
|
| 1510 |
-
field_contracts={source: fields if isinstance(fields, list) else []},
|
| 1511 |
-
allowed_databases={source})
|
| 1512 |
except PermissionError as exc:
|
| 1513 |
raise err(403, "forbidden", str(exc))
|
| 1514 |
except application_registry.ApplicationExistsError as exc:
|
|
|
|
| 531 |
#: of the reserved words below is still a second connection, and its sibling must still be named.
|
| 532 |
#:
|
| 533 |
#: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
|
| 534 |
+
#: `dataPath.ts::dataDatabaseOriginKey` folds `custom` / `loopable` / `blank` / `automation` /
|
| 535 |
+
#: empty into the one Custom group (S79-T60, key `CUSTOM_SOURCE_GROUP`). A connection someone named
|
| 536 |
#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
|
| 537 |
#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
|
| 538 |
_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
|
| 539 |
|
| 540 |
+
#: S79-T60 (QI27): the stable key of the native origin group. The client shows it as "Custom";
|
| 541 |
+
#: no logic compares that label. Must equal `dataPath.ts::CUSTOM_ORIGIN_KEY`.
|
| 542 |
+
CUSTOM_SOURCE_GROUP = "custom"
|
| 543 |
+
#: Native provenance tokens, the legacy "Loopable" from `user_tables` included, fold into the key.
|
| 544 |
+
_NATIVE_GROUP_TOKENS = frozenset({"", CUSTOM_SOURCE_GROUP, "loopable", "blank", "automation"})
|
| 545 |
+
|
| 546 |
+
|
| 547 |
+
def _native_group(group):
|
| 548 |
+
"""The `sourceGroup` to send: a native token becomes `CUSTOM_SOURCE_GROUP`, a connector keeps its name."""
|
| 549 |
+
text = str(group or "").strip()
|
| 550 |
+
return CUSTOM_SOURCE_GROUP if text.lower() in _NATIVE_GROUP_TOKENS else text
|
| 551 |
+
|
| 552 |
#: The registry modules that ARE the designated default connection's data rather than a namespaced
|
| 553 |
#: copy of it: `odoo_relational.namespaced_fields` says so in as many words ("`customer_link`
|
| 554 |
#: targets the compiled registry module `customer_data`, which holds the DEFAULT connection's
|
|
|
|
| 724 |
continue
|
| 725 |
pages.append({"key": e["key"], "label": e["label"],
|
| 726 |
"source": e.get("source") or "Blank",
|
| 727 |
+
"sourceGroup": _native_group(e.get("sourceGroup")), "chrome": "main"})
|
| 728 |
except Exception:
|
| 729 |
# ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT.
|
| 730 |
#
|
|
|
|
| 783 |
_folder = _odoo_folder_label(key, _conn_labels, _conn_default)
|
| 784 |
if _folder:
|
| 785 |
page["sourceGroup"] = _folder
|
| 786 |
+
elif page.get("user_table") or source.strip().lower() in _NATIVE_GROUP_TOKENS:
|
| 787 |
+
page.setdefault("sourceGroup", CUSTOM_SOURCE_GROUP)
|
| 788 |
# ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
|
| 789 |
#
|
| 790 |
# W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS
|
|
|
|
| 1510 |
wizard_kinds = {row["kind"] for row in application_registry.template_contracts(session=session)}
|
| 1511 |
if template_kind not in wizard_kinds:
|
| 1512 |
raise err(400, "bad_application", "application template kind is not recognized")
|
| 1513 |
+
import app_templates
|
| 1514 |
+
source = str(payload.get("sourceDatabase") or app_templates.source(template_kind) or "").strip()
|
| 1515 |
+
sourceless = template_kind in app_templates.BY_KIND and not app_templates.BY_KIND[template_kind]["sources"]
|
| 1516 |
+
if not source and not sourceless:
|
| 1517 |
raise err(400, "bad_application", "sourceDatabase is required")
|
| 1518 |
+
fields = application_registry.chart_field_contracts(session, [source])[source] if source else []
|
| 1519 |
if not session.runtime.available():
|
| 1520 |
raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.")
|
| 1521 |
try:
|
| 1522 |
application = application_registry.create(
|
| 1523 |
session.runtime, session.tenant, session.uname, payload,
|
| 1524 |
+
field_contracts={source: fields if isinstance(fields, list) else []} if source else {},
|
| 1525 |
+
allowed_databases={source} if source else set())
|
| 1526 |
except PermissionError as exc:
|
| 1527 |
raise err(403, "forbidden", str(exc))
|
| 1528 |
except application_registry.ApplicationExistsError as exc:
|
api/routes_odoo_attachments.py
CHANGED
|
@@ -1,28 +1,62 @@
|
|
| 1 |
"""Contract C5 (S78-T14): the delivery photo list and the photo bytes, behind the delivery read wall.
|
| 2 |
|
| 3 |
The work lives in `odoo_attachments.py`. This router decides who may ask and shapes the response.
|
|
|
|
| 4 |
Mounted by `main.py` through `_mount_router("routes_odoo_attachments")`.
|
| 5 |
"""
|
| 6 |
from fastapi import APIRouter, Depends, Path
|
| 7 |
from fastapi.responses import Response
|
| 8 |
|
| 9 |
import odoo_attachments
|
| 10 |
-
from deps import Session,
|
| 11 |
|
| 12 |
router = APIRouter(prefix="/api/v1")
|
| 13 |
|
| 14 |
_MAX_ID = 2 ** 31 - 1
|
| 15 |
|
| 16 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
@router.get("/odoo/deliveries/{picking_id}/photos")
|
| 18 |
-
def delivery_photos(picking_id: int = Path(..., ge=1, le=_MAX_ID),
|
|
|
|
| 19 |
"""`[{id, name, mimetype, url}]`: every photo attachment of one delivery."""
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
|
| 23 |
@router.get("/odoo/attachments/{attachment_id}")
|
| 24 |
-
def delivery_photo(attachment_id: int = Path(..., ge=1, le=_MAX_ID),
|
|
|
|
| 25 |
"""The photo bytes, streamed through and never stored."""
|
|
|
|
|
|
|
|
|
|
| 26 |
body, mimetype, _name = odoo_attachments.read_photo(session, attachment_id)
|
| 27 |
return Response(content=body, media_type=mimetype, headers={
|
| 28 |
"Cache-Control": "private, max-age=300",
|
|
|
|
| 1 |
"""Contract C5 (S78-T14): the delivery photo list and the photo bytes, behind the delivery read wall.
|
| 2 |
|
| 3 |
The work lives in `odoo_attachments.py`. This router decides who may ask and shapes the response.
|
| 4 |
+
Both routes admit the cookie or an external bearer, walled to the key's database scope (D-9926).
|
| 5 |
Mounted by `main.py` through `_mount_router("routes_odoo_attachments")`.
|
| 6 |
"""
|
| 7 |
from fastapi import APIRouter, Depends, Path
|
| 8 |
from fastapi.responses import Response
|
| 9 |
|
| 10 |
import odoo_attachments
|
| 11 |
+
from deps import Session, is_bearer, require_session_or_bearer
|
| 12 |
|
| 13 |
router = APIRouter(prefix="/api/v1")
|
| 14 |
|
| 15 |
_MAX_ID = 2 ** 31 - 1
|
| 16 |
|
| 17 |
|
| 18 |
+
def _key_allowed(session):
|
| 19 |
+
"""The delivery key filter for a bearer: its database scope. None for a cookie session."""
|
| 20 |
+
if not is_bearer(session):
|
| 21 |
+
return None
|
| 22 |
+
from routes_external import _assert_database_scope
|
| 23 |
+
|
| 24 |
+
def allowed(key):
|
| 25 |
+
_assert_database_scope(session, key)
|
| 26 |
+
return allowed
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _bearer_photo_wall(session, attachment_id, key_allowed):
|
| 30 |
+
"""Check a bearer's database scope on the photo's delivery before `read_photo` reads bytes."""
|
| 31 |
+
rows = odoo_attachments._search_read(session, odoo_attachments.ATTACHMENT_MODEL,
|
| 32 |
+
[("id", "=", int(attachment_id))], ["res_model", "res_id"])
|
| 33 |
+
row = (rows or [None])[0]
|
| 34 |
+
res_id = row.get("res_id") if row else None
|
| 35 |
+
# A malformed attachment falls through to read_photo, which answers its own 404.
|
| 36 |
+
if (row and row.get("res_model") == odoo_attachments.PICKING_MODEL
|
| 37 |
+
and type(res_id) is int and res_id > 0):
|
| 38 |
+
odoo_attachments.require_delivery_read(session, res_id, key_allowed)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
@router.get("/odoo/deliveries/{picking_id}/photos")
|
| 42 |
+
def delivery_photos(picking_id: int = Path(..., ge=1, le=_MAX_ID),
|
| 43 |
+
session: Session = Depends(require_session_or_bearer)):
|
| 44 |
"""`[{id, name, mimetype, url}]`: every photo attachment of one delivery."""
|
| 45 |
+
key_allowed = _key_allowed(session)
|
| 46 |
+
if key_allowed is None:
|
| 47 |
+
return odoo_attachments.list_photos(session, picking_id)
|
| 48 |
+
from loopy_links import public_base
|
| 49 |
+
return odoo_attachments.list_photos(session, picking_id, key_allowed=key_allowed,
|
| 50 |
+
base=public_base() or "")
|
| 51 |
|
| 52 |
|
| 53 |
@router.get("/odoo/attachments/{attachment_id}")
|
| 54 |
+
def delivery_photo(attachment_id: int = Path(..., ge=1, le=_MAX_ID),
|
| 55 |
+
session: Session = Depends(require_session_or_bearer)):
|
| 56 |
"""The photo bytes, streamed through and never stored."""
|
| 57 |
+
key_allowed = _key_allowed(session)
|
| 58 |
+
if key_allowed is not None:
|
| 59 |
+
_bearer_photo_wall(session, attachment_id, key_allowed)
|
| 60 |
body, mimetype, _name = odoo_attachments.read_photo(session, attachment_id)
|
| 61 |
return Response(content=body, media_type=mimetype, headers={
|
| 62 |
"Cache-Control": "private, max-age=300",
|
api/routes_opportunity_finder.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Opportunity finder App routes (C3): rising terms, one item, status, a term's trend, settings, validate."""
|
| 2 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 3 |
+
|
| 4 |
+
from deps import Session, err, require_session
|
| 5 |
+
|
| 6 |
+
router = APIRouter(prefix="/api/v1/app-kinds/opportunity_finder", tags=["opportunity_finder"])
|
| 7 |
+
SOURCE = "ut_amzn_brand_analytics"
|
| 8 |
+
TEMPLATE_KIND = "opportunity_finder"
|
| 9 |
+
#: Weeks in a list row's sparkline. Each week is one store read per /screen call.
|
| 10 |
+
SPARK_WEEKS = 8
|
| 11 |
+
SETTING_NAMES = {"min_rise_pct": "min_rise_pct", "minRisePct": "min_rise_pct", "departments": "departments"}
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _module():
|
| 15 |
+
from modules import opportunity_finder
|
| 16 |
+
return opportunity_finder
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 20 |
+
"""Reads need read access to the Amazon brand analytics database."""
|
| 21 |
+
from core import perm_scope
|
| 22 |
+
if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
|
| 23 |
+
raise err(403, "opportunity_finder_forbidden", "Your account cannot read Amazon brand analytics.")
|
| 24 |
+
return session
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _writer(session: Session = Depends(_reader)) -> Session:
|
| 28 |
+
"""Writes also need a role that may edit."""
|
| 29 |
+
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 30 |
+
raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.")
|
| 31 |
+
return session
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _tenant(session):
|
| 35 |
+
return str(getattr(session.runtime, "key", "") or session.tenant)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _user(session):
|
| 39 |
+
return {**session.user, "username": session.uname}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _call(fn, *args, **kwargs):
|
| 43 |
+
try:
|
| 44 |
+
return fn(*args, **kwargs)
|
| 45 |
+
except PermissionError as exc:
|
| 46 |
+
raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.") from exc
|
| 47 |
+
except KeyError as exc:
|
| 48 |
+
raise err(404, "opportunity_finder_not_found", "That search term is not in the opportunity finder.") from exc
|
| 49 |
+
except ValueError as exc:
|
| 50 |
+
raise err(400, "opportunity_finder_invalid", str(exc)) from exc
|
| 51 |
+
except Exception as exc: # noqa: BLE001
|
| 52 |
+
raise err(503, "opportunity_finder_unavailable",
|
| 53 |
+
"The opportunity finder could not be read. Try again shortly.") from exc
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _app(session, app_id):
|
| 57 |
+
"""The Opportunity finder App record this request names, readable by the caller (404 otherwise)."""
|
| 58 |
+
import application_registry
|
| 59 |
+
try:
|
| 60 |
+
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 61 |
+
except Exception as exc: # noqa: BLE001
|
| 62 |
+
raise err(503, "opportunity_finder_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 63 |
+
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 64 |
+
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 65 |
+
raise err(404, "opportunity_finder_app_not_found", "That Opportunity finder App does not exist.")
|
| 66 |
+
return app
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _may_edit(session, app):
|
| 70 |
+
import application_registry
|
| 71 |
+
role = str(session.user.get("role") or "user").strip().lower()
|
| 72 |
+
return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
|
| 76 |
+
"""C3: a read names the App, and the caller must be able to read that App."""
|
| 77 |
+
_app(session, app)
|
| 78 |
+
return session
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
|
| 82 |
+
"""C3: a write names the App, and only an editor of that App may change it."""
|
| 83 |
+
if not _may_edit(session, _app(session, app)):
|
| 84 |
+
raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.")
|
| 85 |
+
return session
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def _with_spark(of, st, items):
|
| 89 |
+
"""Each page item gains `spark`: its rank in the newest SPARK_WEEKS held weeks, oldest first."""
|
| 90 |
+
weeks = of.weeks(None, st=st)[-SPARK_WEEKS:] if SPARK_WEEKS > 0 else []
|
| 91 |
+
held = [(week, of.week_point(None, week, st=st).get("terms") or {}) for week in weeks]
|
| 92 |
+
for item in items:
|
| 93 |
+
term = of.norm(item.get("term"))
|
| 94 |
+
item["spark"] = [{"week": week, "rank": terms[term]} for week, terms in held if term in terms]
|
| 95 |
+
return items
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
@router.get("/screen")
|
| 99 |
+
def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 100 |
+
cursor: str | None = Query(None), session: Session = Depends(_app_reader)):
|
| 101 |
+
"""The rising list for the newest week, with counts, departments, settings and any named cause."""
|
| 102 |
+
of = _module()
|
| 103 |
+
record = _app(session, app)
|
| 104 |
+
st = session.runtime
|
| 105 |
+
listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st)
|
| 106 |
+
listed["items"] = _call(_with_spark, of, st, listed["items"])
|
| 107 |
+
return {"canEdit": _may_edit(session, record), "week": listed["week"], "priorWeek": listed["priorWeek"],
|
| 108 |
+
"weeks": listed["weeks"], "departments": listed["departments"], "counts": listed["counts"],
|
| 109 |
+
"total": listed["total"], "items": listed["items"], "cursor": listed["cursor"],
|
| 110 |
+
"problems": listed["problems"], "statuses": list(of.STATUSES),
|
| 111 |
+
"settings": _call(of.settings, _tenant(session), st=st)}
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
@router.get("/items/{item_id}")
|
| 115 |
+
def item_detail(item_id: str, session: Session = Depends(_app_reader)):
|
| 116 |
+
of = _module()
|
| 117 |
+
return _call(of.item_detail, _tenant(session), session.uname, item_id, st=session.runtime)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
@router.patch("/items/{item_id}")
|
| 121 |
+
def update_item(item_id: str, status: str = Body(...), note: str | None = Body(None),
|
| 122 |
+
owner: str | None = Body(None), due: str | None = Body(None),
|
| 123 |
+
session: Session = Depends(_app_writer)):
|
| 124 |
+
of = _module()
|
| 125 |
+
return _call(of.set_status, _tenant(session), _user(session), item_id, status, note=note, owner=owner,
|
| 126 |
+
due=due, st=session.runtime)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
@router.get("/terms/{item_id}")
|
| 130 |
+
def term_trend(item_id: str, session: Session = Depends(_app_reader)):
|
| 131 |
+
"""A term's weekly rank trend and its newest top 3 clicked ASINs."""
|
| 132 |
+
of = _module()
|
| 133 |
+
return _call(of.term_detail, _tenant(session), item_id, st=session.runtime)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
@router.get("/settings")
|
| 137 |
+
def read_settings(session: Session = Depends(_app_reader)):
|
| 138 |
+
of = _module()
|
| 139 |
+
return _call(of.settings, _tenant(session), st=session.runtime)
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
@router.put("/settings")
|
| 143 |
+
def write_settings(body: dict = Body(...), session: Session = Depends(_app_writer)):
|
| 144 |
+
of = _module()
|
| 145 |
+
body = dict(body or {})
|
| 146 |
+
owner = body.pop("defaultOwner", None)
|
| 147 |
+
unknown = sorted(k for k in body if k not in SETTING_NAMES)
|
| 148 |
+
if unknown:
|
| 149 |
+
raise err(400, "opportunity_finder_invalid", "Unknown setting: " + ", ".join(unknown) + ".")
|
| 150 |
+
values = {SETTING_NAMES[k]: v for k, v in body.items()}
|
| 151 |
+
return _call(of.set_settings, _tenant(session), _user(session), default_owner=owner, st=session.runtime,
|
| 152 |
+
**values)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
@router.get("/validate")
|
| 156 |
+
def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
|
| 157 |
+
session: Session = Depends(_app_reader)):
|
| 158 |
+
of = _module()
|
| 159 |
+
return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime)
|
api/routes_payouts.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Payouts App routes (C3): the overview, settlements with their waterfall and tie, owned items and settings.
|
| 2 |
+
|
| 3 |
+
Every route names the App (`?app=`) and checks the caller may read it and the settlements database.
|
| 4 |
+
Writes also need edit rights on that App and a role other than viewer. `main.py` does not mount this
|
| 5 |
+
router: `app_routes.mount_app_routers` does, from the App contract.
|
| 6 |
+
"""
|
| 7 |
+
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
| 8 |
+
|
| 9 |
+
from deps import Session, err, require_session
|
| 10 |
+
|
| 11 |
+
router = APIRouter(prefix="/api/v1/app-kinds/payouts", tags=["payouts"])
|
| 12 |
+
SOURCE = "ut_amzn_settlements"
|
| 13 |
+
TEMPLATE_KIND = "payouts"
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _module():
|
| 17 |
+
from modules import payouts
|
| 18 |
+
return payouts
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 22 |
+
"""Reads need read access to the Amazon settlements database."""
|
| 23 |
+
from core import perm_scope
|
| 24 |
+
if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
|
| 25 |
+
raise err(403, "payouts_forbidden", "Your account cannot read Amazon settlements.")
|
| 26 |
+
return session
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _writer(session: Session = Depends(_reader)) -> Session:
|
| 30 |
+
"""Writes also need a role that may edit."""
|
| 31 |
+
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 32 |
+
raise err(403, "payouts_read_only", "Your account can view Payouts but not change it.")
|
| 33 |
+
return session
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _tenant(session):
|
| 37 |
+
return str(getattr(session.runtime, "key", "") or session.tenant)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _app(session, app_id):
|
| 41 |
+
"""The Payouts App record this request names, readable by the caller (404 otherwise)."""
|
| 42 |
+
import application_registry
|
| 43 |
+
try:
|
| 44 |
+
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 45 |
+
except Exception as exc: # noqa: BLE001
|
| 46 |
+
raise err(503, "payouts_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 47 |
+
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 48 |
+
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 49 |
+
raise err(404, "payouts_app_not_found", "That Payouts App does not exist.")
|
| 50 |
+
return app
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _may_edit(session, app):
|
| 54 |
+
import application_registry
|
| 55 |
+
role = str(session.user.get("role") or "user").strip().lower()
|
| 56 |
+
return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
| 60 |
+
"""A read names the App the caller can read (404 first, so no App is confirmed), then the database."""
|
| 61 |
+
_app(session, app)
|
| 62 |
+
return _reader(session)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
| 66 |
+
"""A write names the App, and only an editor of that App may change it."""
|
| 67 |
+
found = _app(session, app)
|
| 68 |
+
_writer(_reader(session))
|
| 69 |
+
if not _may_edit(session, found):
|
| 70 |
+
raise err(403, "payouts_read_only", "You can view this App but not change it.")
|
| 71 |
+
return session
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _call(fn, *args, missing="payouts_not_found", missing_message="That settlement does not exist.", **kwargs):
|
| 75 |
+
pay = _module()
|
| 76 |
+
try:
|
| 77 |
+
return fn(*args, **kwargs)
|
| 78 |
+
except PermissionError as exc:
|
| 79 |
+
raise err(403, "payouts_read_only", str(exc)) from exc
|
| 80 |
+
except pay.SourceUnavailable as exc:
|
| 81 |
+
raise err(503, "payouts_no_source", str(exc)) from exc
|
| 82 |
+
except ValueError as exc:
|
| 83 |
+
raise err(400, "payouts_invalid", str(exc)) from exc
|
| 84 |
+
except KeyError as exc:
|
| 85 |
+
raise err(404, missing, missing_message) from exc
|
| 86 |
+
except Exception as exc: # noqa: BLE001
|
| 87 |
+
raise err(503, "payouts_unavailable", "Payouts could not be read. Try again shortly.") from exc
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
@router.get("/overview")
|
| 91 |
+
def read_overview(session: Session = Depends(_app_reader)):
|
| 92 |
+
pay = _module()
|
| 93 |
+
return _call(pay.overview, _tenant(session), st=session.runtime)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
@router.get("/settlements")
|
| 97 |
+
def list_settlements(cursor: str | None = Query(None), limit: int = Query(200, ge=1),
|
| 98 |
+
currency: str | None = Query(None), start: str | None = Query(None),
|
| 99 |
+
end: str | None = Query(None), session: Session = Depends(_app_reader)):
|
| 100 |
+
pay = _module()
|
| 101 |
+
return _call(pay.settlements, _tenant(session), st=session.runtime, currency=currency, start=start, end=end,
|
| 102 |
+
cursor=cursor, limit=limit)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
@router.get("/settlements/{settlement_id}")
|
| 106 |
+
def read_settlement(settlement_id: str, session: Session = Depends(_app_reader)):
|
| 107 |
+
pay = _module()
|
| 108 |
+
return _call(pay.settlement, _tenant(session), settlement_id, st=session.runtime)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
@router.get("/items")
|
| 112 |
+
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 113 |
+
cursor: str | None = Query(None), limit: int = Query(200, ge=1),
|
| 114 |
+
session: Session = Depends(_app_reader)):
|
| 115 |
+
pay = _module()
|
| 116 |
+
listed = _call(pay.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit,
|
| 117 |
+
st=session.runtime)
|
| 118 |
+
return {**listed, "statuses": list(pay.STATUSES), "canEdit": _may_edit(session, _app(session, app))}
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
@router.patch("/items/{item_id}")
|
| 122 |
+
def update_item(item_id: str, status: str = Body(..., embed=True), note: str | None = Body(None, embed=True),
|
| 123 |
+
session: Session = Depends(_app_writer)):
|
| 124 |
+
pay = _module()
|
| 125 |
+
# History records the account name, never a display name.
|
| 126 |
+
actor = {**session.user, "username": session.uname}
|
| 127 |
+
return _call(pay.set_status, _tenant(session), actor, item_id, status, note=note, st=session.runtime,
|
| 128 |
+
missing="payouts_item_not_found", missing_message="That item does not exist.")
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
# --------------------------------------------------------------------------- screen (S79-T27)
|
| 132 |
+
|
| 133 |
+
#: The module's refusal for a workspace with no stored settlements; the screen reads it as empty.
|
| 134 |
+
EMPTY_SOURCE = "No Amazon settlements are stored"
|
| 135 |
+
SCREEN_PAGE = 50
|
| 136 |
+
LIST_FIELDS = ("id", "title", "currency", "start", "end", "depositDate", "amazonTotal", "linesTotal", "gap",
|
| 137 |
+
"reconciled")
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _empty(exc):
|
| 141 |
+
detail = getattr(exc, "detail", None)
|
| 142 |
+
error = detail.get("error") if isinstance(detail, dict) else None
|
| 143 |
+
return (isinstance(error, dict) and error.get("code") == "payouts_no_source"
|
| 144 |
+
and str(error.get("message") or "").startswith(EMPTY_SOURCE))
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _offset(cursor):
|
| 148 |
+
try:
|
| 149 |
+
offset = int(cursor or 0)
|
| 150 |
+
except (TypeError, ValueError):
|
| 151 |
+
offset = -1
|
| 152 |
+
if offset < 0:
|
| 153 |
+
raise err(400, "payouts_invalid", "The cursor is not valid. Start again from the first page.")
|
| 154 |
+
return offset
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _last_deposits(rows, as_of):
|
| 158 |
+
"""The newest deposit on or before `as_of` and its change against the one before in its currency."""
|
| 159 |
+
paid = sorted((r for r in rows if r.get("depositDate") and r["depositDate"] <= as_of
|
| 160 |
+
and r.get("amazonTotal") is not None), key=lambda r: (r["depositDate"], r["id"]), reverse=True)
|
| 161 |
+
if not paid:
|
| 162 |
+
return None
|
| 163 |
+
last = paid[0]
|
| 164 |
+
prev = next((r for r in paid[1:] if r["currency"] == last["currency"]), None)
|
| 165 |
+
return {"settlementId": last["id"], "date": last["depositDate"], "amount": last["amazonTotal"],
|
| 166 |
+
"currency": last["currency"], "previousAmount": None if prev is None else prev["amazonTotal"],
|
| 167 |
+
"previousDate": None if prev is None else prev["depositDate"],
|
| 168 |
+
"change": None if prev is None else round(last["amazonTotal"] - prev["amazonTotal"], 2)}
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
@router.get("/screen")
|
| 172 |
+
def read_screen(app: str = Query(..., min_length=1), currency: str | None = Query(None),
|
| 173 |
+
view: str = Query("all", pattern="^(all|unreconciled)$"), cursor: str | None = Query(None),
|
| 174 |
+
limit: int = Query(SCREEN_PAGE, ge=1), session: Session = Depends(_app_reader)):
|
| 175 |
+
"""The App screen in one read: the KPI strip, tab counts and one page of settlements, newest first. No cap."""
|
| 176 |
+
pay = _module()
|
| 177 |
+
offset = _offset(cursor)
|
| 178 |
+
tenant, st = _tenant(session), session.runtime
|
| 179 |
+
can_edit = _may_edit(session, _app(session, app))
|
| 180 |
+
base = {"canEdit": can_edit, "statuses": list(pay.STATUSES), "view": view, "currency": currency or None}
|
| 181 |
+
try:
|
| 182 |
+
over = _call(pay.overview, tenant, st=st)
|
| 183 |
+
except HTTPException as exc:
|
| 184 |
+
if not _empty(exc):
|
| 185 |
+
raise
|
| 186 |
+
return {**base, "empty": True, "currencies": [], "counts": {"all": 0, "unreconciled": 0},
|
| 187 |
+
"settlements": [], "cursor": None, "kpis": {}}
|
| 188 |
+
listed = _call(pay.settlements, tenant, st=st, currency=currency or None, limit=None)["settlements"]
|
| 189 |
+
counts = {"all": len(listed), "unreconciled": sum(1 for r in listed if not r["reconciled"])}
|
| 190 |
+
rows = listed if view == "all" else [r for r in listed if not r["reconciled"]]
|
| 191 |
+
page = rows[offset:offset + limit]
|
| 192 |
+
opened = over.get("openSettlements") or []
|
| 193 |
+
open_now = next((o for o in opened if currency and o.get("currency") == currency.strip().upper()),
|
| 194 |
+
opened[0] if opened else None)
|
| 195 |
+
return {**base, "empty": not over.get("settlementCount"), "asOf": over.get("asOf"),
|
| 196 |
+
"currencies": over.get("currencies") or [], "economicsCurrency": over.get("economicsCurrency"),
|
| 197 |
+
"counts": counts, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page],
|
| 198 |
+
"cursor": str(offset + limit) if offset + limit < len(rows) else None,
|
| 199 |
+
"kpis": {"nextDeposit": over.get("nextDeposit"), "openSettlement": open_now,
|
| 200 |
+
"lastDeposit": _last_deposits(listed, over.get("asOf") or ""),
|
| 201 |
+
"unreconciled": counts["unreconciled"], "settlements": counts["all"]}}
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
@router.get("/screen/settlements/{settlement_id}")
|
| 205 |
+
def read_screen_settlement(settlement_id: str, app: str = Query(..., min_length=1),
|
| 206 |
+
session: Session = Depends(_app_reader)):
|
| 207 |
+
"""One settlement for the drawer: its waterfall, the tie line and, when unreconciled, its owned item."""
|
| 208 |
+
pay = _module()
|
| 209 |
+
tenant, st = _tenant(session), session.runtime
|
| 210 |
+
can_edit = _may_edit(session, _app(session, app))
|
| 211 |
+
detail = _call(pay.settlement, tenant, settlement_id, st=st)
|
| 212 |
+
item = None
|
| 213 |
+
if not detail.get("reconciled"):
|
| 214 |
+
held = _call(pay.items, tenant, session.user, st=st, limit=None)["items"]
|
| 215 |
+
item = next((i for i in held if str(i.get("id")) == str(settlement_id)), None)
|
| 216 |
+
return {"canEdit": can_edit, "statuses": list(pay.STATUSES), "settlement": detail, "item": item}
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
@router.get("/settings")
|
| 220 |
+
def read_settings(session: Session = Depends(_app_reader)):
|
| 221 |
+
pay = _module()
|
| 222 |
+
return _call(pay.settings, _tenant(session), st=session.runtime)
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
@router.patch("/settings")
|
| 226 |
+
def write_settings(defaultOwner: str | None = Body(None, embed=True), session: Session = Depends(_app_writer)):
|
| 227 |
+
pay = _module()
|
| 228 |
+
return _call(pay.set_settings, _tenant(session), default_owner=defaultOwner, st=session.runtime)
|
api/routes_query.py
CHANGED
|
@@ -1324,7 +1324,9 @@ def _request_model(messages, tools, requested, session, chosen=None, *, before_a
|
|
| 1324 |
turn_audit.requested(turn.get("handle"), active["name"], active["model"])
|
| 1325 |
try:
|
| 1326 |
response = provider_stream.post(request, provider=active, post=requests.post)
|
|
|
|
| 1327 |
except Exception:
|
|
|
|
| 1328 |
if priced:
|
| 1329 |
after_attempt(ticket, None)
|
| 1330 |
raise
|
|
|
|
| 1324 |
turn_audit.requested(turn.get("handle"), active["name"], active["model"])
|
| 1325 |
try:
|
| 1326 |
response = provider_stream.post(request, provider=active, post=requests.post)
|
| 1327 |
+
prov.llm_breaker_record(active["name"], response.status_code)
|
| 1328 |
except Exception:
|
| 1329 |
+
prov.llm_breaker_record(active["name"], 0)
|
| 1330 |
if priced:
|
| 1331 |
after_attempt(ticket, None)
|
| 1332 |
raise
|
api/routes_ranking_diagnostic.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Ranking diagnostic App routes (C3): drop items, one item's cause card, status, ASIN picker, trend, settings, validate."""
|
| 2 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 3 |
+
|
| 4 |
+
from deps import Session, err, require_session
|
| 5 |
+
|
| 6 |
+
router = APIRouter(prefix="/api/v1/app-kinds/ranking_diagnostic", tags=["ranking_diagnostic"])
|
| 7 |
+
SOURCE = "ut_amzn_pricing"
|
| 8 |
+
TEMPLATE_KIND = "ranking_diagnostic"
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _module():
|
| 12 |
+
from modules import ranking_diagnostic
|
| 13 |
+
return ranking_diagnostic
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 17 |
+
"""Reads need read access to the Amazon Sales and Traffic database."""
|
| 18 |
+
from core import perm_scope
|
| 19 |
+
if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
|
| 20 |
+
raise err(403, "ranking_diagnostic_forbidden", "Your account cannot read Amazon sales and traffic.")
|
| 21 |
+
return session
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _writer(session: Session = Depends(_reader)) -> Session:
|
| 25 |
+
"""Writes also need a role that may edit."""
|
| 26 |
+
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 27 |
+
raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.")
|
| 28 |
+
return session
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _tenant(session):
|
| 32 |
+
return str(getattr(session.runtime, "key", "") or session.tenant)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _user(session):
|
| 36 |
+
return {**session.user, "username": session.uname}
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _call(fn, *args, **kwargs):
|
| 40 |
+
try:
|
| 41 |
+
return fn(*args, **kwargs)
|
| 42 |
+
except PermissionError as exc:
|
| 43 |
+
raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.") from exc
|
| 44 |
+
except ValueError as exc:
|
| 45 |
+
raise err(400, "ranking_diagnostic_invalid", str(exc)) from exc
|
| 46 |
+
except KeyError as exc:
|
| 47 |
+
raise err(404, "ranking_diagnostic_not_found", "That item or ASIN is not in the history.") from exc
|
| 48 |
+
except Exception as exc: # noqa: BLE001
|
| 49 |
+
raise err(503, "ranking_diagnostic_unavailable",
|
| 50 |
+
"The ranking diagnostic could not be read. Try again shortly.") from exc
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _app(session, app_id):
|
| 54 |
+
"""The Ranking diagnostic App record this request names, readable by the caller (404 otherwise)."""
|
| 55 |
+
import application_registry
|
| 56 |
+
try:
|
| 57 |
+
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 58 |
+
except Exception as exc: # noqa: BLE001
|
| 59 |
+
raise err(503, "ranking_diagnostic_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 60 |
+
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 61 |
+
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 62 |
+
raise err(404, "ranking_diagnostic_app_not_found", "That Ranking diagnostic App does not exist.")
|
| 63 |
+
return app
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _may_edit(session, app):
|
| 67 |
+
import application_registry
|
| 68 |
+
role = str(session.user.get("role") or "user").strip().lower()
|
| 69 |
+
return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
|
| 73 |
+
"""C3: a read names the App, and the caller must be able to read that App."""
|
| 74 |
+
_app(session, app)
|
| 75 |
+
return session
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
|
| 79 |
+
"""C3: a write names the App, and only an editor of that App may change it."""
|
| 80 |
+
if not _may_edit(session, _app(session, app)):
|
| 81 |
+
raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.")
|
| 82 |
+
return session
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@router.get("/screen")
|
| 86 |
+
def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 87 |
+
week: str | None = Query(None), cursor: str | None = Query(None),
|
| 88 |
+
session: Session = Depends(_app_reader)):
|
| 89 |
+
"""The week's drop list (the newest week by default), with counts, problems and settings."""
|
| 90 |
+
rd = _module()
|
| 91 |
+
record = _app(session, app)
|
| 92 |
+
st = session.runtime
|
| 93 |
+
held = _call(rd.weeks, _tenant(session), st=st)
|
| 94 |
+
listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor,
|
| 95 |
+
week=week or (held[-1] if held else None), st=st)
|
| 96 |
+
return {"canEdit": _may_edit(session, record), "week": listed["week"], "weeks": listed["weeks"],
|
| 97 |
+
"counts": listed["counts"], "items": listed["items"], "cursor": listed["cursor"],
|
| 98 |
+
"problems": listed["problems"], "statuses": list(rd.STATUSES),
|
| 99 |
+
"settings": _call(rd.settings, _tenant(session), st=st)}
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
@router.get("/items/{item_id}")
|
| 103 |
+
def item_detail(item_id: str, session: Session = Depends(_app_reader)):
|
| 104 |
+
rd = _module()
|
| 105 |
+
return _call(rd.item_detail, _tenant(session), session.uname, item_id, st=session.runtime)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@router.patch("/items/{item_id}")
|
| 109 |
+
def update_item(item_id: str, status: str = Body(...), note: str | None = Body(None),
|
| 110 |
+
owner: str | None = Body(None), due: str | None = Body(None),
|
| 111 |
+
session: Session = Depends(_app_writer)):
|
| 112 |
+
rd = _module()
|
| 113 |
+
return _call(rd.set_status, _tenant(session), _user(session), item_id, status, note=note, owner=owner,
|
| 114 |
+
due=due, st=session.runtime)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
@router.get("/asins")
|
| 118 |
+
def asin_picker(week: str | None = Query(None), cursor: str | None = Query(None),
|
| 119 |
+
session: Session = Depends(_app_reader)):
|
| 120 |
+
"""The ASIN picker page. A flagged ASIN carries its cause label and item id for the cause card."""
|
| 121 |
+
rd = _module()
|
| 122 |
+
page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor, st=session.runtime)
|
| 123 |
+
labels = {key: label for key, label, _fix in (*rd.CAUSES, rd.UNEXPLAINED)}
|
| 124 |
+
for row in page["asins"]:
|
| 125 |
+
flagged = bool(row.get("cause") and page["week"])
|
| 126 |
+
row["causeLabel"] = labels.get(row["cause"]) if flagged else None
|
| 127 |
+
row["itemId"] = rd.item_id(page["week"], row["asin"]) if flagged else None
|
| 128 |
+
return page
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
@router.get("/trend/{asin}")
|
| 132 |
+
def asin_trend(asin: str, session: Session = Depends(_app_reader)):
|
| 133 |
+
rd = _module()
|
| 134 |
+
return _call(rd.trend, _tenant(session), asin, st=session.runtime)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
@router.get("/settings")
|
| 138 |
+
def read_settings(session: Session = Depends(_app_reader)):
|
| 139 |
+
rd = _module()
|
| 140 |
+
return _call(rd.settings, _tenant(session), st=session.runtime)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
@router.put("/settings")
|
| 144 |
+
def write_settings(body: dict = Body(...), session: Session = Depends(_app_writer)):
|
| 145 |
+
rd = _module()
|
| 146 |
+
body = dict(body or {})
|
| 147 |
+
owner = body.pop("defaultOwner", None)
|
| 148 |
+
unknown = sorted(k for k in body if k not in rd.DEFAULTS)
|
| 149 |
+
if unknown:
|
| 150 |
+
raise err(400, "ranking_diagnostic_invalid", "Unknown setting: " + ", ".join(unknown) + ".")
|
| 151 |
+
return _call(rd.set_settings, _tenant(session), _user(session), default_owner=owner, st=session.runtime, **body)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
@router.get("/validate")
|
| 155 |
+
def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
|
| 156 |
+
session: Session = Depends(_app_reader)):
|
| 157 |
+
rd = _module()
|
| 158 |
+
return _call(rd.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime)
|
api/routes_reimbursements.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reimbursements App routes (C3): claim items, one item with its case text, status, paid rows, validate."""
|
| 2 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 3 |
+
|
| 4 |
+
from deps import Session, err, require_session
|
| 5 |
+
|
| 6 |
+
router = APIRouter(prefix="/api/v1/app-kinds/reimbursements", tags=["reimbursements"])
|
| 7 |
+
SOURCE = "ut_amzn_reimbursements"
|
| 8 |
+
TEMPLATE_KIND = "reimbursements"
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _module():
|
| 12 |
+
from modules import reimbursements
|
| 13 |
+
return reimbursements
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 17 |
+
"""Reads need read access to the Amazon reimbursements database."""
|
| 18 |
+
from core import perm_scope
|
| 19 |
+
if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
|
| 20 |
+
raise err(403, "reimbursements_forbidden", "Your account cannot read Amazon reimbursements.")
|
| 21 |
+
return session
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _writer(session: Session = Depends(_reader)) -> Session:
|
| 25 |
+
"""Writes also need a role that may edit."""
|
| 26 |
+
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 27 |
+
raise err(403, "reimbursements_read_only", "Your account can view reimbursements but not change them.")
|
| 28 |
+
return session
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _tenant(session):
|
| 32 |
+
return str(getattr(session.runtime, "key", "") or session.tenant)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _user(session):
|
| 36 |
+
return {**session.user, "username": session.uname}
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _call(fn, *args, **kwargs):
|
| 40 |
+
try:
|
| 41 |
+
return fn(*args, **kwargs)
|
| 42 |
+
except PermissionError as exc:
|
| 43 |
+
raise err(403, "reimbursements_read_only", "You can view this App but not change it.") from exc
|
| 44 |
+
except ValueError as exc:
|
| 45 |
+
raise err(400, "reimbursements_invalid", str(exc)) from exc
|
| 46 |
+
except KeyError as exc:
|
| 47 |
+
raise err(404, "reimbursements_item_not_found", "That claim does not exist.") from exc
|
| 48 |
+
except Exception as exc: # noqa: BLE001
|
| 49 |
+
raise err(503, "reimbursements_unavailable", "Reimbursements could not be read. Try again shortly.") from exc
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _app(session, app_id):
|
| 53 |
+
"""The Reimbursements App record this request names, readable by the caller (404 otherwise)."""
|
| 54 |
+
import application_registry
|
| 55 |
+
try:
|
| 56 |
+
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 57 |
+
except Exception as exc: # noqa: BLE001
|
| 58 |
+
raise err(503, "reimbursements_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 59 |
+
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 60 |
+
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 61 |
+
raise err(404, "reimbursements_app_not_found", "That Reimbursements App does not exist.")
|
| 62 |
+
return app
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _may_edit(session, app):
|
| 66 |
+
import application_registry
|
| 67 |
+
role = str(session.user.get("role") or "user").strip().lower()
|
| 68 |
+
return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
|
| 72 |
+
"""C4: a read names the App, and the caller must be able to read that App."""
|
| 73 |
+
_app(session, app)
|
| 74 |
+
return session
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
|
| 78 |
+
"""C4: a write names the App, and only an editor of that App may change it."""
|
| 79 |
+
if not _may_edit(session, _app(session, app)):
|
| 80 |
+
raise err(403, "reimbursements_read_only", "You can view this App but not change it.")
|
| 81 |
+
return session
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
@router.get("/screen")
|
| 85 |
+
def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 86 |
+
cursor: str | None = Query(None), session: Session = Depends(_app_reader)):
|
| 87 |
+
rb = _module()
|
| 88 |
+
record = _app(session, app)
|
| 89 |
+
listed = _call(rb.items, _tenant(session), session.uname, status=status, cursor=cursor, st=session.runtime)
|
| 90 |
+
return {"canEdit": _may_edit(session, record), "counts": listed["counts"], "items": listed["items"],
|
| 91 |
+
"cursor": listed["cursor"], "problems": listed["problems"], "statuses": list(rb.STATUSES)}
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
@router.get("/items/{item_id}")
|
| 95 |
+
def item_detail(item_id: str, session: Session = Depends(_app_reader)):
|
| 96 |
+
rb = _module()
|
| 97 |
+
return _call(rb.item_detail, _tenant(session), session.uname, item_id, st=session.runtime)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
@router.patch("/items/{item_id}")
|
| 101 |
+
def update_item(item_id: str, status: str = Body(...), caseId: str | None = Body(None),
|
| 102 |
+
note: str | None = Body(None), session: Session = Depends(_app_writer)):
|
| 103 |
+
rb = _module()
|
| 104 |
+
return _call(rb.set_status, _tenant(session), _user(session), item_id, status, note=note, case_id=caseId,
|
| 105 |
+
st=session.runtime)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@router.get("/paid")
|
| 109 |
+
def paid_rows(cursor: str | None = Query(None), session: Session = Depends(_app_reader)):
|
| 110 |
+
rb = _module()
|
| 111 |
+
return _call(rb.paid, _tenant(session), session.uname, cursor=cursor, st=session.runtime)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
@router.get("/validate")
|
| 115 |
+
def validate(start: str | None = Query(None), end: str | None = Query(None),
|
| 116 |
+
session: Session = Depends(_app_reader)):
|
| 117 |
+
rb = _module()
|
| 118 |
+
return _call(rb.validate, _tenant(session), start=start, end=end, st=session.runtime)
|
api/routes_restock_planner.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Restock planner App routes (C3): items, one SKU, settings, refresh and the rule 8 tie."""
|
| 2 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 3 |
+
|
| 4 |
+
from deps import Session, err, require_session
|
| 5 |
+
|
| 6 |
+
TEMPLATE_KIND = "restock_planner"
|
| 7 |
+
router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["restock-planner"])
|
| 8 |
+
SOURCE = "ut_amzn_inventory_planning"
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _module():
|
| 12 |
+
from modules import restock_planner
|
| 13 |
+
return restock_planner
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 17 |
+
"""Reads need read access to the inventory planning database."""
|
| 18 |
+
from core import perm_scope
|
| 19 |
+
if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
|
| 20 |
+
raise err(403, "restock_planner_forbidden", "Your account cannot read Amazon inventory planning.")
|
| 21 |
+
return session
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _writer(session: Session = Depends(_reader)) -> Session:
|
| 25 |
+
"""Writes also need a role that may edit."""
|
| 26 |
+
if str(session.user.get("role") or "user").strip().lower() == "viewer":
|
| 27 |
+
raise err(403, "restock_planner_read_only", "You can view this App but not change it.")
|
| 28 |
+
return session
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _tenant(session):
|
| 32 |
+
return str(getattr(session.runtime, "key", "") or session.tenant)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _call(fn, *args, **kwargs):
|
| 36 |
+
try:
|
| 37 |
+
return fn(*args, **kwargs)
|
| 38 |
+
except PermissionError as exc:
|
| 39 |
+
raise err(403, "restock_planner_read_only", "You can view this App but not change it.") from exc
|
| 40 |
+
except ValueError as exc:
|
| 41 |
+
raise err(400, "restock_planner_invalid", str(exc)) from exc
|
| 42 |
+
except KeyError as exc:
|
| 43 |
+
raise err(404, "restock_planner_item_not_found", "That restock item does not exist.") from exc
|
| 44 |
+
except Exception as exc: # noqa: BLE001
|
| 45 |
+
raise err(503, "restock_planner_unavailable", "The restock plan could not be read. Try again shortly.") from exc
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _app(session, app_id):
|
| 49 |
+
"""The Restock planner App record this request names, readable by the caller (404 otherwise)."""
|
| 50 |
+
import application_registry
|
| 51 |
+
try:
|
| 52 |
+
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 53 |
+
except Exception as exc: # noqa: BLE001
|
| 54 |
+
raise err(503, "restock_planner_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 55 |
+
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 56 |
+
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 57 |
+
raise err(404, "restock_planner_app_not_found", "That Restock planner App does not exist.")
|
| 58 |
+
return app
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _may_edit(session, app):
|
| 62 |
+
import application_registry
|
| 63 |
+
role = str(session.user.get("role") or "user").strip().lower()
|
| 64 |
+
return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
|
| 68 |
+
"""A read names the App, and the caller must be able to read that App."""
|
| 69 |
+
_app(session, app)
|
| 70 |
+
return session
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
|
| 74 |
+
"""A write names the App, and only an editor of that App may change it."""
|
| 75 |
+
if not _may_edit(session, _app(session, app)):
|
| 76 |
+
raise err(403, "restock_planner_read_only", "You can view this App but not change it.")
|
| 77 |
+
return session
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def owners(session):
|
| 81 |
+
"""People who may own an item: active, non-service, non-viewer accounts of this tenant, by display name."""
|
| 82 |
+
rp = _module()
|
| 83 |
+
try:
|
| 84 |
+
users = session.runtime.get("users")
|
| 85 |
+
except Exception: # noqa: BLE001
|
| 86 |
+
users = None
|
| 87 |
+
want = _tenant(session).strip().lower()
|
| 88 |
+
names = set()
|
| 89 |
+
for name, rec in (users.items() if isinstance(users, dict) else ()):
|
| 90 |
+
if (not isinstance(rec, dict) or not rec.get("active", True) or rp._is_service(name, rec)
|
| 91 |
+
or str(rec.get("role") or "").strip().lower() == "viewer"
|
| 92 |
+
or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)):
|
| 93 |
+
continue
|
| 94 |
+
names.add(str(rec.get("name") or name).strip())
|
| 95 |
+
return sorted((n for n in names if n), key=str.lower)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def thresholds(session):
|
| 99 |
+
"""The two band thresholds in days of supply, read from the App settings (R49)."""
|
| 100 |
+
conf = _call(_module().settings, _tenant(session), st=session.runtime)
|
| 101 |
+
return {"sendToFbaDays": conf["sendToFbaDays"], "overstockDays": conf["overstockDays"]}
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
@router.get("/items")
|
| 105 |
+
def list_items(app: str = Query(..., min_length=1), band: str | None = Query(None),
|
| 106 |
+
status: str | None = Query(None), cursor: str | None = Query(None),
|
| 107 |
+
limit: int = Query(200, ge=1), q: str | None = Query(None),
|
| 108 |
+
session: Session = Depends(_app_reader)):
|
| 109 |
+
rp = _module()
|
| 110 |
+
listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor, limit=limit,
|
| 111 |
+
band=band, q=q, st=session.runtime)
|
| 112 |
+
return {**listed, "owners": owners(session), "thresholds": thresholds(session),
|
| 113 |
+
"canEdit": _may_edit(session, _app(session, app))}
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
@router.get("/items/{item_id}")
|
| 117 |
+
def read_item(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
|
| 118 |
+
rp = _module()
|
| 119 |
+
detail = _call(rp.item_detail, _tenant(session), item_id, st=session.runtime)
|
| 120 |
+
return {**detail, "thresholds": thresholds(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
@router.patch("/items/{item_id}")
|
| 124 |
+
def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 125 |
+
due: str | None = Body(None), note: str | None = Body(None),
|
| 126 |
+
session: Session = Depends(_app_writer)):
|
| 127 |
+
rp = _module()
|
| 128 |
+
return _call(rp.update_item, _tenant(session), item_id, owner=owner, status=status, due=due, note=note,
|
| 129 |
+
user=session.uname or "unknown", st=session.runtime)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
@router.get("/settings")
|
| 133 |
+
def read_settings(session: Session = Depends(_app_reader)):
|
| 134 |
+
rp = _module()
|
| 135 |
+
return _call(rp.settings, _tenant(session), st=session.runtime)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
@router.patch("/settings")
|
| 139 |
+
def write_settings(sendToFbaDays: float | None = Body(None), overstockDays: float | None = Body(None),
|
| 140 |
+
dueDays: int | None = Body(None), defaultOwner: str | None = Body(None),
|
| 141 |
+
session: Session = Depends(_app_writer)):
|
| 142 |
+
rp = _module()
|
| 143 |
+
return _call(rp.set_settings, _tenant(session), send_to_fba_days=sendToFbaDays, overstock_days=overstockDays,
|
| 144 |
+
due_days=dueDays, default_owner=defaultOwner, st=session.runtime)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
@router.post("/refresh")
|
| 148 |
+
def refresh(session: Session = Depends(_app_writer)):
|
| 149 |
+
rp = _module()
|
| 150 |
+
return _call(rp.refresh, _tenant(session), st=session.runtime)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
@router.get("/validate")
|
| 154 |
+
def validate(session: Session = Depends(_app_reader)):
|
| 155 |
+
rp = _module()
|
| 156 |
+
return _call(rp.validate, _tenant(session), st=session.runtime)
|
api/routes_returns_repeat.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Returns and repeat App routes (C3): summary, SKU segments, SKU detail, owned items and settings.
|
| 2 |
+
|
| 3 |
+
Every endpoint names the App in its path. A caller who cannot read that App gets 404. Item writes
|
| 4 |
+
need a non-viewer account that may read FBA returns. Settings writes need an editor of the App.
|
| 5 |
+
"""
|
| 6 |
+
from fastapi import APIRouter, Body, Depends, Path, Query
|
| 7 |
+
|
| 8 |
+
from deps import Session, err, require_session
|
| 9 |
+
|
| 10 |
+
TEMPLATE_KIND = "returns_repeat"
|
| 11 |
+
router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["returns-repeat"])
|
| 12 |
+
SOURCE = "ut_amzn_fba_customer_returns"
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _module():
|
| 16 |
+
from modules import returns_repeat
|
| 17 |
+
return returns_repeat
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _tenant(session):
|
| 21 |
+
return str(getattr(session.runtime, "key", "") or session.tenant)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _call(fn, *args, **kwargs):
|
| 25 |
+
try:
|
| 26 |
+
return fn(*args, **kwargs)
|
| 27 |
+
except PermissionError as exc:
|
| 28 |
+
raise err(403, "returns_repeat_read_only", str(exc)) from exc
|
| 29 |
+
except ValueError as exc:
|
| 30 |
+
raise err(400, "returns_repeat_invalid", str(exc)) from exc
|
| 31 |
+
except KeyError as exc:
|
| 32 |
+
raise err(404, "returns_repeat_not_found", "That SKU or item does not exist.") from exc
|
| 33 |
+
except Exception as exc: # noqa: BLE001
|
| 34 |
+
raise err(503, "returns_repeat_unavailable", "Returns and repeat could not be read. Try again shortly.") from exc
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _app(session, app_id):
|
| 38 |
+
"""The Returns and repeat App this request names, readable by the caller (404 otherwise)."""
|
| 39 |
+
import application_registry
|
| 40 |
+
try:
|
| 41 |
+
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 42 |
+
except Exception as exc: # noqa: BLE001
|
| 43 |
+
raise err(503, "returns_repeat_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 44 |
+
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 45 |
+
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 46 |
+
raise err(404, "returns_repeat_app_not_found", "That Returns and repeat App does not exist.")
|
| 47 |
+
return app
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _viewer(session):
|
| 51 |
+
return str(session.user.get("role") or "user").strip().lower() == "viewer"
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _may_edit_app(session, app):
|
| 55 |
+
import application_registry
|
| 56 |
+
return not _viewer(session) and application_registry.may_edit(app, session.uname, is_admin=session.admin)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _can_edit_items(session):
|
| 60 |
+
return _module().may_edit(session.runtime, session.user)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _app_reader(app_id: str = Path(..., min_length=1), session: Session = Depends(require_session)) -> Session:
|
| 64 |
+
"""A read names the App, and the caller must read that App and the FBA returns database."""
|
| 65 |
+
_app(session, app_id)
|
| 66 |
+
from core import perm_scope
|
| 67 |
+
if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
|
| 68 |
+
raise err(403, "returns_repeat_forbidden", "Your account cannot read Amazon FBA returns.")
|
| 69 |
+
return session
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _item_writer(session: Session = Depends(_app_reader)) -> Session:
|
| 73 |
+
"""An item write also refuses a viewer."""
|
| 74 |
+
if _viewer(session) or not _can_edit_items(session):
|
| 75 |
+
raise err(403, "returns_repeat_read_only", "You can view this App but not change its items.")
|
| 76 |
+
return session
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _settings_writer(app_id: str = Path(..., min_length=1), session: Session = Depends(_app_reader)) -> Session:
|
| 80 |
+
"""A settings write needs an editor of the App."""
|
| 81 |
+
if not _may_edit_app(session, _app(session, app_id)):
|
| 82 |
+
raise err(403, "returns_repeat_read_only", "You can view this App but not change its settings.")
|
| 83 |
+
return session
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
@router.get("/{app_id}/summary")
|
| 87 |
+
def read_summary(start: str | None = Query(None), end: str | None = Query(None),
|
| 88 |
+
session: Session = Depends(_app_reader)):
|
| 89 |
+
rr = _module()
|
| 90 |
+
out = _call(rr.summary, _tenant(session), start=start, end=end, st=session.runtime)
|
| 91 |
+
return {**out, "canEdit": _can_edit_items(session) and not _viewer(session)}
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
@router.get("/{app_id}/skus")
|
| 95 |
+
def list_skus(segment: str = Query("highest"), start: str | None = Query(None), end: str | None = Query(None),
|
| 96 |
+
cursor: str | None = Query(None), limit: int = Query(200, ge=1), q: str | None = Query(None),
|
| 97 |
+
session: Session = Depends(_app_reader)):
|
| 98 |
+
rr = _module()
|
| 99 |
+
return _call(rr.skus, _tenant(session), segment=segment, start=start, end=end, cursor=cursor,
|
| 100 |
+
limit=limit, q=q, st=session.runtime)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
@router.get("/{app_id}/sku")
|
| 104 |
+
def sku_detail(key: str = Query(..., min_length=1), start: str | None = Query(None),
|
| 105 |
+
end: str | None = Query(None), session: Session = Depends(_app_reader)):
|
| 106 |
+
rr = _module()
|
| 107 |
+
return _call(rr.detail, _tenant(session), key, start=start, end=end, st=session.runtime)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def owners(session):
|
| 111 |
+
"""People who may own an item: active, non-service, non-viewer accounts of this tenant, by display name."""
|
| 112 |
+
rr = _module()
|
| 113 |
+
try:
|
| 114 |
+
users = session.runtime.get("users")
|
| 115 |
+
except Exception: # noqa: BLE001
|
| 116 |
+
users = None
|
| 117 |
+
want = _tenant(session).strip().lower()
|
| 118 |
+
names = set()
|
| 119 |
+
for name, rec in (users.items() if isinstance(users, dict) else ()):
|
| 120 |
+
if (not isinstance(rec, dict) or not rec.get("active", True) or rr._is_service(name, rec)
|
| 121 |
+
or str(rec.get("role") or "").strip().lower() == "viewer"
|
| 122 |
+
or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)):
|
| 123 |
+
continue
|
| 124 |
+
names.add(str(rec.get("name") or name).strip())
|
| 125 |
+
return sorted((n for n in names if n), key=str.lower)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
@router.get("/{app_id}/items")
|
| 129 |
+
def list_items(status: str | None = Query(None), cursor: str | None = Query(None),
|
| 130 |
+
limit: int = Query(200, ge=1), session: Session = Depends(_app_reader)):
|
| 131 |
+
rr = _module()
|
| 132 |
+
listed = _call(rr.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit,
|
| 133 |
+
st=session.runtime)
|
| 134 |
+
return {**listed, "statuses": list(rr.STATUSES), "owners": owners(session),
|
| 135 |
+
"canEdit": _can_edit_items(session) and not _viewer(session)}
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
@router.patch("/{app_id}/items/{item_id}")
|
| 139 |
+
def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 140 |
+
due: str | None = Body(None), note: str | None = Body(None),
|
| 141 |
+
session: Session = Depends(_item_writer)):
|
| 142 |
+
rr = _module()
|
| 143 |
+
return _call(rr.update_item, _tenant(session), session.user, item_id, owner=owner, status=status,
|
| 144 |
+
due=due, note=note, st=session.runtime)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
@router.get("/{app_id}/settings")
|
| 148 |
+
def read_settings(app_id: str, session: Session = Depends(_app_reader)):
|
| 149 |
+
rr = _module()
|
| 150 |
+
out = _call(rr.settings, _tenant(session), st=session.runtime)
|
| 151 |
+
return {**out, "canEdit": _may_edit_app(session, _app(session, app_id))}
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
@router.put("/{app_id}/settings")
|
| 155 |
+
def write_settings(risingRatio: float | None = Body(None), risingMinPct: float | None = Body(None),
|
| 156 |
+
defaultOwner: str | None = Body(None), session: Session = Depends(_settings_writer)):
|
| 157 |
+
rr = _module()
|
| 158 |
+
return _call(rr.set_settings, _tenant(session), rising_ratio=risingRatio, rising_min_pct=risingMinPct,
|
| 159 |
+
default_owner=defaultOwner, st=session.runtime)
|
api/routes_shares.py
CHANGED
|
@@ -1090,6 +1090,97 @@ def my_shares(session: Session = Depends(require_session)):
|
|
| 1090 |
return shares.shared_with(session.uname, st=session.runtime)
|
| 1091 |
|
| 1092 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1093 |
@router.get("/share/diagnostics/{reference}")
|
| 1094 |
def get_view_share_diagnostic(reference: str, session: Session = Depends(require_session)):
|
| 1095 |
"""Read one support reference from this tenant's ring, after administrator admission."""
|
|
@@ -1240,6 +1331,18 @@ def _people(tenant):
|
|
| 1240 |
return sorted(out, key=lambda p: p["name"].lower())
|
| 1241 |
|
| 1242 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1243 |
@router.put("/share/{kind}/{oid}")
|
| 1244 |
def put_share(kind: str, oid: str, body: dict = Body(default=None),
|
| 1245 |
session: Session = Depends(require_session)):
|
|
@@ -1277,6 +1380,9 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
|
|
| 1277 |
# both answerable (*"every editable field has an owner"*), so the `field` arm prints
|
| 1278 |
# them. ⛔ THE CODE WORD `not_owner` IS UNCHANGED — clients match on it — and the
|
| 1279 |
# OTHER THREE KINDS KEEP THIS SENTENCE BYTE FOR BYTE, because their gates assert it.
|
|
|
|
|
|
|
|
|
|
| 1280 |
if kind == "field":
|
| 1281 |
table_key, field_key = shares.split_field_oid(oid)
|
| 1282 |
raise err(403, "not_owner", _field_refusal(session, table_key, field_key))
|
|
|
|
| 1090 |
return shares.shared_with(session.uname, st=session.runtime)
|
| 1091 |
|
| 1092 |
|
| 1093 |
+
def _name_resolver():
|
| 1094 |
+
"""`username -> display name` from ONE registry read; falls back to the username."""
|
| 1095 |
+
try:
|
| 1096 |
+
registry = users.registry() or {}
|
| 1097 |
+
except Exception: # noqa: BLE001
|
| 1098 |
+
registry = {}
|
| 1099 |
+
|
| 1100 |
+
def name(username):
|
| 1101 |
+
who = str(username or "").strip()
|
| 1102 |
+
row = registry.get(who) if isinstance(registry, dict) else None
|
| 1103 |
+
return (str(row.get("name") or "").strip() if isinstance(row, dict) else "") or who
|
| 1104 |
+
return name
|
| 1105 |
+
|
| 1106 |
+
|
| 1107 |
+
def _audience_mark(owner, audience, everyone, viewer):
|
| 1108 |
+
"""The mark an owner-plus-audience object gives `viewer`, or None. Same rule as `shares.mark_from`."""
|
| 1109 |
+
owner = str(owner or "").strip()
|
| 1110 |
+
named = [u for u in dict.fromkeys(str(u or "").strip() for u in audience or ()) if u]
|
| 1111 |
+
if owner and owner == viewer:
|
| 1112 |
+
others = [u for u in named if u != viewer]
|
| 1113 |
+
if not others and not everyone:
|
| 1114 |
+
return None
|
| 1115 |
+
return {"direction": "by_me", "count": len(others), "everyone": bool(everyone)}
|
| 1116 |
+
if everyone or viewer in named:
|
| 1117 |
+
return {"direction": "with_me", "by": owner}
|
| 1118 |
+
return None
|
| 1119 |
+
|
| 1120 |
+
|
| 1121 |
+
def _app_marks(session):
|
| 1122 |
+
"""App marks from the App registry's own record: `createdBy` plus its `share`."""
|
| 1123 |
+
try:
|
| 1124 |
+
import application_registry as _apps
|
| 1125 |
+
rows = (_apps.load(session.runtime).get("applications") or {}).values()
|
| 1126 |
+
except Exception: # noqa: BLE001
|
| 1127 |
+
return {}
|
| 1128 |
+
out = {}
|
| 1129 |
+
for app in rows:
|
| 1130 |
+
if not isinstance(app, dict) or app.get("retired") is True or not app.get("id"):
|
| 1131 |
+
continue
|
| 1132 |
+
share = app.get("share") if isinstance(app.get("share"), dict) else {}
|
| 1133 |
+
mark = _audience_mark(app.get("createdBy"), share.get("users"),
|
| 1134 |
+
share.get("mode") == "tenant", session.uname)
|
| 1135 |
+
if mark:
|
| 1136 |
+
out[f"app:{app['id']}"] = mark
|
| 1137 |
+
return out
|
| 1138 |
+
|
| 1139 |
+
|
| 1140 |
+
def _agent_marks(session):
|
| 1141 |
+
"""Agent marks from each Agent contract's `sharing`, read once."""
|
| 1142 |
+
try:
|
| 1143 |
+
import automation_engine as _engine
|
| 1144 |
+
contracts = _engine._agent_contracts(session.runtime)
|
| 1145 |
+
except Exception: # noqa: BLE001
|
| 1146 |
+
return {}
|
| 1147 |
+
out = {}
|
| 1148 |
+
for aid, rec in (contracts or {}).items():
|
| 1149 |
+
if not isinstance(rec, dict):
|
| 1150 |
+
continue
|
| 1151 |
+
sharing = rec.get("sharing") if isinstance(rec.get("sharing"), dict) else {}
|
| 1152 |
+
audience = sharing.get("sharedWith") if sharing.get("visibility") == "shared" else ()
|
| 1153 |
+
mark = _audience_mark(rec.get("createdBy"), audience, False, session.uname)
|
| 1154 |
+
if mark:
|
| 1155 |
+
out[f"agent:{aid}"] = mark
|
| 1156 |
+
return out
|
| 1157 |
+
|
| 1158 |
+
|
| 1159 |
+
@router.get("/share/marks")
|
| 1160 |
+
def get_share_marks(session: Session = Depends(require_session)):
|
| 1161 |
+
"""`{marks: {"<kind>:<id>": mark}}`: every shared icon a module home draws, in ONE request.
|
| 1162 |
+
|
| 1163 |
+
A mark exists only for the owner (`by_me`, with `count`) or a grantee (`with_me`, with `by`),
|
| 1164 |
+
so no viewer learns of a share they are not part of. View keys are bare view ids, because the
|
| 1165 |
+
grant registry stores them bare. An unreadable grant bucket answers 503 and the client draws
|
| 1166 |
+
no icon. App and Agent marks drop out alone when their own record cannot be read.
|
| 1167 |
+
"""
|
| 1168 |
+
held = shares.share_marks(session.uname, st=session.runtime)
|
| 1169 |
+
if held is None:
|
| 1170 |
+
raise err(503, "marks_unavailable", "Sharing could not be read. Reload to try again.")
|
| 1171 |
+
name = _name_resolver()
|
| 1172 |
+
marks = {}
|
| 1173 |
+
for kind, rows in held.items():
|
| 1174 |
+
for oid, mark in rows.items():
|
| 1175 |
+
marks[f"{kind}:{oid}"] = mark
|
| 1176 |
+
marks.update(_app_marks(session))
|
| 1177 |
+
marks.update(_agent_marks(session))
|
| 1178 |
+
for mark in marks.values():
|
| 1179 |
+
if mark["direction"] == "with_me":
|
| 1180 |
+
mark["by"] = name(mark.get("by")) if mark.get("by") else ""
|
| 1181 |
+
return {"marks": marks}
|
| 1182 |
+
|
| 1183 |
+
|
| 1184 |
@router.get("/share/diagnostics/{reference}")
|
| 1185 |
def get_view_share_diagnostic(reference: str, session: Session = Depends(require_session)):
|
| 1186 |
"""Read one support reference from this tenant's ring, after administrator admission."""
|
|
|
|
| 1331 |
return sorted(out, key=lambda p: p["name"].lower())
|
| 1332 |
|
| 1333 |
|
| 1334 |
+
def _stranger(session, kind, oid):
|
| 1335 |
+
"""True when the caller holds no role and cannot reach the object: `get_share`'s 404 test."""
|
| 1336 |
+
admin = bool(session.admin) and kind != "cohort"
|
| 1337 |
+
if kind == "field":
|
| 1338 |
+
table_key, field_key = shares.split_field_oid(oid)
|
| 1339 |
+
role = _field_permissions.field_role_sources(
|
| 1340 |
+
table_key, field_key, session.uname, is_admin=admin, st=session.runtime)["direct"]
|
| 1341 |
+
else:
|
| 1342 |
+
role = shares.role_for(kind, oid, session.uname, is_admin=admin, st=session.runtime)
|
| 1343 |
+
return role is None and not _can_see_object(session, kind, oid)
|
| 1344 |
+
|
| 1345 |
+
|
| 1346 |
@router.put("/share/{kind}/{oid}")
|
| 1347 |
def put_share(kind: str, oid: str, body: dict = Body(default=None),
|
| 1348 |
session: Session = Depends(require_session)):
|
|
|
|
| 1380 |
# both answerable (*"every editable field has an owner"*), so the `field` arm prints
|
| 1381 |
# them. ⛔ THE CODE WORD `not_owner` IS UNCHANGED — clients match on it — and the
|
| 1382 |
# OTHER THREE KINDS KEEP THIS SENTENCE BYTE FOR BYTE, because their gates assert it.
|
| 1383 |
+
# D-476: a caller with no relationship gets GET's 404, so this door confirms no id.
|
| 1384 |
+
if _stranger(session, kind, oid):
|
| 1385 |
+
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 1386 |
if kind == "field":
|
| 1387 |
table_key, field_key = shares.split_field_oid(oid)
|
| 1388 |
raise err(403, "not_owner", _field_refusal(session, table_key, field_key))
|
api/routes_sheet.py
CHANGED
|
@@ -3774,6 +3774,26 @@ REFRESH_KEEP_SECONDS = 3600
|
|
| 3774 |
_REFRESH_FAILED = {"code": "sheet_unresolved", "message": "This workbook could not be refreshed."}
|
| 3775 |
_REFRESH_TIMEOUT = {"code": "sheet_refresh_timeout",
|
| 3776 |
"message": "This refresh took too long. Try again in a few minutes."}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3777 |
|
| 3778 |
|
| 3779 |
def _refresh_desk_queues(session):
|
|
@@ -3857,36 +3877,78 @@ def _refresh_desk_enqueue(store, tenant, workbook, reader, args, now=None, path=
|
|
| 3857 |
return made["row"]
|
| 3858 |
|
| 3859 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3860 |
def _refresh_desk_claim(store, now=None):
|
| 3861 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3862 |
now = _desk_time.time() if now is None else now
|
| 3863 |
claimed = {}
|
| 3864 |
|
| 3865 |
-
def stale(row):
|
| 3866 |
-
return row.get("state") == "running" and now - float(row.get("startedTs") or 0) > REFRESH_STALE_SECONDS
|
| 3867 |
-
|
| 3868 |
# Read first: an idle poll must never write the store.
|
| 3869 |
rows = _refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now).values()
|
| 3870 |
-
|
| 3871 |
-
|
|
|
|
| 3872 |
return None
|
| 3873 |
|
| 3874 |
def apply(current):
|
| 3875 |
rows = _refresh_desk_rows(current, now)
|
| 3876 |
-
|
| 3877 |
-
if stale(row):
|
| 3878 |
-
if int(row.get("attempts") or 0) < REFRESH_MAX_ATTEMPTS:
|
| 3879 |
-
row["state"] = "queued"
|
| 3880 |
-
else:
|
| 3881 |
-
row.update(state="failed", finishedTs=now, finishedAt=_desk_iso(now),
|
| 3882 |
-
error=dict(_REFRESH_TIMEOUT))
|
| 3883 |
if not any(row.get("state") == "running" for row in rows.values()):
|
| 3884 |
queued = sorted((row for row in rows.values() if row.get("state") == "queued"),
|
| 3885 |
key=_refresh_desk_order)
|
| 3886 |
if queued:
|
| 3887 |
row = queued[0]
|
| 3888 |
row.update(state="running", startedTs=now, startedAt=_desk_iso(now),
|
| 3889 |
-
attempts=int(row.get("attempts") or 0) + 1
|
|
|
|
| 3890 |
claimed["row"] = dict(row)
|
| 3891 |
return {"requests": rows}
|
| 3892 |
|
|
@@ -3894,19 +3956,38 @@ def _refresh_desk_claim(store, now=None):
|
|
| 3894 |
return claimed.get("row")
|
| 3895 |
|
| 3896 |
|
| 3897 |
-
def
|
| 3898 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3899 |
now = _desk_time.time() if now is None else now
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3900 |
|
| 3901 |
def apply(current):
|
| 3902 |
rows = _refresh_desk_rows(current, now)
|
| 3903 |
-
row =
|
| 3904 |
-
if row is not None
|
| 3905 |
row.update(state="failed" if error else "done", finishedTs=now,
|
| 3906 |
-
finishedAt=_desk_iso(now), error=dict(error) if error else None
|
|
|
|
|
|
|
| 3907 |
return {"requests": rows}
|
| 3908 |
|
| 3909 |
store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
|
|
|
|
| 3910 |
|
| 3911 |
|
| 3912 |
def _refresh_desk_entry(store, workbook_id, reader, request_id):
|
|
@@ -6478,13 +6559,16 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
|
|
| 6478 |
|
| 6479 |
|
| 6480 |
_DRILL_MIRRORS = OrderedDict()
|
|
|
|
| 6481 |
_DRILL_MIRROR_LOCK = threading.RLock()
|
| 6482 |
_DRILL_MIRROR_MAX = 6
|
|
|
|
|
|
|
| 6483 |
|
| 6484 |
|
| 6485 |
@contextlib.contextmanager
|
| 6486 |
def _drill_mirror_lock():
|
| 6487 |
-
"""Include
|
| 6488 |
from core import request_timing
|
| 6489 |
timeout = request_timing.remaining_budget()
|
| 6490 |
acquired = (_DRILL_MIRROR_LOCK.acquire() if timeout is None
|
|
@@ -6501,6 +6585,38 @@ def _drill_mirror_lock():
|
|
| 6501 |
_DRILL_MIRROR_LOCK.release()
|
| 6502 |
|
| 6503 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6504 |
def _drill_tokens(session, databases):
|
| 6505 |
"""Unknown freshness disables reuse. No workspace documents are retained here."""
|
| 6506 |
from core import sheet_binding
|
|
@@ -6519,7 +6635,7 @@ def _drill_tokens(session, databases):
|
|
| 6519 |
|
| 6520 |
@contextlib.contextmanager
|
| 6521 |
def _drill_mirror(session, database, window, needed, databases):
|
| 6522 |
-
"""A small reader-scoped LRU of query mirrors
|
| 6523 |
from core import sheet_binding, perm_scope
|
| 6524 |
from core.request_timing import raise_if_resource_error
|
| 6525 |
import datetime
|
|
@@ -6552,8 +6668,10 @@ def _drill_mirror(session, database, window, needed, databases):
|
|
| 6552 |
"_sheet_drill_entries")
|
| 6553 |
absent = object()
|
| 6554 |
previous = {name: getattr(session, name, absent) for name in attrs}
|
| 6555 |
-
with
|
| 6556 |
-
|
|
|
|
|
|
|
| 6557 |
session._sheet_drill_isolated = True
|
| 6558 |
session._sheet_drill_cursor = hit["owned"] if hit else None
|
| 6559 |
session._sheet_drill_entries = hit["mirrors"] if hit else {}
|
|
@@ -6565,21 +6683,30 @@ def _drill_mirror(session, database, window, needed, databases):
|
|
| 6565 |
want = set(needed) | (hit["needed"] if hit else set())
|
| 6566 |
if hit and want <= hit["needed"]:
|
| 6567 |
made = hit["made"]
|
| 6568 |
-
|
|
|
|
|
|
|
| 6569 |
else:
|
| 6570 |
-
|
|
|
|
|
|
|
|
|
|
| 6571 |
made = _mirror(session)(database, metric_window=window, needed=want)
|
| 6572 |
owned = getattr(session, "_sheet_drill_cursor", None)
|
| 6573 |
hit = {"made": made, "needed": want, "owned": owned,
|
| 6574 |
"mirrors": session._sheet_drill_entries}
|
| 6575 |
# A build racing a source change is usable for this request, never reusable.
|
| 6576 |
if token and _drill_tokens(session, databases) == sources:
|
| 6577 |
-
|
| 6578 |
-
|
| 6579 |
-
|
| 6580 |
-
|
| 6581 |
-
|
| 6582 |
-
evicted
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6583 |
else:
|
| 6584 |
token = None
|
| 6585 |
# Related leaves may revisit the primary registry. Keep its table intact and widen
|
|
@@ -6589,8 +6716,16 @@ def _drill_mirror(session, database, window, needed, databases):
|
|
| 6589 |
yield made, token
|
| 6590 |
finally:
|
| 6591 |
owned = getattr(session, "_sheet_drill_cursor", None)
|
| 6592 |
-
|
| 6593 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6594 |
for name, value in previous.items():
|
| 6595 |
if value is absent:
|
| 6596 |
session.__dict__.pop(name, None)
|
|
|
|
| 3774 |
_REFRESH_FAILED = {"code": "sheet_unresolved", "message": "This workbook could not be refreshed."}
|
| 3775 |
_REFRESH_TIMEOUT = {"code": "sheet_refresh_timeout",
|
| 3776 |
"message": "This refresh took too long. Try again in a few minutes."}
|
| 3777 |
+
#: Seconds the supervisor waits past a run's own deadline before it fails the row for the desk.
|
| 3778 |
+
REFRESH_GRACE_SECONDS = 5.0
|
| 3779 |
+
|
| 3780 |
+
|
| 3781 |
+
def _refresh_budget_env():
|
| 3782 |
+
"""`AIOS_SHEET_REFRESH_BUDGET_SECONDS`, else 300. Held below half of REFRESH_STALE_SECONDS."""
|
| 3783 |
+
try:
|
| 3784 |
+
value = float(os.environ.get("AIOS_SHEET_REFRESH_BUDGET_SECONDS") or 300)
|
| 3785 |
+
except ValueError:
|
| 3786 |
+
value = 300.0
|
| 3787 |
+
return min(value if value > 0 else 300.0, REFRESH_STALE_SECONDS / 2)
|
| 3788 |
+
|
| 3789 |
+
|
| 3790 |
+
#: The total budget of one desk run, resolver included. It stays below REFRESH_STALE_SECONDS, so
|
| 3791 |
+
#: a requeue of another process's row never meets an executor that is still inside its budget.
|
| 3792 |
+
REFRESH_BUDGET_SECONDS = _refresh_budget_env()
|
| 3793 |
+
#: This process. A claimed row names its runner, so a reader can tell whose desk holds it.
|
| 3794 |
+
_REFRESH_RUNNER = uuid.uuid4().hex
|
| 3795 |
+
#: Claim token -> {"tenant", "store", "id", "deadline"} for every run this process executes now.
|
| 3796 |
+
_REFRESH_LIVE = {}
|
| 3797 |
|
| 3798 |
|
| 3799 |
def _refresh_desk_queues(session):
|
|
|
|
| 3877 |
return made["row"]
|
| 3878 |
|
| 3879 |
|
| 3880 |
+
def _refresh_desk_overdue(row, now):
|
| 3881 |
+
""""fail", "requeue" or None for one row. A row this process still executes is never requeued."""
|
| 3882 |
+
if row.get("state") != "running":
|
| 3883 |
+
return None
|
| 3884 |
+
live = _REFRESH_LIVE.get(row.get("claim")) if row.get("claim") else None
|
| 3885 |
+
if live is not None:
|
| 3886 |
+
return "fail" if now >= float(live["deadline"]) + REFRESH_GRACE_SECONDS else None
|
| 3887 |
+
if now - float(row.get("startedTs") or 0) > REFRESH_STALE_SECONDS:
|
| 3888 |
+
return "requeue" if int(row.get("attempts") or 0) < REFRESH_MAX_ATTEMPTS else "fail"
|
| 3889 |
+
return None
|
| 3890 |
+
|
| 3891 |
+
|
| 3892 |
+
def _refresh_desk_settle(rows, now, by):
|
| 3893 |
+
"""Requeue or fail every overdue row in place. Returns [(request id, new state)]."""
|
| 3894 |
+
changed = []
|
| 3895 |
+
for row in rows.values():
|
| 3896 |
+
verdict = _refresh_desk_overdue(row, now)
|
| 3897 |
+
if verdict == "requeue":
|
| 3898 |
+
# A dropped claim token stops the old executor's late finish from landing on a new run.
|
| 3899 |
+
row.update(state="queued", claim=None, requeuedBy=by)
|
| 3900 |
+
elif verdict == "fail":
|
| 3901 |
+
row.update(state="failed", finishedTs=now, finishedAt=_desk_iso(now),
|
| 3902 |
+
error=dict(_REFRESH_TIMEOUT), finishedBy=by)
|
| 3903 |
+
if verdict:
|
| 3904 |
+
changed.append((row["id"], row["state"]))
|
| 3905 |
+
return changed
|
| 3906 |
+
|
| 3907 |
+
|
| 3908 |
+
def _refresh_desk_reap(store, now=None, by="supervisor"):
|
| 3909 |
+
"""The stale step on its own: settle every overdue running row. An idle call writes nothing."""
|
| 3910 |
+
now = _desk_time.time() if now is None else now
|
| 3911 |
+
rows = _refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now)
|
| 3912 |
+
if not any(_refresh_desk_overdue(row, now) for row in rows.values()):
|
| 3913 |
+
return []
|
| 3914 |
+
changed = []
|
| 3915 |
+
|
| 3916 |
+
def apply(current):
|
| 3917 |
+
rows = _refresh_desk_rows(current, now)
|
| 3918 |
+
changed[:] = _refresh_desk_settle(rows, now, by)
|
| 3919 |
+
return {"requests": rows}
|
| 3920 |
+
|
| 3921 |
+
store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
|
| 3922 |
+
return changed
|
| 3923 |
+
|
| 3924 |
+
|
| 3925 |
def _refresh_desk_claim(store, now=None):
|
| 3926 |
+
"""Settle overdue rows, then mark the oldest queued request running and return it.
|
| 3927 |
+
|
| 3928 |
+
None while one is already running. The claimed row carries a fresh `claim` token and this
|
| 3929 |
+
process's `runner` id. Only that token can finish it.
|
| 3930 |
+
"""
|
| 3931 |
now = _desk_time.time() if now is None else now
|
| 3932 |
claimed = {}
|
| 3933 |
|
|
|
|
|
|
|
|
|
|
| 3934 |
# Read first: an idle poll must never write the store.
|
| 3935 |
rows = _refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now).values()
|
| 3936 |
+
overdue = {row.get("id") for row in rows if _refresh_desk_overdue(row, now)}
|
| 3937 |
+
busy = any(row.get("state") == "running" and row.get("id") not in overdue for row in rows)
|
| 3938 |
+
if not overdue and (busy or not any(row.get("state") == "queued" for row in rows)):
|
| 3939 |
return None
|
| 3940 |
|
| 3941 |
def apply(current):
|
| 3942 |
rows = _refresh_desk_rows(current, now)
|
| 3943 |
+
_refresh_desk_settle(rows, now, "claim")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3944 |
if not any(row.get("state") == "running" for row in rows.values()):
|
| 3945 |
queued = sorted((row for row in rows.values() if row.get("state") == "queued"),
|
| 3946 |
key=_refresh_desk_order)
|
| 3947 |
if queued:
|
| 3948 |
row = queued[0]
|
| 3949 |
row.update(state="running", startedTs=now, startedAt=_desk_iso(now),
|
| 3950 |
+
attempts=int(row.get("attempts") or 0) + 1,
|
| 3951 |
+
claim=uuid.uuid4().hex, runner=_REFRESH_RUNNER)
|
| 3952 |
claimed["row"] = dict(row)
|
| 3953 |
return {"requests": rows}
|
| 3954 |
|
|
|
|
| 3956 |
return claimed.get("row")
|
| 3957 |
|
| 3958 |
|
| 3959 |
+
def _refresh_desk_owns(row, claim):
|
| 3960 |
+
"""True when `claim` is the row's current token. A caller without a token skips the check."""
|
| 3961 |
+
return claim is None or row.get("claim") == claim
|
| 3962 |
+
|
| 3963 |
+
|
| 3964 |
+
def _refresh_desk_finish(store, request_id, error=None, now=None, claim=None):
|
| 3965 |
+
"""Mark one running request done, or failed with a public `{code, message}`.
|
| 3966 |
+
|
| 3967 |
+
Returns False and writes nothing when the row is no longer running under `claim`.
|
| 3968 |
+
"""
|
| 3969 |
now = _desk_time.time() if now is None else now
|
| 3970 |
+
done = {"ok": False}
|
| 3971 |
+
|
| 3972 |
+
def mine(rows):
|
| 3973 |
+
row = rows.get(request_id)
|
| 3974 |
+
return row if row is not None and row.get("state") == "running" and _refresh_desk_owns(row, claim) else None
|
| 3975 |
+
|
| 3976 |
+
if mine(_refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now)) is None:
|
| 3977 |
+
return False
|
| 3978 |
|
| 3979 |
def apply(current):
|
| 3980 |
rows = _refresh_desk_rows(current, now)
|
| 3981 |
+
row = mine(rows)
|
| 3982 |
+
if row is not None:
|
| 3983 |
row.update(state="failed" if error else "done", finishedTs=now,
|
| 3984 |
+
finishedAt=_desk_iso(now), error=dict(error) if error else None,
|
| 3985 |
+
finishedBy="desk")
|
| 3986 |
+
done["ok"] = True
|
| 3987 |
return {"requests": rows}
|
| 3988 |
|
| 3989 |
store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
|
| 3990 |
+
return done["ok"]
|
| 3991 |
|
| 3992 |
|
| 3993 |
def _refresh_desk_entry(store, workbook_id, reader, request_id):
|
|
|
|
| 6559 |
|
| 6560 |
|
| 6561 |
_DRILL_MIRRORS = OrderedDict()
|
| 6562 |
+
# The registry guard: held only for lookups and bookkeeping, never across a build or a yield.
|
| 6563 |
_DRILL_MIRROR_LOCK = threading.RLock()
|
| 6564 |
_DRILL_MIRROR_MAX = 6
|
| 6565 |
+
# One lease per cache key: a mirror cursor holds connection-scoped temp tables and serves one thread.
|
| 6566 |
+
_DRILL_LEASES = {}
|
| 6567 |
|
| 6568 |
|
| 6569 |
@contextlib.contextmanager
|
| 6570 |
def _drill_mirror_lock():
|
| 6571 |
+
"""Include registry-guard contention in the caller's absolute request budget."""
|
| 6572 |
from core import request_timing
|
| 6573 |
timeout = request_timing.remaining_budget()
|
| 6574 |
acquired = (_DRILL_MIRROR_LOCK.acquire() if timeout is None
|
|
|
|
| 6585 |
_DRILL_MIRROR_LOCK.release()
|
| 6586 |
|
| 6587 |
|
| 6588 |
+
@contextlib.contextmanager
|
| 6589 |
+
def _drill_lease(key):
|
| 6590 |
+
"""Serialize one cache key's cursor. Other keys never wait, and a waiter spends its budget."""
|
| 6591 |
+
from core import request_timing
|
| 6592 |
+
if key is None:
|
| 6593 |
+
# An uncacheable drill owns a private cursor, so it shares nothing to serialize.
|
| 6594 |
+
yield {"entry": None}
|
| 6595 |
+
return
|
| 6596 |
+
with _drill_mirror_lock():
|
| 6597 |
+
lease = _DRILL_LEASES.get(key)
|
| 6598 |
+
if lease is None:
|
| 6599 |
+
lease = _DRILL_LEASES[key] = {"cond": threading.Condition(threading.Lock()),
|
| 6600 |
+
"busy": False, "refs": 0, "entry": None}
|
| 6601 |
+
lease["refs"] += 1
|
| 6602 |
+
try:
|
| 6603 |
+
with lease["cond"]:
|
| 6604 |
+
while lease["busy"]:
|
| 6605 |
+
lease["cond"].wait(request_timing.remaining_budget())
|
| 6606 |
+
lease["busy"] = True
|
| 6607 |
+
try:
|
| 6608 |
+
yield lease
|
| 6609 |
+
finally:
|
| 6610 |
+
with lease["cond"]:
|
| 6611 |
+
lease["busy"] = False
|
| 6612 |
+
lease["cond"].notify_all()
|
| 6613 |
+
finally:
|
| 6614 |
+
with _DRILL_MIRROR_LOCK:
|
| 6615 |
+
lease["refs"] -= 1
|
| 6616 |
+
if not lease["refs"] and _DRILL_LEASES.get(key) is lease:
|
| 6617 |
+
_DRILL_LEASES.pop(key)
|
| 6618 |
+
|
| 6619 |
+
|
| 6620 |
def _drill_tokens(session, databases):
|
| 6621 |
"""Unknown freshness disables reuse. No workspace documents are retained here."""
|
| 6622 |
from core import sheet_binding
|
|
|
|
| 6635 |
|
| 6636 |
@contextlib.contextmanager
|
| 6637 |
def _drill_mirror(session, database, window, needed, databases):
|
| 6638 |
+
"""A small reader-scoped LRU of query mirrors: one lease per key, short registry guard."""
|
| 6639 |
from core import sheet_binding, perm_scope
|
| 6640 |
from core.request_timing import raise_if_resource_error
|
| 6641 |
import datetime
|
|
|
|
| 6668 |
"_sheet_drill_entries")
|
| 6669 |
absent = object()
|
| 6670 |
previous = {name: getattr(session, name, absent) for name in attrs}
|
| 6671 |
+
with _drill_lease(key if token else None) as lease:
|
| 6672 |
+
with _drill_mirror_lock():
|
| 6673 |
+
hit = _DRILL_MIRRORS.get(key) if token else None
|
| 6674 |
+
lease["entry"] = hit
|
| 6675 |
session._sheet_drill_isolated = True
|
| 6676 |
session._sheet_drill_cursor = hit["owned"] if hit else None
|
| 6677 |
session._sheet_drill_entries = hit["mirrors"] if hit else {}
|
|
|
|
| 6683 |
want = set(needed) | (hit["needed"] if hit else set())
|
| 6684 |
if hit and want <= hit["needed"]:
|
| 6685 |
made = hit["made"]
|
| 6686 |
+
with _DRILL_MIRROR_LOCK:
|
| 6687 |
+
if _DRILL_MIRRORS.get(key) is hit:
|
| 6688 |
+
_DRILL_MIRRORS.move_to_end(key)
|
| 6689 |
else:
|
| 6690 |
+
with _DRILL_MIRROR_LOCK:
|
| 6691 |
+
if hit is not None and _DRILL_MIRRORS.get(key) is hit:
|
| 6692 |
+
_DRILL_MIRRORS.pop(key)
|
| 6693 |
+
# The build runs under this key's lease only, so other keys build at once.
|
| 6694 |
made = _mirror(session)(database, metric_window=window, needed=want)
|
| 6695 |
owned = getattr(session, "_sheet_drill_cursor", None)
|
| 6696 |
hit = {"made": made, "needed": want, "owned": owned,
|
| 6697 |
"mirrors": session._sheet_drill_entries}
|
| 6698 |
# A build racing a source change is usable for this request, never reusable.
|
| 6699 |
if token and _drill_tokens(session, databases) == sources:
|
| 6700 |
+
with _DRILL_MIRROR_LOCK:
|
| 6701 |
+
_DRILL_MIRRORS[key] = hit
|
| 6702 |
+
_DRILL_MIRRORS.move_to_end(key)
|
| 6703 |
+
lease["entry"] = hit
|
| 6704 |
+
while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX:
|
| 6705 |
+
gone, evicted = _DRILL_MIRRORS.popitem(last=False)
|
| 6706 |
+
# A leased cursor closes when its holder leaves, never under it.
|
| 6707 |
+
held = _DRILL_LEASES.get(gone)
|
| 6708 |
+
if evicted["owned"] and not (held and held["entry"] is evicted):
|
| 6709 |
+
evicted["owned"][0].close()
|
| 6710 |
else:
|
| 6711 |
token = None
|
| 6712 |
# Related leaves may revisit the primary registry. Keep its table intact and widen
|
|
|
|
| 6716 |
yield made, token
|
| 6717 |
finally:
|
| 6718 |
owned = getattr(session, "_sheet_drill_cursor", None)
|
| 6719 |
+
entry = lease["entry"]
|
| 6720 |
+
with _DRILL_MIRROR_LOCK:
|
| 6721 |
+
lease["entry"] = None
|
| 6722 |
+
kept = [item["owned"] for item in _DRILL_MIRRORS.values()]
|
| 6723 |
+
# The last lease on an uncached or evicted cursor closes it.
|
| 6724 |
+
spent = [owned, entry["owned"] if entry else None]
|
| 6725 |
+
for index, cursor in enumerate(spent):
|
| 6726 |
+
if (cursor and not any(seen is cursor for seen in spent[:index])
|
| 6727 |
+
and not any(item is cursor for item in kept)):
|
| 6728 |
+
cursor[0].close()
|
| 6729 |
for name, value in previous.items():
|
| 6730 |
if value is absent:
|
| 6731 |
session.__dict__.pop(name, None)
|
api/routes_tables.py
CHANGED
|
@@ -3767,8 +3767,7 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
|
|
| 3767 |
# ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY
|
| 3768 |
# by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took`
|
| 3769 |
# asks whether THIS write landed, and this write goes to the definition.
|
| 3770 |
-
stored = dict(
|
| 3771 |
-
.get(str(pid)) or {})
|
| 3772 |
accepted = {k: stored.get(k) for k in updates if k in stored}
|
| 3773 |
|
| 3774 |
def _took(k):
|
|
|
|
| 3767 |
# ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY
|
| 3768 |
# by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took`
|
| 3769 |
# asks whether THIS write landed, and this write goes to the definition.
|
| 3770 |
+
stored = dict(_ut().get_record(table_key, pid, st=session.runtime) or {})
|
|
|
|
| 3771 |
accepted = {k: stored.get(k) for k in updates if k in stored}
|
| 3772 |
|
| 3773 |
def _took(k):
|
api/routes_templates.py
CHANGED
|
@@ -24,6 +24,7 @@ columns listed. The alternative — writing the views anyway — is the `_seed_w
|
|
| 24 |
verbatim: `clean_filter_tree` DROPS a leaf on an unknown column, and a view whose only condition
|
| 25 |
was dropped shows EVERY row under a name that promises a shortlist.
|
| 26 |
"""
|
|
|
|
| 27 |
import dash_norm
|
| 28 |
import uuid
|
| 29 |
|
|
@@ -38,6 +39,10 @@ _TEMPLATE_CATEGORIES = {
|
|
| 38 |
"sheet": "Sheet", "data": "Table", "app": "App", "agents": "Automation", "saved": "Saved",
|
| 39 |
"route": "Operations", "catalog": "Sales", "competitor_watch": "Sales",
|
| 40 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
_TEMPLATE_PREVIEW_LABELS = {**_TEMPLATE_CATEGORIES, "route": "Routing", "catalog": "Catalog",
|
| 42 |
"competitor_watch": "Competitor watch"}
|
| 43 |
|
|
@@ -395,16 +400,21 @@ def _own_template_apps(session):
|
|
| 395 |
return {}
|
| 396 |
|
| 397 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 398 |
def _existing_app(held):
|
| 399 |
"""{id, name, address} of an owned App, where address is the hash the picker navigates to."""
|
| 400 |
-
from urllib.parse import quote
|
| 401 |
if not isinstance(held, dict) or not held.get("id"):
|
| 402 |
return None
|
| 403 |
if held.get("deepLink"):
|
| 404 |
address = "#" + str(held["deepLink"])
|
| 405 |
else:
|
| 406 |
-
address = ("
|
| 407 |
-
+ "&application=" + quote(str(held["id"]), safe=""))
|
| 408 |
return {"id": held["id"], "name": dash_norm.plain(str(held.get("name") or "")),
|
| 409 |
"address": address}
|
| 410 |
|
|
@@ -485,11 +495,19 @@ def _module_templates(session):
|
|
| 485 |
database_states = _database_states(session, definitions, databases)
|
| 486 |
requirement = lambda database, connector="": _requirement(
|
| 487 |
database, definitions, have, database_states, connector)
|
| 488 |
-
sheet = [
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 493 |
for row in saved)
|
| 494 |
data = []
|
| 495 |
for row, database in data_templates:
|
|
@@ -497,8 +515,11 @@ def _module_templates(session):
|
|
| 497 |
connector = "odoo" if row["source"].startswith("odoo_") else row["source"]
|
| 498 |
data.append(_catalog_row(row["key"], row["label"], row["desc"],
|
| 499 |
[requirement(database, connector)], module="data"))
|
| 500 |
-
# These App contracts map fields in the existing wizard
|
| 501 |
-
|
|
|
|
|
|
|
|
|
|
| 502 |
for row in application_templates(session=session)["templates"]]
|
| 503 |
# Owned copies are optional links. They never replace the template's create action.
|
| 504 |
owned = _own_template_apps(session)
|
|
@@ -585,6 +606,10 @@ def apply_sheet_template(key: str, body: dict = Body(default=None),
|
|
| 585 |
created_by=session.uname)
|
| 586 |
# Owners read their persisted values. Use the same permission-scoped
|
| 587 |
# refresh as Sheet so the first open already contains computed cells.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 588 |
refreshed = refresh_sheet(book["id"], session=session)["workbook"]
|
| 589 |
failures = []
|
| 590 |
for tab in refreshed["sheets"]:
|
|
@@ -616,13 +641,14 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
|
|
| 616 |
session: Session = Depends(require_session)):
|
| 617 |
from routes_nav import create_nav_application
|
| 618 |
from urllib.parse import quote
|
|
|
|
| 619 |
|
| 620 |
body = body if isinstance(body, dict) else {}
|
| 621 |
if module == "agents":
|
| 622 |
# A direct call to create_automation does not run its FastAPI dependency.
|
| 623 |
session.require("automation")
|
| 624 |
-
if module == "app" and key
|
| 625 |
-
pass #
|
| 626 |
elif module == "app" and key not in {"route", "catalog"}:
|
| 627 |
raise err(404, "unknown_template", "That template does not exist.")
|
| 628 |
section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None)
|
|
@@ -647,7 +673,36 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
|
|
| 647 |
return {"address": "#/automation?agent=" + quote(result["automation"]["id"], safe=""), **result}
|
| 648 |
# The existing wizard still owns its POST-then-PATCH share sequence. API clients may
|
| 649 |
# dispatch creation here; the existing App writer retains schema and permission walls.
|
| 650 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 651 |
app = result["application"]
|
| 652 |
-
return {"address":
|
| 653 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
verbatim: `clean_filter_tree` DROPS a leaf on an unknown column, and a view whose only condition
|
| 25 |
was dropped shows EVERY row under a name that promises a shortlist.
|
| 26 |
"""
|
| 27 |
+
import app_templates
|
| 28 |
import dash_norm
|
| 29 |
import uuid
|
| 30 |
|
|
|
|
| 39 |
"sheet": "Sheet", "data": "Table", "app": "App", "agents": "Automation", "saved": "Saved",
|
| 40 |
"route": "Operations", "catalog": "Sales", "competitor_watch": "Sales",
|
| 41 |
}
|
| 42 |
+
# C1 connector keys for a shipped Sheet descriptor's `source` (F25: Amazon is `amazon_sp` there).
|
| 43 |
+
_SHEET_CONNECTORS = {"odoo": "odoo", "amazon_sp": "amazon"}
|
| 44 |
+
# C1 fields each App row carries so the client can pick its screen and marks.
|
| 45 |
+
_APP_WIRE_KEYS = ("view", "mark", "connector", "process", "purpose")
|
| 46 |
_TEMPLATE_PREVIEW_LABELS = {**_TEMPLATE_CATEGORIES, "route": "Routing", "catalog": "Catalog",
|
| 47 |
"competitor_watch": "Competitor watch"}
|
| 48 |
|
|
|
|
| 400 |
return {}
|
| 401 |
|
| 402 |
|
| 403 |
+
def _app_address(app_id, source):
|
| 404 |
+
"""The hash an App opens at. An App that reads no database names none."""
|
| 405 |
+
from urllib.parse import quote
|
| 406 |
+
database = "database=" + quote(str(source), safe="") + "&" if source else ""
|
| 407 |
+
return "#/app?" + database + "application=" + quote(str(app_id), safe="")
|
| 408 |
+
|
| 409 |
+
|
| 410 |
def _existing_app(held):
|
| 411 |
"""{id, name, address} of an owned App, where address is the hash the picker navigates to."""
|
|
|
|
| 412 |
if not isinstance(held, dict) or not held.get("id"):
|
| 413 |
return None
|
| 414 |
if held.get("deepLink"):
|
| 415 |
address = "#" + str(held["deepLink"])
|
| 416 |
else:
|
| 417 |
+
address = _app_address(held["id"], held.get("sourceDatabase"))
|
|
|
|
| 418 |
return {"id": held["id"], "name": dash_norm.plain(str(held.get("name") or "")),
|
| 419 |
"address": address}
|
| 420 |
|
|
|
|
| 495 |
database_states = _database_states(session, definitions, databases)
|
| 496 |
requirement = lambda database, connector="": _requirement(
|
| 497 |
database, definitions, have, database_states, connector)
|
| 498 |
+
sheet = []
|
| 499 |
+
for row in sheet_templates.TEMPLATES:
|
| 500 |
+
# QI20: a shipped Sheet template is bundled with its connector and offered only with it.
|
| 501 |
+
connector = _SHEET_CONNECTORS.get(row["source"], row["source"])
|
| 502 |
+
if app_templates.entry_type(connector) not in have:
|
| 503 |
+
continue
|
| 504 |
+
sheet.append({**_catalog_row(row["key"], row["title"], row["answers"],
|
| 505 |
+
[requirement(row["database_hint"], row["source"])]),
|
| 506 |
+
"connector": connector})
|
| 507 |
+
# R34: a user-saved template keeps its databases and names no connector.
|
| 508 |
+
sheet.extend({**_catalog_row(row["key"], row["name"], row["description"],
|
| 509 |
+
[requirement(db) for db in row["databases"]], module="saved"),
|
| 510 |
+
"databases": list(row["databases"])}
|
| 511 |
for row in saved)
|
| 512 |
data = []
|
| 513 |
for row, database in data_templates:
|
|
|
|
| 515 |
connector = "odoo" if row["source"].startswith("odoo_") else row["source"]
|
| 516 |
data.append(_catalog_row(row["key"], row["label"], row["desc"],
|
| 517 |
[requirement(database, connector)], module="data"))
|
| 518 |
+
# These App contracts map fields in the existing wizard. A connector-gated kind is listed
|
| 519 |
+
# only when the registry's offer predicate holds, so it needs no requirement row here.
|
| 520 |
+
# S79-T05: an App card reads its C1 purpose. R33 removed the database and field steps.
|
| 521 |
+
app = [{**_catalog_row(row["kind"], row["label"], row.get("purpose") or row["label"], [], module="app"),
|
| 522 |
+
**{key: row.get(key) for key in _APP_WIRE_KEYS}}
|
| 523 |
for row in application_templates(session=session)["templates"]]
|
| 524 |
# Owned copies are optional links. They never replace the template's create action.
|
| 525 |
owned = _own_template_apps(session)
|
|
|
|
| 606 |
created_by=session.uname)
|
| 607 |
# Owners read their persisted values. Use the same permission-scoped
|
| 608 |
# refresh as Sheet so the first open already contains computed cells.
|
| 609 |
+
import routes_sheet
|
| 610 |
+
if routes_sheet._refresh_desk_queues(session):
|
| 611 |
+
routes_sheet.refresh_sheet_door(book["id"], body={}, session=session)
|
| 612 |
+
return {"workbook": book}
|
| 613 |
refreshed = refresh_sheet(book["id"], session=session)["workbook"]
|
| 614 |
failures = []
|
| 615 |
for tab in refreshed["sheets"]:
|
|
|
|
| 641 |
session: Session = Depends(require_session)):
|
| 642 |
from routes_nav import create_nav_application
|
| 643 |
from urllib.parse import quote
|
| 644 |
+
import application_registry
|
| 645 |
|
| 646 |
body = body if isinstance(body, dict) else {}
|
| 647 |
if module == "agents":
|
| 648 |
# A direct call to create_automation does not run its FastAPI dependency.
|
| 649 |
session.require("automation")
|
| 650 |
+
if module == "app" and key in application_registry.GATED_TEMPLATE_KINDS:
|
| 651 |
+
pass # C1: gated by application_registry.template_contracts(session=...).
|
| 652 |
elif module == "app" and key not in {"route", "catalog"}:
|
| 653 |
raise err(404, "unknown_template", "That template does not exist.")
|
| 654 |
section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None)
|
|
|
|
| 673 |
return {"address": "#/automation?agent=" + quote(result["automation"]["id"], safe=""), **result}
|
| 674 |
# The existing wizard still owns its POST-then-PATCH share sequence. API clients may
|
| 675 |
# dispatch creation here; the existing App writer retains schema and permission walls.
|
| 676 |
+
# S79-T05: the App directory's one-click Add sends no body, so the C1 name and source fill in.
|
| 677 |
+
defaults = {"name": template["name"]}
|
| 678 |
+
if app_templates.source(key):
|
| 679 |
+
defaults["sourceDatabase"] = app_templates.source(key)
|
| 680 |
+
result = create_nav_application({**defaults, **body, "templateKind": key}, session=session)
|
| 681 |
+
else:
|
| 682 |
+
result = _create_sourceless_app({**defaults, **body, "templateKind": key}, session)
|
| 683 |
app = result["application"]
|
| 684 |
+
return {"address": _app_address(app["id"], app.get("sourceDatabase")), **result}
|
| 685 |
+
|
| 686 |
+
|
| 687 |
+
def _create_sourceless_app(payload, session):
|
| 688 |
+
"""S79-T05: create an App whose C1 kind reads no database, behind the nav writer's walls.
|
| 689 |
+
|
| 690 |
+
`create_nav_application` requires a sourceDatabase, so this door calls the registry itself.
|
| 691 |
+
"""
|
| 692 |
+
import application_registry
|
| 693 |
+
|
| 694 |
+
if not session.runtime.available():
|
| 695 |
+
raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.")
|
| 696 |
+
try:
|
| 697 |
+
application = application_registry.create(
|
| 698 |
+
session.runtime, session.tenant, session.uname, payload,
|
| 699 |
+
field_contracts={}, allowed_databases=set())
|
| 700 |
+
except PermissionError as exc:
|
| 701 |
+
raise err(403, "forbidden", str(exc))
|
| 702 |
+
except application_registry.ApplicationRuleError as exc:
|
| 703 |
+
raise err(400, exc.code, str(exc))
|
| 704 |
+
except application_registry.ApplicationError as exc:
|
| 705 |
+
raise err(400, "bad_application", str(exc))
|
| 706 |
+
except Exception:
|
| 707 |
+
raise err(503, "store_unavailable", "the Application was not saved.")
|
| 708 |
+
return {"application": application}
|
api/sheet_warm.py
CHANGED
|
@@ -19,7 +19,7 @@ from datetime import datetime, time as dtime, timedelta, timezone
|
|
| 19 |
from time import perf_counter
|
| 20 |
|
| 21 |
import routes_sheet
|
| 22 |
-
from core import external_access, perms, refresh_proof, shares, sheets, users
|
| 23 |
from deps import Session
|
| 24 |
|
| 25 |
#: The rollover warm starts this long after 00:00 UTC, so the day clock has moved for every reader.
|
|
@@ -335,7 +335,12 @@ def _on_exit():
|
|
| 335 |
# ── the refresh desk: person refreshes the web role queued, run here one at a time ──────────────
|
| 336 |
#: Seconds between two reads of each tenant's request document.
|
| 337 |
DESK_POLL_SECONDS = 5.0
|
| 338 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 339 |
|
| 340 |
|
| 341 |
def _desk_tenants():
|
|
@@ -499,37 +504,65 @@ def _desk_result_log(row, tenant, session, before, result, *, admitted=None):
|
|
| 499 |
pass
|
| 500 |
|
| 501 |
|
| 502 |
-
def desk_run_one(rt, tenant):
|
| 503 |
-
"""Claim and run the oldest queued refresh of one tenant. Returns True when one ran.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 504 |
row = routes_sheet._refresh_desk_claim(rt)
|
| 505 |
if row is None:
|
| 506 |
return False
|
| 507 |
started = perf_counter()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 508 |
error = None
|
| 509 |
book, session, result, admitted = None, None, None, None
|
| 510 |
try:
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 523 |
print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} "
|
| 524 |
f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}"
|
| 525 |
-
+ (f" code={error['code']}" if error else "")
|
|
|
|
| 526 |
return True
|
| 527 |
|
| 528 |
|
| 529 |
-
def
|
| 530 |
-
"""Run every queued refresh of one tenant, oldest first. Returns how many ran."""
|
| 531 |
from harness import runtime
|
| 532 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 533 |
if _desk_skip(rt, tenant):
|
| 534 |
return 0
|
| 535 |
# The revision is taken BEFORE the reads, so a request queued between them still moves it.
|
|
@@ -538,7 +571,7 @@ def desk_drain(tenant):
|
|
| 538 |
except Exception: # noqa: BLE001
|
| 539 |
before = None
|
| 540 |
ran = 0
|
| 541 |
-
while not _STOP.is_set() and desk_run_one(rt, tenant):
|
| 542 |
ran += 1
|
| 543 |
if not ran and before is not None and not routes_sheet._refresh_desk_pending(rt):
|
| 544 |
_DESK["seen"][tenant] = before
|
|
@@ -547,32 +580,99 @@ def desk_drain(tenant):
|
|
| 547 |
return ran
|
| 548 |
|
| 549 |
|
| 550 |
-
def _desk_loop():
|
| 551 |
-
|
|
|
|
|
|
|
|
|
|
| 552 |
try:
|
| 553 |
tenants = _desk_tenants()
|
| 554 |
except Exception as exc: # noqa: BLE001
|
| 555 |
print(f"[sheet-desk] tenant list failed {type(exc).__name__}: {exc}", flush=True)
|
| 556 |
tenants = []
|
| 557 |
for tenant in tenants:
|
| 558 |
-
if _STOP.is_set():
|
| 559 |
return
|
| 560 |
try:
|
| 561 |
-
desk_drain(tenant)
|
| 562 |
except Exception as exc: # noqa: BLE001
|
| 563 |
print(f"[sheet-desk] tenant={tenant} failed {type(exc).__name__}: {exc}", flush=True)
|
| 564 |
_STOP.wait(DESK_POLL_SECONDS)
|
| 565 |
|
| 566 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 567 |
def start_desk():
|
| 568 |
-
"""Start the
|
|
|
|
| 569 |
with _LOCK:
|
| 570 |
thread = _DESK["thread"]
|
| 571 |
-
if thread is
|
| 572 |
-
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
_DESK["
|
|
|
|
|
|
|
| 576 |
return thread
|
| 577 |
|
| 578 |
|
|
|
|
| 19 |
from time import perf_counter
|
| 20 |
|
| 21 |
import routes_sheet
|
| 22 |
+
from core import external_access, perms, refresh_proof, request_timing, shares, sheets, users
|
| 23 |
from deps import Session
|
| 24 |
|
| 25 |
#: The rollover warm starts this long after 00:00 UTC, so the day clock has moved for every reader.
|
|
|
|
| 335 |
# ── the refresh desk: person refreshes the web role queued, run here one at a time ──────────────
|
| 336 |
#: Seconds between two reads of each tenant's request document.
|
| 337 |
DESK_POLL_SECONDS = 5.0
|
| 338 |
+
#: Seconds between two supervisor checks of the desk and the runs it holds.
|
| 339 |
+
SUPERVISOR_POLL_SECONDS = 5.0
|
| 340 |
+
#: Seconds between two supervisor reads of every tenant, for rows another process left running.
|
| 341 |
+
SUPERVISOR_SWEEP_SECONDS = 60.0
|
| 342 |
+
#: generation: the desk loop that may claim. beat: monotonic time the desk last showed progress.
|
| 343 |
+
_DESK = {"thread": None, "seen": {}, "generation": 0, "beat": None, "supervisor": None, "swept": None}
|
| 344 |
|
| 345 |
|
| 346 |
def _desk_tenants():
|
|
|
|
| 504 |
pass
|
| 505 |
|
| 506 |
|
| 507 |
+
def desk_run_one(rt, tenant, generation=None):
|
| 508 |
+
"""Claim and run the oldest queued refresh of one tenant. Returns True when one ran.
|
| 509 |
+
|
| 510 |
+
The run holds REFRESH_BUDGET_SECONDS in total. A desk of a replaced generation claims nothing.
|
| 511 |
+
"""
|
| 512 |
+
if generation is not None and generation != _DESK["generation"]:
|
| 513 |
+
return False
|
| 514 |
+
_DESK["beat"] = time.monotonic()
|
| 515 |
row = routes_sheet._refresh_desk_claim(rt)
|
| 516 |
if row is None:
|
| 517 |
return False
|
| 518 |
started = perf_counter()
|
| 519 |
+
budget = float(routes_sheet.REFRESH_BUDGET_SECONDS)
|
| 520 |
+
deadline = time.monotonic() + budget
|
| 521 |
+
claim = row.get("claim")
|
| 522 |
+
if claim:
|
| 523 |
+
# The supervisor reads this to fail a run that overran its deadline, never to requeue it.
|
| 524 |
+
routes_sheet._REFRESH_LIVE[claim] = {"tenant": tenant, "store": rt, "id": row["id"],
|
| 525 |
+
"deadline": time.time() + budget}
|
| 526 |
error = None
|
| 527 |
book, session, result, admitted = None, None, None, None
|
| 528 |
try:
|
| 529 |
+
try:
|
| 530 |
+
with request_timing.budget_scope(deadline):
|
| 531 |
+
book = sheets.get_workbook(rt, row["workbook"])
|
| 532 |
+
rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt)
|
| 533 |
+
session = reader_session(rt, tenant, row.get("reader"), book, rec)
|
| 534 |
+
if session is None:
|
| 535 |
+
error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
|
| 536 |
+
else:
|
| 537 |
+
admitted = _desk_input_identity(book)
|
| 538 |
+
result = routes_sheet._refresh_desk_run(session, row)
|
| 539 |
+
except Exception as exc: # noqa: BLE001
|
| 540 |
+
timed_out = (isinstance(exc, request_timing.RequestDeadlineExceeded)
|
| 541 |
+
or time.monotonic() >= deadline)
|
| 542 |
+
error = dict(routes_sheet._REFRESH_TIMEOUT) if timed_out else routes_sheet._refresh_desk_error(exc)
|
| 543 |
+
_desk_result_log(row, tenant, session, book, result, admitted=admitted)
|
| 544 |
+
owned = routes_sheet._refresh_desk_finish(rt, row["id"], error, claim=claim)
|
| 545 |
+
finally:
|
| 546 |
+
if claim:
|
| 547 |
+
routes_sheet._REFRESH_LIVE.pop(claim, None)
|
| 548 |
print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} "
|
| 549 |
f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}"
|
| 550 |
+
+ (f" code={error['code']}" if error else "")
|
| 551 |
+
+ (" late=1" if owned is False else ""), flush=True)
|
| 552 |
return True
|
| 553 |
|
| 554 |
|
| 555 |
+
def _desk_runtime(tenant):
|
|
|
|
| 556 |
from harness import runtime
|
| 557 |
+
return runtime.get_runtime(tenant)
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
def desk_drain(tenant, generation=None):
|
| 561 |
+
"""Run every queued refresh of one tenant, oldest first. Returns how many ran."""
|
| 562 |
+
if generation is not None and generation != _DESK["generation"]:
|
| 563 |
+
return 0
|
| 564 |
+
_DESK["beat"] = time.monotonic()
|
| 565 |
+
rt = _desk_runtime(tenant)
|
| 566 |
if _desk_skip(rt, tenant):
|
| 567 |
return 0
|
| 568 |
# The revision is taken BEFORE the reads, so a request queued between them still moves it.
|
|
|
|
| 571 |
except Exception: # noqa: BLE001
|
| 572 |
before = None
|
| 573 |
ran = 0
|
| 574 |
+
while not _STOP.is_set() and desk_run_one(rt, tenant, generation):
|
| 575 |
ran += 1
|
| 576 |
if not ran and before is not None and not routes_sheet._refresh_desk_pending(rt):
|
| 577 |
_DESK["seen"][tenant] = before
|
|
|
|
| 580 |
return ran
|
| 581 |
|
| 582 |
|
| 583 |
+
def _desk_loop(generation=None):
|
| 584 |
+
"""One desk generation. It exits once the supervisor starts a newer one."""
|
| 585 |
+
generation = _DESK["generation"] if generation is None else generation
|
| 586 |
+
while not _STOP.is_set() and _DESK["generation"] == generation:
|
| 587 |
+
_DESK["beat"] = time.monotonic()
|
| 588 |
try:
|
| 589 |
tenants = _desk_tenants()
|
| 590 |
except Exception as exc: # noqa: BLE001
|
| 591 |
print(f"[sheet-desk] tenant list failed {type(exc).__name__}: {exc}", flush=True)
|
| 592 |
tenants = []
|
| 593 |
for tenant in tenants:
|
| 594 |
+
if _STOP.is_set() or _DESK["generation"] != generation:
|
| 595 |
return
|
| 596 |
try:
|
| 597 |
+
desk_drain(tenant, generation)
|
| 598 |
except Exception as exc: # noqa: BLE001
|
| 599 |
print(f"[sheet-desk] tenant={tenant} failed {type(exc).__name__}: {exc}", flush=True)
|
| 600 |
_STOP.wait(DESK_POLL_SECONDS)
|
| 601 |
|
| 602 |
|
| 603 |
+
def _desk_stuck(now):
|
| 604 |
+
"""True when the desk thread died, or showed no progress for longer than one whole run."""
|
| 605 |
+
thread = _DESK["thread"]
|
| 606 |
+
if thread is None:
|
| 607 |
+
return False
|
| 608 |
+
if not thread.is_alive():
|
| 609 |
+
return True
|
| 610 |
+
limit = (float(routes_sheet.REFRESH_BUDGET_SECONDS) + routes_sheet.REFRESH_GRACE_SECONDS
|
| 611 |
+
+ DESK_POLL_SECONDS)
|
| 612 |
+
return _DESK["beat"] is not None and now - _DESK["beat"] > limit
|
| 613 |
+
|
| 614 |
+
|
| 615 |
+
def _desk_replace(reason):
|
| 616 |
+
"""Start a desk of the next generation. The stuck one claims nothing once it returns."""
|
| 617 |
+
import process_role
|
| 618 |
+
with _LOCK:
|
| 619 |
+
_DESK["generation"] += 1
|
| 620 |
+
generation = _DESK["generation"]
|
| 621 |
+
_DESK["beat"] = time.monotonic()
|
| 622 |
+
_DESK["seen"].clear()
|
| 623 |
+
_DESK["thread"] = process_role.start("sheet-refresh-desk", "heavy", _desk_loop, args=(generation,))
|
| 624 |
+
print(f"[sheet-desk] supervisor replaced the desk reason={reason} generation={generation}", flush=True)
|
| 625 |
+
return _DESK["thread"]
|
| 626 |
+
|
| 627 |
+
|
| 628 |
+
def supervise_once():
|
| 629 |
+
"""Fail runs past their deadline, settle rows another process left, and replace a stuck desk."""
|
| 630 |
+
wall, mono = time.time(), time.monotonic()
|
| 631 |
+
stores = {}
|
| 632 |
+
for entry in list(routes_sheet._REFRESH_LIVE.values()):
|
| 633 |
+
if wall >= float(entry["deadline"]) + routes_sheet.REFRESH_GRACE_SECONDS:
|
| 634 |
+
stores[entry["tenant"]] = entry["store"]
|
| 635 |
+
# A read of every tenant at a slower pace finds a row whose runner process died.
|
| 636 |
+
if _DESK["swept"] is None or mono - _DESK["swept"] >= SUPERVISOR_SWEEP_SECONDS:
|
| 637 |
+
_DESK["swept"] = mono
|
| 638 |
+
try:
|
| 639 |
+
for tenant in _desk_tenants():
|
| 640 |
+
if tenant not in stores:
|
| 641 |
+
stores[tenant] = _desk_runtime(tenant)
|
| 642 |
+
except Exception as exc: # noqa: BLE001
|
| 643 |
+
print(f"[sheet-desk] supervisor tenant list failed {type(exc).__name__}: {exc}", flush=True)
|
| 644 |
+
for tenant, rt in stores.items():
|
| 645 |
+
try:
|
| 646 |
+
for request_id, state in routes_sheet._refresh_desk_reap(rt, by="supervisor"):
|
| 647 |
+
print(f"[sheet-desk] supervisor tenant={tenant} request={request_id} state={state}"
|
| 648 |
+
f" code={routes_sheet._REFRESH_TIMEOUT['code']}", flush=True)
|
| 649 |
+
except Exception as exc: # noqa: BLE001
|
| 650 |
+
print(f"[sheet-desk] supervisor tenant={tenant} failed {type(exc).__name__}: {exc}", flush=True)
|
| 651 |
+
if _desk_stuck(time.monotonic()):
|
| 652 |
+
_desk_replace("dead" if not _DESK["thread"].is_alive() else "no_progress")
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
def _supervise_loop():
|
| 656 |
+
while not _STOP.is_set():
|
| 657 |
+
try:
|
| 658 |
+
supervise_once()
|
| 659 |
+
except Exception as exc: # noqa: BLE001
|
| 660 |
+
print(f"[sheet-desk] supervisor failed {type(exc).__name__}: {exc}", flush=True)
|
| 661 |
+
_STOP.wait(SUPERVISOR_POLL_SECONDS)
|
| 662 |
+
|
| 663 |
+
|
| 664 |
def start_desk():
|
| 665 |
+
"""Start the "sheet-refresh-desk" thread and its supervisor where heavy jobs run. Returns the desk."""
|
| 666 |
+
import process_role
|
| 667 |
with _LOCK:
|
| 668 |
thread = _DESK["thread"]
|
| 669 |
+
if thread is None or not thread.is_alive():
|
| 670 |
+
_DESK["beat"] = time.monotonic()
|
| 671 |
+
thread = process_role.start("sheet-refresh-desk", "heavy", _desk_loop, args=(_DESK["generation"],))
|
| 672 |
+
_DESK["thread"] = thread
|
| 673 |
+
supervisor = _DESK["supervisor"]
|
| 674 |
+
if thread is not None and (supervisor is None or not supervisor.is_alive()):
|
| 675 |
+
_DESK["supervisor"] = process_role.start("sheet-refresh-supervisor", "heavy", _supervise_loop)
|
| 676 |
return thread
|
| 677 |
|
| 678 |
|
api/sync_worker.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
| 2 |
|
| 3 |
Run as `python sync_worker.py` from aios-web/api. The web process spawns and supervises it.
|
| 4 |
It keeps its own DuckDB files, so it never opens the web process's royal.duckdb.
|
|
|
|
| 5 |
"""
|
| 6 |
import os
|
| 7 |
import sys
|
|
@@ -22,8 +23,12 @@ def configure(env=None):
|
|
| 22 |
worker_db = worker_dir / web_db.name
|
| 23 |
env["AIOS_DUCKDB_STORE_DIR"] = str(worker_dir)
|
| 24 |
env["AIOS_DUCKDB_PATH"] = str(worker_db)
|
|
|
|
| 25 |
env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "4GB"
|
| 26 |
env["AIOS_DUCKDB_THREADS"] = env.get("AIOS_WORKER_DUCKDB_THREADS") or "1"
|
|
|
|
|
|
|
|
|
|
| 27 |
return worker_db
|
| 28 |
|
| 29 |
|
|
@@ -58,6 +63,9 @@ def main():
|
|
| 58 |
process_role.start_worker_services()
|
| 59 |
# Importing the app starts every role-allowed thread at module level. Startup hooks need a call.
|
| 60 |
import main as app_main # noqa: F401
|
|
|
|
|
|
|
|
|
|
| 61 |
import routes_connectors
|
| 62 |
routes_connectors._start_database_jobs()
|
| 63 |
print(f"[sync-worker] running: {', '.join(sorted(process_role.started_jobs()))}", flush=True)
|
|
|
|
| 2 |
|
| 3 |
Run as `python sync_worker.py` from aios-web/api. The web process spawns and supervises it.
|
| 4 |
It keeps its own DuckDB files, so it never opens the web process's royal.duckdb.
|
| 5 |
+
Its heavy jobs wait until it holds the store's sync writer lease, so two Spaces never both sync.
|
| 6 |
"""
|
| 7 |
import os
|
| 8 |
import sys
|
|
|
|
| 23 |
worker_db = worker_dir / web_db.name
|
| 24 |
env["AIOS_DUCKDB_STORE_DIR"] = str(worker_dir)
|
| 25 |
env["AIOS_DUCKDB_PATH"] = str(worker_db)
|
| 26 |
+
# The launcher (request_class.py --env) sizes these to the box. Unset: the 2 vCPU values.
|
| 27 |
env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "4GB"
|
| 28 |
env["AIOS_DUCKDB_THREADS"] = env.get("AIOS_WORKER_DUCKDB_THREADS") or "1"
|
| 29 |
+
# The worker holds its own file read-write. Mirror versions are for the web workers only.
|
| 30 |
+
env["AIOS_MIRROR_VERSIONS"] = "0"
|
| 31 |
+
env["AIOS_UVICORN_WORKERS"] = "1"
|
| 32 |
return worker_db
|
| 33 |
|
| 34 |
|
|
|
|
| 63 |
process_role.start_worker_services()
|
| 64 |
# Importing the app starts every role-allowed thread at module level. Startup hooks need a call.
|
| 65 |
import main as app_main # noqa: F401
|
| 66 |
+
if getattr(app_main, "_SEED_THREAD", None) is None:
|
| 67 |
+
# No boot restore was scheduled, so no boot hand-off will come. Say so, or web readiness waits.
|
| 68 |
+
process_role.note_boot_restore("skipped")
|
| 69 |
import routes_connectors
|
| 70 |
routes_connectors._start_database_jobs()
|
| 71 |
print(f"[sync-worker] running: {', '.join(sorted(process_role.started_jobs()))}", flush=True)
|
api/test_odoo_rank_refresh.py
CHANGED
|
@@ -198,7 +198,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 198 |
# every pass would ask for a zero wait.
|
| 199 |
clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + cycle[0] * 60,
|
| 200 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 201 |
-
ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None,
|
| 202 |
_configured_mirror_entities=lambda *_: None, freeze_notice=FREEZE_OFF,
|
| 203 |
_resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False,
|
| 204 |
tenant='synthetic', tenantSource='test', tenantSeconds=60,
|
|
@@ -520,7 +520,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 520 |
|
| 521 |
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
|
| 522 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 523 |
-
ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None,
|
| 524 |
_configured_mirror_entities=lambda *_: None,
|
| 525 |
freeze_notice=SimpleNamespace(enabled=lambda: frozen),
|
| 526 |
_resolve_resync_cadence=lambda: dict(
|
|
@@ -795,7 +795,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 795 |
store = self.SlotStore(tenant, {rel.CONFIG_KEY: dict(config or {})})
|
| 796 |
clock = SimpleNamespace(sleep=wait, time=lambda: now[0],
|
| 797 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 798 |
-
ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None,
|
| 799 |
_configured_mirror_entities=lambda *_: None, freeze_notice=FREEZE_OFF,
|
| 800 |
_resolve_resync_cadence=lambda: dict(
|
| 801 |
resolvedSeconds=86400, manual=False, tenant=tenant, tenantSource='test',
|
|
|
|
| 198 |
# every pass would ask for a zero wait.
|
| 199 |
clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + cycle[0] * 60,
|
| 200 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 201 |
+
ns = dict(_restore_sync_request_desks=lambda: None, RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None,
|
| 202 |
_configured_mirror_entities=lambda *_: None, freeze_notice=FREEZE_OFF,
|
| 203 |
_resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False,
|
| 204 |
tenant='synthetic', tenantSource='test', tenantSeconds=60,
|
|
|
|
| 520 |
|
| 521 |
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
|
| 522 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 523 |
+
ns = dict(_restore_sync_request_desks=lambda: None, RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None,
|
| 524 |
_configured_mirror_entities=lambda *_: None,
|
| 525 |
freeze_notice=SimpleNamespace(enabled=lambda: frozen),
|
| 526 |
_resolve_resync_cadence=lambda: dict(
|
|
|
|
| 795 |
store = self.SlotStore(tenant, {rel.CONFIG_KEY: dict(config or {})})
|
| 796 |
clock = SimpleNamespace(sleep=wait, time=lambda: now[0],
|
| 797 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 798 |
+
ns = dict(_restore_sync_request_desks=lambda: None, RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None,
|
| 799 |
_configured_mirror_entities=lambda *_: None, freeze_notice=FREEZE_OFF,
|
| 800 |
_resolve_resync_cadence=lambda: dict(
|
| 801 |
resolvedSeconds=86400, manual=False, tenant=tenant, tenantSource='test',
|