Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- Dockerfile +15 -3
- RELEASES.json +1 -1
- VERSION +1 -1
- api/ai_enrich.py +6 -3
- api/amazon_relational.py +493 -35
- api/application_registry.py +72 -0
- api/assistant_mirror.py +240 -40
- api/connectors_amazon.py +201 -14
- api/content_studio.py +638 -0
- api/deps.py +232 -19
- api/formula_runtime.py +72 -3
- api/main.py +226 -35
- api/odoo_relational.py +491 -34
- api/routes_admin.py +6 -4
- api/routes_auth.py +14 -3
- api/routes_connected_tables.py +7 -3
- api/routes_connectors.py +337 -50
- api/routes_external.py +136 -25
- api/routes_keychain.py +62 -11
- api/routes_module_index.py +64 -5
- api/routes_nav.py +3 -3
- api/routes_odoo_tables.py +454 -47
- api/routes_platform_admin.py +57 -1
- api/routes_query.py +5 -0
- api/routes_sheet.py +881 -260
- api/routes_tables.py +1 -1
- api/routes_templates.py +132 -6
- platform/core/connector_spec.py +304 -1
- platform/core/daily_sales_repair.py +1146 -15
- platform/core/data_binding.py +64 -1
- platform/core/do_usage.py +583 -0
- platform/core/external_tools.py +55 -20
- platform/core/grid_events.py +60 -1
- platform/core/neon_usage.py +590 -0
- platform/core/perm_scope.py +74 -10
- platform/core/perms.py +11 -2
- platform/core/registry.py +10 -0
- platform/core/sheet_binding.py +60 -1
- platform/core/sheet_import.py +177 -4
- platform/core/sheet_structure.py +116 -16
- platform/core/sheets.py +104 -11
- platform/core/store_pg.py +133 -8
- platform/core/user_tables.py +67 -3
- platform/core/users.py +12 -2
- platform/harness/amazon_client.py +1057 -11
- platform/harness/amazon_schema.py +149 -8
- platform/harness/amazon_store.py +0 -0
- platform/harness/datastore.py +53 -4
- platform/harness/runtime.py +8 -0
- platform/harness/telemetry.py +7 -2
Dockerfile
CHANGED
|
@@ -98,6 +98,18 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
|
|
| 98 |
# '*' assumes the container is unreachable except through that trusted proxy, which must replace
|
| 99 |
# client-supplied forwarded headers. For any direct ingress, set FORWARDED_ALLOW_IPS to the actual
|
| 100 |
# proxy IPs/CIDRs before exposing the port; do not carry wildcard trust onto a public listener.
|
| 101 |
-
# Suppress the server banner
|
| 102 |
-
#
|
| 103 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
# '*' assumes the container is unreachable except through that trusted proxy, which must replace
|
| 99 |
# client-supplied forwarded headers. For any direct ingress, set FORWARDED_ALLOW_IPS to the actual
|
| 100 |
# proxy IPs/CIDRs before exposing the port; do not carry wildcard trust onto a public listener.
|
| 101 |
+
# Suppress the server banner. W59-T36 owns the API posture read-back and application host/header
|
| 102 |
+
# policy; these flags are its launcher seam.
|
| 103 |
+
#
|
| 104 |
+
# SDLC 67 close (2026-09-29): `--timeout-keep-alive 5` was the suspected cause of a measured
|
| 105 |
+
# 4-12% 502 rate at the HF proxy -- uvicorn closes an idle keep-alive socket after 5s, and if the
|
| 106 |
+
# proxy in front of it still believes that connection is reusable and forwards a request onto it,
|
| 107 |
+
# the client sees a 502 rather than a fresh connection (the classic reverse-proxy keep-alive race:
|
| 108 |
+
# https://adamcrowder.net/posts/node-express-api-and-aws-alb-502/ describes the same shape against
|
| 109 |
+
# an ALB). The fix is always the same direction -- make the BACKEND's idle timeout longer than
|
| 110 |
+
# whatever the front proxy holds a connection for, never shorter -- because only the side with the
|
| 111 |
+
# longer timeout can be the one to close first. HF's Spaces proxy timeout is not published; 75s is
|
| 112 |
+
# nginx's own keepalive_timeout default and comfortably exceeds the AWS ALB default (60s) other
|
| 113 |
+
# proxies in this class use, so it is the safest available floor without over-fitting to a number
|
| 114 |
+
# we cannot read. Re-aim this value again, never delete the flag, if a measurement finds otherwise.
|
| 115 |
+
CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT} --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"]
|
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "be8fb2f (be8fb2f)",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
be8fb2f (be8fb2f)
|
api/ai_enrich.py
CHANGED
|
@@ -269,12 +269,15 @@ _SYSTEM = ('You fill in ONE cell of a spreadsheet. Answer with the value only: n
|
|
| 269 |
'exactly: UNKNOWN')
|
| 270 |
|
| 271 |
|
| 272 |
-
def _ask(provider, model, prompt, max_tokens, timeout):
|
| 273 |
"""One cell's answer. `(text, tokens_or_None, problem)`; a truthy `problem` means no text.
|
| 274 |
|
| 275 |
β Both request shapes are `ai_review`'s, changed in exactly two ways: they carry the COLUMN's
|
| 276 |
ceiling instead of a hardcoded 300, and they read `usage` back.
|
|
|
|
|
|
|
| 277 |
"""
|
|
|
|
| 278 |
key = (os.environ.get(provider['env']) or '').strip()
|
| 279 |
if not key:
|
| 280 |
return '', None, f"{provider['name']} has no key on this deployment"
|
|
@@ -284,7 +287,7 @@ def _ask(provider, model, prompt, max_tokens, timeout):
|
|
| 284 |
provider['url'], timeout=timeout,
|
| 285 |
headers={'x-api-key': key, 'anthropic-version': '2023-06-01',
|
| 286 |
'content-type': 'application/json'},
|
| 287 |
-
json={'model': model, 'max_tokens': max_tokens, 'system':
|
| 288 |
'messages': [{'role': 'user', 'content': prompt}]})
|
| 289 |
if r.status_code >= 400:
|
| 290 |
return '', None, f"{provider['name']} answered {r.status_code}"
|
|
@@ -300,7 +303,7 @@ def _ask(provider, model, prompt, max_tokens, timeout):
|
|
| 300 |
provider['url'], timeout=timeout,
|
| 301 |
headers={'Authorization': f"Bearer {key}", 'Content-Type': 'application/json'},
|
| 302 |
json={'model': model, 'max_tokens': max_tokens, 'temperature': 0,
|
| 303 |
-
'messages': [{'role': 'system', 'content':
|
| 304 |
{'role': 'user', 'content': prompt}]})
|
| 305 |
if r.status_code >= 400:
|
| 306 |
return '', None, f"{provider['name']} answered {r.status_code}"
|
|
|
|
| 269 |
'exactly: UNKNOWN')
|
| 270 |
|
| 271 |
|
| 272 |
+
def _ask(provider, model, prompt, max_tokens, timeout, system=None):
|
| 273 |
"""One cell's answer. `(text, tokens_or_None, problem)`; a truthy `problem` means no text.
|
| 274 |
|
| 275 |
β Both request shapes are `ai_review`'s, changed in exactly two ways: they carry the COLUMN's
|
| 276 |
ceiling instead of a hardcoded 300, and they read `usage` back.
|
| 277 |
+
S67-T41: `system` replaces the one-cell instruction for a caller that is not filling a cell
|
| 278 |
+
(Content Studio's drafts); every existing caller passes nothing and keeps `_SYSTEM`.
|
| 279 |
"""
|
| 280 |
+
system = system or _SYSTEM
|
| 281 |
key = (os.environ.get(provider['env']) or '').strip()
|
| 282 |
if not key:
|
| 283 |
return '', None, f"{provider['name']} has no key on this deployment"
|
|
|
|
| 287 |
provider['url'], timeout=timeout,
|
| 288 |
headers={'x-api-key': key, 'anthropic-version': '2023-06-01',
|
| 289 |
'content-type': 'application/json'},
|
| 290 |
+
json={'model': model, 'max_tokens': max_tokens, 'system': system,
|
| 291 |
'messages': [{'role': 'user', 'content': prompt}]})
|
| 292 |
if r.status_code >= 400:
|
| 293 |
return '', None, f"{provider['name']} answered {r.status_code}"
|
|
|
|
| 303 |
provider['url'], timeout=timeout,
|
| 304 |
headers={'Authorization': f"Bearer {key}", 'Content-Type': 'application/json'},
|
| 305 |
json={'model': model, 'max_tokens': max_tokens, 'temperature': 0,
|
| 306 |
+
'messages': [{'role': 'system', 'content': system},
|
| 307 |
{'role': 'user', 'content': prompt}]})
|
| 308 |
if r.status_code >= 400:
|
| 309 |
return '', None, f"{provider['name']} answered {r.status_code}"
|
api/amazon_relational.py
CHANGED
|
@@ -28,10 +28,8 @@ AMAZON TABLES: order lines to their order, pricing to its listing. **Nothing her
|
|
| 28 |
removes the [[amazon-conversion-k-is-inverted]] defect class outright: no Odoo unit is ever
|
| 29 |
converted to an Amazon unit, because the two never meet.
|
| 30 |
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
standing rule 1's second half, and it is the likeliest table to hit it β an order-item history is
|
| 34 |
-
one row per line, not per order.
|
| 35 |
"""
|
| 36 |
import hashlib
|
| 37 |
import json
|
|
@@ -45,7 +43,7 @@ if str(_API.parent.parent / "platform") not in sys.path:
|
|
| 45 |
|
| 46 |
import odoo_relational as _odoo # noqa: E402
|
| 47 |
from core import connector_spec # noqa: E402
|
| 48 |
-
from harness import amazon_schema, amazon_store, datastore
|
| 49 |
|
| 50 |
Refused = _odoo.Refused
|
| 51 |
|
|
@@ -119,7 +117,7 @@ def source_attribution(key):
|
|
| 119 |
#: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
|
| 120 |
_DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
|
| 121 |
"settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
|
| 122 |
-
"report_date", "start_date", "end_date", "_time")
|
| 123 |
|
| 124 |
#: β MONEY IS `currency`, NEVER `int`. An Amazon `item-price` is fractional in every marketplace,
|
| 125 |
#: and an int column would round every row of a settlement. The grid shows what the report said and
|
|
@@ -130,6 +128,10 @@ _MONEY = {"item_price", "item_tax", "shipping_price", "shipping_tax", "gift_wrap
|
|
| 130 |
"your_price", "landed_price", "listing_price", "open_date_price",
|
| 131 |
"sales_by_asin_ordered_product_sales_amount",
|
| 132 |
"sales_by_asin_ordered_product_sales_b2_b_amount"}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
|
| 134 |
#: Percentages and rates are fractional too, but they are not money. `currency` is the only
|
| 135 |
#: fractional field type the grid offers, so they take it and the LABEL carries the meaning.
|
|
@@ -150,7 +152,7 @@ def _ftype(col, duck_type):
|
|
| 150 |
"""
|
| 151 |
if any(col == s or col.endswith(s) for s in _DATEISH):
|
| 152 |
return "date"
|
| 153 |
-
if col in _MONEY or any(col.endswith(s) for s in _RATE_SUFFIX):
|
| 154 |
return "currency"
|
| 155 |
base = str(duck_type or "").upper().split("(")[0]
|
| 156 |
if base in ("BIGINT", "INTEGER", "HUGEINT") or base.startswith("DECIMAL"):
|
|
@@ -168,6 +170,116 @@ def mirror_columns(cur, table):
|
|
| 168 |
return []
|
| 169 |
|
| 170 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
def _fields_for(cur, key):
|
| 172 |
"""One preset field per mirror column β the whole schema, derived. See the header."""
|
| 173 |
cols = mirror_columns(cur, SOURCE[key])
|
|
@@ -175,14 +287,31 @@ def _fields_for(cur, key):
|
|
| 175 |
return []
|
| 176 |
names = {c[0] for c in cols}
|
| 177 |
pinned = PINNED.get(key) if PINNED.get(key) in names else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
out = []
|
| 179 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 180 |
out.append(_odoo._preset({
|
| 181 |
"key": name, "label": _label(name), "type": _ftype(name, dtype),
|
| 182 |
"source": "overlay",
|
| 183 |
-
"default": name == pinned or name in _DEFAULT_COLUMNS,
|
| 184 |
"pinned": name == pinned,
|
| 185 |
}, flow="amazon_relational"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
# β BOTH SIDES MUST CARRY THE COLUMN, EVERY TIME. A `measured_at_sync` table's header is not
|
| 187 |
# known until it syncs, so a link to a table that has never run would name a column that does
|
| 188 |
# not exist. **Absent is a normal state here, not a failure** β which is exactly what lets the
|
|
@@ -205,7 +334,8 @@ def _fields_for(cur, key):
|
|
| 205 |
out.append(_odoo._preset({
|
| 206 |
"key": edge["key"], "label": edge["label"],
|
| 207 |
"type": "link", "source": "overlay", "default": True,
|
| 208 |
-
"link": {"table": edge["target"], "on": edge["to_col"], "from": edge["from_col"]
|
|
|
|
| 209 |
}, flow="amazon_relational"))
|
| 210 |
out.append(_odoo._preset(_odoo._refreshed_field(), flow="amazon_relational"))
|
| 211 |
return out
|
|
@@ -222,8 +352,128 @@ _DEFAULT_COLUMNS = {
|
|
| 222 |
}
|
| 223 |
|
| 224 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
def _row_id(row, key):
|
| 226 |
-
"""
|
| 227 |
|
| 228 |
β STABILITY IS THE WHOLE REQUIREMENT, and it is why a row number will not do. `_ensure_table_inplace`
|
| 229 |
merges by `_id`, so an id that changes between syncs turns every re-sync into a full delete and
|
|
@@ -232,6 +482,24 @@ def _row_id(row, key):
|
|
| 232 |
Falls back to a content hash rather than a counter for exactly that reason: a settlement line
|
| 233 |
with no id column still hashes to the same value next week if its content has not changed.
|
| 234 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
for col in IDENTITY.get(key, ()):
|
| 236 |
val = row.get(col)
|
| 237 |
if val not in (None, ""):
|
|
@@ -242,22 +510,76 @@ def _row_id(row, key):
|
|
| 242 |
|
| 243 |
|
| 244 |
def _read(cur, key):
|
| 245 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
cols = [c[0] for c in mirror_columns(cur, SOURCE[key])]
|
| 247 |
if not cols:
|
| 248 |
return []
|
| 249 |
quoted = ", ".join(f'"{c}"' for c in cols)
|
| 250 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
out = []
|
| 252 |
for r in rows:
|
| 253 |
row = {}
|
| 254 |
-
for name, val in zip(
|
| 255 |
-
|
| 256 |
-
|
|
|
|
|
|
|
| 257 |
out.append(row)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 258 |
return out
|
| 259 |
|
| 260 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 261 |
def available(tenant_key="royal-imports"):
|
| 262 |
"""Has this tenant's mirror ever been fed by `amazon_store`? Cheap; never fetches."""
|
| 263 |
try:
|
|
@@ -268,10 +590,8 @@ def available(tenant_key="royal-imports"):
|
|
| 268 |
|
| 269 |
|
| 270 |
def plan(cur, rt=None):
|
| 271 |
-
"""
|
| 272 |
built = {"problems": []}
|
| 273 |
-
ut = _odoo._ut()
|
| 274 |
-
cap = getattr(ut, "MAX_ROWS", 60_000)
|
| 275 |
for key in SOURCE:
|
| 276 |
# β COUNT BEFORE READ. `_read` materialises the whole table as python dicts, so checking
|
| 277 |
# the cap afterwards means the one table too big to keep is the one built in memory first.
|
|
@@ -279,17 +599,8 @@ def plan(cur, rt=None):
|
|
| 279 |
n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 280 |
except Exception: # noqa: BLE001
|
| 281 |
continue # absent = never synced
|
| 282 |
-
if
|
| 283 |
-
|
| 284 |
-
# half, and `ut_amzn_order_items` is the table most likely to reach it: an order
|
| 285 |
-
# history is one row per LINE, so a mid-size seller passes 60,000 in a year.
|
| 286 |
-
built["problems"].append(
|
| 287 |
-
f"{key}: the mirror holds {n:,} rows against MAX_ROWS={cap:,}. Nothing was "
|
| 288 |
-
f"truncated and nothing was written for this table. Cause: this seller's history "
|
| 289 |
-
f"is larger than a materialised ut_* table can hold. Fix: bind it read-through "
|
| 290 |
-
f"off the mirror as `ut_odoo_gl_lines` is "
|
| 291 |
-
f"(odoo_relational.READ_THROUGH_KEYS), which `routes_connected_tables` "
|
| 292 |
-
f"already serves.")
|
| 293 |
continue
|
| 294 |
rows = _read(cur, key)
|
| 295 |
if rows:
|
|
@@ -306,6 +617,9 @@ def plan_fingerprint(cur, built):
|
|
| 306 |
"""
|
| 307 |
tables = []
|
| 308 |
for key in SOURCE:
|
|
|
|
|
|
|
|
|
|
| 309 |
rows = built.get(key) or []
|
| 310 |
canonical = sorted(rows, key=lambda row: json.dumps(
|
| 311 |
row, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str))
|
|
@@ -315,6 +629,62 @@ def plan_fingerprint(cur, built):
|
|
| 315 |
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
| 316 |
|
| 317 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
def apply_plan(rt, built, username="amazon", today=None):
|
| 319 |
"""Create-or-merge every table in `built`, in ONE `flush="sync"` write.
|
| 320 |
|
|
@@ -327,24 +697,103 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 327 |
raise Refused("; ".join(built["problems"]))
|
| 328 |
stamp = today or _odoo._iso_today()
|
| 329 |
written = {}
|
| 330 |
-
cur_mirror = datastore.
|
| 331 |
-
|
| 332 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 333 |
|
| 334 |
def _apply_all(doc):
|
| 335 |
doc = doc if isinstance(doc, dict) else {}
|
|
|
|
| 336 |
for key, label, fields, rows in plans:
|
| 337 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
username, stamp)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 339 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 340 |
# mutable display label or from the process having imported this module.
|
| 341 |
doc[key]["connector"] = source_attribution(key)
|
| 342 |
return doc
|
| 343 |
|
| 344 |
rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
|
|
|
|
| 345 |
return written
|
| 346 |
|
| 347 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 348 |
def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, log=print):
|
| 349 |
"""Open, plan and conditionally apply the Amazon mirror.
|
| 350 |
|
|
@@ -357,7 +806,14 @@ def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, lo
|
|
| 357 |
path = datastore.path_for(tenant)
|
| 358 |
if Path(datastore.DB_PATH) != Path(path):
|
| 359 |
datastore.use_path(path)
|
| 360 |
-
cur = datastore.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 361 |
if not any(mirror_columns(cur, t) for t in SOURCE.values()):
|
| 362 |
log(f"[amazon_relational] {why}: no Amazon tables in {tenant}'s mirror, nothing to spawn "
|
| 363 |
f"(run `python platform/harness/amazon_store.py --sync --tenant {tenant}` first)")
|
|
@@ -381,6 +837,8 @@ def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, lo
|
|
| 381 |
state["changed"] = True
|
| 382 |
state["written"] = apply_plan(rt, applicable)
|
| 383 |
state["applied"] = True
|
|
|
|
|
|
|
| 384 |
for key, counts in state["written"].items():
|
| 385 |
log(f"[amazon_relational] {why}: {key:<28} {counts}")
|
| 386 |
return state
|
|
|
|
| 28 |
removes the [[amazon-conversion-k-is-inverted]] defect class outright: no Odoo unit is ever
|
| 29 |
converted to an Amazon unit, because the two never meet.
|
| 30 |
|
| 31 |
+
History stays in the mirror. Small snapshots may be materialised; a row or byte budget switches
|
| 32 |
+
them to read-through too. The shared document holds their schemas, never a truncated population.
|
|
|
|
|
|
|
| 33 |
"""
|
| 34 |
import hashlib
|
| 35 |
import json
|
|
|
|
| 43 |
|
| 44 |
import odoo_relational as _odoo # noqa: E402
|
| 45 |
from core import connector_spec # noqa: E402
|
| 46 |
+
from harness import amazon_client, amazon_schema, amazon_store, datastore # noqa: E402
|
| 47 |
|
| 48 |
Refused = _odoo.Refused
|
| 49 |
|
|
|
|
| 117 |
#: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
|
| 118 |
_DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
|
| 119 |
"settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
|
| 120 |
+
"report_date", "start_date", "end_date", "_snapshot_date", "month", "_time")
|
| 121 |
|
| 122 |
#: β MONEY IS `currency`, NEVER `int`. An Amazon `item-price` is fractional in every marketplace,
|
| 123 |
#: and an int column would round every row of a settlement. The grid shows what the report said and
|
|
|
|
| 128 |
"your_price", "landed_price", "listing_price", "open_date_price",
|
| 129 |
"sales_by_asin_ordered_product_sales_amount",
|
| 130 |
"sales_by_asin_ordered_product_sales_b2_b_amount"}
|
| 131 |
+
_MONEY.update({"estimated_fee_total", "estimated_referral_fee_per_unit",
|
| 132 |
+
"estimated_variable_closing_fee", "estimated_order_handling_fee_per_order",
|
| 133 |
+
"estimated_pick_pack_fee_per_unit", "estimated_weight_handling_fee_per_unit",
|
| 134 |
+
"expected_fulfillment_fee_per_unit", "estimated_monthly_storage_fee"})
|
| 135 |
|
| 136 |
#: Percentages and rates are fractional too, but they are not money. `currency` is the only
|
| 137 |
#: fractional field type the grid offers, so they take it and the LABEL carries the meaning.
|
|
|
|
| 152 |
"""
|
| 153 |
if any(col == s or col.endswith(s) for s in _DATEISH):
|
| 154 |
return "date"
|
| 155 |
+
if col in _MONEY or col.endswith(("_amount", "_fee")) or any(col.endswith(s) for s in _RATE_SUFFIX):
|
| 156 |
return "currency"
|
| 157 |
base = str(duck_type or "").upper().split("(")[0]
|
| 158 |
if base in ("BIGINT", "INTEGER", "HUGEINT") or base.startswith("DECIMAL"):
|
|
|
|
| 170 |
return []
|
| 171 |
|
| 172 |
|
| 173 |
+
#: ββ S67-T30 (instr 13, R29 / O23) β THE PRODUCT LOOKUPS. Every Amazon database with an ASIN
|
| 174 |
+
#: shows its "Parent ASIN" and "SKU" beside it, and every one with a SKU its "ASIN" and "Parent
|
| 175 |
+
#: ASIN", with no configuration. They are PLAIN TEXT fields whose value the SERVER derives, not
|
| 176 |
+
#: link fields, on purpose: a link renders as a count bubble, is not groupable
|
| 177 |
+
#: (`types.ts:isGroupableField`) and has no SQL binding for a filter (`routes_odoo_tables.
|
| 178 |
+
#: _filter_columns` and `_group_pin` read `spec["cols"]` only). A text field with a `cols`
|
| 179 |
+
#: expression groups, filters and sorts through the machinery every other column already uses.
|
| 180 |
+
#: β The ASIN / SKU column of a table is the `from_col` of its declared listings link (on `asin1`
|
| 181 |
+
#: / `seller_sku`), so the spec's links are the one statement of which column is which. A table
|
| 182 |
+
#: that carries its own `parent_asin` (pricing, economics) gains no derived Parent ASIN; one that
|
| 183 |
+
#: carries its own SKU or ASIN gains no duplicate of it; listings gains Parent ASIN only.
|
| 184 |
+
#: β Parent comes from `amazon_store.asin_parents_sql` ALONE (latest catalog, else economics, else
|
| 185 |
+
#: the ASIN itself), never from `amzn_catalog_items` directly: that table keeps older snapshot days
|
| 186 |
+
#: (D-3031) and five own ASINs have no catalog row. SKU and ASIN come from the LATEST listings day
|
| 187 |
+
#: only, because listings keeps one copy per snapshot date.
|
| 188 |
+
LISTINGS = PREFIX + "listings"
|
| 189 |
+
LOOKUP_PARENT, LOOKUP_SKU, LOOKUP_ASIN = "lookup_parent_asin", "lookup_sku", "lookup_asin"
|
| 190 |
+
LOOKUP_KEYS = frozenset((LOOKUP_PARENT, LOOKUP_SKU, LOOKUP_ASIN))
|
| 191 |
+
_LOOKUP_LABELS = {LOOKUP_PARENT: "Parent ASIN", LOOKUP_SKU: "SKU", LOOKUP_ASIN: "ASIN"}
|
| 192 |
+
_LOOKUP_DESCRIPTIONS = {
|
| 193 |
+
LOOKUP_PARENT: ("The parent ASIN of this row's product, from Amazon's catalog, else the "
|
| 194 |
+
"economics report, else the ASIN itself. Click it to open the listing."),
|
| 195 |
+
LOOKUP_SKU: ("Your seller SKUs for this row's ASIN in the latest listings report. "
|
| 196 |
+
"Click it to open the listing."),
|
| 197 |
+
LOOKUP_ASIN: ("The ASIN of this row's SKU in the latest listings report. "
|
| 198 |
+
"Click it to open the listing."),
|
| 199 |
+
}
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def _product_columns(key):
|
| 203 |
+
"""`(asin_col, sku_col, asin_link_key, sku_link_key)` as the spec declares them for `key`."""
|
| 204 |
+
if key == LISTINGS:
|
| 205 |
+
return "asin1", "seller_sku", None, None # the row IS the listing: Parent ASIN only
|
| 206 |
+
asin = sku = asin_via = sku_via = None
|
| 207 |
+
for edge in connector_spec.edges_for(key):
|
| 208 |
+
if edge["target"] != LISTINGS:
|
| 209 |
+
continue
|
| 210 |
+
if edge["to_col"] == "asin1" and asin is None:
|
| 211 |
+
asin, asin_via = edge["from_col"], edge["key"]
|
| 212 |
+
elif edge["to_col"] == "seller_sku" and sku is None:
|
| 213 |
+
sku, sku_via = edge["from_col"], edge["key"]
|
| 214 |
+
return asin, sku, asin_via, sku_via
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def _latest_listings_sql(listing_names):
|
| 218 |
+
"""`SELECT sku, asin` over the LATEST listings snapshot day, or None without both columns."""
|
| 219 |
+
if not {"seller_sku", "asin1"} <= set(listing_names):
|
| 220 |
+
return None
|
| 221 |
+
table = SOURCE[LISTINGS]
|
| 222 |
+
latest = (f' AND "_snapshot_date" = (SELECT MAX("_snapshot_date") FROM {table})'
|
| 223 |
+
if "_snapshot_date" in listing_names else "")
|
| 224 |
+
return ("SELECT DISTINCT sku, asin FROM (SELECT TRIM(CAST(seller_sku AS VARCHAR)) AS sku, "
|
| 225 |
+
f"TRIM(CAST(asin1 AS VARCHAR)) AS asin FROM {table} "
|
| 226 |
+
f"WHERE seller_sku IS NOT NULL AND asin1 IS NOT NULL{latest}) "
|
| 227 |
+
"WHERE sku <> '' AND asin <> ''")
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def lookups(cur, key, names=None):
|
| 231 |
+
"""`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains.
|
| 232 |
+
|
| 233 |
+
`expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name
|
| 234 |
+
(every reader selects `FROM <table>` unaliased), so it binds to the outer row and never to the
|
| 235 |
+
resolver's own `asin`. DuckDB decorrelates it into a single hash join per query, which is what
|
| 236 |
+
keeps a grouped page of a read-through grid set-based rather than per-row. No match is NULL,
|
| 237 |
+
which every reader serialises as a blank cell. Nothing is capped (standing rule 1).
|
| 238 |
+
"""
|
| 239 |
+
if names is None:
|
| 240 |
+
names = {name for name, _dtype in mirror_columns(cur, SOURCE[key])}
|
| 241 |
+
asin, sku, asin_via, sku_via = _product_columns(key)
|
| 242 |
+
asin = asin if asin in names else None
|
| 243 |
+
sku = sku if sku in names else None
|
| 244 |
+
if not asin and not sku:
|
| 245 |
+
return []
|
| 246 |
+
listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])}
|
| 247 |
+
parents = None if "parent_asin" in names else amazon_store.asin_parents_sql(cur)
|
| 248 |
+
latest = _latest_listings_sql(listing_names)
|
| 249 |
+
table = SOURCE[key]
|
| 250 |
+
out = []
|
| 251 |
+
|
| 252 |
+
def add(lookup_key, after, expr, via):
|
| 253 |
+
if lookup_key in names:
|
| 254 |
+
return
|
| 255 |
+
out.append({"key": lookup_key, "label": _LOOKUP_LABELS[lookup_key], "after": after,
|
| 256 |
+
"expr": expr, "opens": {"via": via} if via else None,
|
| 257 |
+
"description": _LOOKUP_DESCRIPTIONS[lookup_key]})
|
| 258 |
+
|
| 259 |
+
if asin:
|
| 260 |
+
ref = f"TRIM(CAST({table}.{_quoted(asin)} AS VARCHAR))"
|
| 261 |
+
via = asin_via if "asin1" in listing_names else None
|
| 262 |
+
if parents:
|
| 263 |
+
add(LOOKUP_PARENT, asin,
|
| 264 |
+
f"(SELECT _p.parent_asin FROM ({parents}) AS _p WHERE _p.asin = {ref})", via)
|
| 265 |
+
if latest and not sku:
|
| 266 |
+
add(LOOKUP_SKU, asin,
|
| 267 |
+
f"(SELECT string_agg(DISTINCT _l.sku, ', ' ORDER BY _l.sku) "
|
| 268 |
+
f"FROM ({latest}) AS _l WHERE _l.asin = {ref})", via)
|
| 269 |
+
elif sku and latest:
|
| 270 |
+
ref = f"TRIM(CAST({table}.{_quoted(sku)} AS VARCHAR))"
|
| 271 |
+
via = sku_via if "seller_sku" in listing_names else None
|
| 272 |
+
add(LOOKUP_ASIN, sku,
|
| 273 |
+
f"(SELECT string_agg(DISTINCT _l.asin, ', ' ORDER BY _l.asin) "
|
| 274 |
+
f"FROM ({latest}) AS _l WHERE _l.sku = {ref})", via)
|
| 275 |
+
if parents:
|
| 276 |
+
add(LOOKUP_PARENT, sku,
|
| 277 |
+
f"(SELECT string_agg(DISTINCT _p.parent_asin, ', ' ORDER BY _p.parent_asin) "
|
| 278 |
+
f"FROM ({latest}) AS _l JOIN ({parents}) AS _p ON _p.asin = _l.asin "
|
| 279 |
+
f"WHERE _l.sku = {ref})", via)
|
| 280 |
+
return out
|
| 281 |
+
|
| 282 |
+
|
| 283 |
def _fields_for(cur, key):
|
| 284 |
"""One preset field per mirror column β the whole schema, derived. See the header."""
|
| 285 |
cols = mirror_columns(cur, SOURCE[key])
|
|
|
|
| 287 |
return []
|
| 288 |
names = {c[0] for c in cols}
|
| 289 |
pinned = PINNED.get(key) if PINNED.get(key) in names else None
|
| 290 |
+
# S67-T30: each lookup lands IMMEDIATELY after the column it derives from, shown by default,
|
| 291 |
+
# and the ASIN / SKU column it sits beside is shown by default too (inventory's `msku` was not).
|
| 292 |
+
derived = {}
|
| 293 |
+
for lookup in lookups(cur, key, names):
|
| 294 |
+
derived.setdefault(lookup["after"], []).append(lookup)
|
| 295 |
+
product = {c for c in _product_columns(key)[:2] if c in names}
|
| 296 |
out = []
|
| 297 |
+
# The identity column goes FIRST, the rest keep the report's own order. A record reached
|
| 298 |
+
# through a link is titled by its first field (the record drawer's frame label), and a
|
| 299 |
+
# report's first column is whatever Amazon put there (`item_name`, `_data_origin`), so a
|
| 300 |
+
# listing opened from an order must lead with its seller SKU, as Odoo's pinned fields do.
|
| 301 |
+
for name, dtype in sorted(cols, key=lambda col: col[0] != pinned):
|
| 302 |
out.append(_odoo._preset({
|
| 303 |
"key": name, "label": _label(name), "type": _ftype(name, dtype),
|
| 304 |
"source": "overlay",
|
| 305 |
+
"default": name == pinned or name in _DEFAULT_COLUMNS or name in product,
|
| 306 |
"pinned": name == pinned,
|
| 307 |
}, flow="amazon_relational"))
|
| 308 |
+
for lookup in derived.get(name, ()):
|
| 309 |
+
field = {"key": lookup["key"], "label": lookup["label"], "type": "text",
|
| 310 |
+
"source": "overlay", "default": True,
|
| 311 |
+
"description": lookup["description"]}
|
| 312 |
+
if lookup["opens"]:
|
| 313 |
+
field["opens"] = dict(lookup["opens"])
|
| 314 |
+
out.append(_odoo._preset(field, flow="amazon_relational"))
|
| 315 |
# β BOTH SIDES MUST CARRY THE COLUMN, EVERY TIME. A `measured_at_sync` table's header is not
|
| 316 |
# known until it syncs, so a link to a table that has never run would name a column that does
|
| 317 |
# not exist. **Absent is a normal state here, not a failure** β which is exactly what lets the
|
|
|
|
| 334 |
out.append(_odoo._preset({
|
| 335 |
"key": edge["key"], "label": edge["label"],
|
| 336 |
"type": "link", "source": "overlay", "default": True,
|
| 337 |
+
"link": {"table": edge["target"], "on": edge["to_col"], "from": edge["from_col"],
|
| 338 |
+
"single": edge["cardinality"] == "many_to_one"},
|
| 339 |
}, flow="amazon_relational"))
|
| 340 |
out.append(_odoo._preset(_odoo._refreshed_field(), flow="amazon_relational"))
|
| 341 |
return out
|
|
|
|
| 352 |
}
|
| 353 |
|
| 354 |
|
| 355 |
+
#: The original seven used first-field identities. Retained for legacy-id migration
|
| 356 |
+
#: and for distinguishing history from the original small snapshots.
|
| 357 |
+
_LEGACY_IDENTITY_TABLES = frozenset(("orders", "order_items", "listings", "settlements",
|
| 358 |
+
"inventory", "pricing", "brand_analytics"))
|
| 359 |
+
|
| 360 |
+
# The original seven do not declare snapshot/window metadata. Keep their four small
|
| 361 |
+
# tables materialised; every newer non-snapshot report/API is history, including monthly ones.
|
| 362 |
+
READ_THROUGH = frozenset(
|
| 363 |
+
PREFIX + table for table, spec in amazon_client.REPORT_TYPES.items()
|
| 364 |
+
if table in {"settlements", "orders", "order_items", "transactions", "economics", "storage_fees"}
|
| 365 |
+
or (table not in _LEGACY_IDENTITY_TABLES and not spec.get("snapshot")
|
| 366 |
+
and (spec.get("report") or spec.get("api") or spec.get("monthly"))))
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
def _quoted(name):
|
| 370 |
+
return '"' + name.replace('"', '""') + '"'
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
#: ββ HOTFIX 2026-09-27 (D-3284's cost): an Amazon snapshot table larger than this is served
|
| 374 |
+
#: READ-THROUGH, never stored in the tenant document. The per-table `MAX_TABLE_BYTES` (48 MB) let
|
| 375 |
+
#: catalog items (32.9 MB for 2,317 rows), pricing (11.7 MB) and FBA inventory (7.5 MB) in once
|
| 376 |
+
#: connector databases stopped counting toward MAX_TABLES, and `user_tables` doubled to 118 MB.
|
| 377 |
+
#: Every whole-document read and write doubled with it: the 2 GB DigitalOcean staging node ran
|
| 378 |
+
#: out of memory and PostgreSQL restarted (2026-09-27 16:36Z), and on Neon each read is billed at
|
| 379 |
+
#: the document's text size. Small tables stay stored; nothing is capped (standing rule 1).
|
| 380 |
+
AMAZON_STORED_BYTES = 4 * 1024 * 1024
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def _read_through(cur, key, count):
|
| 384 |
+
"""Choose storage before reading rows; estimate wide snapshots with a bounded sample."""
|
| 385 |
+
ut = _odoo._ut()
|
| 386 |
+
if key in READ_THROUGH or count > ut.MAX_ROWS:
|
| 387 |
+
return True
|
| 388 |
+
if not count:
|
| 389 |
+
return False
|
| 390 |
+
names = [name for name, _dtype in mirror_columns(cur, SOURCE[key])]
|
| 391 |
+
sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
|
| 392 |
+
# Include the stored id, refreshed stamp and row-object punctuation, with the same
|
| 393 |
+
# ASCII escaping as the document writer. The largest sampled row is conservative.
|
| 394 |
+
size = max((len(json.dumps(dict(zip(names, row)), ensure_ascii=True,
|
| 395 |
+
default=str).encode("utf-8")) + 96 for row in sample), default=0)
|
| 396 |
+
return count * size > min(ut.MAX_TABLE_BYTES, AMAZON_STORED_BYTES)
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
def missing_source(key):
|
| 400 |
+
return {"missing": (
|
| 401 |
+
f"The Amazon mirror in this container has not loaded Amazon "
|
| 402 |
+
f"{key.removeprefix(PREFIX).replace('_', ' ')} yet. "
|
| 403 |
+
"The next Amazon sync loads it. Nothing is lost.")}
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def _text(value):
|
| 407 |
+
return "" if value is None else str(value)
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
def _number(value):
|
| 411 |
+
return None if value in (None, "") else float(value)
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
def _integer(value):
|
| 415 |
+
return None if value in (None, "") else int(value)
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
def grid_source(cur, key):
|
| 419 |
+
"""One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
|
| 420 |
+
columns = mirror_columns(cur, SOURCE[key])
|
| 421 |
+
if not columns:
|
| 422 |
+
return missing_source(key)
|
| 423 |
+
names = {name for name, _dtype in columns}
|
| 424 |
+
identity = [name for name in IDENTITY[key] if name in names]
|
| 425 |
+
coordinates = ["_report_document_id", "_report_row_number"]
|
| 426 |
+
if all(name in names for name in coordinates):
|
| 427 |
+
identity = list(dict.fromkeys(identity + coordinates))
|
| 428 |
+
if (not identity or identity == ["_report_row_number"]
|
| 429 |
+
or (key in {PREFIX + "settlements", PREFIX + "order_items"}
|
| 430 |
+
and not all(name in names for name in coordinates))):
|
| 431 |
+
identity = sorted(names)
|
| 432 |
+
# Report coordinates qualify line numbers by their document. If the declared identity
|
| 433 |
+
# is incomplete/null, hash the whole row instead of collapsing several unknown lines.
|
| 434 |
+
def digest(keys):
|
| 435 |
+
values = ", ".join(f"CAST({_quoted(name)} AS VARCHAR)" for name in keys)
|
| 436 |
+
return f"sha256(to_json(list_value({values})))"
|
| 437 |
+
complete = " AND ".join(f"NULLIF(CAST({_quoted(name)} AS VARCHAR), '') IS NOT NULL"
|
| 438 |
+
for name in identity)
|
| 439 |
+
digest_sql = f"CASE WHEN {complete} THEN {digest(identity)} ELSE {digest(sorted(names))} END"
|
| 440 |
+
# The shared grid, relation and selection contracts all use numeric pids. Thirteen
|
| 441 |
+
# hex digits retain a stable 52-bit identity that JavaScript represents exactly.
|
| 442 |
+
ident = f"greatest(1, CAST(('0x' || substr(({digest_sql}), 1, 13)) AS BIGINT))"
|
| 443 |
+
cols = {}
|
| 444 |
+
for name, dtype in columns:
|
| 445 |
+
kind = _ftype(name, dtype)
|
| 446 |
+
expr = _quoted(name)
|
| 447 |
+
if kind in ("currency", "int"):
|
| 448 |
+
expr = f"TRY_CAST({expr} AS {'DOUBLE' if kind == 'currency' else 'BIGINT'})"
|
| 449 |
+
cols[name] = (expr, {"currency": _number, "int": _integer}.get(kind, _text))
|
| 450 |
+
# S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a
|
| 451 |
+
# group (count by parent) and a sort compile against them in SQL with no per-row Python.
|
| 452 |
+
for lookup in lookups(cur, key, names):
|
| 453 |
+
cols[lookup["key"]] = (lookup["expr"], _text)
|
| 454 |
+
return {"table": SOURCE[key], "where": "", "id": ident,
|
| 455 |
+
"needs_excluded": False, "cols": cols}
|
| 456 |
+
|
| 457 |
+
|
| 458 |
+
def grid_sources():
|
| 459 |
+
"""Only Amazon read-through grids; no keys at all on an Odoo-only mirror."""
|
| 460 |
+
cur = datastore.ro_cursor()
|
| 461 |
+
try:
|
| 462 |
+
present = {key for key, table in SOURCE.items() if mirror_columns(cur, table)}
|
| 463 |
+
if not present:
|
| 464 |
+
return {}
|
| 465 |
+
keys = set(READ_THROUGH)
|
| 466 |
+
for key in present - keys:
|
| 467 |
+
count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 468 |
+
if _read_through(cur, key, count):
|
| 469 |
+
keys.add(key)
|
| 470 |
+
return {key: grid_source(cur, key) for key in SOURCE if key in keys}
|
| 471 |
+
finally:
|
| 472 |
+
cur.close()
|
| 473 |
+
|
| 474 |
+
|
| 475 |
def _row_id(row, key):
|
| 476 |
+
"""The legacy stored identity, retained to migrate existing snapshot rows.
|
| 477 |
|
| 478 |
β STABILITY IS THE WHOLE REQUIREMENT, and it is why a row number will not do. `_ensure_table_inplace`
|
| 479 |
merges by `_id`, so an id that changes between syncs turns every re-sync into a full delete and
|
|
|
|
| 482 |
Falls back to a content hash rather than a counter for exactly that reason: a settlement line
|
| 483 |
with no id column still hashes to the same value next week if its content has not changed.
|
| 484 |
"""
|
| 485 |
+
if key.removeprefix(PREFIX) not in _LEGACY_IDENTITY_TABLES:
|
| 486 |
+
# These tables retain multiple days/items per SKU/transaction. First-field identity
|
| 487 |
+
# would collapse them in apply_plan even though all rows survived the raw mirror.
|
| 488 |
+
# S66-T40 (lead, observed live): a report whose header spells an identity column
|
| 489 |
+
# differently (the referral fee preview says `seller_sku`, not `sku`) made
|
| 490 |
+
# `row_identity_for` raise, and because `apply_plan` is ONE write for every Amazon table,
|
| 491 |
+
# one such row cost all of them. A row missing a component falls through to the full
|
| 492 |
+
# content hash below, which is still stable and never merges two different rows.
|
| 493 |
+
try:
|
| 494 |
+
identity = amazon_schema.row_identity_for(key.removeprefix(PREFIX), row)
|
| 495 |
+
except ValueError:
|
| 496 |
+
identity = None
|
| 497 |
+
if identity is not None:
|
| 498 |
+
return hashlib.sha256(json.dumps(identity, ensure_ascii=True,
|
| 499 |
+
default=str).encode()).hexdigest()[:32]
|
| 500 |
+
payload = json.dumps({k: row.get(k) for k in sorted(row)}, sort_keys=True,
|
| 501 |
+
ensure_ascii=True, separators=(",", ":"), default=str)
|
| 502 |
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
|
| 503 |
for col in IDENTITY.get(key, ()):
|
| 504 |
val = row.get(col)
|
| 505 |
if val not in (None, ""):
|
|
|
|
| 510 |
|
| 511 |
|
| 512 |
def _read(cur, key):
|
| 513 |
+
"""Small snapshots use the same numeric identity as their read-through grid.
|
| 514 |
+
|
| 515 |
+
A SKU or hex digest used as the stored row key is silently skipped by
|
| 516 |
+
perm_scope._read_user_table, and cannot be opened by the record drawer.
|
| 517 |
+
"""
|
| 518 |
cols = [c[0] for c in mirror_columns(cur, SOURCE[key])]
|
| 519 |
if not cols:
|
| 520 |
return []
|
| 521 |
quoted = ", ".join(f'"{c}"' for c in cols)
|
| 522 |
+
spec = grid_source(cur, key)
|
| 523 |
+
# S67-T30: a stored snapshot carries the same product lookups, computed by the SAME SQL
|
| 524 |
+
# expressions the read-through binding uses, in this one SELECT (never per row).
|
| 525 |
+
derived = [(lookup["key"], lookup["expr"]) for lookup in lookups(cur, key)]
|
| 526 |
+
extra = "".join(f", {expr}" for _k, expr in derived)
|
| 527 |
+
rows = cur.execute(f'SELECT {spec["id"]}, {quoted}{extra} FROM {SOURCE[key]}').fetchall()
|
| 528 |
+
names = cols + [k for k, _expr in derived]
|
| 529 |
out = []
|
| 530 |
for r in rows:
|
| 531 |
row = {}
|
| 532 |
+
for name, val in zip(names, r[1:]):
|
| 533 |
+
# Postgres JSON cannot hold U+0000, and one such byte refused the whole Amazon write.
|
| 534 |
+
row[name] = "" if val is None else (val if isinstance(val, (int, float))
|
| 535 |
+
else str(val).replace("\x00", ""))
|
| 536 |
+
row["_id"] = str(r[0])
|
| 537 |
out.append(row)
|
| 538 |
+
def read_targets(target, column, values):
|
| 539 |
+
target_spec = grid_source(cur, target)
|
| 540 |
+
expr = _quoted(column)
|
| 541 |
+
found = cur.execute(
|
| 542 |
+
f'SELECT {target_spec["id"]}, {expr} FROM {SOURCE[target]} '
|
| 543 |
+
f'WHERE {expr} IN ({",".join("?" for _ in values)})', list(values)).fetchall()
|
| 544 |
+
return [{"pid": pid, column: value} for pid, value in found]
|
| 545 |
+
|
| 546 |
+
fill_link_cells(out, _fields_for(cur, key), read_targets)
|
| 547 |
+
return out
|
| 548 |
+
|
| 549 |
+
|
| 550 |
+
def link_edges(fields):
|
| 551 |
+
"""`[(field, target, from, on)]` for every derived Amazon link `fill_link_cells` fills.
|
| 552 |
+
|
| 553 |
+
HOTFIX 2026-09-28: the ONE predicate, so the HTTP reader can see up front every column it
|
| 554 |
+
will be asked for on a target and read a STORED target once rather than once per edge."""
|
| 555 |
+
out = []
|
| 556 |
+
for field in fields:
|
| 557 |
+
bag = field.get("link") or {}
|
| 558 |
+
target, frm, on = bag.get("table"), bag.get("from"), bag.get("on")
|
| 559 |
+
if (field.get("type") != "link" or target not in SOURCE or not frm or not on
|
| 560 |
+
or bag.get("inverse") or bag.get("through") or bag.get("selectionMode") == "manual"):
|
| 561 |
+
continue
|
| 562 |
+
out.append((field, target, frm, on))
|
| 563 |
return out
|
| 564 |
|
| 565 |
|
| 566 |
+
def fill_link_cells(rows, fields, read_targets):
|
| 567 |
+
"""Populate derived Amazon links with target pids, with one read per edge.
|
| 568 |
+
|
| 569 |
+
The snapshot planner reads its tenant mirror; the HTTP reader supplies a scoped
|
| 570 |
+
target reader. Neither path stores history rows or caps the matching target set.
|
| 571 |
+
"""
|
| 572 |
+
for field, target, frm, on in link_edges(fields):
|
| 573 |
+
values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
|
| 574 |
+
matches = {}
|
| 575 |
+
for row in read_targets(target, on, values) if values else ():
|
| 576 |
+
if row.get(on) not in (None, ""):
|
| 577 |
+
matches.setdefault(str(row[on]), set()).add(int(row["pid"]))
|
| 578 |
+
for row in rows:
|
| 579 |
+
row[field["key"]] = ",".join(str(pid) for pid in sorted(
|
| 580 |
+
matches.get(str(row.get(frm)), ())))
|
| 581 |
+
|
| 582 |
+
|
| 583 |
def available(tenant_key="royal-imports"):
|
| 584 |
"""Has this tenant's mirror ever been fed by `amazon_store`? Cheap; never fetches."""
|
| 585 |
try:
|
|
|
|
| 590 |
|
| 591 |
|
| 592 |
def plan(cur, rt=None):
|
| 593 |
+
"""Rows for small snapshots, schema/count stubs for history or oversized tables."""
|
| 594 |
built = {"problems": []}
|
|
|
|
|
|
|
| 595 |
for key in SOURCE:
|
| 596 |
# β COUNT BEFORE READ. `_read` materialises the whole table as python dicts, so checking
|
| 597 |
# the cap afterwards means the one table too big to keep is the one built in memory first.
|
|
|
|
| 599 |
n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 600 |
except Exception: # noqa: BLE001
|
| 601 |
continue # absent = never synced
|
| 602 |
+
if _read_through(cur, key, n):
|
| 603 |
+
built[key] = {"readThrough": True, "fields": _fields_for(cur, key), "count": n}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 604 |
continue
|
| 605 |
rows = _read(cur, key)
|
| 606 |
if rows:
|
|
|
|
| 617 |
"""
|
| 618 |
tables = []
|
| 619 |
for key in SOURCE:
|
| 620 |
+
if isinstance(built.get(key), dict) and built[key].get("readThrough"):
|
| 621 |
+
tables.append({"key": key, **built[key]})
|
| 622 |
+
continue
|
| 623 |
rows = built.get(key) or []
|
| 624 |
canonical = sorted(rows, key=lambda row: json.dumps(
|
| 625 |
row, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str))
|
|
|
|
| 629 |
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
| 630 |
|
| 631 |
|
| 632 |
+
def _place_lookups(stored, fields):
|
| 633 |
+
"""An EXISTING table's fields with each NEW product lookup placed right after its source.
|
| 634 |
+
|
| 635 |
+
`_ensure_table_inplace` only appends, so a lookup first planned for a table stored before
|
| 636 |
+
S67-T30 would land at the far end of the grid instead of beside the ASIN or SKU it derives
|
| 637 |
+
from. A lookup already stored keeps its place (a person may have moved it) and only has its
|
| 638 |
+
`opens` bag brought up to date, which `_ensure_table_inplace` does not forward-migrate.
|
| 639 |
+
"""
|
| 640 |
+
out = list(stored)
|
| 641 |
+
have = {f.get("key"): i for i, f in enumerate(out)}
|
| 642 |
+
anchor = None
|
| 643 |
+
for field in fields:
|
| 644 |
+
fkey = field.get("key")
|
| 645 |
+
if fkey not in LOOKUP_KEYS:
|
| 646 |
+
anchor = fkey
|
| 647 |
+
continue
|
| 648 |
+
if fkey in have:
|
| 649 |
+
old = out[have[fkey]]
|
| 650 |
+
if (old.get("automation") or {}).get("flowId") == "amazon_relational" \
|
| 651 |
+
and not _odoo._ut().user_edited(old):
|
| 652 |
+
if field.get("opens"):
|
| 653 |
+
old["opens"] = dict(field["opens"])
|
| 654 |
+
else:
|
| 655 |
+
old.pop("opens", None)
|
| 656 |
+
continue
|
| 657 |
+
keys = [f.get("key") for f in out]
|
| 658 |
+
if anchor not in keys:
|
| 659 |
+
continue # no source column stored yet: the append places it
|
| 660 |
+
at = keys.index(anchor) + 1
|
| 661 |
+
while at < len(keys) and keys[at] in LOOKUP_KEYS:
|
| 662 |
+
at += 1 # after any lookup already beside the same source
|
| 663 |
+
out.insert(at, dict(field))
|
| 664 |
+
have = {f.get("key"): i for i, f in enumerate(out)}
|
| 665 |
+
anchor = fkey
|
| 666 |
+
return out
|
| 667 |
+
|
| 668 |
+
|
| 669 |
+
def _lead_with_identity(stored, report, pinned):
|
| 670 |
+
"""An EXISTING table's fields with its identity column moved first, while the order is still
|
| 671 |
+
the report's own.
|
| 672 |
+
|
| 673 |
+
`_ensure_table_inplace` keeps a stored table's field order and only appends, so the
|
| 674 |
+
identity-first order `_fields_for` gives a NEW table never reaches one stored before it. The
|
| 675 |
+
move happens only when the stored mirror columns sit exactly in the report's order: any other
|
| 676 |
+
order is a person's arrangement, and theirs wins.
|
| 677 |
+
"""
|
| 678 |
+
keys = [f.get("key") for f in stored]
|
| 679 |
+
if not pinned or pinned not in keys or keys[0] == pinned:
|
| 680 |
+
return stored
|
| 681 |
+
in_report = set(report)
|
| 682 |
+
if [k for k in keys if k in in_report] != [k for k in report if k in set(keys)]:
|
| 683 |
+
return stored
|
| 684 |
+
return ([f for f in stored if f.get("key") == pinned]
|
| 685 |
+
+ [f for f in stored if f.get("key") != pinned])
|
| 686 |
+
|
| 687 |
+
|
| 688 |
def apply_plan(rt, built, username="amazon", today=None):
|
| 689 |
"""Create-or-merge every table in `built`, in ONE `flush="sync"` write.
|
| 690 |
|
|
|
|
| 697 |
raise Refused("; ".join(built["problems"]))
|
| 698 |
stamp = today or _odoo._iso_today()
|
| 699 |
written = {}
|
| 700 |
+
cur_mirror = datastore.ro_cursor()
|
| 701 |
+
try:
|
| 702 |
+
plans = [(key, LABELS[key], (built[key]["fields"] if isinstance(built[key], dict)
|
| 703 |
+
else _fields_for(cur_mirror, key)), built[key])
|
| 704 |
+
for key in SOURCE if key in built]
|
| 705 |
+
legacy_ids = {}
|
| 706 |
+
report_order = {key: [name for name, _dtype in mirror_columns(cur_mirror, SOURCE[key])]
|
| 707 |
+
for key, _label, _fields, _rows in plans}
|
| 708 |
+
for key, _label, _fields, rows in plans:
|
| 709 |
+
if isinstance(rows, dict):
|
| 710 |
+
continue
|
| 711 |
+
names = report_order[key]
|
| 712 |
+
for row in rows:
|
| 713 |
+
raw = {name: row.get(name) for name in names}
|
| 714 |
+
payload = json.dumps(raw, sort_keys=True, ensure_ascii=True,
|
| 715 |
+
separators=(",", ":"), default=str)
|
| 716 |
+
legacy_ids[(key, row["_id"])] = (
|
| 717 |
+
_row_id(raw, key), hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32])
|
| 718 |
+
finally:
|
| 719 |
+
cur_mirror.close()
|
| 720 |
+
|
| 721 |
+
not_stored = {}
|
| 722 |
|
| 723 |
def _apply_all(doc):
|
| 724 |
doc = doc if isinstance(doc, dict) else {}
|
| 725 |
+
not_stored.clear()
|
| 726 |
for key, label, fields, rows in plans:
|
| 727 |
+
# ββ D-3284 (owner ruling 2026-09-27): S66-T42/T43 created a NEW Amazon table only while
|
| 728 |
+
# the workspace had room under MAX_TABLES and reported the rest `not_stored`; ten Amazon
|
| 729 |
+
# databases stayed missing that way. Connector databases now take no MAX_TABLES slot
|
| 730 |
+
# (`user_tables.counts_toward_table_cap`), so every planned Amazon table is created.
|
| 731 |
+
# `not_stored` stays as bookkeeping: empty, it clears any earlier record.
|
| 732 |
+
# Corrected/removed measured joins retire their machine-owned old fields.
|
| 733 |
+
# Preserve fields a person created or explicitly took over.
|
| 734 |
+
current = doc.get(key) or {}
|
| 735 |
+
wanted = {f["key"] for f in fields}
|
| 736 |
+
# S67-T30: a product lookup the mirror no longer supports retires the same way.
|
| 737 |
+
current["fields"] = _lead_with_identity(
|
| 738 |
+
[f for f in current.get("fields", [])
|
| 739 |
+
if not ((f.get("type") == "link" or f.get("key") in LOOKUP_KEYS)
|
| 740 |
+
and (f.get("automation") or {}).get("flowId") == "amazon_relational"
|
| 741 |
+
and not _odoo._ut().user_edited(f)
|
| 742 |
+
and f.get("key") not in wanted)],
|
| 743 |
+
report_order.get(key) or [], PINNED.get(key))
|
| 744 |
+
if current["fields"]:
|
| 745 |
+
current["fields"] = _place_lookups(current["fields"], fields)
|
| 746 |
+
stub = isinstance(rows, dict) and rows.get("readThrough") is True
|
| 747 |
+
if not stub:
|
| 748 |
+
stored = current.get("rows") or {}
|
| 749 |
+
for row in rows:
|
| 750 |
+
if row["_id"] in stored:
|
| 751 |
+
continue
|
| 752 |
+
for old_id in legacy_ids.get((key, row["_id"]), ()):
|
| 753 |
+
if old_id in stored:
|
| 754 |
+
# Carry authored cells along when replacing an unusable string id.
|
| 755 |
+
stored[row["_id"]] = stored.pop(old_id)
|
| 756 |
+
break
|
| 757 |
+
removed = len((doc.get(key) or {}).get("rows") or {}) if stub else 0
|
| 758 |
+
written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, None if stub else rows,
|
| 759 |
username, stamp)
|
| 760 |
+
if stub:
|
| 761 |
+
doc[key]["readThrough"] = True
|
| 762 |
+
doc[key]["rows"] = {}
|
| 763 |
+
doc[key].pop(_odoo._ut().OVERSIZE_FIELD, None)
|
| 764 |
+
written[key].update(rows=rows["count"], removed=removed, readThrough=True)
|
| 765 |
+
else:
|
| 766 |
+
doc[key].pop("readThrough", None)
|
| 767 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 768 |
# mutable display label or from the process having imported this module.
|
| 769 |
doc[key]["connector"] = source_attribution(key)
|
| 770 |
return doc
|
| 771 |
|
| 772 |
rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
|
| 773 |
+
_record_not_stored({key for key, *_rest in plans}, dict(not_stored))
|
| 774 |
return written
|
| 775 |
|
| 776 |
|
| 777 |
+
def _record_not_stored(planned, not_stored):
|
| 778 |
+
"""Durable per-table marker in the Amazon mirror, read by `connectors_amazon.sync_state`.
|
| 779 |
+
|
| 780 |
+
A table refused at the store's table cap gets `not_stored` with its reason; a table that was
|
| 781 |
+
stored clears any earlier marker. Best effort: the store write above already happened, and a
|
| 782 |
+
marker failure must not turn it into an error.
|
| 783 |
+
"""
|
| 784 |
+
try:
|
| 785 |
+
con = datastore.connect()
|
| 786 |
+
for key in planned:
|
| 787 |
+
table = key[len(PREFIX):]
|
| 788 |
+
entity = amazon_store.entity_key("not_stored_" + table)
|
| 789 |
+
if key in not_stored:
|
| 790 |
+
amazon_store._record_state(con, entity, "not_stored", 0, not_stored[key])
|
| 791 |
+
else:
|
| 792 |
+
con.execute("DELETE FROM _sync_state WHERE entity = ?", [entity])
|
| 793 |
+
except Exception: # noqa: BLE001
|
| 794 |
+
pass
|
| 795 |
+
|
| 796 |
+
|
| 797 |
def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, log=print):
|
| 798 |
"""Open, plan and conditionally apply the Amazon mirror.
|
| 799 |
|
|
|
|
| 806 |
path = datastore.path_for(tenant)
|
| 807 |
if Path(datastore.DB_PATH) != Path(path):
|
| 808 |
datastore.use_path(path)
|
| 809 |
+
cur = datastore.ro_cursor()
|
| 810 |
+
try:
|
| 811 |
+
return _refresh_from_cursor(cur, rt, why, tenant, previous_fingerprint, log, state)
|
| 812 |
+
finally:
|
| 813 |
+
cur.close()
|
| 814 |
+
|
| 815 |
+
|
| 816 |
+
def _refresh_from_cursor(cur, rt, why, tenant, previous_fingerprint, log, state):
|
| 817 |
if not any(mirror_columns(cur, t) for t in SOURCE.values()):
|
| 818 |
log(f"[amazon_relational] {why}: no Amazon tables in {tenant}'s mirror, nothing to spawn "
|
| 819 |
f"(run `python platform/harness/amazon_store.py --sync --tenant {tenant}` first)")
|
|
|
|
| 837 |
state["changed"] = True
|
| 838 |
state["written"] = apply_plan(rt, applicable)
|
| 839 |
state["applied"] = True
|
| 840 |
+
import routes_odoo_tables
|
| 841 |
+
routes_odoo_tables.invalidate_sources()
|
| 842 |
for key, counts in state["written"].items():
|
| 843 |
log(f"[amazon_relational] {why}: {key:<28} {counts}")
|
| 844 |
return state
|
api/application_registry.py
CHANGED
|
@@ -16,6 +16,11 @@ APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
|
|
| 16 |
APPLICATION_WIZARD_TEMPLATE_KINDS = ("route", "catalog")
|
| 17 |
FINANCE_TEMPLATE_KIND = "finance"
|
| 18 |
APPLICATION_TEMPLATE_KINDS = APPLICATION_WIZARD_TEMPLATE_KINDS + (FINANCE_TEMPLATE_KIND,)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
_KEY = "applications_v1"
|
| 20 |
_MAX = 500
|
| 21 |
|
|
@@ -65,6 +70,17 @@ _TEMPLATE_CONTRACTS = {
|
|
| 65 |
"fieldRoles": [],
|
| 66 |
"defaultPlacement": {"section": "app", "order": 0},
|
| 67 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
}
|
| 69 |
|
| 70 |
|
|
@@ -363,6 +379,18 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract
|
|
| 363 |
raise ApplicationError("navPlacement order must be an integer from 0 to 999")
|
| 364 |
return {"templateKind": kind, "financeKey": finance_key, "deepLink": app["deepLink"],
|
| 365 |
"navPlacement": {"section": "app", "order": order}}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 366 |
source = str(payload.get("sourceDatabase") or "").strip()[:120]
|
| 367 |
if not source:
|
| 368 |
raise ApplicationError("sourceDatabase is required")
|
|
@@ -470,12 +498,49 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
|
|
| 470 |
"edit", "share", "delete", "duplicate", "star", "default")})
|
| 471 |
row["starred"] = False
|
| 472 |
row["isDefault"] = False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 473 |
target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
|
| 474 |
row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
|
| 475 |
"database":target_database}
|
| 476 |
return row
|
| 477 |
|
| 478 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 479 |
def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starred=None,
|
| 480 |
allowed_databases=None):
|
| 481 |
doc = load(runtime)
|
|
@@ -491,6 +556,13 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 491 |
rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
|
| 492 |
target_database=database,
|
| 493 |
is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 494 |
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
|
| 495 |
|
| 496 |
|
|
|
|
| 16 |
APPLICATION_WIZARD_TEMPLATE_KINDS = ("route", "catalog")
|
| 17 |
FINANCE_TEMPLATE_KIND = "finance"
|
| 18 |
APPLICATION_TEMPLATE_KINDS = APPLICATION_WIZARD_TEMPLATE_KINDS + (FINANCE_TEMPLATE_KIND,)
|
| 19 |
+
# S67-T40: Content Studio creates its OWN databases (content_studio.py), so it is never offered to
|
| 20 |
+
# the five-step wizard either; its catalogue POST provisions References + Posts and binds Posts.
|
| 21 |
+
# Appended on its own line so the wizard / Finance separation above reads exactly as S65-T10 pins it.
|
| 22 |
+
CONTENT_STUDIO_TEMPLATE_KIND = "content_studio"
|
| 23 |
+
APPLICATION_TEMPLATE_KINDS = APPLICATION_TEMPLATE_KINDS + (CONTENT_STUDIO_TEMPLATE_KIND,)
|
| 24 |
_KEY = "applications_v1"
|
| 25 |
_MAX = 500
|
| 26 |
|
|
|
|
| 70 |
"fieldRoles": [],
|
| 71 |
"defaultPlacement": {"section": "app", "order": 0},
|
| 72 |
},
|
| 73 |
+
# S67-T40 (R17, R21, O3: one declarative row, no new type). S67-T41 owner correction: the App
|
| 74 |
+
# opens on its OWN surface in the App module (`content_studio.app_link`), like Catalog and
|
| 75 |
+
# Route, so the governed type only has to be a real one; its calendar is time.
|
| 76 |
+
"content_studio": {
|
| 77 |
+
"kind": "content_studio",
|
| 78 |
+
"label": "Content Studio",
|
| 79 |
+
"applicationType": "timeseries",
|
| 80 |
+
"displayMode": "timeseries",
|
| 81 |
+
"fieldRoles": [],
|
| 82 |
+
"defaultPlacement": {"section": "app", "order": 0},
|
| 83 |
+
},
|
| 84 |
}
|
| 85 |
|
| 86 |
|
|
|
|
| 379 |
raise ApplicationError("navPlacement order must be an integer from 0 to 999")
|
| 380 |
return {"templateKind": kind, "financeKey": finance_key, "deepLink": app["deepLink"],
|
| 381 |
"navPlacement": {"section": "app", "order": order}}
|
| 382 |
+
if kind == CONTENT_STUDIO_TEMPLATE_KIND:
|
| 383 |
+
# Created server-side by content_studio.create right after it made the Posts database, so
|
| 384 |
+
# the binding is the one database it just created; the link can only open that database.
|
| 385 |
+
source = str(payload.get("sourceDatabase") or "").strip()[:120]
|
| 386 |
+
if not source.startswith("ut_"):
|
| 387 |
+
raise ApplicationError("Content Studio binds the Posts database it created.")
|
| 388 |
+
if allowed_databases is not None and source not in set(allowed_databases):
|
| 389 |
+
raise PermissionError("an Application cannot bind a database you cannot open")
|
| 390 |
+
# S67-T41 owner correction: no stored link. `_public` derives the App's own address from
|
| 391 |
+
# its id, so every Studio, including one created before the correction, opens in App.
|
| 392 |
+
return {"templateKind": kind, "sourceDatabase": source,
|
| 393 |
+
"navPlacement": {"section": "app", "order": 0}}
|
| 394 |
source = str(payload.get("sourceDatabase") or "").strip()[:120]
|
| 395 |
if not source:
|
| 396 |
raise ApplicationError("sourceDatabase is required")
|
|
|
|
| 498 |
"edit", "share", "delete", "duplicate", "star", "default")})
|
| 499 |
row["starred"] = False
|
| 500 |
row["isDefault"] = False
|
| 501 |
+
if row.get("templateKind") == CONTENT_STUDIO_TEMPLATE_KIND:
|
| 502 |
+
# S67-T41 owner correction: a Studio opens on its own App surface, never in Data. The
|
| 503 |
+
# database-bound edit, share, duplicate and default doors cannot reach an App bound to a
|
| 504 |
+
# user table, so they are off; star and delete stay, routed to `/content-studio/apps`.
|
| 505 |
+
row["deepLink"] = f"/app?studio={row['id']}"
|
| 506 |
+
row["capabilities"].update({key: False for key in ("edit", "share", "duplicate", "default")})
|
| 507 |
target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
|
| 508 |
row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
|
| 509 |
"database":target_database}
|
| 510 |
return row
|
| 511 |
|
| 512 |
|
| 513 |
+
#: `core.table_store.RESERVED_MEMBERS`: workspace members that are registries, not users.
|
| 514 |
+
_RESERVED_WORKSPACE_MEMBERS = ("__shared__", "__assigned__", "__audit__")
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
def dependent_view_counts(runtime, database):
|
| 518 |
+
"""S67-T39 / R36: {app_id: number of saved Views on `database` bound to it}, over EVERY
|
| 519 |
+
user's Views in that database's workspace. The Delete App dialog names this count before
|
| 520 |
+
confirm. None when the workspace cannot be read, so the dialog never states a guessed 0.
|
| 521 |
+
"""
|
| 522 |
+
from core.view_templates import workspace_key
|
| 523 |
+
key = workspace_key(database)
|
| 524 |
+
if not key:
|
| 525 |
+
return {}
|
| 526 |
+
try:
|
| 527 |
+
doc = runtime.get(key)
|
| 528 |
+
except Exception:
|
| 529 |
+
return None
|
| 530 |
+
counts = {}
|
| 531 |
+
for member, bucket in (doc.items() if isinstance(doc, dict) else ()):
|
| 532 |
+
if member in _RESERVED_WORKSPACE_MEMBERS or not isinstance(bucket, dict):
|
| 533 |
+
continue
|
| 534 |
+
views = bucket.get("views")
|
| 535 |
+
for view in (views.values() if isinstance(views, dict) else ()):
|
| 536 |
+
config = view.get("config") if isinstance(view, dict) else None
|
| 537 |
+
ref = config.get("application") if isinstance(config, dict) else None
|
| 538 |
+
app_id = str(ref.get("id") or "") if isinstance(ref, dict) else ""
|
| 539 |
+
if app_id:
|
| 540 |
+
counts[app_id] = counts.get(app_id, 0) + 1
|
| 541 |
+
return counts
|
| 542 |
+
|
| 543 |
+
|
| 544 |
def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starred=None,
|
| 545 |
allowed_databases=None):
|
| 546 |
doc = load(runtime)
|
|
|
|
| 556 |
rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
|
| 557 |
target_database=database,
|
| 558 |
is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
|
| 559 |
+
# S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows
|
| 560 |
+
# the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it.
|
| 561 |
+
deletable = [row for row in rows if row["capabilities"].get("delete")]
|
| 562 |
+
if database and deletable:
|
| 563 |
+
counts = dependent_view_counts(runtime, database)
|
| 564 |
+
for row in deletable:
|
| 565 |
+
row["dependentViews"] = None if counts is None else counts.get(row["id"], 0)
|
| 566 |
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
|
| 567 |
|
| 568 |
|
api/assistant_mirror.py
CHANGED
|
@@ -28,6 +28,15 @@ that need them. A module-level import either way is a cycle at startup.
|
|
| 28 |
"""
|
| 29 |
|
| 30 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
class MirrorRefusal(Exception):
|
| 32 |
"""A named, fail-closed refusal with the status and code `deps` should raise it as.
|
| 33 |
|
|
@@ -49,9 +58,14 @@ def _rot():
|
|
| 49 |
return routes_odoo_tables
|
| 50 |
|
| 51 |
|
| 52 |
-
def open_cursor(session):
|
| 53 |
"""`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
|
| 54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
β THE TWO FAILURES ARE NOT THE SAME FAILURE and the statuses differ, exactly as they do on
|
| 56 |
the grid door. `_mirror_cur` has already consumed `DatastoreMismatch` and hands it back as a
|
| 57 |
sentence, so a bare `RuntimeError` escaping it means the mirror is completing its first sync
|
|
@@ -61,7 +75,7 @@ def open_cursor(session):
|
|
| 61 |
"""
|
| 62 |
rot = _rot()
|
| 63 |
try:
|
| 64 |
-
cur, mismatch = rot._mirror_cur(session)
|
| 65 |
except RuntimeError as exc:
|
| 66 |
return None, MirrorRefusal(503, "assistant_store_not_ready", str(exc))
|
| 67 |
except Exception as exc: # noqa: BLE001
|
|
@@ -118,8 +132,26 @@ def is_bound(cur, table_key):
|
|
| 118 |
"""
|
| 119 |
from harness import datastore
|
| 120 |
try:
|
| 121 |
-
|
| 122 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
return False
|
| 124 |
aliases = dict(spec.get("tables") or {})
|
| 125 |
if spec.get("table"):
|
|
@@ -129,8 +161,13 @@ def is_bound(cur, table_key):
|
|
| 129 |
return False
|
| 130 |
|
| 131 |
|
| 132 |
-
def declared_fields(table_key):
|
| 133 |
-
"""`mk_fields()` for one connected grid, or None when this key is not a connected
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
|
| 135 |
β THE STABLE MIRROR CONTRACT, NOT THE STORED DEFINITION. `odoo_relational.apply_plan` built
|
| 136 |
each table's stored `fields` from this same function, so reading it directly is reading the
|
|
@@ -142,10 +179,20 @@ def declared_fields(table_key):
|
|
| 142 |
if key == str(table_key or ""):
|
| 143 |
return [dict(field) for field in (mk_fields() or [])
|
| 144 |
if isinstance(field, dict) and field.get("key")]
|
| 145 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
|
| 147 |
|
| 148 |
-
def answerable_fields(table_key, spec):
|
| 149 |
"""The declared field contract INTERSECTED with the columns the mirror can actually express.
|
| 150 |
|
| 151 |
β THE INTERSECTION IS THE POINT, and dropping it would be the quiet defect. A declared field
|
|
@@ -155,7 +202,8 @@ def answerable_fields(table_key, spec):
|
|
| 155 |
refused - a wrong answer with nothing on screen saying so. Left out, `deps` answers a request
|
| 156 |
for it with a loud `assistant_unknown_field`.
|
| 157 |
"""
|
| 158 |
-
fields
|
|
|
|
| 159 |
if fields is None:
|
| 160 |
return None
|
| 161 |
cols = set((spec or {}).get("cols") or ())
|
|
@@ -197,30 +245,14 @@ def _compiled(nodes, conj, columns, *, code, subject, rot):
|
|
| 197 |
return pred
|
| 198 |
|
| 199 |
|
| 200 |
-
def
|
| 201 |
-
|
| 202 |
-
"""Every permitted row of a mirror-served grid, or a refusal that names its cause.
|
| 203 |
-
|
| 204 |
-
Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
|
| 205 |
-
(`{pid, **cells}`), sorted by pid.
|
| 206 |
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
with
|
| 210 |
-
states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes
|
| 211 |
-
every aggregate from those same records, so a short set does not produce a smaller answer, it
|
| 212 |
-
produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over
|
| 213 |
-
already settled this policy in the same words: REFUSE, NEVER TRIM.
|
| 214 |
-
|
| 215 |
-
β FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and
|
| 216 |
-
the field projection in Python AFTER this returns. Hand it a row missing a column the wall
|
| 217 |
-
names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces
|
| 218 |
-
pass would quietly disagree with the pushdown it is meant to double-check.
|
| 219 |
"""
|
| 220 |
-
from harness import datastore
|
| 221 |
-
rot = _rot()
|
| 222 |
fe = rot._fe()
|
| 223 |
-
|
| 224 |
cols = rot._filter_columns(spec, fields)
|
| 225 |
|
| 226 |
# ββ the administrator's wall, compiled FIRST and pushed down ββββββββββββββββββββββββββββββ
|
|
@@ -267,23 +299,191 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t
|
|
| 267 |
where = f"({where}) AND {pred.sql}" if where else pred.sql
|
| 268 |
params.extend(pred.params)
|
| 269 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 270 |
# β EXACTLY ONE of `table`/`from_sql` - `window` raises if both or neither arrive, so the
|
| 271 |
# spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM.
|
| 272 |
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
|
| 273 |
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 278 |
raise MirrorRefusal(
|
| 279 |
409, "assistant_source_too_large",
|
| 280 |
-
f"this database matches {
|
| 281 |
-
f"
|
| 282 |
-
f"
|
| 283 |
-
f"
|
| 284 |
-
f"again"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
|
| 286 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 287 |
rows = rot._pool_from_window(spec, win, stamp=stamp)
|
| 288 |
rows.sort(key=lambda row: row["pid"])
|
| 289 |
return rows, stamp, int(win["total"])
|
|
|
|
| 28 |
"""
|
| 29 |
|
| 30 |
|
| 31 |
+
#: β S67-T33: the most rows ONE assistant answer materialises in this process. Not a cap on what
|
| 32 |
+
#: the connector may hold (rule 1: the read walks every row up to here, never a 5,000-row window):
|
| 33 |
+
#: it is the memory bound of computing an answer in Python. MEASURED on staging, 2026-09-28: an
|
| 34 |
+
#: unbounded read of one Amazon grid grew this process past 15 GB before it was stopped, which is
|
| 35 |
+
#: the Space's whole memory. Past it, the read REFUSES BY NAME with the cause and the fix (rule 1's
|
| 36 |
+
#: second sentence); server-side aggregates over larger tables are S67-T34 (D-2631).
|
| 37 |
+
ASSISTANT_READ_MAX = 200_000
|
| 38 |
+
|
| 39 |
+
|
| 40 |
class MirrorRefusal(Exception):
|
| 41 |
"""A named, fail-closed refusal with the status and code `deps` should raise it as.
|
| 42 |
|
|
|
|
| 58 |
return routes_odoo_tables
|
| 59 |
|
| 60 |
|
| 61 |
+
def open_cursor(session, table_key=None):
|
| 62 |
"""`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
|
| 63 |
|
| 64 |
+
β S67-T33: `table_key` CHOOSES THE CURSOR BY THE TABLE'S OWN SOURCE, through the grid's
|
| 65 |
+
`_table_cursor` (reached via `_mirror_cur`). An Amazon grid asserts the tenant and opens its
|
| 66 |
+
own mirror without the Odoo first-sync gate, exactly as the grid door has since be3edb6e, so
|
| 67 |
+
the assistant no longer answers 503 for Amazon while Odoo is still backfilling.
|
| 68 |
+
|
| 69 |
β THE TWO FAILURES ARE NOT THE SAME FAILURE and the statuses differ, exactly as they do on
|
| 70 |
the grid door. `_mirror_cur` has already consumed `DatastoreMismatch` and hands it back as a
|
| 71 |
sentence, so a bare `RuntimeError` escaping it means the mirror is completing its first sync
|
|
|
|
| 75 |
"""
|
| 76 |
rot = _rot()
|
| 77 |
try:
|
| 78 |
+
cur, mismatch = rot._mirror_cur(session, table_key)
|
| 79 |
except RuntimeError as exc:
|
| 80 |
return None, MirrorRefusal(503, "assistant_store_not_ready", str(exc))
|
| 81 |
except Exception as exc: # noqa: BLE001
|
|
|
|
| 132 |
"""
|
| 133 |
from harness import datastore
|
| 134 |
try:
|
| 135 |
+
rot = _rot()
|
| 136 |
+
key = str(table_key or "")
|
| 137 |
+
spec = rot._sources().get(key)
|
| 138 |
+
if not spec and key.startswith("ut_amzn_"):
|
| 139 |
+
# S67-T33: a registry built before the Amazon mirror loaded holds no Amazon key.
|
| 140 |
+
# `_source_for` and `_materialised_table` both learn them on demand; so must this.
|
| 141 |
+
rot._learn_amazon_sources()
|
| 142 |
+
spec = rot.GRID_SOURCES.get(key)
|
| 143 |
+
if not spec:
|
| 144 |
+
# A dynamically sized Amazon snapshot is admitted per key by the grid door, not by
|
| 145 |
+
# the provider build, so the registry can lack a key the grid serves (measured on
|
| 146 |
+
# staging: `ut_amzn_fba_inventory` listed unanswerable, then answered). Ask the
|
| 147 |
+
# provider's own per-key resolver, exactly as `require_amazon_mirror` does, without
|
| 148 |
+
# admitting it (the read path admits it through that same door).
|
| 149 |
+
import amazon_relational as amazon
|
| 150 |
+
if key not in amazon.SOURCE:
|
| 151 |
+
return False
|
| 152 |
+
current = amazon.grid_source(cur, key) or {}
|
| 153 |
+
return bool(current) and not current.get("missing")
|
| 154 |
+
if not spec or spec.get("missing"):
|
| 155 |
return False
|
| 156 |
aliases = dict(spec.get("tables") or {})
|
| 157 |
if spec.get("table"):
|
|
|
|
| 161 |
return False
|
| 162 |
|
| 163 |
|
| 164 |
+
def declared_fields(table_key, definition=None):
|
| 165 |
+
"""`mk_fields()` for one connected grid, or None when this key is not a connected grid.
|
| 166 |
+
|
| 167 |
+
β S67-T33: THE SAME FALLBACK THE GRID TAKES (`routes_odoo_tables._compiled_read`). A key that
|
| 168 |
+
is not one of `odoo_relational.TABLES` but IS a registered read-through source (an Amazon grid,
|
| 169 |
+
a generated `ut_odoo_model_*` table) answers from its stored definition's `fields`. Before
|
| 170 |
+
this, those databases were listed as answerable and then refused 409 on every read.
|
| 171 |
|
| 172 |
β THE STABLE MIRROR CONTRACT, NOT THE STORED DEFINITION. `odoo_relational.apply_plan` built
|
| 173 |
each table's stored `fields` from this same function, so reading it directly is reading the
|
|
|
|
| 179 |
if key == str(table_key or ""):
|
| 180 |
return [dict(field) for field in (mk_fields() or [])
|
| 181 |
if isinstance(field, dict) and field.get("key")]
|
| 182 |
+
key = str(table_key or "")
|
| 183 |
+
if not isinstance(definition, dict) or not key.startswith("ut_"):
|
| 184 |
+
return None
|
| 185 |
+
try:
|
| 186 |
+
if _rot()._materialised_table(key):
|
| 187 |
+
return None
|
| 188 |
+
except Exception: # noqa: BLE001
|
| 189 |
+
return None
|
| 190 |
+
fields = [dict(field) for field in (definition.get("fields") or [])
|
| 191 |
+
if isinstance(field, dict) and field.get("key")]
|
| 192 |
+
return fields or None
|
| 193 |
|
| 194 |
|
| 195 |
+
def answerable_fields(table_key, spec, fields=None):
|
| 196 |
"""The declared field contract INTERSECTED with the columns the mirror can actually express.
|
| 197 |
|
| 198 |
β THE INTERSECTION IS THE POINT, and dropping it would be the quiet defect. A declared field
|
|
|
|
| 202 |
refused - a wrong answer with nothing on screen saying so. Left out, `deps` answers a request
|
| 203 |
for it with a loud `assistant_unknown_field`.
|
| 204 |
"""
|
| 205 |
+
if fields is None:
|
| 206 |
+
fields = declared_fields(table_key)
|
| 207 |
if fields is None:
|
| 208 |
return None
|
| 209 |
cols = set((spec or {}).get("cols") or ())
|
|
|
|
| 245 |
return pred
|
| 246 |
|
| 247 |
|
| 248 |
+
def _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys):
|
| 249 |
+
"""`(where, params, cols)`: the ONE compiled predicate every mirror read runs. S67-T34.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 250 |
|
| 251 |
+
Extracted verbatim from `read_rows` so the row walk, the page read and the server-side
|
| 252 |
+
aggregate run the SAME wall and the SAME caller filter, compiled once, by the grid's compiler.
|
| 253 |
+
Three readers with one predicate cannot disagree about which rows the caller may see.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 254 |
"""
|
|
|
|
|
|
|
| 255 |
fe = rot._fe()
|
|
|
|
| 256 |
cols = rot._filter_columns(spec, fields)
|
| 257 |
|
| 258 |
# ββ the administrator's wall, compiled FIRST and pushed down ββββββββββββββββββββββββββββββ
|
|
|
|
| 299 |
where = f"({where}) AND {pred.sql}" if where else pred.sql
|
| 300 |
params.extend(pred.params)
|
| 301 |
|
| 302 |
+
return where, params, cols
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None,
|
| 306 |
+
filter_keys=None, defn=None):
|
| 307 |
+
"""Every permitted row of a mirror-served grid, or a refusal that names its cause.
|
| 308 |
+
|
| 309 |
+
Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
|
| 310 |
+
(`{pid, **cells}`), sorted by pid.
|
| 311 |
+
|
| 312 |
+
β NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a
|
| 313 |
+
connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED
|
| 314 |
+
with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt`
|
| 315 |
+
states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes
|
| 316 |
+
every aggregate from those same records, so a short set does not produce a smaller answer, it
|
| 317 |
+
produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over
|
| 318 |
+
already settled this policy in the same words: REFUSE, NEVER TRIM.
|
| 319 |
+
|
| 320 |
+
β FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and
|
| 321 |
+
the field projection in Python AFTER this returns. Hand it a row missing a column the wall
|
| 322 |
+
names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces
|
| 323 |
+
pass would quietly disagree with the pushdown it is meant to double-check.
|
| 324 |
+
"""
|
| 325 |
+
from harness import datastore
|
| 326 |
+
rot = _rot()
|
| 327 |
+
where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
|
| 328 |
+
|
| 329 |
# β EXACTLY ONE of `table`/`from_sql` - `window` raises if both or neither arrive, so the
|
| 330 |
# spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM.
|
| 331 |
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
|
| 332 |
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
|
| 333 |
+
# ββ S67-T33 / STANDING RULE 1: EVERY ROW, NOT THE FIRST 5,000. The single `WINDOW_MAX`
|
| 334 |
+
# pass this replaced refused any connected database past 5,000 matching rows, which is a cap
|
| 335 |
+
# on a connector's records. The read now walks the whole predicate in `WINDOW_MAX` pages,
|
| 336 |
+
# KEYSET-paged on the spec's own id (the order the pass already used), so a deep page costs
|
| 337 |
+
# one index seek rather than an OFFSET walk. The keyset bound is a pagination cursor, not a
|
| 338 |
+
# user condition: every condition above still comes from the grid's compiler.
|
| 339 |
+
# β STILL NEVER A PARTIAL ANSWER: the first page's `total` is the population, and a walk that
|
| 340 |
+
# ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short.
|
| 341 |
+
first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
|
| 342 |
+
where=where, params=tuple(params), order_by=spec["id"],
|
| 343 |
+
limit=datastore.WINDOW_MAX, cur=cur, **frm)
|
| 344 |
+
total = int(first["total"])
|
| 345 |
+
if total > ASSISTANT_READ_MAX:
|
| 346 |
raise MirrorRefusal(
|
| 347 |
409, "assistant_source_too_large",
|
| 348 |
+
f"this database matches {total:,} rows for that question, and one assistant answer "
|
| 349 |
+
f"is computed over at most {ASSISTANT_READ_MAX:,} rows so that a single question "
|
| 350 |
+
f"cannot exhaust the server's memory. Answering over part of them would report a "
|
| 351 |
+
f"wrong total, so it is refused instead. Narrow it with a filter (for example a date "
|
| 352 |
+
f"range) and ask again, or ask for totals as aggregates, which the server computes "
|
| 353 |
+
f"over every row")
|
| 354 |
+
got = list(first["rows"])
|
| 355 |
+
page = list(first["rows"])
|
| 356 |
+
while len(got) < total and len(page) >= datastore.WINDOW_MAX:
|
| 357 |
+
last = page[-1][0]
|
| 358 |
+
more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?"
|
| 359 |
+
nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
|
| 360 |
+
where=more_where, params=tuple(params) + (last,),
|
| 361 |
+
order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, **frm)
|
| 362 |
+
page = list(nxt["rows"])
|
| 363 |
+
if not page:
|
| 364 |
+
break
|
| 365 |
+
got.extend(page)
|
| 366 |
+
if len(got) != total:
|
| 367 |
+
raise MirrorRefusal(
|
| 368 |
+
409, "assistant_source_changed",
|
| 369 |
+
f"this database changed while it was being read ({len(got):,} rows read against "
|
| 370 |
+
f"{total:,} counted), so the answer would not describe one consistent set of rows. "
|
| 371 |
+
f"Ask again once its connector sync has finished")
|
| 372 |
+
win = {**first, "rows": got, "total": total}
|
| 373 |
+
|
| 374 |
+
# S67-T33: the caller's rows-free definition is LENT, as the grid lends it, so the stamp does
|
| 375 |
+
# not re-read the whole tenant document (measured about 3 s a question on staging).
|
| 376 |
+
stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
|
| 377 |
+
defn=defn if isinstance(defn, dict) else None)
|
| 378 |
+
rows = rot._pool_from_window(spec, win, stamp=stamp)
|
| 379 |
+
rows.sort(key=lambda row: row["pid"])
|
| 380 |
+
return rows, stamp, int(win["total"])
|
| 381 |
+
|
| 382 |
|
| 383 |
+
#: The connector's function names mapped onto the grid's aggregate compiler (`filter_sql`).
|
| 384 |
+
_GRID_AGG = {"sum": "sum", "avg": "average", "count_distinct": "count_distinct"}
|
| 385 |
+
|
| 386 |
+
|
| 387 |
+
def _json_value(rot, value):
|
| 388 |
+
"""A DuckDB aggregate result as plain JSON: a number through the grid's `_agg_number`, a
|
| 389 |
+
date or timestamp as its ISO text, anything else as is. Never a substituted 0."""
|
| 390 |
+
if value is None:
|
| 391 |
+
return None
|
| 392 |
+
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
| 393 |
+
return rot._agg_number(value)
|
| 394 |
+
iso = getattr(value, "isoformat", None)
|
| 395 |
+
if callable(iso):
|
| 396 |
+
return iso()
|
| 397 |
+
try:
|
| 398 |
+
from decimal import Decimal
|
| 399 |
+
if isinstance(value, Decimal):
|
| 400 |
+
return rot._agg_number(value)
|
| 401 |
+
except Exception: # noqa: BLE001
|
| 402 |
+
pass
|
| 403 |
+
return value
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None,
|
| 407 |
+
filter_tree=None, filter_keys=None):
|
| 408 |
+
"""Server-side aggregates over EVERY permitted row, in SQL. S67-T34 (D-616, D-2631).
|
| 409 |
+
|
| 410 |
+
Returns `(values, total)`: `values` is `[{fn, field?, value}]` in `specs` order, `total` the
|
| 411 |
+
count of rows the permitted predicate matches, from `datastore.window`'s own count statement
|
| 412 |
+
over the SAME `where` as the aggregates beside it (the grid's `_aggregate_over` contract).
|
| 413 |
+
|
| 414 |
+
ββ THE PERMISSION FILTER IS INSIDE THE QUERY. `_predicate` compiles the administrator's wall
|
| 415 |
+
and the caller's filter with the grid's compiler, so a sum over a million-row table runs over
|
| 416 |
+
exactly the rows this caller may read and no row is pulled into Python to compute it. There
|
| 417 |
+
is no row ceiling here (rule 1): the answer is a handful of numbers whatever the table size.
|
| 418 |
+
|
| 419 |
+
β AN AGGREGATE MAY NAME ONLY A VISIBLE FIELD. `filter_keys` is the caller's visible list, and
|
| 420 |
+
a field outside it has no expression here at all, so a sum over a hidden column cannot be
|
| 421 |
+
built (the caller's `_normalise_aggregates` has already refused it by name).
|
| 422 |
+
"""
|
| 423 |
+
from harness import datastore
|
| 424 |
+
rot = _rot()
|
| 425 |
+
where, params, cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
|
| 426 |
+
visible = set(filter_keys) if filter_keys is not None else set(cols)
|
| 427 |
+
fs = rot._fs()
|
| 428 |
+
proj, slots = [], []
|
| 429 |
+
for item in specs or ():
|
| 430 |
+
fn, field = item.get("fn"), item.get("field")
|
| 431 |
+
if field is None:
|
| 432 |
+
slots.append(None) # count(*): the window's own total
|
| 433 |
+
continue
|
| 434 |
+
col = cols.get(field) if field in visible else None
|
| 435 |
+
if col is None:
|
| 436 |
+
raise MirrorRefusal(400, "assistant_unknown_field",
|
| 437 |
+
"every requested field must be readable in this database")
|
| 438 |
+
if fn in {"sum", "avg"} and col["type"] not in fs.NUMERIC_TYPES:
|
| 439 |
+
raise MirrorRefusal(400, "query_aggregate_type",
|
| 440 |
+
"sum and avg require a readable numeric field.")
|
| 441 |
+
if fn in _GRID_AGG:
|
| 442 |
+
expr = fs.aggregate_sql(col, _GRID_AGG[fn])
|
| 443 |
+
elif fn in {"min", "max"}:
|
| 444 |
+
expr = (fs.aggregate_sql(col, fn) if col["type"] in fs.NUMERIC_TYPES
|
| 445 |
+
else f"{fn}({col['sql']})")
|
| 446 |
+
elif fn == "count":
|
| 447 |
+
expr = f"count({col['sql']})"
|
| 448 |
+
else:
|
| 449 |
+
raise MirrorRefusal(400, "query_aggregates_invalid",
|
| 450 |
+
"That aggregate is not available.")
|
| 451 |
+
slots.append(len(proj))
|
| 452 |
+
proj.append(f'{expr} AS "a{len(proj)}"')
|
| 453 |
+
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
|
| 454 |
+
win = datastore.window(select=(", ".join(proj) or "1"), where=where, params=tuple(params),
|
| 455 |
+
order_by="1", limit=1, cur=cur, **frm)
|
| 456 |
+
row = (win["rows"] or [()])[0]
|
| 457 |
+
total = int(win["total"])
|
| 458 |
+
out = []
|
| 459 |
+
for item, slot in zip(specs or (), slots):
|
| 460 |
+
value = total if slot is None else (_json_value(rot, row[slot]) if slot < len(row)
|
| 461 |
+
else None)
|
| 462 |
+
out.append({**item, "value": value})
|
| 463 |
+
return out, total
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree=None,
|
| 467 |
+
filter_tree=None, filter_keys=None, defn=None):
|
| 468 |
+
"""ONE page of the permitted rows, paged IN SQL, in pid order. S67-T34 (rule 1).
|
| 469 |
+
|
| 470 |
+
Returns `(rows, stamp, total)` in `read_rows`'s shape. This is how a table past
|
| 471 |
+
`ASSISTANT_READ_MAX` still answers its rows: the page is `LIMIT/OFFSET` over the same
|
| 472 |
+
predicate `read_rows` runs, so only `limit` rows ever reach this process, and `total` is the
|
| 473 |
+
population that predicate matches. It carries no sort other than the pid; a caller asking for
|
| 474 |
+
one past the ceiling is refused by name upstream, because sorting a million rows in Python
|
| 475 |
+
is the memory failure the ceiling exists to prevent.
|
| 476 |
+
"""
|
| 477 |
+
from harness import datastore
|
| 478 |
+
rot = _rot()
|
| 479 |
+
where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
|
| 480 |
+
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
|
| 481 |
+
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
|
| 482 |
+
win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where,
|
| 483 |
+
params=tuple(params), order_by=spec["id"], offset=int(offset),
|
| 484 |
+
limit=int(limit), cur=cur, **frm)
|
| 485 |
+
stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
|
| 486 |
+
defn=defn if isinstance(defn, dict) else None)
|
| 487 |
rows = rot._pool_from_window(spec, win, stamp=stamp)
|
| 488 |
rows.sort(key=lambda row: row["pid"])
|
| 489 |
return rows, stamp, int(win["total"])
|
api/connectors_amazon.py
CHANGED
|
@@ -49,11 +49,80 @@ class AmazonStatusSafetyError(ValueError):
|
|
| 49 |
"""A status response contains a forbidden credential shape."""
|
| 50 |
|
| 51 |
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
"""Scan serialized JSON for C2's credential keys, token shapes and oversized values.
|
| 54 |
|
| 55 |
Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content.
|
| 56 |
The credential key roster is the same one the connector accepts at connect time.
|
|
|
|
| 57 |
"""
|
| 58 |
import json
|
| 59 |
import re
|
|
@@ -73,10 +142,11 @@ def status_payload_has_secret(serialized_payload):
|
|
| 73 |
elif isinstance(value, list):
|
| 74 |
pending.extend(value)
|
| 75 |
elif isinstance(value, str):
|
| 76 |
-
if (len(value) >
|
| 77 |
or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)):
|
| 78 |
return True
|
| 79 |
-
elif value is not None and
|
|
|
|
| 80 |
return True
|
| 81 |
return False
|
| 82 |
|
|
@@ -132,8 +202,51 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 132 |
from datetime import datetime, timezone
|
| 133 |
import duckdb
|
| 134 |
from harness import amazon_store as az
|
|
|
|
| 135 |
from routes_odoo_tables import _sync_entities, stuck_verdict
|
| 136 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
def stamp(value):
|
| 138 |
if not value:
|
| 139 |
return None
|
|
@@ -150,7 +263,7 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 150 |
try:
|
| 151 |
path = az._tenant_mirror_path(tenant_key, rt=rt)
|
| 152 |
if not path.is_file():
|
| 153 |
-
return
|
| 154 |
# ββ A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES
|
| 155 |
# `ConnectionException: Can't open a connection to same database file with a different
|
| 156 |
# configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global
|
|
@@ -174,9 +287,9 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 174 |
mirrored = _mirror_counts(cur, az)
|
| 175 |
except Exception: # noqa: BLE001
|
| 176 |
# No exception text: paths, provider payloads and credentials never become status copy.
|
| 177 |
-
return
|
| 178 |
if entities is None:
|
| 179 |
-
return
|
| 180 |
|
| 181 |
clock = stamp(now) if now is not None else datetime.now(timezone.utc)
|
| 182 |
states, bad_ticks = {}, set()
|
|
@@ -209,12 +322,12 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 209 |
elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"):
|
| 210 |
load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass."
|
| 211 |
|
| 212 |
-
answer =
|
| 213 |
for table in az.TABLES:
|
| 214 |
landed = entities.get(az.entity_key(table)) or {}
|
| 215 |
tick = entities.get(az.entity_key("sync_tick_" + table)) or {}
|
| 216 |
watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {}
|
| 217 |
-
rows = landed.get("rows")
|
| 218 |
if type(rows) is not int or rows < 0:
|
| 219 |
rows = None
|
| 220 |
# Hotfix 2026-09-24: during the first load no per-table marker exists yet, so the screen
|
|
@@ -222,7 +335,9 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 222 |
# own count is the measured answer.
|
| 223 |
if rows is None and table in mirrored:
|
| 224 |
rows = mirrored[table]
|
| 225 |
-
|
|
|
|
|
|
|
| 226 |
if marker.get("phase") == "done"
|
| 227 |
and (when := stamp(marker.get("updatedAt"))) is not None]
|
| 228 |
stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or ""
|
|
@@ -237,8 +352,20 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 237 |
# a promise about when a later resident tick will select this table.
|
| 238 |
next_due = stamp(reason.partition("next due ")[2]) if reason else None
|
| 239 |
unknown = [load_unknown] if load_unknown else []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
if rows is None:
|
| 241 |
-
unknown.append("This
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
# Standing rule 1's second half: a limit that cannot be removed is REPORTED, with cause
|
| 243 |
# and fix. `amazon_relational.plan` refuses a table over MAX_ROWS and said so only in the
|
| 244 |
# server log, so the owner saw an empty database and no reason.
|
|
@@ -246,10 +373,68 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 246 |
if over is not None and over > _stored_table_cap():
|
| 247 |
unknown.append(
|
| 248 |
f"The mirror holds {over:,} rows, more than a stored table can hold "
|
| 249 |
-
f"({_stored_table_cap():,})
|
| 250 |
-
"
|
| 251 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 253 |
answer[table] = {
|
| 254 |
"rowsLoaded": rows,
|
| 255 |
"lastSuccess": iso(max(successes)) if successes else None,
|
|
@@ -260,7 +445,9 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 260 |
"stuck": stuck, "stuckCause": stuck_cause,
|
| 261 |
"unknownCause": " ".join(unknown),
|
| 262 |
}
|
| 263 |
-
|
|
|
|
|
|
|
| 264 |
|
| 265 |
|
| 266 |
#: W55-T19 / R7 / C10 β AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
|
|
|
|
| 49 |
"""A status response contains a forbidden credential shape."""
|
| 50 |
|
| 51 |
|
| 52 |
+
class AmazonStatusRowsRefused(AmazonStatusSafetyError):
|
| 53 |
+
"""S67-T31: some status ROWS were refused; the rest are carried, already guarded.
|
| 54 |
+
|
| 55 |
+
Still an `AmazonStatusSafetyError`, so every caller that refuses on the base class keeps
|
| 56 |
+
refusing. `safe_status` holds only rows that passed the guard and `refused` only table NAMES:
|
| 57 |
+
neither carries the refused content, so a caller may degrade those rows and keep the others.
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
def __init__(self, safe_status, refused):
|
| 61 |
+
super().__init__('Amazon status contains a forbidden credential shape.')
|
| 62 |
+
self.safe_status = safe_status
|
| 63 |
+
self.refused = tuple(refused)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
#: The status guard's scalar ceiling, and the sentence a row wears when its status is refused.
|
| 67 |
+
STATUS_SCALAR_LIMIT = 512
|
| 68 |
+
STATUS_UNAVAILABLE = "Sync status is unavailable for this database."
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def trim_status(value, limit=STATUS_SCALAR_LIMIT):
|
| 72 |
+
"""Cut every string over `limit` to a word-bounded excerpt that fits, recursively.
|
| 73 |
+
|
| 74 |
+
S67-T31: a long, honest cause (staging's Search Query Performance cause measured 935
|
| 75 |
+
characters) used to trip the length branch of the guard and take the whole connector modal
|
| 76 |
+
down with a 500. β Call this only AFTER the credential-shape scan of the UNTRIMMED value
|
| 77 |
+
(`screen_status_rows`): cutting first could slice a token shape out of a string and accept
|
| 78 |
+
the rest, which would trim a refusal into acceptance.
|
| 79 |
+
"""
|
| 80 |
+
if isinstance(value, dict):
|
| 81 |
+
return {key: trim_status(item, limit) for key, item in value.items()}
|
| 82 |
+
if isinstance(value, list):
|
| 83 |
+
return [trim_status(item, limit) for item in value]
|
| 84 |
+
if isinstance(value, tuple):
|
| 85 |
+
return tuple(trim_status(item, limit) for item in value)
|
| 86 |
+
if isinstance(value, str) and len(value) > limit:
|
| 87 |
+
head = value[:limit - 3]
|
| 88 |
+
cut = head.rsplit(" ", 1)[0] if " " in head else head
|
| 89 |
+
return cut.rstrip(" ,;:") + "..."
|
| 90 |
+
return value
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def screen_status_rows(status):
|
| 94 |
+
"""`(safe, refused)` for a `{table: row}` status: each row judged on its own.
|
| 95 |
+
|
| 96 |
+
A row carrying a credential key, an `Atzr|` token or a bearer shape ANYWHERE in its untrimmed
|
| 97 |
+
form is refused whole (NC: never trimmed into acceptance, its value never returned). A row that
|
| 98 |
+
is only too long is trimmed, then held to the full guard again.
|
| 99 |
+
"""
|
| 100 |
+
import json
|
| 101 |
+
|
| 102 |
+
safe, refused = {}, []
|
| 103 |
+
for name, row in (status or {}).items():
|
| 104 |
+
try:
|
| 105 |
+
whole = json.dumps(row, ensure_ascii=False, allow_nan=False)
|
| 106 |
+
if status_payload_has_secret(whole, max_length=None):
|
| 107 |
+
refused.append(name)
|
| 108 |
+
continue
|
| 109 |
+
trimmed = trim_status(row)
|
| 110 |
+
if status_payload_has_secret(json.dumps(trimmed, ensure_ascii=False, allow_nan=False)):
|
| 111 |
+
refused.append(name)
|
| 112 |
+
continue
|
| 113 |
+
except (TypeError, ValueError, RecursionError):
|
| 114 |
+
refused.append(name)
|
| 115 |
+
continue
|
| 116 |
+
safe[name] = trimmed
|
| 117 |
+
return safe, refused
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def status_payload_has_secret(serialized_payload, *, max_length=STATUS_SCALAR_LIMIT):
|
| 121 |
"""Scan serialized JSON for C2's credential keys, token shapes and oversized values.
|
| 122 |
|
| 123 |
Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content.
|
| 124 |
The credential key roster is the same one the connector accepts at connect time.
|
| 125 |
+
`max_length=None` scans for credential shapes only (the pre-trim pass of `screen_status_rows`).
|
| 126 |
"""
|
| 127 |
import json
|
| 128 |
import re
|
|
|
|
| 142 |
elif isinstance(value, list):
|
| 143 |
pending.extend(value)
|
| 144 |
elif isinstance(value, str):
|
| 145 |
+
if ((max_length is not None and len(value) > max_length) or 'Atzr|' in value
|
| 146 |
or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)):
|
| 147 |
return True
|
| 148 |
+
elif (value is not None and max_length is not None
|
| 149 |
+
and len(str(value)) > max_length):
|
| 150 |
return True
|
| 151 |
return False
|
| 152 |
|
|
|
|
| 202 |
from datetime import datetime, timezone
|
| 203 |
import duckdb
|
| 204 |
from harness import amazon_store as az
|
| 205 |
+
from harness.amazon_client import DECLARED_DATASETS
|
| 206 |
from routes_odoo_tables import _sync_entities, stuck_verdict
|
| 207 |
|
| 208 |
+
# S66-T46 (lead): an unreadable deletion ledger is flagged once per row as
|
| 209 |
+
# `purgeScheduleUnknown`, never written into every table's cause, which would push out the
|
| 210 |
+
# reason each table already names (Amazon's refusal, the empty period, the declared reason).
|
| 211 |
+
purge_unknown = False
|
| 212 |
+
try:
|
| 213 |
+
purge_plan = az.scheduled_purge(tenant_key, rt, now=now)
|
| 214 |
+
purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else ""
|
| 215 |
+
except Exception:
|
| 216 |
+
purge_plan, purge_message, purge_unknown = None, "", True
|
| 217 |
+
|
| 218 |
+
def finish(status):
|
| 219 |
+
for row in status.values():
|
| 220 |
+
# A key only when there is something to say: a declared dataset's row stays exactly
|
| 221 |
+
# its declaration while no purge is scheduled (the group (a) status contract).
|
| 222 |
+
if purge_plan:
|
| 223 |
+
row["purgeScheduledAt"] = purge_plan["purgeAt"]
|
| 224 |
+
if purge_unknown and not row.get("declared"):
|
| 225 |
+
row["purgeScheduleUnknown"] = True
|
| 226 |
+
if purge_message:
|
| 227 |
+
cause = row.get("unknownCause") or ""
|
| 228 |
+
# Status scalars have a 512-character safety ceiling. Keep the full existing
|
| 229 |
+
# cause separately if the deletion sentence needs a shorter screen excerpt.
|
| 230 |
+
budget = 512 - len(purge_message) - 1
|
| 231 |
+
if len(cause) > budget:
|
| 232 |
+
row["syncCause"] = cause
|
| 233 |
+
cause = cause[:budget - 3].rsplit(" ", 1)[0] + "..."
|
| 234 |
+
row["unknownCause"] = " ".join(part for part in (cause, purge_message) if part)
|
| 235 |
+
# S67-T31: judged PER ROW. An over-long cause is trimmed (staging's 500 was
|
| 236 |
+
# `search_query_performance.unknownCause` at 935 characters); a credential shape still
|
| 237 |
+
# refuses, now naming only its rows so the caller degrades those and keeps the rest.
|
| 238 |
+
safe, refused = screen_status_rows(status)
|
| 239 |
+
if refused:
|
| 240 |
+
raise AmazonStatusRowsRefused(assert_safe_status(safe), refused)
|
| 241 |
+
return assert_safe_status(safe)
|
| 242 |
+
|
| 243 |
+
declared = {name: {"rowsLoaded": None, "declared": True, "unknownCause": spec["reason"]}
|
| 244 |
+
for name, spec in DECLARED_DATASETS.items()}
|
| 245 |
+
unavailable = {table: {"rowsLoaded": None, "lastSuccess": None, "inProgress": False,
|
| 246 |
+
"unknownCause": "This container has no readable sync status. Sync this database to load its rows."}
|
| 247 |
+
for table in az.TABLES}
|
| 248 |
+
unavailable.update(declared)
|
| 249 |
+
|
| 250 |
def stamp(value):
|
| 251 |
if not value:
|
| 252 |
return None
|
|
|
|
| 263 |
try:
|
| 264 |
path = az._tenant_mirror_path(tenant_key, rt=rt)
|
| 265 |
if not path.is_file():
|
| 266 |
+
return finish(unavailable)
|
| 267 |
# ββ A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES
|
| 268 |
# `ConnectionException: Can't open a connection to same database file with a different
|
| 269 |
# configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global
|
|
|
|
| 287 |
mirrored = _mirror_counts(cur, az)
|
| 288 |
except Exception: # noqa: BLE001
|
| 289 |
# No exception text: paths, provider payloads and credentials never become status copy.
|
| 290 |
+
return finish(unavailable)
|
| 291 |
if entities is None:
|
| 292 |
+
return finish(unavailable)
|
| 293 |
|
| 294 |
clock = stamp(now) if now is not None else datetime.now(timezone.utc)
|
| 295 |
states, bad_ticks = {}, set()
|
|
|
|
| 322 |
elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"):
|
| 323 |
load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass."
|
| 324 |
|
| 325 |
+
answer = dict(declared)
|
| 326 |
for table in az.TABLES:
|
| 327 |
landed = entities.get(az.entity_key(table)) or {}
|
| 328 |
tick = entities.get(az.entity_key("sync_tick_" + table)) or {}
|
| 329 |
watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {}
|
| 330 |
+
rows = mirrored.get(table, landed.get("rows"))
|
| 331 |
if type(rows) is not int or rows < 0:
|
| 332 |
rows = None
|
| 333 |
# Hotfix 2026-09-24: during the first load no per-table marker exists yet, so the screen
|
|
|
|
| 335 |
# own count is the measured answer.
|
| 336 |
if rows is None and table in mirrored:
|
| 337 |
rows = mirrored[table]
|
| 338 |
+
# A refused pass closes its tick (S66-T42); only the watermark can then prove a success.
|
| 339 |
+
refused_now = landed.get("phase") in ("refused", "failed", "throttled")
|
| 340 |
+
successes = [when for marker in ((watermark,) if refused_now else (landed, tick, watermark))
|
| 341 |
if marker.get("phase") == "done"
|
| 342 |
and (when := stamp(marker.get("updatedAt"))) is not None]
|
| 343 |
stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or ""
|
|
|
|
| 352 |
# a promise about when a later resident tick will select this table.
|
| 353 |
next_due = stamp(reason.partition("next due ")[2]) if reason else None
|
| 354 |
unknown = [load_unknown] if load_unknown else []
|
| 355 |
+
proof = entities.get(az.entity_key("reconcile_" + table)) or {}
|
| 356 |
+
try:
|
| 357 |
+
period = json.loads(proof.get("cursor") or "{}")
|
| 358 |
+
if not isinstance(period, dict):
|
| 359 |
+
period = {}
|
| 360 |
+
except (ValueError, TypeError):
|
| 361 |
+
period = {}
|
| 362 |
if rows is None:
|
| 363 |
+
unknown.append("This table has not loaded yet. Sync it to request rows from Amazon.")
|
| 364 |
+
if landed.get("phase") in ("refused", "failed", "throttled"):
|
| 365 |
+
unknown = [str(landed.get("cursor") or "Amazon refused this dataset. Retry sync to refresh its reason.")]
|
| 366 |
+
elif rows == 0 or period.get("count") == 0:
|
| 367 |
+
span = " to ".join(dict.fromkeys(str(period[k]) for k in ("start", "end") if period.get(k)))
|
| 368 |
+
unknown.append("Amazon returned no rows for " + (span or "the last requested period") + ".")
|
| 369 |
# Standing rule 1's second half: a limit that cannot be removed is REPORTED, with cause
|
| 370 |
# and fix. `amazon_relational.plan` refuses a table over MAX_ROWS and said so only in the
|
| 371 |
# server log, so the owner saw an empty database and no reason.
|
|
|
|
| 373 |
if over is not None and over > _stored_table_cap():
|
| 374 |
unknown.append(
|
| 375 |
f"The mirror holds {over:,} rows, more than a stored table can hold "
|
| 376 |
+
f"({_stored_table_cap():,}). This table is served read-through from the mirror "
|
| 377 |
+
"with no row cap.")
|
| 378 |
+
# S66-T42/T43 (lead): a table `amazon_relational.apply_plan` could not create because the
|
| 379 |
+
# workspace is at the store's table cap says so here, with the cause and the fix.
|
| 380 |
+
not_stored = entities.get(az.entity_key("not_stored_" + table)) or {}
|
| 381 |
+
if not_stored.get("phase") == "not_stored":
|
| 382 |
+
unknown.append(str(not_stored.get("cursor") or
|
| 383 |
+
"This table is not in this workspace yet because the workspace is at "
|
| 384 |
+
"its database limit. Its rows are kept in the Amazon mirror."))
|
| 385 |
+
if plan is None and landed.get("phase") not in ("refused", "failed", "throttled"):
|
| 386 |
unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.")
|
| 387 |
+
limit = None
|
| 388 |
+
if table in az.GROUP_A_TABLES and table != "catalog_items":
|
| 389 |
+
marker = entities.get(az.entity_key("limit_" + table)) or {}
|
| 390 |
+
try:
|
| 391 |
+
limit = json.loads(marker.get("cursor") or "{}").get("reason")
|
| 392 |
+
except (ValueError, TypeError, AttributeError):
|
| 393 |
+
limit = "History progress is unreadable. Retry sync to refresh its saved date."
|
| 394 |
+
if limit:
|
| 395 |
+
unknown.append(limit)
|
| 396 |
+
if table == "catalog_items":
|
| 397 |
+
marker = entities.get(az.entity_key("limit_" + table)) or {}
|
| 398 |
+
try:
|
| 399 |
+
limit = json.loads(marker.get("cursor") or "{}").get("reason")
|
| 400 |
+
except (ValueError, TypeError, AttributeError):
|
| 401 |
+
limit = "Catalog request progress is unreadable. Sync Amazon listings and retry."
|
| 402 |
+
limit = limit or ("Amazon Catalog allows 20 ASINs per search and 2 requests per second. "
|
| 403 |
+
"Every own ASIN and page is followed; sync Amazon listings first.")
|
| 404 |
+
unknown.append(limit)
|
| 405 |
+
if table == "listing_items":
|
| 406 |
+
limit = ("Amazon Listings allows 1,000 SKUs per query and 5 requests per second. "
|
| 407 |
+
"Date ranges are split and every page followed. Keep the connection enabled.")
|
| 408 |
+
unknown.append(limit)
|
| 409 |
+
if table == az.SQP_TABLE:
|
| 410 |
+
marker = entities.get(az.SQP_LIMIT_KEY) or {}
|
| 411 |
+
if marker.get("cursor"):
|
| 412 |
+
try:
|
| 413 |
+
detail = json.loads(marker["cursor"])
|
| 414 |
+
limit = az.sqp_limit_sentence(detail)
|
| 415 |
+
unknown.append(limit)
|
| 416 |
+
# S66-T41 (lead): one sentence PER REASON, so a whole refused batch (18
|
| 417 |
+
# ASINs) stays inside `assert_safe_status`'s 512-character scalar ceiling and
|
| 418 |
+
# every refused ASIN is still named on screen.
|
| 419 |
+
by_reason = {}
|
| 420 |
+
for asin, refusal in detail.get("refusedAsins", {}).items():
|
| 421 |
+
by_reason.setdefault(str(refusal), []).append(str(asin))
|
| 422 |
+
for refusal, asins in by_reason.items():
|
| 423 |
+
unknown.append(az.sqp_refusal_message(asins[0], refusal) if len(asins) == 1
|
| 424 |
+
else f"Amazon refused ASINs {', '.join(sorted(asins))} for "
|
| 425 |
+
f"Search Query Performance: {refusal}")
|
| 426 |
+
unknown.extend(detail.get("errors", []))
|
| 427 |
+
if detail.get("deferredReason"):
|
| 428 |
+
reason = detail["deferredReason"]
|
| 429 |
+
if detail.get("partialMonthReason"):
|
| 430 |
+
unknown.append(detail["partialMonthReason"])
|
| 431 |
+
except (ValueError, TypeError, KeyError, AttributeError):
|
| 432 |
+
unknown.append("Search Query Performance limits are unknown because the saved marker could not be read. Retry sync to refresh it.")
|
| 433 |
+
if table in az.GROUP_A_TABLES and len(" ".join(unknown)) > 512:
|
| 434 |
+
# Preserve the dataset's refusal/empty-period and vendor limit. Generic schedule
|
| 435 |
+
# diagnostics can otherwise overflow the status guard and hide every database.
|
| 436 |
+
unknown = [part for part in unknown if part != load_unknown
|
| 437 |
+
and not part.startswith(("The sync schedule is unknown", "This table has not loaded yet."))]
|
| 438 |
answer[table] = {
|
| 439 |
"rowsLoaded": rows,
|
| 440 |
"lastSuccess": iso(max(successes)) if successes else None,
|
|
|
|
| 445 |
"stuck": stuck, "stuckCause": stuck_cause,
|
| 446 |
"unknownCause": " ".join(unknown),
|
| 447 |
}
|
| 448 |
+
if table == az.SQP_TABLE or table in az.GROUP_A_TABLES:
|
| 449 |
+
answer[table]["limit"] = limit
|
| 450 |
+
return finish(answer)
|
| 451 |
|
| 452 |
|
| 453 |
#: W55-T19 / R7 / C10 β AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
|
api/content_studio.py
ADDED
|
@@ -0,0 +1,638 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Content Studio: an App template that creates its own References and Posts databases.
|
| 2 |
+
|
| 3 |
+
S67-T40 (owner instruction 1; rulings R17, R18, R21, R11). Plan-only (round 1 R5): nothing here
|
| 4 |
+
publishes to any network. The template creates two TENANT-OWNED databases and binds the App to
|
| 5 |
+
Posts, opening on a calendar of posts. The App is delete-safe (rule 13, ARCHITECTURE section 24):
|
| 6 |
+
deleting it removes only the App record, so References and Posts and every row in them outlive it.
|
| 7 |
+
|
| 8 |
+
This module is the App's own code. Shared files reach it only through a guarded import
|
| 9 |
+
(`routes_templates.py`), so deleting this file removes the template and nothing else breaks.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
TEMPLATE_KEY = "content_studio"
|
| 14 |
+
LABEL = "Content Studio"
|
| 15 |
+
DESCRIPTION = "Plan posts from your references. Creates References and Posts databases."
|
| 16 |
+
|
| 17 |
+
#: R4: the channels a post is planned for. Channel length limits are ADVISED, never enforced (O4).
|
| 18 |
+
CHANNELS = ["LinkedIn", "X", "Instagram", "TikTok", "YouTube", "Reddit"]
|
| 19 |
+
NETWORKS = CHANNELS + ["Other"]
|
| 20 |
+
STATUSES = ["Idea", "Draft", "Ready", "Posted", "Skipped"]
|
| 21 |
+
CALENDAR_VIEW_ID = "cs_calendar"
|
| 22 |
+
|
| 23 |
+
#: R15 (structure, not words): the three fields a Reference is drafted FROM are extracted from its
|
| 24 |
+
#: pasted text by the per-row AI field machinery (`ai_enrich`), re-run when that text changes.
|
| 25 |
+
_EXTRACT = {
|
| 26 |
+
"hook": ("Hook", "In one sentence, describe the opening move this post uses to earn attention "
|
| 27 |
+
"(for example a surprising number, a contrarian claim, a question). Describe "
|
| 28 |
+
"the technique; do not quote the post. Post: {pasted_text}"),
|
| 29 |
+
"structure": ("Structure", "List the post's structure as 3 to 6 short numbered beats, each "
|
| 30 |
+
"naming what that part does (setup, proof, turn, ask). Describe, "
|
| 31 |
+
"never quote. Post: {pasted_text}"),
|
| 32 |
+
"angle": ("Angle", "In one sentence, name the point of view or argument the post takes and "
|
| 33 |
+
"who it is for. Do not quote the post. Post: {pasted_text}"),
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
#: The platform ladder leads with a REASONING model, which spends a few hundred tokens thinking
|
| 38 |
+
#: before it answers; at 300 the column's ceiling was reached with no answer ("the model had
|
| 39 |
+
#: nothing to answer from"), so each extraction gets room for thought plus a short answer.
|
| 40 |
+
EXTRACT_MAX_TOKENS = 1500
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def reference_fields():
|
| 44 |
+
"""References: URL, network, author, pasted text, tags, hook, structure, angle (T40 done-when)."""
|
| 45 |
+
fields = [
|
| 46 |
+
{"key": "title", "label": "Title", "type": "text"},
|
| 47 |
+
{"key": "url", "label": "URL", "type": "url"},
|
| 48 |
+
{"key": "network", "label": "Network", "type": "select", "options": list(NETWORKS)},
|
| 49 |
+
{"key": "author", "label": "Author", "type": "text"},
|
| 50 |
+
{"key": "pasted_text", "label": "Pasted text", "type": "text"},
|
| 51 |
+
{"key": "tags", "label": "Tags", "type": "multiselect", "options": []},
|
| 52 |
+
]
|
| 53 |
+
for key, (label, prompt) in _EXTRACT.items():
|
| 54 |
+
fields.append({"key": key, "label": label, "type": "ai_enrich",
|
| 55 |
+
"aiEnrich": {"prompt": prompt, "maxTokens": EXTRACT_MAX_TOKENS,
|
| 56 |
+
"trigger": {"mode": "on_change"}, "overwrite": "always"}})
|
| 57 |
+
return fields
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def post_fields(references_key):
|
| 61 |
+
"""Posts: caption, channel, scheduled, status, media, campaign, post URL, References link,
|
| 62 |
+
attribution (T40 done-when). `media` is an `image` field: the store has no attachment type."""
|
| 63 |
+
return [
|
| 64 |
+
{"key": "caption", "label": "Caption", "type": "text"},
|
| 65 |
+
{"key": "channel", "label": "Channel", "type": "select", "options": list(CHANNELS)},
|
| 66 |
+
{"key": "scheduled", "label": "Scheduled", "type": "date"},
|
| 67 |
+
{"key": "status", "label": "Status", "type": "select", "options": list(STATUSES)},
|
| 68 |
+
{"key": "media", "label": "Media", "type": "image"},
|
| 69 |
+
{"key": "campaign", "label": "Campaign", "type": "text"},
|
| 70 |
+
{"key": "post_url", "label": "Post URL", "type": "url"},
|
| 71 |
+
{"key": "references", "label": "References", "type": "link",
|
| 72 |
+
"link": {"table": references_key, "selectionMode": "manual"}},
|
| 73 |
+
{"key": "attribution", "label": "Attribution", "type": "text"},
|
| 74 |
+
]
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
class ContentStudioError(RuntimeError):
|
| 78 |
+
"""A refusal the route turns into a plain sentence; nothing half-created is left behind."""
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _create_table(label, username, fields, st):
|
| 82 |
+
"""`user_tables.create` refuses a duplicate label, so a second Studio gets "Posts 2" and so on."""
|
| 83 |
+
from core import user_tables
|
| 84 |
+
for attempt in range(1, 10):
|
| 85 |
+
name = label if attempt == 1 else f"{label} {attempt}"
|
| 86 |
+
key = user_tables.create(name, username, fields=fields, st=st)
|
| 87 |
+
if key:
|
| 88 |
+
return key
|
| 89 |
+
raise ContentStudioError(f"The {label} database could not be created. "
|
| 90 |
+
"Rename or delete an older one, then try again.")
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def create(session, name=""):
|
| 94 |
+
"""Create References + Posts (tenant-owned), a calendar View on Posts, and the App bound to
|
| 95 |
+
Posts. Returns `{address, application, databases}`; on a failure part-way, the databases this
|
| 96 |
+
call created are deleted again, so a refusal leaves the workspace as it was."""
|
| 97 |
+
import application_registry
|
| 98 |
+
from core import user_tables, view_templates
|
| 99 |
+
from core import table_store
|
| 100 |
+
|
| 101 |
+
st = session.runtime
|
| 102 |
+
user = session.uname
|
| 103 |
+
app_name = " ".join(str(name or "").split())[:120] or LABEL
|
| 104 |
+
created = []
|
| 105 |
+
try:
|
| 106 |
+
references = _create_table("References", user, reference_fields(), st)
|
| 107 |
+
created.append(references)
|
| 108 |
+
posts = _create_table("Posts", user, post_fields(references), st)
|
| 109 |
+
created.append(posts)
|
| 110 |
+
ops = table_store.make(view_templates.workspace_key(posts), st=st)
|
| 111 |
+
ops.save_view(user, {"id": CALENDAR_VIEW_ID, "name": "Calendar", "createdBy": user,
|
| 112 |
+
"config": {"filters": [], "sorts": [],
|
| 113 |
+
"display": {"mode": "calendar", "dateField": "scheduled",
|
| 114 |
+
"titleField": "caption"}}})
|
| 115 |
+
application = application_registry.create(
|
| 116 |
+
st, session.tenant, user,
|
| 117 |
+
{"templateKind": TEMPLATE_KEY, "name": app_name, "sourceDatabase": posts,
|
| 118 |
+
"navPlacement": {"section": "app", "order": 0}},
|
| 119 |
+
allowed_databases=None)
|
| 120 |
+
except Exception:
|
| 121 |
+
for key in reversed(created):
|
| 122 |
+
try:
|
| 123 |
+
user_tables.delete(key, st=st)
|
| 124 |
+
except Exception:
|
| 125 |
+
pass
|
| 126 |
+
raise
|
| 127 |
+
# S67-T41 owner correction: the App opens on its own surface in the App module, not in Data.
|
| 128 |
+
return {"address": "#" + app_link(application["id"]), "application": application,
|
| 129 |
+
"databases": {"references": references, "posts": posts}}
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
# ---------------------------------------------------------------------------------------------
|
| 133 |
+
# S67-T41 (R13, R14, R15, R19, R20, R5, O4): DRAFTING. One Draft row per chosen channel, from
|
| 134 |
+
# several selected References plus a one-line topic, written from their extracted STRUCTURE
|
| 135 |
+
# (hook, structure, angle), never their words, with every author credited. Plan-only: a draft
|
| 136 |
+
# is a Posts row with status Draft; nothing is published anywhere.
|
| 137 |
+
# ---------------------------------------------------------------------------------------------
|
| 138 |
+
import re
|
| 139 |
+
|
| 140 |
+
from fastapi import APIRouter, Body, Depends
|
| 141 |
+
|
| 142 |
+
from deps import Session, err, require_session
|
| 143 |
+
|
| 144 |
+
router = APIRouter(prefix="/content-studio")
|
| 145 |
+
|
| 146 |
+
#: O4: channel limits are ADVICE in the prompt and on screen, never a truncation.
|
| 147 |
+
CHANNEL_ADVICE = {
|
| 148 |
+
"LinkedIn": "LinkedIn post: up to about 3,000 characters, short paragraphs, no hashtag wall.",
|
| 149 |
+
"X": "X post: aim for 280 characters or fewer.",
|
| 150 |
+
"Instagram": "Instagram caption: up to about 2,200 characters; a strong first line.",
|
| 151 |
+
"TikTok": "TikTok caption: short, about 150 characters.",
|
| 152 |
+
"YouTube": "YouTube: a title line, then a two or three sentence description.",
|
| 153 |
+
"Reddit": "Reddit: a title line, then the body; plain, specific, no marketing tone.",
|
| 154 |
+
}
|
| 155 |
+
DRAFT_SYSTEM = (
|
| 156 |
+
"You draft ONE original social media post for the channel named. You are given a one-line "
|
| 157 |
+
"topic and the STRUCTURE of several reference posts: the technique of each hook, its beats "
|
| 158 |
+
"and its angle. Borrow those structural moves; write entirely new sentences. Never copy or "
|
| 159 |
+
"closely paraphrase any sentence from anywhere. Answer with the post text only: no preamble, "
|
| 160 |
+
"no quotation marks around it, no markdown heading. Never use an em dash or an en dash; use "
|
| 161 |
+
"a comma, a colon or a full stop.")
|
| 162 |
+
LEDGER_SURFACE = "field_agent"
|
| 163 |
+
DRAFT_MAX_TOKENS = 3000 # a reasoning model thinks first; see EXTRACT_MAX_TOKENS
|
| 164 |
+
DRAFT_TIMEOUT_S = 60
|
| 165 |
+
#: A shared run of this many words is a copied sentence, whatever surrounds it (R15).
|
| 166 |
+
VERBATIM_WORDS = 8
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _studio(session, app_id):
|
| 170 |
+
"""(app, posts_key, references_key) for a Content Studio App this session may open."""
|
| 171 |
+
import application_registry
|
| 172 |
+
from core import user_tables
|
| 173 |
+
doc = application_registry.load(session.runtime)
|
| 174 |
+
app = doc["applications"].get(str(app_id or ""))
|
| 175 |
+
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KEY
|
| 176 |
+
or not application_registry.may_read(app, session.uname, session.admin)):
|
| 177 |
+
raise err(404, "content_studio_not_found", "That Content Studio does not exist.")
|
| 178 |
+
posts = str(app.get("sourceDatabase") or "")
|
| 179 |
+
defn = user_tables.get(posts, session.runtime) or {}
|
| 180 |
+
link = next((f.get("link") for f in defn.get("fields") or ()
|
| 181 |
+
if isinstance(f, dict) and f.get("key") == "references"), None)
|
| 182 |
+
references = str((link or {}).get("table") or "")
|
| 183 |
+
if not posts or not references:
|
| 184 |
+
raise err(409, "content_studio_incomplete",
|
| 185 |
+
"This Content Studio has lost its Posts or References database.")
|
| 186 |
+
return app, posts, references
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _reference_rows(session, references):
|
| 190 |
+
from core import user_tables
|
| 191 |
+
defn = user_tables.get(references, session.runtime) or {}
|
| 192 |
+
rows = defn.get("rows") or {}
|
| 193 |
+
return {str(rid): (row if isinstance(row, dict) else {}) for rid, row in rows.items()}
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
# ---------------------------------------------------------------------------------------------
|
| 197 |
+
# S67-T42 (R35, standing rule 1): THE MONTHLY DRAFTING ALLOWANCE. Drafting runs on Loopable's
|
| 198 |
+
# platform keys, so each tenant gets a monthly count of drafts (one Draft row = one draft). Past
|
| 199 |
+
# it, Draft pauses with a sentence naming the reset date; the platform admin raises it per tenant
|
| 200 |
+
# (`PUT /content-studio/allowance`). A tenant admin cannot: that would defeat R35.
|
| 201 |
+
# One per-tenant bucket, owned by this module: {"allowance": int (absent = default),
|
| 202 |
+
# "months": {"YYYY-MM": drafts made}}. Month and reset are UTC.
|
| 203 |
+
# ---------------------------------------------------------------------------------------------
|
| 204 |
+
ALLOWANCE_KEY = "content_studio_usage"
|
| 205 |
+
DEFAULT_MONTHLY_DRAFTS = 100
|
| 206 |
+
ALLOWANCE_KEEP_MONTHS = 12
|
| 207 |
+
_MONTH_NAMES = ("January", "February", "March", "April", "May", "June", "July", "August",
|
| 208 |
+
"September", "October", "November", "December")
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _now():
|
| 212 |
+
from datetime import datetime, timezone
|
| 213 |
+
return datetime.now(timezone.utc)
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def _month_key(now):
|
| 217 |
+
return f"{now.year:04d}-{now.month:02d}"
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _reset_on(now):
|
| 221 |
+
"""The first day of next month (UTC): the day this month's count stops counting."""
|
| 222 |
+
from datetime import date
|
| 223 |
+
return date(now.year + 1, 1, 1) if now.month == 12 else date(now.year, now.month + 1, 1)
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def _spoken_date(day):
|
| 227 |
+
return f"{_MONTH_NAMES[day.month - 1]} {day.day}, {day.year}"
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _limit(doc):
|
| 231 |
+
raw = (doc or {}).get("allowance")
|
| 232 |
+
if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0:
|
| 233 |
+
return DEFAULT_MONTHLY_DRAFTS
|
| 234 |
+
return raw
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def allowance_state(doc, now=None):
|
| 238 |
+
"""{used, limit, remaining, resetsOn, paused, message}: the ONE reading of month and reset
|
| 239 |
+
that both the panel and the enforcement use."""
|
| 240 |
+
now = now or _now()
|
| 241 |
+
doc = doc if isinstance(doc, dict) else {}
|
| 242 |
+
months = doc.get("months") if isinstance(doc.get("months"), dict) else {}
|
| 243 |
+
try:
|
| 244 |
+
used = max(0, int(months.get(_month_key(now)) or 0))
|
| 245 |
+
except (TypeError, ValueError):
|
| 246 |
+
used = 0
|
| 247 |
+
limit = _limit(doc)
|
| 248 |
+
remaining = max(0, limit - used)
|
| 249 |
+
reset = _reset_on(now)
|
| 250 |
+
paused = remaining <= 0
|
| 251 |
+
message = (f"This workspace has used this month's drafting allowance. "
|
| 252 |
+
f"It resets on {_spoken_date(reset)}.") if paused else ""
|
| 253 |
+
return {"used": used, "limit": limit, "remaining": remaining,
|
| 254 |
+
"resetsOn": reset.isoformat(), "paused": paused, "message": message}
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def _short_message(remaining):
|
| 258 |
+
drafts = "1 draft" if remaining == 1 else f"{remaining} drafts"
|
| 259 |
+
channels = "1 channel" if remaining == 1 else f"up to {remaining} channels"
|
| 260 |
+
return f"This workspace has {drafts} left this month. Choose {channels}."
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def _refusal(state, wanted):
|
| 264 |
+
"""The 429 a draft of `wanted` drafts earns against `state`, or None when it fits."""
|
| 265 |
+
if state["remaining"] <= 0:
|
| 266 |
+
return err(429, "drafting_allowance_used", state["message"])
|
| 267 |
+
if state["remaining"] < wanted:
|
| 268 |
+
return err(429, "drafting_allowance_short", _short_message(state["remaining"]))
|
| 269 |
+
return None
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _read_allowance(st):
|
| 273 |
+
doc = st.get(ALLOWANCE_KEY)
|
| 274 |
+
return doc if isinstance(doc, dict) else {}
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def _count(st, month, delta):
|
| 278 |
+
"""Add `delta` drafts to `month` in one read-modify-write; keeps the last 12 months."""
|
| 279 |
+
def _up(doc):
|
| 280 |
+
doc = dict(doc) if isinstance(doc, dict) else {}
|
| 281 |
+
months = dict(doc.get("months")) if isinstance(doc.get("months"), dict) else {}
|
| 282 |
+
months[month] = max(0, int(months.get(month) or 0) + int(delta))
|
| 283 |
+
for stale in sorted(months)[:-ALLOWANCE_KEEP_MONTHS]:
|
| 284 |
+
months.pop(stale, None)
|
| 285 |
+
doc["months"] = months
|
| 286 |
+
return doc
|
| 287 |
+
st.update(ALLOWANCE_KEY, _up, flush="sync")
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def reserve_drafts(st, wanted, now=None):
|
| 291 |
+
"""Check AND reserve `wanted` drafts in ONE store update, so two concurrent drafts cannot both
|
| 292 |
+
take the last slot. Raises the 429 (no write) when it does not fit; returns the month key the
|
| 293 |
+
reservation was booked to, for `release_drafts` on a failure."""
|
| 294 |
+
now = now or _now()
|
| 295 |
+
try:
|
| 296 |
+
current = _read_allowance(st)
|
| 297 |
+
except Exception: # noqa: BLE001
|
| 298 |
+
raise err(503, "allowance_unavailable",
|
| 299 |
+
"The drafting allowance could not be checked, so nothing was drafted. Try again.")
|
| 300 |
+
refusal = _refusal(allowance_state(current, now), wanted)
|
| 301 |
+
if refusal is not None: # fast path: a paused workspace writes nothing
|
| 302 |
+
raise refusal
|
| 303 |
+
month = _month_key(now)
|
| 304 |
+
verdict = {}
|
| 305 |
+
|
| 306 |
+
def _up(doc):
|
| 307 |
+
doc = dict(doc) if isinstance(doc, dict) else {}
|
| 308 |
+
refused = _refusal(allowance_state(doc, now), wanted)
|
| 309 |
+
if refused is not None:
|
| 310 |
+
verdict["refusal"] = refused
|
| 311 |
+
return doc
|
| 312 |
+
months = dict(doc.get("months")) if isinstance(doc.get("months"), dict) else {}
|
| 313 |
+
months[month] = max(0, int(months.get(month) or 0)) + int(wanted)
|
| 314 |
+
for stale in sorted(months)[:-ALLOWANCE_KEEP_MONTHS]:
|
| 315 |
+
months.pop(stale, None)
|
| 316 |
+
doc["months"] = months
|
| 317 |
+
return doc
|
| 318 |
+
|
| 319 |
+
try:
|
| 320 |
+
st.update(ALLOWANCE_KEY, _up, flush="sync")
|
| 321 |
+
except Exception: # noqa: BLE001
|
| 322 |
+
raise err(503, "allowance_unavailable",
|
| 323 |
+
"The drafting allowance could not be checked, so nothing was drafted. Try again.")
|
| 324 |
+
if "refusal" in verdict:
|
| 325 |
+
raise verdict["refusal"]
|
| 326 |
+
return month
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def release_drafts(st, month, n):
|
| 330 |
+
"""Hand back a reservation whose drafts were never written (AI failure, copy refusal, store)."""
|
| 331 |
+
try:
|
| 332 |
+
_count(st, month, -int(n))
|
| 333 |
+
except Exception: # noqa: BLE001
|
| 334 |
+
pass
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
@router.put("/allowance")
|
| 338 |
+
def set_allowance(body: dict = Body(default=None), session: Session = Depends(require_session)):
|
| 339 |
+
"""The owner raises (or resets) one tenant's monthly drafting allowance. Platform admin only."""
|
| 340 |
+
import core.platform_admin as platform_admin
|
| 341 |
+
from harness import runtime
|
| 342 |
+
if not platform_admin.is_platform_admin(session.user):
|
| 343 |
+
raise err(403, "forbidden", "your account does not have access to this surface")
|
| 344 |
+
body = body if isinstance(body, dict) else {}
|
| 345 |
+
slug = str(body.get("tenant") or "").strip().lower()
|
| 346 |
+
monthly = body.get("monthly")
|
| 347 |
+
if monthly is not None and (isinstance(monthly, bool) or not isinstance(monthly, int)
|
| 348 |
+
or monthly < 0):
|
| 349 |
+
raise err(400, "bad_allowance",
|
| 350 |
+
"The monthly allowance must be a whole number of drafts, 0 or more.")
|
| 351 |
+
if not slug or slug not in runtime.known_tenants():
|
| 352 |
+
raise err(404, "tenant_not_found", "That workspace does not exist.")
|
| 353 |
+
try:
|
| 354 |
+
st = runtime.get_runtime(slug)
|
| 355 |
+
except KeyError:
|
| 356 |
+
raise err(404, "tenant_not_found", "That workspace does not exist.")
|
| 357 |
+
|
| 358 |
+
def _up(doc):
|
| 359 |
+
doc = dict(doc) if isinstance(doc, dict) else {}
|
| 360 |
+
if monthly is None:
|
| 361 |
+
doc.pop("allowance", None)
|
| 362 |
+
else:
|
| 363 |
+
doc["allowance"] = int(monthly)
|
| 364 |
+
return doc
|
| 365 |
+
|
| 366 |
+
st.update(ALLOWANCE_KEY, _up, flush="sync")
|
| 367 |
+
return {"tenant": slug, "allowance": allowance_state(_read_allowance(st))}
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
@router.get("/references")
|
| 371 |
+
def list_references(app: str = "", session: Session = Depends(require_session)):
|
| 372 |
+
"""The References a Draft can use: id, title, author, network, and whether its structure is in.
|
| 373 |
+
Also this workspace's drafting allowance, so the panel reports the limit before it bites."""
|
| 374 |
+
_app, _posts, references = _studio(session, app)
|
| 375 |
+
rows = _reference_rows(session, references)
|
| 376 |
+
try:
|
| 377 |
+
allowance = allowance_state(_read_allowance(session.runtime))
|
| 378 |
+
except Exception: # noqa: BLE001
|
| 379 |
+
allowance = None # unreadable: the draft route still enforces it
|
| 380 |
+
return {"references": [
|
| 381 |
+
{"id": rid, "title": str(row.get("title") or row.get("url") or "Untitled reference")[:160],
|
| 382 |
+
"author": str(row.get("author") or "")[:120], "network": str(row.get("network") or ""),
|
| 383 |
+
"extracted": bool(str(row.get("structure") or "").strip())}
|
| 384 |
+
for rid, row in rows.items()],
|
| 385 |
+
"channels": list(CHANNELS), "advice": dict(CHANNEL_ADVICE), "allowance": allowance}
|
| 386 |
+
|
| 387 |
+
|
| 388 |
+
# ---------------------------------------------------------------------------------------------
|
| 389 |
+
# S67-T41 OWNER CORRECTION (Telegram update 437342853, 2026-09-28): "Content studio should be an
|
| 390 |
+
# App ... it should act like Catalog / Route where it just uses the data module as what its
|
| 391 |
+
# building on top." So the App opens in the App module on its OWN surface (`#/app?studio=<id>`,
|
| 392 |
+
# `apps/contentStudio/ContentStudioApp.tsx`): a calendar and a board over Posts, the Draft door,
|
| 393 |
+
# and the two databases underneath. Data still holds References and Posts; the App never sends
|
| 394 |
+
# anyone there to use it.
|
| 395 |
+
# β The shared `/applications` doors (`routes_assets.py`) admit only topic-backed databases, so
|
| 396 |
+
# they refuse an App bound to a user table; the read, star and delete this App needs live here,
|
| 397 |
+
# on the registry's own writers, behind the same `_studio` + `may_read` / `may_edit` walls.
|
| 398 |
+
# ---------------------------------------------------------------------------------------------
|
| 399 |
+
def app_link(app_id):
|
| 400 |
+
"""The one address a Content Studio App opens at (App module, its own surface)."""
|
| 401 |
+
return f"/app?studio={app_id}"
|
| 402 |
+
|
| 403 |
+
|
| 404 |
+
#: The Posts fields the surface draws, in the grid's own Field shape so `viewModes.CalendarView`
|
| 405 |
+
#: (by Scheduled) and `KanbanView` (by Status) render them unchanged.
|
| 406 |
+
SURFACE_FIELDS = ("caption", "channel", "scheduled", "status", "attribution", "campaign")
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
def _surface_field(defn):
|
| 410 |
+
kind = {"date": "date", "select": "select"}.get(defn.get("type"), "text")
|
| 411 |
+
field = {"key": defn["key"], "label": str(defn.get("label") or defn["key"]), "type": kind,
|
| 412 |
+
"source": "overlay"}
|
| 413 |
+
if kind == "select":
|
| 414 |
+
field["options"] = [str(o) for o in (defn.get("options") or ())]
|
| 415 |
+
return field
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
def _studio_or_missing(session, app_id):
|
| 419 |
+
"""`_studio`, answering an absent App with the code the App home already reads as 'gone'."""
|
| 420 |
+
try:
|
| 421 |
+
return _studio(session, app_id)
|
| 422 |
+
except Exception as exc: # noqa: BLE001
|
| 423 |
+
if getattr(exc, "status_code", None) == 404:
|
| 424 |
+
raise err(404, "application_not_found", "That Content Studio no longer exists.")
|
| 425 |
+
raise
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
@router.get("/studio")
|
| 429 |
+
def studio(app: str = "", session: Session = Depends(require_session)):
|
| 430 |
+
"""Everything the App surface draws: the App, Posts (surface fields + rows), References'
|
| 431 |
+
count, the statuses in board order, and the drafting allowance."""
|
| 432 |
+
from core import user_tables
|
| 433 |
+
application, posts, references = _studio(session, app)
|
| 434 |
+
if not user_tables.may_open(posts, session.uname, session.admin, st=session.runtime):
|
| 435 |
+
raise err(404, "content_studio_not_found", "That Content Studio does not exist.")
|
| 436 |
+
posts_def = user_tables.get(posts, session.runtime) or {}
|
| 437 |
+
refs_def = user_tables.get(references, session.runtime) or {}
|
| 438 |
+
by_key = {f.get("key"): f for f in posts_def.get("fields") or () if isinstance(f, dict)}
|
| 439 |
+
fields = [_surface_field(by_key[key]) for key in SURFACE_FIELDS if key in by_key]
|
| 440 |
+
rows = []
|
| 441 |
+
for rid, row in (posts_def.get("rows") or {}).items():
|
| 442 |
+
try:
|
| 443 |
+
pid = int(rid)
|
| 444 |
+
except (TypeError, ValueError):
|
| 445 |
+
continue
|
| 446 |
+
row = row if isinstance(row, dict) else {}
|
| 447 |
+
rows.append({"pid": pid, **{key: row.get(key) for key in SURFACE_FIELDS}})
|
| 448 |
+
rows.sort(key=lambda r: r["pid"])
|
| 449 |
+
try:
|
| 450 |
+
allowance = allowance_state(_read_allowance(session.runtime))
|
| 451 |
+
except Exception: # noqa: BLE001
|
| 452 |
+
allowance = None
|
| 453 |
+
return {"app": {"id": application["id"], "name": application.get("name") or LABEL},
|
| 454 |
+
"posts": {"database": posts, "label": str(posts_def.get("label") or "Posts"),
|
| 455 |
+
"fields": fields, "rows": rows},
|
| 456 |
+
"references": {"database": references,
|
| 457 |
+
"label": str(refs_def.get("label") or "References"),
|
| 458 |
+
"count": len(refs_def.get("rows") or {})},
|
| 459 |
+
"statuses": list(STATUSES), "allowance": allowance}
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
@router.patch("/apps/{app_id}")
|
| 463 |
+
def star_app(app_id: str, body: dict = Body(default=None),
|
| 464 |
+
session: Session = Depends(require_session)):
|
| 465 |
+
"""Star or unstar a Content Studio on the App home (the registry's own star writer)."""
|
| 466 |
+
import application_registry
|
| 467 |
+
_app, posts, _references = _studio_or_missing(session, app_id)
|
| 468 |
+
body = body if isinstance(body, dict) else {}
|
| 469 |
+
if set(body) != {"starred"} or not isinstance(body.get("starred"), bool):
|
| 470 |
+
raise err(400, "bad_patch", "Only the star of a Content Studio changes here.")
|
| 471 |
+
try:
|
| 472 |
+
landed = application_registry.mutate(session.runtime, app_id, session.uname, session.admin,
|
| 473 |
+
{"starred": body["starred"]}, allowed_databases=[posts])
|
| 474 |
+
except PermissionError:
|
| 475 |
+
raise err(403, "forbidden", "You cannot change this App.")
|
| 476 |
+
except application_registry.ApplicationError:
|
| 477 |
+
raise err(404, "application_not_found", "That Content Studio no longer exists.")
|
| 478 |
+
return {"application": landed}
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
@router.delete("/apps/{app_id}")
|
| 482 |
+
def delete_app(app_id: str, body: dict = Body(default=None),
|
| 483 |
+
session: Session = Depends(require_session)):
|
| 484 |
+
"""Delete the App and nothing else (rule 13): References and Posts, every row in them, and
|
| 485 |
+
every View on them outlive it. The name is confirmed inside the registry's own update."""
|
| 486 |
+
import application_registry
|
| 487 |
+
application, posts, _references = _studio_or_missing(session, app_id)
|
| 488 |
+
confirm = (body or {}).get("confirmName") if isinstance(body, dict) else None
|
| 489 |
+
if not isinstance(confirm, str) or confirm != application.get("name"):
|
| 490 |
+
raise err(400, "confirm_name", "Enter the exact App name to confirm.")
|
| 491 |
+
|
| 492 |
+
class _NameConfirmed:
|
| 493 |
+
def update(self, key, delete):
|
| 494 |
+
def confirmed(raw):
|
| 495 |
+
deleted = delete(raw)
|
| 496 |
+
if (raw.get("applications") or {}).get(app_id, {}).get("name") != confirm:
|
| 497 |
+
raise application_registry.ApplicationError("the App name changed")
|
| 498 |
+
return deleted
|
| 499 |
+
return session.runtime.update(key, confirmed)
|
| 500 |
+
|
| 501 |
+
try:
|
| 502 |
+
return application_registry.delete(_NameConfirmed(), app_id, session.uname, session.admin,
|
| 503 |
+
allowed_databases=[posts])
|
| 504 |
+
except PermissionError:
|
| 505 |
+
raise err(403, "forbidden", "You no longer have permission to delete this App.")
|
| 506 |
+
except application_registry.ApplicationError:
|
| 507 |
+
raise err(400, "confirm_name", "The App name may have changed. Close this dialog and reload to retry.")
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
def _words(text):
|
| 511 |
+
return re.findall(r"[a-z0-9']+", str(text or "").lower())
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
def copied_sentence(draft, sources):
|
| 515 |
+
"""The first run of VERBATIM_WORDS consecutive words a draft shares with any source, or ''."""
|
| 516 |
+
grams = set()
|
| 517 |
+
for source in sources:
|
| 518 |
+
words = _words(source)
|
| 519 |
+
grams.update(" ".join(words[i:i + VERBATIM_WORDS])
|
| 520 |
+
for i in range(0, max(0, len(words) - VERBATIM_WORDS + 1)))
|
| 521 |
+
words = _words(draft)
|
| 522 |
+
for i in range(0, max(0, len(words) - VERBATIM_WORDS + 1)):
|
| 523 |
+
gram = " ".join(words[i:i + VERBATIM_WORDS])
|
| 524 |
+
if gram in grams:
|
| 525 |
+
return gram
|
| 526 |
+
return ""
|
| 527 |
+
|
| 528 |
+
|
| 529 |
+
def attribution(rows):
|
| 530 |
+
"""Every selected reference's author, credited in plain words (R14 traceability)."""
|
| 531 |
+
names = list(dict.fromkeys(
|
| 532 |
+
str(row.get("author") or row.get("title") or "an unnamed source").strip() for row in rows))
|
| 533 |
+
joined = names[0] if len(names) == 1 else ", ".join(names[:-1]) + " and " + names[-1]
|
| 534 |
+
return f"Structure drawn from posts by {joined}."
|
| 535 |
+
|
| 536 |
+
|
| 537 |
+
def draft_prompt(topic, channel, rows):
|
| 538 |
+
parts = [f"Channel: {channel}. {CHANNEL_ADVICE.get(channel, '')}", f"Topic: {topic}", ""]
|
| 539 |
+
for n, row in enumerate(rows, 1):
|
| 540 |
+
parts.append(f"Reference {n} ({row.get('network') or 'post'}):")
|
| 541 |
+
for key, label in (("hook", "Hook technique"), ("structure", "Structure"), ("angle", "Angle")):
|
| 542 |
+
value = " ".join(str(row.get(key) or "").split())[:900]
|
| 543 |
+
if value:
|
| 544 |
+
parts.append(f" {label}: {value}")
|
| 545 |
+
parts.append("")
|
| 546 |
+
parts.append("Write the post now.")
|
| 547 |
+
return "\n".join(parts)
|
| 548 |
+
|
| 549 |
+
|
| 550 |
+
def draft_posts(session, app_id, reference_ids, topic, channels, *, ask=None):
|
| 551 |
+
"""Generate EVERY channel first; write the Posts rows only if all succeeded (no half row)."""
|
| 552 |
+
topic = " ".join(str(topic or "").split())[:200]
|
| 553 |
+
ids = [str(x) for x in dict.fromkeys(reference_ids or ())][:50]
|
| 554 |
+
wanted = [c for c in dict.fromkeys(channels or ()) if c in CHANNELS]
|
| 555 |
+
if not ids:
|
| 556 |
+
raise err(400, "no_references", "Select at least one reference to draft from.")
|
| 557 |
+
if not topic:
|
| 558 |
+
raise err(400, "no_topic", "Type a one-line topic for the draft.")
|
| 559 |
+
if not wanted:
|
| 560 |
+
raise err(400, "no_channels", "Choose at least one channel.")
|
| 561 |
+
_app, posts, references = _studio(session, app_id)
|
| 562 |
+
rows_by_id = _reference_rows(session, references)
|
| 563 |
+
missing = [rid for rid in ids if rid not in rows_by_id]
|
| 564 |
+
if missing:
|
| 565 |
+
raise err(409, "references_missing", "A selected reference no longer exists. Reload and try again.")
|
| 566 |
+
chosen = [rows_by_id[rid] for rid in ids]
|
| 567 |
+
if not any(str(row.get("structure") or "").strip() for row in chosen):
|
| 568 |
+
raise err(409, "references_not_extracted",
|
| 569 |
+
"The selected references have no extracted structure yet. Paste their text first.")
|
| 570 |
+
# R35: the allowance is checked and reserved BEFORE any provider is resolved or called. One
|
| 571 |
+
# draft per channel; the reservation is handed back unless every row is written.
|
| 572 |
+
month = reserve_drafts(session.runtime, len(wanted))
|
| 573 |
+
try:
|
| 574 |
+
return _generate_and_save(session, posts, ids, topic, wanted, chosen, ask=ask)
|
| 575 |
+
except BaseException:
|
| 576 |
+
release_drafts(session.runtime, month, len(wanted))
|
| 577 |
+
raise
|
| 578 |
+
|
| 579 |
+
|
| 580 |
+
def _generate_and_save(session, posts, ids, topic, wanted, chosen, *, ask=None):
|
| 581 |
+
import ai_enrich
|
| 582 |
+
import ai_review
|
| 583 |
+
import usage_ledger
|
| 584 |
+
from core import user_tables
|
| 585 |
+
|
| 586 |
+
live = ai_review.ladder()
|
| 587 |
+
if not live and ask is None:
|
| 588 |
+
raise err(503, "ai_unavailable", "No AI provider is configured here, so nothing was drafted.")
|
| 589 |
+
provider = (live or [{"name": "injected", "model": "injected"}])[0]
|
| 590 |
+
model = provider["model"]
|
| 591 |
+
caller = ask or ai_enrich._ask
|
| 592 |
+
sources = [row.get("pasted_text") or "" for row in chosen]
|
| 593 |
+
drafts, tokens, calls = [], 0, 0
|
| 594 |
+
try:
|
| 595 |
+
for channel in wanted:
|
| 596 |
+
prompt = draft_prompt(topic, channel, chosen)
|
| 597 |
+
text = ""
|
| 598 |
+
for attempt in range(2):
|
| 599 |
+
answer, used, problem = caller(provider, model, prompt, DRAFT_MAX_TOKENS,
|
| 600 |
+
DRAFT_TIMEOUT_S, system=DRAFT_SYSTEM)
|
| 601 |
+
calls += 1
|
| 602 |
+
tokens += int(used or 0)
|
| 603 |
+
if problem or not str(answer or "").strip():
|
| 604 |
+
raise err(502, "draft_failed", f"The {channel} draft could not be written "
|
| 605 |
+
f"({problem or 'the model returned nothing'}). Nothing was saved.")
|
| 606 |
+
text = ai_enrich.no_dashes(str(answer).strip())
|
| 607 |
+
copied = copied_sentence(text, sources)
|
| 608 |
+
if not copied:
|
| 609 |
+
break
|
| 610 |
+
prompt += f"\n\nYour last draft reused this wording from a reference: \"{copied}\". Rewrite it in new words."
|
| 611 |
+
text = ""
|
| 612 |
+
if not text:
|
| 613 |
+
raise err(502, "draft_copied", f"The {channel} draft kept repeating a reference's "
|
| 614 |
+
"wording, so it was not saved. Nothing was saved.")
|
| 615 |
+
drafts.append({"channel": channel, "caption": text})
|
| 616 |
+
finally:
|
| 617 |
+
if calls:
|
| 618 |
+
# `usage_ledger.SURFACES` names four surfaces and refuses any other, so drafts are
|
| 619 |
+
# booked as `field_agent` (AI-written cells of a user table) until a `content_studio`
|
| 620 |
+
# surface exists there (a finding for the ledger's owner; this module cannot add it).
|
| 621 |
+
usage_ledger.record(LEDGER_SURFACE, provider["name"], model, total=tokens,
|
| 622 |
+
calls=calls, st=session.runtime, user=session.uname)
|
| 623 |
+
credit = attribution(chosen)
|
| 624 |
+
new_rows = [{"caption": d["caption"], "channel": d["channel"], "status": "Draft",
|
| 625 |
+
"references": ",".join(ids), "attribution": credit} for d in drafts]
|
| 626 |
+
created = user_tables.add_rows(posts, new_rows, username=session.uname, st=session.runtime)
|
| 627 |
+
if not created:
|
| 628 |
+
raise err(503, "store_unavailable", "The drafts could not be saved. Nothing was saved.")
|
| 629 |
+
return {"created": list(created) if isinstance(created, (list, tuple)) else created,
|
| 630 |
+
"drafts": drafts, "attribution": credit, "tokens": tokens, "calls": calls,
|
| 631 |
+
"provider": provider["name"], "model": model}
|
| 632 |
+
|
| 633 |
+
|
| 634 |
+
@router.post("/draft")
|
| 635 |
+
def draft(body: dict = Body(default=None), session: Session = Depends(require_session)):
|
| 636 |
+
body = body if isinstance(body, dict) else {}
|
| 637 |
+
return draft_posts(session, body.get("app"), body.get("references"), body.get("topic"),
|
| 638 |
+
body.get("channels"))
|
api/deps.py
CHANGED
|
@@ -232,6 +232,28 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
|
|
| 232 |
else ("product_pool", team_id))
|
| 233 |
cache = getattr(session.runtime, "pool_cache", None)
|
| 234 |
entry = cache.get(cache_key) if isinstance(cache, dict) else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
if (not isinstance(entry, tuple) or len(entry) < 2
|
| 236 |
or not isinstance(entry[1], (list, tuple))
|
| 237 |
or any(not isinstance(row, dict) for row in entry[1])):
|
|
@@ -247,6 +269,44 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
|
|
| 247 |
_ASSISTANT_USER_TABLE_KINDS = frozenset({"materialised_user_table", "mirror_user_table"})
|
| 248 |
|
| 249 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 250 |
def _assistant_table_defn(session: Session, database: str):
|
| 251 |
"""The rows-free definition for one `ut_*` database plus its lent runtime, after `may_open`.
|
| 252 |
|
|
@@ -276,7 +336,9 @@ def _assistant_table_defn(session: Session, database: str):
|
|
| 276 |
if not isinstance(definition, dict):
|
| 277 |
_assistant_refuse(404, "assistant_unknown_database", "that database is not available")
|
| 278 |
definition_runtime = user_tables.lend(session.runtime, user_tables=definitions)
|
| 279 |
-
|
|
|
|
|
|
|
| 280 |
# Deliberately no label/owner in this response: knowing a key must not become an oracle
|
| 281 |
# for somebody else's database.
|
| 282 |
_assistant_refuse(403, "assistant_forbidden",
|
|
@@ -346,7 +408,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
|
|
| 346 |
|
| 347 |
|
| 348 |
def _assistant_mirror_table(session: Session, database: str, requested_fields=None, filters=None,
|
| 349 |
-
*, defn=None, lent=None):
|
| 350 |
"""Read a MIRROR-SERVED user table through the same filter compiler the grid uses (R11 / C7).
|
| 351 |
|
| 352 |
Returns `_assistant_user_table`'s 4-tuple - `(rows, fields, source_version, label)` - so the
|
|
@@ -382,11 +444,24 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 382 |
# THE CHEAP QUESTION FIRST, and it needs no cursor: is this key a connected Odoo grid at all?
|
| 383 |
# A `readThrough` table that is not one has no mirror rows anywhere, and asking the mirror
|
| 384 |
# about it would open a cursor to learn nothing.
|
| 385 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 386 |
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
| 387 |
"this database does not store its rows and it is not one of the "
|
| 388 |
"connected databases the assistant can read from the mirror")
|
| 389 |
-
|
|
|
|
|
|
|
|
|
|
| 390 |
if refusal is not None:
|
| 391 |
_assistant_refuse(refusal.status, refusal.code, refusal.message)
|
| 392 |
spec = assistant_mirror.binding(cur, database)
|
|
@@ -394,7 +469,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 394 |
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
| 395 |
"this database is served live from its connector and this deployment "
|
| 396 |
"has no read-through binding for it yet, so it holds no rows to read")
|
| 397 |
-
fields = assistant_mirror.answerable_fields(database, spec)
|
| 398 |
if not fields:
|
| 399 |
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
| 400 |
"this database's connector mirror carries none of its declared columns "
|
|
@@ -422,12 +497,36 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 422 |
if grant is None:
|
| 423 |
wall_tree = None
|
| 424 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 425 |
try:
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 431 |
except assistant_mirror.MirrorRefusal as exc:
|
| 432 |
_assistant_refuse(exc.status, exc.code, exc.message)
|
| 433 |
raise # unreachable; see above
|
|
@@ -459,7 +558,33 @@ def _assistant_requested_fields(fields, visible_fields):
|
|
| 459 |
return out
|
| 460 |
|
| 461 |
|
| 462 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 463 |
"""Return a detached, permissioned snapshot for one Assistant tool call.
|
| 464 |
|
| 465 |
Signature (the C8 contract for E):
|
|
@@ -468,6 +593,12 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
|
|
| 468 |
permission_scope_applied, fields, filters, records
|
| 469 |
}
|
| 470 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 471 |
`database` is exactly one source. `fields` is an explicit list of visible field keys (or
|
| 472 |
``None`` for every visible field). `filters` is ``None`` or the strict tree accepted by
|
| 473 |
`core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and
|
|
@@ -478,6 +609,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
|
|
| 478 |
from harness import filter_eval
|
| 479 |
|
| 480 |
key = str(database or "").strip()
|
|
|
|
| 481 |
if key in _ASSISTANT_BUILT_INS:
|
| 482 |
rows, canonical_fields, source_version = _assistant_cached_builtin(session, key, fields, filters)
|
| 483 |
label = _ASSISTANT_BUILT_INS[key]["label"]
|
|
@@ -494,12 +626,15 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
|
|
| 494 |
from core import user_tables
|
| 495 |
_defn, _lent = _assistant_table_defn(session, key)
|
| 496 |
if user_tables.materialises(key, st=_lent, defn=_defn):
|
|
|
|
|
|
|
|
|
|
| 497 |
rows, canonical_fields, source_version, label = _assistant_user_table(
|
| 498 |
session, key, fields, filters, defn=_defn, lent=_lent)
|
| 499 |
source_kind = "materialised_user_table"
|
| 500 |
else:
|
| 501 |
rows, canonical_fields, source_version, label = _assistant_mirror_table(
|
| 502 |
-
session, key, fields, filters, defn=_defn, lent=_lent)
|
| 503 |
source_kind = "mirror_user_table"
|
| 504 |
else:
|
| 505 |
_assistant_refuse(404, "assistant_unknown_database", "that database is not available")
|
|
@@ -556,7 +691,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
|
|
| 556 |
records.append({"pid": copy.deepcopy(row["pid"]),
|
| 557 |
**{field: copy.deepcopy(row.get(field)) for field in selected_keys}})
|
| 558 |
|
| 559 |
-
|
| 560 |
"database": key,
|
| 561 |
"label": str(label),
|
| 562 |
"source_kind": source_kind,
|
|
@@ -568,6 +703,11 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
|
|
| 568 |
"filters": copy.deepcopy(requested_filter),
|
| 569 |
"records": records,
|
| 570 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 571 |
|
| 572 |
|
| 573 |
def assistant_permitted_sources(session: Session, databases):
|
|
@@ -672,6 +812,19 @@ def assistant_source_status(session: Session, databases=None):
|
|
| 672 |
mirror_cur = None
|
| 673 |
return mirror_cur
|
| 674 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 675 |
def _definitions():
|
| 676 |
nonlocal definitions, read_definitions
|
| 677 |
if not read_definitions:
|
|
@@ -752,6 +905,10 @@ def assistant_source_status(session: Session, databases=None):
|
|
| 752 |
entry = cache.get(cache_key) if isinstance(cache, dict) else None
|
| 753 |
if isinstance(entry, tuple) and len(entry) >= 2 and isinstance(entry[1], (list, tuple)):
|
| 754 |
row["answerable"] = True
|
|
|
|
|
|
|
|
|
|
|
|
|
| 755 |
else:
|
| 756 |
row["reason"] = "no snapshot loaded yet. Open this database once, then ask"
|
| 757 |
continue
|
|
@@ -762,10 +919,16 @@ def assistant_source_status(session: Session, databases=None):
|
|
| 762 |
if not isinstance(definition, dict):
|
| 763 |
continue
|
| 764 |
lent = user_tables.lend(session.runtime, user_tables=defs)
|
| 765 |
-
|
|
|
|
| 766 |
continue
|
| 767 |
row["permitted"] = True
|
| 768 |
if user_tables.materialises(key, st=lent, defn=definition):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 769 |
row["answerable"] = True
|
| 770 |
continue
|
| 771 |
# ββ W47-T23 / R11 - THE SECOND WAY TO BE ANSWERABLE. This used to end here with a flat
|
|
@@ -778,7 +941,9 @@ def assistant_source_status(session: Session, databases=None):
|
|
| 778 |
# is transient and retrying works, while a grid this deployment has no read-through
|
| 779 |
# binding for will not fix itself by waiting. One sentence for both would make the first
|
| 780 |
# look permanent, which is the failure mode the ruling exists to prevent.
|
| 781 |
-
|
|
|
|
|
|
|
| 782 |
if cur is None:
|
| 783 |
row["reason"] = ("this database is read live from its connector mirror, and that "
|
| 784 |
"mirror is not open on this deployment right now; try again shortly")
|
|
@@ -801,6 +966,49 @@ def assistant_source_status(session: Session, databases=None):
|
|
| 801 |
return out
|
| 802 |
|
| 803 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 804 |
def _user_for(claims):
|
| 805 |
"""The account named by a verified cookie, or None β with the epoch check that makes a
|
| 806 |
stateless cookie revocable.
|
|
@@ -829,11 +1037,16 @@ def _user_for(claims):
|
|
| 829 |
if (qa and claim_tenant == qa.get("tenant")
|
| 830 |
and int(claims.get("e") or 0) == int(qa.get("epoch") or 0)):
|
| 831 |
return qa
|
|
|
|
|
|
|
|
|
|
| 832 |
try:
|
| 833 |
-
reg =
|
| 834 |
-
except Exception:
|
| 835 |
-
|
| 836 |
-
|
|
|
|
|
|
|
| 837 |
if (rec and rec.get("active", True)
|
| 838 |
and int(rec.get("epoch") or 0) == int(claims.get("e") or 0)):
|
| 839 |
pub = users._public(uname, rec)
|
|
|
|
| 232 |
else ("product_pool", team_id))
|
| 233 |
cache = getattr(session.runtime, "pool_cache", None)
|
| 234 |
entry = cache.get(cache_key) if isinstance(cache, dict) else None
|
| 235 |
+
if not isinstance(entry, tuple) and isinstance(cache, dict):
|
| 236 |
+
# ββ S67-T33 / D-778: A DATABASE NEVER OPENED ANSWERS. The snapshot used to exist only
|
| 237 |
+
# once somebody opened the grid in this process, so after every restart the assistant
|
| 238 |
+
# refused a database its caller demonstrably holds. The miss is now filled by the GRID'S
|
| 239 |
+
# OWN pool builder, for the SAME scope key the grid would fill (`derive_pool_scope`
|
| 240 |
+
# above), so this is the grid's first open, not a second reader. It is read-only on Odoo
|
| 241 |
+
# (rule 4) and honours the connector pause exactly as the grid does (503 when paused
|
| 242 |
+
# with nothing cached). Permission and field checks above have already run.
|
| 243 |
+
try:
|
| 244 |
+
if database == "customer_data":
|
| 245 |
+
import routes_customers
|
| 246 |
+
routes_customers._pool_for(session.runtime, team_id, agent)
|
| 247 |
+
else:
|
| 248 |
+
import routes_products
|
| 249 |
+
routes_products._pool_for(session.runtime, team_id)
|
| 250 |
+
except HTTPException:
|
| 251 |
+
raise
|
| 252 |
+
except Exception as exc: # noqa: BLE001
|
| 253 |
+
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
| 254 |
+
f"this database's snapshot could not be built for your current "
|
| 255 |
+
f"scope ({type(exc).__name__}); try again shortly")
|
| 256 |
+
entry = cache.get(cache_key)
|
| 257 |
if (not isinstance(entry, tuple) or len(entry) < 2
|
| 258 |
or not isinstance(entry[1], (list, tuple))
|
| 259 |
or any(not isinstance(row, dict) for row in entry[1])):
|
|
|
|
| 269 |
_ASSISTANT_USER_TABLE_KINDS = frozenset({"materialised_user_table", "mirror_user_table"})
|
| 270 |
|
| 271 |
|
| 272 |
+
def _assistant_nav_opens(session: Session, database: str, lent, definition=None) -> bool:
|
| 273 |
+
"""Does the NAVIGATION offer this `ut_*` database to this caller? S67-T33 / R33.
|
| 274 |
+
|
| 275 |
+
ββ THE AI CONNECTOR'S LIST IS THE NAV'S LIST, BY CONSTRUCTION RATHER THAN BY AGREEMENT.
|
| 276 |
+
`routes_nav.nav` admits a user table when `user_tables.nav_entries` lists it (which is
|
| 277 |
+
`may_open` plus the route-system exclusion), it is not a retired key, and
|
| 278 |
+
`perm_scope.nav_may_open` agrees (an administrator's explicit untick denies). The assistant
|
| 279 |
+
used to ask `may_open` alone, so a database the member's navigation did not show could still
|
| 280 |
+
be listed and read through the connector. This asks the nav's own predicates, in the nav's
|
| 281 |
+
order; it is not a second permission rule.
|
| 282 |
+
β `session.admin` short-circuits leg 1 of `nav_may_open` exactly as the nav's own admin
|
| 283 |
+
session does; `nav_may_open` itself returns True for an administrator.
|
| 284 |
+
"""
|
| 285 |
+
import core.perm_scope as perm_scope
|
| 286 |
+
from core import route_system, user_tables
|
| 287 |
+
|
| 288 |
+
key = str(database or "")
|
| 289 |
+
try:
|
| 290 |
+
from odoo_relational import RETIRED_KEYS as retired
|
| 291 |
+
except Exception: # noqa: BLE001
|
| 292 |
+
retired = ()
|
| 293 |
+
if key in retired:
|
| 294 |
+
return False
|
| 295 |
+
if not user_tables.may_open(key, session.uname, session.admin, st=lent):
|
| 296 |
+
return False
|
| 297 |
+
if isinstance(definition, dict) and (route_system.is_managed(definition)
|
| 298 |
+
or route_system.is_managed(key)):
|
| 299 |
+
return False
|
| 300 |
+
if session.admin:
|
| 301 |
+
return True
|
| 302 |
+
user = getattr(session, "user", None)
|
| 303 |
+
if not isinstance(user, dict):
|
| 304 |
+
# No account record means no explicit per-key deny can exist, and `nav_may_open` leg 3
|
| 305 |
+
# then defers to `may_open`, which has already answered above.
|
| 306 |
+
return True
|
| 307 |
+
return bool(perm_scope.nav_may_open(user, key, st=lent))
|
| 308 |
+
|
| 309 |
+
|
| 310 |
def _assistant_table_defn(session: Session, database: str):
|
| 311 |
"""The rows-free definition for one `ut_*` database plus its lent runtime, after `may_open`.
|
| 312 |
|
|
|
|
| 336 |
if not isinstance(definition, dict):
|
| 337 |
_assistant_refuse(404, "assistant_unknown_database", "that database is not available")
|
| 338 |
definition_runtime = user_tables.lend(session.runtime, user_tables=definitions)
|
| 339 |
+
# S67-T33: the NAV's predicate, not `may_open` alone, so a database the caller's navigation
|
| 340 |
+
# does not show is refused here too (403), not merely left off the list.
|
| 341 |
+
if not _assistant_nav_opens(session, database, definition_runtime, definition):
|
| 342 |
# Deliberately no label/owner in this response: knowing a key must not become an oracle
|
| 343 |
# for somebody else's database.
|
| 344 |
_assistant_refuse(403, "assistant_forbidden",
|
|
|
|
| 408 |
|
| 409 |
|
| 410 |
def _assistant_mirror_table(session: Session, database: str, requested_fields=None, filters=None,
|
| 411 |
+
*, defn=None, lent=None, mode=None, extra=None):
|
| 412 |
"""Read a MIRROR-SERVED user table through the same filter compiler the grid uses (R11 / C7).
|
| 413 |
|
| 414 |
Returns `_assistant_user_table`'s 4-tuple - `(rows, fields, source_version, label)` - so the
|
|
|
|
| 444 |
# THE CHEAP QUESTION FIRST, and it needs no cursor: is this key a connected Odoo grid at all?
|
| 445 |
# A `readThrough` table that is not one has no mirror rows anywhere, and asking the mirror
|
| 446 |
# about it would open a cursor to learn nothing.
|
| 447 |
+
if database.startswith("ut_amzn_"):
|
| 448 |
+
# β S67-T33: THE GRID'S OWN ADMISSION FIRST. A dynamically sized Amazon snapshot enters
|
| 449 |
+
# the source registry only through `require_amazon_mirror`, so without it a database
|
| 450 |
+
# never opened in this process had no binding here (measured on staging for
|
| 451 |
+
# `ut_amzn_fba_inventory`). It refuses by name (503) while the Amazon mirror is unloaded.
|
| 452 |
+
import routes_odoo_tables
|
| 453 |
+
routes_odoo_tables.require_amazon_mirror(database, session, definition=definition)
|
| 454 |
+
# β S67-T33: the stored definition rides along, so an Amazon grid or a generated
|
| 455 |
+
# `ut_odoo_model_*` table answers from its own field contract, as it does on the grid.
|
| 456 |
+
declared = assistant_mirror.declared_fields(database, definition)
|
| 457 |
+
if declared is None:
|
| 458 |
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
| 459 |
"this database does not store its rows and it is not one of the "
|
| 460 |
"connected databases the assistant can read from the mirror")
|
| 461 |
+
# β S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's
|
| 462 |
+
# first sync. Passed only for an Amazon key, which is the one family it changes.
|
| 463 |
+
cur, refusal = (assistant_mirror.open_cursor(session, database)
|
| 464 |
+
if database.startswith("ut_amzn_") else assistant_mirror.open_cursor(session))
|
| 465 |
if refusal is not None:
|
| 466 |
_assistant_refuse(refusal.status, refusal.code, refusal.message)
|
| 467 |
spec = assistant_mirror.binding(cur, database)
|
|
|
|
| 469 |
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
| 470 |
"this database is served live from its connector and this deployment "
|
| 471 |
"has no read-through binding for it yet, so it holds no rows to read")
|
| 472 |
+
fields = assistant_mirror.answerable_fields(database, spec, declared)
|
| 473 |
if not fields:
|
| 474 |
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
| 475 |
"this database's connector mirror carries none of its declared columns "
|
|
|
|
| 497 |
if grant is None:
|
| 498 |
wall_tree = None
|
| 499 |
|
| 500 |
+
filter_keys = [field["key"] for field in visible
|
| 501 |
+
if isinstance(field, dict) and field.get("key")]
|
| 502 |
+
# ββ S67-T34: THREE READS BEHIND THE ONE PREFLIGHT ABOVE, chosen by `mode`, all compiling the
|
| 503 |
+
# SAME wall and filter (`assistant_mirror._predicate`). "schema" reads and counts no row
|
| 504 |
+
# (D-2630: describe answers the field list of a million-row table without counting it);
|
| 505 |
+
# ("aggregate", specs) runs the aggregates IN SQL over every permitted row (D-616, D-2631);
|
| 506 |
+
# ("page", offset, limit) pages IN SQL for a table past the one-pass ceiling (rule 1).
|
| 507 |
try:
|
| 508 |
+
if mode == "schema":
|
| 509 |
+
return [], fields, {"read": "connector mirror"}, definition.get("label") or database
|
| 510 |
+
if isinstance(mode, tuple) and mode and mode[0] == "aggregate":
|
| 511 |
+
values, total = assistant_mirror.aggregate(
|
| 512 |
+
session, database, cur, spec, fields, mode[1],
|
| 513 |
+
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys)
|
| 514 |
+
if isinstance(extra, dict):
|
| 515 |
+
extra["aggregates"], extra["total"] = values, total
|
| 516 |
+
return ([], fields, {"read": "connector mirror", "rows": total},
|
| 517 |
+
definition.get("label") or database)
|
| 518 |
+
if isinstance(mode, tuple) and mode and mode[0] == "page":
|
| 519 |
+
rows, stamp, total = assistant_mirror.read_page(
|
| 520 |
+
session, database, cur, spec, fields, offset=mode[1], limit=mode[2],
|
| 521 |
+
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
|
| 522 |
+
defn=definition)
|
| 523 |
+
if isinstance(extra, dict):
|
| 524 |
+
extra["total"] = total
|
| 525 |
+
else:
|
| 526 |
+
rows, stamp, total = assistant_mirror.read_rows(
|
| 527 |
+
session, database, cur, spec, fields,
|
| 528 |
+
wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
|
| 529 |
+
defn=definition)
|
| 530 |
except assistant_mirror.MirrorRefusal as exc:
|
| 531 |
_assistant_refuse(exc.status, exc.code, exc.message)
|
| 532 |
raise # unreachable; see above
|
|
|
|
| 558 |
return out
|
| 559 |
|
| 560 |
|
| 561 |
+
def _assistant_oversize_reason(database: str, definition, lent) -> str:
|
| 562 |
+
"""The named cause an oversize stored table carries, or "". S67-T33 / rule 1.
|
| 563 |
+
|
| 564 |
+
β AN EVICTED TABLE KEEPS ITS DEFINITION AND LOSES ITS ROWS, so the materialised reader used
|
| 565 |
+
to answer it as a complete table of 0 rows. `user_tables.limit_report` is the one evaluator
|
| 566 |
+
of that stamp (the grid serves the same sentence), so its cause and recommendation are read
|
| 567 |
+
from there rather than re-worded here.
|
| 568 |
+
"""
|
| 569 |
+
from core import user_tables
|
| 570 |
+
|
| 571 |
+
if not user_tables.oversize_of(database, defn=definition if isinstance(definition, dict)
|
| 572 |
+
else {}):
|
| 573 |
+
return ""
|
| 574 |
+
report = user_tables.limit_report(database, st=lent) or {}
|
| 575 |
+
label = str((definition or {}).get("label") or database)
|
| 576 |
+
return (f"{label} is not stored: {report.get('cause') or 'its rows were not stored'}. "
|
| 577 |
+
f"{str(report.get('recommendation') or '').rstrip('.')}. The assistant refuses "
|
| 578 |
+
f"rather than answering 0 rows").replace("\u2014", ",").replace("\u2013", "-")
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
def _assistant_refuse_oversize(database: str, definition, lent) -> None:
|
| 582 |
+
reason = _assistant_oversize_reason(database, definition, lent)
|
| 583 |
+
if reason:
|
| 584 |
+
_assistant_refuse(409, "assistant_source_too_large", reason)
|
| 585 |
+
|
| 586 |
+
|
| 587 |
+
def assistant_read_scope(session: Session, database, fields=None, filters=None, *, mode=None):
|
| 588 |
"""Return a detached, permissioned snapshot for one Assistant tool call.
|
| 589 |
|
| 590 |
Signature (the C8 contract for E):
|
|
|
|
| 593 |
permission_scope_applied, fields, filters, records
|
| 594 |
}
|
| 595 |
|
| 596 |
+
β S67-T34: `mode` changes only HOW a mirror-served database is read, never who may read it
|
| 597 |
+
(every wall below runs unchanged): "schema" returns the field contract with no row read,
|
| 598 |
+
("aggregate", specs) adds `server_aggregates` and `total_rows` computed in SQL over every
|
| 599 |
+
permitted row, ("page", offset, limit) returns one SQL page plus `total_rows`. Every other
|
| 600 |
+
source kind ignores it and answers as before.
|
| 601 |
+
|
| 602 |
`database` is exactly one source. `fields` is an explicit list of visible field keys (or
|
| 603 |
``None`` for every visible field). `filters` is ``None`` or the strict tree accepted by
|
| 604 |
`core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and
|
|
|
|
| 609 |
from harness import filter_eval
|
| 610 |
|
| 611 |
key = str(database or "").strip()
|
| 612 |
+
extra = {}
|
| 613 |
if key in _ASSISTANT_BUILT_INS:
|
| 614 |
rows, canonical_fields, source_version = _assistant_cached_builtin(session, key, fields, filters)
|
| 615 |
label = _ASSISTANT_BUILT_INS[key]["label"]
|
|
|
|
| 626 |
from core import user_tables
|
| 627 |
_defn, _lent = _assistant_table_defn(session, key)
|
| 628 |
if user_tables.materialises(key, st=_lent, defn=_defn):
|
| 629 |
+
# S67-T33: a stored table whose rows were evicted refuses by name, never answers 0.
|
| 630 |
+
# Only on the STORED reader: a read-through table's rows come from the mirror.
|
| 631 |
+
_assistant_refuse_oversize(key, _defn, _lent)
|
| 632 |
rows, canonical_fields, source_version, label = _assistant_user_table(
|
| 633 |
session, key, fields, filters, defn=_defn, lent=_lent)
|
| 634 |
source_kind = "materialised_user_table"
|
| 635 |
else:
|
| 636 |
rows, canonical_fields, source_version, label = _assistant_mirror_table(
|
| 637 |
+
session, key, fields, filters, defn=_defn, lent=_lent, mode=mode, extra=extra)
|
| 638 |
source_kind = "mirror_user_table"
|
| 639 |
else:
|
| 640 |
_assistant_refuse(404, "assistant_unknown_database", "that database is not available")
|
|
|
|
| 691 |
records.append({"pid": copy.deepcopy(row["pid"]),
|
| 692 |
**{field: copy.deepcopy(row.get(field)) for field in selected_keys}})
|
| 693 |
|
| 694 |
+
out = {
|
| 695 |
"database": key,
|
| 696 |
"label": str(label),
|
| 697 |
"source_kind": source_kind,
|
|
|
|
| 703 |
"filters": copy.deepcopy(requested_filter),
|
| 704 |
"records": records,
|
| 705 |
}
|
| 706 |
+
if "aggregates" in extra:
|
| 707 |
+
out["server_aggregates"] = copy.deepcopy(extra["aggregates"])
|
| 708 |
+
if "total" in extra:
|
| 709 |
+
out["total_rows"] = int(extra["total"])
|
| 710 |
+
return out
|
| 711 |
|
| 712 |
|
| 713 |
def assistant_permitted_sources(session: Session, databases):
|
|
|
|
| 812 |
mirror_cur = None
|
| 813 |
return mirror_cur
|
| 814 |
|
| 815 |
+
amazon_cur = {}
|
| 816 |
+
|
| 817 |
+
def _amazon_mirror(key):
|
| 818 |
+
"""ONE Amazon cursor for the batch, chosen by `_table_cursor` for an Amazon key."""
|
| 819 |
+
if "cur" not in amazon_cur:
|
| 820 |
+
try:
|
| 821 |
+
import assistant_mirror
|
| 822 |
+
cur, refusal = assistant_mirror.open_cursor(session, key)
|
| 823 |
+
amazon_cur["cur"] = None if refusal is not None else cur
|
| 824 |
+
except Exception: # noqa: BLE001
|
| 825 |
+
amazon_cur["cur"] = None
|
| 826 |
+
return amazon_cur["cur"]
|
| 827 |
+
|
| 828 |
def _definitions():
|
| 829 |
nonlocal definitions, read_definitions
|
| 830 |
if not read_definitions:
|
|
|
|
| 905 |
entry = cache.get(cache_key) if isinstance(cache, dict) else None
|
| 906 |
if isinstance(entry, tuple) and len(entry) >= 2 and isinstance(entry[1], (list, tuple)):
|
| 907 |
row["answerable"] = True
|
| 908 |
+
elif isinstance(cache, dict):
|
| 909 |
+
# β S67-T33 / D-778: a miss is no longer a refusal; the read builds the snapshot
|
| 910 |
+
# through the grid's own pool builder, so this source is answerable now.
|
| 911 |
+
row["answerable"] = True
|
| 912 |
else:
|
| 913 |
row["reason"] = "no snapshot loaded yet. Open this database once, then ask"
|
| 914 |
continue
|
|
|
|
| 919 |
if not isinstance(definition, dict):
|
| 920 |
continue
|
| 921 |
lent = user_tables.lend(session.runtime, user_tables=defs)
|
| 922 |
+
# β S67-T33 / R33: the NAV's predicate, so this list and the navigation are one set.
|
| 923 |
+
if not _assistant_nav_opens(session, key, lent, definition):
|
| 924 |
continue
|
| 925 |
row["permitted"] = True
|
| 926 |
if user_tables.materialises(key, st=lent, defn=definition):
|
| 927 |
+
oversize = _assistant_oversize_reason(key, definition, lent)
|
| 928 |
+
if oversize:
|
| 929 |
+
# Rule 1's second sentence: listed, with its cause, never answered as 0 rows.
|
| 930 |
+
row["reason"] = oversize
|
| 931 |
+
continue
|
| 932 |
row["answerable"] = True
|
| 933 |
continue
|
| 934 |
# ββ W47-T23 / R11 - THE SECOND WAY TO BE ANSWERABLE. This used to end here with a flat
|
|
|
|
| 941 |
# is transient and retrying works, while a grid this deployment has no read-through
|
| 942 |
# binding for will not fix itself by waiting. One sentence for both would make the first
|
| 943 |
# look permanent, which is the failure mode the ruling exists to prevent.
|
| 944 |
+
# β S67-T33: an Amazon grid asks ITS OWN mirror (`_table_cursor`), never the Odoo
|
| 945 |
+
# first-sync gate, so "not open right now" is never Odoo's backfill answering for Amazon.
|
| 946 |
+
cur = _amazon_mirror(key) if key.startswith("ut_amzn_") else _mirror()
|
| 947 |
if cur is None:
|
| 948 |
row["reason"] = ("this database is read live from its connector mirror, and that "
|
| 949 |
"mirror is not open on this deployment right now; try again shortly")
|
|
|
|
| 966 |
return out
|
| 967 |
|
| 968 |
|
| 969 |
+
#: S67 lane D (S67-T28 finding): the session resolver re-read the WHOLE `users` bucket on every
|
| 970 |
+
#: authenticated request, uncached; on a saturated cluster that read queued ahead of every page.
|
| 971 |
+
#: It is now held against the bucket's durable revision token (`store.revision`), the S67-T28
|
| 972 |
+
#: `_REV_MEMO` pattern: the token is read BEFORE the value, so a write racing the read can only
|
| 973 |
+
#: cache newer bytes under the older token (a miss next time, never a stale hit). Every Postgres
|
| 974 |
+
#: `put` / `update` bumps `rev`, so a deactivation or an epoch rotation misses on the very next
|
| 975 |
+
#: request. β FAIL-CLOSED: no token, or a token read that fails, means a direct read (which still
|
| 976 |
+
#: answers 503 when the store is unreadable); a cached registry is never served in its place.
|
| 977 |
+
#: The key carries the identity of the reader and the store it reads, so a swapped store or a
|
| 978 |
+
#: patched `users.registry` / `store.get` misses instead of replaying another registry.
|
| 979 |
+
_SESSION_REGISTRY = {"key": None, "reg": None}
|
| 980 |
+
_SESSION_REGISTRY_LOCK = __import__("threading").Lock()
|
| 981 |
+
|
| 982 |
+
|
| 983 |
+
def _session_registry():
|
| 984 |
+
"""`users.registry()` for `_user_for`, re-read only when the `users` bucket changed.
|
| 985 |
+
|
| 986 |
+
β POSTGRES ONLY. There `rev` is a durable column every `put` / `update` bumps (`store_pg`),
|
| 987 |
+
and nothing else writes `store_kv`. The HF backend's token is a per-process counter that a
|
| 988 |
+
fixture or a cache-level write can leave unmoved (measured: `verify_scopes` 346/687 with the
|
| 989 |
+
memo on HF), and its `get()` already holds the bucket for the life of the process, so the
|
| 990 |
+
re-read this memo removes does not exist there: it reads straight through.
|
| 991 |
+
"""
|
| 992 |
+
key = None
|
| 993 |
+
try:
|
| 994 |
+
backend = store.backend()
|
| 995 |
+
token = (store.revision("users") or {}).get("token") if backend == "pg" else None
|
| 996 |
+
if token not in (None, ""):
|
| 997 |
+
where = os.environ.get("AIOS_TENANT") or "royal-imports"
|
| 998 |
+
key = (backend, where, id(users.registry), id(store.get), str(token))
|
| 999 |
+
except Exception: # noqa: BLE001 -- fail-closed below
|
| 1000 |
+
key = None
|
| 1001 |
+
if key is not None:
|
| 1002 |
+
with _SESSION_REGISTRY_LOCK:
|
| 1003 |
+
if _SESSION_REGISTRY["key"] == key:
|
| 1004 |
+
return _SESSION_REGISTRY["reg"]
|
| 1005 |
+
reg = users.registry() or {}
|
| 1006 |
+
if key is not None:
|
| 1007 |
+
with _SESSION_REGISTRY_LOCK:
|
| 1008 |
+
_SESSION_REGISTRY["key"], _SESSION_REGISTRY["reg"] = key, copy.deepcopy(reg)
|
| 1009 |
+
return reg
|
| 1010 |
+
|
| 1011 |
+
|
| 1012 |
def _user_for(claims):
|
| 1013 |
"""The account named by a verified cookie, or None β with the epoch check that makes a
|
| 1014 |
stateless cookie revocable.
|
|
|
|
| 1037 |
if (qa and claim_tenant == qa.get("tenant")
|
| 1038 |
and int(claims.get("e") or 0) == int(qa.get("epoch") or 0)):
|
| 1039 |
return qa
|
| 1040 |
+
# β HOTFIX 2026-09-28: AN UNREADABLE REGISTRY IS A 503, NEVER "NO SUCH ACCOUNT". This caught
|
| 1041 |
+
# `Exception` into `{}`, so a store `PoolTimeout` on staging answered a valid cookie with 401
|
| 1042 |
+
# `invalid_session` and the client signed the owner out. Still fail-closed: nobody is admitted.
|
| 1043 |
try:
|
| 1044 |
+
reg = _session_registry()
|
| 1045 |
+
except Exception as exc:
|
| 1046 |
+
print(f"[aios-api] session account store unavailable: {type(exc).__name__}: {exc}")
|
| 1047 |
+
raise err(503, "session_store_unavailable",
|
| 1048 |
+
"your account could not be checked just now. Try again in a moment")
|
| 1049 |
+
rec = copy.deepcopy(reg.get(uname)) # the registry may be the held copy: never mutate it
|
| 1050 |
if (rec and rec.get("active", True)
|
| 1051 |
and int(rec.get("epoch") or 0) == int(claims.get("e") or 0)):
|
| 1052 |
pub = users._public(uname, rec)
|
api/formula_runtime.py
CHANGED
|
@@ -83,6 +83,8 @@ FN_ARITY = {
|
|
| 83 |
"ISNUMBER": (1, 1), "ISTEXT": (1, 1), "ISERROR": (1, 1),
|
| 84 |
"OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
|
| 85 |
"FILTER": (2, 3), "SORT": (1, 4),
|
|
|
|
|
|
|
| 86 |
}
|
| 87 |
# Excel workbooks in the Daily Sales lineage use both spellings. Keep the authored formula text
|
| 88 |
# untouched in the workbook; normalization maps AVG to the same AST/function the catalogue's
|
|
@@ -457,6 +459,15 @@ FN_CATALOGUE = {
|
|
| 457 |
"description": "Returns TRUE when a value is a number.", "contexts": _BOTH},
|
| 458 |
"ISTEXT": {"category": "Information", "signature": "ISTEXT(value)",
|
| 459 |
"description": "Returns TRUE when a value is text.", "contexts": _BOTH},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 460 |
"ISERROR": {"category": "Information", "signature": "ISERROR(value)",
|
| 461 |
"description": "Returns TRUE when a formula would refuse to produce a value.",
|
| 462 |
"contexts": _BOTH},
|
|
@@ -625,6 +636,9 @@ PARITY_FIXTURES = {
|
|
| 625 |
"EXACT": ("EXACT(\"A\", \"a\")", "FALSE"),
|
| 626 |
"ISNUMBER": ("ISNUMBER(1)", "TRUE"),
|
| 627 |
"ISTEXT": ("ISTEXT(\"a\")", "TRUE"),
|
|
|
|
|
|
|
|
|
|
| 628 |
"ISERROR": ("ISERROR(1 / 0)", "TRUE"),
|
| 629 |
}
|
| 630 |
|
|
@@ -917,7 +931,7 @@ RANGE_ARGS = {
|
|
| 917 |
|
| 918 |
#: β THE ARGUMENT IS AN ADDRESS, NOT A VALUE. `ROW(B4)` asks what row `B4` is on and must not read
|
| 919 |
#: what is written there; the cell may well be empty. Handled beside `RANGE_ARGS` in `_eval`.
|
| 920 |
-
ADDR_ARGS = {"ROW": (0,), "COLUMN": (0,)}
|
| 921 |
|
| 922 |
|
| 923 |
def _is_range_position(fn, index):
|
|
@@ -1792,6 +1806,11 @@ def _eval_value(node, row, today, diagnostics=None):
|
|
| 1792 |
return _array_binary(a, b, node["op"], diagnostics)
|
| 1793 |
if isinstance(row, _SheetInputs):
|
| 1794 |
def comparison(value):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1795 |
if isinstance(value, bool): return (2, value)
|
| 1796 |
if isinstance(value, str): return (1, value.casefold())
|
| 1797 |
return (0, _excel_number(value or 0))
|
|
@@ -1858,6 +1877,17 @@ def _eval_value(node, row, today, diagnostics=None):
|
|
| 1858 |
for code in local:
|
| 1859 |
_diagnose(diagnostics, code)
|
| 1860 |
return "" if first is None and isinstance(row, _SheetInputs) and not local else first
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1861 |
if fn == "ISERROR":
|
| 1862 |
# The whole job is to ask whether a refusal happened, so the refusal must stay LOCAL.
|
| 1863 |
local = []
|
|
@@ -1888,6 +1918,11 @@ def _eval_value(node, row, today, diagnostics=None):
|
|
| 1888 |
_diagnose(diagnostics, "bad_value")
|
| 1889 |
return None
|
| 1890 |
args.append(here)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1891 |
return _call(fn, args, today, diagnostics)
|
| 1892 |
# β RANGES FLATTEN HERE, INTO THE SAME FLAT LIST `_call` ALREADY TAKES. `SUM`, `AVERAGE`,
|
| 1893 |
# `COUNT`, `MIN` and `MAX` are declared `(1, MANY)` and simply iterate their argument list,
|
|
@@ -1923,6 +1958,27 @@ def _eval_value(node, row, today, diagnostics=None):
|
|
| 1923 |
raise FormulaError("formula AST node is not supported")
|
| 1924 |
|
| 1925 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1926 |
def _addr_arg(node, diagnostics):
|
| 1927 |
"""The ADDRESS a1 node names, as a key. Anything else is not an address and refuses."""
|
| 1928 |
if isinstance(node, dict) and node.get("t") == "a1":
|
|
@@ -2319,6 +2375,19 @@ def _call(fn, a, today, diagnostics=None):
|
|
| 2319 |
if fn == "FIND":
|
| 2320 |
start = int(nums[2] or 1)-1 if len(a)>2 else 0; pos = _text(a[1]).find(_text(a[0]), start)
|
| 2321 |
return pos+1 if pos >= 0 else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2322 |
if fn in ("REGEX_EXTRACT", "REGEX_REPLACE"):
|
| 2323 |
pattern, subject = _safe_regex(a[1], a[0]); m = re.search(pattern, subject)
|
| 2324 |
if fn == "REGEX_EXTRACT": return m.group(0) if m else None
|
|
@@ -3722,7 +3791,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None):
|
|
| 3722 |
excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code)
|
| 3723 |
if excel: cell["error"]["excelCode"] = excel
|
| 3724 |
else:
|
| 3725 |
-
cell.update(state="fresh", display=
|
| 3726 |
cell.pop("refusal", None)
|
| 3727 |
cell.pop("error", None)
|
| 3728 |
cell.pop("spill", None)
|
|
@@ -3765,7 +3834,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None):
|
|
| 3765 |
base = engine.specs.get(child, {})
|
| 3766 |
style = {**(owner.get("style") or {}), **(base.get("style") or {})}
|
| 3767 |
record = {"kind": "literal", "value": engine.values.get(child), "state": "fresh",
|
| 3768 |
-
"display":
|
| 3769 |
"spillParent": origin, "style": style,
|
| 3770 |
"formulaMeta": {"spillChild": {"version": 1, "origin": origin, "address": child,
|
| 3771 |
**({"base": base} if base else {})}}}
|
|
|
|
| 83 |
"ISNUMBER": (1, 1), "ISTEXT": (1, 1), "ISERROR": (1, 1),
|
| 84 |
"OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
|
| 85 |
"FILTER": (2, 3), "SORT": (1, 4),
|
| 86 |
+
# S67-T24: the reference workbooks' `IF(ISNUMBER(SEARCH(...)))` and `ISFORMULA(...)`.
|
| 87 |
+
"SEARCH": (2, 3), "SEARCHB": (2, 3), "ISFORMULA": (1, 1),
|
| 88 |
}
|
| 89 |
# Excel workbooks in the Daily Sales lineage use both spellings. Keep the authored formula text
|
| 90 |
# untouched in the workbook; normalization maps AVG to the same AST/function the catalogue's
|
|
|
|
| 459 |
"description": "Returns TRUE when a value is a number.", "contexts": _BOTH},
|
| 460 |
"ISTEXT": {"category": "Information", "signature": "ISTEXT(value)",
|
| 461 |
"description": "Returns TRUE when a value is text.", "contexts": _BOTH},
|
| 462 |
+
"ISFORMULA": {"category": "Information", "signature": "ISFORMULA(reference)",
|
| 463 |
+
"description": "Returns TRUE when the cell it names holds a formula.",
|
| 464 |
+
"contexts": _SHEET},
|
| 465 |
+
"SEARCH": {"category": "Text", "signature": "SEARCH(find_text, within_text, [start_num])",
|
| 466 |
+
"description": "Returns the position of one text value inside another, ignoring "
|
| 467 |
+
"case; ? matches one character and * any run.", "contexts": _BOTH},
|
| 468 |
+
"SEARCHB": {"category": "Text", "signature": "SEARCHB(find_text, within_text, [start_num])",
|
| 469 |
+
"description": "Counts like SEARCH; each character is one byte in this workbook.",
|
| 470 |
+
"contexts": _BOTH},
|
| 471 |
"ISERROR": {"category": "Information", "signature": "ISERROR(value)",
|
| 472 |
"description": "Returns TRUE when a formula would refuse to produce a value.",
|
| 473 |
"contexts": _BOTH},
|
|
|
|
| 636 |
"EXACT": ("EXACT(\"A\", \"a\")", "FALSE"),
|
| 637 |
"ISNUMBER": ("ISNUMBER(1)", "TRUE"),
|
| 638 |
"ISTEXT": ("ISTEXT(\"a\")", "TRUE"),
|
| 639 |
+
"SEARCH": ("SEARCH(\"B\", \"abc\")", 2),
|
| 640 |
+
"SEARCHB": ("SEARCHB(\"c?\", \"abcd\")", 3),
|
| 641 |
+
"ISFORMULA": ("ISFORMULA(B2)", "FALSE"),
|
| 642 |
"ISERROR": ("ISERROR(1 / 0)", "TRUE"),
|
| 643 |
}
|
| 644 |
|
|
|
|
| 931 |
|
| 932 |
#: β THE ARGUMENT IS AN ADDRESS, NOT A VALUE. `ROW(B4)` asks what row `B4` is on and must not read
|
| 933 |
#: what is written there; the cell may well be empty. Handled beside `RANGE_ARGS` in `_eval`.
|
| 934 |
+
ADDR_ARGS = {"ROW": (0,), "COLUMN": (0,), "ISFORMULA": (0,)}
|
| 935 |
|
| 936 |
|
| 937 |
def _is_range_position(fn, index):
|
|
|
|
| 1806 |
return _array_binary(a, b, node["op"], diagnostics)
|
| 1807 |
if isinstance(row, _SheetInputs):
|
| 1808 |
def comparison(value):
|
| 1809 |
+
# HOTFIX 2026-09-28: AN EMPTY CELL COMPARES AS EMPTY TEXT, exactly as the browser's
|
| 1810 |
+
# `formulaEval.ts` does ("value" in cell ? cell.value : ""). As a number it made
|
| 1811 |
+
# `=IF(E13="",E12,E13)` FALSE here and TRUE in the browser, so Daily Sales showed a
|
| 1812 |
+
# date the server refused and every binding filtered on row 7 read #ERROR!.
|
| 1813 |
+
if value is None: return (1, "")
|
| 1814 |
if isinstance(value, bool): return (2, value)
|
| 1815 |
if isinstance(value, str): return (1, value.casefold())
|
| 1816 |
return (0, _excel_number(value or 0))
|
|
|
|
| 1877 |
for code in local:
|
| 1878 |
_diagnose(diagnostics, code)
|
| 1879 |
return "" if first is None and isinstance(row, _SheetInputs) and not local else first
|
| 1880 |
+
if fn in ("ISNUMBER", "ISTEXT") and len(node.get("args") or ()) == 1:
|
| 1881 |
+
# β S67-T24: AN IS-FUNCTION NEVER PASSES AN ERROR ON. Excel answers FALSE for
|
| 1882 |
+
# `ISNUMBER(#VALUE!)`, which is what makes `IF(ISNUMBER(SEARCH(x, y)), ...)` (2,512 cells
|
| 1883 |
+
# in the Inventory System's Catalog) take its FALSE branch instead of failing whole.
|
| 1884 |
+
local = []
|
| 1885 |
+
try:
|
| 1886 |
+
value = _eval(node["args"][0], row, today, local)
|
| 1887 |
+
except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error,
|
| 1888 |
+
FormulaError, _ReferenceFault):
|
| 1889 |
+
return False
|
| 1890 |
+
return False if local else _call(fn, [value], today, diagnostics)
|
| 1891 |
if fn == "ISERROR":
|
| 1892 |
# The whole job is to ask whether a refusal happened, so the refusal must stay LOCAL.
|
| 1893 |
local = []
|
|
|
|
| 1918 |
_diagnose(diagnostics, "bad_value")
|
| 1919 |
return None
|
| 1920 |
args.append(here)
|
| 1921 |
+
if fn == "ISFORMULA":
|
| 1922 |
+
# β S67-T24: ASKED OF THE CELL, NOT OF ITS VALUE. Only a Sheet document knows which
|
| 1923 |
+
# cells hold formulas (`SheetCalculation.specs`); a grid row has no formula cells.
|
| 1924 |
+
spec = row.engine.specs.get(args[0]) if isinstance(row, _SheetInputs) and args[0] else None
|
| 1925 |
+
return bool(isinstance(spec, dict) and spec.get("formula"))
|
| 1926 |
return _call(fn, args, today, diagnostics)
|
| 1927 |
# β RANGES FLATTEN HERE, INTO THE SAME FLAT LIST `_call` ALREADY TAKES. `SUM`, `AVERAGE`,
|
| 1928 |
# `COUNT`, `MIN` and `MAX` are declared `(1, MANY)` and simply iterate their argument list,
|
|
|
|
| 1958 |
raise FormulaError("formula AST node is not supported")
|
| 1959 |
|
| 1960 |
|
| 1961 |
+
def _display_text(value):
|
| 1962 |
+
"""A calculated value as the grid shows it: Excel's TRUE / FALSE for a boolean (S67-T24; a
|
| 1963 |
+
Python `str(True)` painted "True"), blank for nothing, else the value's own text."""
|
| 1964 |
+
if value is None:
|
| 1965 |
+
return ""
|
| 1966 |
+
if isinstance(value, bool):
|
| 1967 |
+
return "TRUE" if value else "FALSE"
|
| 1968 |
+
return str(value)
|
| 1969 |
+
|
| 1970 |
+
|
| 1971 |
+
def _search_pattern(text):
|
| 1972 |
+
"""SEARCH's wildcard text as a regex: `?` one character, `*` any run, `~x` a literal x."""
|
| 1973 |
+
out, index = [], 0
|
| 1974 |
+
while index < len(text):
|
| 1975 |
+
ch = text[index]
|
| 1976 |
+
if ch == "~" and index + 1 < len(text) and text[index + 1] in "?*~":
|
| 1977 |
+
out.append(re.escape(text[index + 1])); index += 2; continue
|
| 1978 |
+
out.append(".*?" if ch == "*" else "." if ch == "?" else re.escape(ch)); index += 1
|
| 1979 |
+
return "".join(out)
|
| 1980 |
+
|
| 1981 |
+
|
| 1982 |
def _addr_arg(node, diagnostics):
|
| 1983 |
"""The ADDRESS a1 node names, as a key. Anything else is not an address and refuses."""
|
| 1984 |
if isinstance(node, dict) and node.get("t") == "a1":
|
|
|
|
| 2375 |
if fn == "FIND":
|
| 2376 |
start = int(nums[2] or 1)-1 if len(a)>2 else 0; pos = _text(a[1]).find(_text(a[0]), start)
|
| 2377 |
return pos+1 if pos >= 0 else None
|
| 2378 |
+
if fn in ("SEARCH", "SEARCHB"):
|
| 2379 |
+
# Excel's SEARCH: case-insensitive, `?` one character, `*` any run, `~` escapes the next;
|
| 2380 |
+
# absent text or a start outside the text is #VALUE!. SEARCHB counts bytes only in a
|
| 2381 |
+
# double-byte locale, so in this one it is SEARCH.
|
| 2382 |
+
within = _text(a[1])
|
| 2383 |
+
start = _strict_num(a[2], diagnostics) if len(a) > 2 else 1
|
| 2384 |
+
if start is None or not 1 <= int(start) <= len(within) + 1:
|
| 2385 |
+
_diagnose(diagnostics, "bad_value"); return None
|
| 2386 |
+
found = re.compile(_search_pattern(_text(a[0])), re.IGNORECASE | re.DOTALL).search(
|
| 2387 |
+
within, int(start) - 1)
|
| 2388 |
+
if not found:
|
| 2389 |
+
_diagnose(diagnostics, "bad_value"); return None
|
| 2390 |
+
return found.start() + 1
|
| 2391 |
if fn in ("REGEX_EXTRACT", "REGEX_REPLACE"):
|
| 2392 |
pattern, subject = _safe_regex(a[1], a[0]); m = re.search(pattern, subject)
|
| 2393 |
if fn == "REGEX_EXTRACT": return m.group(0) if m else None
|
|
|
|
| 3791 |
excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code)
|
| 3792 |
if excel: cell["error"]["excelCode"] = excel
|
| 3793 |
else:
|
| 3794 |
+
cell.update(state="fresh", display=_display_text(cell["value"]))
|
| 3795 |
cell.pop("refusal", None)
|
| 3796 |
cell.pop("error", None)
|
| 3797 |
cell.pop("spill", None)
|
|
|
|
| 3834 |
base = engine.specs.get(child, {})
|
| 3835 |
style = {**(owner.get("style") or {}), **(base.get("style") or {})}
|
| 3836 |
record = {"kind": "literal", "value": engine.values.get(child), "state": "fresh",
|
| 3837 |
+
"display": _display_text(engine.values.get(child)),
|
| 3838 |
"spillParent": origin, "style": style,
|
| 3839 |
"formulaMeta": {"spillChild": {"version": 1, "origin": origin, "address": child,
|
| 3840 |
**({"base": base} if base else {})}}}
|
api/main.py
CHANGED
|
@@ -92,6 +92,7 @@ from core import store # noqa: E402
|
|
| 92 |
# 409: a view write whose caller was looking at an older revision. See its handler below.
|
| 93 |
from core import view_audit # noqa: E402
|
| 94 |
from deps import Session, module_gate # noqa: E402
|
|
|
|
| 95 |
|
| 96 |
_WEB_DIST = _HERE.parents[1] / "web" / "dist"
|
| 97 |
|
|
@@ -351,6 +352,21 @@ app.include_router(routes_uploads.router)
|
|
| 351 |
# control is that the enumeration must NAME every router in this file.**
|
| 352 |
app.include_router(routes_templates.router) # item 7 / C12 β template registry, session-gated
|
| 353 |
app.include_router(routes_connectors.router) # item 10 / C11 β the connectors directory
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 354 |
# β routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its
|
| 355 |
# two paths are DELIBERATELY unauthenticated β a form is filled in by someone with no account β so
|
| 356 |
# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token
|
|
@@ -527,6 +543,45 @@ def _assert_contract_floor():
|
|
| 527 |
"Run aios-web/verify_fields_contract.py.")
|
| 528 |
|
| 529 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 530 |
def _startup_notes():
|
| 531 |
"""Say the two things an operator must know, once, at import β never in a response body."""
|
| 532 |
if aios_session.EPHEMERAL_SECRET:
|
|
@@ -535,10 +590,6 @@ def _startup_notes():
|
|
| 535 |
"Set it in production.")
|
| 536 |
|
| 537 |
|
| 538 |
-
_assert_contract_floor()
|
| 539 |
-
_startup_notes()
|
| 540 |
-
|
| 541 |
-
|
| 542 |
# β THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
|
| 543 |
# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
|
| 544 |
# page cannot live behind a password prompt, so the shell's own assets must be public β which they
|
|
@@ -746,6 +797,7 @@ def _pull_amazon(why):
|
|
| 746 |
try:
|
| 747 |
from harness import amazon_store as _amazon
|
| 748 |
from harness import runtime as _runtime
|
|
|
|
| 749 |
# The existing boot/resync projection targets tenant #0. Keep credential, mirror and
|
| 750 |
# projection on that same tenant; never fall back to an operator's environment credential.
|
| 751 |
tenant_key = "royal-imports"
|
|
@@ -758,7 +810,19 @@ def _pull_amazon(why):
|
|
| 758 |
# bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
|
| 759 |
# ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
|
| 760 |
# the markers live in the container's mirror, so every restart began again from orders.
|
| 761 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 762 |
from core import connector_sync as _connector_sync
|
| 763 |
receipt = _connector_sync.request_sync("amazon_sp", tenant_key, requested_by="first load")
|
| 764 |
print(f"[aios-api] amazon sync ({why}): first load "
|
|
@@ -766,6 +830,8 @@ def _pull_amazon(why):
|
|
| 766 |
return True
|
| 767 |
rep = _amazon.sync(tenant_key, rt=rt, scheduled=True,
|
| 768 |
log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}"))
|
|
|
|
|
|
|
| 769 |
for problem in rep.get("problems") or []:
|
| 770 |
print(f"[aios-api] amazon sync PROBLEM ({why}): {problem}")
|
| 771 |
print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
|
|
@@ -1184,7 +1250,7 @@ def _sweep_automation_schemas(why):
|
|
| 1184 |
|
| 1185 |
|
| 1186 |
def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
|
| 1187 |
-
tenant=None, cadence_seconds=None):
|
| 1188 |
"""Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
|
| 1189 |
'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
|
| 1190 |
could have asked" are different diagnoses.
|
|
@@ -1207,13 +1273,14 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1207 |
from core import user_tables as _ut
|
| 1208 |
default_cid = routes_connectors._odoo_connection_slot(_rt, None)
|
| 1209 |
connection_ids = [default_cid]
|
| 1210 |
-
|
| 1211 |
-
|
| 1212 |
-
|
| 1213 |
-
|
| 1214 |
-
|
| 1215 |
-
|
| 1216 |
-
|
|
|
|
| 1217 |
connection_ids = list(dict.fromkeys(cid for cid in connection_ids if cid))
|
| 1218 |
|
| 1219 |
def _health(cid, stage, state, **kwargs):
|
|
@@ -1240,10 +1307,14 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1240 |
# because a gate that counts RAW occurrences of a call would count the one in this
|
| 1241 |
# comment too. That gate happens to strip comments first; the next one may not.
|
| 1242 |
# See `odoo_relational.refresh` and its `all_connections` arm.
|
| 1243 |
-
|
| 1244 |
-
|
| 1245 |
-
|
| 1246 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1247 |
rebuild_errors = []
|
| 1248 |
# ββ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do.
|
| 1249 |
#
|
|
@@ -1319,14 +1390,25 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1319 |
cid = str(cid or default_cid)
|
| 1320 |
report = report if isinstance(report, dict) else {}
|
| 1321 |
failed = report.get("failed")
|
|
|
|
|
|
|
| 1322 |
database_failures, unmatched_failures = (
|
| 1323 |
-
routes_connectors._odoo_result_database_failures(
|
|
|
|
| 1324 |
failure = (routes_connectors._odoo_failure(failed, "materialization")
|
| 1325 |
if failed else None)
|
| 1326 |
-
|
|
|
|
|
|
|
|
|
|
| 1327 |
failure = routes_connectors._odoo_failure(
|
| 1328 |
"one or more databases failed", "materialization")
|
| 1329 |
tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1330 |
database_receipts = report.get("databases")
|
| 1331 |
database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
|
| 1332 |
canonical_receipts = report.get("canonicalDatabases")
|
|
@@ -1395,6 +1477,13 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1395 |
_ut.record_odoo_sync_status(
|
| 1396 |
phase="error" if safe_errors else "ok", errors=safe_errors,
|
| 1397 |
st=_rt, connection_id=cid)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1398 |
meta_state = _rebuild_meta_relational(
|
| 1399 |
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
|
| 1400 |
if isinstance(meta_state_out, dict):
|
|
@@ -1424,26 +1513,96 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1424 |
_ut.record_odoo_sync_status(
|
| 1425 |
phase="error", errors=[f"{failure['code']}: {failure['message']}"],
|
| 1426 |
st=_rt, connection_id=cid)
|
|
|
|
|
|
|
|
|
|
| 1427 |
except Exception:
|
| 1428 |
pass
|
| 1429 |
return False
|
| 1430 |
|
| 1431 |
|
| 1432 |
-
#
|
| 1433 |
-
#
|
| 1434 |
-
# A nonempty all-live zero report is the only evidence strong enough to skip that Odoo half. An
|
| 1435 |
-
# absent report is not zero: it means manual mode, paused source, or a failed/unknown sync, all of
|
| 1436 |
-
# which force the pre-existing rebuild path. `delete_report=None` is one of the ordinary passes
|
| 1437 |
-
# between delete reconciliations; it is unknown, not falsely claimed clean.
|
| 1438 |
_RESYNC_REBUILD_MAX_AGE_SECONDS = 24 * 60 * 60
|
| 1439 |
|
| 1440 |
|
| 1441 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1442 |
"""Return a fail-safe Odoo-rebuild reason, or ``None`` for a proven no-op.
|
| 1443 |
|
| 1444 |
The policy stays pure so it can be tested without Odoo, DuckDB, or PostgreSQL. Hard deletes
|
| 1445 |
are noticed by the existing every-fourth-pass reconciliation; an intervening pass does not
|
| 1446 |
-
pretend that it checked them, preserving the old visibility bound.
|
|
|
|
|
|
|
| 1447 |
"""
|
| 1448 |
if not isinstance(sync_report, dict) or not sync_report:
|
| 1449 |
return "sync-unconfirmed"
|
|
@@ -1451,7 +1610,7 @@ def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now):
|
|
| 1451 |
if not isinstance(report, dict) or report.get("phase") != "live":
|
| 1452 |
return "sync-unconfirmed"
|
| 1453 |
try:
|
| 1454 |
-
if int(report.get("pulled") or 0) > 0:
|
| 1455 |
return "source-change"
|
| 1456 |
except (TypeError, ValueError):
|
| 1457 |
return "sync-unconfirmed"
|
|
@@ -1463,8 +1622,20 @@ def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now):
|
|
| 1463 |
return "delete-change"
|
| 1464 |
except (TypeError, ValueError):
|
| 1465 |
return "sync-unconfirmed"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1466 |
if last_success_at is None or now - last_success_at >= _RESYNC_REBUILD_MAX_AGE_SECONDS:
|
| 1467 |
return "daily-recovery"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1468 |
return None
|
| 1469 |
|
| 1470 |
|
|
@@ -1619,6 +1790,14 @@ def _resolve_resync_cadence():
|
|
| 1619 |
"manual": manual, "disagreement": disagreement}
|
| 1620 |
|
| 1621 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1622 |
def _resync_mirror_pass(manual, passes):
|
| 1623 |
"""ONE bounded mirror pass: DuckDB and the Odoo XML-RPC client, and nothing else.
|
| 1624 |
|
|
@@ -1729,6 +1908,7 @@ def _store_resync_loop():
|
|
| 1729 |
from core import connector_sync
|
| 1730 |
passes = 0
|
| 1731 |
last_relational_success_at = None
|
|
|
|
| 1732 |
last_meta_fingerprint = None
|
| 1733 |
last_meta_applied_at = None
|
| 1734 |
# β AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
|
|
@@ -1844,6 +2024,7 @@ def _store_resync_loop():
|
|
| 1844 |
if _odoo_requested is None:
|
| 1845 |
_odoo_requested = _run_sync_requests()
|
| 1846 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
|
|
|
|
| 1847 |
RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
|
| 1848 |
RESYNC_STATE["lastPassOk"] = _err is None
|
| 1849 |
RESYNC_STATE["lastError"] = _err
|
|
@@ -1896,14 +2077,19 @@ def _store_resync_loop():
|
|
| 1896 |
_meta_previous = last_meta_fingerprint if _meta_reason is None else None
|
| 1897 |
_amzn_previous = last_amzn_fingerprint if _meta_reason is None else None
|
| 1898 |
_reason = _resync_rebuild_reason(sync_report, delete_report,
|
| 1899 |
-
last_relational_success_at, _meta_now
|
|
|
|
| 1900 |
# The scheduled pass may now be quiet because Sync now just advanced this mirror.
|
| 1901 |
# Its zero-change report cannot suppress projection of the explicit Odoo pull.
|
| 1902 |
if _odoo_requested:
|
| 1903 |
_reason = "sync now"
|
| 1904 |
_meta_state = {}
|
| 1905 |
if _reason is not None:
|
| 1906 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1907 |
# Test/recovery harnesses replace the rebuild seam with its historical three-argument
|
| 1908 |
# callable. Bind tenant/cadence whenever the active implementation accepts them,
|
| 1909 |
# without turning an internal TypeError into a second materialization attempt.
|
|
@@ -1913,14 +2099,20 @@ def _store_resync_loop():
|
|
| 1913 |
p.kind == _inspect.Parameter.VAR_KEYWORD for p in _rebuild_params.values())
|
| 1914 |
_rebuild_scope = ({"tenant": _cad["tenant"], "cadence_seconds": _every}
|
| 1915 |
if _rebuild_variadic or "tenant" in _rebuild_params else {})
|
|
|
|
|
|
|
| 1916 |
if _rebuild_odoo_relational(
|
| 1917 |
"resync", _meta_previous, _meta_state, **_rebuild_scope):
|
| 1918 |
-
|
|
|
|
|
|
|
|
|
|
| 1919 |
else:
|
| 1920 |
# A prior successful projection no longer proves the changed mirror was applied.
|
| 1921 |
# Keep recovery due on the next cadence even when that sync has no new rows.
|
| 1922 |
last_relational_success_at = None
|
| 1923 |
else:
|
|
|
|
| 1924 |
# A zero Odoo report says nothing about Meta. Its exact-plan fingerprint decides
|
| 1925 |
# whether this cadence may skip both the full-document write and relation pass.
|
| 1926 |
try:
|
|
@@ -2113,10 +2305,9 @@ if os.environ.get("AIOS_PREWARM") == "1":
|
|
| 2113 |
#
|
| 2114 |
# β THE HOTFIX MOVED THE CADENCE ONLY, 3600 -> 14400 (R11's existing four-hourly preset), which
|
| 2115 |
# is safe at any value because the wait below is `sync_wake_event().wait(_every)` and a Sync now
|
| 2116 |
-
# SETS it.
|
| 2117 |
-
#
|
| 2118 |
-
#
|
| 2119 |
-
# Narrowing it is what would make an hourly cadence nearly free.
|
| 2120 |
# * the whole-document store write itself. A partial-JSONB write is a wave.
|
| 2121 |
# β SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
|
| 2122 |
if os.environ.get("AIOS_STORE_RESYNC") == "1":
|
|
|
|
| 92 |
# 409: a view write whose caller was looking at an older revision. See its handler below.
|
| 93 |
from core import view_audit # noqa: E402
|
| 94 |
from deps import Session, module_gate # noqa: E402
|
| 95 |
+
from deps import require_session as deps_require_session # noqa: E402 (S67-T20 freshness door)
|
| 96 |
|
| 97 |
_WEB_DIST = _HERE.parents[1] / "web" / "dist"
|
| 98 |
|
|
|
|
| 352 |
# control is that the enumeration must NAME every router in this file.**
|
| 353 |
app.include_router(routes_templates.router) # item 7 / C12 β template registry, session-gated
|
| 354 |
app.include_router(routes_connectors.router) # item 10 / C11 β the connectors directory
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
@app.get("/api/v1/connectors/odoo/freshness")
|
| 358 |
+
def odoo_sync_freshness(session: Session = Depends(deps_require_session)):
|
| 359 |
+
"""ββ S67-T20 / C4 β IS THIS WORKSPACE'S ODOO DATA STALE, AND DID THE LAST SYNC FAIL.
|
| 360 |
+
|
| 361 |
+
`{stale, lastGoodSyncAt, schedule, staleAfterSeconds, lastRunAt, phase, frozen, failure}`
|
| 362 |
+
from `odoo_relational.sync_freshness`. The Connectors card and Connectors > Odoo word it.
|
| 363 |
+
β A SIBLING OF `GET /connectors/odoo/status`, not a field on it, because that route is another
|
| 364 |
+
lane's file this wave; the integrator may fold these fields into it (COMBINE-AMEND, S67-T20).
|
| 365 |
+
Same administrator door as that route.
|
| 366 |
+
"""
|
| 367 |
+
routes_connectors._odoo_admin(session)
|
| 368 |
+
import odoo_relational as _rel_fresh
|
| 369 |
+
return _rel_fresh.sync_freshness(session.runtime)
|
| 370 |
# β routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its
|
| 371 |
# two paths are DELIBERATELY unauthenticated β a form is filled in by someone with no account β so
|
| 372 |
# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token
|
|
|
|
| 543 |
"Run aios-web/verify_fields_contract.py.")
|
| 544 |
|
| 545 |
|
| 546 |
+
def _assert_store_guards(cadence=None):
|
| 547 |
+
"""Refuse an unbounded store or a cadence below the deployment's own floor."""
|
| 548 |
+
from core import store_pg
|
| 549 |
+
|
| 550 |
+
budget = store_pg.byte_budget_status()
|
| 551 |
+
if not budget["armed"]:
|
| 552 |
+
raise RuntimeError("byte budget disarmed: " + "; ".join(budget["reasons"]))
|
| 553 |
+
|
| 554 |
+
# Read the deployment independently of the resolver: a broken resolver must
|
| 555 |
+
# not be able to supply both sides of its own comparison.
|
| 556 |
+
raw = os.environ.get("AIOS_STORE_RESYNC_SECONDS")
|
| 557 |
+
deployed = None
|
| 558 |
+
if raw is not None:
|
| 559 |
+
try:
|
| 560 |
+
deployed = int(raw.strip())
|
| 561 |
+
except (TypeError, ValueError):
|
| 562 |
+
raise RuntimeError("resync guard disarmed: AIOS_STORE_RESYNC_SECONDS must be a "
|
| 563 |
+
"positive integer") from None
|
| 564 |
+
if deployed <= 0:
|
| 565 |
+
raise RuntimeError("resync guard disarmed: AIOS_STORE_RESYNC_SECONDS must be a "
|
| 566 |
+
"positive integer")
|
| 567 |
+
if cadence is None:
|
| 568 |
+
cadence = _resolve_resync_cadence()
|
| 569 |
+
resolved = cadence.get("resolvedSeconds")
|
| 570 |
+
if type(resolved) is not int or resolved <= 0:
|
| 571 |
+
raise RuntimeError("resync guard disarmed: resolved cadence must be a positive integer")
|
| 572 |
+
if deployed is not None and resolved < deployed:
|
| 573 |
+
raise RuntimeError(f"resync guard disarmed: resolved cadence {resolved}s is below "
|
| 574 |
+
f"deployed floor {deployed}s")
|
| 575 |
+
deployment = f"{deployed}s deployed" if deployed is not None else "not deployed"
|
| 576 |
+
return {"ok": True, "byte_budget": budget,
|
| 577 |
+
"resync": {"ok": True, "resolved_seconds": resolved, "deployed_seconds": deployed},
|
| 578 |
+
"message": ("Byte budget armed: yes. "
|
| 579 |
+
f"Table {budget['max_table_bytes'] / 1048576:g} MB; "
|
| 580 |
+
f"document {budget['max_document_bytes'] / 1048576:g} MB; "
|
| 581 |
+
f"store ceiling {budget['document_ceiling_bytes'] / 1048576:g} MB. "
|
| 582 |
+
f"Resync: {resolved}s resolved vs {deployment}.")}
|
| 583 |
+
|
| 584 |
+
|
| 585 |
def _startup_notes():
|
| 586 |
"""Say the two things an operator must know, once, at import β never in a response body."""
|
| 587 |
if aios_session.EPHEMERAL_SECRET:
|
|
|
|
| 590 |
"Set it in production.")
|
| 591 |
|
| 592 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 593 |
# β THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
|
| 594 |
# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
|
| 595 |
# page cannot live behind a password prompt, so the shell's own assets must be public β which they
|
|
|
|
| 797 |
try:
|
| 798 |
from harness import amazon_store as _amazon
|
| 799 |
from harness import runtime as _runtime
|
| 800 |
+
import routes_odoo_tables as _rot
|
| 801 |
# The existing boot/resync projection targets tenant #0. Keep credential, mirror and
|
| 802 |
# projection on that same tenant; never fall back to an operator's environment credential.
|
| 803 |
tenant_key = "royal-imports"
|
|
|
|
| 810 |
# bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
|
| 811 |
# ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
|
| 812 |
# the markers live in the container's mirror, so every restart began again from orders.
|
| 813 |
+
first_load_needed = _amazon.needs_first_load(tenant_key, rt=rt)
|
| 814 |
+
# D-3495: `needs_first_load` (and `sync`, below) call `restore_mirror` as their very
|
| 815 |
+
# first step, which writes the Amazon mirror tables directly on the shared datastore
|
| 816 |
+
# instance. Nothing else republished the read-through registry until the Odoo relational
|
| 817 |
+
# rebuild's `refresh_state` ran, minutes later (`_rebuild_odoo_relational`, further down
|
| 818 |
+
# this boot) β so a grid read landing in between (staging, 2026-09-29: `ut_amzn_orders`
|
| 819 |
+
# 409 for ~30 min after "Amazon mirror restored") answered stale or absent. Invalidate
|
| 820 |
+
# right here, cheaply: this only clears the registry and its built flag, so the next read
|
| 821 |
+
# (or `_sources()` below) does a fresh per-table `count(*)` probe against the mirror that
|
| 822 |
+
# was just restored β never the slow `plan`/`apply_plan` write `refresh_state` does.
|
| 823 |
+
_rot.invalidate_sources()
|
| 824 |
+
_rot._sources()
|
| 825 |
+
if first_load_needed:
|
| 826 |
from core import connector_sync as _connector_sync
|
| 827 |
receipt = _connector_sync.request_sync("amazon_sp", tenant_key, requested_by="first load")
|
| 828 |
print(f"[aios-api] amazon sync ({why}): first load "
|
|
|
|
| 830 |
return True
|
| 831 |
rep = _amazon.sync(tenant_key, rt=rt, scheduled=True,
|
| 832 |
log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}"))
|
| 833 |
+
_rot.invalidate_sources()
|
| 834 |
+
_rot._sources()
|
| 835 |
for problem in rep.get("problems") or []:
|
| 836 |
print(f"[aios-api] amazon sync PROBLEM ({why}): {problem}")
|
| 837 |
print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
|
|
|
|
| 1250 |
|
| 1251 |
|
| 1252 |
def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
|
| 1253 |
+
tenant=None, cadence_seconds=None, table_keys=None):
|
| 1254 |
"""Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
|
| 1255 |
'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
|
| 1256 |
could have asked" are different diagnoses.
|
|
|
|
| 1273 |
from core import user_tables as _ut
|
| 1274 |
default_cid = routes_connectors._odoo_connection_slot(_rt, None)
|
| 1275 |
connection_ids = [default_cid]
|
| 1276 |
+
if table_keys is None:
|
| 1277 |
+
try:
|
| 1278 |
+
import core.keychain as _keychain
|
| 1279 |
+
connection_ids.extend(str(row.get("connection_id") or row.get("id"))
|
| 1280 |
+
for row in _keychain.list_entries(_rt)
|
| 1281 |
+
if row.get("type") == "odoo")
|
| 1282 |
+
except Exception: # noqa: BLE001
|
| 1283 |
+
pass
|
| 1284 |
connection_ids = list(dict.fromkeys(cid for cid in connection_ids if cid))
|
| 1285 |
|
| 1286 |
def _health(cid, stage, state, **kwargs):
|
|
|
|
| 1307 |
# because a gate that counts RAW occurrences of a call would count the one in this
|
| 1308 |
# comment too. That gate happens to strip comments first; the next one may not.
|
| 1309 |
# See `odoo_relational.refresh` and its `all_connections` arm.
|
| 1310 |
+
# Only the designated default owns this mirror. A selection must never be lent to
|
| 1311 |
+
# another connection; the full recovery path still refreshes every connection.
|
| 1312 |
+
counts = _rel.refresh(_rt, tenant_key, all_connections=table_keys is None,
|
| 1313 |
+
**({"table_keys": table_keys} if table_keys is not None else {}))
|
| 1314 |
+
if table_keys is None:
|
| 1315 |
+
raw_models, schema_errors = _rel.authoritative_schema_snapshot()
|
| 1316 |
+
_ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
|
| 1317 |
+
errors=schema_errors, st=_rt)
|
| 1318 |
rebuild_errors = []
|
| 1319 |
# ββ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do.
|
| 1320 |
#
|
|
|
|
| 1390 |
cid = str(cid or default_cid)
|
| 1391 |
report = report if isinstance(report, dict) else {}
|
| 1392 |
failed = report.get("failed")
|
| 1393 |
+
# S67-T19: a read-through `limits` notice is the design working, not a failed
|
| 1394 |
+
# database; health reads only the entries that withheld rows (`_rel.health_view`).
|
| 1395 |
database_failures, unmatched_failures = (
|
| 1396 |
+
routes_connectors._odoo_result_database_failures(
|
| 1397 |
+
_rt, cid, _rel.health_view(report)))
|
| 1398 |
failure = (routes_connectors._odoo_failure(failed, "materialization")
|
| 1399 |
if failed else None)
|
| 1400 |
+
# S67-T19: count drift within tolerance is recorded per database (below) and does not
|
| 1401 |
+
# fail the connection; a real failure or a stale mirror's gap still does.
|
| 1402 |
+
if not failure and (_rel.blocking_failures(database_failures, report)
|
| 1403 |
+
or unmatched_failures):
|
| 1404 |
failure = routes_connectors._odoo_failure(
|
| 1405 |
"one or more databases failed", "materialization")
|
| 1406 |
tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
|
| 1407 |
+
for key, delta in sorted(tables.items()):
|
| 1408 |
+
if isinstance(delta, dict):
|
| 1409 |
+
print(f"[aios-api] odoo relational table rebuilt ({why}): {key} "
|
| 1410 |
+
+ str({name: delta.get(name, 0)
|
| 1411 |
+
for name in ("added", "updated", "removed")}))
|
| 1412 |
database_receipts = report.get("databases")
|
| 1413 |
database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
|
| 1414 |
canonical_receipts = report.get("canonicalDatabases")
|
|
|
|
| 1477 |
_ut.record_odoo_sync_status(
|
| 1478 |
phase="error" if safe_errors else "ok", errors=safe_errors,
|
| 1479 |
st=_rt, connection_id=cid)
|
| 1480 |
+
# S67-T20 / C4: the tenant-wide freshness stamp follows the DEFAULT connection, the
|
| 1481 |
+
# one this mirror serves. Best effort: a stamp must never fail the rebuild it records.
|
| 1482 |
+
if cid == str(default_cid):
|
| 1483 |
+
try:
|
| 1484 |
+
_rel.record_sync_outcome(_rt, ok=not safe_errors, at=refreshed_at)
|
| 1485 |
+
except Exception: # noqa: BLE001
|
| 1486 |
+
pass
|
| 1487 |
meta_state = _rebuild_meta_relational(
|
| 1488 |
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
|
| 1489 |
if isinstance(meta_state_out, dict):
|
|
|
|
| 1513 |
_ut.record_odoo_sync_status(
|
| 1514 |
phase="error", errors=[f"{failure['code']}: {failure['message']}"],
|
| 1515 |
st=_rt, connection_id=cid)
|
| 1516 |
+
# S67-T20 / C4: a failed rebuild is a sync that did NOT end well.
|
| 1517 |
+
import odoo_relational as _rel_failed
|
| 1518 |
+
_rel_failed.record_sync_outcome(_rt, ok=False)
|
| 1519 |
except Exception:
|
| 1520 |
pass
|
| 1521 |
return False
|
| 1522 |
|
| 1523 |
|
| 1524 |
+
# D-780: fetched rows include an idempotent cursor overlap. Only mirror content compared with
|
| 1525 |
+
# the last successful apply proves equality; scalar backfills need not move write_date.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1526 |
_RESYNC_REBUILD_MAX_AGE_SECONDS = 24 * 60 * 60
|
| 1527 |
|
| 1528 |
|
| 1529 |
+
# Dependencies of odoo_relational.plan/read_* and the two read-through table specs. Customers
|
| 1530 |
+
# and products are compiled databases, no longer refreshable ut_odoo_* tables. Unknown entities
|
| 1531 |
+
# (including their sources) deliberately select full recovery, never an empty selection.
|
| 1532 |
+
_RESYNC_ENTITY_TABLE_KEYS = {
|
| 1533 |
+
"account_move": ("ut_odoo_invoices", "ut_odoo_bills", "ut_odoo_vendors"),
|
| 1534 |
+
"sale_order": ("ut_odoo_orders",),
|
| 1535 |
+
"res_partner": ("ut_odoo_invoices", "ut_odoo_orders", "ut_odoo_agents", "ut_odoo_vendors"),
|
| 1536 |
+
"account_invoice_line_agent": ("ut_odoo_invoices", "ut_odoo_agents"),
|
| 1537 |
+
"account_account": ("ut_odoo_accounts",),
|
| 1538 |
+
"sale_order_line": ("ut_odoo_order_lines",),
|
| 1539 |
+
"account_move_line": ("ut_odoo_gl_lines",),
|
| 1540 |
+
}
|
| 1541 |
+
|
| 1542 |
+
|
| 1543 |
+
def _resync_source_fingerprints(sync_report, cur=None):
|
| 1544 |
+
"""Count and hash every column, without fetching rows out of the local mirror.
|
| 1545 |
+
|
| 1546 |
+
Use the sync writer's cursor factory (also used for status/validation), never a second
|
| 1547 |
+
DuckDB instance or a tenant-store read. A missing optional table has an explicit fingerprint;
|
| 1548 |
+
a failed read returns unknown and forces recovery. A lent fixture cursor is never closed.
|
| 1549 |
+
"""
|
| 1550 |
+
owned = cur is None
|
| 1551 |
+
try:
|
| 1552 |
+
if owned:
|
| 1553 |
+
from harness import datastore as _ds
|
| 1554 |
+
cur = _ds.connect()
|
| 1555 |
+
cur.execute("BEGIN TRANSACTION")
|
| 1556 |
+
available = {row[0] for row in cur.execute(
|
| 1557 |
+
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'main'"
|
| 1558 |
+
).fetchall()}
|
| 1559 |
+
entities = set(_RESYNC_ENTITY_TABLE_KEYS) | set(sync_report or {})
|
| 1560 |
+
fingerprints = {}
|
| 1561 |
+
for entity in sorted(entities):
|
| 1562 |
+
if entity not in available:
|
| 1563 |
+
fingerprints[entity] = None
|
| 1564 |
+
continue
|
| 1565 |
+
quoted = '"' + entity.replace('"', '""') + '"'
|
| 1566 |
+
fingerprints[entity] = tuple(cur.execute(
|
| 1567 |
+
f"SELECT count(*), coalesce(bit_xor(hash(t)), 0) FROM {quoted} AS t"
|
| 1568 |
+
).fetchone())
|
| 1569 |
+
if owned:
|
| 1570 |
+
cur.execute("COMMIT")
|
| 1571 |
+
return fingerprints
|
| 1572 |
+
except Exception as exc: # noqa: BLE001
|
| 1573 |
+
print(f"[aios-api] odoo source fingerprint unconfirmed ({type(exc).__name__})")
|
| 1574 |
+
return {}
|
| 1575 |
+
finally:
|
| 1576 |
+
if owned and cur is not None:
|
| 1577 |
+
cur.close()
|
| 1578 |
+
|
| 1579 |
+
|
| 1580 |
+
def _resync_changed_entities(previous_fingerprints, current_fingerprints):
|
| 1581 |
+
"""The union catches disappearing entities as well as additions and scalar changes."""
|
| 1582 |
+
previous = previous_fingerprints or {}
|
| 1583 |
+
current = current_fingerprints or {}
|
| 1584 |
+
return sorted(entity for entity in set(previous) | set(current)
|
| 1585 |
+
if entity not in previous or entity not in current
|
| 1586 |
+
or previous[entity] != current[entity])
|
| 1587 |
+
|
| 1588 |
+
|
| 1589 |
+
def _resync_table_keys(changed_entities):
|
| 1590 |
+
"""None means full recovery; an unmapped entity can never suppress an apply."""
|
| 1591 |
+
if not changed_entities or any(entity not in _RESYNC_ENTITY_TABLE_KEYS
|
| 1592 |
+
for entity in changed_entities):
|
| 1593 |
+
return None
|
| 1594 |
+
return sorted({key for entity in changed_entities for key in _RESYNC_ENTITY_TABLE_KEYS[entity]})
|
| 1595 |
+
|
| 1596 |
+
|
| 1597 |
+
def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now,
|
| 1598 |
+
previous_fingerprints=None, current_fingerprints=None):
|
| 1599 |
"""Return a fail-safe Odoo-rebuild reason, or ``None`` for a proven no-op.
|
| 1600 |
|
| 1601 |
The policy stays pure so it can be tested without Odoo, DuckDB, or PostgreSQL. Hard deletes
|
| 1602 |
are noticed by the existing every-fourth-pass reconciliation; an intervening pass does not
|
| 1603 |
+
pretend that it checked them, preserving the old visibility bound. The four-argument legacy
|
| 1604 |
+
caller retains its conservative fetched-row signal; the resident loop always supplies a
|
| 1605 |
+
fingerprint map (an empty map means a failed read, never equality).
|
| 1606 |
"""
|
| 1607 |
if not isinstance(sync_report, dict) or not sync_report:
|
| 1608 |
return "sync-unconfirmed"
|
|
|
|
| 1610 |
if not isinstance(report, dict) or report.get("phase") != "live":
|
| 1611 |
return "sync-unconfirmed"
|
| 1612 |
try:
|
| 1613 |
+
if int(report.get("pulled") or 0) > 0 and current_fingerprints is None:
|
| 1614 |
return "source-change"
|
| 1615 |
except (TypeError, ValueError):
|
| 1616 |
return "sync-unconfirmed"
|
|
|
|
| 1622 |
return "delete-change"
|
| 1623 |
except (TypeError, ValueError):
|
| 1624 |
return "sync-unconfirmed"
|
| 1625 |
+
if current_fingerprints is not None:
|
| 1626 |
+
if not isinstance(current_fingerprints, dict) or not current_fingerprints:
|
| 1627 |
+
return "fingerprint-unconfirmed"
|
| 1628 |
+
if not isinstance(previous_fingerprints, dict) or not previous_fingerprints:
|
| 1629 |
+
return "no-fingerprint"
|
| 1630 |
if last_success_at is None or now - last_success_at >= _RESYNC_REBUILD_MAX_AGE_SECONDS:
|
| 1631 |
return "daily-recovery"
|
| 1632 |
+
if current_fingerprints is not None:
|
| 1633 |
+
# Kept inline so the pure decision can still be AST-extracted on its own by old gates.
|
| 1634 |
+
changed = sorted(entity for entity in set(previous_fingerprints) | set(current_fingerprints)
|
| 1635 |
+
if entity not in previous_fingerprints or entity not in current_fingerprints
|
| 1636 |
+
or previous_fingerprints[entity] != current_fingerprints[entity])
|
| 1637 |
+
if changed:
|
| 1638 |
+
return "source-change: " + ",".join(changed)
|
| 1639 |
return None
|
| 1640 |
|
| 1641 |
|
|
|
|
| 1790 |
"manual": manual, "disagreement": disagreement}
|
| 1791 |
|
| 1792 |
|
| 1793 |
+
# All import-time refusals run before any boot or resync thread can start.
|
| 1794 |
+
_assert_contract_floor()
|
| 1795 |
+
STORE_GUARDS = _assert_store_guards()
|
| 1796 |
+
from core import neon_usage as _neon_usage # noqa: E402
|
| 1797 |
+
_neon_usage.set_guards(STORE_GUARDS)
|
| 1798 |
+
_startup_notes()
|
| 1799 |
+
|
| 1800 |
+
|
| 1801 |
def _resync_mirror_pass(manual, passes):
|
| 1802 |
"""ONE bounded mirror pass: DuckDB and the Odoo XML-RPC client, and nothing else.
|
| 1803 |
|
|
|
|
| 1908 |
from core import connector_sync
|
| 1909 |
passes = 0
|
| 1910 |
last_relational_success_at = None
|
| 1911 |
+
last_source_fingerprints = None
|
| 1912 |
last_meta_fingerprint = None
|
| 1913 |
last_meta_applied_at = None
|
| 1914 |
# β AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
|
|
|
|
| 2024 |
if _odoo_requested is None:
|
| 2025 |
_odoo_requested = _run_sync_requests()
|
| 2026 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
|
| 2027 |
+
source_fingerprints = _resync_source_fingerprints(sync_report)
|
| 2028 |
RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
|
| 2029 |
RESYNC_STATE["lastPassOk"] = _err is None
|
| 2030 |
RESYNC_STATE["lastError"] = _err
|
|
|
|
| 2077 |
_meta_previous = last_meta_fingerprint if _meta_reason is None else None
|
| 2078 |
_amzn_previous = last_amzn_fingerprint if _meta_reason is None else None
|
| 2079 |
_reason = _resync_rebuild_reason(sync_report, delete_report,
|
| 2080 |
+
last_relational_success_at, _meta_now,
|
| 2081 |
+
last_source_fingerprints, source_fingerprints)
|
| 2082 |
# The scheduled pass may now be quiet because Sync now just advanced this mirror.
|
| 2083 |
# Its zero-change report cannot suppress projection of the explicit Odoo pull.
|
| 2084 |
if _odoo_requested:
|
| 2085 |
_reason = "sync now"
|
| 2086 |
_meta_state = {}
|
| 2087 |
if _reason is not None:
|
| 2088 |
+
_table_keys = (_resync_table_keys(_resync_changed_entities(
|
| 2089 |
+
last_source_fingerprints, source_fingerprints))
|
| 2090 |
+
if _reason.startswith("source-change:") else None)
|
| 2091 |
+
print(f"[aios-api] odoo relational rebuild required ({_reason}); "
|
| 2092 |
+
f"table_keys={_table_keys if _table_keys is not None else 'all'}")
|
| 2093 |
# Test/recovery harnesses replace the rebuild seam with its historical three-argument
|
| 2094 |
# callable. Bind tenant/cadence whenever the active implementation accepts them,
|
| 2095 |
# without turning an internal TypeError into a second materialization attempt.
|
|
|
|
| 2099 |
p.kind == _inspect.Parameter.VAR_KEYWORD for p in _rebuild_params.values())
|
| 2100 |
_rebuild_scope = ({"tenant": _cad["tenant"], "cadence_seconds": _every}
|
| 2101 |
if _rebuild_variadic or "tenant" in _rebuild_params else {})
|
| 2102 |
+
if _rebuild_variadic or "table_keys" in _rebuild_params:
|
| 2103 |
+
_rebuild_scope["table_keys"] = _table_keys
|
| 2104 |
if _rebuild_odoo_relational(
|
| 2105 |
"resync", _meta_previous, _meta_state, **_rebuild_scope):
|
| 2106 |
+
last_source_fingerprints = source_fingerprints or None
|
| 2107 |
+
# Partial successes cannot postpone the mandatory full daily recovery forever.
|
| 2108 |
+
if _table_keys is None:
|
| 2109 |
+
last_relational_success_at = _t.time()
|
| 2110 |
else:
|
| 2111 |
# A prior successful projection no longer proves the changed mirror was applied.
|
| 2112 |
# Keep recovery due on the next cadence even when that sync has no new rows.
|
| 2113 |
last_relational_success_at = None
|
| 2114 |
else:
|
| 2115 |
+
print("[aios-api] odoo relational rebuild skipped (unchanged)")
|
| 2116 |
# A zero Odoo report says nothing about Meta. Its exact-plan fingerprint decides
|
| 2117 |
# whether this cadence may skip both the full-document write and relation pass.
|
| 2118 |
try:
|
|
|
|
| 2305 |
#
|
| 2306 |
# β THE HOTFIX MOVED THE CADENCE ONLY, 3600 -> 14400 (R11's existing four-hourly preset), which
|
| 2307 |
# is safe at any value because the wait below is `sync_wake_event().wait(_every)` and a Sync now
|
| 2308 |
+
# SETS it. D-780 now compares local content fingerprints with the last successful apply and
|
| 2309 |
+
# logs unchanged skips; changed mapped entities select only their dependent table keys.
|
| 2310 |
+
# The other structural cost remains outside that skip policy:
|
|
|
|
| 2311 |
# * the whole-document store write itself. A partial-JSONB write is a wave.
|
| 2312 |
# β SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
|
| 2313 |
if os.environ.get("AIOS_STORE_RESYNC") == "1":
|
api/odoo_relational.py
CHANGED
|
@@ -60,6 +60,7 @@ RI_SLUGS = ("", "royal-imports")
|
|
| 60 |
INVOICES_KEY = "ut_odoo_invoices"
|
| 61 |
CUSTOMERS_KEY = "ut_odoo_customers"
|
| 62 |
ORDERS_KEY = "ut_odoo_orders"
|
|
|
|
| 63 |
PRODUCTS_KEY = "ut_odoo_products"
|
| 64 |
#: β WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents,
|
| 65 |
#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to
|
|
@@ -169,6 +170,34 @@ def _refreshed_field():
|
|
| 169 |
"default": False, "description": "When this row was last reconciled against Odoo."}
|
| 170 |
|
| 171 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
def agent_fields():
|
| 173 |
"""One row per SALES AGENT, keyed on the `res.partner` id.
|
| 174 |
|
|
@@ -1854,6 +1883,9 @@ ODOO_TABLE_MODELS = {
|
|
| 1854 |
ACCOUNTS_KEY: "account.account", VENDORS_KEY: "res.partner", BILLS_KEY: "account.move",
|
| 1855 |
ORDER_LINES_KEY: "sale.order.line", GL_LINES_KEY: "account.move.line",
|
| 1856 |
}
|
|
|
|
|
|
|
|
|
|
| 1857 |
|
| 1858 |
|
| 1859 |
def authoritative_schema_snapshot():
|
|
@@ -2589,6 +2621,7 @@ def _aios_grid():
|
|
| 2589 |
#: crashes" posture `routes_odoo_tables._degrade` already takes for the eight Odoo grids.
|
| 2590 |
_REGISTRY_COL_TYPE = {
|
| 2591 |
"int": "BIGINT", "currency": "DOUBLE", "number": "DOUBLE", "pct": "DOUBLE",
|
|
|
|
| 2592 |
"measure": "DOUBLE", "checkbox": "BOOLEAN", "date": "DATE", "datetime": "TIMESTAMP",
|
| 2593 |
}
|
| 2594 |
|
|
@@ -2610,7 +2643,7 @@ def _registry_col_value(value, kind):
|
|
| 2610 |
try:
|
| 2611 |
if kind == "int":
|
| 2612 |
return int(value)
|
| 2613 |
-
if kind in ("currency", "number", "pct", "measure"):
|
| 2614 |
return float(value)
|
| 2615 |
if kind == "checkbox":
|
| 2616 |
return bool(value)
|
|
@@ -2623,7 +2656,7 @@ def _registry_col_value(value, kind):
|
|
| 2623 |
return str(value)
|
| 2624 |
|
| 2625 |
|
| 2626 |
-
def registry_grid_spec(cur, table_key, fields, rows):
|
| 2627 |
"""A `_source_for`-shaped spec for one compiled registry grid, materialised into a throwaway
|
| 2628 |
DuckDB TEMP TABLE on THIS cursor (W52-T11).
|
| 2629 |
|
|
@@ -2637,7 +2670,8 @@ def registry_grid_spec(cur, table_key, fields, rows):
|
|
| 2637 |
request simply gets `CREATE OR REPLACE`d over with that request's own fresh, freshly-scoped
|
| 2638 |
read. Nothing here survives the cursor closing, and nothing here is written to `royal.duckdb`.
|
| 2639 |
|
| 2640 |
-
|
|
|
|
| 2641 |
datum rather than a column a person can hide (`customer_data.pid` IS the `res.partner` id;
|
| 2642 |
`product_data.pid` is `crc32(default_code)` β see `_related_source_value`'s own note), so it
|
| 2643 |
is forced into the materialised columns here even when the caller's field list never
|
|
@@ -2645,10 +2679,10 @@ def registry_grid_spec(cur, table_key, fields, rows):
|
|
| 2645 |
"""
|
| 2646 |
keys = list(dict.fromkeys(
|
| 2647 |
str(f.get("key") or "") for f in (fields or []) if isinstance(f, dict) and f.get("key")))
|
| 2648 |
-
if
|
| 2649 |
-
keys.insert(0,
|
| 2650 |
by_key = {str(f.get("key")): f for f in (fields or []) if isinstance(f, dict)}
|
| 2651 |
-
by_key.setdefault(
|
| 2652 |
|
| 2653 |
table = f"_registry_grid_{table_key}"
|
| 2654 |
cols_sql = ", ".join(
|
|
@@ -2658,9 +2692,15 @@ def registry_grid_spec(cur, table_key, fields, rows):
|
|
| 2658 |
values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys)
|
| 2659 |
for r in (rows or []) if isinstance(r, dict)]
|
| 2660 |
if values:
|
| 2661 |
-
|
| 2662 |
-
|
| 2663 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2664 |
|
| 2665 |
|
| 2666 |
def _declared_from_spec(fields):
|
|
@@ -2766,6 +2806,7 @@ TABLE_SUBJECTS = {
|
|
| 2766 |
# spec row tried to bring it back.
|
| 2767 |
INVOICES_KEY: "odoo:account.move.customer",
|
| 2768 |
ORDERS_KEY: "odoo:sale.order",
|
|
|
|
| 2769 |
AGENTS_KEY: "odoo:res.partner.agent",
|
| 2770 |
ACCOUNTS_KEY: "odoo:account.account",
|
| 2771 |
VENDORS_KEY: "odoo:res.partner.vendor",
|
|
@@ -3150,7 +3191,8 @@ DEFAULT_CONNECTION = "default"
|
|
| 3150 |
#: Every table key this connector owns, un-namespaced. The two RETIRED keys are members so a
|
| 3151 |
#: document that still holds one resolves to its connection rather than reading as a stranger's
|
| 3152 |
#: table β `apply_plan` has to know whose copy it is popping.
|
| 3153 |
-
BASE_TABLE_KEYS = frozenset([key for _b, key, _l, _f in TABLES]
|
|
|
|
| 3154 |
|
| 3155 |
#: `core.user_tables._field_key` truncates a table key at 60 characters SILENTLY, and a truncated
|
| 3156 |
#: key is a key that can collide with another connection's. The longest base key is
|
|
@@ -3577,6 +3619,112 @@ def sync_seconds(rt):
|
|
| 3577 |
return max(int(secs), SYNC_FLOOR_SECONDS)
|
| 3578 |
|
| 3579 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3580 |
def frozen(rt):
|
| 3581 |
"""Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and
|
| 3582 |
stop being refreshed β distinct from PAUSED, which is temporary and keeps the credential."""
|
|
@@ -3697,15 +3845,9 @@ def plan(cur, rt=None, connection_id=None, buckets=None):
|
|
| 3697 |
continue
|
| 3698 |
built[bucket] = reader(cur, excluded)
|
| 3699 |
problems = []
|
| 3700 |
-
|
| 3701 |
-
|
| 3702 |
-
|
| 3703 |
-
if needed and len(existing) + len(needed) > ut.MAX_TABLES:
|
| 3704 |
-
problems.append(
|
| 3705 |
-
f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and "
|
| 3706 |
-
f"needs {len(needed)} more ({', '.join(needed)}) for connection "
|
| 3707 |
-
f"{connection_id or 'default'}; refusing rather than creating "
|
| 3708 |
-
f"part of a linked set")
|
| 3709 |
|
| 3710 |
for bucket, _base, _label, _fields in tables:
|
| 3711 |
key = keys[bucket]
|
|
@@ -3751,6 +3893,221 @@ def plan(cur, rt=None, connection_id=None, buckets=None):
|
|
| 3751 |
return built
|
| 3752 |
|
| 3753 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3754 |
def apply_plan(rt, built, username="automation", today=None, report=None,
|
| 3755 |
connection_id=None, retire=True):
|
| 3756 |
"""Create-or-merge every table in `built` and its rows. Idempotent by construction.
|
|
@@ -3864,10 +4221,8 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
|
|
| 3864 |
subject = table_subject_for(key)
|
| 3865 |
table = cur.get(key)
|
| 3866 |
if table is None:
|
| 3867 |
-
|
| 3868 |
-
|
| 3869 |
-
# successful refresh over a table that does not exist.
|
| 3870 |
-
raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created")
|
| 3871 |
# ββ W33-T41 / item 12a β THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE.
|
| 3872 |
#
|
| 3873 |
# β IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This
|
|
@@ -4195,11 +4550,8 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None,
|
|
| 4195 |
f"{choice['model']}: too large to store, served nowhere until read-through"
|
| 4196 |
for choice in oversized]}
|
| 4197 |
keys = [str(choice["key"]) for choice in selected]
|
| 4198 |
-
|
| 4199 |
-
|
| 4200 |
-
raise Refused(f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} databases and "
|
| 4201 |
-
f"needs {len(missing)} more enabled Odoo models; nothing was created")
|
| 4202 |
-
source = (rt.odoo_source() if source_connection_id is None
|
| 4203 |
else rt.odoo_source(source_connection_id))
|
| 4204 |
if source is None:
|
| 4205 |
raise Refused("the selected Odoo connection is unavailable, so no discovered model was read")
|
|
@@ -4312,6 +4664,93 @@ def available(rt=None, tenant_key=None, connection_id=None):
|
|
| 4312 |
return False
|
| 4313 |
|
| 4314 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4315 |
def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
|
| 4316 |
today=None, retire=True, on_apply=None, source_client=None):
|
| 4317 |
"""Refresh each enabled database independently; retain failed databases unchanged.
|
|
@@ -4598,6 +5037,16 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
|
|
| 4598 |
return out
|
| 4599 |
|
| 4600 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4601 |
def refresh_all(rt, tenant, username="automation", cur=None, today=None):
|
| 4602 |
"""Refresh EVERY Odoo connection this tenant holds, one at a time (R4 / R7 / C4).
|
| 4603 |
|
|
@@ -4634,12 +5083,20 @@ def refresh_all(rt, tenant, username="automation", cur=None, today=None):
|
|
| 4634 |
primary = None
|
| 4635 |
for entry_cid in order:
|
| 4636 |
named = None if (entry_cid is None or entry_cid == default) else entry_cid
|
| 4637 |
-
|
| 4638 |
-
|
| 4639 |
-
|
| 4640 |
-
|
| 4641 |
-
|
| 4642 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4643 |
reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
|
| 4644 |
if named is None and primary is None:
|
| 4645 |
primary = one
|
|
|
|
| 60 |
INVOICES_KEY = "ut_odoo_invoices"
|
| 61 |
CUSTOMERS_KEY = "ut_odoo_customers"
|
| 62 |
ORDERS_KEY = "ut_odoo_orders"
|
| 63 |
+
SHIPMENTS_KEY = "ut_odoo_shipments"
|
| 64 |
PRODUCTS_KEY = "ut_odoo_products"
|
| 65 |
#: β WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents,
|
| 66 |
#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to
|
|
|
|
| 170 |
"default": False, "description": "When this row was last reconciled against Odoo."}
|
| 171 |
|
| 172 |
|
| 173 |
+
def shipment_fields():
|
| 174 |
+
"""Outgoing pickings, with elapsed delivery time measured at actual completion."""
|
| 175 |
+
return [_preset({"source": "overlay", **field}) for field in (
|
| 176 |
+
{"key": "picking_no", "type": "text", "label": "Shipment", "pinned": True,
|
| 177 |
+
"default": True},
|
| 178 |
+
{"key": "odoo_id", "type": "int", "label": "Odoo ID", "default": False,
|
| 179 |
+
"description": "The `stock.picking` id. Also this row's id."},
|
| 180 |
+
{"key": "origin", "type": "text", "label": "Source document", "default": True},
|
| 181 |
+
{"key": "order_no", "type": "text", "label": "Order", "default": True},
|
| 182 |
+
{"key": "order_id", "type": "int", "label": "Odoo order id", "default": False},
|
| 183 |
+
{"key": "order_link", "type": "link", "label": "Order record", "default": True,
|
| 184 |
+
"link": {"on": "odoo_id", "from": "order_id", "table": ORDERS_KEY}},
|
| 185 |
+
{"key": "customer", "type": "text", "label": "Customer", "default": True},
|
| 186 |
+
{"key": "partner_id", "type": "int", "label": "Odoo partner id", "default": False},
|
| 187 |
+
{"key": "customer_link", "type": "link", "label": "Customer record", "default": False,
|
| 188 |
+
"link": {"on": "pid", "from": "partner_id", "table": "customer_data"}},
|
| 189 |
+
{"key": "state", "type": "select", "label": "State", "default": True,
|
| 190 |
+
"options": ["draft", "waiting", "confirmed", "assigned", "done", "cancel"]},
|
| 191 |
+
{"key": "scheduled_date", "type": "date", "label": "Scheduled date", "default": True},
|
| 192 |
+
{"key": "date_done", "type": "date", "label": "Ship date", "default": True},
|
| 193 |
+
{"key": "order_date", "type": "date", "label": "Order date", "default": True},
|
| 194 |
+
{"key": "delivery_days", "type": "float", "agg": "avg", "label": "Delivery days",
|
| 195 |
+
"default": True, "description": "Elapsed days from the sale order date to the actual "
|
| 196 |
+
"shipment completion. Blank when either date is missing."},
|
| 197 |
+
_refreshed_field(),
|
| 198 |
+
)]
|
| 199 |
+
|
| 200 |
+
|
| 201 |
def agent_fields():
|
| 202 |
"""One row per SALES AGENT, keyed on the `res.partner` id.
|
| 203 |
|
|
|
|
| 1883 |
ACCOUNTS_KEY: "account.account", VENDORS_KEY: "res.partner", BILLS_KEY: "account.move",
|
| 1884 |
ORDER_LINES_KEY: "sale.order.line", GL_LINES_KEY: "account.move.line",
|
| 1885 |
}
|
| 1886 |
+
# Shipments deliberately stay outside this map and TABLES: its consumers route mapped models
|
| 1887 |
+
# through the DuckDB refresh and suppress their discovered-model reads. There is no picking
|
| 1888 |
+
# mirror. The live shipment writer below stamps its model directly on the stored connector.
|
| 1889 |
|
| 1890 |
|
| 1891 |
def authoritative_schema_snapshot():
|
|
|
|
| 2621 |
#: crashes" posture `routes_odoo_tables._degrade` already takes for the eight Odoo grids.
|
| 2622 |
_REGISTRY_COL_TYPE = {
|
| 2623 |
"int": "BIGINT", "currency": "DOUBLE", "number": "DOUBLE", "pct": "DOUBLE",
|
| 2624 |
+
"float": "DOUBLE",
|
| 2625 |
"measure": "DOUBLE", "checkbox": "BOOLEAN", "date": "DATE", "datetime": "TIMESTAMP",
|
| 2626 |
}
|
| 2627 |
|
|
|
|
| 2643 |
try:
|
| 2644 |
if kind == "int":
|
| 2645 |
return int(value)
|
| 2646 |
+
if kind in ("currency", "number", "pct", "measure", "float"):
|
| 2647 |
return float(value)
|
| 2648 |
if kind == "checkbox":
|
| 2649 |
return bool(value)
|
|
|
|
| 2656 |
return str(value)
|
| 2657 |
|
| 2658 |
|
| 2659 |
+
def registry_grid_spec(cur, table_key, fields, rows, *, id_key="pid", bulk=False):
|
| 2660 |
"""A `_source_for`-shaped spec for one compiled registry grid, materialised into a throwaway
|
| 2661 |
DuckDB TEMP TABLE on THIS cursor (W52-T11).
|
| 2662 |
|
|
|
|
| 2670 |
request simply gets `CREATE OR REPLACE`d over with that request's own fresh, freshly-scoped
|
| 2671 |
read. Nothing here survives the cursor closing, and nothing here is written to `royal.duckdb`.
|
| 2672 |
|
| 2673 |
+
The default identity is `pid`; stored shipments explicitly use `odoo_id` instead.
|
| 2674 |
+
Both registry grains carry `pid` as a raw row
|
| 2675 |
datum rather than a column a person can hide (`customer_data.pid` IS the `res.partner` id;
|
| 2676 |
`product_data.pid` is `crc32(default_code)` β see `_related_source_value`'s own note), so it
|
| 2677 |
is forced into the materialised columns here even when the caller's field list never
|
|
|
|
| 2679 |
"""
|
| 2680 |
keys = list(dict.fromkeys(
|
| 2681 |
str(f.get("key") or "") for f in (fields or []) if isinstance(f, dict) and f.get("key")))
|
| 2682 |
+
if id_key not in keys:
|
| 2683 |
+
keys.insert(0, id_key)
|
| 2684 |
by_key = {str(f.get("key")): f for f in (fields or []) if isinstance(f, dict)}
|
| 2685 |
+
by_key.setdefault(id_key, {"type": "int"})
|
| 2686 |
|
| 2687 |
table = f"_registry_grid_{table_key}"
|
| 2688 |
cols_sql = ", ".join(
|
|
|
|
| 2692 |
values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys)
|
| 2693 |
for r in (rows or []) if isinstance(r, dict)]
|
| 2694 |
if values:
|
| 2695 |
+
if bulk:
|
| 2696 |
+
# Column arrays avoid 40,000 Python/DuckDB execute round trips for stored shipments.
|
| 2697 |
+
selects = ", ".join(
|
| 2698 |
+
f'unnest(?)::{_REGISTRY_COL_TYPE.get(by_key[k].get("type"), "VARCHAR")}' for k in keys)
|
| 2699 |
+
cur.execute(f'INSERT INTO "{table}" SELECT {selects}', [list(col) for col in zip(*values)])
|
| 2700 |
+
else:
|
| 2701 |
+
placeholders = ", ".join(["?"] * len(keys))
|
| 2702 |
+
cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values)
|
| 2703 |
+
return {"table": table, "id": f'"{id_key}"', "cols": {k: (f'"{k}"', str) for k in keys}}
|
| 2704 |
|
| 2705 |
|
| 2706 |
def _declared_from_spec(fields):
|
|
|
|
| 2806 |
# spec row tried to bring it back.
|
| 2807 |
INVOICES_KEY: "odoo:account.move.customer",
|
| 2808 |
ORDERS_KEY: "odoo:sale.order",
|
| 2809 |
+
SHIPMENTS_KEY: "odoo:stock.picking.outgoing",
|
| 2810 |
AGENTS_KEY: "odoo:res.partner.agent",
|
| 2811 |
ACCOUNTS_KEY: "odoo:account.account",
|
| 2812 |
VENDORS_KEY: "odoo:res.partner.vendor",
|
|
|
|
| 3191 |
#: Every table key this connector owns, un-namespaced. The two RETIRED keys are members so a
|
| 3192 |
#: document that still holds one resolves to its connection rather than reading as a stranger's
|
| 3193 |
#: table β `apply_plan` has to know whose copy it is popping.
|
| 3194 |
+
BASE_TABLE_KEYS = frozenset([key for _b, key, _l, _f in TABLES]
|
| 3195 |
+
+ list(RETIRED_KEYS) + [SHIPMENTS_KEY])
|
| 3196 |
|
| 3197 |
#: `core.user_tables._field_key` truncates a table key at 60 characters SILENTLY, and a truncated
|
| 3198 |
#: key is a key that can collide with another connection's. The longest base key is
|
|
|
|
| 3619 |
return max(int(secs), SYNC_FLOOR_SECONDS)
|
| 3620 |
|
| 3621 |
|
| 3622 |
+
#: ββ S67-T20 / C4 β how far past its cadence a sync may run before its data is called stale.
|
| 3623 |
+
#: Two hours of grace on top of the interval: a daily sync is stale after 26 hours (C4's number),
|
| 3624 |
+
#: long enough that one slow pass does not raise a warning and short enough that a dead sync is
|
| 3625 |
+
#: named on the next day, not three weeks later as staging's was.
|
| 3626 |
+
STALE_GRACE_SECONDS = 2 * 3600
|
| 3627 |
+
|
| 3628 |
+
|
| 3629 |
+
def _utc_now():
|
| 3630 |
+
return _dt.datetime.now(_dt.timezone.utc)
|
| 3631 |
+
|
| 3632 |
+
|
| 3633 |
+
def _parse_iso(value):
|
| 3634 |
+
"""An ISO timestamp as an aware UTC datetime, or None when it is not one."""
|
| 3635 |
+
if not value:
|
| 3636 |
+
return None
|
| 3637 |
+
try:
|
| 3638 |
+
at = _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
| 3639 |
+
except ValueError:
|
| 3640 |
+
return None
|
| 3641 |
+
return at if at.tzinfo else at.replace(tzinfo=_dt.timezone.utc)
|
| 3642 |
+
|
| 3643 |
+
|
| 3644 |
+
def record_sync_outcome(rt, ok, at=None):
|
| 3645 |
+
"""Stamp this tenant's last sync, and its last GOOD sync when `ok`, on the connector's config.
|
| 3646 |
+
|
| 3647 |
+
β ON THE CONFIG DOCUMENT, NOT ON `odoo_sync_status`, because that document is REPLACED whole
|
| 3648 |
+
on every phase write (`core.user_tables.record_odoo_sync_status`), so a stamp kept there would
|
| 3649 |
+
be erased by the next `syncing`. Every writer of this config document updates it in place.
|
| 3650 |
+
"""
|
| 3651 |
+
stamp = at or _utc_now().isoformat()
|
| 3652 |
+
|
| 3653 |
+
def _up(cur):
|
| 3654 |
+
cur = dict(cur) if isinstance(cur, dict) else {}
|
| 3655 |
+
cur["lastSyncAt"] = stamp
|
| 3656 |
+
if ok:
|
| 3657 |
+
cur["lastGoodSyncAt"] = stamp
|
| 3658 |
+
return cur
|
| 3659 |
+
|
| 3660 |
+
rt.update(CONFIG_KEY, _up, flush="sync")
|
| 3661 |
+
return stamp
|
| 3662 |
+
|
| 3663 |
+
|
| 3664 |
+
def _effective_sync_seconds(preset):
|
| 3665 |
+
"""The interval a sync can actually keep: the tenant's preset, floored, and never faster than
|
| 3666 |
+
the deployment's own resync floor (`AIOS_STORE_RESYNC_SECONDS`), which `main.py` applies with
|
| 3667 |
+
`max()`. None for `manual`: a person who asked not to be synced has no schedule to fall
|
| 3668 |
+
behind."""
|
| 3669 |
+
secs = SYNC_PRESETS.get(preset)
|
| 3670 |
+
if secs is None:
|
| 3671 |
+
return None
|
| 3672 |
+
secs = max(int(secs), SYNC_FLOOR_SECONDS)
|
| 3673 |
+
import os
|
| 3674 |
+
try:
|
| 3675 |
+
env = int(str(os.environ.get("AIOS_STORE_RESYNC_SECONDS") or "").strip() or 0)
|
| 3676 |
+
except ValueError:
|
| 3677 |
+
env = 0
|
| 3678 |
+
return max(secs, env)
|
| 3679 |
+
|
| 3680 |
+
|
| 3681 |
+
def sync_freshness(rt, now=None):
|
| 3682 |
+
"""C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run
|
| 3683 |
+
fail. PURE over the store reads.
|
| 3684 |
+
|
| 3685 |
+
`stale` is true when the last good sync is older than the effective cadence plus
|
| 3686 |
+
`STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected
|
| 3687 |
+
(frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is
|
| 3688 |
+
the last run's own reason, its code prefix removed, when that run ended in error.
|
| 3689 |
+
"""
|
| 3690 |
+
now = now or _utc_now()
|
| 3691 |
+
try:
|
| 3692 |
+
raw = rt.get(CONFIG_KEY) if rt is not None else None
|
| 3693 |
+
except Exception: # noqa: BLE001
|
| 3694 |
+
raw = None
|
| 3695 |
+
raw = raw if isinstance(raw, dict) else {}
|
| 3696 |
+
cfg = read_config(rt)
|
| 3697 |
+
try:
|
| 3698 |
+
status = rt.get(_ut().ODOO_SYNC_STATUS_KEY) if rt is not None else None
|
| 3699 |
+
except Exception: # noqa: BLE001
|
| 3700 |
+
status = None
|
| 3701 |
+
status = status if isinstance(status, dict) else {}
|
| 3702 |
+
phase = str(status.get("phase") or "unknown")
|
| 3703 |
+
last_good = raw.get("lastGoodSyncAt") or None
|
| 3704 |
+
# Before the first stamp this deployment writes, a status that last read `ok` IS a good sync.
|
| 3705 |
+
if not last_good and phase == "ok":
|
| 3706 |
+
last_good = status.get("lastRun") or None
|
| 3707 |
+
every = _effective_sync_seconds(cfg["syncEvery"])
|
| 3708 |
+
allowed = None if every is None else every + STALE_GRACE_SECONDS
|
| 3709 |
+
stale = False
|
| 3710 |
+
if allowed is not None and not cfg["frozen"]:
|
| 3711 |
+
good_at = _parse_iso(last_good)
|
| 3712 |
+
stale = good_at is None or (now - good_at).total_seconds() > allowed
|
| 3713 |
+
failure = None
|
| 3714 |
+
errors = status.get("errors") if isinstance(status.get("errors"), list) else []
|
| 3715 |
+
if phase == "error" and errors:
|
| 3716 |
+
text = str(errors[0])
|
| 3717 |
+
code, _sep, message = text.partition(": ")
|
| 3718 |
+
if not _sep:
|
| 3719 |
+
code, message = "sync_failed", text
|
| 3720 |
+
failure = {"code": code.strip()[:80], "message": message.strip()[:300],
|
| 3721 |
+
"at": status.get("lastRun") or None}
|
| 3722 |
+
return {"stale": bool(stale), "lastGoodSyncAt": last_good,
|
| 3723 |
+
"schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
|
| 3724 |
+
"lastRunAt": status.get("lastRun") or None, "phase": phase,
|
| 3725 |
+
"frozen": cfg["frozen"], "failure": failure}
|
| 3726 |
+
|
| 3727 |
+
|
| 3728 |
def frozen(rt):
|
| 3729 |
"""Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and
|
| 3730 |
stop being refreshed β distinct from PAUSED, which is temporary and keeps the credential."""
|
|
|
|
| 3845 |
continue
|
| 3846 |
built[bucket] = reader(cur, excluded)
|
| 3847 |
problems = []
|
| 3848 |
+
# ββ D-3284 (owner ruling 2026-09-27): connector databases take NO `MAX_TABLES` slot
|
| 3849 |
+
# (`user_tables.counts_toward_table_cap`), so the cap can no longer refuse, or half-spawn, a
|
| 3850 |
+
# linked set. The refusal that stood here counted every database, people's and connectors'.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3851 |
|
| 3852 |
for bucket, _base, _label, _fields in tables:
|
| 3853 |
key = keys[bucket]
|
|
|
|
| 3893 |
return built
|
| 3894 |
|
| 3895 |
|
| 3896 |
+
def _shipment_datetime(value):
|
| 3897 |
+
"""Odoo's naive datetimes are UTC; preserve seconds before deriving date cells."""
|
| 3898 |
+
if not value:
|
| 3899 |
+
return None
|
| 3900 |
+
parsed = _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
| 3901 |
+
return (parsed.replace(tzinfo=_dt.timezone.utc) if parsed.tzinfo is None
|
| 3902 |
+
else parsed.astimezone(_dt.timezone.utc))
|
| 3903 |
+
|
| 3904 |
+
|
| 3905 |
+
def _shipment_relation_id(value):
|
| 3906 |
+
if isinstance(value, (list, tuple)):
|
| 3907 |
+
value = value[0] if value else None
|
| 3908 |
+
return int(value) if value else None
|
| 3909 |
+
|
| 3910 |
+
|
| 3911 |
+
def read_shipments_live(client):
|
| 3912 |
+
"""All outgoing pickings in id pages, then their sale orders in bounded read batches.
|
| 3913 |
+
|
| 3914 |
+
Only guarded read methods are used. The guarded OdooClient has no ``read`` convenience
|
| 3915 |
+
method, so sale-order batches use search_read with an explicit id domain as well.
|
| 3916 |
+
"""
|
| 3917 |
+
pickings, last_id = [], 0
|
| 3918 |
+
fields = ["name", "origin", "sale_id", "partner_id", "state", "scheduled_date", "date_done"]
|
| 3919 |
+
while True:
|
| 3920 |
+
domain = [("picking_type_code", "=", "outgoing")]
|
| 3921 |
+
if last_id:
|
| 3922 |
+
domain.append(("id", ">", last_id))
|
| 3923 |
+
page = client.search_read("stock.picking", domain, fields,
|
| 3924 |
+
limit=MODEL_READ_PAGE, order="id")
|
| 3925 |
+
if not page:
|
| 3926 |
+
break
|
| 3927 |
+
for picking in page:
|
| 3928 |
+
rid = int(picking["id"])
|
| 3929 |
+
if rid <= last_id:
|
| 3930 |
+
raise Refused("stock.picking: source ids did not advance; no shipments written")
|
| 3931 |
+
last_id = rid
|
| 3932 |
+
pickings.append(picking)
|
| 3933 |
+
# Continue until an empty page, even if the server returned less than requested.
|
| 3934 |
+
|
| 3935 |
+
ids = sorted({oid for picking in pickings
|
| 3936 |
+
if (oid := _shipment_relation_id(picking.get("sale_id"))) is not None})
|
| 3937 |
+
orders = {}
|
| 3938 |
+
for start in range(0, len(ids), MODEL_READ_PAGE):
|
| 3939 |
+
chunk = ids[start:start + MODEL_READ_PAGE]
|
| 3940 |
+
batch = client.search_read("sale.order", [("id", "in", chunk)],
|
| 3941 |
+
["name", "date_order"], limit=len(chunk), order="id")
|
| 3942 |
+
if len(batch) != len(chunk) or {int(row["id"]) for row in batch} != set(chunk):
|
| 3943 |
+
raise Refused("sale.order: referenced shipment orders were not all readable; "
|
| 3944 |
+
"the prior database was preserved. Check Odoo read access and retry.")
|
| 3945 |
+
orders.update((int(row["id"]), row) for row in batch)
|
| 3946 |
+
|
| 3947 |
+
rows, stamp = {}, _iso_today()
|
| 3948 |
+
for picking in pickings:
|
| 3949 |
+
rid = int(picking["id"])
|
| 3950 |
+
oid = _shipment_relation_id(picking.get("sale_id"))
|
| 3951 |
+
partner = picking.get("partner_id")
|
| 3952 |
+
pid = _shipment_relation_id(partner)
|
| 3953 |
+
order = orders.get(oid) or {}
|
| 3954 |
+
ordered = _shipment_datetime(order.get("date_order"))
|
| 3955 |
+
done = _shipment_datetime(picking.get("date_done"))
|
| 3956 |
+
scheduled = _shipment_datetime(picking.get("scheduled_date"))
|
| 3957 |
+
rows[str(rid)] = {
|
| 3958 |
+
"picking_no": picking.get("name") or "", "odoo_id": rid,
|
| 3959 |
+
"origin": picking.get("origin") or "", "order_no": order.get("name") or "",
|
| 3960 |
+
"order_id": oid, **({"order_link": str(oid)} if oid is not None else {}),
|
| 3961 |
+
"customer": (partner[1] or "")
|
| 3962 |
+
if isinstance(partner, (list, tuple)) and len(partner) > 1 else "",
|
| 3963 |
+
"partner_id": pid, **({"customer_link": str(pid)} if pid is not None else {}),
|
| 3964 |
+
"state": picking.get("state") or "",
|
| 3965 |
+
"scheduled_date": scheduled.date().isoformat() if scheduled else "",
|
| 3966 |
+
"date_done": done.date().isoformat() if done else "",
|
| 3967 |
+
"order_date": ordered.date().isoformat() if ordered else "",
|
| 3968 |
+
"delivery_days": (done - ordered).total_seconds() / 86400
|
| 3969 |
+
if done and ordered else None,
|
| 3970 |
+
"refreshed": stamp,
|
| 3971 |
+
}
|
| 3972 |
+
return rows
|
| 3973 |
+
|
| 3974 |
+
|
| 3975 |
+
def reconcile_shipments(rows, client, year):
|
| 3976 |
+
"""Independent source count for completed outgoing shipments in one UTC calendar year."""
|
| 3977 |
+
year = int(year)
|
| 3978 |
+
start, end = f"{year:04d}-01-01", f"{year + 1:04d}-01-01"
|
| 3979 |
+
values = rows.values() if isinstance(rows, dict) else rows
|
| 3980 |
+
count = sum(1 for row in values if row.get("state") == "done"
|
| 3981 |
+
and start <= (row.get("date_done") or "") < end)
|
| 3982 |
+
source_count = client.search_count("stock.picking", [
|
| 3983 |
+
("picking_type_code", "=", "outgoing"), ("state", "=", "done"),
|
| 3984 |
+
("date_done", ">=", start + " 00:00:00"), ("date_done", "<", end + " 00:00:00")])
|
| 3985 |
+
return {"year": year, "rowCount": count, "sourceCount": source_count,
|
| 3986 |
+
"ok": type(source_count) is int and source_count == count}
|
| 3987 |
+
|
| 3988 |
+
|
| 3989 |
+
def _shipment_problems(rows, existing, key):
|
| 3990 |
+
ut = _ut()
|
| 3991 |
+
problems = []
|
| 3992 |
+
stored = len((existing.get(key) or {}).get("rows") or {})
|
| 3993 |
+
if stored and len(rows) < stored * MAX_SHRINK:
|
| 3994 |
+
problems.append(f"{key}: read returned {len(rows)} rows against {stored} stored; "
|
| 3995 |
+
"preserved the prior database. Check Odoo read access and retry sync.")
|
| 3996 |
+
# D-3284: no MAX_TABLES check here; a connector database takes no slot.
|
| 3997 |
+
size = ut.rows_bytes(rows)
|
| 3998 |
+
if size > ut.MAX_TABLE_BYTES:
|
| 3999 |
+
problems.append(f"{key}: {size} bytes exceeds MAX_TABLE_BYTES={ut.MAX_TABLE_BYTES}; "
|
| 4000 |
+
"the prior database was preserved. Use read-through storage before retrying.")
|
| 4001 |
+
return problems
|
| 4002 |
+
|
| 4003 |
+
|
| 4004 |
+
def plan_shipments(rt, connection_id=None, client=None, today=None):
|
| 4005 |
+
"""Prepare shipments without writing a store; injected clients keep gates hermetic."""
|
| 4006 |
+
if client is None:
|
| 4007 |
+
source = rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id)
|
| 4008 |
+
if source is None:
|
| 4009 |
+
raise Refused("the selected Odoo connection is unavailable; no shipments read")
|
| 4010 |
+
from core.odoo import get_odoo
|
| 4011 |
+
with source._bound():
|
| 4012 |
+
return plan_shipments(rt, connection_id, get_odoo(), today)
|
| 4013 |
+
_cid, namespace = connection_namespace(rt, connection_id)
|
| 4014 |
+
rows = read_shipments_live(client)
|
| 4015 |
+
for row in rows.values():
|
| 4016 |
+
row["refreshed"] = today or _iso_today()
|
| 4017 |
+
if namespace:
|
| 4018 |
+
row.pop("customer_link", None) # customer_data belongs to the default connection
|
| 4019 |
+
key = table_key_for(SHIPMENTS_KEY, namespace)
|
| 4020 |
+
result = {"rows": rows, "problems": _shipment_problems(
|
| 4021 |
+
rows, rt.get(_ut().STORE_KEY) or {}, key)}
|
| 4022 |
+
if namespace:
|
| 4023 |
+
result["connection_id"] = namespace
|
| 4024 |
+
return result
|
| 4025 |
+
|
| 4026 |
+
|
| 4027 |
+
def apply_shipments(rt, planned, username="automation", today=None):
|
| 4028 |
+
"""Publish via the relational writer, refusing partial reads before touching stored rows."""
|
| 4029 |
+
from copy import deepcopy
|
| 4030 |
+
from core import connector_sync
|
| 4031 |
+
if planned.get("problems"):
|
| 4032 |
+
raise Refused("; ".join(planned["problems"]))
|
| 4033 |
+
ut, stamp = _ut(), today or _iso_today()
|
| 4034 |
+
namespace = planned.get("connection_id")
|
| 4035 |
+
key = table_key_for(SHIPMENTS_KEY, namespace)
|
| 4036 |
+
rows = {rid: {**row, "refreshed": stamp} for rid, row in planned["rows"].items()}
|
| 4037 |
+
problems = _shipment_problems(rows, rt.get(ut.STORE_KEY) or {}, key)
|
| 4038 |
+
if problems:
|
| 4039 |
+
raise Refused("; ".join(problems))
|
| 4040 |
+
written = {}
|
| 4041 |
+
|
| 4042 |
+
def apply(current):
|
| 4043 |
+
current = current if isinstance(current, dict) else {}
|
| 4044 |
+
# Recheck the current document under the store's update lock, including concurrent growth.
|
| 4045 |
+
problems = _shipment_problems(rows, current, key)
|
| 4046 |
+
if problems:
|
| 4047 |
+
raise Refused("; ".join(problems))
|
| 4048 |
+
current = dict(current)
|
| 4049 |
+
if key in current:
|
| 4050 |
+
current[key] = deepcopy(current[key])
|
| 4051 |
+
previous = (current.get(key) or {}).get("rows") or {}
|
| 4052 |
+
link_only_changes = sum(
|
| 4053 |
+
1 for rid, row in rows.items() if rid in previous
|
| 4054 |
+
and any(link in previous[rid] and link not in row
|
| 4055 |
+
for link in ("order_link", "customer_link"))
|
| 4056 |
+
and all(str(previous[rid].get(k, "")) == str(v)
|
| 4057 |
+
for k, v in row.items() if k != "refreshed"))
|
| 4058 |
+
new = key not in current
|
| 4059 |
+
written[key] = _ensure_table_inplace(
|
| 4060 |
+
current, key, table_label_for("Odoo shipments", connection_label(rt, namespace)),
|
| 4061 |
+
namespaced_fields(shipment_fields(), namespace),
|
| 4062 |
+
[{**row, "_id": rid} for rid, row in rows.items()], username, stamp)
|
| 4063 |
+
if new:
|
| 4064 |
+
current[key].pop("created", None) # match the existing staging definition
|
| 4065 |
+
for rid, row in rows.items():
|
| 4066 |
+
for link in ("order_link", "customer_link"):
|
| 4067 |
+
if link not in row:
|
| 4068 |
+
current[key]["rows"][rid].pop(link, None)
|
| 4069 |
+
written[key]["updated"] += link_only_changes
|
| 4070 |
+
current[key]["connector"] = {
|
| 4071 |
+
**connector_sync.database_identity(
|
| 4072 |
+
"odoo", connection_for_table_key(key, rt), key, "stock.picking"),
|
| 4073 |
+
"key": "odoo", "readOnly": True}
|
| 4074 |
+
# Existing user cells also count toward the unchanged byte budget.
|
| 4075 |
+
problems = _shipment_problems(current[key]["rows"], current, key)
|
| 4076 |
+
if problems:
|
| 4077 |
+
raise Refused("; ".join(problems))
|
| 4078 |
+
return current
|
| 4079 |
+
|
| 4080 |
+
rt.update(ut.STORE_KEY, apply, flush="sync")
|
| 4081 |
+
ut.register_connected(key)
|
| 4082 |
+
return written
|
| 4083 |
+
|
| 4084 |
+
|
| 4085 |
+
def refresh_shipments(rt, *, connection_id=None, year=None, today=None,
|
| 4086 |
+
username="automation", client=None):
|
| 4087 |
+
"""Reconcile before publication, then independently count the rows read back from storage."""
|
| 4088 |
+
if client is None:
|
| 4089 |
+
source = rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id)
|
| 4090 |
+
if source is None:
|
| 4091 |
+
raise Refused("the selected Odoo connection is unavailable; no shipments read")
|
| 4092 |
+
from core.odoo import get_odoo
|
| 4093 |
+
with source._bound():
|
| 4094 |
+
return refresh_shipments(rt, connection_id=connection_id, year=year, today=today,
|
| 4095 |
+
username=username, client=get_odoo())
|
| 4096 |
+
stamp = today or _iso_today()
|
| 4097 |
+
year = int(year) if year is not None else int(stamp[:4])
|
| 4098 |
+
planned = plan_shipments(rt, connection_id, client, stamp)
|
| 4099 |
+
reconciled = reconcile_shipments(planned["rows"], client, year)
|
| 4100 |
+
if not reconciled["ok"]:
|
| 4101 |
+
raise Refused(f"stock.picking: {year} done outgoing shipments do not reconcile "
|
| 4102 |
+
f"({reconciled['rowCount']} planned, {reconciled['sourceCount']} source); "
|
| 4103 |
+
"nothing written")
|
| 4104 |
+
applied = apply_shipments(rt, planned, username, stamp)
|
| 4105 |
+
key = table_key_for(SHIPMENTS_KEY, planned.get("connection_id"))
|
| 4106 |
+
stored = ((rt.get(_ut().STORE_KEY, fresh=True) or {}).get(key) or {}).get("rows") or {}
|
| 4107 |
+
return {"planned": planned, "reconcile_planned": reconciled, "apply": applied,
|
| 4108 |
+
"stored_rows": stored, "reconcile_stored": reconcile_shipments(stored, client, year)}
|
| 4109 |
+
|
| 4110 |
+
|
| 4111 |
def apply_plan(rt, built, username="automation", today=None, report=None,
|
| 4112 |
connection_id=None, retire=True):
|
| 4113 |
"""Create-or-merge every table in `built` and its rows. Idempotent by construction.
|
|
|
|
| 4221 |
subject = table_subject_for(key)
|
| 4222 |
table = cur.get(key)
|
| 4223 |
if table is None:
|
| 4224 |
+
# D-3284 (owner ruling 2026-09-27): every table this writer creates is a connector
|
| 4225 |
+
# database (Odoo, Meta, Amazon), and those take no MAX_TABLES slot, so no cap refusal here.
|
|
|
|
|
|
|
| 4226 |
# ββ W33-T41 / item 12a β THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE.
|
| 4227 |
#
|
| 4228 |
# β IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This
|
|
|
|
| 4550 |
f"{choice['model']}: too large to store, served nowhere until read-through"
|
| 4551 |
for choice in oversized]}
|
| 4552 |
keys = [str(choice["key"]) for choice in selected]
|
| 4553 |
+
# D-3284: enabled Odoo models are connector databases and take no MAX_TABLES slot.
|
| 4554 |
+
source =(rt.odoo_source() if source_connection_id is None
|
|
|
|
|
|
|
|
|
|
| 4555 |
else rt.odoo_source(source_connection_id))
|
| 4556 |
if source is None:
|
| 4557 |
raise Refused("the selected Odoo connection is unavailable, so no discovered model was read")
|
|
|
|
| 4664 |
return False
|
| 4665 |
|
| 4666 |
|
| 4667 |
+
#: β S67-T19 β the `limits` effects that REPORT a serving choice rather than withheld rows.
|
| 4668 |
+
#: `read_through` says a database is served THROUGH the mirror with no ceiling at all
|
| 4669 |
+
#: (`core.user_tables.limit_report`), which is the design working, not a failure.
|
| 4670 |
+
INFORMATIONAL_LIMIT_EFFECTS = frozenset({"read_through"})
|
| 4671 |
+
|
| 4672 |
+
|
| 4673 |
+
def failing_limits(limits):
|
| 4674 |
+
"""Only the `limits` entries that describe rows a pass did NOT serve, PURE.
|
| 4675 |
+
|
| 4676 |
+
ββ STAGING'S `materialization_failed` WAS THIS, NAMED 2026-09-28 FROM ITS OWN RECEIPTS. The
|
| 4677 |
+
rebuild handed its whole `limits` map to the health classifier, which reads EVERY entry as a
|
| 4678 |
+
materialization failure, so the two read-through grains (`ut_odoo_order_lines`,
|
| 4679 |
+
`ut_odoo_gl_lines`, "no ceiling applies to them") failed the connection on every single sync
|
| 4680 |
+
and `odoo_sync_status` could never read `ok`. The report keeps them; only health drops them.
|
| 4681 |
+
"""
|
| 4682 |
+
if not isinstance(limits, dict):
|
| 4683 |
+
return limits
|
| 4684 |
+
return {key: entry for key, entry in limits.items()
|
| 4685 |
+
if not (isinstance(entry, dict)
|
| 4686 |
+
and entry.get("effect") in INFORMATIONAL_LIMIT_EFFECTS)}
|
| 4687 |
+
|
| 4688 |
+
|
| 4689 |
+
#: β S67-T19 β the reconciliation DRIFT a good sync may carry: Odoo keeps taking invoices while a
|
| 4690 |
+
#: rebuild runs, and a hard delete leaves a mirror row until `reconcile_deletes` (every fourth
|
| 4691 |
+
#: pass). Measured on a mirror pulled 25 minutes before its rebuild: invoices 32,980 vs 32,978,
|
| 4692 |
+
#: orders 34,059 vs 34,058, agents 17 vs 18, GL lines 1,024,102 vs 1,024,178, customers 3,715 vs
|
| 4693 |
+
#: 3,716. A stale mirror is nothing like that: staging's unpulled seed was 626 orders and 732
|
| 4694 |
+
#: invoices short. The receipt keeps its exact counts either way; this only decides whether the
|
| 4695 |
+
#: CONNECTION's sync counts as good.
|
| 4696 |
+
RECONCILE_DRIFT_ROWS = 10
|
| 4697 |
+
RECONCILE_DRIFT_RATIO = 0.002
|
| 4698 |
+
|
| 4699 |
+
|
| 4700 |
+
def reconciliation_drift(receipt):
|
| 4701 |
+
"""`(difference, tolerance)` for a count-reconciliation receipt, or None when it has no counts."""
|
| 4702 |
+
if not isinstance(receipt, dict):
|
| 4703 |
+
return None
|
| 4704 |
+
source = receipt.get("sourceCount")
|
| 4705 |
+
if isinstance(receipt.get("unexplainedCount"), int):
|
| 4706 |
+
diff = int(receipt["unexplainedCount"])
|
| 4707 |
+
else:
|
| 4708 |
+
mirror = receipt.get("mirrorCount")
|
| 4709 |
+
if not isinstance(source, int) or not isinstance(mirror, int):
|
| 4710 |
+
return None
|
| 4711 |
+
diff = abs(source - mirror)
|
| 4712 |
+
base = source if isinstance(source, int) and source > 0 else 0
|
| 4713 |
+
return diff, max(RECONCILE_DRIFT_ROWS, int(base * RECONCILE_DRIFT_RATIO))
|
| 4714 |
+
|
| 4715 |
+
|
| 4716 |
+
def blocking_failures(database_failures, report):
|
| 4717 |
+
"""The database failures that make a sync NOT good, PURE.
|
| 4718 |
+
|
| 4719 |
+
A reconciliation failure whose count drift is within `reconciliation_drift`'s tolerance is
|
| 4720 |
+
recorded against its database and does not fail the connection; every other failure does.
|
| 4721 |
+
"""
|
| 4722 |
+
receipts = {}
|
| 4723 |
+
for name in ("databases", "canonicalDatabases"):
|
| 4724 |
+
group = report.get(name) if isinstance(report, dict) else None
|
| 4725 |
+
if isinstance(group, dict):
|
| 4726 |
+
receipts.update(group)
|
| 4727 |
+
out = {}
|
| 4728 |
+
for key, failure in (database_failures or {}).items():
|
| 4729 |
+
if isinstance(failure, dict) and failure.get("code") == "reconciliation_incomplete":
|
| 4730 |
+
drift = reconciliation_drift(receipts.get(key))
|
| 4731 |
+
if drift is not None and drift[0] <= drift[1]:
|
| 4732 |
+
continue
|
| 4733 |
+
out[key] = failure
|
| 4734 |
+
return out
|
| 4735 |
+
|
| 4736 |
+
|
| 4737 |
+
def health_view(report):
|
| 4738 |
+
"""The refresh report as the sync-health classifier must read it, PURE.
|
| 4739 |
+
|
| 4740 |
+
Identical to `report` except that `limits` carries only `failing_limits`; the returned report
|
| 4741 |
+
itself (the log line, the Sync report) still names every read-through database.
|
| 4742 |
+
"""
|
| 4743 |
+
if not isinstance(report, dict) or not isinstance(report.get("limits"), dict):
|
| 4744 |
+
return report
|
| 4745 |
+
view = dict(report)
|
| 4746 |
+
remaining = failing_limits(report["limits"])
|
| 4747 |
+
if remaining:
|
| 4748 |
+
view["limits"] = remaining
|
| 4749 |
+
else:
|
| 4750 |
+
view.pop("limits", None)
|
| 4751 |
+
return view
|
| 4752 |
+
|
| 4753 |
+
|
| 4754 |
def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
|
| 4755 |
today=None, retire=True, on_apply=None, source_client=None):
|
| 4756 |
"""Refresh each enabled database independently; retain failed databases unchanged.
|
|
|
|
| 5037 |
return out
|
| 5038 |
|
| 5039 |
|
| 5040 |
+
#: β S67-T19 β the store errors a connection refresh retries ONCE. Staging's rebuild at
|
| 5041 |
+
#: 2026-09-28 16:03Z failed whole with `ProtocolViolation` (SQLSTATE 08P01), which is how the
|
| 5042 |
+
#: DigitalOcean pool (PgBouncer) reports its own timeouts while every lane container shares it;
|
| 5043 |
+
#: nothing was written and the next scheduled pass was a day away. `refresh` is idempotent, so a
|
| 5044 |
+
#: second attempt after a pause is safe; a refusal or any other exception is never retried.
|
| 5045 |
+
TRANSIENT_STORE_ERRORS = frozenset({"ProtocolViolation", "OperationalError", "PoolTimeout",
|
| 5046 |
+
"ConnectionTimeout", "AdminShutdown"})
|
| 5047 |
+
REFRESH_RETRY_SECONDS = 20
|
| 5048 |
+
|
| 5049 |
+
|
| 5050 |
def refresh_all(rt, tenant, username="automation", cur=None, today=None):
|
| 5051 |
"""Refresh EVERY Odoo connection this tenant holds, one at a time (R4 / R7 / C4).
|
| 5052 |
|
|
|
|
| 5083 |
primary = None
|
| 5084 |
for entry_cid in order:
|
| 5085 |
named = None if (entry_cid is None or entry_cid == default) else entry_cid
|
| 5086 |
+
for attempt in (1, 2):
|
| 5087 |
+
try:
|
| 5088 |
+
one = refresh(rt, tenant, username=username,
|
| 5089 |
+
cur=(cur if named is None else None),
|
| 5090 |
+
today=today, connection_id=named)
|
| 5091 |
+
break
|
| 5092 |
+
except Exception as exc: # noqa: BLE001
|
| 5093 |
+
# S67-T19: one retry for a TRANSIENT store error only; every other failure is
|
| 5094 |
+
# recorded exactly as before.
|
| 5095 |
+
if attempt == 1 and type(exc).__name__ in TRANSIENT_STORE_ERRORS:
|
| 5096 |
+
_time.sleep(REFRESH_RETRY_SECONDS)
|
| 5097 |
+
continue
|
| 5098 |
+
one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"}
|
| 5099 |
+
break
|
| 5100 |
reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
|
| 5101 |
if named is None and primary is None:
|
| 5102 |
primary = one
|
api/routes_admin.py
CHANGED
|
@@ -605,7 +605,8 @@ def _perm_modules(session, surfaces=False):
|
|
| 605 |
skeys = set()
|
| 606 |
if surfaces:
|
| 607 |
# ββ OWNER RULING 2026-08-18 β the Assistant and Agents surfaces are
|
| 608 |
-
# governable ACCESS toggles in the ACCOUNT editor
|
|
|
|
| 609 |
# defaults CLOSED because the other consumer of this list is the CHANNEL-AGENT editor
|
| 610 |
# (`routes_slack.get_channel_agent`), whose principal is enforced per DATABASE
|
| 611 |
# (`agent_rows`/`agent_may_read`) and never opens an app surface: an Assistant toggle
|
|
@@ -869,11 +870,11 @@ def _module_fields(key, session=None):
|
|
| 869 |
import aios_grid
|
| 870 |
key = str(key or "")
|
| 871 |
if key in {s["key"] for s in perms.governable_surface_modules()}:
|
| 872 |
-
# β A SURFACE (Assistant, Agents) has NO field schema, and `[]` is the honest
|
| 873 |
# answer rather than the wave-26 empty-option trap: the editor's schemaless rendering
|
| 874 |
# is a deliberate state (access toggle only, R9), and `_clean_perms` refuses any
|
| 875 |
# filter or hiddenFields validated against this empty vocabulary. An unknown key
|
| 876 |
-
# still RAISES below
|
| 877 |
return []
|
| 878 |
if key.startswith("ut_"):
|
| 879 |
if session is None:
|
|
@@ -2337,7 +2338,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2337 |
# save of ANY unrelated change would write the explicit deny for real. `nav_may_open`
|
| 2338 |
# is the evaluator the nav and the route gate actually ask, legacy fallback included,
|
| 2339 |
# so the box shows what the account can reach and an untouched save stays a no-op.
|
| 2340 |
-
|
|
|
|
| 2341 |
if any(m.get("surface") and m["key"] == k for m in modules) else
|
| 2342 |
perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
|
| 2343 |
# ββ W38-T19 β `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN
|
|
|
|
| 605 |
skeys = set()
|
| 606 |
if surfaces:
|
| 607 |
# ββ OWNER RULING 2026-08-18 β the Assistant and Agents surfaces are
|
| 608 |
+
# governable ACCESS toggles in the ACCOUNT editor, alongside R3/C6's AI apps (MCP)
|
| 609 |
+
# gateway capability. `surfaces` is caller-explicit and
|
| 610 |
# defaults CLOSED because the other consumer of this list is the CHANNEL-AGENT editor
|
| 611 |
# (`routes_slack.get_channel_agent`), whose principal is enforced per DATABASE
|
| 612 |
# (`agent_rows`/`agent_may_read`) and never opens an app surface: an Assistant toggle
|
|
|
|
| 870 |
import aios_grid
|
| 871 |
key = str(key or "")
|
| 872 |
if key in {s["key"] for s in perms.governable_surface_modules()}:
|
| 873 |
+
# β A SURFACE (Assistant, Agents, AI apps) has NO field schema, and `[]` is the honest
|
| 874 |
# answer rather than the wave-26 empty-option trap: the editor's schemaless rendering
|
| 875 |
# is a deliberate state (access toggle only, R9), and `_clean_perms` refuses any
|
| 876 |
# filter or hiddenFields validated against this empty vocabulary. An unknown key
|
| 877 |
+
# still RAISES below: this arm admits the page surfaces and R3/C6's gateway capability.
|
| 878 |
return []
|
| 879 |
if key.startswith("ut_"):
|
| 880 |
if session is None:
|
|
|
|
| 2338 |
# save of ANY unrelated change would write the explicit deny for real. `nav_may_open`
|
| 2339 |
# is the evaluator the nav and the route gate actually ask, legacy fallback included,
|
| 2340 |
# so the box shows what the account can reach and an untouched save stays a no-op.
|
| 2341 |
+
# AI apps also uses this evaluator: absence grants even for a restricted legacy list.
|
| 2342 |
+
_default = (perm_scope.nav_may_open(users._public(uname, rec), k, st=session.runtime)
|
| 2343 |
if any(m.get("surface") and m["key"] == k for m in modules) else
|
| 2344 |
perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
|
| 2345 |
# ββ W38-T19 β `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN
|
api/routes_auth.py
CHANGED
|
@@ -69,6 +69,17 @@ def _throttled(key, now=None):
|
|
| 69 |
raise _auth_store_down("_throttled", exc)
|
| 70 |
|
| 71 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
def _note_failure(key, now=None):
|
| 73 |
if key is None:
|
| 74 |
return 0
|
|
@@ -196,7 +207,7 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 196 |
raise err(429, "too_many_attempts",
|
| 197 |
f"too many failed attempts. Try again in {wait} seconds")
|
| 198 |
if not user:
|
| 199 |
-
user =
|
| 200 |
if not user:
|
| 201 |
# CONSTANT-TIME-ISH 401. `users.verify` returns immediately for an unknown username (no
|
| 202 |
# record, no PBKDF2) and burns ~200k iterations for a known one, so the response time
|
|
@@ -356,7 +367,7 @@ def _mfa_credentials(uname, password):
|
|
| 356 |
print(f"[aios-api] {exc}")
|
| 357 |
raise err(429, "too_many_attempts",
|
| 358 |
f"too many failed attempts. Try again in {wait} seconds")
|
| 359 |
-
user =
|
| 360 |
if not user:
|
| 361 |
try:
|
| 362 |
users._hash(password or "x", "00" * 16)
|
|
@@ -620,7 +631,7 @@ def change_password(body: dict = Body(default=None)):
|
|
| 620 |
print(f"[aios-api] {exc}")
|
| 621 |
raise err(429, "too_many_attempts",
|
| 622 |
f"too many failed attempts. Try again in {wait} seconds")
|
| 623 |
-
user =
|
| 624 |
if not user:
|
| 625 |
try:
|
| 626 |
users._hash(pw or "x", "00" * 16)
|
|
|
|
| 69 |
raise _auth_store_down("_throttled", exc)
|
| 70 |
|
| 71 |
|
| 72 |
+
def _verify(uname, pw):
|
| 73 |
+
"""`users.verify`, with an UNREADABLE registry answered 503 before `_note_failure` runs.
|
| 74 |
+
|
| 75 |
+
HOTFIX 2026-09-28: a store `PoolTimeout` used to read as "no such account", so a correct
|
| 76 |
+
password got 401 `invalid_credentials` and was charged to the lockout budget."""
|
| 77 |
+
try:
|
| 78 |
+
return users.verify(uname, pw) if (uname and pw) else None
|
| 79 |
+
except users.RegistryUnavailable as exc:
|
| 80 |
+
raise _auth_store_down("verify", exc)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
def _note_failure(key, now=None):
|
| 84 |
if key is None:
|
| 85 |
return 0
|
|
|
|
| 207 |
raise err(429, "too_many_attempts",
|
| 208 |
f"too many failed attempts. Try again in {wait} seconds")
|
| 209 |
if not user:
|
| 210 |
+
user = _verify(uname, pw)
|
| 211 |
if not user:
|
| 212 |
# CONSTANT-TIME-ISH 401. `users.verify` returns immediately for an unknown username (no
|
| 213 |
# record, no PBKDF2) and burns ~200k iterations for a known one, so the response time
|
|
|
|
| 367 |
print(f"[aios-api] {exc}")
|
| 368 |
raise err(429, "too_many_attempts",
|
| 369 |
f"too many failed attempts. Try again in {wait} seconds")
|
| 370 |
+
user = _verify(uname, password)
|
| 371 |
if not user:
|
| 372 |
try:
|
| 373 |
users._hash(password or "x", "00" * 16)
|
|
|
|
| 631 |
print(f"[aios-api] {exc}")
|
| 632 |
raise err(429, "too_many_attempts",
|
| 633 |
f"too many failed attempts. Try again in {wait} seconds")
|
| 634 |
+
user = _verify(uname, pw)
|
| 635 |
if not user:
|
| 636 |
try:
|
| 637 |
users._hash(pw or "x", "00" * 16)
|
api/routes_connected_tables.py
CHANGED
|
@@ -30,7 +30,7 @@ behind green gates in wave 23, and wave 28 proved a MOUNTED router can still be
|
|
| 30 |
(`session.is_admin`, an attribute that does not exist, raising above the route's own `try`). So
|
| 31 |
`verify_meta` both enumerates `main.app.routes` for this path AND calls it.
|
| 32 |
"""
|
| 33 |
-
from fastapi import APIRouter, Depends, Query
|
| 34 |
|
| 35 |
from deps import Session, err, require_session
|
| 36 |
|
|
@@ -45,6 +45,8 @@ def connected_table_identity(table_key: str, session: Session = Depends(require_
|
|
| 45 |
|
| 46 |
# Definitions only: this endpoint neither reads rows nor bypasses the table permission wall.
|
| 47 |
definition = _defn_or_refuse(session, table_key, defs_only=True)
|
|
|
|
|
|
|
| 48 |
identity = user_tables.connector_database_identity(
|
| 49 |
table_key, rt=session.runtime, definition=definition)
|
| 50 |
if identity is None:
|
|
@@ -54,6 +56,7 @@ def connected_table_identity(table_key: str, session: Session = Depends(require_
|
|
| 54 |
|
| 55 |
|
| 56 |
@router.get("/connected-tables/{table_key}/rows")
|
|
|
|
| 57 |
def connected_table_rows(table_key: str,
|
| 58 |
offset: int = Query(default=0, ge=0),
|
| 59 |
limit: int = Query(default=0),
|
|
@@ -64,7 +67,8 @@ def connected_table_rows(table_key: str,
|
|
| 64 |
related: str = Query(default=None),
|
| 65 |
groupBy: str = Query(default=None),
|
| 66 |
groupKey: str = Query(default=None),
|
| 67 |
-
session: Session = Depends(require_session)
|
|
|
|
| 68 |
"""ONE WINDOW over a connected grid β `{fields, rows, total, totalUnfiltered, offset, limit,
|
| 69 |
limits, identity, recordsMutable}`, whichever connector the rows come from.
|
| 70 |
|
|
@@ -92,4 +96,4 @@ def connected_table_rows(table_key: str,
|
|
| 92 |
return routes_odoo_tables.odoo_table_rows(
|
| 93 |
table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
|
| 94 |
sorts=sorts, search=search, related=related, groupBy=groupBy, groupKey=groupKey,
|
| 95 |
-
session=session)
|
|
|
|
| 30 |
(`session.is_admin`, an attribute that does not exist, raising above the route's own `try`). So
|
| 31 |
`verify_meta` both enumerates `main.app.routes` for this path AND calls it.
|
| 32 |
"""
|
| 33 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 34 |
|
| 35 |
from deps import Session, err, require_session
|
| 36 |
|
|
|
|
| 45 |
|
| 46 |
# Definitions only: this endpoint neither reads rows nor bypasses the table permission wall.
|
| 47 |
definition = _defn_or_refuse(session, table_key, defs_only=True)
|
| 48 |
+
from routes_odoo_tables import require_amazon_mirror
|
| 49 |
+
require_amazon_mirror(table_key, session, definition)
|
| 50 |
identity = user_tables.connector_database_identity(
|
| 51 |
table_key, rt=session.runtime, definition=definition)
|
| 52 |
if identity is None:
|
|
|
|
| 56 |
|
| 57 |
|
| 58 |
@router.get("/connected-tables/{table_key}/rows")
|
| 59 |
+
@router.post("/connected-tables/{table_key}/rows")
|
| 60 |
def connected_table_rows(table_key: str,
|
| 61 |
offset: int = Query(default=0, ge=0),
|
| 62 |
limit: int = Query(default=0),
|
|
|
|
| 67 |
related: str = Query(default=None),
|
| 68 |
groupBy: str = Query(default=None),
|
| 69 |
groupKey: str = Query(default=None),
|
| 70 |
+
session: Session = Depends(require_session),
|
| 71 |
+
body: dict = Body(default=None)):
|
| 72 |
"""ONE WINDOW over a connected grid β `{fields, rows, total, totalUnfiltered, offset, limit,
|
| 73 |
limits, identity, recordsMutable}`, whichever connector the rows come from.
|
| 74 |
|
|
|
|
| 96 |
return routes_odoo_tables.odoo_table_rows(
|
| 97 |
table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
|
| 98 |
sorts=sorts, search=search, related=related, groupBy=groupBy, groupKey=groupKey,
|
| 99 |
+
session=session, body=body)
|
api/routes_connectors.py
CHANGED
|
@@ -21,14 +21,17 @@ that reported another user's Gmail as "connected" would be both wrong and a smal
|
|
| 21 |
from __future__ import annotations
|
| 22 |
|
| 23 |
import os
|
|
|
|
| 24 |
import ssl
|
| 25 |
import time
|
| 26 |
import threading
|
| 27 |
import xmlrpc.client
|
| 28 |
from concurrent.futures import ThreadPoolExecutor
|
| 29 |
from datetime import datetime, timezone
|
|
|
|
|
|
|
| 30 |
|
| 31 |
-
from fastapi import APIRouter, Body, Depends
|
| 32 |
|
| 33 |
import oauth_connect
|
| 34 |
from deps import Session, err, require_session, running_main
|
|
@@ -83,6 +86,75 @@ from routes_monid import router as monid_router
|
|
| 83 |
router = APIRouter(prefix="/api/v1")
|
| 84 |
|
| 85 |
CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps"))
|
| 87 |
SCRAPER_RECIPE_MAX_ROWS = 30
|
| 88 |
SCRAPER_RUN_HISTORY_LIMIT = 20
|
|
@@ -254,9 +326,11 @@ TOKEN_CONNECTORS = [
|
|
| 254 |
"hint": "Create a Production app client in the Solution Provider Portal, self authorize "
|
| 255 |
"it, then paste client_id, client_secret, refresh_token, seller_id and "
|
| 256 |
"marketplace_id.",
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
|
|
|
|
|
|
| 260 |
]
|
| 261 |
|
| 262 |
#: ββ WAVE 31 Β· R7 β THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
|
|
@@ -374,6 +448,11 @@ _register_probers()
|
|
| 374 |
# W48C-T08 / C6 β the graph is a reader-scoped DTO over tables that exist in THIS workspace,
|
| 375 |
# never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family;
|
| 376 |
# `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 377 |
def _graph_refusal(connection_key, reason, status="refused"):
|
| 378 |
return {"connectionKey": str(connection_key or ""), "status": status,
|
| 379 |
"reason": str(reason or ""), "tables": [], "relationships": []}
|
|
@@ -526,7 +605,12 @@ def relationship_graphs(rows, user, rt, uname, is_admin=False):
|
|
| 526 |
graphs = {}
|
| 527 |
connected = set()
|
| 528 |
for connection_key, row in row_by_key.items():
|
| 529 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 530 |
connected.add(connection_key)
|
| 531 |
graphs[connection_key] = _graph_refusal(
|
| 532 |
connection_key,
|
|
@@ -906,9 +990,21 @@ def directory(rt, uname, is_admin=False, incomplete=None):
|
|
| 906 |
else "keychain_personal"),
|
| 907 |
# Said once, on the row, rather than in a paragraph somewhere (R13): storing the
|
| 908 |
# key is real and useful, and it is not the same as the data being on screen.
|
| 909 |
-
|
| 910 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 911 |
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 912 |
|
| 913 |
contribute("token", _tokens,
|
| 914 |
"the token-connector registry could not be composed",
|
|
@@ -2150,19 +2246,21 @@ def _schema_catalog(session):
|
|
| 2150 |
return _schema_catalog_rt(session.runtime)
|
| 2151 |
|
| 2152 |
|
| 2153 |
-
def _odoo_raw_models(rt, snapshot=None, connection_id=None):
|
| 2154 |
"""Raw fields from stamped discovery, or the legacy snapshot before discovery exists."""
|
| 2155 |
-
if
|
| 2156 |
-
|
| 2157 |
-
|
| 2158 |
-
|
| 2159 |
-
|
| 2160 |
-
|
| 2161 |
-
discovered = odoo_relational.readable_model_catalog(rt, connection_id)
|
| 2162 |
-
except Exception: # noqa: BLE001
|
| 2163 |
-
discovered = {}
|
| 2164 |
if not discovered.get("capturedAt"):
|
| 2165 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2166 |
# A stamp means this tenant-bound account has spoken. Even a partial stamped set is safer
|
| 2167 |
# than adding snapshot-only models that the current account never proved it may read.
|
| 2168 |
return {str(item.get("model") or ""): dict(item.get("fields") or {})
|
|
@@ -2170,7 +2268,7 @@ def _odoo_raw_models(rt, snapshot=None, connection_id=None):
|
|
| 2170 |
if str(item.get("model") or "")}
|
| 2171 |
|
| 2172 |
|
| 2173 |
-
def _schema_catalog_rt(rt, database_key=None, connection_id=None):
|
| 2174 |
"""The same catalogue, over a RUNTIME rather than a session.
|
| 2175 |
|
| 2176 |
β SPLIT IN W49-T12 SO THE DETAIL ROUTE CAN REUSE IT RATHER THAN RE-DERIVE IT. The connector
|
|
@@ -2179,10 +2277,9 @@ def _schema_catalog_rt(rt, database_key=None, connection_id=None):
|
|
| 2179 |
already paid for twice ([[one-question-two-normalizers]]). `_schema_catalog` keeps its
|
| 2180 |
signature so every existing caller is untouched.
|
| 2181 |
"""
|
| 2182 |
-
import core.user_tables as user_tables
|
| 2183 |
enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}
|
| 2184 |
-
|
| 2185 |
-
|
| 2186 |
if raw_models:
|
| 2187 |
rows = []
|
| 2188 |
for model, raw_fields in raw_models.items():
|
|
@@ -3089,34 +3186,72 @@ def _detail_keychain(rt, uname, is_admin, row):
|
|
| 3089 |
"connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None}
|
| 3090 |
|
| 3091 |
|
| 3092 |
-
def _odoo_catalog_by_model(rt, connection_id=None):
|
| 3093 |
"""`{odoo model: (fields, excluded)}` from the schema catalogue this module already serves.
|
| 3094 |
|
| 3095 |
`excluded` is a DIFF, not a guess: every raw field the snapshot captured that the catalogue did
|
| 3096 |
not offer. Today that diff is empty, which is the honest answer, and it fills itself the moment
|
| 3097 |
the catalogue starts filtering instead of needing somebody to remember.
|
| 3098 |
"""
|
| 3099 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3100 |
offered = {}
|
| 3101 |
-
for row in _schema_catalog_rt(rt, connection_id=connection_id)
|
|
|
|
| 3102 |
offered.setdefault(str(row.get("model") or ""), []).append(row)
|
| 3103 |
-
|
| 3104 |
-
for model, raw_fields in
|
| 3105 |
rows = offered.get(str(model), [])
|
| 3106 |
offered_raw = {str(row.get("rawName") or "") for row in rows}
|
| 3107 |
-
|
| 3108 |
[{"key": str(row.get("key") or ""),
|
| 3109 |
"label": str(row.get("label") or ""),
|
| 3110 |
"type": str(row.get("type") or "text"),
|
| 3111 |
"source_field": str(row.get("rawName") or ""),
|
| 3112 |
-
"enabled":
|
| 3113 |
for row in rows],
|
| 3114 |
[{"source_field": str(name),
|
| 3115 |
"reason": "This source field was captured from Odoo but the field catalogue did not "
|
| 3116 |
"offer it."}
|
| 3117 |
for name in sorted(raw_fields or {}) if str(name) not in offered_raw],
|
| 3118 |
)
|
| 3119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3120 |
|
| 3121 |
|
| 3122 |
def _spec_field(item, name):
|
|
@@ -3495,7 +3630,8 @@ def _scraper_databases(rt, is_admin, uname):
|
|
| 3495 |
return _named_databases(rt, is_admin, uname, rows)
|
| 3496 |
|
| 3497 |
|
| 3498 |
-
def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None
|
|
|
|
| 3499 |
"""C4's `databases` half: what this connector materialises, and the fields behind each one.
|
| 3500 |
|
| 3501 |
β ODOO IS THE ONLY INTROSPECTABLE CONNECTOR TODAY and every other card answers `[]`. That is
|
|
@@ -3523,19 +3659,34 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
|
|
| 3523 |
reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
|
| 3524 |
is walled, per row, by `canRename`.
|
| 3525 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3526 |
if str(connector_key) == "scraper":
|
| 3527 |
-
return _scraper_databases(rt, is_admin, uname)
|
| 3528 |
if not connector_directory.is_introspectable(connector_key):
|
| 3529 |
return []
|
| 3530 |
spec = _connector_spec_tables(connector_key)
|
| 3531 |
if str(connector_key) == "amazon_sp":
|
| 3532 |
-
return
|
|
|
|
| 3533 |
try:
|
| 3534 |
import odoo_relational
|
| 3535 |
namespace = (odoo_relational.connection_namespace(rt, connection_id)[1]
|
| 3536 |
if connection_id else None)
|
| 3537 |
choices = odoo_relational.grid_choices(rt, connection_id)
|
| 3538 |
models = dict(getattr(odoo_relational, "ODOO_TABLE_MODELS", {}) or {})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3539 |
# This read never contacts Odoo. A 499-model exact pass measured 67.87 s after the
|
| 3540 |
# catalogue was warm, so modal detail must never turn a missing cache into inline work.
|
| 3541 |
census = odoo_relational.population_census(rt, namespace)
|
|
@@ -3547,9 +3698,11 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
|
|
| 3547 |
census_at = str(census.get("capturedAt") or "")
|
| 3548 |
counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
|
| 3549 |
catalog = {}
|
| 3550 |
-
if is_admin:
|
| 3551 |
try:
|
| 3552 |
-
catalog = _odoo_catalog_by_model(rt, namespace
|
|
|
|
|
|
|
| 3553 |
except Exception: # noqa: BLE001
|
| 3554 |
catalog = {}
|
| 3555 |
enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {}
|
|
@@ -3576,9 +3729,9 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
|
|
| 3576 |
key = str(choice.get("key") or "")
|
| 3577 |
resolved_connection_id = resolve_connection_id(key)
|
| 3578 |
model = str(choice.get("model") or models.get(key) or "")
|
| 3579 |
-
|
| 3580 |
scoped_fields = []
|
| 3581 |
-
for field in
|
| 3582 |
field_key = _odoo_field_key(rt, key, model, field["source_field"],
|
| 3583 |
connection_id=resolved_connection_id)
|
| 3584 |
scoped_fields.append({**field, "key": field_key,
|
|
@@ -3612,7 +3765,7 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
|
|
| 3612 |
"fields": scoped_fields,
|
| 3613 |
"excluded": [dict(item) for item in excluded],
|
| 3614 |
"sharesFieldsWith": []})
|
| 3615 |
-
return _named_databases(rt, is_admin, uname, out)
|
| 3616 |
|
| 3617 |
|
| 3618 |
def _amazon_report_name(table):
|
|
@@ -3662,7 +3815,17 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 3662 |
enabled = enablement.get("databases") or {}
|
| 3663 |
cat = amazon_schema.catalogue()
|
| 3664 |
tables = cat.get("tables") or {}
|
| 3665 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3666 |
|
| 3667 |
# Which OTHER declared tables are filled from the SAME report document. Derived from the
|
| 3668 |
# report map so a table added to `REPORT_TYPES` gets this disclosure for free.
|
|
@@ -3833,12 +3996,14 @@ def _detail_introspection(rt, is_admin, row):
|
|
| 3833 |
captured = ""
|
| 3834 |
census = {}
|
| 3835 |
try:
|
| 3836 |
-
import core.user_tables as user_tables
|
| 3837 |
-
snapshot = user_tables.odoo_schema_snapshot(rt)
|
| 3838 |
import odoo_relational
|
| 3839 |
discovered = odoo_relational.readable_model_catalog(rt)
|
| 3840 |
census = odoo_relational.population_census(rt)
|
| 3841 |
has_discovery = bool(discovered.get("capturedAt"))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3842 |
complete = (discovered.get("complete") is True if has_discovery
|
| 3843 |
else snapshot.get("complete") is True)
|
| 3844 |
captured = str((discovered.get("capturedAt") if has_discovery
|
|
@@ -4130,13 +4295,16 @@ def _scraper_database_run(rt, uname, is_admin, database_key):
|
|
| 4130 |
return {"key": key, "state": state, "counts": counts, "receipt": receipt}
|
| 4131 |
|
| 4132 |
|
| 4133 |
-
def connector_detail(rt, user, uname, is_admin, connector_key
|
|
|
|
| 4134 |
"""The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
|
| 4135 |
|
| 4136 |
Pure over the session for the same reason `directory` is: a gate can drive it without a
|
| 4137 |
request, so the modal's payload is provable without a browser.
|
| 4138 |
"""
|
| 4139 |
budget = _ConnectorDetailBudget()
|
|
|
|
|
|
|
| 4140 |
rows = directory(rt, uname, bool(is_admin))
|
| 4141 |
budget.check()
|
| 4142 |
key = str(connector_key or "")
|
|
@@ -4144,15 +4312,23 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
|
|
| 4144 |
if str(candidate.get("key") or "") == key), None)
|
| 4145 |
if row is None:
|
| 4146 |
return None
|
| 4147 |
-
|
| 4148 |
-
|
| 4149 |
-
|
|
|
|
|
|
|
|
|
|
| 4150 |
budget.check()
|
| 4151 |
# β `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a
|
| 4152 |
# question about a PERSON and a table, not about an administrator flag alone.
|
| 4153 |
-
databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or "")
|
|
|
|
|
|
|
|
|
|
| 4154 |
budget.check()
|
| 4155 |
overview = _detail_overview(row, databases, graph)
|
|
|
|
|
|
|
| 4156 |
budget.check()
|
| 4157 |
keychain = _detail_keychain(rt, uname, bool(is_admin), row)
|
| 4158 |
budget.check()
|
|
@@ -4163,6 +4339,9 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
|
|
| 4163 |
return _safe_connector_status(key, {"overview": overview,
|
| 4164 |
"keychain": keychain,
|
| 4165 |
"databases": databases,
|
|
|
|
|
|
|
|
|
|
| 4166 |
"syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"}
|
| 4167 |
if is_admin and key in connector_sync.providers() else None),
|
| 4168 |
"materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
|
|
@@ -4182,6 +4361,36 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
|
|
| 4182 |
"relationships": graph})
|
| 4183 |
|
| 4184 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4185 |
def _set_amazon_database_enablement(database_key, body, session):
|
| 4186 |
"""The testable core of Amazon's tenant-scoped administrator enablement door."""
|
| 4187 |
if not session.admin:
|
|
@@ -4253,22 +4462,100 @@ def scraper_database_run(database_key: str, session: Session = Depends(require_s
|
|
| 4253 |
# the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
|
| 4254 |
# collide with this three-segment path at all.
|
| 4255 |
@router.get("/connectors/{connector_key}/detail")
|
| 4256 |
-
def connectors_detail(connector_key: str, session: Session = Depends(require_session)
|
|
|
|
|
|
|
|
|
|
| 4257 |
"""C4 / R26: everything the connector modal shows, for THIS reader, in one read.
|
| 4258 |
|
| 4259 |
Session-gated exactly like the directory it is opened from: a member may look at what the
|
| 4260 |
workspace is connected to. What a member does NOT get is the tenant-wide credential door
|
| 4261 |
(`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue.
|
| 4262 |
"""
|
| 4263 |
-
|
| 4264 |
-
|
| 4265 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4266 |
if detail is None:
|
| 4267 |
raise err(404, "unknown_connector",
|
| 4268 |
"this workspace has no connector with that key")
|
| 4269 |
return detail
|
| 4270 |
|
| 4271 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4272 |
class _ReplacementConflict(Exception):
|
| 4273 |
"""The addressed keychain row changed between replacement preparation and its store update."""
|
| 4274 |
|
|
|
|
| 21 |
from __future__ import annotations
|
| 22 |
|
| 23 |
import os
|
| 24 |
+
import copy
|
| 25 |
import ssl
|
| 26 |
import time
|
| 27 |
import threading
|
| 28 |
import xmlrpc.client
|
| 29 |
from concurrent.futures import ThreadPoolExecutor
|
| 30 |
from datetime import datetime, timezone
|
| 31 |
+
from collections import OrderedDict
|
| 32 |
+
from typing import Annotated, Literal
|
| 33 |
|
| 34 |
+
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
| 35 |
|
| 36 |
import oauth_connect
|
| 37 |
from deps import Session, err, require_session, running_main
|
|
|
|
| 86 |
router = APIRouter(prefix="/api/v1")
|
| 87 |
|
| 88 |
CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
|
| 89 |
+
_ODOO_DETAIL_CATALOG_CACHE = OrderedDict()
|
| 90 |
+
_ODOO_DETAIL_CATALOG_LOCK = threading.Lock()
|
| 91 |
+
_ODOO_DETAIL_CATALOG_CACHE_SIZE = 16
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class _ConnectorDetailReads:
|
| 95 |
+
"""Request-local copies only; credentials, grants and enablements never outlive a read.
|
| 96 |
+
|
| 97 |
+
Keep projections separate from whole documents and honour fresh reads. HF projections still
|
| 98 |
+
download the whole document on a cold store; removing that requires a store-layer change.
|
| 99 |
+
"""
|
| 100 |
+
|
| 101 |
+
def __init__(self, runtime):
|
| 102 |
+
self.runtime = runtime
|
| 103 |
+
self.documents = {}
|
| 104 |
+
self.projections = {}
|
| 105 |
+
|
| 106 |
+
def get(self, name, fresh=False):
|
| 107 |
+
if fresh:
|
| 108 |
+
return self.runtime.get(name, fresh=True)
|
| 109 |
+
if name not in self.documents:
|
| 110 |
+
self.documents[name] = self.runtime.get(name)
|
| 111 |
+
return copy.deepcopy(self.documents[name])
|
| 112 |
+
|
| 113 |
+
def get_projection(self, name, drop=()):
|
| 114 |
+
key = (name, tuple(sorted(drop)))
|
| 115 |
+
if key not in self.projections:
|
| 116 |
+
self.projections[key] = self.runtime.get_projection(name, drop=drop)
|
| 117 |
+
return copy.deepcopy(self.projections[key])
|
| 118 |
+
|
| 119 |
+
def __getattr__(self, name):
|
| 120 |
+
return getattr(self.runtime, name)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
class _DatabasePage:
|
| 124 |
+
"""Select before hydrating Odoo fields; the total always describes the unsliced list."""
|
| 125 |
+
|
| 126 |
+
def __init__(self, offset=0, limit=50, database_key=None):
|
| 127 |
+
self.offset, self.limit, self.database_key = offset, limit, database_key
|
| 128 |
+
self.total = 0
|
| 129 |
+
self.creates = []
|
| 130 |
+
|
| 131 |
+
def select(self, rows, *, overview_rows=None):
|
| 132 |
+
self.total = len(rows)
|
| 133 |
+
self.creates = [str(row.get("label") or "")
|
| 134 |
+
for row in (rows if overview_rows is None else overview_rows)
|
| 135 |
+
if row.get("enabled") is True and row.get("label")]
|
| 136 |
+
if self.database_key is not None:
|
| 137 |
+
return [row for row in rows if row.get("key") == self.database_key]
|
| 138 |
+
return rows[self.offset:self.offset + self.limit]
|
| 139 |
+
|
| 140 |
+
def dto(self):
|
| 141 |
+
next_offset = self.offset + self.limit
|
| 142 |
+
return {"offset": self.offset, "limit": self.limit, "total": self.total,
|
| 143 |
+
"next": next_offset if next_offset < self.total else None}
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _detail_page(databases_offset, databases_limit, fields):
|
| 147 |
+
if fields not in ("summary", "full"):
|
| 148 |
+
raise err(422, "invalid_database_fields", "Fields must be summary or full.")
|
| 149 |
+
if databases_offset is None and databases_limit is None and fields == "full":
|
| 150 |
+
return None
|
| 151 |
+
offset = 0 if databases_offset is None else databases_offset
|
| 152 |
+
limit = 50 if databases_limit is None else databases_limit
|
| 153 |
+
if (type(offset) is not int or offset < 0 or type(limit) is not int or limit < 1):
|
| 154 |
+
raise err(422, "invalid_database_page", "Use a nonnegative offset and a positive limit.")
|
| 155 |
+
return _DatabasePage(offset, limit)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps"))
|
| 159 |
SCRAPER_RECIPE_MAX_ROWS = 30
|
| 160 |
SCRAPER_RUN_HISTORY_LIMIT = 20
|
|
|
|
| 326 |
"hint": "Create a Production app client in the Solution Provider Portal, self authorize "
|
| 327 |
"it, then paste client_id, client_secret, refresh_token, seller_id and "
|
| 328 |
"marketplace_id.",
|
| 329 |
+
# S66-T27: storing is offline (`keychain._test_amazon_entry` probes only on an explicit
|
| 330 |
+
# Test), so this no longer claims the key was verified before it was stored.
|
| 331 |
+
"note": "Choose Test in the Keychain tab to verify the credential live against your "
|
| 332 |
+
"marketplace participations. Bulk reads use the Reports API, so a backfill is one "
|
| 333 |
+
"document rather than a page loop."},
|
| 334 |
]
|
| 335 |
|
| 336 |
#: ββ WAVE 31 Β· R7 β THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
|
|
|
|
| 448 |
# W48C-T08 / C6 β the graph is a reader-scoped DTO over tables that exist in THIS workspace,
|
| 449 |
# never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family;
|
| 450 |
# `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter.
|
| 451 |
+
#: The directory states that mean "this workspace holds this connection": connected, stored but
|
| 452 |
+
#: asking to be reconnected, or stored past its rotation deadline (`ConnectorsPage`'s `held`).
|
| 453 |
+
_GRAPH_HELD_STATES = frozenset(("connected", "reconnect", "expired"))
|
| 454 |
+
|
| 455 |
+
|
| 456 |
def _graph_refusal(connection_key, reason, status="refused"):
|
| 457 |
return {"connectionKey": str(connection_key or ""), "status": status,
|
| 458 |
"reason": str(reason or ""), "tables": [], "relationships": []}
|
|
|
|
| 605 |
graphs = {}
|
| 606 |
connected = set()
|
| 607 |
for connection_key, row in row_by_key.items():
|
| 608 |
+
# β S67-T31 / D-718 β A HELD CONNECTION HAS A GRAPH. Staging's Amazon card reads
|
| 609 |
+
# `reconnect` (its stored key has no passing test on record) while 38 `ut_amzn_*` tables
|
| 610 |
+
# are materialised, and the old `== "connected"` test refused the graph as "Connect this
|
| 611 |
+
# provider" over a connection that is plainly there. The nodes are still only tables that
|
| 612 |
+
# exist and that this reader may open, so admitting the held states draws no template.
|
| 613 |
+
if row.get("state") in _GRAPH_HELD_STATES:
|
| 614 |
connected.add(connection_key)
|
| 615 |
graphs[connection_key] = _graph_refusal(
|
| 616 |
connection_key,
|
|
|
|
| 990 |
else "keychain_personal"),
|
| 991 |
# Said once, on the row, rather than in a paragraph somewhere (R13): storing the
|
| 992 |
# key is real and useful, and it is not the same as the data being on screen.
|
| 993 |
+
# β S66-T27 / S67-T31 β EXCEPT AMAZON, whose data DOES land (38 `ut_amzn_*`
|
| 994 |
+
# tables on staging), so the generic sentence told an administrator the opposite
|
| 995 |
+
# of the truth. Amazon carries its own authored note.
|
| 996 |
+
# S67 lane D: so does Meta Ads, whose authored note this line used to overwrite
|
| 997 |
+
# with the generic one; every card now shows its own (Stripe's and Shopify's own
|
| 998 |
+
# note IS the generic sentence, so theirs is unchanged).
|
| 999 |
+
"note": t["note"],
|
| 1000 |
})
|
| 1001 |
+
# S66-T27 β WHAT `reconnect` MEANS, said on the card. On Amazon it is "a key is stored
|
| 1002 |
+
# but no passing Test is on record for it" (`keychain.entry_verified`), measured on
|
| 1003 |
+
# staging's admin view with the data still syncing. The bare word read as a broken
|
| 1004 |
+
# connection with no way to tell what to do.
|
| 1005 |
+
if rows[-1]["state"] == "reconnect" and t["entry"] == "amazon_sp":
|
| 1006 |
+
rows[-1]["needs"] = ("Amazon access for the stored key is not confirmed yet. "
|
| 1007 |
+
"Choose Test in the Keychain tab, or replace the key.")
|
| 1008 |
|
| 1009 |
contribute("token", _tokens,
|
| 1010 |
"the token-connector registry could not be composed",
|
|
|
|
| 2246 |
return _schema_catalog_rt(session.runtime)
|
| 2247 |
|
| 2248 |
|
| 2249 |
+
def _odoo_raw_models(rt, snapshot=None, connection_id=None, *, discovered=None):
|
| 2250 |
"""Raw fields from stamped discovery, or the legacy snapshot before discovery exists."""
|
| 2251 |
+
if discovered is None:
|
| 2252 |
+
try:
|
| 2253 |
+
import odoo_relational
|
| 2254 |
+
discovered = odoo_relational.readable_model_catalog(rt, connection_id)
|
| 2255 |
+
except Exception: # noqa: BLE001
|
| 2256 |
+
discovered = {}
|
|
|
|
|
|
|
|
|
|
| 2257 |
if not discovered.get("capturedAt"):
|
| 2258 |
+
if connection_id is not None:
|
| 2259 |
+
return {}
|
| 2260 |
+
if snapshot is None:
|
| 2261 |
+
import core.user_tables as user_tables
|
| 2262 |
+
snapshot = user_tables.odoo_schema_snapshot(rt)
|
| 2263 |
+
return dict((snapshot or {}).get("models") or {})
|
| 2264 |
# A stamp means this tenant-bound account has spoken. Even a partial stamped set is safer
|
| 2265 |
# than adding snapshot-only models that the current account never proved it may read.
|
| 2266 |
return {str(item.get("model") or ""): dict(item.get("fields") or {})
|
|
|
|
| 2268 |
if str(item.get("model") or "")}
|
| 2269 |
|
| 2270 |
|
| 2271 |
+
def _schema_catalog_rt(rt, database_key=None, connection_id=None, *, raw_models=None):
|
| 2272 |
"""The same catalogue, over a RUNTIME rather than a session.
|
| 2273 |
|
| 2274 |
β SPLIT IN W49-T12 SO THE DETAIL ROUTE CAN REUSE IT RATHER THAN RE-DERIVE IT. The connector
|
|
|
|
| 2277 |
already paid for twice ([[one-question-two-normalizers]]). `_schema_catalog` keeps its
|
| 2278 |
signature so every existing caller is untouched.
|
| 2279 |
"""
|
|
|
|
| 2280 |
enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}
|
| 2281 |
+
if raw_models is None:
|
| 2282 |
+
raw_models = _odoo_raw_models(rt, connection_id=connection_id)
|
| 2283 |
if raw_models:
|
| 2284 |
rows = []
|
| 2285 |
for model, raw_fields in raw_models.items():
|
|
|
|
| 3186 |
"connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None}
|
| 3187 |
|
| 3188 |
|
| 3189 |
+
def _odoo_catalog_by_model(rt, connection_id=None, *, models=None):
|
| 3190 |
"""`{odoo model: (fields, excluded)}` from the schema catalogue this module already serves.
|
| 3191 |
|
| 3192 |
`excluded` is a DIFF, not a guess: every raw field the snapshot captured that the catalogue did
|
| 3193 |
not offer. Today that diff is empty, which is the honest answer, and it fills itself the moment
|
| 3194 |
the catalogue starts filtering instead of needing somebody to remember.
|
| 3195 |
"""
|
| 3196 |
+
import odoo_relational
|
| 3197 |
+
discovered = odoo_relational.readable_model_catalog(rt, connection_id)
|
| 3198 |
+
marker = ("discovered", discovered.get("version"), discovered.get("capturedAt"),
|
| 3199 |
+
discovered.get("complete"))
|
| 3200 |
+
snapshot = None
|
| 3201 |
+
if not discovered.get("capturedAt"):
|
| 3202 |
+
import core.user_tables as user_tables
|
| 3203 |
+
snapshot = user_tables.odoo_schema_snapshot(rt) if connection_id is None else {}
|
| 3204 |
+
marker = ("snapshot", snapshot.get("version"), snapshot.get("capturedAt"),
|
| 3205 |
+
snapshot.get("complete"))
|
| 3206 |
+
owner = rt.runtime if isinstance(rt, _ConnectorDetailReads) else rt
|
| 3207 |
+
cache_key = (id(owner), str(connection_id or ""), marker)
|
| 3208 |
+
wanted = None if models is None else set(models)
|
| 3209 |
+
# Cache only normalized metadata, never permissions or field switch values. Keep the owner
|
| 3210 |
+
# alive with the entry so Python cannot reuse a runtime id for another tenant's catalogue.
|
| 3211 |
+
with _ODOO_DETAIL_CATALOG_LOCK:
|
| 3212 |
+
entry = _ODOO_DETAIL_CATALOG_CACHE.get(cache_key) if marker[2] else None
|
| 3213 |
+
cached = dict(entry[1]) if entry else {}
|
| 3214 |
+
if entry:
|
| 3215 |
+
_ODOO_DETAIL_CATALOG_CACHE.move_to_end(cache_key)
|
| 3216 |
+
raw_models = _odoo_raw_models(rt, snapshot, connection_id, discovered=discovered)
|
| 3217 |
+
if wanted is not None:
|
| 3218 |
+
raw_models = {model: fields for model, fields in raw_models.items() if model in wanted}
|
| 3219 |
+
missing = {model: fields for model, fields in raw_models.items() if model not in cached}
|
| 3220 |
offered = {}
|
| 3221 |
+
for row in (_schema_catalog_rt(rt, connection_id=connection_id, raw_models=missing)
|
| 3222 |
+
if missing else ()):
|
| 3223 |
offered.setdefault(str(row.get("model") or ""), []).append(row)
|
| 3224 |
+
built = {}
|
| 3225 |
+
for model, raw_fields in missing.items():
|
| 3226 |
rows = offered.get(str(model), [])
|
| 3227 |
offered_raw = {str(row.get("rawName") or "") for row in rows}
|
| 3228 |
+
built[str(model)] = (
|
| 3229 |
[{"key": str(row.get("key") or ""),
|
| 3230 |
"label": str(row.get("label") or ""),
|
| 3231 |
"type": str(row.get("type") or "text"),
|
| 3232 |
"source_field": str(row.get("rawName") or ""),
|
| 3233 |
+
"enabled": False}
|
| 3234 |
for row in rows],
|
| 3235 |
[{"source_field": str(name),
|
| 3236 |
"reason": "This source field was captured from Odoo but the field catalogue did not "
|
| 3237 |
"offer it."}
|
| 3238 |
for name in sorted(raw_fields or {}) if str(name) not in offered_raw],
|
| 3239 |
)
|
| 3240 |
+
if marker[2]:
|
| 3241 |
+
with _ODOO_DETAIL_CATALOG_LOCK:
|
| 3242 |
+
entry = _ODOO_DETAIL_CATALOG_CACHE.get(cache_key)
|
| 3243 |
+
merged = {**(entry[1] if entry else {}), **built}
|
| 3244 |
+
_ODOO_DETAIL_CATALOG_CACHE[cache_key] = (owner, merged)
|
| 3245 |
+
_ODOO_DETAIL_CATALOG_CACHE.move_to_end(cache_key)
|
| 3246 |
+
while len(_ODOO_DETAIL_CATALOG_CACHE) > _ODOO_DETAIL_CATALOG_CACHE_SIZE:
|
| 3247 |
+
_ODOO_DETAIL_CATALOG_CACHE.popitem(last=False)
|
| 3248 |
+
cached.update(built)
|
| 3249 |
+
enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}
|
| 3250 |
+
return {model: ([{**field, "enabled": connection_id is None
|
| 3251 |
+
and enabled.get(field["key"]) is True}
|
| 3252 |
+
for field in cached[model][0]],
|
| 3253 |
+
[dict(item) for item in cached[model][1]])
|
| 3254 |
+
for model in raw_models}
|
| 3255 |
|
| 3256 |
|
| 3257 |
def _spec_field(item, name):
|
|
|
|
| 3630 |
return _named_databases(rt, is_admin, uname, rows)
|
| 3631 |
|
| 3632 |
|
| 3633 |
+
def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None, *,
|
| 3634 |
+
page=None, fields="full"):
|
| 3635 |
"""C4's `databases` half: what this connector materialises, and the fields behind each one.
|
| 3636 |
|
| 3637 |
β ODOO IS THE ONLY INTROSPECTABLE CONNECTOR TODAY and every other card answers `[]`. That is
|
|
|
|
| 3659 |
reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
|
| 3660 |
is walled, per row, by `canRename`.
|
| 3661 |
"""
|
| 3662 |
+
def select(rows):
|
| 3663 |
+
return page.select(rows) if page is not None else rows
|
| 3664 |
+
|
| 3665 |
+
def disclose(rows):
|
| 3666 |
+
if fields == "summary":
|
| 3667 |
+
return [{**row, "fields": [], "excluded": [], "fieldsDeferred": True}
|
| 3668 |
+
for row in rows]
|
| 3669 |
+
return rows
|
| 3670 |
+
|
| 3671 |
if str(connector_key) == "scraper":
|
| 3672 |
+
return disclose(select(_scraper_databases(rt, is_admin, uname)))
|
| 3673 |
if not connector_directory.is_introspectable(connector_key):
|
| 3674 |
return []
|
| 3675 |
spec = _connector_spec_tables(connector_key)
|
| 3676 |
if str(connector_key) == "amazon_sp":
|
| 3677 |
+
return disclose(select(_named_databases(
|
| 3678 |
+
rt, is_admin, uname, _amazon_databases(rt, is_admin, spec))))
|
| 3679 |
try:
|
| 3680 |
import odoo_relational
|
| 3681 |
namespace = (odoo_relational.connection_namespace(rt, connection_id)[1]
|
| 3682 |
if connection_id else None)
|
| 3683 |
choices = odoo_relational.grid_choices(rt, connection_id)
|
| 3684 |
models = dict(getattr(odoo_relational, "ODOO_TABLE_MODELS", {}) or {})
|
| 3685 |
+
if page is not None:
|
| 3686 |
+
choices = page.select(choices, overview_rows=schema_overlay.merge_database_labels([
|
| 3687 |
+
{**choice, "label": str((spec.get(choice.get("baseKey") or choice["key"])
|
| 3688 |
+
or {}).get("label") or choice.get("label") or choice["key"])}
|
| 3689 |
+
for choice in choices], _database_name_overrides(rt)))
|
| 3690 |
# This read never contacts Odoo. A 499-model exact pass measured 67.87 s after the
|
| 3691 |
# catalogue was warm, so modal detail must never turn a missing cache into inline work.
|
| 3692 |
census = odoo_relational.population_census(rt, namespace)
|
|
|
|
| 3698 |
census_at = str(census.get("capturedAt") or "")
|
| 3699 |
counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
|
| 3700 |
catalog = {}
|
| 3701 |
+
if is_admin and fields == "full" and choices:
|
| 3702 |
try:
|
| 3703 |
+
catalog = _odoo_catalog_by_model(rt, namespace, models={
|
| 3704 |
+
str(choice.get("model") or models.get(choice.get("key")) or "")
|
| 3705 |
+
for choice in choices})
|
| 3706 |
except Exception: # noqa: BLE001
|
| 3707 |
catalog = {}
|
| 3708 |
enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {}
|
|
|
|
| 3729 |
key = str(choice.get("key") or "")
|
| 3730 |
resolved_connection_id = resolve_connection_id(key)
|
| 3731 |
model = str(choice.get("model") or models.get(key) or "")
|
| 3732 |
+
field_rows, excluded = catalog.get(model, ([], []))
|
| 3733 |
scoped_fields = []
|
| 3734 |
+
for field in field_rows:
|
| 3735 |
field_key = _odoo_field_key(rt, key, model, field["source_field"],
|
| 3736 |
connection_id=resolved_connection_id)
|
| 3737 |
scoped_fields.append({**field, "key": field_key,
|
|
|
|
| 3765 |
"fields": scoped_fields,
|
| 3766 |
"excluded": [dict(item) for item in excluded],
|
| 3767 |
"sharesFieldsWith": []})
|
| 3768 |
+
return disclose(_named_databases(rt, is_admin, uname, out))
|
| 3769 |
|
| 3770 |
|
| 3771 |
def _amazon_report_name(table):
|
|
|
|
| 3815 |
enabled = enablement.get("databases") or {}
|
| 3816 |
cat = amazon_schema.catalogue()
|
| 3817 |
tables = cat.get("tables") or {}
|
| 3818 |
+
# β S67-T31 β A STATUS THE GUARD REFUSES COSTS ITS OWN ROW, NEVER THE MODAL. `sync_state`
|
| 3819 |
+
# trims an over-long cause itself; a credential shape still refuses, and the refusal carries
|
| 3820 |
+
# the guarded rows that passed, so only the refused tables read "Sync status is unavailable
|
| 3821 |
+
# for this database." (a missing status is exactly what `_amazon_database_sync_health`
|
| 3822 |
+
# degrades). Any other failure reading status degrades every row, never the whole page.
|
| 3823 |
+
try:
|
| 3824 |
+
states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
|
| 3825 |
+
except cx.AmazonStatusSafetyError as refused:
|
| 3826 |
+
states = dict(getattr(refused, "safe_status", None) or {})
|
| 3827 |
+
except Exception: # noqa: BLE001
|
| 3828 |
+
states = {}
|
| 3829 |
|
| 3830 |
# Which OTHER declared tables are filled from the SAME report document. Derived from the
|
| 3831 |
# report map so a table added to `REPORT_TYPES` gets this disclosure for free.
|
|
|
|
| 3996 |
captured = ""
|
| 3997 |
census = {}
|
| 3998 |
try:
|
|
|
|
|
|
|
| 3999 |
import odoo_relational
|
| 4000 |
discovered = odoo_relational.readable_model_catalog(rt)
|
| 4001 |
census = odoo_relational.population_census(rt)
|
| 4002 |
has_discovery = bool(discovered.get("capturedAt"))
|
| 4003 |
+
snapshot = {}
|
| 4004 |
+
if not has_discovery:
|
| 4005 |
+
import core.user_tables as user_tables
|
| 4006 |
+
snapshot = user_tables.odoo_schema_snapshot(rt)
|
| 4007 |
complete = (discovered.get("complete") is True if has_discovery
|
| 4008 |
else snapshot.get("complete") is True)
|
| 4009 |
captured = str((discovered.get("capturedAt") if has_discovery
|
|
|
|
| 4295 |
return {"key": key, "state": state, "counts": counts, "receipt": receipt}
|
| 4296 |
|
| 4297 |
|
| 4298 |
+
def connector_detail(rt, user, uname, is_admin, connector_key, *, databases_offset=None,
|
| 4299 |
+
databases_limit=None, fields="full"):
|
| 4300 |
"""The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
|
| 4301 |
|
| 4302 |
Pure over the session for the same reason `directory` is: a gate can drive it without a
|
| 4303 |
request, so the modal's payload is provable without a browser.
|
| 4304 |
"""
|
| 4305 |
budget = _ConnectorDetailBudget()
|
| 4306 |
+
page = _detail_page(databases_offset, databases_limit, fields)
|
| 4307 |
+
rt = _ConnectorDetailReads(rt)
|
| 4308 |
rows = directory(rt, uname, bool(is_admin))
|
| 4309 |
budget.check()
|
| 4310 |
key = str(connector_key or "")
|
|
|
|
| 4312 |
if str(candidate.get("key") or "") == key), None)
|
| 4313 |
if row is None:
|
| 4314 |
return None
|
| 4315 |
+
# Summary pages defer the independent graph as well, otherwise an all-table field graph
|
| 4316 |
+
# would recreate the unbounded payload beside the paged database list.
|
| 4317 |
+
graph = (None if fields == "summary" else
|
| 4318 |
+
relationship_graphs([row], user, rt, uname, bool(is_admin)).get(key)
|
| 4319 |
+
or _graph_refusal(key, "No actual relationship graph was returned for this connection. "
|
| 4320 |
+
"A provider template is not shown."))
|
| 4321 |
budget.check()
|
| 4322 |
# β `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a
|
| 4323 |
# question about a PERSON and a table, not about an administrator flag alone.
|
| 4324 |
+
databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or ""),
|
| 4325 |
+
page=page, fields=fields)
|
| 4326 |
+
if key == "amazon_sp":
|
| 4327 |
+
databases = _screen_amazon_database_rows(databases)
|
| 4328 |
budget.check()
|
| 4329 |
overview = _detail_overview(row, databases, graph)
|
| 4330 |
+
if page is not None and page.total:
|
| 4331 |
+
overview["creates"] = page.creates
|
| 4332 |
budget.check()
|
| 4333 |
keychain = _detail_keychain(rt, uname, bool(is_admin), row)
|
| 4334 |
budget.check()
|
|
|
|
| 4339 |
return _safe_connector_status(key, {"overview": overview,
|
| 4340 |
"keychain": keychain,
|
| 4341 |
"databases": databases,
|
| 4342 |
+
**({"databasesPage": page.dto()} if page is not None else {}),
|
| 4343 |
+
**({"relationshipsPath": f"/connectors/{key}/relationships"}
|
| 4344 |
+
if fields == "summary" else {}),
|
| 4345 |
"syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"}
|
| 4346 |
if is_admin and key in connector_sync.providers() else None),
|
| 4347 |
"materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
|
|
|
|
| 4361 |
"relationships": graph})
|
| 4362 |
|
| 4363 |
|
| 4364 |
+
def _screen_amazon_database_rows(rows):
|
| 4365 |
+
"""S67-T31: each Amazon database row answers the status guard ON ITS OWN.
|
| 4366 |
+
|
| 4367 |
+
`_safe_connector_status` still guards the whole detail below, and it used to be the first and
|
| 4368 |
+
only judge, so one oversized scalar in one row aborted every row, the Overview and the
|
| 4369 |
+
Keychain with a bare 500. Here an over-long string is trimmed, and a row that still fails (a
|
| 4370 |
+
credential shape anywhere in it) keeps only its key, its shipped name and its switch, with
|
| 4371 |
+
"Sync status is unavailable for this database." as its status. The refused value is dropped,
|
| 4372 |
+
never echoed.
|
| 4373 |
+
"""
|
| 4374 |
+
import connectors_amazon as cx # noqa: PLC0415 - provider owns the guard
|
| 4375 |
+
listed = list(rows or [])
|
| 4376 |
+
safe, _refused = cx.screen_status_rows(dict(enumerate(listed)))
|
| 4377 |
+
out = []
|
| 4378 |
+
for index, row in enumerate(listed):
|
| 4379 |
+
if index in safe:
|
| 4380 |
+
out.append(safe[index])
|
| 4381 |
+
continue
|
| 4382 |
+
key = str((row or {}).get("key") or "")
|
| 4383 |
+
enabled = (row or {}).get("enabled") is True
|
| 4384 |
+
label = LABELS_AMAZON.get(key) or key
|
| 4385 |
+
out.append({"key": key, "label": label, "defaultLabel": label, "native": "",
|
| 4386 |
+
"enabled": enabled,
|
| 4387 |
+
**({"sync": None} if "sync" in (row or {}) else {}),
|
| 4388 |
+
"syncHealth": _amazon_database_sync_health(None, enabled),
|
| 4389 |
+
"fields": [], "excluded": [], "sharesFieldsWith": [],
|
| 4390 |
+
**({"fieldsDeferred": True} if (row or {}).get("fieldsDeferred") else {})})
|
| 4391 |
+
return out
|
| 4392 |
+
|
| 4393 |
+
|
| 4394 |
def _set_amazon_database_enablement(database_key, body, session):
|
| 4395 |
"""The testable core of Amazon's tenant-scoped administrator enablement door."""
|
| 4396 |
if not session.admin:
|
|
|
|
| 4462 |
# the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
|
| 4463 |
# collide with this three-segment path at all.
|
| 4464 |
@router.get("/connectors/{connector_key}/detail")
|
| 4465 |
+
def connectors_detail(connector_key: str, session: Session = Depends(require_session),
|
| 4466 |
+
databases_offset: Annotated[int | None, Query(ge=0)] = None,
|
| 4467 |
+
databases_limit: Annotated[int | None, Query(ge=1)] = None,
|
| 4468 |
+
fields: Literal["summary", "full"] = "full"):
|
| 4469 |
"""C4 / R26: everything the connector modal shows, for THIS reader, in one read.
|
| 4470 |
|
| 4471 |
Session-gated exactly like the directory it is opened from: a member may look at what the
|
| 4472 |
workspace is connected to. What a member does NOT get is the tenant-wide credential door
|
| 4473 |
(`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue.
|
| 4474 |
"""
|
| 4475 |
+
try:
|
| 4476 |
+
detail = connector_detail(
|
| 4477 |
+
session.runtime, session.user, session.uname,
|
| 4478 |
+
bool(session.admin), connector_key, databases_offset=databases_offset,
|
| 4479 |
+
databases_limit=databases_limit, fields=fields)
|
| 4480 |
+
except HTTPException:
|
| 4481 |
+
raise
|
| 4482 |
+
except Exception as exc: # noqa: BLE001
|
| 4483 |
+
raise _plain_read_failure(exc, "connector_detail_failed",
|
| 4484 |
+
"The connector details could not be read.")
|
| 4485 |
if detail is None:
|
| 4486 |
raise err(404, "unknown_connector",
|
| 4487 |
"this workspace has no connector with that key")
|
| 4488 |
return detail
|
| 4489 |
|
| 4490 |
|
| 4491 |
+
def _plain_read_failure(exc, code, sentence):
|
| 4492 |
+
"""S67-T31: an unexpected failure on a connector read answers the app's JSON error shape.
|
| 4493 |
+
|
| 4494 |
+
The owner's modal read "The server answered 500." because a guard refusal escaped as a bare
|
| 4495 |
+
`Internal Server Error` with no body. Every read the modal makes now answers
|
| 4496 |
+
`{error: {code, message}}` with a plain cause instead; the traceback goes to the server log,
|
| 4497 |
+
and no exception text (which can echo a payload) reaches the wire.
|
| 4498 |
+
"""
|
| 4499 |
+
import logging # noqa: PLC0415
|
| 4500 |
+
logging.getLogger(__name__).exception("connector read failed: %s", code)
|
| 4501 |
+
name = type(exc).__name__
|
| 4502 |
+
if name in ("StoreUnavailable", "PoolTimeout"):
|
| 4503 |
+
return err(503, "store_unavailable",
|
| 4504 |
+
"The workspace store did not answer in time, so nothing could be read.")
|
| 4505 |
+
try:
|
| 4506 |
+
import connectors_amazon as cx # noqa: PLC0415
|
| 4507 |
+
if isinstance(exc, cx.AmazonStatusSafetyError):
|
| 4508 |
+
return err(500, "connector_status_withheld",
|
| 4509 |
+
"This connector's status could not be shown safely, so it was withheld.")
|
| 4510 |
+
except Exception: # noqa: BLE001
|
| 4511 |
+
pass
|
| 4512 |
+
return err(500, code, sentence)
|
| 4513 |
+
|
| 4514 |
+
|
| 4515 |
+
def _detail_row(rt, uname, is_admin, connector_key):
|
| 4516 |
+
row = next((row for row in directory(rt, uname, is_admin)
|
| 4517 |
+
if row.get("key") == connector_key), None)
|
| 4518 |
+
if row is None:
|
| 4519 |
+
raise err(404, "unknown_connector", "this workspace has no connector with that key")
|
| 4520 |
+
return row
|
| 4521 |
+
|
| 4522 |
+
|
| 4523 |
+
@router.get("/connectors/{connector_key}/databases/{database_key}/fields")
|
| 4524 |
+
def connector_database_fields(connector_key: str, database_key: str,
|
| 4525 |
+
session: Session = Depends(require_session)):
|
| 4526 |
+
"""The same field disclosure and database access as detail, for one database only."""
|
| 4527 |
+
rt = _ConnectorDetailReads(session.runtime)
|
| 4528 |
+
try:
|
| 4529 |
+
_detail_row(rt, session.uname, bool(session.admin), connector_key)
|
| 4530 |
+
rows = _detail_databases(rt, bool(session.admin), connector_key, uname=session.uname,
|
| 4531 |
+
page=_DatabasePage(database_key=database_key))
|
| 4532 |
+
if not rows:
|
| 4533 |
+
raise err(404, "unknown_database",
|
| 4534 |
+
"This connector has no visible database with that key.")
|
| 4535 |
+
return _safe_connector_status(connector_key, {
|
| 4536 |
+
"key": database_key, "fields": rows[0]["fields"], "excluded": rows[0]["excluded"]})
|
| 4537 |
+
except HTTPException:
|
| 4538 |
+
raise
|
| 4539 |
+
except Exception as exc: # noqa: BLE001
|
| 4540 |
+
raise _plain_read_failure(exc, "connector_fields_failed",
|
| 4541 |
+
"The fields of this database could not be read.")
|
| 4542 |
+
|
| 4543 |
+
|
| 4544 |
+
@router.get("/connectors/{connector_key}/relationships")
|
| 4545 |
+
def connector_relationships(connector_key: str, session: Session = Depends(require_session)):
|
| 4546 |
+
"""Load the permitted graph when the summary client's Relationships tab opens."""
|
| 4547 |
+
rt = _ConnectorDetailReads(session.runtime)
|
| 4548 |
+
try:
|
| 4549 |
+
row = _detail_row(rt, session.uname, bool(session.admin), connector_key)
|
| 4550 |
+
return _safe_connector_status(connector_key, relationship_graphs(
|
| 4551 |
+
[row], session.user, rt, session.uname, bool(session.admin))[connector_key])
|
| 4552 |
+
except HTTPException:
|
| 4553 |
+
raise
|
| 4554 |
+
except Exception as exc: # noqa: BLE001
|
| 4555 |
+
raise _plain_read_failure(exc, "connector_relationships_failed",
|
| 4556 |
+
"The relationship graph could not be read.")
|
| 4557 |
+
|
| 4558 |
+
|
| 4559 |
class _ReplacementConflict(Exception):
|
| 4560 |
"""The addressed keychain row changed between replacement preparation and its store update."""
|
| 4561 |
|
api/routes_external.py
CHANGED
|
@@ -19,7 +19,7 @@ from starlette.concurrency import run_in_threadpool
|
|
| 19 |
import deps
|
| 20 |
import aios_session
|
| 21 |
from deps import Session, err, require_session
|
| 22 |
-
from core import audit, external_access, external_oauth, external_tools, sheets
|
| 23 |
from harness import runtime
|
| 24 |
|
| 25 |
|
|
@@ -621,14 +621,16 @@ def require_external_session(
|
|
| 621 |
def require_external_analyst(
|
| 622 |
session: Session = Depends(require_external_session),
|
| 623 |
) -> Session:
|
| 624 |
-
"""
|
| 625 |
-
|
|
|
|
|
|
|
| 626 |
return session
|
| 627 |
|
| 628 |
|
| 629 |
def _can_use_analyst(session: Session) -> bool:
|
| 630 |
try:
|
| 631 |
-
|
| 632 |
except HTTPException as exc:
|
| 633 |
if exc.status_code == 403:
|
| 634 |
return False
|
|
@@ -726,8 +728,12 @@ def _resolve_external_scope(
|
|
| 726 |
if include_schema:
|
| 727 |
if row["answerable"] or strict_schema:
|
| 728 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 729 |
scoped = deps.assistant_read_scope(
|
| 730 |
-
session, database, fields=None, filters=None,
|
| 731 |
)
|
| 732 |
except HTTPException as exc:
|
| 733 |
# HOTFIX 2026-09-24: ONE refusing source (measured: a 996,203-row mirror
|
|
@@ -999,7 +1005,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str:
|
|
| 999 |
f" '{surface_url}/mcp'",
|
| 1000 |
"```",
|
| 1001 |
"",
|
| 1002 |
-
("Sheet tools require
|
| 1003 |
"read by the server; pass rev to also protect the revision you read. Cell writes "
|
| 1004 |
"are previewed before confirmation. A partially refused patch changes no cells."),
|
| 1005 |
"",
|
|
@@ -1043,7 +1049,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str:
|
|
| 1043 |
@router.post("/keys", status_code=201)
|
| 1044 |
def create_key(body=Body(default=None), session: Session = Depends(require_session)):
|
| 1045 |
"""Mint a credential for the signed-in human. Bearer auth is not accepted here."""
|
| 1046 |
-
|
| 1047 |
label = body.get("label") if isinstance(body, dict) else None
|
| 1048 |
access = body.get("access") if isinstance(body, dict) else None
|
| 1049 |
databases = body.get("databases") if isinstance(body, dict) else None
|
|
@@ -1182,7 +1188,7 @@ def oauth_metadata(request: Request):
|
|
| 1182 |
@router.post("/oauth/clients", status_code=201)
|
| 1183 |
def create_oauth_client(body=Body(default=None), session: Session = Depends(require_session)):
|
| 1184 |
"""Register one user-owned public client; PKCE replaces a shared client secret."""
|
| 1185 |
-
|
| 1186 |
body = body if isinstance(body, dict) else {}
|
| 1187 |
try:
|
| 1188 |
client = external_oauth.register_client(
|
|
@@ -1470,16 +1476,18 @@ def oauth_authorize(
|
|
| 1470 |
if redirect_uri not in meta["r"]:
|
| 1471 |
return _refusal_page("The app sent a callback it did not register. Start the "
|
| 1472 |
"connection again from the app.")
|
|
|
|
|
|
|
|
|
|
| 1473 |
try:
|
| 1474 |
-
session.require("analyst")
|
| 1475 |
_require_oauth_resource(request, resource)
|
| 1476 |
if response_type != "code":
|
| 1477 |
raise external_oauth.ExternalOAuthError("oauth_response_type_invalid")
|
| 1478 |
external_oauth._normalise_state(state)
|
| 1479 |
external_oauth._normalise_challenge(code_challenge, code_challenge_method)
|
| 1480 |
except HTTPException:
|
| 1481 |
-
return _refusal_page("
|
| 1482 |
-
"
|
| 1483 |
except external_access.ExternalAccessError:
|
| 1484 |
return _refusal_page("The app sent an incomplete sign-in request. Start the "
|
| 1485 |
"connection again from the app.")
|
|
@@ -1490,7 +1498,9 @@ def oauth_authorize(
|
|
| 1490 |
return _consent_page(request, session, meta, fields)
|
| 1491 |
if session is None:
|
| 1492 |
raise err(401, "no_session", "sign in to continue")
|
| 1493 |
-
|
|
|
|
|
|
|
| 1494 |
try:
|
| 1495 |
audience = _require_oauth_resource(request, resource)
|
| 1496 |
result = external_oauth.authorize(
|
|
@@ -1532,11 +1542,9 @@ async def oauth_consent(request: Request, session: Session = Depends(require_ses
|
|
| 1532 |
databases = chosen
|
| 1533 |
else:
|
| 1534 |
databases = external_access.ALL_DATABASES
|
| 1535 |
-
|
| 1536 |
-
|
| 1537 |
-
|
| 1538 |
-
return _refusal_page("Your Loopable role cannot connect AI apps. Ask an administrator.",
|
| 1539 |
-
403)
|
| 1540 |
try:
|
| 1541 |
audience = _require_oauth_resource(request, fields.get("resource"))
|
| 1542 |
client = external_oauth.adopt_dynamic_client(
|
|
@@ -1719,7 +1727,7 @@ def external_setup(request: Request):
|
|
| 1719 |
@router.get("/skill.md")
|
| 1720 |
def portable_skill(request: Request, session: Session = Depends(require_session)):
|
| 1721 |
"""Download current governed external read and build instructions."""
|
| 1722 |
-
|
| 1723 |
document = _portable_skill_document(session, str(request.base_url))
|
| 1724 |
return Response(
|
| 1725 |
content=document,
|
|
@@ -1750,7 +1758,7 @@ def whoami(session: Session = Depends(require_external_analyst)):
|
|
| 1750 |
@router.get("/scope/databases")
|
| 1751 |
def scope_databases(session: Session = Depends(require_session)):
|
| 1752 |
"""List the databases this signed-in maker could put inside a new credential's scope."""
|
| 1753 |
-
|
| 1754 |
return {"databases": [
|
| 1755 |
{key: row[key] for key in ("database", "label", "permitted", "answerable")}
|
| 1756 |
for row in _resolve_external_scope(session)
|
|
@@ -1817,8 +1825,11 @@ def _query_database(session: Session, database: str, payload: dict):
|
|
| 1817 |
|
| 1818 |
# This is the same permissioned record reader the in-app Assistant uses. It resolves the
|
| 1819 |
# database, field and row walls on every call before this adapter sorts, caps, or aggregates.
|
|
|
|
|
|
|
| 1820 |
scoped = deps.assistant_read_scope(
|
| 1821 |
-
session, database, fields=None, filters=payload.get("filters"))
|
|
|
|
| 1822 |
visible_fields = list(scoped.get("fields") or [])
|
| 1823 |
visible_keys = {
|
| 1824 |
field.get("key") for field in visible_fields
|
|
@@ -1847,6 +1858,104 @@ def _query_database(session: Session, database: str, payload: dict):
|
|
| 1847 |
sorts.append({"field": rule["field"], "direction": rule["direction"]})
|
| 1848 |
aggregates = _normalise_aggregates(payload.get("aggregates"), visible_keys)
|
| 1849 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1850 |
rows = list(scoped.get("records") or [])
|
| 1851 |
inspected_keys = (set(selected_keys)
|
| 1852 |
| {rule["field"] for rule in sorts}
|
|
@@ -1897,7 +2006,8 @@ def _query_database(session: Session, database: str, payload: dict):
|
|
| 1897 |
"filters": copy.deepcopy(scoped.get("filters")),
|
| 1898 |
"sorts": copy.deepcopy(sorts),
|
| 1899 |
"records": records,
|
| 1900 |
-
"aggregates":
|
|
|
|
| 1901 |
"page": {"limit": limit, "returned": len(records), "offset": offset,
|
| 1902 |
"has_more": next_cursor is not None, "next_cursor": next_cursor},
|
| 1903 |
"caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
|
|
@@ -2730,7 +2840,7 @@ def _tool_context(session: Session, *, preview=False) -> external_tools.ToolCont
|
|
| 2730 |
def _strong_confirmation_required(name, args):
|
| 2731 |
if name == "database_object":
|
| 2732 |
return args.get("action") in {"record_delete", "field_create", "field_update", "field_delete"}
|
| 2733 |
-
if name in {"view_delete", "app_delete"}:
|
| 2734 |
return True
|
| 2735 |
if name == "sheet_write_cells":
|
| 2736 |
return any(isinstance(cell, dict) and cell.get("kind") == "empty"
|
|
@@ -2913,7 +3023,7 @@ def _confirm_change(session, args):
|
|
| 2913 |
"Enter the exact stronger confirmation phrase shown in the preview.")
|
| 2914 |
_confirmation_already_used(session, payload)
|
| 2915 |
|
| 2916 |
-
# Freshly resolve the account,
|
| 2917 |
# database walls and writer semantics before claiming the nonce or touching customer state.
|
| 2918 |
fresh_user = external_access.resolve_active_user(session.uname, session.tenant)
|
| 2919 |
fresh_session = Session(session.tenant, fresh_user, copy.deepcopy(session.claims), session.runtime)
|
|
@@ -3100,7 +3210,7 @@ def _handle_mcp(req, session: Session, base_url: str = ""):
|
|
| 3100 |
return protocol_error(rid, -32602, "MCP params must be an object.")
|
| 3101 |
name = params.get("name")
|
| 3102 |
try:
|
| 3103 |
-
# Same
|
| 3104 |
# remains MCP content. Only the unchanged bearer dependency produces an HTTP 401.
|
| 3105 |
require_external_analyst(session)
|
| 3106 |
if method == "initialize":
|
|
@@ -3163,7 +3273,7 @@ def _handle_mcp(req, session: Session, base_url: str = ""):
|
|
| 3163 |
|
| 3164 |
@router.post("/mcp")
|
| 3165 |
async def external_mcp(request: Request, session: Session = Depends(require_external_session)):
|
| 3166 |
-
"""Remote MCP without a browser session, with the read doors' maker and
|
| 3167 |
try:
|
| 3168 |
req = await request.json()
|
| 3169 |
except (ValueError, UnicodeError):
|
|
@@ -3177,6 +3287,7 @@ async def external_mcp(request: Request, session: Session = Depends(require_exte
|
|
| 3177 |
# A new write in the registry without a matching literal raises at import, before serving.
|
| 3178 |
_WRITE_TOOLS = frozenset({
|
| 3179 |
"sheet_create_workbook", "sheet_add_tab", "sheet_write_cells", "sheet_set_binding",
|
|
|
|
| 3180 |
"sheet_set_styles", "sheet_set_dimensions", "sheet_set_validation", "sheet_set_freeze",
|
| 3181 |
"sheet_add_rule", "sheet_set_chart", "sheet_structure",
|
| 3182 |
"sheet_save_template",
|
|
|
|
| 19 |
import deps
|
| 20 |
import aios_session
|
| 21 |
from deps import Session, err, require_session
|
| 22 |
+
from core import audit, external_access, external_oauth, external_tools, perm_scope, sheets
|
| 23 |
from harness import runtime
|
| 24 |
|
| 25 |
|
|
|
|
| 621 |
def require_external_analyst(
|
| 622 |
session: Session = Depends(require_external_session),
|
| 623 |
) -> Session:
|
| 624 |
+
"""Check the separate AI apps grant on every request, including existing bearer tokens."""
|
| 625 |
+
reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
|
| 626 |
+
if reason:
|
| 627 |
+
raise err(403, "forbidden", reason)
|
| 628 |
return session
|
| 629 |
|
| 630 |
|
| 631 |
def _can_use_analyst(session: Session) -> bool:
|
| 632 |
try:
|
| 633 |
+
require_external_analyst(session)
|
| 634 |
except HTTPException as exc:
|
| 635 |
if exc.status_code == 403:
|
| 636 |
return False
|
|
|
|
| 728 |
if include_schema:
|
| 729 |
if row["answerable"] or strict_schema:
|
| 730 |
try:
|
| 731 |
+
# β S67-T34 (D-2630): THE FIELD CONTRACT, NOT THE ROWS. Describing a
|
| 732 |
+
# database used to read (and count) every row first, so the largest
|
| 733 |
+
# connected table refused its own schema. "schema" runs every wall and
|
| 734 |
+
# reads no row; sources that are not mirror-served ignore it.
|
| 735 |
scoped = deps.assistant_read_scope(
|
| 736 |
+
session, database, fields=None, filters=None, mode="schema",
|
| 737 |
)
|
| 738 |
except HTTPException as exc:
|
| 739 |
# HOTFIX 2026-09-24: ONE refusing source (measured: a 996,203-row mirror
|
|
|
|
| 1005 |
f" '{surface_url}/mcp'",
|
| 1006 |
"```",
|
| 1007 |
"",
|
| 1008 |
+
("Sheet tools require AI apps (MCP) and Sheet access. Geometry patches carry the revision "
|
| 1009 |
"read by the server; pass rev to also protect the revision you read. Cell writes "
|
| 1010 |
"are previewed before confirmation. A partially refused patch changes no cells."),
|
| 1011 |
"",
|
|
|
|
| 1049 |
@router.post("/keys", status_code=201)
|
| 1050 |
def create_key(body=Body(default=None), session: Session = Depends(require_session)):
|
| 1051 |
"""Mint a credential for the signed-in human. Bearer auth is not accepted here."""
|
| 1052 |
+
require_external_analyst(session)
|
| 1053 |
label = body.get("label") if isinstance(body, dict) else None
|
| 1054 |
access = body.get("access") if isinstance(body, dict) else None
|
| 1055 |
databases = body.get("databases") if isinstance(body, dict) else None
|
|
|
|
| 1188 |
@router.post("/oauth/clients", status_code=201)
|
| 1189 |
def create_oauth_client(body=Body(default=None), session: Session = Depends(require_session)):
|
| 1190 |
"""Register one user-owned public client; PKCE replaces a shared client secret."""
|
| 1191 |
+
require_external_analyst(session)
|
| 1192 |
body = body if isinstance(body, dict) else {}
|
| 1193 |
try:
|
| 1194 |
client = external_oauth.register_client(
|
|
|
|
| 1476 |
if redirect_uri not in meta["r"]:
|
| 1477 |
return _refusal_page("The app sent a callback it did not register. Start the "
|
| 1478 |
"connection again from the app.")
|
| 1479 |
+
reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
|
| 1480 |
+
if reason:
|
| 1481 |
+
return _refusal_page(reason, 403)
|
| 1482 |
try:
|
|
|
|
| 1483 |
_require_oauth_resource(request, resource)
|
| 1484 |
if response_type != "code":
|
| 1485 |
raise external_oauth.ExternalOAuthError("oauth_response_type_invalid")
|
| 1486 |
external_oauth._normalise_state(state)
|
| 1487 |
external_oauth._normalise_challenge(code_challenge, code_challenge_method)
|
| 1488 |
except HTTPException:
|
| 1489 |
+
return _refusal_page("The app requested an unavailable resource. Start the "
|
| 1490 |
+
"connection again from the app.", 403)
|
| 1491 |
except external_access.ExternalAccessError:
|
| 1492 |
return _refusal_page("The app sent an incomplete sign-in request. Start the "
|
| 1493 |
"connection again from the app.")
|
|
|
|
| 1498 |
return _consent_page(request, session, meta, fields)
|
| 1499 |
if session is None:
|
| 1500 |
raise err(401, "no_session", "sign in to continue")
|
| 1501 |
+
reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
|
| 1502 |
+
if reason:
|
| 1503 |
+
return _refusal_page(reason, 403)
|
| 1504 |
try:
|
| 1505 |
audience = _require_oauth_resource(request, resource)
|
| 1506 |
result = external_oauth.authorize(
|
|
|
|
| 1542 |
databases = chosen
|
| 1543 |
else:
|
| 1544 |
databases = external_access.ALL_DATABASES
|
| 1545 |
+
reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
|
| 1546 |
+
if reason:
|
| 1547 |
+
return _refusal_page(reason, 403)
|
|
|
|
|
|
|
| 1548 |
try:
|
| 1549 |
audience = _require_oauth_resource(request, fields.get("resource"))
|
| 1550 |
client = external_oauth.adopt_dynamic_client(
|
|
|
|
| 1727 |
@router.get("/skill.md")
|
| 1728 |
def portable_skill(request: Request, session: Session = Depends(require_session)):
|
| 1729 |
"""Download current governed external read and build instructions."""
|
| 1730 |
+
require_external_analyst(session)
|
| 1731 |
document = _portable_skill_document(session, str(request.base_url))
|
| 1732 |
return Response(
|
| 1733 |
content=document,
|
|
|
|
| 1758 |
@router.get("/scope/databases")
|
| 1759 |
def scope_databases(session: Session = Depends(require_session)):
|
| 1760 |
"""List the databases this signed-in maker could put inside a new credential's scope."""
|
| 1761 |
+
require_external_analyst(session)
|
| 1762 |
return {"databases": [
|
| 1763 |
{key: row[key] for key in ("database", "label", "permitted", "answerable")}
|
| 1764 |
for row in _resolve_external_scope(session)
|
|
|
|
| 1825 |
|
| 1826 |
# This is the same permissioned record reader the in-app Assistant uses. It resolves the
|
| 1827 |
# database, field and row walls on every call before this adapter sorts, caps, or aggregates.
|
| 1828 |
+
# β S67-T34: a MIRROR-SERVED database is first resolved schema-only (every wall, no row), so
|
| 1829 |
+
# its aggregates run in SQL over every permitted row and its rows are read only as needed.
|
| 1830 |
scoped = deps.assistant_read_scope(
|
| 1831 |
+
session, database, fields=None, filters=payload.get("filters"), mode="schema")
|
| 1832 |
+
mirror = scoped.get("source_kind") == "mirror_user_table"
|
| 1833 |
visible_fields = list(scoped.get("fields") or [])
|
| 1834 |
visible_keys = {
|
| 1835 |
field.get("key") for field in visible_fields
|
|
|
|
| 1858 |
sorts.append({"field": rule["field"], "direction": rule["direction"]})
|
| 1859 |
aggregates = _normalise_aggregates(payload.get("aggregates"), visible_keys)
|
| 1860 |
|
| 1861 |
+
server_aggregates = None
|
| 1862 |
+
paged_total = None
|
| 1863 |
+
if mirror:
|
| 1864 |
+
# ββ S67-T34 (D-616, D-2631): THE PERMISSION FILTER IS INSIDE THE QUERY. The aggregates
|
| 1865 |
+
# are computed by the mirror in SQL over exactly the rows this caller may read, never
|
| 1866 |
+
# over a fetched slice, and never by pulling a million rows into this process.
|
| 1867 |
+
if aggregates:
|
| 1868 |
+
summary = deps.assistant_read_scope(
|
| 1869 |
+
session, database, fields=None, filters=payload.get("filters"),
|
| 1870 |
+
mode=("aggregate", aggregates))
|
| 1871 |
+
server_aggregates = summary.get("server_aggregates")
|
| 1872 |
+
import assistant_mirror
|
| 1873 |
+
# The summary already counted the permitted population, so a table past the one-pass
|
| 1874 |
+
# ceiling goes straight to SQL paging instead of pulling a first 5,000-row page only to
|
| 1875 |
+
# refuse it.
|
| 1876 |
+
known = (summary.get("total_rows") if aggregates else None)
|
| 1877 |
+
try:
|
| 1878 |
+
if known is not None and int(known) > assistant_mirror.ASSISTANT_READ_MAX:
|
| 1879 |
+
raise err(409, "assistant_source_too_large",
|
| 1880 |
+
f"this database matches {int(known):,} rows for that question, "
|
| 1881 |
+
f"past the {assistant_mirror.ASSISTANT_READ_MAX:,} rows one answer "
|
| 1882 |
+
f"holds in memory")
|
| 1883 |
+
scoped = deps.assistant_read_scope(
|
| 1884 |
+
session, database, fields=None, filters=payload.get("filters"))
|
| 1885 |
+
except HTTPException as exc:
|
| 1886 |
+
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 1887 |
+
if info.get("code") != "assistant_source_too_large":
|
| 1888 |
+
raise
|
| 1889 |
+
# β RULE 1: PAST THE ONE-PASS CEILING THE ROWS ARE PAGED IN SQL, NOT REFUSED. Only
|
| 1890 |
+
# this page reaches the process. A sort cannot be honoured at this size without
|
| 1891 |
+
# holding every row, so it is refused by name with the fix, never silently dropped.
|
| 1892 |
+
if sorts:
|
| 1893 |
+
raise err(409, "query_sort_too_large",
|
| 1894 |
+
f"{info.get('message') or 'This database is too large to sort here.'} "
|
| 1895 |
+
f"Sorting is refused at this size because it would need every row in "
|
| 1896 |
+
f"memory. Ask without sorts to page it in record order, or narrow it "
|
| 1897 |
+
f"with a filter and sort again.") from exc
|
| 1898 |
+
paged_total = True
|
| 1899 |
+
if paged_total:
|
| 1900 |
+
offset = (0 if payload.get("cursor") is None else None)
|
| 1901 |
+
page_digest = _json_digest({
|
| 1902 |
+
"database": database, "fields": selected_keys, "filters": payload.get("filters"),
|
| 1903 |
+
"sorts": [], "aggregates": aggregates, "limit": limit, "paged": True,
|
| 1904 |
+
})
|
| 1905 |
+
first = deps.assistant_read_scope(
|
| 1906 |
+
session, database, fields=None, filters=payload.get("filters"), mode=("page", 0, 1))
|
| 1907 |
+
snap = _json_digest({
|
| 1908 |
+
"source_version": first.get("source_version"),
|
| 1909 |
+
"visible_fields": [field["key"] for field in visible_fields
|
| 1910 |
+
if isinstance(field, dict) and field.get("key") in visible_keys],
|
| 1911 |
+
"total_rows": first.get("total_rows"),
|
| 1912 |
+
})
|
| 1913 |
+
if offset is None:
|
| 1914 |
+
offset = _decode_query_cursor(session, payload.get("cursor"), page_digest, snap)
|
| 1915 |
+
total_rows = int(first.get("total_rows") or 0)
|
| 1916 |
+
if offset > total_rows:
|
| 1917 |
+
raise err(409, "query_cursor_stale",
|
| 1918 |
+
"The permitted result set changed. Start again without a cursor.")
|
| 1919 |
+
page = deps.assistant_read_scope(
|
| 1920 |
+
session, database, fields=None, filters=payload.get("filters"),
|
| 1921 |
+
mode=("page", offset, limit))
|
| 1922 |
+
page_rows = list(page.get("records") or [])
|
| 1923 |
+
for row in page_rows:
|
| 1924 |
+
_assert_finite_query_value({"pid": row.get("pid"),
|
| 1925 |
+
**{key: row.get(key) for key in selected_keys}})
|
| 1926 |
+
records = [{"pid": copy.deepcopy(row.get("pid")),
|
| 1927 |
+
**{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
|
| 1928 |
+
for row in page_rows]
|
| 1929 |
+
next_offset = offset + limit
|
| 1930 |
+
next_cursor = (_encode_query_cursor(session, page_digest, snap, next_offset)
|
| 1931 |
+
if next_offset < total_rows else None)
|
| 1932 |
+
result = {
|
| 1933 |
+
"database": page.get("database"), "label": page.get("label"),
|
| 1934 |
+
"source_kind": page.get("source_kind"),
|
| 1935 |
+
"source_version": copy.deepcopy(page.get("source_version")),
|
| 1936 |
+
"retrieved_at": page.get("retrieved_at"), "permission_scope_applied": True,
|
| 1937 |
+
"fields": [copy.deepcopy(field) for field in visible_fields
|
| 1938 |
+
if field.get("key") in selected_keys],
|
| 1939 |
+
"filters": copy.deepcopy(page.get("filters")), "sorts": [],
|
| 1940 |
+
"records": records, "aggregates": copy.deepcopy(server_aggregates or []),
|
| 1941 |
+
"page": {"limit": limit, "returned": len(records), "offset": offset,
|
| 1942 |
+
"has_more": next_cursor is not None, "next_cursor": next_cursor},
|
| 1943 |
+
"caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
|
| 1944 |
+
"max_aggregates": _QUERY_MAX_AGGREGATES,
|
| 1945 |
+
"max_filter_depth": _QUERY_MAX_FILTER_DEPTH,
|
| 1946 |
+
"max_filter_nodes": _QUERY_MAX_FILTER_NODES,
|
| 1947 |
+
"max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS},
|
| 1948 |
+
}
|
| 1949 |
+
_assert_finite_query_value(result)
|
| 1950 |
+
audit.record("external.query", session.tenant, session.uname, {
|
| 1951 |
+
"surface": "ai_api",
|
| 1952 |
+
"authentication": session.claims.get("external_auth_type", "api_key"),
|
| 1953 |
+
"field_count": len(selected_keys), "filter_applied": page.get("filters") is not None,
|
| 1954 |
+
"sort_count": 0, "aggregate_count": len(aggregates), "page_size": limit,
|
| 1955 |
+
"returned_count": len(records), "cursor_used": offset > 0,
|
| 1956 |
+
})
|
| 1957 |
+
return result
|
| 1958 |
+
|
| 1959 |
rows = list(scoped.get("records") or [])
|
| 1960 |
inspected_keys = (set(selected_keys)
|
| 1961 |
| {rule["field"] for rule in sorts}
|
|
|
|
| 2006 |
"filters": copy.deepcopy(scoped.get("filters")),
|
| 2007 |
"sorts": copy.deepcopy(sorts),
|
| 2008 |
"records": records,
|
| 2009 |
+
"aggregates": (copy.deepcopy(server_aggregates) if server_aggregates is not None
|
| 2010 |
+
else _aggregate_records(rows, aggregates)),
|
| 2011 |
"page": {"limit": limit, "returned": len(records), "offset": offset,
|
| 2012 |
"has_more": next_cursor is not None, "next_cursor": next_cursor},
|
| 2013 |
"caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
|
|
|
|
| 2840 |
def _strong_confirmation_required(name, args):
|
| 2841 |
if name == "database_object":
|
| 2842 |
return args.get("action") in {"record_delete", "field_create", "field_update", "field_delete"}
|
| 2843 |
+
if name in {"view_delete", "app_delete", "sheet_delete_tab"}:
|
| 2844 |
return True
|
| 2845 |
if name == "sheet_write_cells":
|
| 2846 |
return any(isinstance(cell, dict) and cell.get("kind") == "empty"
|
|
|
|
| 3023 |
"Enter the exact stronger confirmation phrase shown in the preview.")
|
| 3024 |
_confirmation_already_used(session, payload)
|
| 3025 |
|
| 3026 |
+
# Freshly resolve the account, AI apps grant, credential revocation/label, credential scope,
|
| 3027 |
# database walls and writer semantics before claiming the nonce or touching customer state.
|
| 3028 |
fresh_user = external_access.resolve_active_user(session.uname, session.tenant)
|
| 3029 |
fresh_session = Session(session.tenant, fresh_user, copy.deepcopy(session.claims), session.runtime)
|
|
|
|
| 3210 |
return protocol_error(rid, -32602, "MCP params must be an object.")
|
| 3211 |
name = params.get("name")
|
| 3212 |
try:
|
| 3213 |
+
# Same AI apps wall as the read doors, inside the dispatcher so a grant refusal
|
| 3214 |
# remains MCP content. Only the unchanged bearer dependency produces an HTTP 401.
|
| 3215 |
require_external_analyst(session)
|
| 3216 |
if method == "initialize":
|
|
|
|
| 3273 |
|
| 3274 |
@router.post("/mcp")
|
| 3275 |
async def external_mcp(request: Request, session: Session = Depends(require_external_session)):
|
| 3276 |
+
"""Remote MCP without a browser session, with the read doors' maker and AI apps wall."""
|
| 3277 |
try:
|
| 3278 |
req = await request.json()
|
| 3279 |
except (ValueError, UnicodeError):
|
|
|
|
| 3287 |
# A new write in the registry without a matching literal raises at import, before serving.
|
| 3288 |
_WRITE_TOOLS = frozenset({
|
| 3289 |
"sheet_create_workbook", "sheet_add_tab", "sheet_write_cells", "sheet_set_binding",
|
| 3290 |
+
"sheet_delete_tab",
|
| 3291 |
"sheet_set_styles", "sheet_set_dimensions", "sheet_set_validation", "sheet_set_freeze",
|
| 3292 |
"sheet_add_rule", "sheet_set_chart", "sheet_structure",
|
| 3293 |
"sheet_save_template",
|
api/routes_keychain.py
CHANGED
|
@@ -782,23 +782,63 @@ def _credential_revocation(row, rt):
|
|
| 782 |
|
| 783 |
|
| 784 |
def _revocation_residue(entry_type):
|
| 785 |
-
"""
|
| 786 |
-
|
| 787 |
-
Clearing the app's materialised copy is not evidence that the raw reports disappeared. Mirror
|
| 788 |
-
deletion stays operator-only because it is permanent, exact-tenant work outside the product
|
| 789 |
-
store transaction.
|
| 790 |
-
"""
|
| 791 |
if entry_type != "amazon_sp":
|
| 792 |
return []
|
| 793 |
return [{
|
| 794 |
"where": "The tenant DuckDB mirror",
|
| 795 |
"why": "Connection removal clears product databases but not the raw Amazon mirror.",
|
| 796 |
-
"fix": "
|
| 797 |
"platform/harness/amazon_store.py --purge --tenant <tenant> "
|
| 798 |
"--confirm-tenant <tenant> and retain the residue-free receipt.",
|
| 799 |
}]
|
| 800 |
|
| 801 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 802 |
def _write_revocation_warning(rt, entry_id, warning=None):
|
| 803 |
"""Prune expired consent on every write; failed persistence requires explicit confirmation."""
|
| 804 |
now = int(time.time())
|
|
@@ -852,6 +892,7 @@ def list_keychain(session: Session = Depends(require_session)):
|
|
| 852 |
@router.post("/admin/keychain", status_code=201)
|
| 853 |
def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)):
|
| 854 |
kc = _kc()
|
|
|
|
| 855 |
body = body or {}
|
| 856 |
if not session.runtime.available():
|
| 857 |
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
|
@@ -921,6 +962,11 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 921 |
except Exception: # noqa: BLE001
|
| 922 |
pass
|
| 923 |
elif etype == "amazon_sp":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 924 |
# D-29 / intake F6: the HTTP thread only records intent. The resident resync thread
|
| 925 |
# remains the mirror's single writer. A queue failure must not undo a stored credential.
|
| 926 |
try:
|
|
@@ -939,7 +985,7 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 939 |
raise err(503, "store_unavailable",
|
| 940 |
"the key was not saved; its sharing setting could not be stored, so "
|
| 941 |
"nothing was kept. Try again.")
|
| 942 |
-
return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}}
|
| 943 |
|
| 944 |
|
| 945 |
@router.put("/admin/keychain/{entry_id}")
|
|
@@ -1089,14 +1135,16 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
|
|
| 1089 |
raise err(503, "store_unavailable", "The delete did not land. Try again.")
|
| 1090 |
if not purges:
|
| 1091 |
_write_revocation_warning(rt, entry_id)
|
|
|
|
|
|
|
| 1092 |
if footprint["rowsKnown"]:
|
| 1093 |
-
return {"ok": True}
|
| 1094 |
# Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes
|
| 1095 |
# that incomplete cleanup explicit instead of returning an invented "nothing to purge".
|
| 1096 |
return {"ok": True, "registry": registered, "footprint": footprint,
|
| 1097 |
"purged": {"tables": [], "rows": 0, "rowsKnown": False,
|
| 1098 |
"refused": [], "failed": []},
|
| 1099 |
-
"residue": _revocation_residue(row["type"])}
|
| 1100 |
from core import user_tables as ut
|
| 1101 |
try:
|
| 1102 |
result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt)
|
|
@@ -1105,11 +1153,14 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
|
|
| 1105 |
result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [],
|
| 1106 |
"failed": [{"key": key, "error": type(exc).__name__} for key in tables]}
|
| 1107 |
_write_revocation_warning(rt, entry_id)
|
|
|
|
|
|
|
|
|
|
| 1108 |
return {"ok": True,
|
| 1109 |
"purged": {"tables": result["purged"], "rows": result["rows"],
|
| 1110 |
"rowsKnown": result["rowsKnown"], "refused": result["refused"],
|
| 1111 |
"failed": result["failed"]},
|
| 1112 |
-
"residue": _revocation_residue(row["type"])}
|
| 1113 |
|
| 1114 |
|
| 1115 |
@router.post("/admin/keychain/{entry_id}/test")
|
|
|
|
| 782 |
|
| 783 |
|
| 784 |
def _revocation_residue(entry_type):
|
| 785 |
+
"""Fallback cleanup instructions when automatic retention could not be scheduled."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 786 |
if entry_type != "amazon_sp":
|
| 787 |
return []
|
| 788 |
return [{
|
| 789 |
"where": "The tenant DuckDB mirror",
|
| 790 |
"why": "Connection removal clears product databases but not the raw Amazon mirror.",
|
| 791 |
+
"fix": "If automatic deletion could not be scheduled, after every Amazon credential is revoked an operator must run "
|
| 792 |
"platform/harness/amazon_store.py --purge --tenant <tenant> "
|
| 793 |
"--confirm-tenant <tenant> and retain the residue-free receipt.",
|
| 794 |
}]
|
| 795 |
|
| 796 |
|
| 797 |
+
def _amazon_lifecycle_warning(session, action):
|
| 798 |
+
"""Access already changed. Warn and audit a lifecycle failure without turning it into 5xx."""
|
| 799 |
+
rt = session.runtime
|
| 800 |
+
removing = action == "revocation"
|
| 801 |
+
warning = ("The Amazon credential was removed, but its automatic data deletion could not be scheduled. "
|
| 802 |
+
"Contact an administrator to arrange deletion within 30 days."
|
| 803 |
+
if removing else
|
| 804 |
+
"The Amazon credential was saved, but its pending data deletion could not be cancelled. "
|
| 805 |
+
"Contact an administrator before the scheduled deletion date.")
|
| 806 |
+
kind = "credential.revoked" if removing else "credential.added"
|
| 807 |
+
try:
|
| 808 |
+
from core import audit
|
| 809 |
+
detail = {"connector": "amazon_sp",
|
| 810 |
+
"deletion_trigger_recorded" if removing else "deletion_cancellation_recorded": False}
|
| 811 |
+
if audit.record(kind, rt.key, session.uname, detail) is None:
|
| 812 |
+
raise RuntimeError("lifecycle audit could not be saved")
|
| 813 |
+
except Exception:
|
| 814 |
+
try:
|
| 815 |
+
from harness import amazon_store as az
|
| 816 |
+
az._append_ledger(rt.key, rt, "audit_failures",
|
| 817 |
+
{"tenant": rt.key, "kind": kind, "checked_at": az._control_stamp()})
|
| 818 |
+
except Exception:
|
| 819 |
+
pass
|
| 820 |
+
warning += " The audit event could not be saved."
|
| 821 |
+
print(f"Amazon lifecycle warning: {warning}")
|
| 822 |
+
return {"warning": warning}
|
| 823 |
+
|
| 824 |
+
|
| 825 |
+
def _amazon_deletion_receipt(session, *, product_purge_succeeded=True):
|
| 826 |
+
try:
|
| 827 |
+
if not product_purge_succeeded:
|
| 828 |
+
raise RuntimeError("product cleanup is incomplete")
|
| 829 |
+
from harness import amazon_store as az
|
| 830 |
+
rt = session.runtime
|
| 831 |
+
az.record_deletion_trigger(rt.key, "revocation", rt=rt, principal=session.uname)
|
| 832 |
+
plan = az.scheduled_purge(rt.key, rt)
|
| 833 |
+
return {"purgeScheduledAt": plan["purgeAt"], "deadlineAt": plan["deadlineAt"],
|
| 834 |
+
"residue": [{"where": "Amazon data in this workspace",
|
| 835 |
+
"why": az.deletion_schedule_message(plan),
|
| 836 |
+
"fix": "Add the Amazon credential again before the scheduled date to cancel deletion."}]}
|
| 837 |
+
except Exception:
|
| 838 |
+
return {**_amazon_lifecycle_warning(session, "revocation"),
|
| 839 |
+
"residue": _revocation_residue("amazon_sp")}
|
| 840 |
+
|
| 841 |
+
|
| 842 |
def _write_revocation_warning(rt, entry_id, warning=None):
|
| 843 |
"""Prune expired consent on every write; failed persistence requires explicit confirmation."""
|
| 844 |
now = int(time.time())
|
|
|
|
| 892 |
@router.post("/admin/keychain", status_code=201)
|
| 893 |
def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)):
|
| 894 |
kc = _kc()
|
| 895 |
+
lifecycle = {}
|
| 896 |
body = body or {}
|
| 897 |
if not session.runtime.available():
|
| 898 |
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
|
|
|
| 962 |
except Exception: # noqa: BLE001
|
| 963 |
pass
|
| 964 |
elif etype == "amazon_sp":
|
| 965 |
+
try:
|
| 966 |
+
from harness import amazon_store as az
|
| 967 |
+
az.cancel_deletion_trigger(session.runtime.key, rt=session.runtime, principal=session.uname)
|
| 968 |
+
except Exception:
|
| 969 |
+
lifecycle = _amazon_lifecycle_warning(session, "cancellation")
|
| 970 |
# D-29 / intake F6: the HTTP thread only records intent. The resident resync thread
|
| 971 |
# remains the mirror's single writer. A queue failure must not undo a stored credential.
|
| 972 |
try:
|
|
|
|
| 985 |
raise err(503, "store_unavailable",
|
| 986 |
"the key was not saved; its sharing setting could not be stored, so "
|
| 987 |
"nothing was kept. Try again.")
|
| 988 |
+
return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}, **lifecycle}
|
| 989 |
|
| 990 |
|
| 991 |
@router.put("/admin/keychain/{entry_id}")
|
|
|
|
| 1135 |
raise err(503, "store_unavailable", "The delete did not land. Try again.")
|
| 1136 |
if not purges:
|
| 1137 |
_write_revocation_warning(rt, entry_id)
|
| 1138 |
+
lifecycle = (_amazon_deletion_receipt(session, product_purge_succeeded=footprint["rowsKnown"])
|
| 1139 |
+
if row["type"] == "amazon_sp" else {})
|
| 1140 |
if footprint["rowsKnown"]:
|
| 1141 |
+
return {"ok": True, **lifecycle}
|
| 1142 |
# Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes
|
| 1143 |
# that incomplete cleanup explicit instead of returning an invented "nothing to purge".
|
| 1144 |
return {"ok": True, "registry": registered, "footprint": footprint,
|
| 1145 |
"purged": {"tables": [], "rows": 0, "rowsKnown": False,
|
| 1146 |
"refused": [], "failed": []},
|
| 1147 |
+
"residue": _revocation_residue(row["type"]), **lifecycle}
|
| 1148 |
from core import user_tables as ut
|
| 1149 |
try:
|
| 1150 |
result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt)
|
|
|
|
| 1153 |
result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [],
|
| 1154 |
"failed": [{"key": key, "error": type(exc).__name__} for key in tables]}
|
| 1155 |
_write_revocation_warning(rt, entry_id)
|
| 1156 |
+
lifecycle = (_amazon_deletion_receipt(
|
| 1157 |
+
session, product_purge_succeeded=not result["failed"] and not result["refused"])
|
| 1158 |
+
if row["type"] == "amazon_sp" else {})
|
| 1159 |
return {"ok": True,
|
| 1160 |
"purged": {"tables": result["purged"], "rows": result["rows"],
|
| 1161 |
"rowsKnown": result["rowsKnown"], "refused": result["refused"],
|
| 1162 |
"failed": result["failed"]},
|
| 1163 |
+
"residue": _revocation_residue(row["type"]), **lifecycle}
|
| 1164 |
|
| 1165 |
|
| 1166 |
@router.post("/admin/keychain/{entry_id}/test")
|
api/routes_module_index.py
CHANGED
|
@@ -1,12 +1,15 @@
|
|
| 1 |
-
"""
|
| 2 |
import copy
|
| 3 |
|
| 4 |
from fastapi import APIRouter, Body, Depends, HTTPException
|
| 5 |
|
| 6 |
from core.sheets import SheetFolderRefusal, validate_folder_tree
|
| 7 |
from deps import Session, err, module_gate, require_session
|
|
|
|
| 8 |
import routes_assets
|
| 9 |
import routes_automation
|
|
|
|
|
|
|
| 10 |
import voice_control_plane as voice
|
| 11 |
|
| 12 |
router = APIRouter(prefix="/api/v1/module-index")
|
|
@@ -17,16 +20,22 @@ def _sentence(value):
|
|
| 17 |
return " ".join(str(value).replace("\u2014", "-").replace("\u2013", "-").split())
|
| 18 |
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
def _refuse(exc):
|
| 21 |
status = {"module_index_unknown": 404, "module_index_user_forbidden": 403,
|
| 22 |
"module_index_item_forbidden": 403, "index_revision_stale": 409}.get(exc.code, 400)
|
| 23 |
-
return err(status, exc.code, _sentence(exc.message))
|
| 24 |
|
| 25 |
|
| 26 |
def _access(module: str, session: Session = Depends(require_session)):
|
| 27 |
-
if module not in {"app", "agents"}:
|
| 28 |
raise _refuse(SheetFolderRefusal(
|
| 29 |
-
"module_index_unknown", f'Folder index "{module}" is not supported; use app or
|
| 30 |
# App has the same session-only door as GET /applications; its rows have their own wall.
|
| 31 |
if module == "agents":
|
| 32 |
try:
|
|
@@ -47,11 +56,28 @@ def _owner(session, user):
|
|
| 47 |
|
| 48 |
|
| 49 |
def _readable_ids(module, session):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
# The projection is hygiene; the per-item list door is the wall. The client renders
|
| 51 |
# only listed items, so an unverifiable placement renders no tile. Use local voice
|
| 52 |
# records, never audience-option assembly, just to verify placement ids.
|
| 53 |
if module == "app":
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
# ββ W62-T17 β THE ID-ONLY ACCESSOR THIS COMMENT USED TO ASK FOR NOW EXISTS. It read
|
| 56 |
# *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until
|
| 57 |
# it has an id-only accessor"*, and the whole Agents rail payload was being built to answer a
|
|
@@ -94,11 +120,41 @@ def _available(session):
|
|
| 94 |
raise err(503, "store_unavailable", "The tenant store is unavailable. Reload folders to retry.")
|
| 95 |
|
| 96 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
@router.get("/{module}")
|
| 98 |
def read_module_index(module: str, user: str = "", session: Session = Depends(_access)):
|
| 99 |
try:
|
| 100 |
owner = _owner(session, user)
|
| 101 |
_available(session)
|
|
|
|
| 102 |
doc = session.runtime.get("module_index_" + module) or {}
|
| 103 |
bucket = (doc.get("users") or {}).get(owner) or {}
|
| 104 |
ids = _readable_ids(module, session) if bucket.get("itemFolders") else set()
|
|
@@ -117,6 +173,7 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
|
|
| 117 |
raise SheetFolderRefusal("module_index_user_forbidden",
|
| 118 |
f'The folder tree for "{owner}" can only be changed by its owner.')
|
| 119 |
_available(session)
|
|
|
|
| 120 |
raw = body or {}
|
| 121 |
expect_rev = raw.get("rev")
|
| 122 |
if type(expect_rev) is not int or expect_rev < 0:
|
|
@@ -141,6 +198,8 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
|
|
| 141 |
raise SheetFolderRefusal("index_revision_stale",
|
| 142 |
"Folders changed somewhere else. Reload folders and try again.")
|
| 143 |
saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
|
|
|
|
|
|
|
| 144 |
if collapsed is not None:
|
| 145 |
saved["collapsedGroups"] = collapsed
|
| 146 |
elif "collapsedGroups" in (users.get(owner) or {}):
|
|
|
|
| 1 |
+
"""Personal App, Agents and Data folder trees in the tenant store."""
|
| 2 |
import copy
|
| 3 |
|
| 4 |
from fastapi import APIRouter, Body, Depends, HTTPException
|
| 5 |
|
| 6 |
from core.sheets import SheetFolderRefusal, validate_folder_tree
|
| 7 |
from deps import Session, err, module_gate, require_session
|
| 8 |
+
import application_registry
|
| 9 |
import routes_assets
|
| 10 |
import routes_automation
|
| 11 |
+
from routes_applications import _may_open
|
| 12 |
+
from routes_nav import _placeable_top_keys
|
| 13 |
import voice_control_plane as voice
|
| 14 |
|
| 15 |
router = APIRouter(prefix="/api/v1/module-index")
|
|
|
|
| 20 |
return " ".join(str(value).replace("\u2014", "-").replace("\u2013", "-").split())
|
| 21 |
|
| 22 |
|
| 23 |
+
# D-3282 (standing rule 1): a limit the server cannot lift is reported with its cause and the fix.
|
| 24 |
+
_REFUSAL_SENTENCES = {
|
| 25 |
+
"too_many_folders": "Folders hold at most 200 per rail. Delete or merge a folder, then try again.",
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
|
| 29 |
def _refuse(exc):
|
| 30 |
status = {"module_index_unknown": 404, "module_index_user_forbidden": 403,
|
| 31 |
"module_index_item_forbidden": 403, "index_revision_stale": 409}.get(exc.code, 400)
|
| 32 |
+
return err(status, exc.code, _sentence(_REFUSAL_SENTENCES.get(exc.code, exc.message)))
|
| 33 |
|
| 34 |
|
| 35 |
def _access(module: str, session: Session = Depends(require_session)):
|
| 36 |
+
if module not in {"app", "agents", "data"}:
|
| 37 |
raise _refuse(SheetFolderRefusal(
|
| 38 |
+
"module_index_unknown", f'Folder index "{module}" is not supported; use app, agents or data.'))
|
| 39 |
# App has the same session-only door as GET /applications; its rows have their own wall.
|
| 40 |
if module == "agents":
|
| 41 |
try:
|
|
|
|
| 56 |
|
| 57 |
|
| 58 |
def _readable_ids(module, session):
|
| 59 |
+
if module == "data":
|
| 60 |
+
keys, enumerated = _placeable_top_keys(session)
|
| 61 |
+
if not enumerated:
|
| 62 |
+
raise err(503, "store_unavailable", "Databases could not be loaded. Reload folders to retry.")
|
| 63 |
+
return keys
|
| 64 |
# The projection is hygiene; the per-item list door is the wall. The client renders
|
| 65 |
# only listed items, so an unverifiable placement renders no tile. Use local voice
|
| 66 |
# records, never audience-option assembly, just to verify placement ids.
|
| 67 |
if module == "app":
|
| 68 |
+
# S67-T38 (O24): the App home also renders the seeded Finance projections, so the wall
|
| 69 |
+
# admits their ids through the SAME grant filter /nav/applications uses. Never ungated:
|
| 70 |
+
# a Finance card the session cannot open is not placeable.
|
| 71 |
+
stored = {row["id"] for row in routes_assets.application_index(session=session)["applications"]}
|
| 72 |
+
finance = application_registry.finance_application_records(
|
| 73 |
+
may_open=lambda permission: _may_open(session, permission))
|
| 74 |
+
# S67-T41 owner correction: a Content Studio App is bound to a USER table, which the
|
| 75 |
+
# topic-only `/applications` index refuses, so its tile arrives from /nav/applications.
|
| 76 |
+
# Its id is admitted through the registry's own `may_read` (the same read that lists it).
|
| 77 |
+
studios = {row["id"] for row in application_registry.list_apps(
|
| 78 |
+
session.runtime, session.uname, session.admin)
|
| 79 |
+
if row.get("templateKind") == application_registry.CONTENT_STUDIO_TEMPLATE_KIND}
|
| 80 |
+
return stored | {row["id"] for row in finance} | studios
|
| 81 |
# ββ W62-T17 β THE ID-ONLY ACCESSOR THIS COMMENT USED TO ASK FOR NOW EXISTS. It read
|
| 82 |
# *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until
|
| 83 |
# it has an id-only accessor"*, and the whole Agents rail payload was being built to answer a
|
|
|
|
| 120 |
raise err(503, "store_unavailable", "The tenant store is unavailable. Reload folders to retry.")
|
| 121 |
|
| 122 |
|
| 123 |
+
def _carry_data_index(module, owner, session):
|
| 124 |
+
"""Copy the legacy bucket once, inside the index transaction; never clean or write nav_prefs.
|
| 125 |
+
|
| 126 |
+
Presence, including an empty bucket, is authoritative. Revision one makes an unread rev-zero
|
| 127 |
+
PUT stale even when it is the first request after upgrade (W61-T19's wholesale-erase guard).
|
| 128 |
+
"""
|
| 129 |
+
if module != "data":
|
| 130 |
+
return
|
| 131 |
+
doc = session.runtime.get("module_index_data") or {}
|
| 132 |
+
if owner in (doc.get("users") or {}):
|
| 133 |
+
return
|
| 134 |
+
legacy = (session.runtime.get("nav_prefs") or {}).get(owner) or {}
|
| 135 |
+
|
| 136 |
+
def carry(current):
|
| 137 |
+
state = copy.deepcopy(current) if isinstance(current, dict) else {}
|
| 138 |
+
users = state.setdefault("users", {})
|
| 139 |
+
if owner not in users:
|
| 140 |
+
users[owner] = {
|
| 141 |
+
"folders": [{**folder, "parent": None} for folder in legacy.get("folders", [])],
|
| 142 |
+
"itemFolders": copy.deepcopy(legacy.get("placement", {})),
|
| 143 |
+
"rev": 1,
|
| 144 |
+
"carriedFrom": "nav_prefs",
|
| 145 |
+
}
|
| 146 |
+
state["schema"] = 1
|
| 147 |
+
return state
|
| 148 |
+
|
| 149 |
+
session.runtime.update("module_index_data", carry, flush="sync")
|
| 150 |
+
|
| 151 |
+
|
| 152 |
@router.get("/{module}")
|
| 153 |
def read_module_index(module: str, user: str = "", session: Session = Depends(_access)):
|
| 154 |
try:
|
| 155 |
owner = _owner(session, user)
|
| 156 |
_available(session)
|
| 157 |
+
_carry_data_index(module, owner, session)
|
| 158 |
doc = session.runtime.get("module_index_" + module) or {}
|
| 159 |
bucket = (doc.get("users") or {}).get(owner) or {}
|
| 160 |
ids = _readable_ids(module, session) if bucket.get("itemFolders") else set()
|
|
|
|
| 173 |
raise SheetFolderRefusal("module_index_user_forbidden",
|
| 174 |
f'The folder tree for "{owner}" can only be changed by its owner.')
|
| 175 |
_available(session)
|
| 176 |
+
_carry_data_index(module, owner, session)
|
| 177 |
raw = body or {}
|
| 178 |
expect_rev = raw.get("rev")
|
| 179 |
if type(expect_rev) is not int or expect_rev < 0:
|
|
|
|
| 198 |
raise SheetFolderRefusal("index_revision_stale",
|
| 199 |
"Folders changed somewhere else. Reload folders and try again.")
|
| 200 |
saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
|
| 201 |
+
if "carriedFrom" in (users.get(owner) or {}):
|
| 202 |
+
saved["carriedFrom"] = users[owner]["carriedFrom"]
|
| 203 |
if collapsed is not None:
|
| 204 |
saved["collapsedGroups"] = collapsed
|
| 205 |
elif "collapsedGroups" in (users.get(owner) or {}):
|
api/routes_nav.py
CHANGED
|
@@ -528,7 +528,7 @@ def _granted_view_ids(session):
|
|
| 528 |
#:
|
| 529 |
#: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
|
| 530 |
#: `dataPath.ts::dataDatabaseOrigin` folds `loopable` / `blank` / `automation` / empty into the
|
| 531 |
-
#: "Loopable" group
|
| 532 |
#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
|
| 533 |
#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
|
| 534 |
_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
|
|
@@ -704,7 +704,7 @@ def nav(session: Session = Depends(require_session)):
|
|
| 704 |
continue
|
| 705 |
pages.append({"key": e["key"], "label": e["label"],
|
| 706 |
"source": e.get("source") or "Blank",
|
| 707 |
-
"sourceGroup": e.get("sourceGroup") or "
|
| 708 |
except Exception:
|
| 709 |
# ββ W31-T11 (owner item 6b) β STILL SWALLOWED, NO LONGER SILENT.
|
| 710 |
#
|
|
@@ -757,7 +757,7 @@ def nav(session: Session = Depends(require_session)):
|
|
| 757 |
if _folder:
|
| 758 |
page["sourceGroup"] = _folder
|
| 759 |
elif page.get("user_table") or source in {"", "Blank", "Custom"}:
|
| 760 |
-
page.setdefault("sourceGroup", "
|
| 761 |
# ββ W35-T43 (rulings R4/R7) β THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
|
| 762 |
#
|
| 763 |
# W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS
|
|
|
|
| 528 |
#:
|
| 529 |
#: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
|
| 530 |
#: `dataPath.ts::dataDatabaseOrigin` folds `loopable` / `blank` / `automation` / empty into the
|
| 531 |
+
#: "Loopable" group; the legacy "Custom" group is reserved too. A connection someone named
|
| 532 |
#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
|
| 533 |
#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
|
| 534 |
_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
|
|
|
|
| 704 |
continue
|
| 705 |
pages.append({"key": e["key"], "label": e["label"],
|
| 706 |
"source": e.get("source") or "Blank",
|
| 707 |
+
"sourceGroup": e.get("sourceGroup") or "Loopable", "chrome": "main"})
|
| 708 |
except Exception:
|
| 709 |
# ββ W31-T11 (owner item 6b) β STILL SWALLOWED, NO LONGER SILENT.
|
| 710 |
#
|
|
|
|
| 757 |
if _folder:
|
| 758 |
page["sourceGroup"] = _folder
|
| 759 |
elif page.get("user_table") or source in {"", "Blank", "Custom"}:
|
| 760 |
+
page.setdefault("sourceGroup", "Loopable")
|
| 761 |
# ββ W35-T43 (rulings R4/R7) β THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
|
| 762 |
#
|
| 763 |
# W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS
|
api/routes_odoo_tables.py
CHANGED
|
@@ -23,6 +23,7 @@ import hashlib
|
|
| 23 |
import hmac
|
| 24 |
import json
|
| 25 |
import math
|
|
|
|
| 26 |
import time
|
| 27 |
from contextlib import contextmanager
|
| 28 |
|
|
@@ -66,7 +67,7 @@ def _rt():
|
|
| 66 |
# takes a mirror cursor with NO tenant gate at all (`is_royal` appears only as an `applicable`
|
| 67 |
# FIELD in its response), so any authenticated tenant reached tenant #0's file and was served row
|
| 68 |
# counts from it. That is D-169 firing in production, and it is what `_mirror_cur` closes.
|
| 69 |
-
def _mirror_cur(session):
|
| 70 |
"""A mirror cursor for THIS session's tenant, or a refusal that says which failure it is.
|
| 71 |
|
| 72 |
Returns `(cursor, mismatch_sentence)` β exactly one of the two is None, so a caller cannot
|
|
@@ -88,11 +89,32 @@ def _mirror_cur(session):
|
|
| 88 |
"""
|
| 89 |
rtm = _rt()
|
| 90 |
try:
|
| 91 |
-
return
|
| 92 |
except rtm.DatastoreMismatch as e:
|
| 93 |
return None, str(e)
|
| 94 |
|
| 95 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
@router.post("/odoo-tables/refresh")
|
| 97 |
def refresh_odoo_tables(session: Session = Depends(require_session)):
|
| 98 |
"""Rebuild all four locked Odoo databases from the tenant's mirror β customers, products,
|
|
@@ -178,10 +200,15 @@ def odoo_tables_status(brief: bool = Query(False),
|
|
| 178 |
rel = _rel()
|
| 179 |
if brief:
|
| 180 |
sources = _sources()
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 185 |
|
| 186 |
import core.user_tables as user_tables
|
| 187 |
# The detailed operator view genuinely needs rows to report counts/freshness, but it needs
|
|
@@ -596,6 +623,28 @@ GRID_SOURCES = {}
|
|
| 596 |
_SOURCE_PROVIDERS = []
|
| 597 |
_SOURCE_ANALYSES = {}
|
| 598 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 599 |
|
| 600 |
def register_source_provider(fn):
|
| 601 |
"""Add a `() -> {table_key: spec}` builder to the connected-grid registry.
|
|
@@ -641,20 +690,60 @@ def _sources():
|
|
| 641 |
β STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and
|
| 642 |
importing that module at file-import time would drag the Odoo layer into every process that
|
| 643 |
mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 644 |
"""
|
| 645 |
-
|
|
|
|
| 646 |
return GRID_SOURCES
|
| 647 |
-
|
|
|
|
|
|
|
|
|
|
| 648 |
try:
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 655 |
GRID_SOURCES.update(extra)
|
| 656 |
_ut().register_connected(*extra)
|
| 657 |
-
return GRID_SOURCES
|
| 658 |
|
| 659 |
|
| 660 |
def _odoo_sources():
|
|
@@ -1014,7 +1103,67 @@ def _odoo_spawned():
|
|
| 1014 |
#: Odoo registers itself, at import, exactly as any other connector does. β The order providers
|
| 1015 |
#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
|
| 1016 |
#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1017 |
register_source_provider(_odoo_sources)
|
|
|
|
| 1018 |
register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned)
|
| 1019 |
|
| 1020 |
|
|
@@ -1243,8 +1392,18 @@ def _source_for(cur, table_key):
|
|
| 1243 |
if module_cache_source(key) is not None:
|
| 1244 |
return None
|
| 1245 |
spec = _sources().get(key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1246 |
if not spec:
|
| 1247 |
return None
|
|
|
|
|
|
|
| 1248 |
from harness import datastore
|
| 1249 |
for opt in (spec.get("optional") or ()):
|
| 1250 |
wants = tuple(opt.get("absent") or ())
|
|
@@ -1271,6 +1430,10 @@ def _source_for(cur, table_key):
|
|
| 1271 |
for alias, tname in aliases.items():
|
| 1272 |
cols = datastore.columns_of(tname, cur=cur)
|
| 1273 |
if not cols:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1274 |
return None # the mirror has no such table on this deployment
|
| 1275 |
have_by_alias[alias] = cols
|
| 1276 |
excluded = ""
|
|
@@ -1624,11 +1787,133 @@ def _related_source_value(session, entry, record):
|
|
| 1624 |
|
| 1625 |
|
| 1626 |
def _materialised_table(table_key):
|
| 1627 |
-
"""Storage dispatch, independent of the browser's fallible status roster.
|
| 1628 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1629 |
|
|
|
|
|
|
|
|
|
|
| 1630 |
|
| 1631 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1632 |
"""R50 / C4: admit an intermediate after the caller has established both end grants.
|
| 1633 |
|
| 1634 |
The session may already open both ends of a declared two-hop link. Its intermediate
|
|
@@ -1636,7 +1921,7 @@ def _traversal_defn_or_refuse(session, table_key):
|
|
| 1636 |
read still applies the row wall, hidden-field closure, BU scope and mirror guard.
|
| 1637 |
"""
|
| 1638 |
ut = _ut()
|
| 1639 |
-
st = ut.lend_defs(session.runtime)
|
| 1640 |
defn = ut.get(table_key, st=st)
|
| 1641 |
if not defn:
|
| 1642 |
raise err(404, "unknown_table", "that database does not exist")
|
|
@@ -1644,21 +1929,26 @@ def _traversal_defn_or_refuse(session, table_key):
|
|
| 1644 |
|
| 1645 |
|
| 1646 |
def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
|
| 1647 |
-
traversal_of=None):
|
| 1648 |
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
|
| 1649 |
|
| 1650 |
Join columns may be hidden on the wire. Only identities leave the resolver; the native
|
| 1651 |
payload builders still apply their complete hidden-field closure before returning cells.
|
| 1652 |
With traversal_of, the caller must already have established both ends of the declared
|
| 1653 |
edge named by that key. Only join values leave this intermediate read, never a cell.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1654 |
"""
|
| 1655 |
import core.perm_scope as perm_scope
|
| 1656 |
import core.relations as relations
|
| 1657 |
|
| 1658 |
columns = set(columns) | {relations.IDENTITY_COL}
|
| 1659 |
if not _materialised_table(table):
|
| 1660 |
-
read = (_compiled_read(table, session, traversal_intermediate=True)
|
| 1661 |
-
if traversal_of is not None else _compiled_read(table, session))
|
| 1662 |
spec = read["spec"]
|
| 1663 |
bindings = dict(spec["cols"], pid=(spec["id"], int))
|
| 1664 |
if columns - set(bindings):
|
|
@@ -1681,11 +1971,16 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
|
|
| 1681 |
|
| 1682 |
# This is an internal join, never a cell response. Apply the same permanent wall on the
|
| 1683 |
# complete contract before taking any join value, including a join on a hidden column.
|
| 1684 |
-
if traversal_of is None and not perm_scope.may_read(
|
|
|
|
| 1685 |
raise err(403, "forbidden_table", "This account cannot open the linked database.")
|
| 1686 |
if module_cache_source(table) is not None:
|
| 1687 |
session.require(table)
|
| 1688 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1689 |
rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime)
|
| 1690 |
if where:
|
| 1691 |
raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.")
|
|
@@ -1730,6 +2025,11 @@ def _relation_leg_ids(session, source, source_ids, entry):
|
|
| 1730 |
# identity. Treating a missing/hidden/blank source value as an empty match would
|
| 1731 |
# turn "the join input is unavailable" into the business claim "no related rows".
|
| 1732 |
if source_col != relations.IDENTITY_COL and not values:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1733 |
raise err(400, "unknown_relation", relations.unresolvable_source_refusal(source, entry))
|
| 1734 |
if (manual and not reverse) or (inverse and reverse):
|
| 1735 |
values = set().union(*(_relation_cell_ids(row.get(source_col)) for row in source_rows))
|
|
@@ -1794,6 +2094,10 @@ def _relation_target_ids(session, target, related):
|
|
| 1794 |
if not _may_open(session, leg["target"]):
|
| 1795 |
raise err(403, "forbidden_table",
|
| 1796 |
f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1797 |
ids = _relation_leg_ids(session, table, ids, leg)
|
| 1798 |
return ids
|
| 1799 |
|
|
@@ -2464,7 +2768,8 @@ def _may_see_view(session, vid, store):
|
|
| 2464 |
|
| 2465 |
|
| 2466 |
def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
|
| 2467 |
-
search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False
|
|
|
|
| 2468 |
"""EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
|
| 2469 |
|
| 2470 |
Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits,
|
|
@@ -2524,17 +2829,25 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 2524 |
# tenant #0's eight databases is read-through. Left un-applied, an administrator could set a
|
| 2525 |
# filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows
|
| 2526 |
# used to be. So the wall is APPLIED here, in SQL, three paragraphs down.
|
|
|
|
|
|
|
| 2527 |
if traversal_intermediate:
|
| 2528 |
-
defn = _traversal_defn_or_refuse(session, table_key)
|
| 2529 |
else:
|
| 2530 |
-
defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
|
| 2531 |
fields = None
|
| 2532 |
for _bucket, key, _label, mk_fields in rel.TABLES:
|
| 2533 |
if key == table_key:
|
| 2534 |
fields = mk_fields()
|
| 2535 |
break
|
| 2536 |
if fields is None:
|
| 2537 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2538 |
|
| 2539 |
# ββ W31-T45 / D-169 β THE SECOND WALL, and it asks a question `_defn_or_refuse` above cannot.
|
| 2540 |
# That wall asks "may this SESSION open this DATABASE"; this one asks "is the file this process
|
|
@@ -2549,7 +2862,8 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 2549 |
# turn the loudest refusal in the system into a spinner. The guard therefore runs ABOVE the
|
| 2550 |
# try, not inside it.
|
| 2551 |
try:
|
| 2552 |
-
cur, mismatch = _mirror_cur(session)
|
|
|
|
| 2553 |
except RuntimeError as e:
|
| 2554 |
# β Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the
|
| 2555 |
# mismatch subclass, which is what makes this broad clause safe to keep here.
|
|
@@ -2787,7 +3101,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 2787 |
|
| 2788 |
|
| 2789 |
@contextmanager
|
| 2790 |
-
def _materialised_read(table_key, session, query, related):
|
| 2791 |
"""One compiler for native rows and their aggregate sibling, over the walled payload.
|
| 2792 |
|
| 2793 |
The temporary connection contains only this request's already stripped, pinned rows and is
|
|
@@ -2806,6 +3120,10 @@ def _materialised_read(table_key, session, query, related):
|
|
| 2806 |
else:
|
| 2807 |
from routes_tables import table_rows
|
| 2808 |
payload = table_rows(table_key, session=session, related=related)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2809 |
fields = payload["fields"]
|
| 2810 |
with duckdb.connect(":memory:") as cur:
|
| 2811 |
cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)",
|
|
@@ -2882,13 +3200,38 @@ def _group_pin(cols, group_by, group_key):
|
|
| 2882 |
return f"({gexpr}) = ?", [str(key)]
|
| 2883 |
|
| 2884 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2885 |
def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
|
| 2886 |
-
groupBy=None, groupKey=None):
|
| 2887 |
from harness import datastore
|
| 2888 |
|
| 2889 |
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
|
| 2890 |
"sorts": _json_arg(sorts, "sorts"), "search": search})
|
| 2891 |
-
with _materialised_read(table_key, session, query, related) as read:
|
| 2892 |
spec = read["spec"]
|
| 2893 |
order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
|
| 2894 |
where, params = read["where"], list(read["params"])
|
|
@@ -2912,6 +3255,7 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
|
| 2912 |
|
| 2913 |
|
| 2914 |
@router.get("/odoo-tables/{table_key}/rows")
|
|
|
|
| 2915 |
def odoo_table_rows(table_key: str,
|
| 2916 |
offset: int = Query(default=0, ge=0),
|
| 2917 |
limit: int = Query(default=0),
|
|
@@ -2922,7 +3266,8 @@ def odoo_table_rows(table_key: str,
|
|
| 2922 |
related: str = Query(default=None),
|
| 2923 |
groupBy: str = Query(default=None),
|
| 2924 |
groupKey: str = Query(default=None),
|
| 2925 |
-
session: Session = Depends(require_session)
|
|
|
|
| 2926 |
"""ONE WINDOW over a connected grid, read straight from the mirror (contract C2).
|
| 2927 |
|
| 2928 |
`{fields, rows, total, totalUnfiltered, offset, limit, limits, identity, recordsMutable}` β
|
|
@@ -2940,19 +3285,36 @@ def odoo_table_rows(table_key: str,
|
|
| 2940 |
argument over: `?related={"table":"customer_data","record":1234,"via":{...}}`. It narrows this
|
| 2941 |
window to the rows linked to that ONE record, ahead of `filters`, and `total` /
|
| 2942 |
`totalUnfiltered` count inside the pin rather than around it.
|
| 2943 |
-
|
| 2944 |
-
|
| 2945 |
-
|
| 2946 |
-
|
| 2947 |
-
|
|
|
|
| 2948 |
"""
|
| 2949 |
import aios_grid
|
| 2950 |
import core.perm_scope as perm_scope
|
| 2951 |
from harness import datastore
|
| 2952 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2953 |
if _materialised_table(table_key):
|
| 2954 |
return _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
| 2955 |
-
sorts, search, related, groupBy, groupKey)
|
| 2956 |
|
| 2957 |
# ββ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
|
| 2958 |
# hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
|
|
@@ -2960,8 +3322,12 @@ def odoo_table_rows(table_key: str,
|
|
| 2960 |
# is a page of* - and that is provable only while there is one of it. What stays here is what
|
| 2961 |
# a WINDOW alone needs: the total order, the projection, the overlay merge and the two
|
| 2962 |
# sentences a page earns (`window_clamped`, the deep-page walk).
|
|
|
|
|
|
|
|
|
|
| 2963 |
read = _compiled_read(table_key, session, filters=filters, filterConj=filterConj,
|
| 2964 |
-
sorts=sorts, search=search, related=related)
|
|
|
|
| 2965 |
defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"]
|
| 2966 |
cols, limits, frm = read["cols"], read["limits"], read["frm"]
|
| 2967 |
base_where, wall_params = read["base_where"], read["wall_params"]
|
|
@@ -3010,6 +3376,40 @@ def odoo_table_rows(table_key: str,
|
|
| 3010 |
rows_src = _pool_from_window(spec, win,
|
| 3011 |
stamp=mirror_stamp(spec, cur, table_key, defn=defn, rt=
|
| 3012 |
session.runtime))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3013 |
overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
|
| 3014 |
|
| 3015 |
# ββ W30-T28 β THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
|
|
@@ -3025,11 +3425,14 @@ def odoo_table_rows(table_key: str,
|
|
| 3025 |
# the wall and the predicate, so the argument it demands is the list we just fetched.
|
| 3026 |
# β It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already
|
| 3027 |
# answered "may this session open this surface".
|
| 3028 |
-
|
|
|
|
|
|
|
|
|
|
| 3029 |
if shared_defs:
|
| 3030 |
fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()]
|
| 3031 |
for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src],
|
| 3032 |
-
st=
|
| 3033 |
overlays.setdefault(pid, {}).update(cells)
|
| 3034 |
# RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
|
| 3035 |
# ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
|
|
@@ -3335,6 +3738,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 3335 |
# hand: `filters` a list, `filterConj` in and/or, `search` text under 1,000 characters. A
|
| 3336 |
# second spelling of "what a predicate on the wire looks like" is how two of them come apart.
|
| 3337 |
query = _selection_query(payload)
|
|
|
|
| 3338 |
aggs = payload.get("aggs")
|
| 3339 |
if aggs is None:
|
| 3340 |
aggs = []
|
|
@@ -3348,7 +3752,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 3348 |
raise err(400, "bad_selection", "groupBy must be a column id")
|
| 3349 |
|
| 3350 |
if _materialised_table(table_key):
|
| 3351 |
-
with _materialised_read(table_key, session, query, payload.get("related")) as read:
|
| 3352 |
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
|
| 3353 |
read["params"], read["frm"], aggs, group_by=group_by,
|
| 3354 |
limits=read["limits"])
|
|
@@ -3359,9 +3763,12 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 3359 |
# who assumed both were bodies would have wired the rows and left the summaries answering over
|
| 3360 |
# the entire database under a tab that names one record. `_compiled_read` accepts either shape,
|
| 3361 |
# so the two doors cannot pin differently without this argument being deleted.
|
|
|
|
|
|
|
| 3362 |
read = _compiled_read(table_key, session, tree=query["filters"],
|
| 3363 |
filterConj=query["filterConj"], search=query["search"],
|
| 3364 |
-
related=payload.get("related"))
|
|
|
|
| 3365 |
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
|
| 3366 |
read["params"], read["frm"], aggs, group_by=group_by,
|
| 3367 |
limits=read["limits"])
|
|
@@ -3487,7 +3894,7 @@ def population(table_key, cur=None, rt=None):
|
|
| 3487 |
from harness import datastore
|
| 3488 |
if rt is not None:
|
| 3489 |
rt.assert_datastore_matches() # W31-T45: the same wall the row doors carry
|
| 3490 |
-
cur = cur if cur is not None else
|
| 3491 |
spec = _source_for(cur, table_key)
|
| 3492 |
if spec is None:
|
| 3493 |
return None
|
|
@@ -3520,9 +3927,9 @@ def whole_pool(table_key, cur=None, rt=None):
|
|
| 3520 |
"""
|
| 3521 |
from harness import datastore
|
| 3522 |
if rt is not None:
|
| 3523 |
-
cur = cur if cur is not None else
|
| 3524 |
rt.assert_datastore_matches() # also covers a cursor the CALLER opened and lent
|
| 3525 |
-
cur = cur if cur is not None else
|
| 3526 |
spec = _source_for(cur, table_key)
|
| 3527 |
if spec is None:
|
| 3528 |
raise TooBigToMaterialise(
|
|
|
|
| 23 |
import hmac
|
| 24 |
import json
|
| 25 |
import math
|
| 26 |
+
import threading
|
| 27 |
import time
|
| 28 |
from contextlib import contextmanager
|
| 29 |
|
|
|
|
| 67 |
# takes a mirror cursor with NO tenant gate at all (`is_royal` appears only as an `applicable`
|
| 68 |
# FIELD in its response), so any authenticated tenant reached tenant #0's file and was served row
|
| 69 |
# counts from it. That is D-169 firing in production, and it is what `_mirror_cur` closes.
|
| 70 |
+
def _mirror_cur(session, table_key=None):
|
| 71 |
"""A mirror cursor for THIS session's tenant, or a refusal that says which failure it is.
|
| 72 |
|
| 73 |
Returns `(cursor, mismatch_sentence)` β exactly one of the two is None, so a caller cannot
|
|
|
|
| 89 |
"""
|
| 90 |
rtm = _rt()
|
| 91 |
try:
|
| 92 |
+
return _table_cursor(session.runtime, table_key), None
|
| 93 |
except rtm.DatastoreMismatch as e:
|
| 94 |
return None, str(e)
|
| 95 |
|
| 96 |
|
| 97 |
+
def _table_cursor(rt, table_key=None):
|
| 98 |
+
"""THE one place a mirror cursor is chosen FOR A TABLE: tenant assertion first, then the gate.
|
| 99 |
+
|
| 100 |
+
Amazon sync is independent of Odoo's `ready()` phases, so a `ut_amzn_*` grid asserts the same
|
| 101 |
+
tenant and then opens the loader's local mirror without that Odoo gate; every other table keeps
|
| 102 |
+
`mirror_cursor`, which refuses until the Odoo backfill is live (partial totals are worse than a
|
| 103 |
+
retry).
|
| 104 |
+
β HOTFIX 2026-09-27: this rule lived only in `_mirror_cur`, so the window route served Amazon
|
| 105 |
+
rows while `whole_pool` (every `GET /tables/<key>/rows` and the workspace envelope of a
|
| 106 |
+
read-through grid) still answered `503 store_not_ready` on a staging mirror with 0 of 8 Odoo
|
| 107 |
+
sources ready. Every door now asks here, so the rule cannot hold in one door and not another.
|
| 108 |
+
`rt=None` is the unguarded legacy caller `whole_pool`/`population` already documents.
|
| 109 |
+
"""
|
| 110 |
+
from harness import datastore
|
| 111 |
+
if str(table_key or "").startswith("ut_amzn_"):
|
| 112 |
+
if rt is not None:
|
| 113 |
+
rt.assert_datastore_matches()
|
| 114 |
+
return datastore.ro_cursor()
|
| 115 |
+
return _rt().mirror_cursor(rt) if rt is not None else datastore.ro_con()
|
| 116 |
+
|
| 117 |
+
|
| 118 |
@router.post("/odoo-tables/refresh")
|
| 119 |
def refresh_odoo_tables(session: Session = Depends(require_session)):
|
| 120 |
"""Rebuild all four locked Odoo databases from the tenant's mirror β customers, products,
|
|
|
|
| 200 |
rel = _rel()
|
| 201 |
if brief:
|
| 202 |
sources = _sources()
|
| 203 |
+
tables = {key: {"readThrough": key in sources}
|
| 204 |
+
for _bucket, key, _label, _fields in rel.TABLES}
|
| 205 |
+
# S66-T43 (lead, observed 2026-09-25): the grid asks THIS answer which tables read
|
| 206 |
+
# through, and it listed Odoo's roster only, so an Amazon read-through table opened on the
|
| 207 |
+
# ordinary door and painted its empty stub. Every Amazon key the registered provider
|
| 208 |
+
# serves is read-through; none is listed when this container's mirror holds no Amazon rows.
|
| 209 |
+
tables.update({key: {"readThrough": True} for key in sources
|
| 210 |
+
if str(key).startswith("ut_amzn_") and key not in tables})
|
| 211 |
+
return {"ok": True, "tables": tables}
|
| 212 |
|
| 213 |
import core.user_tables as user_tables
|
| 214 |
# The detailed operator view genuinely needs rows to report counts/freshness, but it needs
|
|
|
|
| 623 |
_SOURCE_PROVIDERS = []
|
| 624 |
_SOURCE_ANALYSES = {}
|
| 625 |
|
| 626 |
+
#: S67-T28: `GRID_SOURCES` is PUBLISHED WHOLE, never seen half built. The first build asks every
|
| 627 |
+
#: provider in turn, and the Amazon one runs count(*) queries; the old loop added each provider's
|
| 628 |
+
#: keys to the shared dict as it went, and returned the dict to any caller that found it
|
| 629 |
+
#: non-empty. So a concurrent request saw Odoo's keys with no Amazon key, answered
|
| 630 |
+
#: `_materialised_table("ut_amzn_orders") == True`, and a 559,363-row read-through target was read
|
| 631 |
+
#: as a stored table: a first-page 500 (lane D, 2026-09-28). Now a build fills a LOCAL dict under
|
| 632 |
+
#: this lock and publishes it in one `dict.update`; a reader waits on the lock until
|
| 633 |
+
#: `_SOURCES_BUILT` says a whole build is visible. An admission (`require_amazon_mirror`, the
|
| 634 |
+
#: Amazon fallback) happens only after a build has been published, so it can never stand in
|
| 635 |
+
#: for one either.
|
| 636 |
+
_SOURCES_LOCK = threading.RLock()
|
| 637 |
+
_SOURCES_BUILT = False
|
| 638 |
+
_SOURCES_BUILDER = None # the thread inside a build, so a re-entrant call cannot recurse
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
def invalidate_sources():
|
| 642 |
+
"""Forget measured bindings after a connector publishes a new mirror schema."""
|
| 643 |
+
global _SOURCES_BUILT
|
| 644 |
+
with _SOURCES_LOCK:
|
| 645 |
+
_SOURCES_BUILT = False
|
| 646 |
+
GRID_SOURCES.clear()
|
| 647 |
+
|
| 648 |
|
| 649 |
def register_source_provider(fn):
|
| 650 |
"""Add a `() -> {table_key: spec}` builder to the connected-grid registry.
|
|
|
|
| 690 |
β STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and
|
| 691 |
importing that module at file-import time would drag the Odoo layer into every process that
|
| 692 |
mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
|
| 693 |
+
|
| 694 |
+
β S67-T28: built into a LOCAL dict under `_SOURCES_LOCK` and published in one step (see the
|
| 695 |
+
lock's comment). An empty registry still means "build" (gates clear it to force a rebuild),
|
| 696 |
+
and so does one that holds only admissions made since the last invalidation.
|
| 697 |
"""
|
| 698 |
+
global _SOURCES_BUILT, _SOURCES_BUILDER
|
| 699 |
+
if _SOURCES_BUILT and GRID_SOURCES:
|
| 700 |
return GRID_SOURCES
|
| 701 |
+
with _SOURCES_LOCK:
|
| 702 |
+
if (_SOURCES_BUILT and GRID_SOURCES) or _SOURCES_BUILDER == threading.get_ident():
|
| 703 |
+
return GRID_SOURCES
|
| 704 |
+
_SOURCES_BUILDER = threading.get_ident()
|
| 705 |
try:
|
| 706 |
+
built = {}
|
| 707 |
+
for build in list(_SOURCE_PROVIDERS):
|
| 708 |
+
try:
|
| 709 |
+
extra = build() or {}
|
| 710 |
+
except Exception: # noqa: BLE001
|
| 711 |
+
# β ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo
|
| 712 |
+
# grids must not go dark because a newer connector's module is unhappy, and
|
| 713 |
+
# vice versa.
|
| 714 |
+
continue
|
| 715 |
+
if extra:
|
| 716 |
+
built.update(extra)
|
| 717 |
+
_ut().register_connected(*extra)
|
| 718 |
+
# A key admitted since the last invalidation was measured at least as recently as
|
| 719 |
+
# this build, so it stands; everything else comes from the build. ONE update, so a
|
| 720 |
+
# lock-free reader of the dict sees the old entry or the new one, never a gap.
|
| 721 |
+
GRID_SOURCES.update({**built, **GRID_SOURCES})
|
| 722 |
+
_SOURCES_BUILT = True
|
| 723 |
+
finally:
|
| 724 |
+
_SOURCES_BUILDER = None
|
| 725 |
+
return GRID_SOURCES
|
| 726 |
+
|
| 727 |
+
|
| 728 |
+
def _learn_amazon_sources():
|
| 729 |
+
"""Publish the Amazon provider's keys into a registry that was built without any of them.
|
| 730 |
+
|
| 731 |
+
HOTFIX 2026-09-27 (moved here by S67-T28 so every door shares it): a registry built at boot
|
| 732 |
+
BEFORE the Amazon mirror restore holds no Amazon key, and only `refresh_state` invalidates
|
| 733 |
+
it. `_source_for` learned them on demand; `_materialised_table` did not, so a read-through
|
| 734 |
+
Amazon LINK TARGET (orders, 559k rows) was dispatched as a stored table and refused as more
|
| 735 |
+
than one window. A registry that already knows one Amazon key is not re-asked.
|
| 736 |
+
"""
|
| 737 |
+
_sources()
|
| 738 |
+
if any(k.startswith("ut_amzn_") for k in list(GRID_SOURCES)):
|
| 739 |
+
return
|
| 740 |
+
extra = _amazon_sources()
|
| 741 |
+
if not extra:
|
| 742 |
+
return
|
| 743 |
+
with _SOURCES_LOCK:
|
| 744 |
+
if not any(k.startswith("ut_amzn_") for k in list(GRID_SOURCES)):
|
| 745 |
GRID_SOURCES.update(extra)
|
| 746 |
_ut().register_connected(*extra)
|
|
|
|
| 747 |
|
| 748 |
|
| 749 |
def _odoo_sources():
|
|
|
|
| 1103 |
#: Odoo registers itself, at import, exactly as any other connector does. β The order providers
|
| 1104 |
#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
|
| 1105 |
#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
|
| 1106 |
+
def _amazon_sources():
|
| 1107 |
+
try:
|
| 1108 |
+
import amazon_relational
|
| 1109 |
+
return amazon_relational.grid_sources()
|
| 1110 |
+
except Exception:
|
| 1111 |
+
import logging
|
| 1112 |
+
logging.getLogger(__name__).exception("Amazon grid sources could not be loaded")
|
| 1113 |
+
return {}
|
| 1114 |
+
|
| 1115 |
+
|
| 1116 |
+
def require_amazon_mirror(table_key, session, definition=None, st=None):
|
| 1117 |
+
"""Refuse a persisted Amazon stub whose mirror has not loaded, even on a cold registry.
|
| 1118 |
+
|
| 1119 |
+
The provider contributes no keys on an Odoo-only mirror. A persisted read-through
|
| 1120 |
+
definition still must refuse there instead of falling through to its empty stored rows.
|
| 1121 |
+
S67-T28: `st` is a definitions snapshot the caller already holds (`_page_defs`), so the
|
| 1122 |
+
window route stops paying a second definitions projection for this one check.
|
| 1123 |
+
"""
|
| 1124 |
+
if not str(table_key).startswith("ut_amzn_"):
|
| 1125 |
+
return
|
| 1126 |
+
import amazon_relational as amazon
|
| 1127 |
+
from routes_tables import _defn_or_refuse
|
| 1128 |
+
|
| 1129 |
+
if table_key not in amazon.SOURCE:
|
| 1130 |
+
return
|
| 1131 |
+
if definition is None:
|
| 1132 |
+
definition = _defn_or_refuse(session, table_key, st=st, defs_only=True,
|
| 1133 |
+
scope_applied=True)
|
| 1134 |
+
try:
|
| 1135 |
+
cur, mismatch = _mirror_cur(session, table_key)
|
| 1136 |
+
except RuntimeError as exc:
|
| 1137 |
+
raise err(503, "store_not_ready", str(exc))
|
| 1138 |
+
if mismatch:
|
| 1139 |
+
raise err(409, "cross_tenant_store", mismatch)
|
| 1140 |
+
try:
|
| 1141 |
+
spec = _sources().get(table_key)
|
| 1142 |
+
if (spec is not None or definition.get("readThrough")
|
| 1143 |
+
or table_key in amazon.READ_THROUGH):
|
| 1144 |
+
# Check current table presence even if a binding was cached before a restart.
|
| 1145 |
+
current = amazon.grid_source(cur, table_key)
|
| 1146 |
+
if current.get("missing"):
|
| 1147 |
+
raise err(503, "amazon_mirror_not_loaded", current["missing"])
|
| 1148 |
+
# A dynamically oversized snapshot may have a durable stub before this process
|
| 1149 |
+
# has measured it. Admit its binding without changing another source's registry.
|
| 1150 |
+
# β S67-T28: ONLY INTO A PUBLISHED BUILD, and never as the registry's only Amazon
|
| 1151 |
+
# key. Admitted into an empty or Amazon-less registry, this one key used to make
|
| 1152 |
+
# `_sources()` skip its build, so the grid resolved while its link targets
|
| 1153 |
+
# (orders, read-through) were dispatched as stored tables: a first-page 500.
|
| 1154 |
+
if spec is None or spec.get("missing"):
|
| 1155 |
+
_learn_amazon_sources()
|
| 1156 |
+
with _SOURCES_LOCK:
|
| 1157 |
+
_sources()
|
| 1158 |
+
held = GRID_SOURCES.get(table_key)
|
| 1159 |
+
if held is None or held.get("missing"):
|
| 1160 |
+
GRID_SOURCES[table_key] = current
|
| 1161 |
+
finally:
|
| 1162 |
+
cur.close()
|
| 1163 |
+
|
| 1164 |
+
|
| 1165 |
register_source_provider(_odoo_sources)
|
| 1166 |
+
register_source_provider(_amazon_sources)
|
| 1167 |
register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned)
|
| 1168 |
|
| 1169 |
|
|
|
|
| 1392 |
if module_cache_source(key) is not None:
|
| 1393 |
return None
|
| 1394 |
spec = _sources().get(key)
|
| 1395 |
+
if not spec and key.startswith("ut_amzn_"):
|
| 1396 |
+
# HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no
|
| 1397 |
+
# Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync
|
| 1398 |
+
# runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every
|
| 1399 |
+
# read-through Amazon grid answered "no read-through binding". Ask the Amazon provider
|
| 1400 |
+
# alone, once it can answer; a registry that already knows one Amazon key is not re-asked.
|
| 1401 |
+
_learn_amazon_sources()
|
| 1402 |
+
spec = GRID_SOURCES.get(key)
|
| 1403 |
if not spec:
|
| 1404 |
return None
|
| 1405 |
+
if spec.get("missing"):
|
| 1406 |
+
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
| 1407 |
from harness import datastore
|
| 1408 |
for opt in (spec.get("optional") or ()):
|
| 1409 |
wants = tuple(opt.get("absent") or ())
|
|
|
|
| 1430 |
for alias, tname in aliases.items():
|
| 1431 |
cols = datastore.columns_of(tname, cur=cur)
|
| 1432 |
if not cols:
|
| 1433 |
+
if key.startswith("ut_amzn_"):
|
| 1434 |
+
import amazon_relational
|
| 1435 |
+
raise err(503, "amazon_mirror_not_loaded",
|
| 1436 |
+
amazon_relational.missing_source(key)["missing"])
|
| 1437 |
return None # the mirror has no such table on this deployment
|
| 1438 |
have_by_alias[alias] = cols
|
| 1439 |
excluded = ""
|
|
|
|
| 1787 |
|
| 1788 |
|
| 1789 |
def _materialised_table(table_key):
|
| 1790 |
+
"""Storage dispatch, independent of the browser's fallible status roster.
|
| 1791 |
+
|
| 1792 |
+
β S67-T28: an Amazon key the registry does not hold is asked of the Amazon provider first
|
| 1793 |
+
(`_learn_amazon_sources`, the same fallback `_source_for` takes), so a registry built before
|
| 1794 |
+
the Amazon mirror loaded cannot dispatch a read-through Amazon table as a stored one.
|
| 1795 |
+
"""
|
| 1796 |
+
if module_cache_source(table_key) is not None:
|
| 1797 |
+
return True
|
| 1798 |
+
if table_key in _sources():
|
| 1799 |
+
return False
|
| 1800 |
+
if str(table_key).startswith("ut_amzn_"):
|
| 1801 |
+
_learn_amazon_sources()
|
| 1802 |
+
return table_key not in GRID_SOURCES
|
| 1803 |
+
return True
|
| 1804 |
+
|
| 1805 |
+
|
| 1806 |
+
#: β S67-T28 (R38, D-3480 point 1) β the tenant's table DEFINITIONS and a stored link target's
|
| 1807 |
+
#: ROWS, each held against the durable `user_tables` revision it was read at. A grouped Amazon page
|
| 1808 |
+
#: used to re-read the definitions projection once for its own wall and once more for EVERY link
|
| 1809 |
+
#: target, plus the stored listings entry, on every page; on staging each of those reads is one
|
| 1810 |
+
#: large JSONB projection (measured 33 to 186 s a page under load, 2026-09-28). Every `put` and
|
| 1811 |
+
#: `update` bumps the revision column in the same statement, so an unchanged token proves the
|
| 1812 |
+
#: bytes unchanged; the token is read BEFORE the value, so a write racing the read can only cache
|
| 1813 |
+
#: newer bytes under the older token (a miss next time, never a stale hit). β Nothing cached here
|
| 1814 |
+
#: is a permission: the wall (`may_read`, `apply_row_scope`, the compiled predicate) is evaluated
|
| 1815 |
+
#: per request against the session's own user, only the bytes it evaluates over are reused.
|
| 1816 |
+
_REV_MEMO: dict = {}
|
| 1817 |
+
_REV_MEMO_MAX = 64
|
| 1818 |
+
# Route handlers run on a thread pool: the lock makes the evict (`next(iter(...))`) and the
|
| 1819 |
+
# insert one step, so a concurrent insert can never raise "dictionary changed size".
|
| 1820 |
+
_REV_MEMO_LOCK = threading.Lock()
|
| 1821 |
+
#: "No token was handed in, read one." Distinct from None, which means "unreadable, hold nothing".
|
| 1822 |
+
#: It is the `...` literal (a singleton) so a signature can spell its default `...` and still
|
| 1823 |
+
#: define when `verify_grid_ux.py` execs `_relation_rows` on its own, without this module name.
|
| 1824 |
+
_UNREAD = ...
|
| 1825 |
+
|
| 1826 |
+
|
| 1827 |
+
def _user_tables_token(rt):
|
| 1828 |
+
"""The durable change token of this tenant's `user_tables` bucket, or None when unreadable."""
|
| 1829 |
+
try:
|
| 1830 |
+
return rt.revision(_ut().STORE_KEY)["token"]
|
| 1831 |
+
except Exception: # noqa: BLE001
|
| 1832 |
+
return None
|
| 1833 |
+
|
| 1834 |
+
|
| 1835 |
+
def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
|
| 1836 |
+
"""`read()`, reused while this tenant's `user_tables` revision is unchanged.
|
| 1837 |
+
|
| 1838 |
+
β `clone` runs on the way in AND on every hit, so a caller that edits what it was handed
|
| 1839 |
+
cannot edit the next request's answer. `keep(value)` decides what may be held: an empty
|
| 1840 |
+
answer never is (it is the shape a failed read takes, and caching it would make one outage
|
| 1841 |
+
outlive itself), and neither is the whole-document fallback `all_defs` takes on a failure.
|
| 1842 |
+
`token` is one this REQUEST already read (`_page_defs`), so a page pays one revision round
|
| 1843 |
+
trip, not one per memo. It was read before any value this call reads, which is the only
|
| 1844 |
+
ordering the staleness argument above needs.
|
| 1845 |
+
"""
|
| 1846 |
+
if token is _UNREAD:
|
| 1847 |
+
token = _user_tables_token(rt)
|
| 1848 |
+
tenant = getattr(rt, "key", None)
|
| 1849 |
+
slot = (kind, tenant, name)
|
| 1850 |
+
if token is not None and tenant:
|
| 1851 |
+
with _REV_MEMO_LOCK:
|
| 1852 |
+
hit = _REV_MEMO.get(slot)
|
| 1853 |
+
if hit is not None and hit[0] == token:
|
| 1854 |
+
return clone(hit[1])
|
| 1855 |
+
value = read()
|
| 1856 |
+
if token is not None and tenant and keep(value):
|
| 1857 |
+
held = (token, clone(value))
|
| 1858 |
+
with _REV_MEMO_LOCK:
|
| 1859 |
+
if slot not in _REV_MEMO and len(_REV_MEMO) >= _REV_MEMO_MAX:
|
| 1860 |
+
_REV_MEMO.pop(next(iter(_REV_MEMO)), None)
|
| 1861 |
+
_REV_MEMO[slot] = held
|
| 1862 |
+
return value
|
| 1863 |
+
|
| 1864 |
+
|
| 1865 |
+
def _page_defs(session):
|
| 1866 |
+
"""ONE lent definitions snapshot for a whole `/rows` page: its own wall and every link target's.
|
| 1867 |
+
|
| 1868 |
+
Returns `(st, token)`: the lent snapshot, and the `user_tables` revision it was checked
|
| 1869 |
+
against (None when unreadable), which the page hands to `_stored_link_rows` so a stored link
|
| 1870 |
+
target does not pay a second revision round trip.
|
| 1871 |
+
The same `lend_defs` the table wall takes for itself (`_defn_or_refuse`, D-213), read once per
|
| 1872 |
+
`user_tables` revision instead of once per question. Kept only when every definition really is
|
| 1873 |
+
a projection (`store.is_projected`), because `all_defs` silently falls back to the whole
|
| 1874 |
+
28 MB document on a failed read and that must never be held, let alone copied per request.
|
| 1875 |
+
β The snapshot is a fresh `_Lent` per call, so the `object_shares` and `__shared` buckets it
|
| 1876 |
+
lends lazily live for this one request and are never carried to the next.
|
| 1877 |
+
"""
|
| 1878 |
+
import copy
|
| 1879 |
+
import core.store as store
|
| 1880 |
+
|
| 1881 |
+
ut = _ut()
|
| 1882 |
+
rt = session.runtime
|
| 1883 |
+
token = _user_tables_token(rt)
|
| 1884 |
+
|
| 1885 |
+
def keep(v):
|
| 1886 |
+
tables = [d for d in (v or {}).values() if isinstance(d, dict)]
|
| 1887 |
+
return bool(tables) and all(store.is_projected(d) for d in tables)
|
| 1888 |
|
| 1889 |
+
defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt),
|
| 1890 |
+
keep=keep, clone=copy.deepcopy, token=token)
|
| 1891 |
+
return ut.lend(rt, **{ut.STORE_KEY: defs}), token
|
| 1892 |
|
| 1893 |
+
|
| 1894 |
+
def _stored_link_rows(session, table, token=_UNREAD):
|
| 1895 |
+
"""`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision.
|
| 1896 |
+
|
| 1897 |
+
β Only a table whose rows live in `user_tables` qualifies: a registered row source (a module
|
| 1898 |
+
cache, an app reader) or a read-through binding answers from somewhere this token does not
|
| 1899 |
+
watch, so those read exactly as before. The wall is NOT applied here; the caller applies
|
| 1900 |
+
`apply_row_scope` for its own user on every request. Holding it across users is sound only
|
| 1901 |
+
because `perm_scope._read_user_table` never consults `user` for a `ut_*` key: every
|
| 1902 |
+
user-dependent step (hidden fields, row wall, BU scope) sits above it, in the caller.
|
| 1903 |
+
"""
|
| 1904 |
+
import core.perm_scope as perm_scope
|
| 1905 |
+
|
| 1906 |
+
rt = session.runtime
|
| 1907 |
+
if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES
|
| 1908 |
+
or module_cache_source(table) is not None or not _materialised_table(table)):
|
| 1909 |
+
return perm_scope._read(table, session.user, rt)
|
| 1910 |
+
return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt),
|
| 1911 |
+
keep=lambda v: bool(v[1]),
|
| 1912 |
+
clone=lambda v: ([dict(f) for f in v[0]], [dict(r) for r in v[1]]),
|
| 1913 |
+
token=token)
|
| 1914 |
+
|
| 1915 |
+
|
| 1916 |
+
def _traversal_defn_or_refuse(session, table_key, st=None):
|
| 1917 |
"""R50 / C4: admit an intermediate after the caller has established both end grants.
|
| 1918 |
|
| 1919 |
The session may already open both ends of a declared two-hop link. Its intermediate
|
|
|
|
| 1921 |
read still applies the row wall, hidden-field closure, BU scope and mirror guard.
|
| 1922 |
"""
|
| 1923 |
ut = _ut()
|
| 1924 |
+
st = st if st is not None else ut.lend_defs(session.runtime)
|
| 1925 |
defn = ut.get(table_key, st=st)
|
| 1926 |
if not defn:
|
| 1927 |
raise err(404, "unknown_table", "that database does not exist")
|
|
|
|
| 1929 |
|
| 1930 |
|
| 1931 |
def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
|
| 1932 |
+
traversal_of=None, st=None, rev=...): # `...` IS `_UNREAD`
|
| 1933 |
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
|
| 1934 |
|
| 1935 |
Join columns may be hidden on the wire. Only identities leave the resolver; the native
|
| 1936 |
payload builders still apply their complete hidden-field closure before returning cells.
|
| 1937 |
With traversal_of, the caller must already have established both ends of the declared
|
| 1938 |
edge named by that key. Only join values leave this intermediate read, never a cell.
|
| 1939 |
+
S67-T28: `st` is a lent DEFINITIONS snapshot the caller already holds (`_page_defs`); the wall
|
| 1940 |
+
is asked of it instead of re-reading every definition per target. A stored target's rows come
|
| 1941 |
+
through `_stored_link_rows`, reused per `user_tables` revision; its wall still runs here.
|
| 1942 |
+
`rev` is the revision token `_page_defs` already read for this request; absent, the stored
|
| 1943 |
+
target is read directly, as before S67-T28.
|
| 1944 |
"""
|
| 1945 |
import core.perm_scope as perm_scope
|
| 1946 |
import core.relations as relations
|
| 1947 |
|
| 1948 |
columns = set(columns) | {relations.IDENTITY_COL}
|
| 1949 |
if not _materialised_table(table):
|
| 1950 |
+
read = (_compiled_read(table, session, traversal_intermediate=True, st=st)
|
| 1951 |
+
if traversal_of is not None else _compiled_read(table, session, st=st))
|
| 1952 |
spec = read["spec"]
|
| 1953 |
bindings = dict(spec["cols"], pid=(spec["id"], int))
|
| 1954 |
if columns - set(bindings):
|
|
|
|
| 1971 |
|
| 1972 |
# This is an internal join, never a cell response. Apply the same permanent wall on the
|
| 1973 |
# complete contract before taking any join value, including a join on a hidden column.
|
| 1974 |
+
if traversal_of is None and not perm_scope.may_read(
|
| 1975 |
+
session.user, table, st=st if st is not None else session.runtime):
|
| 1976 |
raise err(403, "forbidden_table", "This account cannot open the linked database.")
|
| 1977 |
if module_cache_source(table) is not None:
|
| 1978 |
session.require(table)
|
| 1979 |
+
# The revision memo serves a caller that handed this request's token (the page's link fills,
|
| 1980 |
+
# S67-T28); any other caller reads exactly as before, which is also what lets
|
| 1981 |
+
# `verify_grid_ux.py` exec this function alone with a stubbed `perm_scope`.
|
| 1982 |
+
fields, rows = (perm_scope._read(table, session.user, session.runtime) if rev is ...
|
| 1983 |
+
else _stored_link_rows(session, table, token=rev))
|
| 1984 |
rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime)
|
| 1985 |
if where:
|
| 1986 |
raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.")
|
|
|
|
| 2025 |
# identity. Treating a missing/hidden/blank source value as an empty match would
|
| 2026 |
# turn "the join input is unavailable" into the business claim "no related rows".
|
| 2027 |
if source_col != relations.IDENTITY_COL and not values:
|
| 2028 |
+
# A visible Amazon row with an empty business key has no related records.
|
| 2029 |
+
# A missing/inaccessible source row must retain the ordinary refusal.
|
| 2030 |
+
if (source.startswith("ut_amzn_") and target.startswith("ut_amzn_")
|
| 2031 |
+
and source_rows and all(source_col in row for row in source_rows)):
|
| 2032 |
+
return set()
|
| 2033 |
raise err(400, "unknown_relation", relations.unresolvable_source_refusal(source, entry))
|
| 2034 |
if (manual and not reverse) or (inverse and reverse):
|
| 2035 |
values = set().union(*(_relation_cell_ids(row.get(source_col)) for row in source_rows))
|
|
|
|
| 2094 |
if not _may_open(session, leg["target"]):
|
| 2095 |
raise err(403, "forbidden_table",
|
| 2096 |
f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
|
| 2097 |
+
if not ids and table.startswith("ut_amzn_") and leg["target"].startswith("ut_amzn_"):
|
| 2098 |
+
# An earlier, authorised Amazon hop matched nothing. There is no next
|
| 2099 |
+
# source record to read, and that is an empty relation, not a missing key.
|
| 2100 |
+
continue
|
| 2101 |
ids = _relation_leg_ids(session, table, ids, leg)
|
| 2102 |
return ids
|
| 2103 |
|
|
|
|
| 2768 |
|
| 2769 |
|
| 2770 |
def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
|
| 2771 |
+
search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False,
|
| 2772 |
+
st=None):
|
| 2773 |
"""EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
|
| 2774 |
|
| 2775 |
Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits,
|
|
|
|
| 2829 |
# tenant #0's eight databases is read-through. Left un-applied, an administrator could set a
|
| 2830 |
# filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows
|
| 2831 |
# used to be. So the wall is APPLIED here, in SQL, three paragraphs down.
|
| 2832 |
+
# S67-T28: `st` lends a definitions snapshot the caller already holds (`_page_defs`), the
|
| 2833 |
+
# same `st=` `_defn_or_refuse` has always accepted. None keeps its own `lend_defs` read.
|
| 2834 |
if traversal_intermediate:
|
| 2835 |
+
defn = _traversal_defn_or_refuse(session, table_key, st=st)
|
| 2836 |
else:
|
| 2837 |
+
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 2838 |
fields = None
|
| 2839 |
for _bucket, key, _label, mk_fields in rel.TABLES:
|
| 2840 |
if key == table_key:
|
| 2841 |
fields = mk_fields()
|
| 2842 |
break
|
| 2843 |
if fields is None:
|
| 2844 |
+
if table_key in _sources():
|
| 2845 |
+
fields = defn.get("fields") or []
|
| 2846 |
+
if not fields:
|
| 2847 |
+
raise err(409, "schema_not_loaded",
|
| 2848 |
+
"This connected database has no saved fields yet. Sync its connector first.")
|
| 2849 |
+
else:
|
| 2850 |
+
raise err(404, "not_connected", "that database is not a connected grid")
|
| 2851 |
|
| 2852 |
# ββ W31-T45 / D-169 β THE SECOND WALL, and it asks a question `_defn_or_refuse` above cannot.
|
| 2853 |
# That wall asks "may this SESSION open this DATABASE"; this one asks "is the file this process
|
|
|
|
| 2862 |
# turn the loudest refusal in the system into a spinner. The guard therefore runs ABOVE the
|
| 2863 |
# try, not inside it.
|
| 2864 |
try:
|
| 2865 |
+
cur, mismatch = (_mirror_cur(session, table_key) if table_key.startswith("ut_amzn_")
|
| 2866 |
+
else _mirror_cur(session))
|
| 2867 |
except RuntimeError as e:
|
| 2868 |
# β Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the
|
| 2869 |
# mismatch subclass, which is what makes this broad clause safe to keep here.
|
|
|
|
| 3101 |
|
| 3102 |
|
| 3103 |
@contextmanager
|
| 3104 |
+
def _materialised_read(table_key, session, query, related, ids=None):
|
| 3105 |
"""One compiler for native rows and their aggregate sibling, over the walled payload.
|
| 3106 |
|
| 3107 |
The temporary connection contains only this request's already stripped, pinned rows and is
|
|
|
|
| 3120 |
else:
|
| 3121 |
from routes_tables import table_rows
|
| 3122 |
payload = table_rows(table_key, session=session, related=related)
|
| 3123 |
+
if ids is not None:
|
| 3124 |
+
wanted = set(ids)
|
| 3125 |
+
payload = {**payload, "rows": [row for row in payload["rows"]
|
| 3126 |
+
if int(row["pid"]) in wanted]}
|
| 3127 |
fields = payload["fields"]
|
| 3128 |
with duckdb.connect(":memory:") as cur:
|
| 3129 |
cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)",
|
|
|
|
| 3200 |
return f"({gexpr}) = ?", [str(key)]
|
| 3201 |
|
| 3202 |
|
| 3203 |
+
def _record_ids(body):
|
| 3204 |
+
"""An explicit empty set stays empty. Never truncate a drill's population."""
|
| 3205 |
+
if "ids" not in body:
|
| 3206 |
+
return None
|
| 3207 |
+
ids = body["ids"]
|
| 3208 |
+
if not isinstance(ids, list) or any(type(pid) is not int for pid in ids):
|
| 3209 |
+
raise err(400, "bad_record_ids", "ids must be a list of integer record IDs")
|
| 3210 |
+
if any(pid < -(2 ** 63) or pid >= 2 ** 63 for pid in ids):
|
| 3211 |
+
raise err(400, "bad_record_ids", "Record IDs must fit a signed 64-bit integer")
|
| 3212 |
+
return list(dict.fromkeys(ids))
|
| 3213 |
+
|
| 3214 |
+
|
| 3215 |
+
def _restrict_row_ids(read, ids):
|
| 3216 |
+
"""Intersect the walled read with one bound array, independent of the ID count."""
|
| 3217 |
+
if ids is None:
|
| 3218 |
+
return read
|
| 3219 |
+
pin = f'{read["spec"]["id"]} IN (SELECT unnest(?::BIGINT[]))'
|
| 3220 |
+
narrowed = dict(read)
|
| 3221 |
+
for where_key, params_key in (("where", "params"), ("base_where", "wall_params")):
|
| 3222 |
+
where = read[where_key]
|
| 3223 |
+
narrowed[where_key] = f"({where}) AND {pin}" if where else pin
|
| 3224 |
+
narrowed[params_key] = list(read[params_key]) + [ids]
|
| 3225 |
+
return narrowed
|
| 3226 |
+
|
| 3227 |
+
|
| 3228 |
def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
|
| 3229 |
+
groupBy=None, groupKey=None, ids=None):
|
| 3230 |
from harness import datastore
|
| 3231 |
|
| 3232 |
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
|
| 3233 |
"sorts": _json_arg(sorts, "sorts"), "search": search})
|
| 3234 |
+
with _materialised_read(table_key, session, query, related, ids=ids) as read:
|
| 3235 |
spec = read["spec"]
|
| 3236 |
order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
|
| 3237 |
where, params = read["where"], list(read["params"])
|
|
|
|
| 3255 |
|
| 3256 |
|
| 3257 |
@router.get("/odoo-tables/{table_key}/rows")
|
| 3258 |
+
@router.post("/odoo-tables/{table_key}/rows")
|
| 3259 |
def odoo_table_rows(table_key: str,
|
| 3260 |
offset: int = Query(default=0, ge=0),
|
| 3261 |
limit: int = Query(default=0),
|
|
|
|
| 3266 |
related: str = Query(default=None),
|
| 3267 |
groupBy: str = Query(default=None),
|
| 3268 |
groupKey: str = Query(default=None),
|
| 3269 |
+
session: Session = Depends(require_session),
|
| 3270 |
+
body: dict = Body(default=None)):
|
| 3271 |
"""ONE WINDOW over a connected grid, read straight from the mirror (contract C2).
|
| 3272 |
|
| 3273 |
`{fields, rows, total, totalUnfiltered, offset, limit, limits, identity, recordsMutable}` β
|
|
|
|
| 3285 |
argument over: `?related={"table":"customer_data","record":1234,"via":{...}}`. It narrows this
|
| 3286 |
window to the rows linked to that ONE record, ahead of `filters`, and `total` /
|
| 3287 |
`totalUnfiltered` count inside the pin rather than around it.
|
| 3288 |
+
S66-T48: POST is the same window with one more narrowing, a JSON body `{"ids": [...]}`: a
|
| 3289 |
+
cell's record set, which can run to thousands of ids and would overflow a URL. Every other
|
| 3290 |
+
part of the question stays in the query string, spelled exactly as GET spells it (one
|
| 3291 |
+
vocabulary, one parser). The set narrows the rows AND their counts before LIMIT/OFFSET, so
|
| 3292 |
+
`total` and the pager describe the cell's records. The connected-tables alias forwards the
|
| 3293 |
+
body.
|
| 3294 |
"""
|
| 3295 |
import aios_grid
|
| 3296 |
import core.perm_scope as perm_scope
|
| 3297 |
from harness import datastore
|
| 3298 |
|
| 3299 |
+
# A direct Python caller omits the Body sentinel, and a GET has no body: both read unpinned.
|
| 3300 |
+
ids = None
|
| 3301 |
+
if isinstance(body, dict):
|
| 3302 |
+
# A window part sent in the body would be IGNORED, and an ignored offset or filter is a
|
| 3303 |
+
# wrong page that looks right. Refuse it by name instead.
|
| 3304 |
+
if set(body) - {"ids"}:
|
| 3305 |
+
raise err(400, "bad_window", "Only ids travel in the request body. Send offset, "
|
| 3306 |
+
"limit, filters, sorts and search in the query string.")
|
| 3307 |
+
ids = _record_ids(body)
|
| 3308 |
+
|
| 3309 |
+
# S67-T28: an Amazon grid's page snapshot is taken BEFORE the mirror check, so that check
|
| 3310 |
+
# asks it instead of reading its own definitions projection (one store round trip fewer on
|
| 3311 |
+
# every Amazon page). Not reordered past it: the check may register this grid's binding,
|
| 3312 |
+
# which is what `_materialised_table` below then reads.
|
| 3313 |
+
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 3314 |
+
require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
| 3315 |
if _materialised_table(table_key):
|
| 3316 |
return _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
| 3317 |
+
sorts, search, related, groupBy, groupKey, ids=ids)
|
| 3318 |
|
| 3319 |
# ββ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
|
| 3320 |
# hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
|
|
|
|
| 3322 |
# is a page of* - and that is provable only while there is one of it. What stays here is what
|
| 3323 |
# a WINDOW alone needs: the total order, the projection, the overlay merge and the two
|
| 3324 |
# sentences a page earns (`window_clamped`, the deep-page walk).
|
| 3325 |
+
# S67-T28 (D-3480 point 1): ONE definitions snapshot answers this page's own wall and every
|
| 3326 |
+
# link target's below, re-read only when `user_tables` has moved (`_page_defs`).
|
| 3327 |
+
page_st, page_rev = page if page is not None else _page_defs(session)
|
| 3328 |
read = _compiled_read(table_key, session, filters=filters, filterConj=filterConj,
|
| 3329 |
+
sorts=sorts, search=search, related=related, st=page_st)
|
| 3330 |
+
read = _restrict_row_ids(read, ids)
|
| 3331 |
defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"]
|
| 3332 |
cols, limits, frm = read["cols"], read["limits"], read["frm"]
|
| 3333 |
base_where, wall_params = read["base_where"], read["wall_params"]
|
|
|
|
| 3376 |
rows_src = _pool_from_window(spec, win,
|
| 3377 |
stamp=mirror_stamp(spec, cur, table_key, defn=defn, rt=
|
| 3378 |
session.runtime))
|
| 3379 |
+
if table_key.startswith("ut_amzn_"):
|
| 3380 |
+
import amazon_relational
|
| 3381 |
+
from routes_grid import _may_open
|
| 3382 |
+
|
| 3383 |
+
# ββ HOTFIX 2026-09-28: ONE READ PER STORED TARGET PER REQUEST. FBA customer returns
|
| 3384 |
+
# links listings twice (`seller_sku`, `asin1`); each edge read the whole stored table, so
|
| 3385 |
+
# every page paid two tenant-document reads (~3 s each on staging, /rows 9.5 s against
|
| 3386 |
+
# /aggregates' 1 s) and a burst of group-page clicks starved the store pool until sign-in
|
| 3387 |
+
# failed. A read-through target (orders, 559k rows) keeps its `match=` SQL read per edge:
|
| 3388 |
+
# reading it whole to filter here would be far worse.
|
| 3389 |
+
_opened, _stored = {}, {}
|
| 3390 |
+
_stored_cols = {}
|
| 3391 |
+
for _f, _target, _frm, _on in amazon_relational.link_edges(fields):
|
| 3392 |
+
_stored_cols.setdefault(_target, set()).add(_on)
|
| 3393 |
+
|
| 3394 |
+
def read_amazon_targets(target, column, values):
|
| 3395 |
+
if target not in _opened:
|
| 3396 |
+
_opened[target] = _may_open(session, target, st=page_st)
|
| 3397 |
+
if not _opened[target]:
|
| 3398 |
+
return []
|
| 3399 |
+
if not _materialised_table(target):
|
| 3400 |
+
return _relation_rows(session, target, [column], match=(column, values),
|
| 3401 |
+
st=page_st, rev=page_rev)
|
| 3402 |
+
if target not in _stored:
|
| 3403 |
+
_stored[target] = _relation_rows(
|
| 3404 |
+
session, target, sorted(_stored_cols.get(target, set()) | {column}),
|
| 3405 |
+
st=page_st, rev=page_rev)
|
| 3406 |
+
wanted = {str(value) for value in values}
|
| 3407 |
+
# The same string comparison `_relation_rows`' own `match=` applies to a stored table.
|
| 3408 |
+
return [row for row in _stored[target] if str(row.get(column)) in wanted]
|
| 3409 |
+
|
| 3410 |
+
# Fields are the current reader's contract; target rows cross the same wall
|
| 3411 |
+
# as /related pins. No unscoped SQL subquery can leak a hidden target's id.
|
| 3412 |
+
amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets)
|
| 3413 |
overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
|
| 3414 |
|
| 3415 |
# ββ W30-T28 β THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
|
|
|
|
| 3425 |
# the wall and the predicate, so the argument it demands is the list we just fetched.
|
| 3426 |
# β It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already
|
| 3427 |
# answered "may this session open this surface".
|
| 3428 |
+
# S67-T28: through `page_st`, which lends a `__shared` bucket lazily ONCE for this request
|
| 3429 |
+
# (`user_tables._Lent`), so `fields` and `cells` share one read of the stratum instead of two.
|
| 3430 |
+
# Both only read it (`shared_overlay._read` copies the top level, `cells` copies each row).
|
| 3431 |
+
shared_defs = _so().fields(table_key, st=page_st)
|
| 3432 |
if shared_defs:
|
| 3433 |
fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()]
|
| 3434 |
for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src],
|
| 3435 |
+
st=page_st).items():
|
| 3436 |
overlays.setdefault(pid, {}).update(cells)
|
| 3437 |
# RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
|
| 3438 |
# ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
|
|
|
|
| 3738 |
# hand: `filters` a list, `filterConj` in and/or, `search` text under 1,000 characters. A
|
| 3739 |
# second spelling of "what a predicate on the wire looks like" is how two of them come apart.
|
| 3740 |
query = _selection_query(payload)
|
| 3741 |
+
ids = _record_ids(payload)
|
| 3742 |
aggs = payload.get("aggs")
|
| 3743 |
if aggs is None:
|
| 3744 |
aggs = []
|
|
|
|
| 3752 |
raise err(400, "bad_selection", "groupBy must be a column id")
|
| 3753 |
|
| 3754 |
if _materialised_table(table_key):
|
| 3755 |
+
with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
|
| 3756 |
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
|
| 3757 |
read["params"], read["frm"], aggs, group_by=group_by,
|
| 3758 |
limits=read["limits"])
|
|
|
|
| 3763 |
# who assumed both were bodies would have wired the rows and left the summaries answering over
|
| 3764 |
# the entire database under a tab that names one record. `_compiled_read` accepts either shape,
|
| 3765 |
# so the two doors cannot pin differently without this argument being deleted.
|
| 3766 |
+
# S67-T28: the same page snapshot `/rows` takes, so opening a grouped view (its group counts
|
| 3767 |
+
# come from here) no longer re-reads every definition while `user_tables` is unchanged.
|
| 3768 |
read = _compiled_read(table_key, session, tree=query["filters"],
|
| 3769 |
filterConj=query["filterConj"], search=query["search"],
|
| 3770 |
+
related=payload.get("related"), st=_page_defs(session)[0])
|
| 3771 |
+
read = _restrict_row_ids(read, ids)
|
| 3772 |
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
|
| 3773 |
read["params"], read["frm"], aggs, group_by=group_by,
|
| 3774 |
limits=read["limits"])
|
|
|
|
| 3894 |
from harness import datastore
|
| 3895 |
if rt is not None:
|
| 3896 |
rt.assert_datastore_matches() # W31-T45: the same wall the row doors carry
|
| 3897 |
+
cur = cur if cur is not None else _table_cursor(rt, table_key)
|
| 3898 |
spec = _source_for(cur, table_key)
|
| 3899 |
if spec is None:
|
| 3900 |
return None
|
|
|
|
| 3927 |
"""
|
| 3928 |
from harness import datastore
|
| 3929 |
if rt is not None:
|
| 3930 |
+
cur = cur if cur is not None else _table_cursor(rt, table_key)
|
| 3931 |
rt.assert_datastore_matches() # also covers a cursor the CALLER opened and lent
|
| 3932 |
+
cur = cur if cur is not None else _table_cursor(None, table_key)
|
| 3933 |
spec = _source_for(cur, table_key)
|
| 3934 |
if spec is None:
|
| 3935 |
raise TooBigToMaterialise(
|
api/routes_platform_admin.py
CHANGED
|
@@ -37,16 +37,60 @@ import subprocess
|
|
| 37 |
import sys
|
| 38 |
import time
|
| 39 |
from pathlib import Path
|
|
|
|
| 40 |
|
| 41 |
from fastapi import APIRouter, Body, Depends, Header, Request
|
| 42 |
|
| 43 |
import core.platform_admin as platform_admin
|
|
|
|
|
|
|
| 44 |
import core.store as store
|
| 45 |
import operator_control
|
| 46 |
import voice_control_plane as voice
|
| 47 |
from deps import Session, err, require_session, users
|
| 48 |
|
| 49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
|
| 52 |
def _now_iso():
|
|
@@ -736,6 +780,18 @@ def platform_automations(tenant: str = "", session: Session = Depends(padmin_gat
|
|
| 736 |
"resident": resident}}
|
| 737 |
|
| 738 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 739 |
@router.get("/aws")
|
| 740 |
def platform_aws(days: int = 7, session: Session = Depends(padmin_gate)):
|
| 741 |
"""AWS automation usage, verbatim, or an honest note explaining its absence."""
|
|
|
|
| 37 |
import sys
|
| 38 |
import time
|
| 39 |
from pathlib import Path
|
| 40 |
+
from contextlib import asynccontextmanager
|
| 41 |
|
| 42 |
from fastapi import APIRouter, Body, Depends, Header, Request
|
| 43 |
|
| 44 |
import core.platform_admin as platform_admin
|
| 45 |
+
import core.neon_usage as neon_usage
|
| 46 |
+
import core.do_usage as do_usage
|
| 47 |
import core.store as store
|
| 48 |
import operator_control
|
| 49 |
import voice_control_plane as voice
|
| 50 |
from deps import Session, err, require_session, users
|
| 51 |
|
| 52 |
+
@asynccontextmanager
|
| 53 |
+
async def _neon_lifespan(app):
|
| 54 |
+
_start_neon_sampler()
|
| 55 |
+
_start_do_sampler()
|
| 56 |
+
try:
|
| 57 |
+
yield
|
| 58 |
+
finally:
|
| 59 |
+
neon_usage.stop_sampler()
|
| 60 |
+
do_usage.stop_sampler()
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
router = APIRouter(prefix="/api/v1/platform-admin", lifespan=_neon_lifespan)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _start_neon_sampler():
|
| 67 |
+
# These callbacks are invoked only by the sampler after its management API
|
| 68 |
+
# awake check. In particular, the card's GET never resolves a store runtime.
|
| 69 |
+
def load():
|
| 70 |
+
from harness import runtime
|
| 71 |
+
return runtime.get_runtime("loopable").get(neon_usage.STORE_KEY)
|
| 72 |
+
|
| 73 |
+
def persist(document):
|
| 74 |
+
from harness import runtime
|
| 75 |
+
runtime.get_runtime("loopable").update(neon_usage.STORE_KEY, lambda _: document, flush="sync")
|
| 76 |
+
|
| 77 |
+
neon_usage.start_sampler(load=load, persist=persist)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _start_do_sampler():
|
| 81 |
+
def load():
|
| 82 |
+
from harness import runtime
|
| 83 |
+
return runtime.get_runtime("loopable").get(do_usage.STORE_KEY)
|
| 84 |
+
|
| 85 |
+
def persist(document):
|
| 86 |
+
from harness import runtime
|
| 87 |
+
runtime.get_runtime("loopable").update(do_usage.STORE_KEY, lambda _: document, flush="sync")
|
| 88 |
+
|
| 89 |
+
do_usage.start_sampler(load=load, persist=persist)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
# FastAPI 0.139 carries a router lifespan into include_router; legacy startup
|
| 93 |
+
# event registration does not fire there. Defining it does no work at import.
|
| 94 |
|
| 95 |
|
| 96 |
def _now_iso():
|
|
|
|
| 780 |
"resident": resident}}
|
| 781 |
|
| 782 |
|
| 783 |
+
@router.get("/neon")
|
| 784 |
+
def platform_neon(session: Session = Depends(padmin_gate)):
|
| 785 |
+
"""Project billing totals plus the sampler's bounded, in-memory history."""
|
| 786 |
+
return neon_usage.report()
|
| 787 |
+
|
| 788 |
+
|
| 789 |
+
@router.get("/do")
|
| 790 |
+
def platform_do(session: Session = Depends(padmin_gate)):
|
| 791 |
+
"""DigitalOcean management-plane billing, with no database read on this path."""
|
| 792 |
+
return do_usage.report()
|
| 793 |
+
|
| 794 |
+
|
| 795 |
@router.get("/aws")
|
| 796 |
def platform_aws(days: int = 7, session: Session = Depends(padmin_gate)):
|
| 797 |
"""AWS automation usage, verbatim, or an honest note explaining its absence."""
|
api/routes_query.py
CHANGED
|
@@ -410,6 +410,7 @@ def _spec_schema(field_keys):
|
|
| 410 |
"filters": {"type": "array", "items": {"type": "object", "properties": {
|
| 411 |
"colId": col, "op": {"type": "string", "enum": sorted(_grid().FILTER_OPS)},
|
| 412 |
"value": {"type": "string"},
|
|
|
|
| 413 |
"rhs": {"type": "object", "properties": {
|
| 414 |
"kind": {"type": "string", "enum": ["field"]}, "colId": col,
|
| 415 |
}, "required": ["kind", "colId"]},
|
|
@@ -897,6 +898,10 @@ def _validate(spec, fields):
|
|
| 897 |
for sent, kept in zip(raw_filters, filters):
|
| 898 |
if sent.get("rhs") and not kept.get("rhs"):
|
| 899 |
return None, "that column cannot be compared against another column", "rhs_dropped"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 900 |
|
| 901 |
aggregation = raw_aggregation if isinstance(raw_aggregation, dict) else {}
|
| 902 |
op = str(aggregation.get("op") or "").lower()
|
|
|
|
| 410 |
"filters": {"type": "array", "items": {"type": "object", "properties": {
|
| 411 |
"colId": col, "op": {"type": "string", "enum": sorted(_grid().FILTER_OPS)},
|
| 412 |
"value": {"type": "string"},
|
| 413 |
+
"values": {"type": "array", "items": {"type": "string"}},
|
| 414 |
"rhs": {"type": "object", "properties": {
|
| 415 |
"kind": {"type": "string", "enum": ["field"]}, "colId": col,
|
| 416 |
}, "required": ["kind", "colId"]},
|
|
|
|
| 898 |
for sent, kept in zip(raw_filters, filters):
|
| 899 |
if sent.get("rhs") and not kept.get("rhs"):
|
| 900 |
return None, "that column cannot be compared against another column", "rhs_dropped"
|
| 901 |
+
# D-658: an empty multi-value leaf survives cleaning but narrows no records.
|
| 902 |
+
if sent.get("op") in _grid().MULTI_VALUE_OPS and (
|
| 903 |
+
not sent.get("values") or not kept.get("values")):
|
| 904 |
+
return None, "that filter needs at least one non-blank value", "values_missing"
|
| 905 |
|
| 906 |
aggregation = raw_aggregation if isinstance(raw_aggregation, dict) else {}
|
| 907 |
op = str(aggregation.get("op") or "").lower()
|
api/routes_sheet.py
CHANGED
|
@@ -35,6 +35,7 @@ while its surface is a 404.
|
|
| 35 |
(standing rule 2), and each names both the cause and the subject.
|
| 36 |
"""
|
| 37 |
|
|
|
|
| 38 |
import hashlib
|
| 39 |
import json
|
| 40 |
import os
|
|
@@ -115,14 +116,67 @@ def _mirror(session: Session):
|
|
| 115 |
# β `TABLES` ITSELF IS UNTOUCHED β it has other consumers (`declared_schema_fields`,
|
| 116 |
# `_odoo_sources`' `register_connected`) that must keep seeing exactly eight rows.
|
| 117 |
is_registry_grid = fields is None and table_key in rel._REGISTRY_GRID_FIELDS
|
| 118 |
-
|
| 119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
try:
|
| 121 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
except RuntimeError as exc:
|
| 123 |
raise err(503, "store_not_ready", str(exc))
|
| 124 |
if mismatch:
|
| 125 |
raise err(409, "cross_tenant_store", mismatch)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
if is_registry_grid:
|
| 127 |
# ββ ONE ASSEMBLY PER REQUEST PER (DATABASE, WINDOW), NOT ONE PER CELL. `refresh_sheet`
|
| 128 |
# resolves a binding CELL BY CELL, and this branch now costs a full `/workspace`
|
|
@@ -165,6 +219,8 @@ def _mirror(session: Session):
|
|
| 165 |
memo[slot] = (None if want is None else set(want), made)
|
| 166 |
return made
|
| 167 |
spec = rot._source_for(cur, table_key)
|
|
|
|
|
|
|
| 168 |
if spec is None:
|
| 169 |
raise err(409, "not_read_through",
|
| 170 |
f"{table_key} has no read-through binding on this deployment, so a cell cannot "
|
|
@@ -179,6 +235,49 @@ def _mirror(session: Session):
|
|
| 179 |
return _for
|
| 180 |
|
| 181 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
def _registry_assembly(session: Session, table_key: str):
|
| 183 |
"""The `g` dict `/workspace` builds for one compiled registry grid, or a NAMED refusal.
|
| 184 |
|
|
@@ -254,157 +353,144 @@ def _registry_assembly(session: Session, table_key: str):
|
|
| 254 |
"shown rather than shown from an older set of columns. Try again in a moment.")
|
| 255 |
|
| 256 |
|
| 257 |
-
def
|
| 258 |
-
"""
|
| 259 |
-
from core.relations import BRIDGES
|
| 260 |
-
return [bridge for bridge in BRIDGES if bridge.get("target") == database
|
| 261 |
-
and bridge.get("source") == "customer_data" and bridge.get("served")
|
| 262 |
-
and bridge.get("cardinality") == "one_to_many" and not bridge.get("where")]
|
| 263 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
|
| 265 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
from core import perm_scope
|
| 267 |
-
_require_database(session,
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
fields = [f for f in fields if f.get("key") not in hidden
|
| 274 |
-
and f.get("type") not in {"formula", "link", "lookup", "rollup", "metric"}
|
| 275 |
-
and not f.get("measure")]
|
| 276 |
-
return g, fields
|
| 277 |
|
| 278 |
|
| 279 |
-
def
|
| 280 |
-
|
|
|
|
| 281 |
from modules import sheet as sheet_mod
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
|
|
|
|
|
|
|
|
|
| 297 |
try:
|
| 298 |
-
|
| 299 |
-
except
|
| 300 |
-
raise
|
| 301 |
-
"the declared relation column is unavailable on this database.")
|
| 302 |
-
columns[key] = {"sql": key, "type": "int", "aggregate": False}
|
| 303 |
-
return key
|
| 304 |
|
| 305 |
|
| 306 |
@router.get("/sheets/filter-fields/{database}")
|
| 307 |
def linked_filter_fields(database: str, session: Session = Depends(require_session)):
|
| 308 |
-
"""
|
| 309 |
-
from
|
| 310 |
database = _require_database(session, database)
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
|
|
|
|
|
|
| 318 |
try:
|
| 319 |
-
|
| 320 |
-
|
| 321 |
except HTTPException as exc:
|
| 322 |
-
if exc.
|
|
|
|
|
|
|
| 323 |
continue
|
| 324 |
raise
|
| 325 |
-
|
| 326 |
continue
|
| 327 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
for field in fields:
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
"
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
def _linked_filter_scope(session, database, nodes, cur, spec, columns, *, today=None):
|
| 338 |
-
"""Read linked values in Python, then give the relation-blind compiler flat predicates."""
|
| 339 |
-
from modules import sheet as sheet_mod
|
| 340 |
-
if not sheet_mod.linked_filter_ids(nodes):
|
| 341 |
-
return nodes
|
| 342 |
-
import aios_grid
|
| 343 |
-
from core import perm_scope, shared_overlay, view_templates
|
| 344 |
-
from harness import filter_eval, filter_sql
|
| 345 |
-
|
| 346 |
-
bridges = {b["key"]: b for b in _linked_customer_bridges(database)}
|
| 347 |
-
related = {}
|
| 348 |
-
|
| 349 |
-
def resolve(leaf):
|
| 350 |
-
subject = leaf["colId"]
|
| 351 |
-
bridge_key, field_key = sheet_mod.linked_field_parts(subject)
|
| 352 |
-
bridge = bridges.get(bridge_key)
|
| 353 |
-
if bridge is None:
|
| 354 |
-
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 355 |
-
"this database no longer has that available customer link.")
|
| 356 |
-
try:
|
| 357 |
-
relation_column = _linked_relation_column(session, bridge, cur, spec, columns)
|
| 358 |
-
if bridge_key not in related:
|
| 359 |
-
g, fields = _linked_customer_fields(session, bridge)
|
| 360 |
-
# Use the rows door's wall, including its shared-field enrichment. The pool's
|
| 361 |
-
# identity is pid; it must never be confused with the orders' partner_id.
|
| 362 |
-
rows = perm_scope.apply_row_scope(g["rows_src"], session.user, bridge["source"],
|
| 363 |
-
aios_grid.FIELDS, st=session.runtime)
|
| 364 |
-
related[bridge_key] = (g, fields, rows)
|
| 365 |
-
g, fields, source_rows = related[bridge_key]
|
| 366 |
-
except HTTPException as exc:
|
| 367 |
-
if exc.status_code != 403:
|
| 368 |
-
raise
|
| 369 |
-
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 370 |
-
"this account cannot read the linked customer database.")
|
| 371 |
-
field = next((f for f in fields if f.get("key") == field_key), None)
|
| 372 |
-
if field is None:
|
| 373 |
-
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 374 |
-
"the field is hidden, removed, or has no readable per-record value.")
|
| 375 |
-
local = {**leaf, "colId": field_key}
|
| 376 |
-
# Validate before evaluating even an empty pool, so malformed/stale leaves refuse
|
| 377 |
-
# rather than looking like a genuine zero. Cell slots were resolved by W60-T06 above.
|
| 378 |
-
probe_columns = {field_key: {"sql": "linked_value", "type": sheet_mod.mirror_field_type(field)}}
|
| 379 |
-
try:
|
| 380 |
-
compiled = sheet_mod.compile_scope([local], "and", probe_columns, today=today)
|
| 381 |
-
if (local.get("op") not in filter_sql.FILTER_OPS or compiled is None
|
| 382 |
-
or field_key not in compiled.columns_used):
|
| 383 |
-
raise ValueError("unusable linked condition")
|
| 384 |
-
except (ValueError, sheet_mod.SheetRefusal):
|
| 385 |
-
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 386 |
-
"its value or operator cannot be applied to this field.")
|
| 387 |
-
storage_key = view_templates.workspace_key(bridge["source"])
|
| 388 |
-
try:
|
| 389 |
-
definitions = shared_overlay.fields(storage_key, st=session.runtime)
|
| 390 |
-
rows = aios_grid.rows_from_pool(source_rows, fields, (g.get("ws") or {}).get("overlays"),
|
| 391 |
-
derived=g.get("derived"))
|
| 392 |
-
if field_key in definitions:
|
| 393 |
-
cells = shared_overlay.cells(storage_key, [r["pid"] for r in source_rows], st=session.runtime)
|
| 394 |
-
rows = [{**row, field_key: (cells.get(str(int(row["pid"]))) or {}).get(field_key, "")}
|
| 395 |
-
for row in rows]
|
| 396 |
-
ctx = filter_eval.EvalCtx(today=today)
|
| 397 |
-
ids = [row[bridge["source_col"]] for row in rows
|
| 398 |
-
if filter_eval.permits({"conj": "and", "nodes": [local]}, row, fields, ctx)]
|
| 399 |
-
return sheet_mod.linked_id_predicate(relation_column, ids)
|
| 400 |
-
except Exception:
|
| 401 |
-
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 402 |
-
"the linked customer values could not be read. Retry after the data is available.")
|
| 403 |
-
|
| 404 |
-
try:
|
| 405 |
-
return sheet_mod.rewrite_linked_filters(nodes, resolve)
|
| 406 |
-
except sheet_mod.SheetRefusal as exc:
|
| 407 |
-
raise err(409, exc.code, exc.message)
|
| 408 |
|
| 409 |
|
| 410 |
def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
|
|
@@ -448,6 +534,19 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
|
|
| 448 |
derived, bounded, limits = _metric_derived(session, table_key, g, metric_window)
|
| 449 |
rows = aios_grid.rows_from_pool(g["rows_src"], fields, (g.get("ws") or {}).get("overlays"),
|
| 450 |
derived=derived)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 451 |
# The columns this cell can read, in the assembly's own order. `registry_grid_spec` forces
|
| 452 |
# `pid` in on its own, because `build_ids_query` needs `spec["id"]` to be a real column.
|
| 453 |
kept = (fields if needed is None
|
|
@@ -815,6 +914,9 @@ def _refuse(exc):
|
|
| 815 |
return HTTPException(status_code=409, detail={"error": body})
|
| 816 |
if exc.code in _FORBIDDEN_CODES:
|
| 817 |
return err(403, exc.code, exc.message)
|
|
|
|
|
|
|
|
|
|
| 818 |
status = 404 if exc.code in _MISSING_CODES else 400
|
| 819 |
return err(status, exc.code, exc.message)
|
| 820 |
|
|
@@ -1022,17 +1124,43 @@ def _has_cell_inputs(binding):
|
|
| 1022 |
related_nodes, _ = _fe_parts(related.get("filter") if isinstance(related, dict) else None)
|
| 1023 |
nodes = [*nodes, *related_nodes]
|
| 1024 |
# Linked values and related-table grants can change independently of the orders' cache key.
|
| 1025 |
-
if sheet_mod.linked_filter_ids(nodes)
|
|
|
|
| 1026 |
return True
|
| 1027 |
return any(sheet_mod.filter_cell_nodes(nodes))
|
| 1028 |
|
| 1029 |
|
| 1030 |
-
def _refresh_context(book, binding, memo=None):
|
| 1031 |
-
"""Authored inputs and relative dates invalidate refresh, but never an open's snapshot.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1032 |
from core.sheet_binding import fingerprint
|
| 1033 |
from datetime import date
|
| 1034 |
payload = {}
|
| 1035 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1036 |
if memo is None:
|
| 1037 |
memo = {}
|
| 1038 |
if "inputs" not in memo:
|
|
@@ -1143,8 +1271,11 @@ def _reresolved_for_reader(session: Session, book, *, endings_only=False):
|
|
| 1143 |
# and MCP read agree on stored cells, figures and formats` measured exactly that pair: D1 and
|
| 1144 |
# F1 fresh 30.0 / 60.0 from MCP against stale-and-blank from the UI. A non-browser consumer has
|
| 1145 |
# no client engine to cover for it, so the blank is what it would have shown.
|
|
|
|
|
|
|
|
|
|
| 1146 |
stale_formula = any(
|
| 1147 |
-
cell.get("kind") == "formula" and (cell.get("state")
|
| 1148 |
for tab in (fresh.get("sheets") or []) for cell in (tab.get("cells") or {}).values())
|
| 1149 |
if changed_values or stale_formula:
|
| 1150 |
from modules import sheet as sheet_mod
|
|
@@ -1760,10 +1891,17 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
|
|
| 1760 |
A formula sitting on a tab nobody has open still shows its last answer until that tab is
|
| 1761 |
opened or the workbook is refreshed, exactly as a stale database-bound cell does.
|
| 1762 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1763 |
from core import sheets
|
| 1764 |
from modules import sheet as sheet_mod
|
| 1765 |
import formula_runtime as fr
|
|
|
|
| 1766 |
|
|
|
|
| 1767 |
try:
|
| 1768 |
book = sheets.get_workbook(_store(session), workbook_id)
|
| 1769 |
found = sheets.find_tab(book, tab_ref)
|
|
@@ -1777,17 +1915,43 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
|
|
| 1777 |
(found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes}
|
| 1778 |
_recompute_formulas(book, fr, sheet_mod,
|
| 1779 |
cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
|
| 1780 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1781 |
ending_cells = set()
|
| 1782 |
refreshed_entries = []
|
| 1783 |
context_memo = {}
|
| 1784 |
-
|
|
|
|
|
|
|
| 1785 |
for addr, cell in (source.get("cells") or {}).items():
|
| 1786 |
binding = cell.get("binding") or {}
|
| 1787 |
-
if cell.get("kind") != "bound" or not _has_cell_inputs(binding):
|
|
|
|
|
|
|
|
|
|
| 1788 |
continue
|
| 1789 |
try:
|
| 1790 |
-
result = _resolve_cell_binding(session, binding, book, source.get("id")
|
|
|
|
| 1791 |
cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
|
| 1792 |
cell["refreshed_at"] = _now_iso()
|
| 1793 |
if result.get("limits"):
|
|
@@ -1795,6 +1959,8 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
|
|
| 1795 |
else:
|
| 1796 |
cell.pop("limits", None)
|
| 1797 |
cell.pop("refusal", None)
|
|
|
|
|
|
|
| 1798 |
except HTTPException as exc:
|
| 1799 |
info = ((exc.detail or {}).get("error") or {})
|
| 1800 |
cell.update(_binding_refusal(
|
|
@@ -1804,7 +1970,8 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
|
|
| 1804 |
ending_cells.add((source.get("id"), addr))
|
| 1805 |
refreshed_entries.append((str(source.get("id")) + "!" + addr,
|
| 1806 |
_binding_scope_key(session, str(binding.get("database") or ""), binding),
|
| 1807 |
-
{"tokens": {}, "context": _refresh_context(book, binding, context_memo
|
|
|
|
| 1808 |
"cell": {k: cell[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
|
| 1809 |
if k in cell}}))
|
| 1810 |
if ending_cells:
|
|
@@ -1813,6 +1980,11 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
|
|
| 1813 |
{c.get("binding", {}).get("database") for t in book.get("sheets", [])
|
| 1814 |
for c in t.get("cells", {}).values() if c.get("kind") == "bound"})
|
| 1815 |
_recompute_formulas(book, fr, sheet_mod)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1816 |
found = sheets.find_tab(book, tab_ref)
|
| 1817 |
out = {}
|
| 1818 |
for addr, cell in (found.get("cells") or {}).items():
|
|
@@ -1986,12 +2158,25 @@ def unshare_sheet(workbook_id: str, who: str, session: Session = Depends(require
|
|
| 1986 |
|
| 1987 |
|
| 1988 |
@router.post("/sheets/{workbook_id}/refresh")
|
| 1989 |
-
def refresh_sheet(workbook_id: str,
|
|
|
|
| 1990 |
"""C2/R4: Refresh resolves only databases whose token or binding inputs changed.
|
| 1991 |
|
| 1992 |
Viewers refresh their own server cache. Only editors save the workbook document.
|
| 1993 |
One token is read per distinct database on the unchanged path, regardless of cell count.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1994 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1995 |
from copy import deepcopy
|
| 1996 |
from core import sheets, sheet_binding
|
| 1997 |
from modules import sheet as sheet_mod
|
|
@@ -2019,10 +2204,12 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 2019 |
|
| 2020 |
def input_databases(binding, tab_ref):
|
| 2021 |
nonlocal input_graph
|
| 2022 |
-
|
| 2023 |
-
|
|
|
|
|
|
|
| 2024 |
refs = [str((node.get(key) or {}).get("ref") or "")
|
| 2025 |
-
for node in sheet_mod.filter_cell_nodes(
|
| 2026 |
for key in ("valueRef", "value2Ref") if key in node]
|
| 2027 |
period = binding.get("period") or {}
|
| 2028 |
if period.get("ref"):
|
|
@@ -2032,15 +2219,25 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 2032 |
tabs = book.get("sheets") or []
|
| 2033 |
names = _tab_scope_names(tabs)
|
| 2034 |
if input_graph is None:
|
| 2035 |
-
|
| 2036 |
-
|
| 2037 |
-
|
| 2038 |
-
|
| 2039 |
-
|
| 2040 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2041 |
documents, engine = input_graph
|
| 2042 |
home = _resolve_home_tab(book, tabs, names, tab_ref)
|
| 2043 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2044 |
while pending:
|
| 2045 |
key = pending.pop()
|
| 2046 |
if key in visited:
|
|
@@ -2048,7 +2245,14 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 2048 |
visited.add(key)
|
| 2049 |
cell = documents.get(key) or {}
|
| 2050 |
if cell.get("kind") == "bound":
|
| 2051 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2052 |
pending.extend(engine.consumed.get(key, ()))
|
| 2053 |
return deps
|
| 2054 |
|
|
@@ -2069,16 +2273,15 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 2069 |
|
| 2070 |
def dependencies(db, binding, tab_ref):
|
| 2071 |
deps = {db}
|
| 2072 |
-
|
| 2073 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2074 |
if key not in related_memo:
|
| 2075 |
-
related_memo[key] = _binding_related_entry(session, db,
|
| 2076 |
deps.add(related_memo[key]["target"])
|
| 2077 |
-
nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
|
| 2078 |
-
linked = sheet_mod.linked_filter_ids(nodes)
|
| 2079 |
-
if linked:
|
| 2080 |
-
names = {sheet_mod.linked_field_parts(key)[0] for key in linked}
|
| 2081 |
-
deps.update(b["source"] for b in _linked_customer_bridges(db) if b["key"] in names)
|
| 2082 |
deps.update(input_databases(binding, tab_ref))
|
| 2083 |
return deps
|
| 2084 |
|
|
@@ -2098,11 +2301,16 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 2098 |
hit = cache.get("refreshCells", {}).get(str(tab_ref) + "!" + addr, {}).get(scope_key)
|
| 2099 |
sources = {db: token(db) for db in dependencies(db, binding, tab_ref)}
|
| 2100 |
if (hit and all(sources.values()) and hit.get("tokens") == sources
|
| 2101 |
-
and hit.get("context") == _refresh_context(book, binding, context_memo
|
|
|
|
| 2102 |
and hit["cell"].get("state") == "fresh"):
|
| 2103 |
return hit["cell"]
|
| 2104 |
return None
|
| 2105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2106 |
for tab in book.get("sheets") or []:
|
| 2107 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 2108 |
if cell.get("kind") != "bound" or _is_view_binding(cell):
|
|
@@ -2110,8 +2318,11 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 2110 |
binding = cell.get("binding") or {}
|
| 2111 |
db = str(binding.get("database") or "")
|
| 2112 |
cell_key = str(tab.get("id")) + "!" + addr
|
|
|
|
|
|
|
| 2113 |
scope_key = _binding_scope_key(session, db, binding)
|
| 2114 |
-
context = _refresh_context(book, binding, context_memo)
|
|
|
|
| 2115 |
hit = cache.get("refreshCells", {}).get(cell_key, {}).get(scope_key)
|
| 2116 |
source_tokens = {}
|
| 2117 |
try:
|
|
@@ -2131,12 +2342,16 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 2131 |
status.setdefault(db, "skipped")
|
| 2132 |
continue
|
| 2133 |
status[db] = "resolved"
|
|
|
|
| 2134 |
previous = getattr(session, "_sheet_refresh_input", None)
|
| 2135 |
session._sheet_refresh_input = cached_input
|
| 2136 |
try:
|
| 2137 |
out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
|
| 2138 |
finally:
|
| 2139 |
session._sheet_refresh_input = previous
|
|
|
|
|
|
|
|
|
|
| 2140 |
limits = list(out.get("limits") or [])
|
| 2141 |
if not all(source_tokens.values()):
|
| 2142 |
limits.append({"code": "change_token_unavailable",
|
|
@@ -2147,8 +2362,10 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 2147 |
"refreshed_at": stamp}
|
| 2148 |
if limits:
|
| 2149 |
snapshot["limits"] = limits
|
| 2150 |
-
except HTTPException as exc:
|
| 2151 |
-
info = (
|
|
|
|
|
|
|
| 2152 |
snapshot = _binding_refusal(info.get("code", "sheet_unresolved"),
|
| 2153 |
info.get("message", "This cell could not be refreshed."))
|
| 2154 |
status[db] = "refused"
|
|
@@ -2179,9 +2396,15 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 2179 |
tab_ref, addr = cell_key.split("!", 1)
|
| 2180 |
_snapshot_cell(sheets.find_tab(book, tab_ref)["cells"][addr], entry["cell"])
|
| 2181 |
|
|
|
|
| 2182 |
started = perf_counter()
|
| 2183 |
_recompute_formulas(book, fr, sheet_mod)
|
| 2184 |
recalc_ms = round((perf_counter() - started) * 1000.0, 1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2185 |
if editable:
|
| 2186 |
try:
|
| 2187 |
saved = sheets.put_workbook(store, workbook_id,
|
|
@@ -2196,6 +2419,13 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 2196 |
saved["refreshedFor"] = str(session.uname or "")
|
| 2197 |
if entries:
|
| 2198 |
sheet_binding.save_refresh_cache(store, workbook_id, entries, status)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2199 |
return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms,
|
| 2200 |
"binding_refresh": {"databases": status, "token_reads": token_reads}}
|
| 2201 |
|
|
@@ -2341,6 +2571,10 @@ _METRIC_COLUMN_CREATE_LOCK = threading.RLock()
|
|
| 2341 |
def _forget_sheet_database(session, database):
|
| 2342 |
"""A field write invalidates both request-local schema and materialised-column memos."""
|
| 2343 |
getattr(session, "_sheet_assemblies", {}).pop(database, None)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2344 |
mirrors = getattr(session, "_sheet_mirrors", {})
|
| 2345 |
for slot in list(mirrors):
|
| 2346 |
if slot[0] == database:
|
|
@@ -2467,49 +2701,361 @@ def _ending_window(session, binding, *, book=None, tab=None, today=None, seen=()
|
|
| 2467 |
return sheet_mod.period_window(period, today, cell_date=date)
|
| 2468 |
|
| 2469 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2470 |
def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(),
|
| 2471 |
-
code="ending_cell_unreadable"):
|
| 2472 |
-
"""Shared cell read for Ending and Filter, including scoped bound and formula inputs.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2473 |
from modules import sheet as sheet_mod
|
| 2474 |
import formula_runtime as fr
|
| 2475 |
|
| 2476 |
def refuse(reason):
|
| 2477 |
raise sheet_mod.SheetRefusal(code, ref or "(blank)", reason)
|
| 2478 |
|
|
|
|
|
|
|
|
|
|
| 2479 |
if not re.fullmatch(r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref):
|
| 2480 |
refuse("its reference is invalid.")
|
| 2481 |
if book is None:
|
| 2482 |
refuse("the source workbook is unavailable.")
|
| 2483 |
tabs = book.get("sheets") or []
|
| 2484 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2485 |
home = _resolve_home_tab(book, tabs, names, tab)
|
| 2486 |
key = _asked_key(ref, home, fr)
|
| 2487 |
if key in seen:
|
| 2488 |
refuse("the reference is circular.")
|
| 2489 |
-
cells =
|
| 2490 |
-
|
|
|
|
| 2491 |
cell = cells.get(key)
|
| 2492 |
if not isinstance(cell, dict) or cell.get("kind") == "empty":
|
| 2493 |
refuse("the cell or its sheet is missing or blank.")
|
| 2494 |
if cell.get("kind") == "bound":
|
| 2495 |
try:
|
| 2496 |
-
|
| 2497 |
-
|
| 2498 |
-
value = (cached["value"] if cached is not None else _resolve_binding(
|
| 2499 |
-
session, cell.get("binding"), today=today, book=book,
|
| 2500 |
-
tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"])
|
| 2501 |
except HTTPException as exc:
|
| 2502 |
reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
|
| 2503 |
refuse(reason)
|
| 2504 |
elif cell.get("kind") == "formula":
|
| 2505 |
# The existing formula graph discovers actual inputs, including dynamic references.
|
| 2506 |
# A database input starts refused, so a cached value can never bypass this reader's wall.
|
| 2507 |
-
bound =
|
| 2508 |
-
|
| 2509 |
-
|
| 2510 |
-
|
| 2511 |
-
|
| 2512 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2513 |
while True:
|
| 2514 |
dependencies, pending = set(), [key]
|
| 2515 |
while pending:
|
|
@@ -2525,11 +3071,8 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
|
|
| 2525 |
for dependency in needed:
|
| 2526 |
attempted.add(dependency)
|
| 2527 |
try:
|
| 2528 |
-
|
| 2529 |
-
|
| 2530 |
-
if result is None:
|
| 2531 |
-
result = _resolve_binding(session, bound[dependency].get("binding"), today=today,
|
| 2532 |
-
book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency))
|
| 2533 |
changes[dependency] = {"value": result["value"]}
|
| 2534 |
except HTTPException as exc:
|
| 2535 |
info = ((exc.detail or {}).get("error") or {})
|
|
@@ -2551,6 +3094,13 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
|
|
| 2551 |
if cell.get("state") == "stale":
|
| 2552 |
refuse("the cell needs to be refreshed.")
|
| 2553 |
value = cell.get("value", cell.get("text"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2554 |
return value
|
| 2555 |
|
| 2556 |
|
|
@@ -2558,7 +3108,8 @@ def _filter_cell_values(session, nodes, *, book=None, tab=None, today=None, seen
|
|
| 2558 |
from modules import sheet as sheet_mod
|
| 2559 |
try:
|
| 2560 |
return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell(
|
| 2561 |
-
session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable"
|
|
|
|
| 2562 |
except sheet_mod.SheetRefusal as exc:
|
| 2563 |
raise err(400, exc.code, exc.message)
|
| 2564 |
|
|
@@ -2602,25 +3153,14 @@ def _resolve_cell_binding(session, binding, book, tab, **kwargs):
|
|
| 2602 |
return _resolve_binding(session, binding, **kwargs)
|
| 2603 |
|
| 2604 |
|
| 2605 |
-
def _binding_related_entry(session, database,
|
| 2606 |
"""Resolve one authored edge key against the permission-scoped declarations."""
|
| 2607 |
-
|
| 2608 |
-
|
| 2609 |
-
from core import perm_scope, relations, user_tables
|
| 2610 |
-
from routes_grid import _source_fields
|
| 2611 |
-
|
| 2612 |
-
fields = _source_fields(session, database)
|
| 2613 |
-
defs = {}
|
| 2614 |
-
for key, definition in (user_tables.all_defs(st=session.runtime) or {}).items():
|
| 2615 |
-
if perm_scope.nav_may_open(session.user, key, st=session.runtime):
|
| 2616 |
-
declared = definition.get("fields") or []
|
| 2617 |
-
hidden = perm_scope.hidden_keys(session.user, key, declared, st=session.runtime)
|
| 2618 |
-
defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
|
| 2619 |
-
entries = relations.related(database, fields=fields, defs=defs)
|
| 2620 |
-
name = binding["related"]["via"]
|
| 2621 |
matches = [entry for entry in entries if entry["key"] == name]
|
| 2622 |
via = matches[0]["via"] if len(matches) == 1 else {"key": name}
|
| 2623 |
-
|
|
|
|
| 2624 |
if entry is None:
|
| 2625 |
candidates = [item["key"] for item in relations.via_candidates(entries, via)
|
| 2626 |
if perm_scope.nav_may_open(session.user, item["target"], st=session.runtime)]
|
|
@@ -2640,23 +3180,20 @@ def _binding_related_entry(session, database, binding):
|
|
| 2640 |
declaration = next((f for f in defs.get(target, {}).get("fields", [])
|
| 2641 |
if f.get("key") == entry["via"].get("field")), {})
|
| 2642 |
link = declaration.get("link") or {}
|
| 2643 |
-
if link.get("through") or link.get("selectionMode") == "manual"
|
|
|
|
| 2644 |
raise err(409, "relation_source_unresolvable",
|
| 2645 |
f"Relationship '{name}' does not declare a direct column join. "
|
| 2646 |
"Choose a relationship with one readable column on each database.")
|
| 2647 |
return entry
|
| 2648 |
|
| 2649 |
|
| 2650 |
-
def _binding_related_query(session,
|
| 2651 |
-
today=None, book=None, tab=None, ending_seen=()):
|
| 2652 |
"""Materialise the far side on the near cursor, then hand metadata to the compiler."""
|
| 2653 |
-
if entry is None:
|
| 2654 |
-
return None
|
| 2655 |
from core import perm_scope, relations
|
| 2656 |
from modules import sheet as sheet_mod
|
| 2657 |
|
| 2658 |
-
|
| 2659 |
-
name = binding["related"]["via"]
|
| 2660 |
local_key, remote_key = entry["via"].get("from"), entry["via"].get("on")
|
| 2661 |
|
| 2662 |
def map_join(key, table, source_spec, cols, side):
|
|
@@ -2682,11 +3219,7 @@ def _binding_related_query(session, binding, entry, cur, spec, columns, *, windo
|
|
| 2682 |
wall = ((perm_scope.entry(session.user, target) or {}).get("filter")
|
| 2683 |
if perm_scope.row_scope_applies(session.user, target) else None)
|
| 2684 |
wall_nodes, wall_conj = _fe_parts(wall)
|
| 2685 |
-
|
| 2686 |
-
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
|
| 2687 |
-
today=today, seen=ending_seen)
|
| 2688 |
-
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
| 2689 |
-
| sheet_mod.filter_columns_named(wall_nodes) | {remote_key})
|
| 2690 |
try:
|
| 2691 |
rcur, rspec, rcols = _mirror(session)(target, metric_window=window, needed=needed)
|
| 2692 |
except HTTPException as exc:
|
|
@@ -2703,13 +3236,69 @@ def _binding_related_query(session, binding, entry, cur, spec, columns, *, windo
|
|
| 2703 |
hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [],
|
| 2704 |
st=session.runtime)
|
| 2705 |
rcols = {key: value for key, value in rcols.items() if key not in hidden}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2706 |
map_join(remote_key, target, rspec, rcols, "target")
|
| 2707 |
-
|
|
|
|
|
|
|
|
|
|
| 2708 |
return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key,
|
| 2709 |
"spec": rspec, "columns": rcols, "where": entry.get("where"),
|
| 2710 |
-
"scope_nodes": scope_nodes, "scope_conj": scope_conj,
|
| 2711 |
"wall_nodes": wall_nodes, "wall_conj": wall_conj,
|
| 2712 |
-
"limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2713 |
|
| 2714 |
|
| 2715 |
def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
|
|
@@ -2760,12 +3349,7 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 2760 |
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 2761 |
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 2762 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 2763 |
-
|
| 2764 |
-
if scope.get("view_id"):
|
| 2765 |
-
scope_nodes, scope_conj = _view_filter(session, database, scope["view_id"],
|
| 2766 |
-
scope.get("view_name", ""), memo=view_memo)
|
| 2767 |
-
else:
|
| 2768 |
-
scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
|
| 2769 |
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
|
| 2770 |
today=today, seen=ending_seen)
|
| 2771 |
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
|
@@ -2773,12 +3357,9 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 2773 |
if binding["aggregate"].get("field"):
|
| 2774 |
needed.add(str(binding["aggregate"]["field"]))
|
| 2775 |
|
| 2776 |
-
|
| 2777 |
-
if related_entry:
|
| 2778 |
-
needed.add(related_entry["via"].get("from"))
|
| 2779 |
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 2780 |
cols = dict(cols)
|
| 2781 |
-
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
|
| 2782 |
|
| 2783 |
# β THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
|
| 2784 |
# real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
|
|
@@ -2817,9 +3398,7 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 2817 |
# spec's column expressions; applying it a second time would double-count the exclusion and
|
| 2818 |
# break every joined spec, whose partner column is `sol.order_partner_id` or `aml.partner_id`.
|
| 2819 |
try:
|
| 2820 |
-
related =
|
| 2821 |
-
session, binding, related_entry, cur, spec, cols, window=window, today=today,
|
| 2822 |
-
book=book, tab=tab, ending_seen=ending_seen)
|
| 2823 |
sql, params, limits = sheet_mod.build_aggregate_query(
|
| 2824 |
spec, binding, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
|
| 2825 |
wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, related=related)
|
|
@@ -2922,6 +3501,11 @@ def preview_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 2922 |
binding editor show a number before anyone commits the cell. R10 may create a missing Metric
|
| 2923 |
field through the database's ordinary field writer; reusing a column needs no field write.
|
| 2924 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2925 |
raw = dict(body or {})
|
| 2926 |
binding = raw.get("binding") or {}
|
| 2927 |
book = None
|
|
@@ -3029,6 +3613,11 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 3029 |
β `rev` BUMPS, BECAUSE THIS PERSISTS. It goes back in the answer and the client adopts it, or
|
| 3030 |
its next PATCH would carry a revision the server has already moved past.
|
| 3031 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3032 |
from core import sheets
|
| 3033 |
|
| 3034 |
try:
|
|
@@ -3162,9 +3751,10 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
|
|
| 3162 |
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 3163 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 3164 |
scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
|
|
|
|
| 3165 |
# W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
|
| 3166 |
for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
|
| 3167 |
-
if not key.startswith(sheet_mod.
|
| 3168 |
return JSONResponse(status_code=400, content={
|
| 3169 |
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
|
| 3170 |
today = date.today()
|
|
@@ -3173,10 +3763,13 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
|
|
| 3173 |
window = _ending_window(session, binding, book=book, today=today)
|
| 3174 |
needed = (set(requested) | sorted_keys | sheet_mod.filter_columns_named(scope_nodes)
|
| 3175 |
| sheet_mod.filter_columns_named(wall_nodes))
|
|
|
|
| 3176 |
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 3177 |
cols = dict(cols)
|
| 3178 |
-
|
| 3179 |
for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
|
|
|
|
|
|
|
| 3180 |
if key in hidden or not (cols.get(key) or {}).get("sql"):
|
| 3181 |
return JSONResponse(status_code=400, content={
|
| 3182 |
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
|
|
@@ -3186,7 +3779,8 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
|
|
| 3186 |
raise ValueError("unapplied row wall")
|
| 3187 |
count_sql, params, limits = sheet_mod.build_aggregate_query(
|
| 3188 |
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 3189 |
-
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today
|
|
|
|
| 3190 |
if limits:
|
| 3191 |
raise ValueError("unapplied view filter")
|
| 3192 |
except sheet_mod.SheetRefusal as exc:
|
|
@@ -3227,8 +3821,20 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
|
|
| 3227 |
|
| 3228 |
|
| 3229 |
@router.get("/sheets/{workbook_id}/cells/{addr}/drill")
|
| 3230 |
-
|
| 3231 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3232 |
|
| 3233 |
β A FORMULA CELL REFUSES BY NAME AND DOES NOT DRILL TO ONE OPERAND (R17). Picking an input
|
| 3234 |
would be a guess presented as an answer: `=B4-B8` has two, and neither is what the person
|
|
@@ -3249,10 +3855,18 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
|
|
| 3249 |
|
| 3250 |
key = str(addr or "").replace("$", "").upper()
|
| 3251 |
cell = None
|
| 3252 |
-
|
| 3253 |
-
|
| 3254 |
-
|
| 3255 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3256 |
if not isinstance(cell, dict):
|
| 3257 |
raise err(404, "cell_missing", f"Cell {key} is not filled in.")
|
| 3258 |
if cell.get("kind") == "formula":
|
|
@@ -3282,11 +3896,7 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
|
|
| 3282 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 3283 |
|
| 3284 |
scope = binding["scope"]
|
| 3285 |
-
|
| 3286 |
-
scope_nodes, scope_conj = _view_filter(session, database, scope["view_id"],
|
| 3287 |
-
scope.get("view_name", ""))
|
| 3288 |
-
else:
|
| 3289 |
-
scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
|
| 3290 |
|
| 3291 |
# β THE SAME PERIOD THE NUMBER WAS COMPUTED AT (W53-T13). The drill probes `COUNT(*)` and never
|
| 3292 |
# reads the bound column, but its FILTER may name a Metric column, and a mirror opened at a
|
|
@@ -3297,25 +3907,20 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
|
|
| 3297 |
# window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from
|
| 3298 |
# the column set instead, where the time actually is.
|
| 3299 |
try:
|
| 3300 |
-
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=
|
| 3301 |
-
window = _ending_window(session, binding, book=book, tab=
|
| 3302 |
except sheet_mod.SheetRefusal as exc:
|
| 3303 |
raise err(400, exc.code, exc.message)
|
| 3304 |
-
related_entry = _binding_related_entry(session, database, binding)
|
| 3305 |
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
| 3306 |
| sheet_mod.filter_columns_named(wall_nodes))
|
| 3307 |
-
|
| 3308 |
-
needed.add(related_entry["via"].get("from"))
|
| 3309 |
cur, spec, cols = _mirror(session)(
|
| 3310 |
database, metric_window=window,
|
| 3311 |
needed=needed)
|
| 3312 |
cols = dict(cols)
|
| 3313 |
-
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols)
|
| 3314 |
|
| 3315 |
try:
|
| 3316 |
-
related =
|
| 3317 |
-
session, binding, related_entry, cur, spec, cols, window=window,
|
| 3318 |
-
book=book, tab=tab.get("id"))
|
| 3319 |
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
|
| 3320 |
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 3321 |
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
|
|
@@ -3330,10 +3935,21 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
|
|
| 3330 |
"connection. Edit the scope or ask an administrator to update the restriction.")
|
| 3331 |
total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
|
| 3332 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3333 |
record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
|
| 3334 |
|
| 3335 |
out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
|
| 3336 |
-
"limits": limits}
|
| 3337 |
if scope.get("view_id"):
|
| 3338 |
out["view_id"] = scope["view_id"]
|
| 3339 |
else:
|
|
@@ -3442,6 +4058,11 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
|
|
| 3442 |
database work, and a batch's cost is dominated by the number of distinct (database, period)
|
| 3443 |
pairs in it rather than by the number of cells.
|
| 3444 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3445 |
from core import sheets
|
| 3446 |
import formula_runtime as fr
|
| 3447 |
|
|
|
|
| 35 |
(standing rule 2), and each names both the cause and the subject.
|
| 36 |
"""
|
| 37 |
|
| 38 |
+
import contextlib
|
| 39 |
import hashlib
|
| 40 |
import json
|
| 41 |
import os
|
|
|
|
| 116 |
# β `TABLES` ITSELF IS UNTOUCHED β it has other consumers (`declared_schema_fields`,
|
| 117 |
# `_odoo_sources`' `register_connected`) that must keep seeing exactly eight rows.
|
| 118 |
is_registry_grid = fields is None and table_key in rel._REGISTRY_GRID_FIELDS
|
| 119 |
+
is_shipments = table_key == rel.SHIPMENTS_KEY
|
| 120 |
+
# ββ S66-T43 β THE THIRD ROSTER IS THE READ-THROUGH REGISTRY ITSELF. Lane G made every
|
| 121 |
+
# windowed Amazon table a `user_tables` stub served from the mirror through a provider in
|
| 122 |
+
# `routes_odoo_tables._sources()`, the same registry the Data grid's rows door reads. A key
|
| 123 |
+
# found there is exactly as connected as one in `TABLES`, and its FIELDS are the stub's own
|
| 124 |
+
# (`defn["fields"]`), which is where the rows door takes them from too.
|
| 125 |
+
# β `require_amazon_mirror` runs FIRST for an Amazon key: it admits a stub on a cold
|
| 126 |
+
# registry and refuses 503 with cause and fix when the mirror has not loaded the table,
|
| 127 |
+
# instead of this door calling a real Amazon table "not connected".
|
| 128 |
+
# β A STORED Amazon snapshot (under `MAX_ROWS`, e.g. listings) is served from the mirror
|
| 129 |
+
# too: its stored ids ARE the mirror's (`amazon_relational._read` stores `grid_source`'s
|
| 130 |
+
# id), and a relation from one Amazon table to another must resolve on ONE cursor
|
| 131 |
+
# (`_binding_related_query` refuses `rcur is not cur`), so both paths agree.
|
| 132 |
+
connector_defn = None
|
| 133 |
+
if fields is None and not is_registry_grid and not is_shipments:
|
| 134 |
+
connector_defn = _connector_definition(session, table_key)
|
| 135 |
+
if connector_defn is None:
|
| 136 |
+
raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
|
| 137 |
+
fields = list(connector_defn.get("fields") or [])
|
| 138 |
+
if not fields:
|
| 139 |
+
raise err(409, "schema_not_loaded",
|
| 140 |
+
f"{table_key} has no saved fields yet. Sync its connector first.")
|
| 141 |
try:
|
| 142 |
+
# The Amazon mirror opens without Odoo's ready() gate, exactly as the rows door does.
|
| 143 |
+
if str(table_key).startswith("ut_amzn_"):
|
| 144 |
+
# ONE Amazon cursor per request: `datastore.ro_cursor()` opens a new one per call,
|
| 145 |
+
# and a related condition joins two Amazon tables on the same cursor.
|
| 146 |
+
cur, mismatch = getattr(session, "_sheet_amzn_cursor", None) or (None, None)
|
| 147 |
+
if cur is None:
|
| 148 |
+
cur, mismatch = rot._mirror_cur(session, table_key)
|
| 149 |
+
if cur is not None:
|
| 150 |
+
session._sheet_amzn_cursor = (cur, None)
|
| 151 |
+
else:
|
| 152 |
+
cur, mismatch = rot._mirror_cur(session)
|
| 153 |
except RuntimeError as exc:
|
| 154 |
raise err(503, "store_not_ready", str(exc))
|
| 155 |
if mismatch:
|
| 156 |
raise err(409, "cross_tenant_store", mismatch)
|
| 157 |
+
if is_shipments:
|
| 158 |
+
from core import user_tables, perm_scope
|
| 159 |
+
from modules import sheet as sheet_mod
|
| 160 |
+
memo = getattr(session, "_sheet_stored_mirrors", None)
|
| 161 |
+
if memo is None:
|
| 162 |
+
memo = session._sheet_stored_mirrors = {}
|
| 163 |
+
if table_key not in memo:
|
| 164 |
+
stored = (user_tables.get(table_key, st=session.runtime) or {}).get("rows") or {}
|
| 165 |
+
# The store keeps records as {row id: row}. Iterating that dict as a list yields only
|
| 166 |
+
# the ids, which materialises an EMPTY table and answers every cell blank.
|
| 167 |
+
rows = [row for row in (stored.values() if isinstance(stored, dict) else stored)
|
| 168 |
+
if isinstance(row, dict)]
|
| 169 |
+
if not rows:
|
| 170 |
+
raise err(409, "sheet_database_empty",
|
| 171 |
+
f"{table_key} has no stored shipment rows. Sync shipments before binding this cell.")
|
| 172 |
+
fields = rel.shipment_fields()
|
| 173 |
+
hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
|
| 174 |
+
fields = [field for field in fields if field["key"] not in hidden]
|
| 175 |
+
spec_fields = sheet_mod.mirror_fields(fields)
|
| 176 |
+
spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows,
|
| 177 |
+
id_key="odoo_id", bulk=True)
|
| 178 |
+
memo[table_key] = (cur, {**spec, "fields": fields}, rot._filter_columns(spec, spec_fields))
|
| 179 |
+
return memo[table_key]
|
| 180 |
if is_registry_grid:
|
| 181 |
# ββ ONE ASSEMBLY PER REQUEST PER (DATABASE, WINDOW), NOT ONE PER CELL. `refresh_sheet`
|
| 182 |
# resolves a binding CELL BY CELL, and this branch now costs a full `/workspace`
|
|
|
|
| 219 |
memo[slot] = (None if want is None else set(want), made)
|
| 220 |
return made
|
| 221 |
spec = rot._source_for(cur, table_key)
|
| 222 |
+
if spec is None and connector_defn is not None and str(table_key).startswith("ut_amzn_"):
|
| 223 |
+
spec = _amazon_stored_spec(cur, table_key)
|
| 224 |
if spec is None:
|
| 225 |
raise err(409, "not_read_through",
|
| 226 |
f"{table_key} has no read-through binding on this deployment, so a cell cannot "
|
|
|
|
| 235 |
return _for
|
| 236 |
|
| 237 |
|
| 238 |
+
def _connector_definition(session: Session, table_key: str):
|
| 239 |
+
"""The stored stub of a read-through CONNECTOR grid (S66-T43), or None when it is not one.
|
| 240 |
+
|
| 241 |
+
Only a key the read-through registry serves qualifies (`routes_odoo_tables._sources()`), or an
|
| 242 |
+
Amazon table the connector declares (`amazon_relational.SOURCE`, stored or read-through), so an
|
| 243 |
+
ordinary typed database still answers `not_connected`. Memoised on the Session for the reason
|
| 244 |
+
`_mirror` memoises: `resolve_bindings` asks once per CELL and the Session dies with the request.
|
| 245 |
+
β `scope_applied=True` because every Sheet read compiles the row wall into its own SQL.
|
| 246 |
+
"""
|
| 247 |
+
import routes_odoo_tables as rot
|
| 248 |
+
import routes_tables
|
| 249 |
+
memo = getattr(session, "_sheet_connector_defs", None)
|
| 250 |
+
if memo is None:
|
| 251 |
+
memo = session._sheet_connector_defs = {}
|
| 252 |
+
if table_key in memo:
|
| 253 |
+
return memo[table_key]
|
| 254 |
+
key = str(table_key or "")
|
| 255 |
+
if not key.startswith("ut_"):
|
| 256 |
+
return None
|
| 257 |
+
defn = None
|
| 258 |
+
if key.startswith("ut_amzn_"):
|
| 259 |
+
import amazon_relational
|
| 260 |
+
if key not in amazon_relational.SOURCE:
|
| 261 |
+
return None
|
| 262 |
+
defn = routes_tables._defn_or_refuse(session, key, defs_only=True, scope_applied=True)
|
| 263 |
+
rot.require_amazon_mirror(key, session, defn)
|
| 264 |
+
elif key not in rot._sources():
|
| 265 |
+
return None
|
| 266 |
+
if defn is None:
|
| 267 |
+
defn = routes_tables._defn_or_refuse(session, key, defs_only=True, scope_applied=True)
|
| 268 |
+
memo[table_key] = defn
|
| 269 |
+
return defn
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _amazon_stored_spec(cur, table_key):
|
| 273 |
+
"""The mirror binding for a STORED Amazon snapshot, or the 503 that names the missing load."""
|
| 274 |
+
import amazon_relational
|
| 275 |
+
spec = amazon_relational.grid_source(cur, table_key)
|
| 276 |
+
if spec.get("missing"):
|
| 277 |
+
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
| 278 |
+
return spec
|
| 279 |
+
|
| 280 |
+
|
| 281 |
def _registry_assembly(session: Session, table_key: str):
|
| 282 |
"""The `g` dict `/workspace` builds for one compiled registry grid, or a NAMED refusal.
|
| 283 |
|
|
|
|
| 353 |
"shown rather than shown from an older set of columns. Try again in a moment.")
|
| 354 |
|
| 355 |
|
| 356 |
+
def _related_declarations(session, database):
|
| 357 |
+
"""The same permission-scoped catalogue for reads, rewrites and the picker.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 358 |
|
| 359 |
+
HOTFIX 2026-09-29: answered once per request inside `_sheet_request_scope`. Every
|
| 360 |
+
`related:` leaf used to re-read `user_tables.all_defs()` and re-scope every definition, once
|
| 361 |
+
per BINDING, so a Refresh of Daily Sales (~470 cells filtered on the customer's pricelist) read
|
| 362 |
+
the whole catalogue ~470 times (SDLC 67 S67-T21 finding).
|
| 363 |
+
"""
|
| 364 |
+
memo = getattr(session, "_sheet_read_memo", None)
|
| 365 |
+
slot = ("related", database)
|
| 366 |
+
if isinstance(memo, dict) and slot in memo:
|
| 367 |
+
return memo[slot]
|
| 368 |
+
from core import perm_scope, relations, user_tables
|
| 369 |
+
from routes_grid import _source_fields
|
| 370 |
|
| 371 |
+
fields = _source_fields(session, database)
|
| 372 |
+
defs = {}
|
| 373 |
+
for key, definition in (user_tables.all_defs(st=session.runtime) or {}).items():
|
| 374 |
+
if perm_scope.nav_may_open(session.user, key, st=session.runtime):
|
| 375 |
+
declared = definition.get("fields") or []
|
| 376 |
+
hidden = perm_scope.hidden_keys(session.user, key, declared, st=session.runtime)
|
| 377 |
+
defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
|
| 378 |
+
entries = relations.related(database, fields=fields, defs=defs)
|
| 379 |
+
# S66-T13 / C2: `relations.bridge_entries` lists a bridge only from its SOURCE, so
|
| 380 |
+
# `ut_odoo_orders` has no edge back to `customer_data` although every one-release
|
| 381 |
+
# `linked:bridge_customer_orders:<field>` leaf stands on exactly that edge. Read from the
|
| 382 |
+
# target the columns swap, as `reverse_links` swaps them. A bridge carrying `where` is
|
| 383 |
+
# skipped as `linked:` always skipped it, because that predicate narrows this side's rows.
|
| 384 |
+
seen = {relations.edge_identity(entry) for entry in entries}
|
| 385 |
+
for bridge in relations.BRIDGES:
|
| 386 |
+
if bridge.get("target") != database or bridge.get("where"):
|
| 387 |
+
continue
|
| 388 |
+
entry = {"key": bridge["key"], "label": bridge.get("source_label") or bridge["source"],
|
| 389 |
+
"via": {"direction": "bridge", "key": bridge["key"],
|
| 390 |
+
"on": bridge["source_col"], "from": bridge["target_col"]},
|
| 391 |
+
"target": bridge["source"], "served": bool(bridge.get("served"))}
|
| 392 |
+
if relations.edge_identity(entry) not in seen:
|
| 393 |
+
seen.add(relations.edge_identity(entry))
|
| 394 |
+
entries.append(entry)
|
| 395 |
+
if isinstance(memo, dict):
|
| 396 |
+
memo[slot] = (fields, defs, entries)
|
| 397 |
+
return fields, defs, entries
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def _legacy_linked_keys(database, entry):
|
| 401 |
+
"""The one-release `linked:` bridge keys an offered relationship replaces."""
|
| 402 |
+
from core import relations
|
| 403 |
+
via = entry.get("via") or {}
|
| 404 |
+
return [bridge["key"] for bridge in relations.BRIDGES
|
| 405 |
+
if bridge.get("target") == database and bridge.get("source") == entry.get("target")
|
| 406 |
+
and bridge.get("target_col") == via.get("from")
|
| 407 |
+
and bridge.get("source_col") == via.get("on") and not bridge.get("where")]
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
def _related_condition_fields(session, database):
|
| 411 |
from core import perm_scope
|
| 412 |
+
_require_database(session, database)
|
| 413 |
+
fields = list(_registry_assembly(session, database).get("fields") or [])
|
| 414 |
+
hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime)
|
| 415 |
+
return [field for field in fields if field.get("key") not in hidden
|
| 416 |
+
and field.get("type") not in {"formula", "link", "lookup", "rollup", "metric", "measure"}
|
| 417 |
+
and not field.get("measure")]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 418 |
|
| 419 |
|
| 420 |
+
def _rewrite_related_filters(session, database, nodes):
|
| 421 |
+
"""Read the one-release linked spelling as an ordinary related condition."""
|
| 422 |
+
from core import relations
|
| 423 |
from modules import sheet as sheet_mod
|
| 424 |
+
if not sheet_mod.linked_filter_ids(nodes):
|
| 425 |
+
return nodes
|
| 426 |
+
_fields, _defs, entries = _related_declarations(session, database)
|
| 427 |
+
bridges = {b["key"]: b for b in relations.BRIDGES if b.get("target") == database}
|
| 428 |
+
|
| 429 |
+
def resolve(leaf):
|
| 430 |
+
bridge_key, field_key = sheet_mod.linked_field_parts(leaf["colId"])
|
| 431 |
+
bridge = bridges.get(bridge_key)
|
| 432 |
+
entry = next((entry for entry in entries if bridge
|
| 433 |
+
and entry["target"] == bridge["source"]
|
| 434 |
+
and entry["via"].get("from") == bridge.get("target_col")
|
| 435 |
+
and entry["via"].get("on") == bridge.get("source_col")
|
| 436 |
+
and not entry.get("where")), None)
|
| 437 |
+
if entry is None:
|
| 438 |
+
raise sheet_mod.SheetRefusal("linked_filter_unreadable", leaf["colId"],
|
| 439 |
+
"its declared relationship is unavailable.")
|
| 440 |
+
return {**leaf, "colId": f'related:{entry["key"]}:{field_key}'}
|
| 441 |
+
|
| 442 |
try:
|
| 443 |
+
return sheet_mod.rewrite_linked_filters(nodes, resolve)
|
| 444 |
+
except sheet_mod.SheetRefusal as exc:
|
| 445 |
+
raise err(409, exc.code, exc.message)
|
|
|
|
|
|
|
|
|
|
| 446 |
|
| 447 |
|
| 448 |
@router.get("/sheets/filter-fields/{database}")
|
| 449 |
def linked_filter_fields(database: str, session: Session = Depends(require_session)):
|
| 450 |
+
"""Offer ordinary conditions grouped by their readable related database."""
|
| 451 |
+
from core import perm_scope, relations
|
| 452 |
database = _require_database(session, database)
|
| 453 |
+
_fields, defs, entries = _related_declarations(session, database)
|
| 454 |
+
offered, databases, seen = [], [], set()
|
| 455 |
+
for candidate in entries:
|
| 456 |
+
if not candidate.get("served") or not perm_scope.nav_may_open(
|
| 457 |
+
session.user, candidate["target"], st=session.runtime):
|
| 458 |
+
continue
|
| 459 |
+
identity = relations.edge_identity(candidate)
|
| 460 |
+
if identity in seen:
|
| 461 |
+
continue
|
| 462 |
try:
|
| 463 |
+
entry = _binding_related_entry(session, database, candidate["key"])
|
| 464 |
+
fields = _related_condition_fields(session, entry["target"])
|
| 465 |
except HTTPException as exc:
|
| 466 |
+
code = ((exc.detail or {}).get("error") or {}).get("code") if isinstance(exc.detail, dict) else None
|
| 467 |
+
if exc.status_code == 403 or code in {
|
| 468 |
+
"relation_source_unresolvable", "relation_target_unavailable", "unknown_relation"}:
|
| 469 |
continue
|
| 470 |
raise
|
| 471 |
+
if not entry["via"].get("from") or not entry["via"].get("on"):
|
| 472 |
continue
|
| 473 |
+
seen.add(identity)
|
| 474 |
+
target = entry["target"]
|
| 475 |
+
from core import registry
|
| 476 |
+
from routes_nav import _read_nav_meta
|
| 477 |
+
topic = registry.BY_KEY.get(target) or {}
|
| 478 |
+
label = (_read_nav_meta(session.runtime).get(target, {}).get("name")
|
| 479 |
+
or defs.get(target, {}).get("label")
|
| 480 |
+
or topic.get("label") or target)
|
| 481 |
+
metadata = {"via": entry["key"], "database": target, "databaseLabel": label,
|
| 482 |
+
"relationLabel": entry.get("label") or entry["key"]}
|
| 483 |
+
databases.append(metadata)
|
| 484 |
+
# The panel maps a saved one-release `linked:` leaf onto its related key with these.
|
| 485 |
+
legacy = _legacy_linked_keys(database, entry)
|
| 486 |
for field in fields:
|
| 487 |
+
field_label = field.get("label") or field["key"]
|
| 488 |
+
offered.append({**field, "key": f'related:{entry["key"]}:{field["key"]}',
|
| 489 |
+
"label": f'{label} via {metadata["relationLabel"]}: {field_label}',
|
| 490 |
+
"related": {**metadata, "field": field["key"], "fieldLabel": field_label,
|
| 491 |
+
"legacyKeys": [f"linked:{key}:{field['key']}"
|
| 492 |
+
for key in legacy]}})
|
| 493 |
+
return {"fields": offered, "databases": databases}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 494 |
|
| 495 |
|
| 496 |
def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
|
|
|
|
| 534 |
derived, bounded, limits = _metric_derived(session, table_key, g, metric_window)
|
| 535 |
rows = aios_grid.rows_from_pool(g["rows_src"], fields, (g.get("ws") or {}).get("overlays"),
|
| 536 |
derived=derived)
|
| 537 |
+
if table_key == "customer_data" and (needed is None or "agent_id" in needed):
|
| 538 |
+
from core import perm_scope
|
| 539 |
+
agent_field = {"key": "agent_id", "type": "int", "label": "Current agent ID"}
|
| 540 |
+
hidden = perm_scope.hidden_keys(session.user, table_key, [*fields, agent_field], st=session.runtime)
|
| 541 |
+
if "agent_id" not in hidden and "agent" not in hidden:
|
| 542 |
+
# Join only to already-scoped customer rows. pid is the actual res.partner id.
|
| 543 |
+
have = rel.columns(cur, "res_partner")
|
| 544 |
+
if "agent_id" in have:
|
| 545 |
+
agents = dict(cur.execute("SELECT id, agent_id FROM res_partner").fetchall())
|
| 546 |
+
rows = [{**row, "agent_id": agents.get(row.get("pid"))} for row in rows]
|
| 547 |
+
fields = [field for field in fields if field.get("key") != "agent_id"] + [agent_field]
|
| 548 |
+
elif needed is not None and "agent_id" in needed:
|
| 549 |
+
raise err(409, "related_invalid", "customer_data.agent_id is unavailable in the partner mirror.")
|
| 550 |
# The columns this cell can read, in the assembly's own order. `registry_grid_spec` forces
|
| 551 |
# `pid` in on its own, because `build_ids_query` needs `spec["id"]` to be a real column.
|
| 552 |
kept = (fields if needed is None
|
|
|
|
| 914 |
return HTTPException(status_code=409, detail={"error": body})
|
| 915 |
if exc.code in _FORBIDDEN_CODES:
|
| 916 |
return err(403, exc.code, exc.message)
|
| 917 |
+
if exc.code == "workbook_unavailable":
|
| 918 |
+
# The store did not answer; the workbook is not gone. 503 tells the client to retry.
|
| 919 |
+
return err(503, exc.code, exc.message)
|
| 920 |
status = 404 if exc.code in _MISSING_CODES else 400
|
| 921 |
return err(status, exc.code, exc.message)
|
| 922 |
|
|
|
|
| 1124 |
related_nodes, _ = _fe_parts(related.get("filter") if isinstance(related, dict) else None)
|
| 1125 |
nodes = [*nodes, *related_nodes]
|
| 1126 |
# Linked values and related-table grants can change independently of the orders' cache key.
|
| 1127 |
+
if (sheet_mod.linked_filter_ids(nodes) or sheet_mod.related_filter_ids(nodes)
|
| 1128 |
+
or related is not None):
|
| 1129 |
return True
|
| 1130 |
return any(sheet_mod.filter_cell_nodes(nodes))
|
| 1131 |
|
| 1132 |
|
| 1133 |
+
def _refresh_context(book, binding, memo=None, *, tab=None, session=None):
|
| 1134 |
+
"""Authored inputs and relative dates invalidate refresh, but never an open's snapshot.
|
| 1135 |
+
|
| 1136 |
+
HOTFIX 2026-09-29 (owner: "ONLY when we update ANY cells, its related cells also have to
|
| 1137 |
+
update"): with its `tab` and a request scope, a binding's inputs are the cells it actually
|
| 1138 |
+
READS, transitively (`_input_closure`), not the whole workbook. The whole-workbook fingerprint
|
| 1139 |
+
meant one edit anywhere invalidated every one of Daily Sales' ~470 bindings, so every Refresh
|
| 1140 |
+
after any edit re-resolved all of them. The tab list rides the fingerprint, so a rename or a
|
| 1141 |
+
reorder still invalidates everything. Without a scope the old whole-workbook reading stands.
|
| 1142 |
+
"""
|
| 1143 |
from core.sheet_binding import fingerprint
|
| 1144 |
from datetime import date
|
| 1145 |
payload = {}
|
| 1146 |
+
graph = None
|
| 1147 |
+
if session is not None and tab is not None and _has_cell_inputs(binding):
|
| 1148 |
+
import formula_runtime as fr
|
| 1149 |
+
graph = _read_graph(session, book, fr, None)
|
| 1150 |
+
if graph is not None:
|
| 1151 |
+
import formula_runtime as fr
|
| 1152 |
+
tabs = book.get("sheets") or []
|
| 1153 |
+
home = _resolve_home_tab(book, tabs, graph["names"], tab)
|
| 1154 |
+
engine = _read_engine(graph, book, fr, None)[0]
|
| 1155 |
+
closure = _input_closure(graph, engine, fr, _qualified_refs(binding, home, fr, graph))
|
| 1156 |
+
authored = {}
|
| 1157 |
+
for key in sorted(closure):
|
| 1158 |
+
cell = graph["cells"].get(key)
|
| 1159 |
+
authored[key] = None if not isinstance(cell, dict) else {
|
| 1160 |
+
k: cell.get(k) for k in ("kind", "text", "formula", "binding", "value")
|
| 1161 |
+
if k != "value" or cell.get("kind") in ("literal", "empty")}
|
| 1162 |
+
payload["inputs"] = fingerprint([[[t.get("id"), t.get("name")] for t in tabs], authored])
|
| 1163 |
+
elif _has_cell_inputs(binding):
|
| 1164 |
if memo is None:
|
| 1165 |
memo = {}
|
| 1166 |
if "inputs" not in memo:
|
|
|
|
| 1271 |
# and MCP read agree on stored cells, figures and formats` measured exactly that pair: D1 and
|
| 1272 |
# F1 fresh 30.0 / 60.0 from MCP against stale-and-blank from the UI. A non-browser consumer has
|
| 1273 |
# no client engine to cover for it, so the blank is what it would have shown.
|
| 1274 |
+
# HOTFIX 2026-09-29: a `refused` formula is a COMPUTED answer (an error is a result), so it no
|
| 1275 |
+
# longer forces a whole-workbook recompute on every tab read. Only a formula with no answer at
|
| 1276 |
+
# all (`stale`, or no value stored) does. One `#DIV/0!` anywhere used to cost every read ~1 s.
|
| 1277 |
stale_formula = any(
|
| 1278 |
+
cell.get("kind") == "formula" and (cell.get("state") == "stale" or "value" not in cell)
|
| 1279 |
for tab in (fresh.get("sheets") or []) for cell in (tab.get("cells") or {}).values())
|
| 1280 |
if changed_values or stale_formula:
|
| 1281 |
from modules import sheet as sheet_mod
|
|
|
|
| 1891 |
A formula sitting on a tab nobody has open still shows its last answer until that tab is
|
| 1892 |
opened or the workbook is refreshed, exactly as a stale database-bound cell does.
|
| 1893 |
"""
|
| 1894 |
+
with _sheet_request_scope(session):
|
| 1895 |
+
return _recomputed_tab_cells_scoped(session, workbook_id, tab_ref, changes=changes)
|
| 1896 |
+
|
| 1897 |
+
|
| 1898 |
+
def _recomputed_tab_cells_scoped(session: Session, workbook_id: str, tab_ref: str, *, changes=None):
|
| 1899 |
from core import sheets
|
| 1900 |
from modules import sheet as sheet_mod
|
| 1901 |
import formula_runtime as fr
|
| 1902 |
+
from time import perf_counter
|
| 1903 |
|
| 1904 |
+
started = perf_counter()
|
| 1905 |
try:
|
| 1906 |
book = sheets.get_workbook(_store(session), workbook_id)
|
| 1907 |
found = sheets.find_tab(book, tab_ref)
|
|
|
|
| 1915 |
(found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes}
|
| 1916 |
_recompute_formulas(book, fr, sheet_mod,
|
| 1917 |
cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
|
| 1918 |
+
# HOTFIX 2026-09-29 (owner: "we should only update or refresh cells that are affected by an
|
| 1919 |
+
# operation"): a commit re-resolves ONLY the database cells whose inputs read what it changed.
|
| 1920 |
+
# This loop used to resolve EVERY bound cell with a cell input on EVERY commit, whatever was
|
| 1921 |
+
# edited: ~470 SQL resolves (each rebuilding the whole formula graph) per keystroke on Daily
|
| 1922 |
+
# Sales. `_dependent_bound_keys` walks the formula graph forward from the edited cells.
|
| 1923 |
+
graph = _read_graph(session, book, fr, None)
|
| 1924 |
+
targets, mode = None, "all"
|
| 1925 |
+
if edits is not None and graph is not None:
|
| 1926 |
+
try:
|
| 1927 |
+
targets = _dependent_bound_keys(book, graph, fr, set(edits))
|
| 1928 |
+
# A bound cell written by this commit resolves too, as it always did.
|
| 1929 |
+
targets |= {key for key in edits if key in graph["bound"]}
|
| 1930 |
+
mode = "dependents"
|
| 1931 |
+
except Exception as exc: # noqa: BLE001
|
| 1932 |
+
# Unprovable dependents fall back to the old whole-workbook pass: slow, never wrong.
|
| 1933 |
+
try:
|
| 1934 |
+
import harness.telemetry as _tel
|
| 1935 |
+
_tel.error("api:sheet-dependents", exc)
|
| 1936 |
+
except Exception: # noqa: BLE001
|
| 1937 |
+
pass
|
| 1938 |
+
targets, mode = None, "all"
|
| 1939 |
ending_cells = set()
|
| 1940 |
refreshed_entries = []
|
| 1941 |
context_memo = {}
|
| 1942 |
+
view_memo = {}
|
| 1943 |
+
names = _tab_scope_names(book.get("sheets") or [])
|
| 1944 |
+
for source, name in zip(book.get("sheets") or [], names):
|
| 1945 |
for addr, cell in (source.get("cells") or {}).items():
|
| 1946 |
binding = cell.get("binding") or {}
|
| 1947 |
+
if cell.get("kind") != "bound" or _is_view_binding(cell) or not _has_cell_inputs(binding):
|
| 1948 |
+
continue
|
| 1949 |
+
key = fr.a1_key(addr, name)
|
| 1950 |
+
if targets is not None and key not in targets:
|
| 1951 |
continue
|
| 1952 |
try:
|
| 1953 |
+
result = _resolve_cell_binding(session, binding, book, source.get("id"),
|
| 1954 |
+
view_memo=view_memo)
|
| 1955 |
cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
|
| 1956 |
cell["refreshed_at"] = _now_iso()
|
| 1957 |
if result.get("limits"):
|
|
|
|
| 1959 |
else:
|
| 1960 |
cell.pop("limits", None)
|
| 1961 |
cell.pop("refusal", None)
|
| 1962 |
+
if graph is not None:
|
| 1963 |
+
graph["bound_values"][key] = result
|
| 1964 |
except HTTPException as exc:
|
| 1965 |
info = ((exc.detail or {}).get("error") or {})
|
| 1966 |
cell.update(_binding_refusal(
|
|
|
|
| 1970 |
ending_cells.add((source.get("id"), addr))
|
| 1971 |
refreshed_entries.append((str(source.get("id")) + "!" + addr,
|
| 1972 |
_binding_scope_key(session, str(binding.get("database") or ""), binding),
|
| 1973 |
+
{"tokens": {}, "context": _refresh_context(book, binding, context_memo,
|
| 1974 |
+
tab=source.get("id"), session=session),
|
| 1975 |
"cell": {k: cell[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
|
| 1976 |
if k in cell}}))
|
| 1977 |
if ending_cells:
|
|
|
|
| 1980 |
{c.get("binding", {}).get("database") for t in book.get("sheets", [])
|
| 1981 |
for c in t.get("cells", {}).values() if c.get("kind") == "bound"})
|
| 1982 |
_recompute_formulas(book, fr, sheet_mod)
|
| 1983 |
+
stats = _read_stats(session) or {}
|
| 1984 |
+
print(f"[sheet] commit wb={workbook_id} tab={tab_ref} edited={len(edits or {})} mode={mode} "
|
| 1985 |
+
f"bound_resolved={len(ending_cells)} engines={stats.get('engines', 0)} "
|
| 1986 |
+
f"input_hits={stats.get('input_hits', 0)} "
|
| 1987 |
+
f"ms={round((perf_counter() - started) * 1000.0)}", flush=True)
|
| 1988 |
found = sheets.find_tab(book, tab_ref)
|
| 1989 |
out = {}
|
| 1990 |
for addr, cell in (found.get("cells") or {}).items():
|
|
|
|
| 2158 |
|
| 2159 |
|
| 2160 |
@router.post("/sheets/{workbook_id}/refresh")
|
| 2161 |
+
def refresh_sheet(workbook_id: str, body: dict = Body(default=None),
|
| 2162 |
+
session: Session = Depends(require_session)):
|
| 2163 |
"""C2/R4: Refresh resolves only databases whose token or binding inputs changed.
|
| 2164 |
|
| 2165 |
Viewers refresh their own server cache. Only editors save the workbook document.
|
| 2166 |
One token is read per distinct database on the unchanged path, regardless of cell count.
|
| 2167 |
+
|
| 2168 |
+
HOTFIX 2026-09-29: an optional body `{"cells": ["<tab id>!<ADDR>", ...]}` refreshes ONLY
|
| 2169 |
+
those bound cells (the open-time repair of the few cells this reader has no answer for, and
|
| 2170 |
+
the cells a row or column edit left stale). Every other cell keeps its stored answer.
|
| 2171 |
"""
|
| 2172 |
+
wanted = (body or {}).get("cells") if isinstance(body, dict) else None
|
| 2173 |
+
only = ({str(item) for item in wanted if isinstance(item, str) and "!" in item}
|
| 2174 |
+
if isinstance(wanted, list) else None)
|
| 2175 |
+
with _sheet_request_scope(session):
|
| 2176 |
+
return _refresh_sheet_scoped(workbook_id, session, only=only)
|
| 2177 |
+
|
| 2178 |
+
|
| 2179 |
+
def _refresh_sheet_scoped(workbook_id: str, session: Session, *, only=None):
|
| 2180 |
from copy import deepcopy
|
| 2181 |
from core import sheets, sheet_binding
|
| 2182 |
from modules import sheet as sheet_mod
|
|
|
|
| 2204 |
|
| 2205 |
def input_databases(binding, tab_ref):
|
| 2206 |
nonlocal input_graph
|
| 2207 |
+
binding = dict(binding)
|
| 2208 |
+
if "related" in binding:
|
| 2209 |
+
binding = sheet_mod.clean_binding(binding)
|
| 2210 |
+
nodes, _ = _binding_filter_nodes(session, binding["database"], binding, memo=view_memo)
|
| 2211 |
refs = [str((node.get(key) or {}).get("ref") or "")
|
| 2212 |
+
for node in sheet_mod.filter_cell_nodes(nodes)
|
| 2213 |
for key in ("valueRef", "value2Ref") if key in node]
|
| 2214 |
period = binding.get("period") or {}
|
| 2215 |
if period.get("ref"):
|
|
|
|
| 2219 |
tabs = book.get("sheets") or []
|
| 2220 |
names = _tab_scope_names(tabs)
|
| 2221 |
if input_graph is None:
|
| 2222 |
+
shared = _read_graph(session, book, fr, None)
|
| 2223 |
+
if shared is not None:
|
| 2224 |
+
# The request's one engine (bound cells pending): its edges are the same edges.
|
| 2225 |
+
input_graph = shared["cells"], _read_engine(shared, book, fr, None)[0]
|
| 2226 |
+
else:
|
| 2227 |
+
documents = {fr.a1_key(addr, name): cell for tab, name in zip(tabs, names)
|
| 2228 |
+
for addr, cell in (tab.get("cells") or {}).items()}
|
| 2229 |
+
engine = fr.SheetCalculation({key: fr._document_spec(cell, key.rpartition("!")[0])
|
| 2230 |
+
for key, cell in documents.items()}, sheets=names,
|
| 2231 |
+
named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
|
| 2232 |
+
input_graph = documents, engine
|
| 2233 |
documents, engine = input_graph
|
| 2234 |
home = _resolve_home_tab(book, tabs, names, tab_ref)
|
| 2235 |
+
shared = _read_graph(session, book, fr, None)
|
| 2236 |
+
pending = ([key for ref in refs for key in
|
| 2237 |
+
(_formula_slot_keys(shared, fr, ref, home) if ref.startswith("=")
|
| 2238 |
+
else [_asked_key(ref, home, fr)])] if shared is not None
|
| 2239 |
+
else [_asked_key(ref, home, fr) for ref in refs if not ref.startswith("=")])
|
| 2240 |
+
visited, deps = set(), set()
|
| 2241 |
while pending:
|
| 2242 |
key = pending.pop()
|
| 2243 |
if key in visited:
|
|
|
|
| 2245 |
visited.add(key)
|
| 2246 |
cell = documents.get(key) or {}
|
| 2247 |
if cell.get("kind") == "bound":
|
| 2248 |
+
source = dict(cell.get("binding") or {})
|
| 2249 |
+
if "related" in source:
|
| 2250 |
+
source = sheet_mod.clean_binding(source)
|
| 2251 |
+
source_db = str(source.get("database") or "")
|
| 2252 |
+
deps.add(source_db)
|
| 2253 |
+
source_nodes, _ = _binding_filter_nodes(session, source_db, source, memo=view_memo)
|
| 2254 |
+
deps.update(_binding_related_entry(session, source_db, via)["target"]
|
| 2255 |
+
for via in sheet_mod.related_vias(source_nodes))
|
| 2256 |
pending.extend(engine.consumed.get(key, ()))
|
| 2257 |
return deps
|
| 2258 |
|
|
|
|
| 2273 |
|
| 2274 |
def dependencies(db, binding, tab_ref):
|
| 2275 |
deps = {db}
|
| 2276 |
+
binding = dict(binding)
|
| 2277 |
+
if "related" in binding:
|
| 2278 |
+
binding = sheet_mod.clean_binding(binding)
|
| 2279 |
+
nodes, _ = _binding_filter_nodes(session, db, binding, memo=view_memo)
|
| 2280 |
+
for via in sheet_mod.related_vias(nodes):
|
| 2281 |
+
key = (db, via)
|
| 2282 |
if key not in related_memo:
|
| 2283 |
+
related_memo[key] = _binding_related_entry(session, db, via)
|
| 2284 |
deps.add(related_memo[key]["target"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2285 |
deps.update(input_databases(binding, tab_ref))
|
| 2286 |
return deps
|
| 2287 |
|
|
|
|
| 2301 |
hit = cache.get("refreshCells", {}).get(str(tab_ref) + "!" + addr, {}).get(scope_key)
|
| 2302 |
sources = {db: token(db) for db in dependencies(db, binding, tab_ref)}
|
| 2303 |
if (hit and all(sources.values()) and hit.get("tokens") == sources
|
| 2304 |
+
and hit.get("context") == _refresh_context(book, binding, context_memo,
|
| 2305 |
+
tab=tab_ref, session=session)
|
| 2306 |
and hit["cell"].get("state") == "fresh"):
|
| 2307 |
return hit["cell"]
|
| 2308 |
return None
|
| 2309 |
|
| 2310 |
+
started = perf_counter()
|
| 2311 |
+
resolved_count = 0
|
| 2312 |
+
scope_names = {id(tab): name for tab, name in
|
| 2313 |
+
zip(book.get("sheets") or [], _tab_scope_names(book.get("sheets") or []))}
|
| 2314 |
for tab in book.get("sheets") or []:
|
| 2315 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 2316 |
if cell.get("kind") != "bound" or _is_view_binding(cell):
|
|
|
|
| 2318 |
binding = cell.get("binding") or {}
|
| 2319 |
db = str(binding.get("database") or "")
|
| 2320 |
cell_key = str(tab.get("id")) + "!" + addr
|
| 2321 |
+
if only is not None and cell_key not in only:
|
| 2322 |
+
continue
|
| 2323 |
scope_key = _binding_scope_key(session, db, binding)
|
| 2324 |
+
context = _refresh_context(book, binding, context_memo, tab=tab.get("id"),
|
| 2325 |
+
session=session)
|
| 2326 |
hit = cache.get("refreshCells", {}).get(cell_key, {}).get(scope_key)
|
| 2327 |
source_tokens = {}
|
| 2328 |
try:
|
|
|
|
| 2342 |
status.setdefault(db, "skipped")
|
| 2343 |
continue
|
| 2344 |
status[db] = "resolved"
|
| 2345 |
+
resolved_count += 1
|
| 2346 |
previous = getattr(session, "_sheet_refresh_input", None)
|
| 2347 |
session._sheet_refresh_input = cached_input
|
| 2348 |
try:
|
| 2349 |
out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
|
| 2350 |
finally:
|
| 2351 |
session._sheet_refresh_input = previous
|
| 2352 |
+
shared = _read_graph(session, book, fr, None)
|
| 2353 |
+
if shared is not None:
|
| 2354 |
+
shared["bound_values"][fr.a1_key(addr, scope_names.get(id(tab), ""))] = out
|
| 2355 |
limits = list(out.get("limits") or [])
|
| 2356 |
if not all(source_tokens.values()):
|
| 2357 |
limits.append({"code": "change_token_unavailable",
|
|
|
|
| 2362 |
"refreshed_at": stamp}
|
| 2363 |
if limits:
|
| 2364 |
snapshot["limits"] = limits
|
| 2365 |
+
except (HTTPException, sheet_mod.SheetRefusal) as exc:
|
| 2366 |
+
info = ({"code": exc.code, "message": exc.message}
|
| 2367 |
+
if isinstance(exc, sheet_mod.SheetRefusal)
|
| 2368 |
+
else ((exc.detail or {}).get("error") or {}))
|
| 2369 |
snapshot = _binding_refusal(info.get("code", "sheet_unresolved"),
|
| 2370 |
info.get("message", "This cell could not be refreshed."))
|
| 2371 |
status[db] = "refused"
|
|
|
|
| 2396 |
tab_ref, addr = cell_key.split("!", 1)
|
| 2397 |
_snapshot_cell(sheets.find_tab(book, tab_ref)["cells"][addr], entry["cell"])
|
| 2398 |
|
| 2399 |
+
resolve_ms = round((perf_counter() - started) * 1000.0)
|
| 2400 |
started = perf_counter()
|
| 2401 |
_recompute_formulas(book, fr, sheet_mod)
|
| 2402 |
recalc_ms = round((perf_counter() - started) * 1000.0, 1)
|
| 2403 |
+
stats = _read_stats(session) or {}
|
| 2404 |
+
print(f"[sheet] refresh wb={workbook_id} cells={'all' if only is None else len(only)} "
|
| 2405 |
+
f"bound_resolved={resolved_count} engines={stats.get('engines', 0)} "
|
| 2406 |
+
f"input_hits={stats.get('input_hits', 0)} resolve_ms={resolve_ms} "
|
| 2407 |
+
f"recalc_ms={recalc_ms}", flush=True)
|
| 2408 |
if editable:
|
| 2409 |
try:
|
| 2410 |
saved = sheets.put_workbook(store, workbook_id,
|
|
|
|
| 2419 |
saved["refreshedFor"] = str(session.uname or "")
|
| 2420 |
if entries:
|
| 2421 |
sheet_binding.save_refresh_cache(store, workbook_id, entries, status)
|
| 2422 |
+
if only is not None:
|
| 2423 |
+
# β C5: a targeted answer carries ONLY the cells it resolved for this caller. The rest of
|
| 2424 |
+
# the document holds the owner's stored answers, which a scoped reader must never adopt.
|
| 2425 |
+
return {"workbook": {key: saved.get(key) for key in ("id", "name", "rev", "refreshed_at")},
|
| 2426 |
+
"cells": {cell_key: entry["cell"] for cell_key, _, entry in entries},
|
| 2427 |
+
"recalc_ms": recalc_ms,
|
| 2428 |
+
"binding_refresh": {"databases": status, "token_reads": token_reads}}
|
| 2429 |
return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms,
|
| 2430 |
"binding_refresh": {"databases": status, "token_reads": token_reads}}
|
| 2431 |
|
|
|
|
| 2571 |
def _forget_sheet_database(session, database):
|
| 2572 |
"""A field write invalidates both request-local schema and materialised-column memos."""
|
| 2573 |
getattr(session, "_sheet_assemblies", {}).pop(database, None)
|
| 2574 |
+
memo = getattr(session, "_sheet_read_memo", None)
|
| 2575 |
+
if isinstance(memo, dict):
|
| 2576 |
+
for slot in [slot for slot in memo if isinstance(slot, tuple) and slot[0] == "related"]:
|
| 2577 |
+
memo.pop(slot, None)
|
| 2578 |
mirrors = getattr(session, "_sheet_mirrors", {})
|
| 2579 |
for slot in list(mirrors):
|
| 2580 |
if slot[0] == database:
|
|
|
|
| 2701 |
return sheet_mod.period_window(period, today, cell_date=date)
|
| 2702 |
|
| 2703 |
|
| 2704 |
+
@contextlib.contextmanager
|
| 2705 |
+
def _sheet_request_scope(session):
|
| 2706 |
+
"""HOTFIX 2026-09-29: one memo for one Sheet request, dropped when the request ends.
|
| 2707 |
+
|
| 2708 |
+
β The Session is built fresh per request (`deps.require_session`), but gates reuse one
|
| 2709 |
+
SimpleNamespace across calls, so the memo is OPT-IN, installed here and removed on exit.
|
| 2710 |
+
Inside it the workbook's authored cells cannot change: every door that opens a scope reads one
|
| 2711 |
+
document and resolves against it, so a cell read or a bound input answered once is the answer
|
| 2712 |
+
for the rest of the request. β Never open a scope around code that edits cells between two
|
| 2713 |
+
reads of the same document object (the Daily Sales repair projects in rounds and does not).
|
| 2714 |
+
"""
|
| 2715 |
+
previous = getattr(session, "_sheet_read_memo", None)
|
| 2716 |
+
memo = {"stats": {"engines": 0, "resolves": 0, "input_hits": 0}}
|
| 2717 |
+
try:
|
| 2718 |
+
session._sheet_read_memo = memo
|
| 2719 |
+
except Exception: # noqa: BLE001
|
| 2720 |
+
yield None
|
| 2721 |
+
return
|
| 2722 |
+
try:
|
| 2723 |
+
yield memo
|
| 2724 |
+
finally:
|
| 2725 |
+
session._sheet_read_memo = previous
|
| 2726 |
+
|
| 2727 |
+
|
| 2728 |
+
def _request_scoped(route):
|
| 2729 |
+
"""Run a route inside `_sheet_request_scope`, keeping its signature for FastAPI and its body in
|
| 2730 |
+
place for the source gates that read it."""
|
| 2731 |
+
import functools
|
| 2732 |
+
import inspect as _inspect
|
| 2733 |
+
signature = _inspect.signature(route)
|
| 2734 |
+
|
| 2735 |
+
@functools.wraps(route)
|
| 2736 |
+
def scoped(*args, **kwargs):
|
| 2737 |
+
session = signature.bind_partial(*args, **kwargs).arguments.get("session")
|
| 2738 |
+
if session is None:
|
| 2739 |
+
return route(*args, **kwargs)
|
| 2740 |
+
with _sheet_request_scope(session):
|
| 2741 |
+
return route(*args, **kwargs)
|
| 2742 |
+
return scoped
|
| 2743 |
+
|
| 2744 |
+
|
| 2745 |
+
def _read_stats(session):
|
| 2746 |
+
memo = getattr(session, "_sheet_read_memo", None)
|
| 2747 |
+
return memo.get("stats") if isinstance(memo, dict) else None
|
| 2748 |
+
|
| 2749 |
+
|
| 2750 |
+
def _read_graph(session, book, fr, today):
|
| 2751 |
+
"""This request's cell map and formula graph for `book`, or None outside a request scope."""
|
| 2752 |
+
memo = getattr(session, "_sheet_read_memo", None)
|
| 2753 |
+
if not isinstance(memo, dict) or not isinstance(book, dict):
|
| 2754 |
+
return None
|
| 2755 |
+
slot = ("graph", id(book), str(today or ""))
|
| 2756 |
+
graph = memo.get(slot)
|
| 2757 |
+
if graph is None or graph["book"] is not book:
|
| 2758 |
+
graph = memo[slot] = _new_read_graph(book, fr, memo.get("stats"))
|
| 2759 |
+
return graph
|
| 2760 |
+
|
| 2761 |
+
|
| 2762 |
+
def _new_read_graph(book, fr, stats=None):
|
| 2763 |
+
tabs = book.get("sheets") or []
|
| 2764 |
+
names = _tab_scope_names(tabs)
|
| 2765 |
+
cells = {fr.a1_key(addr, name): cell for source, name in zip(tabs, names)
|
| 2766 |
+
for addr, cell in (source.get("cells") or {}).items() if isinstance(cell, dict)}
|
| 2767 |
+
return {"book": book, "names": names, "cells": cells,
|
| 2768 |
+
"bound": {k: c for k, c in cells.items() if c.get("kind") == "bound"},
|
| 2769 |
+
"engine": None, "attempted": set(), "failures": {}, "bound_values": {},
|
| 2770 |
+
"formulas": {}, "stats": stats}
|
| 2771 |
+
|
| 2772 |
+
|
| 2773 |
+
#: Where a filter formula is evaluated: a private one-formula engine, never a workbook address.
|
| 2774 |
+
_FORMULA_SLOT_ADDR = "XFD1048576"
|
| 2775 |
+
|
| 2776 |
+
|
| 2777 |
+
def _formula_slot(graph, fr, formula, home):
|
| 2778 |
+
"""A filter formula, compiled once per request: (slot key, cells it reads, ranges it reads).
|
| 2779 |
+
|
| 2780 |
+
HOTFIX 2026-09-29 (owner: "make formula native inside the Cell database Filter so it can point
|
| 2781 |
+
to an existing date but we can use something like a date function refer to a cell"). A filter
|
| 2782 |
+
value slot holds a cell reference (`F7`) OR a formula (`=DATE($E$24,1,1)`), so a monthly total
|
| 2783 |
+
needs no helper cell holding its first and last day. `cells` is None when it does not compile.
|
| 2784 |
+
"""
|
| 2785 |
+
cache = graph.setdefault("formulas", {})
|
| 2786 |
+
slot = (str(home).upper(), formula)
|
| 2787 |
+
if slot not in cache:
|
| 2788 |
+
vkey = fr.a1_key(_FORMULA_SLOT_ADDR, home)
|
| 2789 |
+
book = graph["book"]
|
| 2790 |
+
try:
|
| 2791 |
+
probe = fr.SheetCalculation({vkey: {"formula": formula, "sheet": home}},
|
| 2792 |
+
sheets=graph["names"], named_ranges=book.get("namedRanges"),
|
| 2793 |
+
date_system=book.get("dateSystem", "1900"))
|
| 2794 |
+
compiled = probe.compiled.get(vkey)
|
| 2795 |
+
except Exception: # noqa: BLE001
|
| 2796 |
+
compiled = None
|
| 2797 |
+
cache[slot] = (vkey, set(compiled["cells"]) if compiled else None,
|
| 2798 |
+
list(compiled["ranges"]) if compiled else [])
|
| 2799 |
+
return cache[slot]
|
| 2800 |
+
|
| 2801 |
+
|
| 2802 |
+
def _formula_slot_keys(graph, fr, formula, home):
|
| 2803 |
+
"""Every workbook cell a filter formula reads (range members included), or [] if unreadable."""
|
| 2804 |
+
_vkey, reads, ranges = _formula_slot(graph, fr, formula, home)
|
| 2805 |
+
keys = set(reads or ())
|
| 2806 |
+
if ranges:
|
| 2807 |
+
engine = _read_engine(graph, graph["book"], fr, None)[0]
|
| 2808 |
+
for bounds in ranges:
|
| 2809 |
+
keys.update(engine.positions.within(bounds))
|
| 2810 |
+
return sorted(keys)
|
| 2811 |
+
|
| 2812 |
+
|
| 2813 |
+
def _read_binding_formula(session, formula, *, book, tab, today, seen, code, as_date):
|
| 2814 |
+
"""A filter / ending value computed by a formula over this workbook's cells, never stored."""
|
| 2815 |
+
from modules import sheet as sheet_mod
|
| 2816 |
+
import formula_runtime as fr
|
| 2817 |
+
|
| 2818 |
+
def refuse(reason):
|
| 2819 |
+
exc = sheet_mod.SheetRefusal(code, formula, reason)
|
| 2820 |
+
exc.message = exc.message.replace("from cell =", "from formula =", 1)
|
| 2821 |
+
exc.args = (exc.message,)
|
| 2822 |
+
raise exc
|
| 2823 |
+
|
| 2824 |
+
graph = _read_graph(session, book, fr, today) or _new_read_graph(book, fr)
|
| 2825 |
+
home = _resolve_home_tab(book, book.get("sheets") or [], graph["names"], tab)
|
| 2826 |
+
vkey, reads, _ranges = _formula_slot(graph, fr, formula, home)
|
| 2827 |
+
if reads is None:
|
| 2828 |
+
refuse("the formula could not be read.")
|
| 2829 |
+
specs = {vkey: {"formula": formula, "sheet": home}}
|
| 2830 |
+
reasons = {}
|
| 2831 |
+
for key in _formula_slot_keys(graph, fr, formula, home):
|
| 2832 |
+
if key == vkey:
|
| 2833 |
+
continue
|
| 2834 |
+
if key in seen:
|
| 2835 |
+
refuse("the reference is circular.")
|
| 2836 |
+
cell = graph["cells"].get(key)
|
| 2837 |
+
if not isinstance(cell, dict) or cell.get("kind") == "empty":
|
| 2838 |
+
if key in reads:
|
| 2839 |
+
# A cell the formula names directly must hold something: a deleted input refuses
|
| 2840 |
+
# by name, as a cell reference does, never a year-zero date. The browser twin is
|
| 2841 |
+
# `sheetState.readFormulaSlot`. A blank range member stays blank.
|
| 2842 |
+
refuse(f"cell {key.rpartition('!')[2]} is blank.")
|
| 2843 |
+
continue
|
| 2844 |
+
try:
|
| 2845 |
+
specs[key] = {"value": _read_binding_cell(session, key, book=book, tab=tab, today=today,
|
| 2846 |
+
seen=seen, code=code),
|
| 2847 |
+
**({"style": cell["style"]} if isinstance(cell.get("style"), dict) else {})}
|
| 2848 |
+
except sheet_mod.SheetRefusal as exc:
|
| 2849 |
+
specs[key] = {"refused": exc.code}
|
| 2850 |
+
reasons[key] = exc.message
|
| 2851 |
+
probe = fr.SheetCalculation(specs, sheets=graph["names"], today=today,
|
| 2852 |
+
named_ranges=book.get("namedRanges"),
|
| 2853 |
+
date_system=book.get("dateSystem", "1900"))
|
| 2854 |
+
if probe.errors.get(vkey):
|
| 2855 |
+
error_code = probe.errors[vkey]
|
| 2856 |
+
refuse(next(iter(reasons.values()), None) or fr._sheet_error_message(error_code))
|
| 2857 |
+
value = probe.values.get(vkey)
|
| 2858 |
+
if value is None or isinstance(value, str) and not value.strip():
|
| 2859 |
+
refuse("its result is blank.")
|
| 2860 |
+
if (as_date and isinstance(value, (int, float)) and not isinstance(value, bool)
|
| 2861 |
+
and sheet_mod.date_formatted_cell({"style": {"format": probe.formats.get(vkey)}})):
|
| 2862 |
+
from harness import windows as W
|
| 2863 |
+
day = W.ending_cell_date(value, book.get("dateSystem", "1900"))
|
| 2864 |
+
if not day:
|
| 2865 |
+
refuse("its value is not a real date.")
|
| 2866 |
+
return day
|
| 2867 |
+
return value
|
| 2868 |
+
|
| 2869 |
+
|
| 2870 |
+
def _read_engine(graph, book, fr, today):
|
| 2871 |
+
"""ONE formula engine per request, database inputs pending until a read needs them."""
|
| 2872 |
+
if graph["engine"] is None:
|
| 2873 |
+
specs = {k: ({"refused": "ending_input_pending"} if k in graph["bound"]
|
| 2874 |
+
else fr._document_spec(c, k.rpartition("!")[0]))
|
| 2875 |
+
for k, c in graph["cells"].items()}
|
| 2876 |
+
graph["engine"] = fr.SheetCalculation(
|
| 2877 |
+
specs, sheets=graph["names"], today=today,
|
| 2878 |
+
named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
|
| 2879 |
+
stats = graph.get("stats")
|
| 2880 |
+
if stats is not None:
|
| 2881 |
+
stats["engines"] += 1
|
| 2882 |
+
return graph["engine"], graph["attempted"], graph["failures"]
|
| 2883 |
+
|
| 2884 |
+
|
| 2885 |
+
def _bound_input(session, key, cell, *, book, today, seen, graph):
|
| 2886 |
+
"""A bound cell read as an INPUT (a filter value, an ending date, a formula's operand).
|
| 2887 |
+
|
| 2888 |
+
Refresh's certified cache first, then this request's memo, then one real resolve. A refusal
|
| 2889 |
+
is memoized as the refusal: the same binding over the same document and the same database
|
| 2890 |
+
cannot answer differently twice in one request.
|
| 2891 |
+
"""
|
| 2892 |
+
lookup = getattr(session, "_sheet_refresh_input", None)
|
| 2893 |
+
cached = lookup(key) if lookup is not None else None
|
| 2894 |
+
if cached is not None:
|
| 2895 |
+
return cached
|
| 2896 |
+
memo = graph["bound_values"] if graph is not None else None
|
| 2897 |
+
if memo is not None and key in memo:
|
| 2898 |
+
stats = _read_stats(session)
|
| 2899 |
+
if stats is not None:
|
| 2900 |
+
stats["input_hits"] += 1
|
| 2901 |
+
hit = memo[key]
|
| 2902 |
+
if isinstance(hit, HTTPException):
|
| 2903 |
+
raise hit
|
| 2904 |
+
return hit
|
| 2905 |
+
try:
|
| 2906 |
+
result = _resolve_binding(session, cell.get("binding"), today=today, book=book,
|
| 2907 |
+
tab=key.rpartition("!")[0], ending_seen=seen)
|
| 2908 |
+
except HTTPException as exc:
|
| 2909 |
+
if memo is not None:
|
| 2910 |
+
memo[key] = exc
|
| 2911 |
+
raise
|
| 2912 |
+
if memo is not None:
|
| 2913 |
+
memo[key] = result
|
| 2914 |
+
return result
|
| 2915 |
+
|
| 2916 |
+
|
| 2917 |
+
def _binding_input_refs(binding):
|
| 2918 |
+
"""Every cell reference a binding reads: filter values at any depth and a period's ref."""
|
| 2919 |
+
from modules import sheet as sheet_mod
|
| 2920 |
+
if not isinstance(binding, dict):
|
| 2921 |
+
return []
|
| 2922 |
+
nodes = []
|
| 2923 |
+
for part in (binding.get("scope"), binding.get("related")):
|
| 2924 |
+
if isinstance(part, dict):
|
| 2925 |
+
found, _ = _fe_parts(part.get("filter"))
|
| 2926 |
+
nodes.extend(found)
|
| 2927 |
+
refs = [str((node.get(slot) or {}).get("ref") or "")
|
| 2928 |
+
for node in sheet_mod.filter_cell_nodes(nodes)
|
| 2929 |
+
for slot in ("valueRef", "value2Ref") if isinstance(node.get(slot), dict)]
|
| 2930 |
+
period = binding.get("period")
|
| 2931 |
+
if isinstance(period, dict) and period.get("ref"):
|
| 2932 |
+
refs.append(str(period["ref"]))
|
| 2933 |
+
return [ref for ref in refs if ref.strip()]
|
| 2934 |
+
|
| 2935 |
+
|
| 2936 |
+
def _qualified_refs(binding, home, fr, graph=None):
|
| 2937 |
+
out = []
|
| 2938 |
+
for ref in _binding_input_refs(binding):
|
| 2939 |
+
if ref.startswith("="):
|
| 2940 |
+
if graph is not None:
|
| 2941 |
+
out.extend(_formula_slot_keys(graph, fr, ref, home))
|
| 2942 |
+
continue
|
| 2943 |
+
try:
|
| 2944 |
+
out.append(_asked_key(ref, home, fr))
|
| 2945 |
+
except Exception: # noqa: BLE001
|
| 2946 |
+
continue
|
| 2947 |
+
return out
|
| 2948 |
+
|
| 2949 |
+
|
| 2950 |
+
def _dependent_bound_keys(book, graph, fr, edited):
|
| 2951 |
+
"""HOTFIX 2026-09-29 (owner: "ONLY when we update ANY cells, its related cells also have to
|
| 2952 |
+
update, this includes the cell database"): the bound cells an edit can change, and no other.
|
| 2953 |
+
|
| 2954 |
+
Walks FORWARD from the edited keys over the formula graph's own reverse edges (the same three
|
| 2955 |
+
`SheetCalculation.update` walks: static reads, dynamic reads, range consumers) plus one more
|
| 2956 |
+
edge the formula graph cannot see: a bound cell's filter value / period reference. So editing
|
| 2957 |
+
`F6` reaches `F7 = G6-1`, and every binding filtering on `F7`, and every formula over those.
|
| 2958 |
+
β STRUCTURE, NEVER A VALUE DIFF: the stored values are last computed, not current, so an
|
| 2959 |
+
edit reverted to its old value would look unchanged and leave a binding on the edited answer.
|
| 2960 |
+
"""
|
| 2961 |
+
from collections import deque
|
| 2962 |
+
engine = _read_engine(graph, book, fr, None)[0]
|
| 2963 |
+
readers = {}
|
| 2964 |
+
for key, cell in graph["bound"].items():
|
| 2965 |
+
if _is_view_binding(cell):
|
| 2966 |
+
continue
|
| 2967 |
+
for ref in _qualified_refs(cell.get("binding"), key.rpartition("!")[0], fr, graph):
|
| 2968 |
+
readers.setdefault(ref, set()).add(key)
|
| 2969 |
+
seen, queue = set(edited), deque(edited)
|
| 2970 |
+
while queue:
|
| 2971 |
+
key = queue.popleft()
|
| 2972 |
+
owners = (set(engine.read_by.get(key, ())) | set(engine.consumed_by.get(key, ()))
|
| 2973 |
+
| set(engine.ranges.at(key, engine.stats)) | readers.get(key, set()))
|
| 2974 |
+
for owner in owners:
|
| 2975 |
+
if owner not in seen:
|
| 2976 |
+
seen.add(owner)
|
| 2977 |
+
queue.append(owner)
|
| 2978 |
+
return {key for key in seen if key in graph["bound"] and key not in edited}
|
| 2979 |
+
|
| 2980 |
+
|
| 2981 |
+
def _input_closure(graph, engine, fr, roots):
|
| 2982 |
+
"""Every cell a set of references reads, transitively, through formulas and bound inputs."""
|
| 2983 |
+
seen, pending = set(), list(roots)
|
| 2984 |
+
while pending:
|
| 2985 |
+
key = pending.pop()
|
| 2986 |
+
if key in seen:
|
| 2987 |
+
continue
|
| 2988 |
+
seen.add(key)
|
| 2989 |
+
cell = graph["cells"].get(key)
|
| 2990 |
+
if isinstance(cell, dict) and cell.get("kind") == "bound":
|
| 2991 |
+
pending.extend(_qualified_refs(cell.get("binding"), key.rpartition("!")[0], fr, graph))
|
| 2992 |
+
continue
|
| 2993 |
+
pending.extend(engine.reads.get(key, ()))
|
| 2994 |
+
pending.extend(engine.consumed.get(key, ()))
|
| 2995 |
+
compiled = engine.compiled.get(key)
|
| 2996 |
+
for bounds in (compiled or {}).get("ranges") or ():
|
| 2997 |
+
pending.extend(engine.positions.within(bounds))
|
| 2998 |
+
return seen
|
| 2999 |
+
|
| 3000 |
+
|
| 3001 |
def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(),
|
| 3002 |
+
code="ending_cell_unreadable", as_date=False):
|
| 3003 |
+
"""Shared cell read for Ending and Filter, including scoped bound and formula inputs.
|
| 3004 |
+
|
| 3005 |
+
S67-T10: with ``as_date`` a date-formatted number (typed, or a formula's result such as
|
| 3006 |
+
46293) is returned as its ISO day, so a Filter condition compares it as that date for every
|
| 3007 |
+
operator; a plain number with no date format is returned unchanged.
|
| 3008 |
+
"""
|
| 3009 |
from modules import sheet as sheet_mod
|
| 3010 |
import formula_runtime as fr
|
| 3011 |
|
| 3012 |
def refuse(reason):
|
| 3013 |
raise sheet_mod.SheetRefusal(code, ref or "(blank)", reason)
|
| 3014 |
|
| 3015 |
+
if str(ref or "").strip().startswith("=") and book is not None:
|
| 3016 |
+
return _read_binding_formula(session, str(ref).strip(), book=book, tab=tab, today=today,
|
| 3017 |
+
seen=seen, code=code, as_date=as_date)
|
| 3018 |
if not re.fullmatch(r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref):
|
| 3019 |
refuse("its reference is invalid.")
|
| 3020 |
if book is None:
|
| 3021 |
refuse("the source workbook is unavailable.")
|
| 3022 |
tabs = book.get("sheets") or []
|
| 3023 |
+
# HOTFIX 2026-09-29 (owner: "MAKE IT FAST"): one request reads ONE graph. Before this, every
|
| 3024 |
+
# filter reference of every bound cell rebuilt the whole workbook's cell map and, for a formula
|
| 3025 |
+
# reference, a whole-workbook SheetCalculation (about 0.6 s each): Daily Sales has ~470 bound
|
| 3026 |
+
# cells reading `$E$10` and `F7`, so one commit or one Refresh paid ~900 engine builds.
|
| 3027 |
+
graph = _read_graph(session, book, fr, today)
|
| 3028 |
+
names = graph["names"] if graph is not None else _tab_scope_names(tabs)
|
| 3029 |
home = _resolve_home_tab(book, tabs, names, tab)
|
| 3030 |
key = _asked_key(ref, home, fr)
|
| 3031 |
if key in seen:
|
| 3032 |
refuse("the reference is circular.")
|
| 3033 |
+
cells = graph["cells"] if graph is not None else {
|
| 3034 |
+
fr.a1_key(addr, name): cell for source, name in zip(tabs, names)
|
| 3035 |
+
for addr, cell in (source.get("cells") or {}).items()}
|
| 3036 |
cell = cells.get(key)
|
| 3037 |
if not isinstance(cell, dict) or cell.get("kind") == "empty":
|
| 3038 |
refuse("the cell or its sheet is missing or blank.")
|
| 3039 |
if cell.get("kind") == "bound":
|
| 3040 |
try:
|
| 3041 |
+
value = _bound_input(session, key, cell, book=book, today=today,
|
| 3042 |
+
seen=(*seen, key), graph=graph)["value"]
|
|
|
|
|
|
|
|
|
|
| 3043 |
except HTTPException as exc:
|
| 3044 |
reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
|
| 3045 |
refuse(reason)
|
| 3046 |
elif cell.get("kind") == "formula":
|
| 3047 |
# The existing formula graph discovers actual inputs, including dynamic references.
|
| 3048 |
# A database input starts refused, so a cached value can never bypass this reader's wall.
|
| 3049 |
+
bound = graph["bound"] if graph is not None else {
|
| 3050 |
+
k: c for k, c in cells.items() if c.get("kind") == "bound"}
|
| 3051 |
+
if graph is not None:
|
| 3052 |
+
engine, attempted, failures = _read_engine(graph, book, fr, today)
|
| 3053 |
+
else:
|
| 3054 |
+
specs = {k: ({"refused": "ending_input_pending"} if k in bound
|
| 3055 |
+
else fr._document_spec(c, k.rpartition("!")[0])) for k, c in cells.items()}
|
| 3056 |
+
engine = fr.SheetCalculation(specs, sheets=names, today=today,
|
| 3057 |
+
named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
|
| 3058 |
+
attempted, failures = set(), {}
|
| 3059 |
while True:
|
| 3060 |
dependencies, pending = set(), [key]
|
| 3061 |
while pending:
|
|
|
|
| 3071 |
for dependency in needed:
|
| 3072 |
attempted.add(dependency)
|
| 3073 |
try:
|
| 3074 |
+
result = _bound_input(session, dependency, bound[dependency], book=book,
|
| 3075 |
+
today=today, seen=(*seen, key, dependency), graph=graph)
|
|
|
|
|
|
|
|
|
|
| 3076 |
changes[dependency] = {"value": result["value"]}
|
| 3077 |
except HTTPException as exc:
|
| 3078 |
info = ((exc.detail or {}).get("error") or {})
|
|
|
|
| 3094 |
if cell.get("state") == "stale":
|
| 3095 |
refuse("the cell needs to be refreshed.")
|
| 3096 |
value = cell.get("value", cell.get("text"))
|
| 3097 |
+
if (as_date and isinstance(value, (int, float)) and not isinstance(value, bool)
|
| 3098 |
+
and sheet_mod.date_formatted_cell(cell)):
|
| 3099 |
+
from harness import windows as W
|
| 3100 |
+
day = W.ending_cell_date(value, book.get("dateSystem", "1900"))
|
| 3101 |
+
if not day:
|
| 3102 |
+
refuse("its value is not a real date.")
|
| 3103 |
+
return day
|
| 3104 |
return value
|
| 3105 |
|
| 3106 |
|
|
|
|
| 3108 |
from modules import sheet as sheet_mod
|
| 3109 |
try:
|
| 3110 |
return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell(
|
| 3111 |
+
session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable",
|
| 3112 |
+
as_date=True))
|
| 3113 |
except sheet_mod.SheetRefusal as exc:
|
| 3114 |
raise err(400, exc.code, exc.message)
|
| 3115 |
|
|
|
|
| 3153 |
return _resolve_binding(session, binding, **kwargs)
|
| 3154 |
|
| 3155 |
|
| 3156 |
+
def _binding_related_entry(session, database, name):
|
| 3157 |
"""Resolve one authored edge key against the permission-scoped declarations."""
|
| 3158 |
+
from core import perm_scope, relations
|
| 3159 |
+
fields, defs, entries = _related_declarations(session, database)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3160 |
matches = [entry for entry in entries if entry["key"] == name]
|
| 3161 |
via = matches[0]["via"] if len(matches) == 1 else {"key": name}
|
| 3162 |
+
# `pick` over the SCOPED list, which also carries the target-side bridge entries above.
|
| 3163 |
+
entry = relations.pick(entries, via)
|
| 3164 |
if entry is None:
|
| 3165 |
candidates = [item["key"] for item in relations.via_candidates(entries, via)
|
| 3166 |
if perm_scope.nav_may_open(session.user, item["target"], st=session.runtime)]
|
|
|
|
| 3180 |
declaration = next((f for f in defs.get(target, {}).get("fields", [])
|
| 3181 |
if f.get("key") == entry["via"].get("field")), {})
|
| 3182 |
link = declaration.get("link") or {}
|
| 3183 |
+
if (link.get("through") or link.get("selectionMode") == "manual"
|
| 3184 |
+
or not entry["via"].get("from") or not entry["via"].get("on")):
|
| 3185 |
raise err(409, "relation_source_unresolvable",
|
| 3186 |
f"Relationship '{name}' does not declare a direct column join. "
|
| 3187 |
"Choose a relationship with one readable column on each database.")
|
| 3188 |
return entry
|
| 3189 |
|
| 3190 |
|
| 3191 |
+
def _binding_related_query(session, database, entry, cur, spec, columns, *, fields, window=None):
|
|
|
|
| 3192 |
"""Materialise the far side on the near cursor, then hand metadata to the compiler."""
|
|
|
|
|
|
|
| 3193 |
from core import perm_scope, relations
|
| 3194 |
from modules import sheet as sheet_mod
|
| 3195 |
|
| 3196 |
+
target, name = entry["target"], entry["key"]
|
|
|
|
| 3197 |
local_key, remote_key = entry["via"].get("from"), entry["via"].get("on")
|
| 3198 |
|
| 3199 |
def map_join(key, table, source_spec, cols, side):
|
|
|
|
| 3219 |
wall = ((perm_scope.entry(session.user, target) or {}).get("filter")
|
| 3220 |
if perm_scope.row_scope_applies(session.user, target) else None)
|
| 3221 |
wall_nodes, wall_conj = _fe_parts(wall)
|
| 3222 |
+
needed = set(fields) | sheet_mod.filter_columns_named(wall_nodes) | {remote_key}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3223 |
try:
|
| 3224 |
rcur, rspec, rcols = _mirror(session)(target, metric_window=window, needed=needed)
|
| 3225 |
except HTTPException as exc:
|
|
|
|
| 3236 |
hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [],
|
| 3237 |
st=session.runtime)
|
| 3238 |
rcols = {key: value for key, value in rcols.items() if key not in hidden}
|
| 3239 |
+
declared = {field["key"]: field for field in rspec.get("fields") or []}
|
| 3240 |
+
|
| 3241 |
+
def unknown_field(key):
|
| 3242 |
+
return err(409, "related_field_unknown",
|
| 3243 |
+
f"Field '{key}' on relationship '{name}' is hidden, removed, or has no "
|
| 3244 |
+
"readable per-record value. Choose a readable field on this relationship.")
|
| 3245 |
+
|
| 3246 |
+
for key in sorted(fields):
|
| 3247 |
+
field = declared.get(key) or {}
|
| 3248 |
+
if (key in hidden
|
| 3249 |
+
or field.get("type") in {"formula", "link", "lookup", "rollup", "metric", "measure"}
|
| 3250 |
+
or field.get("measure")):
|
| 3251 |
+
raise unknown_field(key)
|
| 3252 |
map_join(remote_key, target, rspec, rcols, "target")
|
| 3253 |
+
for key in sorted(fields):
|
| 3254 |
+
expr = (rcols.get(key) or {}).get("sql")
|
| 3255 |
+
if not expr or str(expr).strip().upper() == "NULL":
|
| 3256 |
+
raise unknown_field(key)
|
| 3257 |
return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key,
|
| 3258 |
"spec": rspec, "columns": rcols, "where": entry.get("where"),
|
|
|
|
| 3259 |
"wall_nodes": wall_nodes, "wall_conj": wall_conj,
|
| 3260 |
+
"limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in fields]}
|
| 3261 |
+
|
| 3262 |
+
|
| 3263 |
+
def _related_leaf_bags(session, database, nodes, needed):
|
| 3264 |
+
"""Prepare joins before the local mirror, then materialise their bags on its cursor."""
|
| 3265 |
+
from modules import sheet as sheet_mod
|
| 3266 |
+
entries, fields = {}, {}
|
| 3267 |
+
try:
|
| 3268 |
+
for key in sorted(sheet_mod.related_filter_ids(nodes)):
|
| 3269 |
+
via, field = sheet_mod.related_field_parts(key)
|
| 3270 |
+
if via not in entries:
|
| 3271 |
+
entries[via] = _binding_related_entry(session, database, via)
|
| 3272 |
+
needed.add(entries[via]["via"].get("from"))
|
| 3273 |
+
fields.setdefault(via, set()).add(field)
|
| 3274 |
+
needed.difference_update(sheet_mod.related_filter_ids(nodes))
|
| 3275 |
+
except sheet_mod.SheetRefusal as exc:
|
| 3276 |
+
raise err(409, exc.code, exc.message)
|
| 3277 |
+
|
| 3278 |
+
def materialise(cur, spec, columns, *, window=None):
|
| 3279 |
+
return {via: _binding_related_query(session, database, entry, cur, spec, columns,
|
| 3280 |
+
fields=fields[via], window=window)
|
| 3281 |
+
for via, entry in entries.items()}
|
| 3282 |
+
return materialise
|
| 3283 |
+
|
| 3284 |
+
|
| 3285 |
+
def _binding_filter_nodes(session, database, binding, *, memo=None):
|
| 3286 |
+
"""A saved view and an authored extra scope are always intersected."""
|
| 3287 |
+
scope = binding.get("scope") or {}
|
| 3288 |
+
nodes, conj = _fe_parts(scope.get("filter"))
|
| 3289 |
+
from modules import sheet as sheet_mod
|
| 3290 |
+
if sheet_mod.linked_filter_ids(nodes):
|
| 3291 |
+
nodes = _rewrite_related_filters(session, database, nodes)
|
| 3292 |
+
binding["scope"] = {**scope, "filter": {"conj": conj, "nodes": nodes}}
|
| 3293 |
+
if scope.get("view_id"):
|
| 3294 |
+
view_nodes, view_conj = _view_filter(session, database, scope["view_id"],
|
| 3295 |
+
scope.get("view_name", ""), memo=memo)
|
| 3296 |
+
if nodes:
|
| 3297 |
+
nodes, conj = [{"conj": view_conj, "children": view_nodes},
|
| 3298 |
+
{"conj": conj, "children": nodes}], "and"
|
| 3299 |
+
else:
|
| 3300 |
+
nodes, conj = view_nodes, view_conj
|
| 3301 |
+
return _rewrite_related_filters(session, database, nodes), conj
|
| 3302 |
|
| 3303 |
|
| 3304 |
def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
|
|
|
|
| 3349 |
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 3350 |
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 3351 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 3352 |
+
scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding, memo=view_memo)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3353 |
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
|
| 3354 |
today=today, seen=ending_seen)
|
| 3355 |
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
|
|
|
| 3357 |
if binding["aggregate"].get("field"):
|
| 3358 |
needed.add(str(binding["aggregate"]["field"]))
|
| 3359 |
|
| 3360 |
+
related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
|
|
|
|
|
|
|
| 3361 |
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 3362 |
cols = dict(cols)
|
|
|
|
| 3363 |
|
| 3364 |
# β THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
|
| 3365 |
# real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
|
|
|
|
| 3398 |
# spec's column expressions; applying it a second time would double-count the exclusion and
|
| 3399 |
# break every joined spec, whose partner column is `sol.order_partner_id` or `aml.partner_id`.
|
| 3400 |
try:
|
| 3401 |
+
related = related_bags(cur, spec, cols, window=window)
|
|
|
|
|
|
|
| 3402 |
sql, params, limits = sheet_mod.build_aggregate_query(
|
| 3403 |
spec, binding, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
|
| 3404 |
wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, related=related)
|
|
|
|
| 3501 |
binding editor show a number before anyone commits the cell. R10 may create a missing Metric
|
| 3502 |
field through the database's ordinary field writer; reusing a column needs no field write.
|
| 3503 |
"""
|
| 3504 |
+
with _sheet_request_scope(session):
|
| 3505 |
+
return _preview_cell_scoped(workbook_id, addr, body, session)
|
| 3506 |
+
|
| 3507 |
+
|
| 3508 |
+
def _preview_cell_scoped(workbook_id: str, addr: str, body, session: Session):
|
| 3509 |
raw = dict(body or {})
|
| 3510 |
binding = raw.get("binding") or {}
|
| 3511 |
book = None
|
|
|
|
| 3613 |
β `rev` BUMPS, BECAUSE THIS PERSISTS. It goes back in the answer and the client adopts it, or
|
| 3614 |
its next PATCH would carry a revision the server has already moved past.
|
| 3615 |
"""
|
| 3616 |
+
with _sheet_request_scope(session):
|
| 3617 |
+
return _compute_cell_scoped(workbook_id, addr, body, session)
|
| 3618 |
+
|
| 3619 |
+
|
| 3620 |
+
def _compute_cell_scoped(workbook_id: str, addr: str, body, session: Session):
|
| 3621 |
from core import sheets
|
| 3622 |
|
| 3623 |
try:
|
|
|
|
| 3751 |
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 3752 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 3753 |
scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
|
| 3754 |
+
scope_nodes = _rewrite_related_filters(session, database, scope_nodes)
|
| 3755 |
# W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
|
| 3756 |
for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
|
| 3757 |
+
if not key.startswith(sheet_mod.RELATED_FILTER_PREFIX) and key not in visible:
|
| 3758 |
return JSONResponse(status_code=400, content={
|
| 3759 |
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
|
| 3760 |
today = date.today()
|
|
|
|
| 3763 |
window = _ending_window(session, binding, book=book, today=today)
|
| 3764 |
needed = (set(requested) | sorted_keys | sheet_mod.filter_columns_named(scope_nodes)
|
| 3765 |
| sheet_mod.filter_columns_named(wall_nodes))
|
| 3766 |
+
related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
|
| 3767 |
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 3768 |
cols = dict(cols)
|
| 3769 |
+
related = related_bags(cur, spec, cols, window=window)
|
| 3770 |
for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
|
| 3771 |
+
if key.startswith(sheet_mod.RELATED_FILTER_PREFIX):
|
| 3772 |
+
continue
|
| 3773 |
if key in hidden or not (cols.get(key) or {}).get("sql"):
|
| 3774 |
return JSONResponse(status_code=400, content={
|
| 3775 |
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
|
|
|
|
| 3779 |
raise ValueError("unapplied row wall")
|
| 3780 |
count_sql, params, limits = sheet_mod.build_aggregate_query(
|
| 3781 |
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 3782 |
+
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today,
|
| 3783 |
+
related=related)
|
| 3784 |
if limits:
|
| 3785 |
raise ValueError("unapplied view filter")
|
| 3786 |
except sheet_mod.SheetRefusal as exc:
|
|
|
|
| 3821 |
|
| 3822 |
|
| 3823 |
@router.get("/sheets/{workbook_id}/cells/{addr}/drill")
|
| 3824 |
+
@_request_scoped
|
| 3825 |
+
def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
|
| 3826 |
+
session: Session = Depends(require_session)):
|
| 3827 |
+
"""`{database, view_id?, filter?, record_ids, count, id_limit}` for `LinkGridModal` (R6, R13, R17).
|
| 3828 |
+
|
| 3829 |
+
ββ S66-T48 β `tab` (an id or a name, as `GET /sheets/{id}/tabs/{tab}` accepts) NAMES THE TAB
|
| 3830 |
+
THE CELL IS ON. Without it the address was resolved against the FIRST tab holding it, so a
|
| 3831 |
+
Daily Sales `Summary (SR)` cell drilled `Summary (All)`'s same address. Omitted, the old
|
| 3832 |
+
first-tab rule still answers, so an older client keeps working.
|
| 3833 |
+
|
| 3834 |
+
ββ S66-T48 β NO CAP ON A CONNECTOR DRILL (standing rule 1). `record_ids` is every matching
|
| 3835 |
+
id; the popup pages them and sends the set in a POST body. Only a database whose records a
|
| 3836 |
+
person types (`records_mutable`) keeps `user_tables.MAX_ROWS`, and then `id_limit` names it so
|
| 3837 |
+
the popup says "showing X of N" rather than presenting X as the whole.
|
| 3838 |
|
| 3839 |
β A FORMULA CELL REFUSES BY NAME AND DOES NOT DRILL TO ONE OPERAND (R17). Picking an input
|
| 3840 |
would be a guess presented as an answer: `=B4-B8` has two, and neither is what the person
|
|
|
|
| 3855 |
|
| 3856 |
key = str(addr or "").replace("$", "").upper()
|
| 3857 |
cell = None
|
| 3858 |
+
home = None
|
| 3859 |
+
if str(tab or "").strip():
|
| 3860 |
+
try:
|
| 3861 |
+
home = sheets.find_tab(book, str(tab).strip())
|
| 3862 |
+
except sheets.SheetRefusal as exc:
|
| 3863 |
+
raise _refuse(exc)
|
| 3864 |
+
cell = (home.get("cells") or {}).get(key)
|
| 3865 |
+
else:
|
| 3866 |
+
for candidate in book.get("sheets") or ():
|
| 3867 |
+
if key in (candidate.get("cells") or {}):
|
| 3868 |
+
home, cell = candidate, candidate["cells"][key]
|
| 3869 |
+
break
|
| 3870 |
if not isinstance(cell, dict):
|
| 3871 |
raise err(404, "cell_missing", f"Cell {key} is not filled in.")
|
| 3872 |
if cell.get("kind") == "formula":
|
|
|
|
| 3896 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 3897 |
|
| 3898 |
scope = binding["scope"]
|
| 3899 |
+
scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3900 |
|
| 3901 |
# β THE SAME PERIOD THE NUMBER WAS COMPUTED AT (W53-T13). The drill probes `COUNT(*)` and never
|
| 3902 |
# reads the bound column, but its FILTER may name a Metric column, and a mirror opened at a
|
|
|
|
| 3907 |
# window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from
|
| 3908 |
# the column set instead, where the time actually is.
|
| 3909 |
try:
|
| 3910 |
+
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=home.get("id"))
|
| 3911 |
+
window = _ending_window(session, binding, book=book, tab=home.get("id"))
|
| 3912 |
except sheet_mod.SheetRefusal as exc:
|
| 3913 |
raise err(400, exc.code, exc.message)
|
|
|
|
| 3914 |
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
| 3915 |
| sheet_mod.filter_columns_named(wall_nodes))
|
| 3916 |
+
related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
|
|
|
|
| 3917 |
cur, spec, cols = _mirror(session)(
|
| 3918 |
database, metric_window=window,
|
| 3919 |
needed=needed)
|
| 3920 |
cols = dict(cols)
|
|
|
|
| 3921 |
|
| 3922 |
try:
|
| 3923 |
+
related = related_bags(cur, spec, cols, window=window)
|
|
|
|
|
|
|
| 3924 |
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
|
| 3925 |
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 3926 |
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
|
|
|
|
| 3935 |
"connection. Edit the scope or ask an administrator to update the restriction.")
|
| 3936 |
total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
|
| 3937 |
|
| 3938 |
+
# ββ S66-T48 β THE CAP IS DECIDED BY THE DATA'S SOURCE, NOT BY THE POPUP. A connector's records
|
| 3939 |
+
# (`records_mutable` False) are listed in full: the popup renders one page and the id set
|
| 3940 |
+
# travels in a POST body, which were the two reasons the old 500 id cap gave. A typed database
|
| 3941 |
+
# keeps `MAX_ROWS`, asked only when the count passes it (the predicate reads the store), and
|
| 3942 |
+
# the limit is RETURNED so the popup states it instead of enforcing it silently.
|
| 3943 |
+
from core import user_tables
|
| 3944 |
+
id_limit = None
|
| 3945 |
+
if (int(total or 0) > user_tables.MAX_ROWS
|
| 3946 |
+
and user_tables.records_mutable(database, st=session.runtime)):
|
| 3947 |
+
id_limit = user_tables.MAX_ROWS
|
| 3948 |
+
ids_sql = f"{ids_sql} LIMIT {int(id_limit)}"
|
| 3949 |
record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
|
| 3950 |
|
| 3951 |
out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
|
| 3952 |
+
"id_limit": id_limit, "limits": limits}
|
| 3953 |
if scope.get("view_id"):
|
| 3954 |
out["view_id"] = scope["view_id"]
|
| 3955 |
else:
|
|
|
|
| 4058 |
database work, and a batch's cost is dominated by the number of distinct (database, period)
|
| 4059 |
pairs in it rather than by the number of cells.
|
| 4060 |
"""
|
| 4061 |
+
with _sheet_request_scope(session):
|
| 4062 |
+
return _resolve_bindings_scoped(workbook_id, body, session)
|
| 4063 |
+
|
| 4064 |
+
|
| 4065 |
+
def _resolve_bindings_scoped(workbook_id: str, body, session: Session):
|
| 4066 |
from core import sheets
|
| 4067 |
import formula_runtime as fr
|
| 4068 |
|
api/routes_tables.py
CHANGED
|
@@ -2116,7 +2116,7 @@ def create_table(body: dict = Body(default=None),
|
|
| 2116 |
if not key:
|
| 2117 |
raise err(400, "refused",
|
| 2118 |
f"could not create it. The name may be empty or this tenant already has "
|
| 2119 |
-
f"{ut.MAX_TABLES} databases")
|
| 2120 |
return {"key": key}
|
| 2121 |
|
| 2122 |
|
|
|
|
| 2116 |
if not key:
|
| 2117 |
raise err(400, "refused",
|
| 2118 |
f"could not create it. The name may be empty or this tenant already has "
|
| 2119 |
+
f"{ut.MAX_TABLES} databases of its own (connected-source databases do not count)")
|
| 2120 |
return {"key": key}
|
| 2121 |
|
| 2122 |
|
api/routes_templates.py
CHANGED
|
@@ -80,8 +80,8 @@ def list_templates(table: str = "", session: Session = Depends(require_session),
|
|
| 80 |
`missing_columns`, which is the same function the POST below re-runs).
|
| 81 |
"""
|
| 82 |
if module:
|
| 83 |
-
if module not in ("sheet", "data", "app"):
|
| 84 |
-
raise err(400, "unknown_module", "Choose Sheet, Data or
|
| 85 |
return _module_templates(session)
|
| 86 |
fields, source = _target_or_refuse(session, table)
|
| 87 |
vt = _templates()
|
|
@@ -137,6 +137,16 @@ def apply_template(key: str, body: dict = Body(default=None),
|
|
| 137 |
import core.table_store as table_store
|
| 138 |
ops = table_store.make(ws_key, st=session.runtime)
|
| 139 |
applied = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
for view in tpl.get('views') or ():
|
| 141 |
# β A COPY PER APPLY, and `createdBy` STAMPED HERE. The registry's dicts are module-level
|
| 142 |
# β mutating one would write this caller's username into the template every other tenant
|
|
@@ -145,6 +155,10 @@ def apply_template(key: str, body: dict = Body(default=None),
|
|
| 145 |
# side.
|
| 146 |
payload = dict(view)
|
| 147 |
payload['config'] = dict(view.get('config') or {})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
payload['createdBy'] = session.uname
|
| 149 |
saved = ops.save_view(session.uname, payload)
|
| 150 |
applied.append({"id": payload['id'], "name": saved.get('name') or payload['name']})
|
|
@@ -265,8 +279,68 @@ def _data_template_database(template, definitions):
|
|
| 265 |
return ""
|
| 266 |
|
| 267 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
def _module_templates(session):
|
| 269 |
-
from core import sheet_templates, user_tables
|
| 270 |
from routes_connectors import _entry_types
|
| 271 |
from routes_keychain import env_odoo_available
|
| 272 |
from routes_nav import application_templates
|
|
@@ -281,9 +355,13 @@ def _module_templates(session):
|
|
| 281 |
saved = _saved_sheet_templates(session.runtime)
|
| 282 |
data_templates = [(row, _data_template_database(row, definitions))
|
| 283 |
for row in _templates().TEMPLATES]
|
|
|
|
|
|
|
|
|
|
| 284 |
databases = {row["database_hint"] for row in sheet_templates.TEMPLATES}
|
| 285 |
databases.update(db for row in saved for db in row["databases"])
|
| 286 |
databases.update(database for _row, database in data_templates if database)
|
|
|
|
| 287 |
database_states = _database_states(session, definitions, databases)
|
| 288 |
requirement = lambda database, connector="": _requirement(
|
| 289 |
database, definitions, have, database_states, connector)
|
|
@@ -305,9 +383,25 @@ def _module_templates(session):
|
|
| 305 |
app.extend(_catalog_row("finance:" + row["key"], row["label"], row["description"], [],
|
| 306 |
creation_mode="post")
|
| 307 |
for row in finance_templates(may_open=lambda permission: _may_open(session, permission)))
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 311 |
|
| 312 |
|
| 313 |
@router.get("/sheet-templates")
|
|
@@ -412,6 +506,9 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
|
|
| 412 |
from urllib.parse import quote
|
| 413 |
|
| 414 |
body = body if isinstance(body, dict) else {}
|
|
|
|
|
|
|
|
|
|
| 415 |
if module == "app" and key.startswith("finance:"):
|
| 416 |
import application_registry
|
| 417 |
from modules.finance_apps import finance_application
|
|
@@ -447,6 +544,16 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
|
|
| 447 |
raise err(404, "unknown_template", "That template does not exist.")
|
| 448 |
if not template["creatable"]:
|
| 449 |
raise err(409, "template_unavailable", "Connect the required source and switch on its databases.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
if module == "sheet":
|
| 451 |
database = template["requires"][0]["database"] if template["requires"] else ""
|
| 452 |
result = apply_sheet_template(key, {"database": database, "name": body.get("name", "")}, session)
|
|
@@ -455,9 +562,28 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
|
|
| 455 |
database = template["requires"][0]["database"]
|
| 456 |
result = apply_template(key, {"table": database}, session)
|
| 457 |
return {"address": "#/" + quote(database, safe=""), **result}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 458 |
# The existing wizard still owns its POST-then-PATCH share sequence. API clients may
|
| 459 |
# dispatch creation here; the existing App writer retains schema and permission walls.
|
| 460 |
result = create_nav_application({**body, "templateKind": key}, session=session)
|
| 461 |
app = result["application"]
|
| 462 |
return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""),
|
| 463 |
**result}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
`missing_columns`, which is the same function the POST below re-runs).
|
| 81 |
"""
|
| 82 |
if module:
|
| 83 |
+
if module not in ("sheet", "data", "app", "agents"):
|
| 84 |
+
raise err(400, "unknown_module", "Choose Sheet, Data, App or Agents.")
|
| 85 |
return _module_templates(session)
|
| 86 |
fields, source = _target_or_refuse(session, table)
|
| 87 |
vt = _templates()
|
|
|
|
| 137 |
import core.table_store as table_store
|
| 138 |
ops = table_store.make(ws_key, st=session.runtime)
|
| 139 |
applied = []
|
| 140 |
+
# S67-T39 (R12, D-2233): a template View bound to an App that no longer exists instantiates as
|
| 141 |
+
# a plain view of its database; the live App ids are read once, and an unreadable registry
|
| 142 |
+
# keeps the reference (the read projection in `grid_events.table_workspace` still masks it).
|
| 143 |
+
try:
|
| 144 |
+
import application_registry
|
| 145 |
+
raw_registry = session.runtime.get(application_registry._KEY)
|
| 146 |
+
live_apps = (set(raw_registry["applications"]) if isinstance(raw_registry, dict)
|
| 147 |
+
and isinstance(raw_registry.get("applications"), dict) else None)
|
| 148 |
+
except Exception:
|
| 149 |
+
live_apps = None
|
| 150 |
for view in tpl.get('views') or ():
|
| 151 |
# β A COPY PER APPLY, and `createdBy` STAMPED HERE. The registry's dicts are module-level
|
| 152 |
# β mutating one would write this caller's username into the template every other tenant
|
|
|
|
| 155 |
# side.
|
| 156 |
payload = dict(view)
|
| 157 |
payload['config'] = dict(view.get('config') or {})
|
| 158 |
+
dead_ref = payload['config'].get('application')
|
| 159 |
+
if (live_apps is not None and isinstance(dead_ref, dict)
|
| 160 |
+
and str(dead_ref.get('id') or '') not in live_apps):
|
| 161 |
+
payload['config'].pop('application')
|
| 162 |
payload['createdBy'] = session.uname
|
| 163 |
saved = ops.save_view(session.uname, payload)
|
| 164 |
applied.append({"id": payload['id'], "name": saved.get('name') or payload['name']})
|
|
|
|
| 279 |
return ""
|
| 280 |
|
| 281 |
|
| 282 |
+
def _agent_templates():
|
| 283 |
+
"""Fresh starter definitions; saving one never arms its schedule."""
|
| 284 |
+
import os
|
| 285 |
+
|
| 286 |
+
base = (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").strip().rstrip("/")
|
| 287 |
+
|
| 288 |
+
def report(instruction, route):
|
| 289 |
+
return {"id": "act_report", "kind": "ai_agent", "config": {
|
| 290 |
+
"url": base + "/#/" + route if base else "",
|
| 291 |
+
"instruction": instruction,
|
| 292 |
+
"field": "", "timeoutMs": 20000, "maxSteps": 6}}
|
| 293 |
+
|
| 294 |
+
def starter(key, name, description, cron, databases, actions):
|
| 295 |
+
return {"key": key, "name": name, "description": description, "databases": databases,
|
| 296 |
+
"definition": {"name": name, "kind": "plain", "trigger": {"key": "schedule"},
|
| 297 |
+
"schedule": {"cron": cron, "enabled": False},
|
| 298 |
+
"flow": {"actions": actions}}}
|
| 299 |
+
|
| 300 |
+
return [
|
| 301 |
+
starter("odoo_sync_check", "Odoo sync check", "Check Odoo database freshness each day.",
|
| 302 |
+
"0 8 * * *", ["ut_odoo_orders"], [
|
| 303 |
+
{"id": "act_find", "kind": "find_records", "config": {
|
| 304 |
+
"table": "ut_odoo_orders", "cond": None, "limit": 25}},
|
| 305 |
+
report("Read the Odoo connection's database sync statuses in this workspace. "
|
| 306 |
+
"Report tables whose last successful sync is older than 24 hours or failed. "
|
| 307 |
+
"Name each table and the observed timestamp or error. If freshness cannot "
|
| 308 |
+
"be read, report it as unknown. Only read; do not sync, change settings, "
|
| 309 |
+
"edit records or send messages.", "connectors")]),
|
| 310 |
+
starter("ar_follow_up_digest", "AR follow-up digest", "Draft a weekly digest of overdue invoices.",
|
| 311 |
+
"0 9 * * 1", ["ut_odoo_invoices"], [
|
| 312 |
+
# The condition vocabulary orders numbers only. Evaluate due dates in the
|
| 313 |
+
# report at run time, never against a date frozen when the template was saved.
|
| 314 |
+
{"id": "act_find", "kind": "find_records", "config": {
|
| 315 |
+
"table": "ut_odoo_invoices",
|
| 316 |
+
"cond": {"field": "residual", "op": ">", "value": 0}, "limit": 100}},
|
| 317 |
+
report("Read open invoices in this workspace's Odoo invoices database. Compare "
|
| 318 |
+
"due_date with today's date and include only overdue invoices with residual "
|
| 319 |
+
"above zero. Draft a follow-up digest grouped by customer, with invoice, "
|
| 320 |
+
"due date, open amount and suggested next action. State any missing data "
|
| 321 |
+
"or incomplete coverage. Only read; do not edit records or send messages.",
|
| 322 |
+
"ut_odoo_invoices")]),
|
| 323 |
+
starter("data_health_watcher", "Data-health watcher", "Check for stale or empty databases each day.",
|
| 324 |
+
"0 7 * * *", [], [
|
| 325 |
+
report("Read the databases visible in this workspace. Flag empty databases and "
|
| 326 |
+
"those whose last successful refresh is older than 24 hours. Report each "
|
| 327 |
+
"database name, observed row count and refresh time when available. "
|
| 328 |
+
"Treat missing freshness information as unknown, never as healthy. "
|
| 329 |
+
"Only read; do not edit records, change settings or send messages.", "data")]),
|
| 330 |
+
]
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def _content_studio():
|
| 334 |
+
"""The Content Studio App's own module, or None when it is absent (rule 13: delete-safe)."""
|
| 335 |
+
try:
|
| 336 |
+
import content_studio
|
| 337 |
+
except ImportError:
|
| 338 |
+
return None
|
| 339 |
+
return content_studio
|
| 340 |
+
|
| 341 |
+
|
| 342 |
def _module_templates(session):
|
| 343 |
+
from core import registry, sheet_templates, user_tables
|
| 344 |
from routes_connectors import _entry_types
|
| 345 |
from routes_keychain import env_odoo_available
|
| 346 |
from routes_nav import application_templates
|
|
|
|
| 355 |
saved = _saved_sheet_templates(session.runtime)
|
| 356 |
data_templates = [(row, _data_template_database(row, definitions))
|
| 357 |
for row in _templates().TEMPLATES]
|
| 358 |
+
# S67-T36 (R25): an archived Agents module offers no agent template; the create route stays.
|
| 359 |
+
agent_templates = (_agent_templates() if _may_open(session, "automation")
|
| 360 |
+
and not registry.is_archived("automation") else [])
|
| 361 |
databases = {row["database_hint"] for row in sheet_templates.TEMPLATES}
|
| 362 |
databases.update(db for row in saved for db in row["databases"])
|
| 363 |
databases.update(database for _row, database in data_templates if database)
|
| 364 |
+
databases.update(db for row in agent_templates for db in row["databases"])
|
| 365 |
database_states = _database_states(session, definitions, databases)
|
| 366 |
requirement = lambda database, connector="": _requirement(
|
| 367 |
database, definitions, have, database_states, connector)
|
|
|
|
| 383 |
app.extend(_catalog_row("finance:" + row["key"], row["label"], row["description"], [],
|
| 384 |
creation_mode="post")
|
| 385 |
for row in finance_templates(may_open=lambda permission: _may_open(session, permission)))
|
| 386 |
+
sections = [{"module": "sheet", "templates": sheet},
|
| 387 |
+
{"module": "data", "templates": data},
|
| 388 |
+
{"module": "app", "templates": app}]
|
| 389 |
+
# S67-T40 (R18): every tenant's App catalogue offers Content Studio, which creates its own
|
| 390 |
+
# databases. Guarded (rule 13): without the module the catalogue simply omits the row.
|
| 391 |
+
# β Appended AFTER `sections` (it is the App section's own list, so the order stays Finance,
|
| 392 |
+
# then Studio): `verify_shell_rail.py` scopes its S65-T10 needles to the span from the Finance
|
| 393 |
+
# offer to `sections = [`, and its finance-offer-wizard control mutates the FIRST
|
| 394 |
+
# `creation_mode="post"` in this file. A Studio row inside that span, or above it, masks it.
|
| 395 |
+
studio = _content_studio()
|
| 396 |
+
if studio is not None:
|
| 397 |
+
app.append(_catalog_row(studio.TEMPLATE_KEY, studio.LABEL, studio.DESCRIPTION, [],
|
| 398 |
+
creation_mode="post"))
|
| 399 |
+
if agent_templates:
|
| 400 |
+
sections.append({"module": "agents", "templates": [
|
| 401 |
+
_catalog_row(row["key"], row["name"], row["description"],
|
| 402 |
+
[requirement(db, "odoo") for db in row["databases"]], creation_mode="post")
|
| 403 |
+
for row in agent_templates]})
|
| 404 |
+
return {"sections": sections}
|
| 405 |
|
| 406 |
|
| 407 |
@router.get("/sheet-templates")
|
|
|
|
| 506 |
from urllib.parse import quote
|
| 507 |
|
| 508 |
body = body if isinstance(body, dict) else {}
|
| 509 |
+
if module == "agents":
|
| 510 |
+
# A direct call to create_automation does not run its FastAPI dependency.
|
| 511 |
+
session.require("automation")
|
| 512 |
if module == "app" and key.startswith("finance:"):
|
| 513 |
import application_registry
|
| 514 |
from modules.finance_apps import finance_application
|
|
|
|
| 544 |
raise err(404, "unknown_template", "That template does not exist.")
|
| 545 |
if not template["creatable"]:
|
| 546 |
raise err(409, "template_unavailable", "Connect the required source and switch on its databases.")
|
| 547 |
+
studio = _content_studio() if module == "app" else None
|
| 548 |
+
if studio is not None and key == studio.TEMPLATE_KEY:
|
| 549 |
+
if not session.runtime.available():
|
| 550 |
+
raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was created.")
|
| 551 |
+
try:
|
| 552 |
+
return studio.create(session, body.get("name") or "")
|
| 553 |
+
except studio.ContentStudioError as exc:
|
| 554 |
+
raise err(409, "content_studio_refused", str(exc))
|
| 555 |
+
except PermissionError as exc:
|
| 556 |
+
raise err(403, "forbidden", str(exc))
|
| 557 |
if module == "sheet":
|
| 558 |
database = template["requires"][0]["database"] if template["requires"] else ""
|
| 559 |
result = apply_sheet_template(key, {"database": database, "name": body.get("name", "")}, session)
|
|
|
|
| 562 |
database = template["requires"][0]["database"]
|
| 563 |
result = apply_template(key, {"table": database}, session)
|
| 564 |
return {"address": "#/" + quote(database, safe=""), **result}
|
| 565 |
+
if module == "agents":
|
| 566 |
+
from routes_automation import create_automation
|
| 567 |
+
|
| 568 |
+
definition = next(row["definition"] for row in _agent_templates() if row["key"] == key)
|
| 569 |
+
result = create_automation(definition, session=session)
|
| 570 |
+
return {"address": "#/automation?agent=" + quote(result["automation"]["id"], safe=""), **result}
|
| 571 |
# The existing wizard still owns its POST-then-PATCH share sequence. API clients may
|
| 572 |
# dispatch creation here; the existing App writer retains schema and permission walls.
|
| 573 |
result = create_nav_application({**body, "templateKind": key}, session=session)
|
| 574 |
app = result["application"]
|
| 575 |
return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""),
|
| 576 |
**result}
|
| 577 |
+
|
| 578 |
+
# S67-T41: Content Studio's own routes (`/api/v1/content-studio/*`) ride this router because the
|
| 579 |
+
# App wires no mount of its own into main.py; GUARDED, so deleting content_studio.py drops the
|
| 580 |
+
# routes and nothing else (rule 13).
|
| 581 |
+
# β An inline guarded import, NOT a call to `_content_studio()`: this tail belongs to the
|
| 582 |
+
# dispatcher block, which `verify_home.py`'s W61-T23 declaration-order control moves ABOVE that
|
| 583 |
+
# helper's definition, where a call is a NameError that crashes the whole gate.
|
| 584 |
+
try:
|
| 585 |
+
import content_studio as _studio_module
|
| 586 |
+
except ImportError:
|
| 587 |
+
_studio_module = None
|
| 588 |
+
if _studio_module is not None and getattr(_studio_module, "router", None) is not None:
|
| 589 |
+
router.include_router(_studio_module.router)
|
platform/core/connector_spec.py
CHANGED
|
@@ -278,7 +278,7 @@ AMAZON_TABLES = (
|
|
| 278 |
# β THE MONEY-TO-ORDER BRIDGE β what turns "an order happened" into "this is what we were
|
| 279 |
# actually paid for it". The V2 settlement report carries the order id per line.
|
| 280 |
links=(
|
| 281 |
-
("
|
| 282 |
("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 283 |
),
|
| 284 |
),
|
|
@@ -320,6 +320,309 @@ AMAZON_TABLES = (
|
|
| 320 |
# about this product", and a link would be misleading.
|
| 321 |
links=(),
|
| 322 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 323 |
)
|
| 324 |
|
| 325 |
|
|
|
|
| 278 |
# β THE MONEY-TO-ORDER BRIDGE β what turns "an order happened" into "this is what we were
|
| 279 |
# actually paid for it". The V2 settlement report carries the order id per line.
|
| 280 |
links=(
|
| 281 |
+
("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
|
| 282 |
("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 283 |
),
|
| 284 |
),
|
|
|
|
| 320 |
# about this product", and a link would be misleading.
|
| 321 |
links=(),
|
| 322 |
),
|
| 323 |
+
ConnectorTableSpec(
|
| 324 |
+
key=_AMZN + "economics", native="analytics_economics_2024_03_15",
|
| 325 |
+
source="amzn_economics", label="Amazon economics", pinned="msku",
|
| 326 |
+
identity=("start_date", "msku", "marketplace_id"),
|
| 327 |
+
links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 328 |
+
("child_asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 329 |
+
),
|
| 330 |
+
ConnectorTableSpec(
|
| 331 |
+
key=_AMZN + "transactions", native="finances/2024-06-19/transactions",
|
| 332 |
+
source="amzn_transactions", label="Amazon financial transactions", pinned="transaction_id",
|
| 333 |
+
identity=("transaction_id", "item_position"),
|
| 334 |
+
links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
|
| 335 |
+
("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 336 |
+
("asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 337 |
+
),
|
| 338 |
+
ConnectorTableSpec(
|
| 339 |
+
key=_AMZN + "fba_fee_estimates", native="GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA",
|
| 340 |
+
source="amzn_fba_fee_estimates", label="Amazon FBA fee estimates", pinned="sku",
|
| 341 |
+
identity=("_snapshot_date", "sku", "amazon_store"),
|
| 342 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 343 |
+
("asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 344 |
+
),
|
| 345 |
+
ConnectorTableSpec(
|
| 346 |
+
key=_AMZN + "referral_fees", native="GET_REFERRAL_FEE_PREVIEW_REPORT",
|
| 347 |
+
source="amzn_referral_fees", label="Amazon referral fee preview", pinned="seller_sku",
|
| 348 |
+
identity=("_snapshot_date", "seller_sku"),
|
| 349 |
+
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 350 |
+
("asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 351 |
+
),
|
| 352 |
+
ConnectorTableSpec(
|
| 353 |
+
key=_AMZN + "storage_fees", native="GET_FBA_STORAGE_FEE_CHARGES_DATA",
|
| 354 |
+
source="amzn_storage_fees", label="Amazon storage fees", pinned="asin",
|
| 355 |
+
identity=("month", "asin", "fnsku", "fulfillment_center"),
|
| 356 |
+
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 357 |
+
),
|
| 358 |
+
ConnectorTableSpec(
|
| 359 |
+
key=_AMZN + "search_query_performance", native="GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
|
| 360 |
+
source="amzn_search_query_performance", label="Amazon search query performance",
|
| 361 |
+
pinned="search_query_data_search_query",
|
| 362 |
+
identity=("start_date", "end_date", "asin", "search_query_data_search_query"),
|
| 363 |
+
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 364 |
+
),
|
| 365 |
+
ConnectorTableSpec(
|
| 366 |
+
key=_AMZN + "long_term_storage_fees", native="GET_FBA_FULFILLMENT_LONGTERM_STORAGE_FEE_CHARGES_DATA",
|
| 367 |
+
source="amzn_long_term_storage_fees", label="Amazon long term storage fees", pinned="sku",
|
| 368 |
+
identity=("_period_start", "sku", "fnsku", "asin", "_report_row_number"),
|
| 369 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 370 |
+
),
|
| 371 |
+
ConnectorTableSpec(
|
| 372 |
+
key=_AMZN + "storage_overage_fees", native="GET_FBA_OVERAGE_FEE_CHARGES_DATA",
|
| 373 |
+
source="amzn_storage_overage_fees", label="Amazon storage overage fees", pinned="storage_type",
|
| 374 |
+
identity=("_period_start", "storage_type", "_report_row_number"),
|
| 375 |
+
links=(),
|
| 376 |
+
),
|
| 377 |
+
ConnectorTableSpec(
|
| 378 |
+
key=_AMZN + "reimbursements", native="GET_FBA_REIMBURSEMENTS_DATA",
|
| 379 |
+
source="amzn_reimbursements", label="Amazon reimbursements", pinned="reimbursement_id",
|
| 380 |
+
identity=("_period_start", "reimbursement_id", "sku", "asin", "amazon_order_id", "_report_row_number"),
|
| 381 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"), ("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),),
|
| 382 |
+
),
|
| 383 |
+
ConnectorTableSpec(
|
| 384 |
+
key=_AMZN + "fba_customer_returns", native="GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
|
| 385 |
+
source="amzn_fba_customer_returns", label="Amazon FBA customer returns", pinned="order_id",
|
| 386 |
+
identity=("_period_start", "order_id", "sku", "fnsku", "asin", "_report_row_number"),
|
| 387 |
+
links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 388 |
+
),
|
| 389 |
+
ConnectorTableSpec(
|
| 390 |
+
key=_AMZN + "replacements", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA",
|
| 391 |
+
source="amzn_replacements", label="Amazon replacements", pinned="replacement_amazon_order_id",
|
| 392 |
+
identity=("_period_start", "replacement_amazon_order_id", "original_amazon_order_id", "sku", "asin", "_report_row_number"),
|
| 393 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 394 |
+
),
|
| 395 |
+
ConnectorTableSpec(
|
| 396 |
+
key=_AMZN + "fba_shipped_sales", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_SALES_DATA",
|
| 397 |
+
source="amzn_fba_shipped_sales", label="Amazon FBA shipped sales", pinned="amazon_order_id",
|
| 398 |
+
identity=("_period_start", "amazon_order_id", "sku", "fnsku", "asin", "_report_row_number"),
|
| 399 |
+
links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 400 |
+
),
|
| 401 |
+
ConnectorTableSpec(
|
| 402 |
+
key=_AMZN + "fba_shipment_promotions", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA",
|
| 403 |
+
source="amzn_fba_shipment_promotions", label="Amazon FBA shipment promotions", pinned="amazon_order_id",
|
| 404 |
+
identity=("_period_start", "amazon_order_id", "sku", "_report_row_number"),
|
| 405 |
+
links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),),
|
| 406 |
+
),
|
| 407 |
+
ConnectorTableSpec(
|
| 408 |
+
key=_AMZN + "inventory_ledger_detail", native="GET_LEDGER_DETAIL_VIEW_DATA",
|
| 409 |
+
source="amzn_inventory_ledger_detail", label="Amazon inventory ledger detail", pinned="msku",
|
| 410 |
+
identity=("_period_start", "msku", "fnsku", "asin", "_report_row_number"),
|
| 411 |
+
links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 412 |
+
),
|
| 413 |
+
ConnectorTableSpec(
|
| 414 |
+
key=_AMZN + "inventory_planning", native="GET_FBA_INVENTORY_PLANNING_DATA",
|
| 415 |
+
source="amzn_inventory_planning", label="Amazon inventory planning", pinned="sku",
|
| 416 |
+
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 417 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 418 |
+
),
|
| 419 |
+
ConnectorTableSpec(
|
| 420 |
+
key=_AMZN + "manage_inventory", native="GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA",
|
| 421 |
+
source="amzn_manage_inventory", label="Amazon manage inventory", pinned="sku",
|
| 422 |
+
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 423 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 424 |
+
),
|
| 425 |
+
ConnectorTableSpec(
|
| 426 |
+
key=_AMZN + "reserved_inventory", native="GET_RESERVED_INVENTORY_DATA",
|
| 427 |
+
source="amzn_reserved_inventory", label="Amazon reserved inventory", pinned="sku",
|
| 428 |
+
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 429 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 430 |
+
),
|
| 431 |
+
ConnectorTableSpec(
|
| 432 |
+
key=_AMZN + "restock_recommendations", native="GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT",
|
| 433 |
+
source="amzn_restock_recommendations", label="Amazon restock recommendations", pinned="merchant_sku",
|
| 434 |
+
identity=("_snapshot_date", "merchant_sku", "fnsku", "asin", "_report_row_number"),
|
| 435 |
+
links=(("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 436 |
+
("asin", _AMZN + "listings", "asin1", "many_to_one")),
|
| 437 |
+
),
|
| 438 |
+
ConnectorTableSpec(
|
| 439 |
+
key=_AMZN + "stranded_inventory", native="GET_STRANDED_INVENTORY_UI_DATA",
|
| 440 |
+
source="amzn_stranded_inventory", label="Amazon stranded inventory", pinned="sku",
|
| 441 |
+
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 442 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 443 |
+
),
|
| 444 |
+
ConnectorTableSpec(
|
| 445 |
+
key=_AMZN + "inbound_noncompliance", native="GET_FBA_FULFILLMENT_INBOUND_NONCOMPLIANCE_DATA",
|
| 446 |
+
source="amzn_inbound_noncompliance", label="Amazon inbound noncompliance", pinned="shipment_id",
|
| 447 |
+
identity=("_period_start", "shipment_id", "sku", "asin", "_report_row_number"),
|
| 448 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 449 |
+
),
|
| 450 |
+
ConnectorTableSpec(
|
| 451 |
+
key=_AMZN + "inventory_expiration_dates", native="GET_FBA_FULFILLMENT_INVENTORY_EXPIRATION_DATE_DATA",
|
| 452 |
+
source="amzn_inventory_expiration_dates", label="Amazon inventory expiration dates", pinned="sku",
|
| 453 |
+
identity=("_snapshot_date", "sku", "fnsku", "_report_row_number"),
|
| 454 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 455 |
+
),
|
| 456 |
+
ConnectorTableSpec(
|
| 457 |
+
key=_AMZN + "recommended_removals", native="GET_FBA_RECOMMENDED_REMOVAL_DATA",
|
| 458 |
+
source="amzn_recommended_removals", label="Amazon recommended removals", pinned="sku",
|
| 459 |
+
identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 460 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 461 |
+
),
|
| 462 |
+
ConnectorTableSpec(
|
| 463 |
+
key=_AMZN + "removal_orders", native="GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
|
| 464 |
+
source="amzn_removal_orders", label="Amazon removal orders", pinned="order_id",
|
| 465 |
+
identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
|
| 466 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 467 |
+
),
|
| 468 |
+
ConnectorTableSpec(
|
| 469 |
+
key=_AMZN + "removal_shipments", native="GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
|
| 470 |
+
source="amzn_removal_shipments", label="Amazon removal shipments", pinned="order_id",
|
| 471 |
+
identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
|
| 472 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 473 |
+
),
|
| 474 |
+
ConnectorTableSpec(
|
| 475 |
+
key=_AMZN + "subscribe_save_forecast", native="GET_FBA_SNS_FORECAST_DATA",
|
| 476 |
+
source="amzn_subscribe_save_forecast", label="Amazon Subscribe and Save forecast", pinned="sku",
|
| 477 |
+
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 478 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 479 |
+
),
|
| 480 |
+
ConnectorTableSpec(
|
| 481 |
+
key=_AMZN + "subscribe_save_performance", native="GET_FBA_SNS_PERFORMANCE_DATA",
|
| 482 |
+
source="amzn_subscribe_save_performance", label="Amazon Subscribe and Save performance", pinned="sku",
|
| 483 |
+
identity=("_period_start", "sku", "asin", "_report_row_number"),
|
| 484 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 485 |
+
),
|
| 486 |
+
ConnectorTableSpec(
|
| 487 |
+
key=_AMZN + "remote_fulfillment_eligibility", native="GET_REMOTE_FULFILLMENT_ELIGIBILITY",
|
| 488 |
+
source="amzn_remote_fulfillment_eligibility", label="Amazon remote fulfillment eligibility", pinned="sku",
|
| 489 |
+
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 490 |
+
links=(), # Measured header has no product key; do not declare a guessed join.
|
| 491 |
+
),
|
| 492 |
+
ConnectorTableSpec(
|
| 493 |
+
key=_AMZN + "search_catalog_performance", native="GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT",
|
| 494 |
+
source="amzn_search_catalog_performance", label="Amazon search catalog performance", pinned="asin",
|
| 495 |
+
identity=("_period_start", "asin", "_report_row_number"),
|
| 496 |
+
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 497 |
+
),
|
| 498 |
+
ConnectorTableSpec(
|
| 499 |
+
key=_AMZN + "market_basket", native="GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT",
|
| 500 |
+
source="amzn_market_basket", label="Amazon market basket", pinned="asin",
|
| 501 |
+
identity=("_period_start", "asin", "purchased_with_asin", "_report_row_number"),
|
| 502 |
+
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 503 |
+
),
|
| 504 |
+
ConnectorTableSpec(
|
| 505 |
+
key=_AMZN + "repeat_purchase", native="GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT",
|
| 506 |
+
source="amzn_repeat_purchase", label="Amazon repeat purchase", pinned="asin",
|
| 507 |
+
identity=("_period_start", "asin", "_report_row_number"),
|
| 508 |
+
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 509 |
+
),
|
| 510 |
+
ConnectorTableSpec(
|
| 511 |
+
key=_AMZN + "promotion_performance", native="GET_PROMOTION_PERFORMANCE_REPORT",
|
| 512 |
+
source="amzn_promotion_performance", label="Amazon promotion performance", pinned="promotion_id",
|
| 513 |
+
identity=("_period_start", "promotion_id", "_report_row_number"),
|
| 514 |
+
links=(),
|
| 515 |
+
),
|
| 516 |
+
ConnectorTableSpec(
|
| 517 |
+
key=_AMZN + "coupon_performance", native="GET_COUPON_PERFORMANCE_REPORT",
|
| 518 |
+
source="amzn_coupon_performance", label="Amazon coupon performance", pinned="coupon_id",
|
| 519 |
+
identity=("_period_start", "coupon_id", "_report_row_number"),
|
| 520 |
+
links=(),
|
| 521 |
+
),
|
| 522 |
+
ConnectorTableSpec(
|
| 523 |
+
key=_AMZN + "account_health", native="GET_V2_SELLER_PERFORMANCE_REPORT",
|
| 524 |
+
source="amzn_account_health", label="Amazon account health", pinned="marketplace_id",
|
| 525 |
+
identity=("_snapshot_date", "marketplace_id", "_json_root", "_report_row_number"),
|
| 526 |
+
links=(),
|
| 527 |
+
),
|
| 528 |
+
ConnectorTableSpec(
|
| 529 |
+
key=_AMZN + "inactive_listings", native="GET_MERCHANT_LISTINGS_INACTIVE_DATA",
|
| 530 |
+
source="amzn_inactive_listings", label="Amazon inactive listings", pinned="seller_sku",
|
| 531 |
+
identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
|
| 532 |
+
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 533 |
+
("asin1", _AMZN + "listings", "asin1", "many_to_one")),
|
| 534 |
+
),
|
| 535 |
+
ConnectorTableSpec(
|
| 536 |
+
key=_AMZN + "cancelled_listings", native="GET_MERCHANT_CANCELLED_LISTINGS_DATA",
|
| 537 |
+
source="amzn_cancelled_listings", label="Amazon cancelled listings", pinned="seller_sku",
|
| 538 |
+
identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
|
| 539 |
+
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 540 |
+
("asin1", _AMZN + "listings", "asin1", "many_to_one")),
|
| 541 |
+
),
|
| 542 |
+
ConnectorTableSpec(
|
| 543 |
+
key=_AMZN + "suppressed_listings", native="GET_MERCHANTS_LISTINGS_FYP_REPORT",
|
| 544 |
+
source="amzn_suppressed_listings", label="Amazon suppressed listings", pinned="sku",
|
| 545 |
+
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 546 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 547 |
+
),
|
| 548 |
+
ConnectorTableSpec(
|
| 549 |
+
key=_AMZN + "listing_defects", native="GET_MERCHANT_LISTINGS_DEFECT_DATA",
|
| 550 |
+
source="amzn_listing_defects", label="Amazon listing defects", pinned="sku",
|
| 551 |
+
identity=("_snapshot_date", "sku", "_report_row_number"),
|
| 552 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 553 |
+
),
|
| 554 |
+
ConnectorTableSpec(
|
| 555 |
+
key=_AMZN + "sold_listings", native="GET_CONVERGED_FLAT_FILE_SOLD_LISTINGS_DATA",
|
| 556 |
+
source="amzn_sold_listings", label="Amazon sold listings", pinned="sku",
|
| 557 |
+
identity=("_snapshot_date", "sku", "_report_row_number"),
|
| 558 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 559 |
+
),
|
| 560 |
+
ConnectorTableSpec(
|
| 561 |
+
key=_AMZN + "mfn_returns", native="GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
|
| 562 |
+
source="amzn_mfn_returns", label="Amazon MFN returns", pinned="order_id",
|
| 563 |
+
identity=("_period_start", "order_id", "sku", "asin", "_report_row_number"),
|
| 564 |
+
links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 565 |
+
),
|
| 566 |
+
ConnectorTableSpec(
|
| 567 |
+
key=_AMZN + "b2b_recommended", native="GET_B2B_PRODUCT_OPPORTUNITIES_RECOMMENDED_FOR_YOU",
|
| 568 |
+
source="amzn_b2b_recommended", label="Amazon B2B product opportunities", pinned="asin",
|
| 569 |
+
identity=("_snapshot_date", "asin", "_report_row_number"),
|
| 570 |
+
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 571 |
+
),
|
| 572 |
+
ConnectorTableSpec(
|
| 573 |
+
key=_AMZN + "b2b_not_yet_on_amazon", native="GET_B2B_PRODUCT_OPPORTUNITIES_NOT_YET_ON_AMAZON",
|
| 574 |
+
source="amzn_b2b_not_yet_on_amazon", label="Amazon B2B products not yet on Amazon", pinned="ean",
|
| 575 |
+
identity=("_snapshot_date", "ean", "upc", "part_number", "_report_row_number"),
|
| 576 |
+
links=(),
|
| 577 |
+
),
|
| 578 |
+
ConnectorTableSpec(
|
| 579 |
+
key=_AMZN + "geo_opportunities", native="GET_FLAT_FILE_GEO_OPPORTUNITIES",
|
| 580 |
+
source="amzn_geo_opportunities", label="Amazon geo opportunities", pinned="asin",
|
| 581 |
+
identity=("_snapshot_date", "asin", "_report_row_number"),
|
| 582 |
+
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 583 |
+
),
|
| 584 |
+
ConnectorTableSpec(
|
| 585 |
+
key=_AMZN + "out_of_stock_mfn", native="GET_OUT_OF_STOCK_MFN_INFORMATION",
|
| 586 |
+
source="amzn_out_of_stock_mfn", label="Amazon out of stock MFN listings", pinned="sku",
|
| 587 |
+
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 588 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 589 |
+
),
|
| 590 |
+
ConnectorTableSpec(
|
| 591 |
+
key=_AMZN + "b2b_quotes", native="RFQD_BULK_DOWNLOAD",
|
| 592 |
+
source="amzn_b2b_quotes", label="Amazon B2B quantity discounts", pinned="sku",
|
| 593 |
+
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 594 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 595 |
+
),
|
| 596 |
+
ConnectorTableSpec(
|
| 597 |
+
key=_AMZN + "fee_discounts", native="FEE_DISCOUNTS_REPORT",
|
| 598 |
+
source="amzn_fee_discounts", label="Amazon fee discounts", pinned="sku",
|
| 599 |
+
identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 600 |
+
links=(), # The measured empty report supplies metadata only.
|
| 601 |
+
),
|
| 602 |
+
ConnectorTableSpec(
|
| 603 |
+
key=_AMZN + "order_metrics", native="sales/v1/orderMetrics",
|
| 604 |
+
source="amzn_order_metrics", label="Amazon order metrics", pinned="interval",
|
| 605 |
+
identity=("_period_start", "interval", "_report_row_number"),
|
| 606 |
+
links=(),
|
| 607 |
+
),
|
| 608 |
+
ConnectorTableSpec(
|
| 609 |
+
key=_AMZN + "fba_inventory", native="fba/inventory/v1/summaries",
|
| 610 |
+
source="amzn_fba_inventory", label="Amazon FBA inventory summaries", pinned="seller_sku",
|
| 611 |
+
identity=("_snapshot_date", "seller_sku", "fn_sku", "asin", "_report_row_number"),
|
| 612 |
+
links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 613 |
+
),
|
| 614 |
+
ConnectorTableSpec(
|
| 615 |
+
key=_AMZN + "listing_items", native="listings/2021-08-01/items/{sellerId}",
|
| 616 |
+
source="amzn_listing_items", label="Amazon listing status and issues", pinned="sku",
|
| 617 |
+
identity=("_snapshot_date", "sku", "_report_row_number"),
|
| 618 |
+
links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
|
| 619 |
+
),
|
| 620 |
+
ConnectorTableSpec(
|
| 621 |
+
key=_AMZN + "catalog_items", native="catalog/2022-04-01/items",
|
| 622 |
+
source="amzn_catalog_items", label="Amazon catalog attributes", pinned="asin",
|
| 623 |
+
identity=("_snapshot_date", "asin", "_report_row_number"),
|
| 624 |
+
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 625 |
+
),
|
| 626 |
)
|
| 627 |
|
| 628 |
|
platform/core/daily_sales_repair.py
CHANGED
|
@@ -40,6 +40,39 @@ _SUMMARY_RATIO_CASCADES = {
|
|
| 40 |
"R71": "=R69-R70", "R72": "=R71/R70", "R73": "=R72-Q72",
|
| 41 |
}
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
def operation_for(workbook_id):
|
| 45 |
"""Bind a repair operation to the workbook the caller names."""
|
|
@@ -141,10 +174,27 @@ def _validate_binding(name, binding, *, window_kind=None):
|
|
| 141 |
# Accept the same explicit, nonempty conjunction used by rows 108-113. Requiring
|
| 142 |
# both referenced date bounds keeps a missing period from becoming an unbounded sum.
|
| 143 |
nodes = tree.get("nodes") if isinstance(tree, dict) else None
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
if not has_view and not has_filter:
|
| 149 |
raise RepairRefusal(
|
| 150 |
"repair_binding_scope_required",
|
|
@@ -158,7 +208,7 @@ def _validate_binding(name, binding, *, window_kind=None):
|
|
| 158 |
period = binding.get("period")
|
| 159 |
if period is None and has_filter and not has_view:
|
| 160 |
bounds = {}
|
| 161 |
-
for node in
|
| 162 |
slot = node.get("valueRef")
|
| 163 |
if slot is not None:
|
| 164 |
if not isinstance(slot, dict) or set(slot) != {"ref"}:
|
|
@@ -281,6 +331,70 @@ def check_summary_formula_errors(book):
|
|
| 281 |
"negative_control": {"planted": len(expected), "found": len(found)}}
|
| 282 |
|
| 283 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 284 |
def _formula_error_comparison(before, after, edits, reported_errors=()):
|
| 285 |
"""Reject new errors anywhere and every residual error in explicitly planned cells.
|
| 286 |
|
|
@@ -331,6 +445,21 @@ def _cell_map(book):
|
|
| 331 |
for address, cell in (tab.get("cells") or {}).items()}
|
| 332 |
|
| 333 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 334 |
def _checked_projection(projected, edits, formula_check):
|
| 335 |
"""Make formula output part of the plan, never an unpersisted preflight side effect."""
|
| 336 |
if not callable(formula_check):
|
|
@@ -351,6 +480,10 @@ def _checked_projection(projected, edits, formula_check):
|
|
| 351 |
if before.get(slot) == after.get(slot):
|
| 352 |
continue
|
| 353 |
row = planned.get(slot)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 354 |
if row is None or row["kind"] == "recalculated":
|
| 355 |
previous, recalculated = before.get(slot), after.get(slot)
|
| 356 |
# Recalculation may refresh a healthy formula's result, but may not repair
|
|
@@ -365,7 +498,8 @@ def _checked_projection(projected, edits, formula_check):
|
|
| 365 |
!= {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated}:
|
| 366 |
raise RepairRefusal(
|
| 367 |
"repair_unplanned_formula_change",
|
| 368 |
-
"Formula validation would change
|
|
|
|
| 369 |
desired = sheets.clean_cell(_preserve_visual(previous, recalculated), slot[1])
|
| 370 |
if row is None:
|
| 371 |
sheet = sheet_names[slot[0]]
|
|
@@ -382,7 +516,8 @@ def _checked_projection(projected, edits, formula_check):
|
|
| 382 |
or row["after"].get("kind") != "formula":
|
| 383 |
raise RepairRefusal(
|
| 384 |
"repair_unplanned_formula_change",
|
| 385 |
-
"Formula validation would change
|
|
|
|
| 386 |
recalculated = after.get(slot)
|
| 387 |
if not isinstance(recalculated, dict) \
|
| 388 |
or (row["kind"] == "source"
|
|
@@ -441,10 +576,10 @@ def _summary_date(value):
|
|
| 441 |
"Every summary date must be a real ISO date from the sheet or mirror.") from exc
|
| 442 |
|
| 443 |
|
| 444 |
-
def _resolved(binding, book, resolve_binding):
|
| 445 |
if not callable(resolve_binding):
|
| 446 |
raise RepairRefusal("repair_binding_resolver_required", "The summary needs its scoped source resolver.")
|
| 447 |
-
result = resolve_binding(copy.deepcopy(binding), book, _tab(book,
|
| 448 |
if not isinstance(result, dict) or "value" not in result or result.get("value") is None:
|
| 449 |
raise RepairRefusal("repair_binding_unreadable", "A summary source did not return a value.")
|
| 450 |
if result.get("limits"):
|
|
@@ -599,9 +734,9 @@ def _build_summary_plan(current, request, resolve_binding, formula_check):
|
|
| 599 |
"scope": {"filter": {"conj": "and", "nodes": [
|
| 600 |
{"colId": "invoice_date", "op": "within", "dateWindow": {"kind": kind},
|
| 601 |
"valueRef": {"ref": end_ref}},
|
| 602 |
-
{"colId": "move_type", "op": "eq", "value": "out_invoice"}
|
| 603 |
-
|
| 604 |
-
|
| 605 |
_validate_binding(col + str(row_number), binding)
|
| 606 |
value = _resolved(binding, projected, resolve_binding)
|
| 607 |
if type(value) not in (int, float) or not math.isfinite(value):
|
|
@@ -685,6 +820,919 @@ def validate(book, *, windows, mirror_as_of, partner_ids, invoice_validate, form
|
|
| 685 |
return checks
|
| 686 |
|
| 687 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 688 |
def _build_plan(current, request, resolve_binding):
|
| 689 |
if not isinstance(request.get("source_cells"), list) or not request["source_cells"]:
|
| 690 |
raise RepairRefusal(
|
|
@@ -877,15 +1925,62 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
|
|
| 877 |
raise RepairRefusal("repair_revision_mismatch",
|
| 878 |
"The named workbook is not at the expected repair revision.")
|
| 879 |
digest = sheets.workbook_digest(current)
|
|
|
|
|
|
|
|
|
|
| 880 |
summary_evidence = None
|
| 881 |
summary_errors_before = None
|
| 882 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 883 |
summary_errors_before = check_summary_formula_errors(current)
|
| 884 |
edits, summary_evidence = _build_summary_plan(current, request, resolve_binding, formula_check)
|
| 885 |
else:
|
| 886 |
edits = _build_plan(current, request, resolve_binding)
|
| 887 |
projected = _project(current, edits)
|
| 888 |
projected, projected_errors = _checked_projection(projected, edits, formula_check)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 889 |
comparison = _formula_error_comparison(current, projected, edits, projected_errors)
|
| 890 |
summary_errors_after = (check_summary_formula_errors(projected)
|
| 891 |
if summary_evidence is not None else None)
|
|
@@ -896,7 +1991,11 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
|
|
| 896 |
"The projected repair introduces a formula error or leaves one in its repair scope, "
|
| 897 |
"so no repair was applied.")
|
| 898 |
visual_before = visual_metadata(current)
|
| 899 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 900 |
raise RepairRefusal("repair_visual_metadata_changed",
|
| 901 |
"The projected repair changes Daily Sales visual metadata.")
|
| 902 |
|
|
@@ -928,9 +2027,13 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
|
|
| 928 |
or _cell_map(checked_readback) != _cell_map(readback):
|
| 929 |
raise RepairRefusal("repair_readback_formula_state",
|
| 930 |
"The repaired workbook did not read back calculation-current.")
|
| 931 |
-
if
|
| 932 |
raise RepairRefusal("repair_readback_visual_metadata",
|
| 933 |
"Daily Sales visual metadata changed during repair.")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 934 |
if summary_evidence is not None:
|
| 935 |
tab = _tab(readback, SUMMARY_TAB)
|
| 936 |
if any((tab.get("cells", {}).get(col + str(row)) or {}).get("kind")
|
|
@@ -963,8 +2066,36 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
|
|
| 963 |
"bindings": bindings,
|
| 964 |
"formula_errors": formula_error_census(readback),
|
| 965 |
"formula_error_comparison": readback_comparison,
|
|
|
|
| 966 |
"visual_metadata_preserved": True,
|
| 967 |
}
|
| 968 |
if summary_evidence is not None:
|
| 969 |
result["summary_block"] = summary_evidence
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 970 |
return result
|
|
|
|
| 40 |
"R71": "=R69-R70", "R72": "=R71/R70", "R73": "=R72-Q72",
|
| 41 |
}
|
| 42 |
|
| 43 |
+
# Exact authored formulas at the staging snapshot; an owner edit is never an oracle.
|
| 44 |
+
# Q64 deliberately keeps its formula but participates in the recalculation/write plan.
|
| 45 |
+
_FORMULA_REPAIRS = {
|
| 46 |
+
("Summary (SR)", "A1", '=CONCATENATE(Print!C6," Sales Report")'):
|
| 47 |
+
('=CONCATENATE(C6," Sales Report")', "resolved", "C6 holds this tab's sales representative."),
|
| 48 |
+
("Summary (SR)", "W8", "=Print!C6"):
|
| 49 |
+
("=C6", "resolved", "C6 holds this tab's sales representative."),
|
| 50 |
+
("Summary (SR)", "Q39", '=_xlfn.XLOOKUP(Print!$C$6,Print!C14:C30,Print!E14:E30,"")'):
|
| 51 |
+
('=""', "deliberate blank", "The Print target table is absent from this workbook."),
|
| 52 |
+
("Summary (SR)", "O43", "=Q39-O41"):
|
| 53 |
+
('=IFERROR(Q39-O41,"")', "deliberate blank", "The target in Q39 is unavailable."),
|
| 54 |
+
("Summary (SR)", "R64", '=COUNTIF(Uncollectible!E7:E1000,"*")'):
|
| 55 |
+
('=""', "deliberate blank", "The Uncollectible tab is absent from this workbook."),
|
| 56 |
+
("Summary (SR)", "R65", '=COUNTIF(Closed!E7:E1000,"*")'):
|
| 57 |
+
('=""', "deliberate blank", "The Closed tab is absent from this workbook."),
|
| 58 |
+
("Summary (SR)", "Q64", "=R64"):
|
| 59 |
+
("=R64", "deliberate blank", "The unchanged reference follows the deliberate blank in R64."),
|
| 60 |
+
("Summary (SR)", "F71", "=(F77/F66) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1))"):
|
| 61 |
+
('=IFERROR((F77/F66) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1)),"")',
|
| 62 |
+
"deliberate blank", "The ratio is blank while its denominator is unavailable."),
|
| 63 |
+
("Summary (SR)", "F72", "=(F78/F69) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1))"):
|
| 64 |
+
('=IFERROR((F78/F69) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1)),"")',
|
| 65 |
+
"deliberate blank", "The ratio is blank while its denominator is unavailable."),
|
| 66 |
+
("Summary (SR)", "F73", "=F71-F72"):
|
| 67 |
+
('=IFERROR(F71-F72,"")', "deliberate blank", "The difference is blank while either ratio is unavailable."),
|
| 68 |
+
("Summary (SR)", "Q63", "=R63"):
|
| 69 |
+
('=IF(ISBLANK(R63),"",R63)', "deliberate blank", "An empty R63 has no figure to report."),
|
| 70 |
+
("Summary (SR)", "R68", "=R67-Q67"):
|
| 71 |
+
('=IFERROR(R67-Q67,"")', "deliberate blank", "The difference is blank while R67 is unavailable."),
|
| 72 |
+
(SUMMARY_TAB, "F52", "=G51-F51"):
|
| 73 |
+
('=IFERROR(G51-F51,"")', "deliberate blank", "The difference is blank while F51 is unavailable."),
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
|
| 77 |
def operation_for(workbook_id):
|
| 78 |
"""Bind a repair operation to the workbook the caller names."""
|
|
|
|
| 174 |
# Accept the same explicit, nonempty conjunction used by rows 108-113. Requiring
|
| 175 |
# both referenced date bounds keeps a missing period from becoming an unbounded sum.
|
| 176 |
nodes = tree.get("nodes") if isinstance(tree, dict) else None
|
| 177 |
+
def conjunction_leaves(items):
|
| 178 |
+
if not isinstance(items, list) or not items:
|
| 179 |
+
return None
|
| 180 |
+
leaves = []
|
| 181 |
+
for node in items:
|
| 182 |
+
if not isinstance(node, dict):
|
| 183 |
+
return None
|
| 184 |
+
if "children" in node:
|
| 185 |
+
children = conjunction_leaves(node["children"])
|
| 186 |
+
if node.get("conj", "and") != "and" or children is None:
|
| 187 |
+
return None
|
| 188 |
+
leaves.extend(children)
|
| 189 |
+
elif node.get("colId") and node.get("op") and (
|
| 190 |
+
"value" in node or isinstance(node.get("valueRef"), dict)):
|
| 191 |
+
leaves.append(node)
|
| 192 |
+
else:
|
| 193 |
+
return None
|
| 194 |
+
return leaves
|
| 195 |
+
|
| 196 |
+
leaves = conjunction_leaves(nodes)
|
| 197 |
+
has_filter = leaves is not None and tree.get("conj") == "and"
|
| 198 |
if not has_view and not has_filter:
|
| 199 |
raise RepairRefusal(
|
| 200 |
"repair_binding_scope_required",
|
|
|
|
| 208 |
period = binding.get("period")
|
| 209 |
if period is None and has_filter and not has_view:
|
| 210 |
bounds = {}
|
| 211 |
+
for node in leaves:
|
| 212 |
slot = node.get("valueRef")
|
| 213 |
if slot is not None:
|
| 214 |
if not isinstance(slot, dict) or set(slot) != {"ref"}:
|
|
|
|
| 331 |
"negative_control": {"planted": len(expected), "found": len(found)}}
|
| 332 |
|
| 333 |
|
| 334 |
+
def _recalculated_formula_cycles(book, formula_check):
|
| 335 |
+
checked = copy.deepcopy(book)
|
| 336 |
+
try:
|
| 337 |
+
result = formula_check(checked)
|
| 338 |
+
except Exception as exc:
|
| 339 |
+
raise RepairRefusal("repair_cycle_check_failed",
|
| 340 |
+
"The formula engine could not complete the cycle check.") from exc
|
| 341 |
+
if isinstance(result, dict) and isinstance(result.get("book"), dict):
|
| 342 |
+
checked = result["book"]
|
| 343 |
+
# The engine gives all members of a cyclic component the same origin. Preserve
|
| 344 |
+
# authored tab names in the report; engine lookup keys are upper case.
|
| 345 |
+
groups = {}
|
| 346 |
+
for tab in checked.get("sheets") or ():
|
| 347 |
+
for address, cell in (tab.get("cells") or {}).items():
|
| 348 |
+
if not isinstance(cell, dict) or cell.get("kind") != "formula":
|
| 349 |
+
continue
|
| 350 |
+
error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
|
| 351 |
+
if not isinstance(error, dict) or error.get("code") != "circular_reference":
|
| 352 |
+
continue
|
| 353 |
+
label = str(tab.get("name") or "") + "!" + address
|
| 354 |
+
origin = str(error.get("origin") or label).upper()
|
| 355 |
+
groups.setdefault(origin, set()).add(label)
|
| 356 |
+
return sorted(sorted(addresses) for addresses in groups.values())
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def formula_cycles(book, formula_check):
|
| 360 |
+
"""Recalculate a copy and report the engine's circular references by component.
|
| 361 |
+
|
| 362 |
+
The injected checker is the same API-side engine used by the repair. Every call
|
| 363 |
+
first proves that both a two-cell cycle and a self-loop are detectable, without
|
| 364 |
+
trusting stored errors or changing the supplied workbook.
|
| 365 |
+
"""
|
| 366 |
+
if not callable(formula_check):
|
| 367 |
+
raise RepairRefusal("repair_cycle_check_required",
|
| 368 |
+
"The cycle check requires a formula engine.")
|
| 369 |
+
probe = {"sheets": [{"id": "cycle-probe", "name": "Cycle probe", "cells": {
|
| 370 |
+
address: {"kind": "formula", "state": "stale", "formula": formula}
|
| 371 |
+
for address, formula in (("A1", "=B1"), ("B1", "=A1"), ("C1", "=C1"))
|
| 372 |
+
}}]}
|
| 373 |
+
expected = [["Cycle probe!A1", "Cycle probe!B1"], ["Cycle probe!C1"]]
|
| 374 |
+
found = _recalculated_formula_cycles(probe, formula_check)
|
| 375 |
+
if found != expected:
|
| 376 |
+
raise RepairRefusal("repair_cycle_census_blind",
|
| 377 |
+
"The cycle check failed to find its planted formula cycles.")
|
| 378 |
+
cycles = _recalculated_formula_cycles(book, formula_check)
|
| 379 |
+
return {"count": len(cycles), "cycles": cycles,
|
| 380 |
+
"negative_control": {"planted": len(expected), "found": len(found), "cycles": found}}
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def _formula_cycle_comparison(before, after, formula_check):
|
| 384 |
+
"""Diagnostic evidence only: keep execute's existing acceptance/refusal contract.
|
| 385 |
+
|
| 386 |
+
Legacy callers may omit the engine. A missing or blind checker is explicitly
|
| 387 |
+
reported as a named refusal, never represented as a successful zero census.
|
| 388 |
+
"""
|
| 389 |
+
def report(book):
|
| 390 |
+
try:
|
| 391 |
+
return formula_cycles(book, formula_check)
|
| 392 |
+
except RepairRefusal as exc:
|
| 393 |
+
return {"refusal": {"code": exc.code, "message": exc.message}}
|
| 394 |
+
|
| 395 |
+
return {"before": report(before), "after": report(after)}
|
| 396 |
+
|
| 397 |
+
|
| 398 |
def _formula_error_comparison(before, after, edits, reported_errors=()):
|
| 399 |
"""Reject new errors anywhere and every residual error in explicitly planned cells.
|
| 400 |
|
|
|
|
| 445 |
for address, cell in (tab.get("cells") or {}).items()}
|
| 446 |
|
| 447 |
|
| 448 |
+
def _same_unplanned_error(previous, recalculated):
|
| 449 |
+
"""True when recalculation leaves an erroring formula with the same formula and error code."""
|
| 450 |
+
def code(cell):
|
| 451 |
+
error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
|
| 452 |
+
return error.get("code") if isinstance(error, dict) else None
|
| 453 |
+
|
| 454 |
+
return (isinstance(previous, dict) and isinstance(recalculated, dict)
|
| 455 |
+
and previous.get("kind") == recalculated.get("kind") == "formula"
|
| 456 |
+
and isinstance(previous.get("formula"), str)
|
| 457 |
+
and previous.get("formula") == recalculated.get("formula")
|
| 458 |
+
and code(previous) is not None and code(previous) == code(recalculated)
|
| 459 |
+
and {key: previous[key] for key in _VISUAL_CELL_KEYS if key in previous}
|
| 460 |
+
== {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated})
|
| 461 |
+
|
| 462 |
+
|
| 463 |
def _checked_projection(projected, edits, formula_check):
|
| 464 |
"""Make formula output part of the plan, never an unpersisted preflight side effect."""
|
| 465 |
if not callable(formula_check):
|
|
|
|
| 480 |
if before.get(slot) == after.get(slot):
|
| 481 |
continue
|
| 482 |
row = planned.get(slot)
|
| 483 |
+
if row is None and _same_unplanned_error(before.get(slot), after.get(slot)):
|
| 484 |
+
# An unrelated formula that was an error and recalculates to the SAME error is
|
| 485 |
+
# neither repaired nor broken by this plan, so it stays exactly as stored.
|
| 486 |
+
continue
|
| 487 |
if row is None or row["kind"] == "recalculated":
|
| 488 |
previous, recalculated = before.get(slot), after.get(slot)
|
| 489 |
# Recalculation may refresh a healthy formula's result, but may not repair
|
|
|
|
| 498 |
!= {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated}:
|
| 499 |
raise RepairRefusal(
|
| 500 |
"repair_unplanned_formula_change",
|
| 501 |
+
"Formula validation would change " + str(sheet_names.get(slot[0]) or slot[0])
|
| 502 |
+
+ "!" + slot[1] + ", a cell outside the supplied source diff.")
|
| 503 |
desired = sheets.clean_cell(_preserve_visual(previous, recalculated), slot[1])
|
| 504 |
if row is None:
|
| 505 |
sheet = sheet_names[slot[0]]
|
|
|
|
| 516 |
or row["after"].get("kind") != "formula":
|
| 517 |
raise RepairRefusal(
|
| 518 |
"repair_unplanned_formula_change",
|
| 519 |
+
"Formula validation would change " + str(sheet_names.get(slot[0]) or slot[0])
|
| 520 |
+
+ "!" + slot[1] + ", a cell outside the supplied source diff.")
|
| 521 |
recalculated = after.get(slot)
|
| 522 |
if not isinstance(recalculated, dict) \
|
| 523 |
or (row["kind"] == "source"
|
|
|
|
| 576 |
"Every summary date must be a real ISO date from the sheet or mirror.") from exc
|
| 577 |
|
| 578 |
|
| 579 |
+
def _resolved(binding, book, resolve_binding, *, tab_name=SUMMARY_TAB):
|
| 580 |
if not callable(resolve_binding):
|
| 581 |
raise RepairRefusal("repair_binding_resolver_required", "The summary needs its scoped source resolver.")
|
| 582 |
+
result = resolve_binding(copy.deepcopy(binding), book, _tab(book, tab_name))
|
| 583 |
if not isinstance(result, dict) or "value" not in result or result.get("value") is None:
|
| 584 |
raise RepairRefusal("repair_binding_unreadable", "A summary source did not return a value.")
|
| 585 |
if result.get("limits"):
|
|
|
|
| 734 |
"scope": {"filter": {"conj": "and", "nodes": [
|
| 735 |
{"colId": "invoice_date", "op": "within", "dateWindow": {"kind": kind},
|
| 736 |
"valueRef": {"ref": end_ref}},
|
| 737 |
+
{"colId": "move_type", "op": "eq", "value": "out_invoice"},
|
| 738 |
+
{"colId": "related:customer_link:pricelist", "op": "eq",
|
| 739 |
+
"valueRef": {"ref": scope_ref}}]}}}
|
| 740 |
_validate_binding(col + str(row_number), binding)
|
| 741 |
value = _resolved(binding, projected, resolve_binding)
|
| 742 |
if type(value) not in (int, float) or not math.isfinite(value):
|
|
|
|
| 820 |
return checks
|
| 821 |
|
| 822 |
|
| 823 |
+
def _binding_ref_slots(value):
|
| 824 |
+
"""Yield both operand slots at any depth without changing other binding metadata."""
|
| 825 |
+
if isinstance(value, dict):
|
| 826 |
+
for key, child in value.items():
|
| 827 |
+
if key in {"valueRef", "value2Ref"} and isinstance(child, dict):
|
| 828 |
+
yield child
|
| 829 |
+
yield from _binding_ref_slots(child)
|
| 830 |
+
elif isinstance(value, list):
|
| 831 |
+
for child in value:
|
| 832 |
+
yield from _binding_ref_slots(child)
|
| 833 |
+
|
| 834 |
+
|
| 835 |
+
def _has_legacy_related(value):
|
| 836 |
+
def linked(node):
|
| 837 |
+
if isinstance(node, dict):
|
| 838 |
+
key = node.get("colId")
|
| 839 |
+
return ((isinstance(key, str) and key.startswith("linked:"))
|
| 840 |
+
or any(linked(child) for child in node.values()))
|
| 841 |
+
return isinstance(node, list) and any(linked(child) for child in node)
|
| 842 |
+
|
| 843 |
+
return isinstance(value, dict) and ("related" in value or linked(value))
|
| 844 |
+
|
| 845 |
+
|
| 846 |
+
def _remap_cell(book, tab_name, ref):
|
| 847 |
+
"""Read an explicitly supplied A1 ref, relative to the migrated cell's tab."""
|
| 848 |
+
match = re.fullmatch(r"(?:(?:'((?:[^']|'')+)'|([^'!]+))!)?(\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6})",
|
| 849 |
+
ref) if isinstance(ref, str) else None
|
| 850 |
+
if not match:
|
| 851 |
+
raise RepairRefusal("repair_ref_remap_invalid", "Invalid remap reference " + str(ref) + ".")
|
| 852 |
+
name = (match[1].replace("''", "'") if match[1] else match[2]) or tab_name
|
| 853 |
+
try:
|
| 854 |
+
return (_tab(book, name).get("cells") or {}).get(_addr(match[3])) or {}
|
| 855 |
+
except RepairRefusal as exc:
|
| 856 |
+
raise RepairRefusal("repair_ref_remap_invalid",
|
| 857 |
+
"Remap reference " + ref + " cannot be read.") from exc
|
| 858 |
+
|
| 859 |
+
|
| 860 |
+
def _build_related_migration_plan(current, request, resolve_binding, formula_check=None):
|
| 861 |
+
"""Migrate by content, proving explicit moved helpers and every stored answer first."""
|
| 862 |
+
from core.sheet_binding import BindingMetadataRefusal, rewrite_legacy_related
|
| 863 |
+
|
| 864 |
+
migration = request["related_migration"]
|
| 865 |
+
if not isinstance(migration, dict) or not isinstance(migration.get("ref_remap", {}), dict):
|
| 866 |
+
raise RepairRefusal("repair_ref_remap_invalid", "Supply a reference remap object for the migration.")
|
| 867 |
+
if any(key in request for key in ("summary_block", "source_cells", "binding_cells")):
|
| 868 |
+
raise RepairRefusal("repair_migration_plan_conflict", "Run the related migration as its own repair.")
|
| 869 |
+
remap = migration.get("ref_remap", {})
|
| 870 |
+
edits = []
|
| 871 |
+
for tab in current.get("sheets") or ():
|
| 872 |
+
for address, before in (tab.get("cells") or {}).items():
|
| 873 |
+
if not isinstance(before, dict) or before.get("kind") != "bound" \
|
| 874 |
+
or not isinstance(before.get("binding"), dict) \
|
| 875 |
+
or not _has_legacy_related(before["binding"]):
|
| 876 |
+
continue
|
| 877 |
+
try:
|
| 878 |
+
binding = rewrite_legacy_related(before["binding"], linked=True)
|
| 879 |
+
except BindingMetadataRefusal as exc:
|
| 880 |
+
raise RepairRefusal(exc.code, exc.message) from exc
|
| 881 |
+
if _has_legacy_related(binding):
|
| 882 |
+
raise RepairRefusal("related_invalid", "A nested legacy relationship cannot be migrated.")
|
| 883 |
+
edits.append({"kind": "binding", "name": tab["name"] + "!" + address,
|
| 884 |
+
"sheet": tab["name"], "address": address, "before": copy.deepcopy(before),
|
| 885 |
+
"after": {**copy.deepcopy(before), "binding": binding}})
|
| 886 |
+
# A retry with the same remap must be a no-op, not an unused-remap refusal.
|
| 887 |
+
if not edits:
|
| 888 |
+
return [], []
|
| 889 |
+
if not str(request.get("refreshed_at") or "").strip():
|
| 890 |
+
raise RepairRefusal("repair_timestamp_required", "The migration needs an explicit refresh timestamp.")
|
| 891 |
+
edits.sort(key=lambda row: (row["sheet"].casefold(), row["address"]))
|
| 892 |
+
recalculated = []
|
| 893 |
+
|
| 894 |
+
def current_after_recalculation(tab_name, ref, stored):
|
| 895 |
+
# A structure change leaves a helper formula `stale` until the next recalculation, so the
|
| 896 |
+
# formula engine, not the stored flag, decides whether the stored text is still its answer.
|
| 897 |
+
if not callable(formula_check):
|
| 898 |
+
return False
|
| 899 |
+
if not recalculated:
|
| 900 |
+
checked = formula_check(copy.deepcopy(current))
|
| 901 |
+
recalculated.append(checked.get("book") if isinstance(checked, dict) else None)
|
| 902 |
+
if not isinstance(recalculated[0], dict):
|
| 903 |
+
return False
|
| 904 |
+
again = _remap_cell(recalculated[0], tab_name, ref)
|
| 905 |
+
return (again.get("kind") == "formula" and again.get("state") == "fresh"
|
| 906 |
+
and not again.get("error") and not again.get("refusal")
|
| 907 |
+
and again.get("value") == stored.get("value"))
|
| 908 |
+
|
| 909 |
+
used = set()
|
| 910 |
+
for row in edits:
|
| 911 |
+
for slot in _binding_ref_slots(row["after"]["binding"]):
|
| 912 |
+
ref = slot.get("ref")
|
| 913 |
+
if not isinstance(ref, str) or ref not in remap:
|
| 914 |
+
continue
|
| 915 |
+
target = remap[ref]
|
| 916 |
+
source_cell = _remap_cell(current, row["sheet"], ref)
|
| 917 |
+
target_cell = _remap_cell(current, row["sheet"], target)
|
| 918 |
+
source_empty = (source_cell.get("kind", "empty") in {"empty", "literal"}
|
| 919 |
+
and source_cell.get("value") in (None, "")
|
| 920 |
+
and not any(source_cell.get(key) for key in ("formula", "binding", "text")))
|
| 921 |
+
current_text = target_cell.get("state") == "fresh" or (
|
| 922 |
+
target_cell.get("state") == "stale"
|
| 923 |
+
and current_after_recalculation(row["sheet"], target, target_cell))
|
| 924 |
+
if not source_empty or target_cell.get("kind") != "formula" \
|
| 925 |
+
or not str(target_cell.get("formula") or "").startswith("=") \
|
| 926 |
+
or not current_text \
|
| 927 |
+
or target_cell.get("error") or target_cell.get("refusal") \
|
| 928 |
+
or not isinstance(target_cell.get("value"), str) or not target_cell["value"].strip():
|
| 929 |
+
raise RepairRefusal(
|
| 930 |
+
"repair_ref_remap_invalid",
|
| 931 |
+
"Remap " + ref + " to " + str(target) + " on " + row["sheet"]
|
| 932 |
+
+ " requires an empty source and a current formula with a non-empty string value.")
|
| 933 |
+
used.add(ref)
|
| 934 |
+
slot["ref"] = target
|
| 935 |
+
unused = set(remap) - used
|
| 936 |
+
if unused:
|
| 937 |
+
raise RepairRefusal("repair_ref_remap_unused",
|
| 938 |
+
"No migrated binding uses remap reference " + ", ".join(sorted(map(str, unused))) + ".")
|
| 939 |
+
|
| 940 |
+
projected = _project(current, edits)
|
| 941 |
+
migrated = []
|
| 942 |
+
for row in edits:
|
| 943 |
+
value = _resolved(row["after"]["binding"], projected, resolve_binding, tab_name=row["sheet"])
|
| 944 |
+
before_value = row["before"].get("value")
|
| 945 |
+
numeric = type(value) in (int, float) and type(before_value) in (int, float)
|
| 946 |
+
equal = (math.isfinite(value) and math.isfinite(before_value)
|
| 947 |
+
and math.isclose(value, before_value, rel_tol=1e-9, abs_tol=1e-6)) if numeric else (
|
| 948 |
+
type(value) is type(before_value) and value == before_value)
|
| 949 |
+
if not equal:
|
| 950 |
+
raise RepairRefusal("repair_migration_value_changed",
|
| 951 |
+
row["name"] + " would change from " + repr(before_value)
|
| 952 |
+
+ " to " + repr(value) + "; no migration was applied.")
|
| 953 |
+
desired = _preserve_visual(row["before"], {
|
| 954 |
+
**row["after"], "kind": "bound", "state": "fresh", "value": value,
|
| 955 |
+
"display": str(value), "refreshed_at": request["refreshed_at"]})
|
| 956 |
+
cleaned = sheets.clean_cell(desired, row["address"])
|
| 957 |
+
if cleaned != desired:
|
| 958 |
+
raise RepairRefusal("repair_migration_metadata_changed",
|
| 959 |
+
row["name"] + " cannot be migrated without changing its saved metadata.")
|
| 960 |
+
row["after"] = cleaned
|
| 961 |
+
_tab(projected, row["sheet"])["cells"][row["address"]] = copy.deepcopy(cleaned)
|
| 962 |
+
migrated.append({"sheet": row["sheet"], "address": row["address"],
|
| 963 |
+
"before_value": before_value, "after_value": value})
|
| 964 |
+
return edits, migrated
|
| 965 |
+
|
| 966 |
+
|
| 967 |
+
def _formula_repair_census(book):
|
| 968 |
+
errors = formula_error_census(book)
|
| 969 |
+
return {name: {"count": sum(row["sheet"] == name for row in errors),
|
| 970 |
+
"errors": [row for row in errors if row["sheet"] == name]}
|
| 971 |
+
for name in (SUMMARY_TAB, "Summary (SR)")}
|
| 972 |
+
|
| 973 |
+
|
| 974 |
+
def _check_formula_repair_errors(book):
|
| 975 |
+
"""All codes count, including blank, missing tabs and circular references."""
|
| 976 |
+
for name, address, _before in _FORMULA_REPAIRS:
|
| 977 |
+
cell = (_tab(book, name).get("cells") or {}).get(address) or {}
|
| 978 |
+
if cell.get("kind") != "formula" or cell.get("state") != "fresh" \
|
| 979 |
+
or cell.get("error") or cell.get("refusal") or cell.get("value") is None:
|
| 980 |
+
raise RepairRefusal("repair_error_cell_unresolved",
|
| 981 |
+
name + "!" + address + " did not recalculate to a non-error value.")
|
| 982 |
+
census = _formula_repair_census(book)
|
| 983 |
+
remaining = [row for tab in census.values() for row in tab["errors"]]
|
| 984 |
+
if remaining:
|
| 985 |
+
raise RepairRefusal("repair_formula_errors_remain",
|
| 986 |
+
"Formula errors remain: " + json.dumps(remaining, sort_keys=True) + ".")
|
| 987 |
+
return census
|
| 988 |
+
|
| 989 |
+
|
| 990 |
+
def _build_formula_repair_plan(current, request, formula_check):
|
| 991 |
+
"""Restore the two stored marker masters and the exact, enumerated error formulas."""
|
| 992 |
+
from openpyxl.formula.translate import Translator
|
| 993 |
+
|
| 994 |
+
if not isinstance(request["formula_repair"], dict) or any(
|
| 995 |
+
key in request for key in ("related_migration", "summary_block", "source_cells", "binding_cells")):
|
| 996 |
+
raise RepairRefusal("repair_formula_plan_conflict", "Run the formula repair as its own repair.")
|
| 997 |
+
if not callable(formula_check):
|
| 998 |
+
raise RepairRefusal("repair_formula_check_required", "The formula repair requires a formula engine.")
|
| 999 |
+
edits, evidence = [], []
|
| 1000 |
+
|
| 1001 |
+
def append(name, address, formula):
|
| 1002 |
+
before = copy.deepcopy((_tab(current, name).get("cells") or {}).get(address))
|
| 1003 |
+
desired = None
|
| 1004 |
+
if formula is not None:
|
| 1005 |
+
desired = {**copy.deepcopy(before or {}), "kind": "formula",
|
| 1006 |
+
"formula": formula, "state": "stale", "value": None}
|
| 1007 |
+
for key in ("error", "refusal", "display"):
|
| 1008 |
+
desired.pop(key, None)
|
| 1009 |
+
after = _preserve_visual(before, desired)
|
| 1010 |
+
if after is not None:
|
| 1011 |
+
after = sheets.clean_cell(after, address)
|
| 1012 |
+
# Even unchanged formula text must be planned: Q64's old error is repaired by R64.
|
| 1013 |
+
if before != after:
|
| 1014 |
+
edits.append({"kind": "derived", "name": name + "!" + address,
|
| 1015 |
+
"sheet": name, "address": address, "before": before, "after": after})
|
| 1016 |
+
|
| 1017 |
+
def empty(cell):
|
| 1018 |
+
return (cell.get("kind", "empty") in {"empty", "literal"}
|
| 1019 |
+
and cell.get("value") in (None, "")
|
| 1020 |
+
and not any(cell.get(key) for key in ("formula", "binding", "text")))
|
| 1021 |
+
|
| 1022 |
+
cached_markers = []
|
| 1023 |
+
for name, marker_row, last_copy in ((SUMMARY_TAB, 4, 6), ("Summary (SR)", 5, 7)):
|
| 1024 |
+
cells = _tab(current, name).get("cells") or {}
|
| 1025 |
+
source, origin = "E" + str(marker_row), "F" + str(marker_row)
|
| 1026 |
+
stored = cells.get(source) or {}
|
| 1027 |
+
master = stored.get("formula") if stored.get("kind") == "formula" else None
|
| 1028 |
+
retry = empty(stored)
|
| 1029 |
+
if retry:
|
| 1030 |
+
# A completed repair retains the verbatim master in F, after clearing E.
|
| 1031 |
+
held = cells.get(origin) or {}
|
| 1032 |
+
master = held.get("formula") if held.get("kind") == "formula" else None
|
| 1033 |
+
if not isinstance(master, str) or not master.startswith('=IF(' + source + '="",'):
|
| 1034 |
+
raise RepairRefusal("repair_marker_master_missing",
|
| 1035 |
+
name + "!" + source + " must hold its self-referencing marker master.")
|
| 1036 |
+
translated = {origin: master}
|
| 1037 |
+
translated.update({chr(col) + str(marker_row): Translator(master, origin=origin).translate_formula(
|
| 1038 |
+
chr(col) + str(marker_row)) for col in range(ord("G"), ord("Q") + 1)})
|
| 1039 |
+
if retry and (not all(empty(cells.get("E" + str(row)) or {})
|
| 1040 |
+
for row in range(3, last_copy + 1)) or not all(
|
| 1041 |
+
(cells.get(address) or {}).get("kind") == "formula"
|
| 1042 |
+
and cells[address].get("formula") == formula for address, formula in translated.items())):
|
| 1043 |
+
raise RepairRefusal("repair_marker_master_missing",
|
| 1044 |
+
name + "!" + source + " is missing and the restored marker row is incomplete.")
|
| 1045 |
+
for address, formula in translated.items():
|
| 1046 |
+
before = cells.get(address) or {}
|
| 1047 |
+
exact_formula = before.get("kind") == "formula" and before.get("formula") == formula
|
| 1048 |
+
# Excel's cached month-end markers (H4 2026-06-30, M4 2026-07-31, Q4 2026-08-31 on
|
| 1049 |
+
# staging) are typed ISO dates; one is accepted only if the restored formula
|
| 1050 |
+
# recalculates to exactly it (checked after projection, below), never by address.
|
| 1051 |
+
cached_date = (before.get("kind") == "literal" and isinstance(before.get("value"), str)
|
| 1052 |
+
and re.fullmatch(r"\d{4}-\d{2}-\d{2}", before["value"]) is not None)
|
| 1053 |
+
if not (empty(before) or cached_date or exact_formula):
|
| 1054 |
+
raise RepairRefusal("repair_marker_row_occupied",
|
| 1055 |
+
name + "!" + address + " contains an owner value outside the marker repair.")
|
| 1056 |
+
if cached_date:
|
| 1057 |
+
cached_markers.append((name, address, before["value"]))
|
| 1058 |
+
append(name, address, formula)
|
| 1059 |
+
for row in range(3, last_copy + 1):
|
| 1060 |
+
append(name, "E" + str(row), None)
|
| 1061 |
+
|
| 1062 |
+
for (name, address, expected), (formula, resolution, reason) in _FORMULA_REPAIRS.items():
|
| 1063 |
+
before = (_tab(current, name).get("cells") or {}).get(address) or {}
|
| 1064 |
+
if before.get("kind") != "formula" or before.get("formula") not in (expected, formula):
|
| 1065 |
+
raise RepairRefusal("repair_error_cell_changed",
|
| 1066 |
+
name + "!" + address + " no longer holds the expected error formula or its repair.")
|
| 1067 |
+
evidence.append({"tab": name, "address": address, "before": before["formula"],
|
| 1068 |
+
"after": formula, "resolution": resolution, "reason": reason})
|
| 1069 |
+
append(name, address, formula)
|
| 1070 |
+
|
| 1071 |
+
def checked(book):
|
| 1072 |
+
result = formula_check(book)
|
| 1073 |
+
if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
|
| 1074 |
+
raise RepairRefusal("repair_formula_check_required",
|
| 1075 |
+
"The formula repair requires a recalculated workbook from its engine.")
|
| 1076 |
+
# Name unresolved targets before the generic unplanned-change guard can mask them.
|
| 1077 |
+
_check_formula_repair_errors(result["book"])
|
| 1078 |
+
return result
|
| 1079 |
+
|
| 1080 |
+
projected, _errors = _checked_projection(_project(current, edits), edits, checked)
|
| 1081 |
+
for name, address, cached in cached_markers:
|
| 1082 |
+
got = ((_tab(projected, name).get("cells") or {}).get(address) or {}).get("value")
|
| 1083 |
+
if got != cached:
|
| 1084 |
+
raise RepairRefusal("repair_marker_row_occupied",
|
| 1085 |
+
name + "!" + address + " holds " + cached + " but its restored marker "
|
| 1086 |
+
"recalculates to " + json.dumps(got) + ", so it is an owner value.")
|
| 1087 |
+
cycles = formula_cycles(projected, formula_check)
|
| 1088 |
+
if cycles["count"]:
|
| 1089 |
+
raise RepairRefusal("repair_formula_errors_remain",
|
| 1090 |
+
"Formula cycles remain: " + json.dumps(cycles["cycles"]) + ".")
|
| 1091 |
+
return [row for row in edits if row["before"] != row["after"]], evidence
|
| 1092 |
+
|
| 1093 |
+
|
| 1094 |
+
def _daily_sales_formula_checker(formula_check):
|
| 1095 |
+
"""Keep number-format inference from repainting this content-only repair."""
|
| 1096 |
+
if not callable(formula_check):
|
| 1097 |
+
raise RepairRefusal("repair_formula_check_required", "Daily Sales requires a formula engine.")
|
| 1098 |
+
|
| 1099 |
+
def checked(book):
|
| 1100 |
+
before = _cell_map(book)
|
| 1101 |
+
result = formula_check(book)
|
| 1102 |
+
if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
|
| 1103 |
+
raise RepairRefusal("repair_formula_check_required",
|
| 1104 |
+
"Daily Sales requires a recalculated workbook from its engine.")
|
| 1105 |
+
for tab in result["book"].get("sheets") or ():
|
| 1106 |
+
for address, cell in (tab.get("cells") or {}).items():
|
| 1107 |
+
prior = before.get((str(tab.get("name") or "").casefold(), address)) or {}
|
| 1108 |
+
inferred = (cell.get("formulaMeta") or {}).get("inferredFormat")
|
| 1109 |
+
if cell.get("kind") != "formula" or not isinstance(inferred, dict):
|
| 1110 |
+
continue
|
| 1111 |
+
style, old_style = cell.get("style") or {}, prior.get("style") or {}
|
| 1112 |
+
if inferred.get("format") == style.get("format") and {
|
| 1113 |
+
k: v for k, v in style.items() if k != "format"} == {
|
| 1114 |
+
k: v for k, v in old_style.items() if k != "format"}:
|
| 1115 |
+
if "style" in prior:
|
| 1116 |
+
cell["style"] = copy.deepcopy(prior["style"])
|
| 1117 |
+
else:
|
| 1118 |
+
cell.pop("style", None)
|
| 1119 |
+
meta = cell.get("formulaMeta") or {}
|
| 1120 |
+
old_inferred = (prior.get("formulaMeta") or {}).get("inferredFormat")
|
| 1121 |
+
if old_inferred:
|
| 1122 |
+
meta["inferredFormat"] = copy.deepcopy(old_inferred)
|
| 1123 |
+
else:
|
| 1124 |
+
meta.pop("inferredFormat", None)
|
| 1125 |
+
if not meta:
|
| 1126 |
+
cell.pop("formulaMeta", None)
|
| 1127 |
+
return result
|
| 1128 |
+
return checked
|
| 1129 |
+
|
| 1130 |
+
|
| 1131 |
+
def _daily_sales_invariants(book, formula_check):
|
| 1132 |
+
census = _formula_repair_census(book)
|
| 1133 |
+
if any(row["count"] for row in census.values()):
|
| 1134 |
+
raise RepairRefusal("repair_formula_errors_remain",
|
| 1135 |
+
"Daily Sales formula errors remain: " + json.dumps(census, sort_keys=True))
|
| 1136 |
+
for name in (SUMMARY_TAB, "Summary (SR)"):
|
| 1137 |
+
for address, cell in (_tab(book, name).get("cells") or {}).items():
|
| 1138 |
+
if cell.get("kind") == "formula" and cell.get("state") != "fresh":
|
| 1139 |
+
raise RepairRefusal("repair_formula_check_required", name + "!" + address
|
| 1140 |
+
+ " has not been recalculated.")
|
| 1141 |
+
cycles = formula_cycles(book, formula_check)
|
| 1142 |
+
if cycles["count"]:
|
| 1143 |
+
raise RepairRefusal("repair_formula_errors_remain", "Daily Sales formula cycles remain.")
|
| 1144 |
+
return census
|
| 1145 |
+
|
| 1146 |
+
|
| 1147 |
+
def _build_daily_sales_bindings_plan(current, request, resolve_binding, formula_check):
|
| 1148 |
+
"""R18-R21: locate the authored report by labels and resolve every proposed binding."""
|
| 1149 |
+
from calendar import month_name, monthrange
|
| 1150 |
+
from openpyxl.formula.translate import Translator
|
| 1151 |
+
from openpyxl.utils.cell import column_index_from_string, get_column_letter
|
| 1152 |
+
|
| 1153 |
+
if not isinstance(request["daily_sales_bindings"], dict) or any(key in request for key in (
|
| 1154 |
+
"formula_repair", "related_migration", "summary_block", "source_cells", "binding_cells")):
|
| 1155 |
+
raise RepairRefusal("repair_daily_sales_plan_conflict", "Run Daily Sales bindings as their own repair.")
|
| 1156 |
+
if not callable(formula_check) or not callable(resolve_binding):
|
| 1157 |
+
raise RepairRefusal("repair_binding_resolver_required",
|
| 1158 |
+
"Daily Sales bindings require the resolver and formula engine.")
|
| 1159 |
+
if not str(request.get("refreshed_at") or "").strip():
|
| 1160 |
+
raise RepairRefusal("repair_timestamp_required", "Daily Sales bindings require a refresh timestamp.")
|
| 1161 |
+
tab = _tab(current, SUMMARY_TAB)
|
| 1162 |
+
_tab(current, "Summary (SR)")
|
| 1163 |
+
cells = tab.get("cells") or {}
|
| 1164 |
+
|
| 1165 |
+
def label(text, column="D", lo=0, hi=sheets.DOCUMENT_MAX_ROWS + 1):
|
| 1166 |
+
found = [address for address, cell in cells.items()
|
| 1167 |
+
if re.fullmatch(column + r"[1-9][0-9]*", address)
|
| 1168 |
+
and lo < int(address[len(column):]) < hi
|
| 1169 |
+
and str(cell.get("value") or "").strip() == text]
|
| 1170 |
+
if len(found) != 1:
|
| 1171 |
+
raise RepairRefusal("repair_label_missing", SUMMARY_TAB + " label '" + text
|
| 1172 |
+
+ "' must occur exactly once in its report block.")
|
| 1173 |
+
return found[0]
|
| 1174 |
+
|
| 1175 |
+
def row(text, **kwargs):
|
| 1176 |
+
return int(re.search(r"\d+$", label(text, **kwargs))[0])
|
| 1177 |
+
|
| 1178 |
+
ending = row("End. Date")
|
| 1179 |
+
monthly = row("Monthly metrics")
|
| 1180 |
+
allocated = row("Allocated client summary")
|
| 1181 |
+
frequency = row("Average order frequency", lo=allocated)
|
| 1182 |
+
orders = row("Number of Orders", lo=frequency)
|
| 1183 |
+
aov = row("Avg. Order Value", lo=orders)
|
| 1184 |
+
delivery = row("Average Delivery Time", lo=aov)
|
| 1185 |
+
sales = row("Total Sales", hi=monthly)
|
| 1186 |
+
blocks = {}
|
| 1187 |
+
for name, start, end in (("sales", sales, monthly), ("frequency", frequency, orders),
|
| 1188 |
+
("orders", orders, aov), ("aov", aov, delivery),
|
| 1189 |
+
("delivery", delivery, sheets.DOCUMENT_MAX_ROWS + 1)):
|
| 1190 |
+
blocks[name] = [row(text, lo=start, hi=end) for text in YTD_NAMES]
|
| 1191 |
+
allocated_row = row("No. of allocated client", lo=allocated, hi=frequency)
|
| 1192 |
+
purchasing_row = row("No. of purchasing client", lo=allocated, hi=frequency)
|
| 1193 |
+
previous_row = row("Previous value", lo=allocated, hi=frequency)
|
| 1194 |
+
percent_row = row("% of purchasing client", lo=allocated, hi=frequency)
|
| 1195 |
+
helpers = [address for address, cell in cells.items() if cell.get("kind") == "formula"
|
| 1196 |
+
and str(cell.get("formula") or "").startswith('=IF($D$9="RI","Royal 1 (USD)"')]
|
| 1197 |
+
if len(helpers) != 1:
|
| 1198 |
+
raise RepairRefusal("repair_label_missing", "Daily Sales BU helper formula must occur exactly once.")
|
| 1199 |
+
helper = helpers[0]
|
| 1200 |
+
columns = []
|
| 1201 |
+
for address, cell in cells.items():
|
| 1202 |
+
match = re.fullmatch(r"([A-Z]+)([1-9][0-9]*)", address)
|
| 1203 |
+
if match and int(match[2]) == ending and column_index_from_string(match[1]) >= 6:
|
| 1204 |
+
try:
|
| 1205 |
+
_summary_date(cell.get("value"))
|
| 1206 |
+
except RepairRefusal:
|
| 1207 |
+
continue
|
| 1208 |
+
columns.append(match[1])
|
| 1209 |
+
columns.sort(key=column_index_from_string)
|
| 1210 |
+
if tuple(columns) != SUMMARY_COLUMNS:
|
| 1211 |
+
raise RepairRefusal("repair_label_missing", "End. Date requires one date for each F through R column.")
|
| 1212 |
+
edits, observed = [], []
|
| 1213 |
+
|
| 1214 |
+
def cache_or_empty(cell):
|
| 1215 |
+
return (cell.get("kind", "empty") in {"literal", "empty"}
|
| 1216 |
+
and (cell.get("value") in (None, "") or type(cell.get("value")) in (int, float))
|
| 1217 |
+
and not any(cell.get(key) for key in ("formula", "binding", "text")))
|
| 1218 |
+
|
| 1219 |
+
def append(address, desired, kind):
|
| 1220 |
+
before = copy.deepcopy(cells.get(address))
|
| 1221 |
+
after = sheets.clean_cell(_preserve_visual(before, desired), address)
|
| 1222 |
+
edits.append({"kind": kind, "name": SUMMARY_TAB + "!" + address,
|
| 1223 |
+
"sheet": SUMMARY_TAB, "address": address, "before": before, "after": after})
|
| 1224 |
+
|
| 1225 |
+
def formula_fill(origin, targets):
|
| 1226 |
+
master = (cells.get(origin) or {}).get("formula")
|
| 1227 |
+
if not isinstance(master, str) or not master.startswith("="):
|
| 1228 |
+
raise RepairRefusal("repair_formula_master_missing", SUMMARY_TAB + "!" + origin
|
| 1229 |
+
+ " must hold its authored formula master.")
|
| 1230 |
+
for address in targets:
|
| 1231 |
+
text = Translator(master, origin=origin).translate_formula(address)
|
| 1232 |
+
before = cells.get(address) or {}
|
| 1233 |
+
if not (cache_or_empty(before) or before.get("kind") == "formula"
|
| 1234 |
+
and before.get("formula") == text):
|
| 1235 |
+
raise RepairRefusal("repair_owner_value", SUMMARY_TAB + "!" + address
|
| 1236 |
+
+ " contains an owner value outside this formula fill.")
|
| 1237 |
+
append(address, {"kind": "formula", "state": "stale", "formula": text}, "derived")
|
| 1238 |
+
|
| 1239 |
+
def bind(address, database, fn, nodes, field=None):
|
| 1240 |
+
aggregate = {"fn": fn, **({"field": field} if field else {})}
|
| 1241 |
+
binding = sheets.clean_cell({"kind": "bound", "binding": {
|
| 1242 |
+
"database": database, "aggregate": aggregate,
|
| 1243 |
+
"scope": {"filter": {"conj": "and", "nodes": nodes}}}}, address)["binding"]
|
| 1244 |
+
before = cells.get(address) or {}
|
| 1245 |
+
if not (cache_or_empty(before) or before.get("kind") == "bound" and before.get("binding") == binding):
|
| 1246 |
+
raise RepairRefusal("repair_owner_value", SUMMARY_TAB + "!" + address
|
| 1247 |
+
+ " contains an owner value outside this binding plan.")
|
| 1248 |
+
try:
|
| 1249 |
+
result = resolve_binding(copy.deepcopy(binding), copy.deepcopy(current), copy.deepcopy(tab))
|
| 1250 |
+
except Exception as exc:
|
| 1251 |
+
detail = getattr(exc, "detail", None)
|
| 1252 |
+
info = (detail.get("error") or {}) if isinstance(detail, dict) else {}
|
| 1253 |
+
code = getattr(exc, "code", None) or info.get("code") or "repair_binding_unreadable"
|
| 1254 |
+
raise RepairRefusal(code, SUMMARY_TAB + "!" + address + " binding resolution refused.") from exc
|
| 1255 |
+
if not isinstance(result, dict) or "value" not in result or result.get("refusal") \
|
| 1256 |
+
or result.get("error") or result.get("state") not in (None, "fresh"):
|
| 1257 |
+
raise RepairRefusal("repair_binding_unreadable", SUMMARY_TAB + "!" + address
|
| 1258 |
+
+ " did not resolve a fresh bound value.")
|
| 1259 |
+
if result.get("limits"):
|
| 1260 |
+
raise RepairRefusal("repair_binding_limits", SUMMARY_TAB + "!" + address
|
| 1261 |
+
+ " has unapplied or limited binding conditions.")
|
| 1262 |
+
value = result["value"]
|
| 1263 |
+
if not (value is None and fn == "avg") and (
|
| 1264 |
+
type(value) not in (int, float) or not math.isfinite(value)):
|
| 1265 |
+
raise RepairRefusal("repair_binding_unreadable", SUMMARY_TAB + "!" + address
|
| 1266 |
+
+ " must resolve a finite number.")
|
| 1267 |
+
prior = before.get("value")
|
| 1268 |
+
if before.get("kind") == "bound" and before.get("binding") == binding \
|
| 1269 |
+
and type(prior) in (int, float) and type(value) in (int, float) \
|
| 1270 |
+
and math.isclose(prior, value, rel_tol=1e-9, abs_tol=1e-9):
|
| 1271 |
+
# A float SUM/AVG can differ in its last bits between runs (parallel summation
|
| 1272 |
+
# order: 168809.64999999997 then 168809.65). It is the same number, so the stored
|
| 1273 |
+
# one is kept and a rerun changes nothing.
|
| 1274 |
+
value = prior
|
| 1275 |
+
desired = {"kind": "bound", "state": "fresh", "binding": binding, "value": value,
|
| 1276 |
+
"display": "" if value is None else str(value), "refreshed_at": request["refreshed_at"]}
|
| 1277 |
+
if all(before.get(key) == desired[key] for key in ("kind", "state", "binding", "value", "display")) \
|
| 1278 |
+
and not any(before.get(key) for key in ("limits", "refusal", "error")):
|
| 1279 |
+
desired["refreshed_at"] = before.get("refreshed_at", request["refreshed_at"])
|
| 1280 |
+
append(address, desired, "binding")
|
| 1281 |
+
observed.append({"address": address, "binding": binding, "value": value})
|
| 1282 |
+
|
| 1283 |
+
def bu(via=""):
|
| 1284 |
+
return {"colId": ("related:" + via + ":" if via else "") + "pricelist",
|
| 1285 |
+
"op": "eq", "valueRef": {"ref": helper}}
|
| 1286 |
+
|
| 1287 |
+
def window(field, col, kind):
|
| 1288 |
+
return {"colId": field, "op": "within", "valueRef": {"ref": col + str(ending)},
|
| 1289 |
+
"dateWindow": {"kind": kind}}
|
| 1290 |
+
|
| 1291 |
+
for half, label_column, months in (("H1", "D", range(1, 7)), ("H2", "I", range(7, 13))):
|
| 1292 |
+
header = label(half, column=label_column, lo=monthly, hi=allocated)
|
| 1293 |
+
header_row = int(re.search(r"\d+$", header)[0])
|
| 1294 |
+
base_col = column_index_from_string(label_column)
|
| 1295 |
+
month_rows = [row(month_name[month], column=label_column, lo=header_row, hi=allocated)
|
| 1296 |
+
for month in months]
|
| 1297 |
+
for offset in (1, 2):
|
| 1298 |
+
value_col = get_column_letter(base_col + offset)
|
| 1299 |
+
year = (cells.get(value_col + str(header_row)) or {}).get("value")
|
| 1300 |
+
numeric_year = (type(year) in (int, float) and math.isfinite(year) and int(year) == year)
|
| 1301 |
+
if not (numeric_year or isinstance(year, str) and year.isdigit()) \
|
| 1302 |
+
or not 1900 <= int(year) <= 9999:
|
| 1303 |
+
raise RepairRefusal("repair_label_missing", half + " requires a calendar year at "
|
| 1304 |
+
+ value_col + str(header_row) + ".")
|
| 1305 |
+
year = int(year)
|
| 1306 |
+
for month, month_row in zip(months, month_rows):
|
| 1307 |
+
bind(value_col + str(month_row), "ut_odoo_invoices", "sum", [
|
| 1308 |
+
{"colId": "invoice_date", "op": "between", "value": f"{year:04d}-{month:02d}-01",
|
| 1309 |
+
"value2": f"{year:04d}-{month:02d}-{monthrange(year, month)[1]:02d}"},
|
| 1310 |
+
{"colId": "move_type", "op": "eq", "value": "out_invoice"}, bu("customer_link")],
|
| 1311 |
+
"amount_untaxed")
|
| 1312 |
+
growth = get_column_letter(base_col + 3)
|
| 1313 |
+
formula_fill(growth + str(month_rows[0]), [growth + str(r) for r in month_rows])
|
| 1314 |
+
|
| 1315 |
+
for col in columns:
|
| 1316 |
+
bind(col + str(allocated_row), "customer_data", "count", [bu(), {"conj": "and", "children": [
|
| 1317 |
+
{"colId": "related:invoices:agent_id", "op": "eq", "rhs": {"colId": "agent_id"}},
|
| 1318 |
+
{"colId": "related:invoices:invoice_date", "op": "lte", "valueRef": {"ref": col + str(ending)}},
|
| 1319 |
+
{"colId": "related:invoices:move_type", "op": "eq", "value": "out_invoice"}]}])
|
| 1320 |
+
for i, kind in enumerate(YTD_NAMES.values()):
|
| 1321 |
+
bind(col + str((purchasing_row, previous_row)[i]), "customer_data", "count", [bu(),
|
| 1322 |
+
{"conj": "and", "children": [window("related:invoices:invoice_date", col, kind),
|
| 1323 |
+
{"colId": "related:invoices:move_type", "op": "eq", "value": "out_invoice"}]}])
|
| 1324 |
+
bind(col + str(blocks["orders"][i]), "ut_odoo_orders", "count", [
|
| 1325 |
+
window("order_date", col, kind), {"colId": "state", "op": "isAnyOf", "values": ["sale", "done"]},
|
| 1326 |
+
bu("bridge_customer_orders")])
|
| 1327 |
+
bind(col + str(blocks["delivery"][i]), "ut_odoo_shipments", "avg", [
|
| 1328 |
+
{"colId": "state", "op": "eq", "value": "done"}, window("date_done", col, kind),
|
| 1329 |
+
bu("customer_link")], "delivery_days")
|
| 1330 |
+
for target_row in [percent_row, *blocks["frequency"], *blocks["aov"]]:
|
| 1331 |
+
formula_fill("F" + str(target_row), [col + str(target_row) for col in columns])
|
| 1332 |
+
projected, _errors = _checked_projection(_project(current, edits), edits, formula_check)
|
| 1333 |
+
_daily_sales_invariants(projected, formula_check)
|
| 1334 |
+
return [edit for edit in edits if edit["before"] != edit["after"]], {
|
| 1335 |
+
"helper": helper, "ending_row": ending, "observed": observed}
|
| 1336 |
+
|
| 1337 |
+
|
| 1338 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1339 |
+
# ββ S67-T21 / R34 β EVERY METRIC COMPUTED; ONLY THE OWNER'S OWN INPUTS STAY TYPED
|
| 1340 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1341 |
+
#
|
| 1342 |
+
# The owner's weekly scorecard read pasted numbers: the weekly block and the middle of a dozen
|
| 1343 |
+
# rows were values copied once and never recomputed. This mode applies an explicit, fully named
|
| 1344 |
+
# cell plan (each cell a formula, a Cell Database binding or a blank), resolved and recalculated
|
| 1345 |
+
# BEFORE a byte moves, and REFUSES unless the projected workbook keeps no hardcoded number but the
|
| 1346 |
+
# owner's inputs. The plan itself is built by the operator from the Excel reference; this module is
|
| 1347 |
+
# what makes it safe to apply: owner inputs and labels are untouchable, every binding resolves to
|
| 1348 |
+
# a fresh value with no limits, the formula census stays empty, and the backup lands first.
|
| 1349 |
+
|
| 1350 |
+
SR_TAB = "Summary (SR)"
|
| 1351 |
+
|
| 1352 |
+
#: R34 (owner, 2026-09-28), verbatim cells. Summary (All)'s six are the owner's inputs; Summary
|
| 1353 |
+
#: (SR)'s are the only numbers the Excel file itself types on that tab (its targets and goals).
|
| 1354 |
+
OWNER_INPUTS = {
|
| 1355 |
+
SUMMARY_TAB: frozenset({"Q25", "T20", "T37", "T38", "T51", "T55"}),
|
| 1356 |
+
SR_TAB: frozenset({"T24", "T25", "T34", "T67", "T71", "T105", "T112"}),
|
| 1357 |
+
}
|
| 1358 |
+
_PLANNED_KINDS = ("formula", "bound", "empty")
|
| 1359 |
+
#: The tab-level visual keys a plan may DECLARE (O42: match the Excel file's freeze). Anything else
|
| 1360 |
+
#: visual still refuses, exactly as every content-only repair above does.
|
| 1361 |
+
_PLANNED_TAB_VISUAL_KEYS = ("freeze",)
|
| 1362 |
+
|
| 1363 |
+
|
| 1364 |
+
def hardcoded_numbers(book):
|
| 1365 |
+
"""`{tab name: {address: number}}` for every literal numeric cell, PURE. R34's census."""
|
| 1366 |
+
out = {}
|
| 1367 |
+
for tab in book.get("sheets") or ():
|
| 1368 |
+
found = {}
|
| 1369 |
+
for address, cell in (tab.get("cells") or {}).items():
|
| 1370 |
+
value = (cell or {}).get("value")
|
| 1371 |
+
if (cell or {}).get("kind") == "literal" and type(value) in (int, float):
|
| 1372 |
+
found[address] = value
|
| 1373 |
+
out[str(tab.get("name") or "")] = found
|
| 1374 |
+
return out
|
| 1375 |
+
|
| 1376 |
+
|
| 1377 |
+
def _bound_value(value, fn):
|
| 1378 |
+
"""A resolved binding's value as the document stores it, or a refusal.
|
| 1379 |
+
|
| 1380 |
+
β A `min`/`max` over a date column answers a DATE ("Latest data" is `max(invoice_date)`), which
|
| 1381 |
+
is stored as its ISO text, the shape every formula date in this workbook already has.
|
| 1382 |
+
"""
|
| 1383 |
+
import datetime as _dtm
|
| 1384 |
+
if isinstance(value, (_dtm.date, _dtm.datetime)):
|
| 1385 |
+
if fn not in ("min", "max"):
|
| 1386 |
+
raise RepairRefusal("repair_binding_unreadable", "Only min or max may answer a date.")
|
| 1387 |
+
return value.isoformat()[:10]
|
| 1388 |
+
if isinstance(value, str) and fn in ("min", "max") and re.fullmatch(r"\d{4}-\d{2}-\d{2}.*", value):
|
| 1389 |
+
return value[:10]
|
| 1390 |
+
if value is None and fn in ("avg", "min", "max"):
|
| 1391 |
+
return None
|
| 1392 |
+
if type(value) not in (int, float) or not math.isfinite(value):
|
| 1393 |
+
raise RepairRefusal("repair_binding_unreadable", "A binding must resolve a finite number.")
|
| 1394 |
+
return value
|
| 1395 |
+
|
| 1396 |
+
|
| 1397 |
+
def _visual_without(book, planned_notes, keys=("note",)):
|
| 1398 |
+
"""`visual_metadata` with each planned cell's declared visual `keys` removed (a note for
|
| 1399 |
+
T21, a sparkline for T23), so a declared change is the only visual change a plan may make."""
|
| 1400 |
+
shaped = copy.deepcopy(book)
|
| 1401 |
+
for tab in shaped.get("sheets") or ():
|
| 1402 |
+
name = str(tab.get("name") or "").casefold()
|
| 1403 |
+
for address, cell in (tab.get("cells") or {}).items():
|
| 1404 |
+
if (name, address) in planned_notes and isinstance(cell, dict):
|
| 1405 |
+
for key in keys:
|
| 1406 |
+
cell.pop(key, None)
|
| 1407 |
+
return visual_metadata(shaped)
|
| 1408 |
+
|
| 1409 |
+
|
| 1410 |
+
def _build_computed_metrics_plan(current, request, resolve_binding, formula_check):
|
| 1411 |
+
"""R34: validate and resolve an explicit per-cell plan; refuse unless it leaves no hardcode."""
|
| 1412 |
+
spec = request["computed_metrics"]
|
| 1413 |
+
if not isinstance(spec, dict) or any(key in request for key in (
|
| 1414 |
+
"formula_repair", "related_migration", "summary_block", "source_cells",
|
| 1415 |
+
"binding_cells", "daily_sales_bindings")):
|
| 1416 |
+
raise RepairRefusal("repair_metrics_plan_conflict",
|
| 1417 |
+
"Run the computed metrics plan as its own repair.")
|
| 1418 |
+
if not callable(formula_check) or not callable(resolve_binding):
|
| 1419 |
+
raise RepairRefusal("repair_binding_resolver_required",
|
| 1420 |
+
"Computed metrics require the resolver and the formula engine.")
|
| 1421 |
+
refreshed_at = str(request.get("refreshed_at") or "").strip()
|
| 1422 |
+
if not refreshed_at:
|
| 1423 |
+
raise RepairRefusal("repair_timestamp_required", "Computed metrics require a refresh timestamp.")
|
| 1424 |
+
rows = spec.get("cells")
|
| 1425 |
+
if not isinstance(rows, list) or not rows:
|
| 1426 |
+
raise RepairRefusal("repair_metrics_plan_empty", "The computed metrics plan names no cell.")
|
| 1427 |
+
edits, observed, seen, planned_notes = [], [], set(), set()
|
| 1428 |
+
# β A binding may read a cell this same plan writes (a month's helper start/end dates, a
|
| 1429 |
+
# year header), so bindings resolve against the workbook WITH the planned formulas and blanks
|
| 1430 |
+
# applied and recalculated, never against the stale current one.
|
| 1431 |
+
staged = []
|
| 1432 |
+
for row in rows:
|
| 1433 |
+
want = row.get("cell") if isinstance(row, dict) else None
|
| 1434 |
+
if isinstance(want, dict) and want.get("kind") in ("formula", "empty"):
|
| 1435 |
+
tab = _tab(current, str(row.get("sheet") or ""))
|
| 1436 |
+
address = _addr(row.get("address"))
|
| 1437 |
+
before = copy.deepcopy((tab.get("cells") or {}).get(address))
|
| 1438 |
+
cell = ({"kind": "formula", "state": "stale", "formula": str(want.get("formula") or "")}
|
| 1439 |
+
if want["kind"] == "formula" else {"kind": "empty", "state": "typed"})
|
| 1440 |
+
staged.append({"sheet": str(tab.get("name")), "address": address, "before": before,
|
| 1441 |
+
"after": sheets.clean_cell(_preserve_visual(before, cell) or cell, address)})
|
| 1442 |
+
def recalculated(extra):
|
| 1443 |
+
projected = _project(current, staged + extra)
|
| 1444 |
+
checked = formula_check(copy.deepcopy(projected))
|
| 1445 |
+
return checked["book"] if isinstance(checked, dict) and isinstance(
|
| 1446 |
+
checked.get("book"), dict) else projected
|
| 1447 |
+
|
| 1448 |
+
def resolve_row(row, book):
|
| 1449 |
+
sheet = str(_tab(current, str(row.get("sheet") or "")).get("name"))
|
| 1450 |
+
address = _addr(row.get("address"))
|
| 1451 |
+
binding = sheets.clean_cell({"kind": "bound", "binding": row["cell"].get("binding")},
|
| 1452 |
+
address)["binding"]
|
| 1453 |
+
try:
|
| 1454 |
+
result = resolve_binding(copy.deepcopy(binding), copy.deepcopy(book),
|
| 1455 |
+
copy.deepcopy(_tab(book, sheet)))
|
| 1456 |
+
except Exception as exc: # noqa: BLE001
|
| 1457 |
+
detail = getattr(exc, "detail", None)
|
| 1458 |
+
info = (detail.get("error") or {}) if isinstance(detail, dict) else {}
|
| 1459 |
+
code = getattr(exc, "code", None) or info.get("code") or "repair_binding_unreadable"
|
| 1460 |
+
return None, RepairRefusal(code, sheet + "!" + address + " binding resolution refused.")
|
| 1461 |
+
return (binding, result), None
|
| 1462 |
+
|
| 1463 |
+
# β ROUNDS, because a binding may read a date another binding produces through formulas
|
| 1464 |
+
# (Summary (SR)'s week axis runs E12 `max(invoice_date)` -> E15 -> R8 -> F8 -> F31's window).
|
| 1465 |
+
# Each round resolves what it can read, the next projection carries those values, and only a
|
| 1466 |
+
# cell whose filter input was not yet readable waits; no progress in a round refuses by name.
|
| 1467 |
+
resolved, pending = {}, [row for row in rows if isinstance(row, dict)
|
| 1468 |
+
and isinstance(row.get("cell"), dict)
|
| 1469 |
+
and row["cell"].get("kind") == "bound"]
|
| 1470 |
+
for _round in range(6):
|
| 1471 |
+
if not pending:
|
| 1472 |
+
break
|
| 1473 |
+
extra = []
|
| 1474 |
+
for (sheet, address), (binding, result) in resolved.items():
|
| 1475 |
+
value = result.get("value") if isinstance(result, dict) else None
|
| 1476 |
+
try:
|
| 1477 |
+
value = _bound_value(value, (binding.get("aggregate") or {}).get("fn"))
|
| 1478 |
+
except RepairRefusal:
|
| 1479 |
+
value = None
|
| 1480 |
+
extra.append({"sheet": sheet, "address": address, "before": None,
|
| 1481 |
+
"after": sheets.clean_cell({"kind": "bound", "state": "fresh",
|
| 1482 |
+
"binding": binding, "value": value,
|
| 1483 |
+
"display": "" if value is None else str(value)},
|
| 1484 |
+
address)})
|
| 1485 |
+
book = recalculated(extra)
|
| 1486 |
+
waiting, first_error = [], None
|
| 1487 |
+
for row in pending:
|
| 1488 |
+
got, error = resolve_row(row, book)
|
| 1489 |
+
if got is not None:
|
| 1490 |
+
resolved[(str(_tab(current, str(row["sheet"])).get("name")),
|
| 1491 |
+
_addr(row.get("address")))] = got
|
| 1492 |
+
elif error.code == "filter_cell_unreadable":
|
| 1493 |
+
waiting.append(row)
|
| 1494 |
+
first_error = first_error or error
|
| 1495 |
+
else:
|
| 1496 |
+
raise error
|
| 1497 |
+
if len(waiting) == len(pending):
|
| 1498 |
+
raise first_error
|
| 1499 |
+
pending = waiting
|
| 1500 |
+
if pending:
|
| 1501 |
+
raise RepairRefusal("filter_cell_unreadable", "A binding's filter cell never became readable.")
|
| 1502 |
+
for row in rows:
|
| 1503 |
+
if not isinstance(row, dict):
|
| 1504 |
+
raise RepairRefusal("repair_metrics_cell_invalid", "Every planned cell must be an object.")
|
| 1505 |
+
sheet_name = str(row.get("sheet") or "").strip()
|
| 1506 |
+
address = _addr(row.get("address"))
|
| 1507 |
+
tab = _tab(current, sheet_name)
|
| 1508 |
+
sheet_name = str(tab.get("name") or sheet_name)
|
| 1509 |
+
slot = (sheet_name.casefold(), address)
|
| 1510 |
+
if slot in seen:
|
| 1511 |
+
raise RepairRefusal("repair_edit_duplicate", "A planned cell may be named only once.")
|
| 1512 |
+
seen.add(slot)
|
| 1513 |
+
if address in OWNER_INPUTS.get(sheet_name, ()):
|
| 1514 |
+
raise RepairRefusal("repair_owner_input", sheet_name + "!" + address
|
| 1515 |
+
+ " is one of the owner's typed inputs and stays typed.")
|
| 1516 |
+
cells = tab.get("cells") or {}
|
| 1517 |
+
before = copy.deepcopy(cells.get(address))
|
| 1518 |
+
prior_value = (before or {}).get("value")
|
| 1519 |
+
want = row.get("cell")
|
| 1520 |
+
if not isinstance(want, dict) or want.get("kind") not in (*_PLANNED_KINDS, "note_only"):
|
| 1521 |
+
raise RepairRefusal("repair_metrics_cell_invalid", sheet_name + "!" + address
|
| 1522 |
+
+ " must be planned as a formula, a binding, a blank or a note.")
|
| 1523 |
+
kind = want["kind"]
|
| 1524 |
+
# A typed ISO date is a value (Summary (SR)'s frozen E13/E15), never a label.
|
| 1525 |
+
if (before or {}).get("kind") == "literal" and isinstance(prior_value, str) \
|
| 1526 |
+
and prior_value.strip() and kind != "note_only" \
|
| 1527 |
+
and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", prior_value.strip()):
|
| 1528 |
+
raise RepairRefusal("repair_label_protected", sheet_name + "!" + address
|
| 1529 |
+
+ " holds a label, which a metrics plan never replaces.")
|
| 1530 |
+
if kind == "note_only":
|
| 1531 |
+
desired = copy.deepcopy(before) if before else {"kind": "empty", "state": "typed"}
|
| 1532 |
+
elif kind == "formula":
|
| 1533 |
+
text = str(want.get("formula") or "")
|
| 1534 |
+
if not text.startswith("="):
|
| 1535 |
+
raise RepairRefusal("repair_metrics_cell_invalid", sheet_name + "!" + address
|
| 1536 |
+
+ " needs a formula that starts with =.")
|
| 1537 |
+
desired = _preserve_visual(before, {"kind": "formula", "state": "stale", "formula": text})
|
| 1538 |
+
elif kind == "empty":
|
| 1539 |
+
desired = _preserve_visual(before, {"kind": "empty", "state": "typed"}) \
|
| 1540 |
+
or {"kind": "empty", "state": "typed"}
|
| 1541 |
+
else:
|
| 1542 |
+
binding, result = resolved[(sheet_name, address)]
|
| 1543 |
+
fn = (binding.get("aggregate") or {}).get("fn")
|
| 1544 |
+
if not isinstance(result, dict) or "value" not in result or result.get("refusal") \
|
| 1545 |
+
or result.get("error") or result.get("state") not in (None, "fresh"):
|
| 1546 |
+
raise RepairRefusal("repair_binding_unreadable", sheet_name + "!" + address
|
| 1547 |
+
+ " did not resolve a fresh bound value.")
|
| 1548 |
+
if result.get("limits"):
|
| 1549 |
+
raise RepairRefusal("repair_binding_limits", sheet_name + "!" + address
|
| 1550 |
+
+ " has unapplied or limited binding conditions.")
|
| 1551 |
+
value = _bound_value(result["value"], fn)
|
| 1552 |
+
if (before or {}).get("kind") == "bound" and before.get("binding") == binding \
|
| 1553 |
+
and type(prior_value) in (int, float) and type(value) in (int, float) \
|
| 1554 |
+
and math.isclose(prior_value, value, rel_tol=1e-9, abs_tol=1e-9):
|
| 1555 |
+
value = prior_value # the same number; a rerun changes nothing
|
| 1556 |
+
desired = _preserve_visual(before, {
|
| 1557 |
+
"kind": "bound", "state": "fresh", "binding": binding, "value": value,
|
| 1558 |
+
"display": "" if value is None else str(value), "refreshed_at": refreshed_at})
|
| 1559 |
+
observed.append({"sheet": sheet_name, "address": address, "value": value})
|
| 1560 |
+
if "note" in row:
|
| 1561 |
+
note = str(row.get("note") or "").strip()
|
| 1562 |
+
if note:
|
| 1563 |
+
desired["note"] = note
|
| 1564 |
+
else:
|
| 1565 |
+
desired.pop("note", None)
|
| 1566 |
+
planned_notes.add(slot)
|
| 1567 |
+
after = sheets.clean_cell(desired, address)
|
| 1568 |
+
# A planned formula is `derived`: the recalculation below is what fills its value, and
|
| 1569 |
+
# `_checked_projection` accepts a recalculated planned cell only under that kind.
|
| 1570 |
+
edit_kind = {"formula": "derived", "bound": "binding"}.get(kind, "metric")
|
| 1571 |
+
edits.append({"kind": edit_kind, "name": sheet_name + "!" + address, "sheet": sheet_name,
|
| 1572 |
+
"address": address, "before": before, "after": after})
|
| 1573 |
+
tab_visual = spec.get("tab_visual") or {}
|
| 1574 |
+
if not isinstance(tab_visual, dict):
|
| 1575 |
+
raise RepairRefusal("repair_metrics_visual_invalid", "Tab visual changes must be an object.")
|
| 1576 |
+
for sheet_name, keys in tab_visual.items():
|
| 1577 |
+
_tab(current, sheet_name)
|
| 1578 |
+
if not isinstance(keys, dict) or not keys or set(keys) - set(_PLANNED_TAB_VISUAL_KEYS):
|
| 1579 |
+
raise RepairRefusal("repair_metrics_visual_invalid",
|
| 1580 |
+
"A metrics plan may declare only a tab's freeze.")
|
| 1581 |
+
projected, _errors = _checked_projection(_project(current, edits), edits, formula_check)
|
| 1582 |
+
_daily_sales_invariants(projected, formula_check)
|
| 1583 |
+
remaining = {name: sorted(set(found) - OWNER_INPUTS.get(name, frozenset()))
|
| 1584 |
+
for name, found in hardcoded_numbers(projected).items()
|
| 1585 |
+
if name in OWNER_INPUTS}
|
| 1586 |
+
if any(remaining.values()):
|
| 1587 |
+
raise RepairRefusal("repair_hardcoded_remain", "Hardcoded numbers would remain: "
|
| 1588 |
+
+ json.dumps(remaining, sort_keys=True))
|
| 1589 |
+
return [edit for edit in edits if edit["before"] != edit["after"]], {
|
| 1590 |
+
"observed": observed, "planned_notes": sorted(f"{s}!{a}" for s, a in planned_notes),
|
| 1591 |
+
"tab_visual": copy.deepcopy(tab_visual),
|
| 1592 |
+
"hardcoded_before": hardcoded_numbers(current), "hardcoded_after": hardcoded_numbers(projected)}
|
| 1593 |
+
|
| 1594 |
+
|
| 1595 |
+
def _apply_tab_visual(st, workbook_id, tab_visual, *, actor="", actor_name=""):
|
| 1596 |
+
"""Write the DECLARED tab-level visual keys through the ordinary document door, then prove
|
| 1597 |
+
nothing else moved. Returns the readback snapshot."""
|
| 1598 |
+
doc = sheets.workbook_snapshot(st, workbook_id)
|
| 1599 |
+
before_cells = {str(t.get("name")): copy.deepcopy(t.get("cells") or {}) for t in doc["sheets"]}
|
| 1600 |
+
before_visual = visual_metadata(doc)
|
| 1601 |
+
for sheet_name, keys in tab_visual.items():
|
| 1602 |
+
tab = _tab(doc, sheet_name)
|
| 1603 |
+
for key, value in keys.items():
|
| 1604 |
+
tab[key] = copy.deepcopy(value)
|
| 1605 |
+
sheets.put_workbook(st, workbook_id, doc, expect_rev=int(doc.get("rev") or 0),
|
| 1606 |
+
actor=actor, actor_name=actor_name, lossless=True)
|
| 1607 |
+
readback = sheets.workbook_snapshot(st, workbook_id)
|
| 1608 |
+
for sheet_name, keys in tab_visual.items():
|
| 1609 |
+
tab = _tab(readback, sheet_name)
|
| 1610 |
+
for key, value in keys.items():
|
| 1611 |
+
if _canonical(tab.get(key)) != _canonical(value):
|
| 1612 |
+
raise RepairRefusal("repair_visual_readback", sheet_name + " " + key
|
| 1613 |
+
+ " did not read back as declared.")
|
| 1614 |
+
if {str(t.get("name")): t.get("cells") or {} for t in readback["sheets"]} != before_cells:
|
| 1615 |
+
raise RepairRefusal("repair_visual_readback", "A declared visual change moved cell content.")
|
| 1616 |
+
expected = copy.deepcopy(before_visual)
|
| 1617 |
+
for item in expected:
|
| 1618 |
+
for key, value in (tab_visual.get(item.get("name")) or {}).items():
|
| 1619 |
+
item[key] = copy.deepcopy(value)
|
| 1620 |
+
if visual_metadata(readback) != _canonical(expected):
|
| 1621 |
+
raise RepairRefusal("repair_visual_readback", "An undeclared visual property changed.")
|
| 1622 |
+
return readback
|
| 1623 |
+
|
| 1624 |
+
|
| 1625 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1626 |
+
# ββ S67-T22 / S67-T23 β DECLARED VISUAL REPAIRS: Excel's rule colours and its 3M sparklines
|
| 1627 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1628 |
+
#
|
| 1629 |
+
# Every repair above REFUSES a visual change, on purpose. These two tickets ARE visual changes, so
|
| 1630 |
+
# they get their own mode that names exactly what may move: a tab's `rules` (T22, O31) and a cell's
|
| 1631 |
+
# `sparkline` (T23, O32). Cell content, every other visual key and every other cell must read back
|
| 1632 |
+
# unchanged; the backup lands before anything is written.
|
| 1633 |
+
|
| 1634 |
+
_VISUAL_REPAIR_TAB_KEYS = ("rules",)
|
| 1635 |
+
|
| 1636 |
+
|
| 1637 |
+
def _build_visual_repair_plan(current, request):
|
| 1638 |
+
spec = request["visual_repair"]
|
| 1639 |
+
if not isinstance(spec, dict) or any(key in request for key in (
|
| 1640 |
+
"formula_repair", "related_migration", "summary_block", "source_cells",
|
| 1641 |
+
"binding_cells", "daily_sales_bindings", "computed_metrics")):
|
| 1642 |
+
raise RepairRefusal("repair_visual_plan_conflict", "Run the visual repair as its own repair.")
|
| 1643 |
+
tab_visual = copy.deepcopy(spec.get("tab_visual") or {})
|
| 1644 |
+
for sheet_name, keys in tab_visual.items():
|
| 1645 |
+
_tab(current, sheet_name)
|
| 1646 |
+
if not isinstance(keys, dict) or not keys or set(keys) - set(_VISUAL_REPAIR_TAB_KEYS):
|
| 1647 |
+
raise RepairRefusal("repair_visual_invalid", "A visual repair may declare only a tab's rules.")
|
| 1648 |
+
keys["rules"] = sheets._clean_rules(keys["rules"], lossless=True)
|
| 1649 |
+
edits, seen = [], set()
|
| 1650 |
+
for row in spec.get("cells") or ():
|
| 1651 |
+
if not isinstance(row, dict) or "sparkline" not in row:
|
| 1652 |
+
raise RepairRefusal("repair_visual_invalid", "A visual repair cell names only its sparkline.")
|
| 1653 |
+
tab = _tab(current, str(row.get("sheet") or ""))
|
| 1654 |
+
sheet_name, address = str(tab.get("name")), _addr(row.get("address"))
|
| 1655 |
+
if (sheet_name.casefold(), address) in seen:
|
| 1656 |
+
raise RepairRefusal("repair_edit_duplicate", "A repaired cell may be named only once.")
|
| 1657 |
+
seen.add((sheet_name.casefold(), address))
|
| 1658 |
+
before = copy.deepcopy((tab.get("cells") or {}).get(address))
|
| 1659 |
+
desired = copy.deepcopy(before) if before else {"kind": "empty", "state": "typed"}
|
| 1660 |
+
if row["sparkline"] is None:
|
| 1661 |
+
desired.pop("sparkline", None)
|
| 1662 |
+
else:
|
| 1663 |
+
clean = sheets._clean_sparkline(row["sparkline"])
|
| 1664 |
+
if clean is None:
|
| 1665 |
+
raise RepairRefusal("repair_visual_invalid", sheet_name + "!" + address
|
| 1666 |
+
+ " names a sparkline that cannot be drawn.")
|
| 1667 |
+
desired["sparkline"] = clean
|
| 1668 |
+
after = sheets.clean_cell(desired, address)
|
| 1669 |
+
content = ("kind", "state", "value", "display", "formula", "binding", "text")
|
| 1670 |
+
if any((before or {}).get(key) != after.get(key) for key in content if key != "state") \
|
| 1671 |
+
and not (before is None and after.get("kind") == "empty"):
|
| 1672 |
+
raise RepairRefusal("repair_visual_content_changed", sheet_name + "!" + address
|
| 1673 |
+
+ " would change content, which a visual repair never does.")
|
| 1674 |
+
edits.append({"kind": "visual", "name": sheet_name + "!" + address, "sheet": sheet_name,
|
| 1675 |
+
"address": address, "before": before, "after": after})
|
| 1676 |
+
return [e for e in edits if e["before"] != e["after"]], {
|
| 1677 |
+
"tab_visual": tab_visual, "sparkline_cells": sorted(e["name"] for e in edits)}
|
| 1678 |
+
|
| 1679 |
+
def _execute_visual_repair(st, backup_store, request, current, revision, digest, workbook_id,
|
| 1680 |
+
*, actor="", actor_name=""):
|
| 1681 |
+
"""Back up, then write ONLY the declared sparklines (exact cell edits) and tab rules."""
|
| 1682 |
+
edits, evidence = _build_visual_repair_plan(current, request)
|
| 1683 |
+
if not edits and not evidence["tab_visual"]:
|
| 1684 |
+
return {"workbook_id": workbook_id, "revision_before": revision, "revision_after": revision,
|
| 1685 |
+
"applied": 0, "noop": True, "visual_repair": evidence}
|
| 1686 |
+
planned = {(row["sheet"].casefold(), row["address"]) for row in edits}
|
| 1687 |
+
projected = _project(current, edits)
|
| 1688 |
+
if _visual_without(projected, planned, ("sparkline",)) != \
|
| 1689 |
+
_visual_without(current, planned, ("sparkline",)):
|
| 1690 |
+
raise RepairRefusal("repair_visual_metadata_changed",
|
| 1691 |
+
"The projected repair changes a visual property it did not declare.")
|
| 1692 |
+
exact = [{key: copy.deepcopy(row[key]) for key in ("sheet", "address", "before", "after")}
|
| 1693 |
+
for row in edits]
|
| 1694 |
+
diff_digest = sheets.workbook_digest({"edits": exact, "tab_visual": evidence["tab_visual"]})
|
| 1695 |
+
backup_key = _write_backup(backup_store, current, digest, diff_digest, workbook_id)
|
| 1696 |
+
revision_after = revision
|
| 1697 |
+
if exact:
|
| 1698 |
+
sheets.apply_exact_cell_edits(st, workbook_id, exact, revision, digest,
|
| 1699 |
+
actor=actor, actor_name=actor_name)
|
| 1700 |
+
readback = sheets.workbook_snapshot(st, workbook_id)
|
| 1701 |
+
if int(readback.get("rev") or 0) != revision + 1:
|
| 1702 |
+
raise RepairRefusal("repair_readback_revision",
|
| 1703 |
+
"The repaired workbook revision did not read back as expected.")
|
| 1704 |
+
for row in edits:
|
| 1705 |
+
actual = (_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"])
|
| 1706 |
+
if actual != sheets.clean_cell(row["after"], row["address"]):
|
| 1707 |
+
raise RepairRefusal("repair_readback_cell",
|
| 1708 |
+
row["sheet"] + "!" + row["address"] + " did not read back exactly.")
|
| 1709 |
+
revision_after = int(readback["rev"])
|
| 1710 |
+
if evidence["tab_visual"]:
|
| 1711 |
+
revision_after = int(_apply_tab_visual(st, workbook_id, evidence["tab_visual"],
|
| 1712 |
+
actor=actor, actor_name=actor_name)["rev"])
|
| 1713 |
+
return {"workbook_id": workbook_id, "revision_before": revision,
|
| 1714 |
+
"revision_after": revision_after, "applied": len(exact), "backup_key": backup_key,
|
| 1715 |
+
"backup_digest": digest, "diff_digest": diff_digest, "visual_repair": evidence}
|
| 1716 |
+
|
| 1717 |
+
|
| 1718 |
+
def restore_backup(st, backup_store, backup_key, *, workbook_id, actor="", actor_name=""):
|
| 1719 |
+
"""NC for every mode here: put a backup's workbook back and prove it round-trips."""
|
| 1720 |
+
record = backup_store.get(backup_key)
|
| 1721 |
+
if not isinstance(record, dict) or record.get("workbook_id") != workbook_id:
|
| 1722 |
+
raise RepairRefusal("repair_backup_missing", "That backup does not exist for this workbook.")
|
| 1723 |
+
saved = copy.deepcopy(record["workbook"])
|
| 1724 |
+
current = sheets.workbook_snapshot(st, workbook_id)
|
| 1725 |
+
sheets.put_workbook(st, workbook_id, saved, expect_rev=int(current.get("rev") or 0),
|
| 1726 |
+
actor=actor, actor_name=actor_name, lossless=True)
|
| 1727 |
+
readback = sheets.workbook_snapshot(st, workbook_id)
|
| 1728 |
+
same = ({str(t.get("name")): t.get("cells") or {} for t in readback["sheets"]}
|
| 1729 |
+
== {str(t.get("name")): t.get("cells") or {} for t in saved["sheets"]}
|
| 1730 |
+
and visual_metadata(readback) == visual_metadata(saved))
|
| 1731 |
+
if not same:
|
| 1732 |
+
raise RepairRefusal("repair_restore_unverified", "The restored workbook did not round-trip.")
|
| 1733 |
+
return readback
|
| 1734 |
+
|
| 1735 |
+
|
| 1736 |
def _build_plan(current, request, resolve_binding):
|
| 1737 |
if not isinstance(request.get("source_cells"), list) or not request["source_cells"]:
|
| 1738 |
raise RepairRefusal(
|
|
|
|
| 1925 |
raise RepairRefusal("repair_revision_mismatch",
|
| 1926 |
"The named workbook is not at the expected repair revision.")
|
| 1927 |
digest = sheets.workbook_digest(current)
|
| 1928 |
+
if "visual_repair" in request:
|
| 1929 |
+
return _execute_visual_repair(st, backup_store, request, current, revision, digest,
|
| 1930 |
+
target_workbook_id, actor=actor, actor_name=actor_name)
|
| 1931 |
summary_evidence = None
|
| 1932 |
summary_errors_before = None
|
| 1933 |
+
migrated = None
|
| 1934 |
+
formula_evidence = None
|
| 1935 |
+
daily_evidence = None
|
| 1936 |
+
metrics_evidence = None
|
| 1937 |
+
if "computed_metrics" in request:
|
| 1938 |
+
formula_check = _daily_sales_formula_checker(formula_check)
|
| 1939 |
+
edits, metrics_evidence = _build_computed_metrics_plan(
|
| 1940 |
+
current, request, resolve_binding, formula_check)
|
| 1941 |
+
if not edits and not metrics_evidence["tab_visual"]:
|
| 1942 |
+
return {"workbook_id": target_workbook_id, "revision_before": revision,
|
| 1943 |
+
"revision_after": revision, "applied": 0, "noop": True,
|
| 1944 |
+
"computed_metrics": metrics_evidence}
|
| 1945 |
+
elif "daily_sales_bindings" in request:
|
| 1946 |
+
formula_check = _daily_sales_formula_checker(formula_check)
|
| 1947 |
+
edits, daily_evidence = _build_daily_sales_bindings_plan(
|
| 1948 |
+
current, request, resolve_binding, formula_check)
|
| 1949 |
+
if not edits:
|
| 1950 |
+
census = _daily_sales_invariants(current, formula_check)
|
| 1951 |
+
return {"workbook_id": target_workbook_id, "revision_before": revision,
|
| 1952 |
+
"revision_after": revision, "applied": 0, "noop": True,
|
| 1953 |
+
"daily_sales_bindings": daily_evidence, "formula_errors": [],
|
| 1954 |
+
"formula_error_census": {"before": census, "after": copy.deepcopy(census)},
|
| 1955 |
+
"formula_cycles": _formula_cycle_comparison(current, current, formula_check),
|
| 1956 |
+
"visual_metadata_preserved": True}
|
| 1957 |
+
elif "formula_repair" in request:
|
| 1958 |
+
edits, formula_evidence = _build_formula_repair_plan(current, request, formula_check)
|
| 1959 |
+
if not edits:
|
| 1960 |
+
census = _formula_repair_census(current)
|
| 1961 |
+
return {"workbook_id": target_workbook_id, "revision_before": revision,
|
| 1962 |
+
"revision_after": revision, "applied": 0, "noop": True,
|
| 1963 |
+
"formula_repair": formula_evidence, "formula_errors": formula_error_census(current),
|
| 1964 |
+
"formula_error_census": {"before": census, "after": copy.deepcopy(census)},
|
| 1965 |
+
"formula_cycles": _formula_cycle_comparison(current, current, formula_check),
|
| 1966 |
+
"visual_metadata_preserved": True}
|
| 1967 |
+
elif "related_migration" in request:
|
| 1968 |
+
edits, migrated = _build_related_migration_plan(
|
| 1969 |
+
current, request, resolve_binding, formula_check)
|
| 1970 |
+
if not edits:
|
| 1971 |
+
return {"workbook_id": target_workbook_id, "revision_before": revision,
|
| 1972 |
+
"revision_after": revision, "applied": 0, "migrated": [], "noop": True}
|
| 1973 |
+
elif "summary_block" in request:
|
| 1974 |
summary_errors_before = check_summary_formula_errors(current)
|
| 1975 |
edits, summary_evidence = _build_summary_plan(current, request, resolve_binding, formula_check)
|
| 1976 |
else:
|
| 1977 |
edits = _build_plan(current, request, resolve_binding)
|
| 1978 |
projected = _project(current, edits)
|
| 1979 |
projected, projected_errors = _checked_projection(projected, edits, formula_check)
|
| 1980 |
+
if formula_evidence is not None:
|
| 1981 |
+
_check_formula_repair_errors(projected)
|
| 1982 |
+
if daily_evidence is not None:
|
| 1983 |
+
_daily_sales_invariants(projected, formula_check)
|
| 1984 |
comparison = _formula_error_comparison(current, projected, edits, projected_errors)
|
| 1985 |
summary_errors_after = (check_summary_formula_errors(projected)
|
| 1986 |
if summary_evidence is not None else None)
|
|
|
|
| 1991 |
"The projected repair introduces a formula error or leaves one in its repair scope, "
|
| 1992 |
"so no repair was applied.")
|
| 1993 |
visual_before = visual_metadata(current)
|
| 1994 |
+
planned_notes = set()
|
| 1995 |
+
if metrics_evidence is not None:
|
| 1996 |
+
planned_notes = {tuple(item.split("!", 1)) for item in metrics_evidence["planned_notes"]}
|
| 1997 |
+
planned_notes = {(sheet.casefold(), address) for sheet, address in planned_notes}
|
| 1998 |
+
if _visual_without(projected, planned_notes) != _visual_without(current, planned_notes):
|
| 1999 |
raise RepairRefusal("repair_visual_metadata_changed",
|
| 2000 |
"The projected repair changes Daily Sales visual metadata.")
|
| 2001 |
|
|
|
|
| 2027 |
or _cell_map(checked_readback) != _cell_map(readback):
|
| 2028 |
raise RepairRefusal("repair_readback_formula_state",
|
| 2029 |
"The repaired workbook did not read back calculation-current.")
|
| 2030 |
+
if _visual_without(readback, planned_notes) != _visual_without(current, planned_notes):
|
| 2031 |
raise RepairRefusal("repair_readback_visual_metadata",
|
| 2032 |
"Daily Sales visual metadata changed during repair.")
|
| 2033 |
+
if formula_evidence is not None:
|
| 2034 |
+
_check_formula_repair_errors(readback)
|
| 2035 |
+
if daily_evidence is not None:
|
| 2036 |
+
_daily_sales_invariants(readback, formula_check)
|
| 2037 |
if summary_evidence is not None:
|
| 2038 |
tab = _tab(readback, SUMMARY_TAB)
|
| 2039 |
if any((tab.get("cells", {}).get(col + str(row)) or {}).get("kind")
|
|
|
|
| 2066 |
"bindings": bindings,
|
| 2067 |
"formula_errors": formula_error_census(readback),
|
| 2068 |
"formula_error_comparison": readback_comparison,
|
| 2069 |
+
"formula_cycles": _formula_cycle_comparison(current, readback, formula_check),
|
| 2070 |
"visual_metadata_preserved": True,
|
| 2071 |
}
|
| 2072 |
if summary_evidence is not None:
|
| 2073 |
result["summary_block"] = summary_evidence
|
| 2074 |
+
if formula_evidence is not None:
|
| 2075 |
+
result["formula_repair"] = formula_evidence
|
| 2076 |
+
result["formula_error_census"] = {
|
| 2077 |
+
"before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
|
| 2078 |
+
if migrated is not None:
|
| 2079 |
+
result["migrated"] = [
|
| 2080 |
+
{**row, "after_value": _tab(readback, row["sheet"])["cells"][row["address"]]["value"]}
|
| 2081 |
+
for row in migrated]
|
| 2082 |
+
if daily_evidence is not None:
|
| 2083 |
+
result["daily_sales_bindings"] = daily_evidence
|
| 2084 |
+
result["formula_error_census"] = {
|
| 2085 |
+
"before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
|
| 2086 |
+
if metrics_evidence is not None:
|
| 2087 |
+
_daily_sales_invariants(readback, formula_check)
|
| 2088 |
+
remaining = {name: sorted(set(found) - OWNER_INPUTS.get(name, frozenset()))
|
| 2089 |
+
for name, found in hardcoded_numbers(readback).items() if name in OWNER_INPUTS}
|
| 2090 |
+
if any(remaining.values()):
|
| 2091 |
+
raise RepairRefusal("repair_hardcoded_remain", "Hardcoded numbers remain after the write: "
|
| 2092 |
+
+ json.dumps(remaining, sort_keys=True))
|
| 2093 |
+
if metrics_evidence["tab_visual"]:
|
| 2094 |
+
visual_readback = _apply_tab_visual(st, target_workbook_id, metrics_evidence["tab_visual"],
|
| 2095 |
+
actor=actor, actor_name=actor_name)
|
| 2096 |
+
result["revision_after"] = visual_readback["rev"]
|
| 2097 |
+
metrics_evidence["hardcoded_readback"] = hardcoded_numbers(readback)
|
| 2098 |
+
result["computed_metrics"] = metrics_evidence
|
| 2099 |
+
result["formula_error_census"] = {
|
| 2100 |
+
"before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
|
| 2101 |
return result
|
platform/core/data_binding.py
CHANGED
|
@@ -167,6 +167,69 @@ def local_override():
|
|
| 167 |
return (not is_space()) and _env(_LOCAL_OPT_IN_VAR) == '1'
|
| 168 |
|
| 169 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
def default_store():
|
| 171 |
"""What `OS_DATA_REPO` resolves to when it is unset.
|
| 172 |
|
|
@@ -221,7 +284,7 @@ def write_refusal(repo):
|
|
| 221 |
|
| 222 |
def check_write(repo, operation='write'):
|
| 223 |
"""Raise `StoreWriteRefused` unless this deployment may write `repo`. The one enforcement door."""
|
| 224 |
-
reason = write_refusal(repo)
|
| 225 |
if reason:
|
| 226 |
raise StoreWriteRefused(f'{operation} to {repo!r} refused: {reason}')
|
| 227 |
|
|
|
|
| 167 |
return (not is_space()) and _env(_LOCAL_OPT_IN_VAR) == '1'
|
| 168 |
|
| 169 |
|
| 170 |
+
#: W-219. Inherited by every child a gate process starts, so a subprocess a gate spawns is a gate too.
|
| 171 |
+
_GATE_PROCESS_VAR = 'AIOS_GATE_PROCESS'
|
| 172 |
+
|
| 173 |
+
#: The directories the product gate battery lives in (`aios-web/api`, `aios-web/web`, `aios-web`,
|
| 174 |
+
#: `platform`). `ops/verify_*.py` are OPERATOR tools, some of which read or probe a live store on
|
| 175 |
+
#: purpose, so they are not gates here.
|
| 176 |
+
_GATE_DIRS = frozenset({'api', 'web', 'aios-web', 'platform'})
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def _argv_gate():
|
| 180 |
+
"""The `verify_*.py` gate script this process was started as, or `''`."""
|
| 181 |
+
import sys
|
| 182 |
+
argv0 = str((getattr(sys, 'argv', None) or [''])[0] or '')
|
| 183 |
+
name = os.path.basename(argv0).lower()
|
| 184 |
+
if not (name.startswith('verify_') and name.endswith('.py')):
|
| 185 |
+
return ''
|
| 186 |
+
parent = os.path.basename(os.path.dirname(os.path.abspath(argv0))).lower()
|
| 187 |
+
return name if parent in _GATE_DIRS else ''
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def gate_process():
|
| 191 |
+
"""The gate this process is, or `''`. W-219: a gate writes FIXTURE stores only, never a real one.
|
| 192 |
+
|
| 193 |
+
ββ WHY THIS LIVES AT THE STORE DOOR AND NOT IN EACH GATE. A lane container carries the exact
|
| 194 |
+
isolated staging posture `store_pg.write_refusal` is written to ALLOW (`STORE_BACKEND=pg`, the
|
| 195 |
+
staging `DATABASE_URL`, `AIOS_DEPLOYMENT_ID=fsanyoto/loopable`, `AIOS_SANDBOX_TENANTS=
|
| 196 |
+
royal-imports`), and `_fake_store.installed` swaps module functions a bound `PgStore` never
|
| 197 |
+
calls. So `verify_meta.py` run in SDLC 61 wrote staging's `t_royal_imports` through the real
|
| 198 |
+
app, and a pin at the top of each gate (W-219, W-240) only covers the gates somebody pinned:
|
| 199 |
+
`verify_amazon` and `verify_odoo_relational` carry none. Asking the PROCESS covers every gate,
|
| 200 |
+
including one written tomorrow.
|
| 201 |
+
|
| 202 |
+
Two ways to be one: started as `verify_*.py` from a battery directory (every battery row and
|
| 203 |
+
every lane runs `python <dir>/verify_<x>.py`), or `AIOS_GATE_PROCESS=1` inherited from one.
|
| 204 |
+
β There is no way OUT through the environment, because a lane container exports whatever it
|
| 205 |
+
was handed: the only off switch is an in-process patch of this function, which is what a gate
|
| 206 |
+
testing the PRODUCT rules does.
|
| 207 |
+
"""
|
| 208 |
+
name = _argv_gate()
|
| 209 |
+
if name:
|
| 210 |
+
return name
|
| 211 |
+
return 'a process started by a gate' if _env(_GATE_PROCESS_VAR) == '1' else ''
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
if _argv_gate():
|
| 215 |
+
os.environ[_GATE_PROCESS_VAR] = '1' # W-219: a child this gate spawns is a gate too
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def gate_write_refusal(repo):
|
| 219 |
+
"""`None` unless this is a gate process writing a PRODUCTION store, else why not. W-219.
|
| 220 |
+
|
| 221 |
+
Consulted by `check_write` BEFORE `write_refusal`, so neither `AIOS_ALLOW_PRODUCTION_DATA=1`
|
| 222 |
+
on a laptop nor a production identity opens a production store to a gate.
|
| 223 |
+
"""
|
| 224 |
+
who = gate_process()
|
| 225 |
+
rid = str(repo or '').strip()
|
| 226 |
+
if not who or rid not in PRODUCTION_STORES:
|
| 227 |
+
return None
|
| 228 |
+
return (f'this process is a verification gate ({who}), and a gate never writes {rid}, which '
|
| 229 |
+
f'holds real customer data. A gate writes fixture stores only: empty HF_TOKEN and '
|
| 230 |
+
f'point OS_DATA_REPO at a fixture store (W-219).')
|
| 231 |
+
|
| 232 |
+
|
| 233 |
def default_store():
|
| 234 |
"""What `OS_DATA_REPO` resolves to when it is unset.
|
| 235 |
|
|
|
|
| 284 |
|
| 285 |
def check_write(repo, operation='write'):
|
| 286 |
"""Raise `StoreWriteRefused` unless this deployment may write `repo`. The one enforcement door."""
|
| 287 |
+
reason = gate_write_refusal(repo) or write_refusal(repo)
|
| 288 |
if reason:
|
| 289 |
raise StoreWriteRefused(f'{operation} to {repo!r} refused: {reason}')
|
| 290 |
|
platform/core/do_usage.py
ADDED
|
@@ -0,0 +1,583 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""DigitalOcean billing observations, never SQL or a database wake-up probe.
|
| 2 |
+
|
| 3 |
+
The card uses GETs and memory only. The optional sampler receives storage
|
| 4 |
+
callbacks from the host and shares Neon's management-plane store-awake guard.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import copy
|
| 9 |
+
import json
|
| 10 |
+
import math
|
| 11 |
+
import os
|
| 12 |
+
import re
|
| 13 |
+
import threading
|
| 14 |
+
from datetime import datetime, timezone
|
| 15 |
+
from urllib.error import HTTPError, URLError
|
| 16 |
+
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
| 17 |
+
|
| 18 |
+
from core.neon_usage import send_telegram, store_is_awake
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
API_BASE = "https://api.digitalocean.com/v2"
|
| 22 |
+
HTTP_TIMEOUT = 5
|
| 23 |
+
DEFAULT_CLUSTER_ID = "0c4049b4-8541-43a7-ab6e-a69c34a8c604"
|
| 24 |
+
STORE_KEY = "platform_do_usage_samples"
|
| 25 |
+
MAX_SAMPLES = 240
|
| 26 |
+
FIXED_PRICES = {"db-s-1vcpu-1gb": {"usd": 15.15, "source": "R28 fixed monthly price per node"}}
|
| 27 |
+
CLUSTER_SOURCE = "DigitalOcean databases API"
|
| 28 |
+
BALANCE_SOURCE = "DigitalOcean balance API (daily snapshot, account-wide)"
|
| 29 |
+
BILLING_SOURCE = "DigitalOcean billing history API"
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class UsageReadError(Exception):
|
| 33 |
+
"""Fixed, credential-free cause safe to show to an operator."""
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class _NoRedirect(HTTPRedirectHandler):
|
| 37 |
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
| 38 |
+
return None
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _get(path, token):
|
| 42 |
+
try:
|
| 43 |
+
request = Request(API_BASE + path, headers={"Authorization": "Bearer " + token}, method="GET")
|
| 44 |
+
with build_opener(_NoRedirect()).open(request, timeout=HTTP_TIMEOUT) as response:
|
| 45 |
+
payload = json.loads(response.read(1_000_001))
|
| 46 |
+
if not isinstance(payload, dict):
|
| 47 |
+
raise ValueError()
|
| 48 |
+
return payload
|
| 49 |
+
except HTTPError as exc:
|
| 50 |
+
raise UsageReadError(f"DigitalOcean API returned HTTP {exc.code}.") from None
|
| 51 |
+
except TimeoutError:
|
| 52 |
+
raise UsageReadError("DigitalOcean API timed out.") from None
|
| 53 |
+
except URLError as exc:
|
| 54 |
+
cause = "timed out" if isinstance(exc.reason, TimeoutError) else "could not be reached"
|
| 55 |
+
raise UsageReadError(f"DigitalOcean API {cause}.") from None
|
| 56 |
+
except Exception:
|
| 57 |
+
# Do not return the URL, response body or exception text: any can contain credentials.
|
| 58 |
+
raise UsageReadError("DigitalOcean API returned an unreadable response.") from None
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _now():
|
| 62 |
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _stamp(value):
|
| 66 |
+
try:
|
| 67 |
+
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
| 68 |
+
return parsed.astimezone(timezone.utc) if parsed.tzinfo else None
|
| 69 |
+
except (ValueError, TypeError):
|
| 70 |
+
return None
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _text(value):
|
| 74 |
+
if not isinstance(value, str) or not value.strip():
|
| 75 |
+
raise ValueError()
|
| 76 |
+
return value.replace("\u2014", "; ").replace("\u2013", " to ")[:512]
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _money(value):
|
| 80 |
+
if isinstance(value, bool) or value is None:
|
| 81 |
+
raise ValueError()
|
| 82 |
+
number = float(value)
|
| 83 |
+
if not math.isfinite(number):
|
| 84 |
+
raise ValueError()
|
| 85 |
+
return number
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def read_account():
|
| 89 |
+
"""Independently read each API block; allowlist fields to exclude connection secrets."""
|
| 90 |
+
read_at = _now()
|
| 91 |
+
cluster = {"available": False, "cause": None, "source": CLUSTER_SOURCE, "read_at": read_at,
|
| 92 |
+
**dict.fromkeys(("id", "name", "size", "num_nodes", "region", "status", "version", "storage_size_mib"))}
|
| 93 |
+
balance = {"available": False, "cause": None, "source": BALANCE_SOURCE, "read_at": read_at,
|
| 94 |
+
**dict.fromkeys(("month_to_date_usage", "account_balance", "month_to_date_balance", "generated_at"))}
|
| 95 |
+
billing = {"available": False, "cause": None, "source": BILLING_SOURCE, "read_at": read_at}
|
| 96 |
+
price = {"monthly_usd": None, "source": None, "read_at": read_at, "cause": None}
|
| 97 |
+
result = {"available": False, "cause": None, "read_at": read_at, "cluster": cluster,
|
| 98 |
+
"balance": balance, "billing": billing, "billing_history": [], "fixed_price": price}
|
| 99 |
+
token = os.environ.get("AIOS_DO_USAGE_TOKEN") or os.environ.get("DIGITALOCEAN_TOKEN", "")
|
| 100 |
+
if not token:
|
| 101 |
+
cause = "No DigitalOcean token is configured on this deployment, so usage cannot be read."
|
| 102 |
+
result["cause"] = price["cause"] = cause
|
| 103 |
+
for block in (cluster, balance, billing):
|
| 104 |
+
block["cause"] = cause
|
| 105 |
+
return result
|
| 106 |
+
selected = os.environ.get("AIOS_DO_CLUSTER_ID", DEFAULT_CLUSTER_ID)
|
| 107 |
+
|
| 108 |
+
try:
|
| 109 |
+
if not re.fullmatch(r"[a-zA-Z0-9-]{1,128}", selected):
|
| 110 |
+
raise UsageReadError("AIOS_DO_CLUSTER_ID is invalid; the cluster could not be read.")
|
| 111 |
+
database = _get("/databases/" + selected, token).get("database")
|
| 112 |
+
if not isinstance(database, dict) or database.get("id") != selected:
|
| 113 |
+
raise ValueError()
|
| 114 |
+
fields = {key: _text(database.get(key)) for key in ("id", "name", "size", "region", "status", "version")}
|
| 115 |
+
nodes = database.get("num_nodes")
|
| 116 |
+
if type(nodes) is not int or nodes < 1:
|
| 117 |
+
raise ValueError()
|
| 118 |
+
storage = database.get("storage_size_mib")
|
| 119 |
+
fields.update(num_nodes=nodes, storage_size_mib=storage if type(storage) is int and storage >= 0 else None)
|
| 120 |
+
cluster.update(fields, available=True, read_at=_now())
|
| 121 |
+
quoted = FIXED_PRICES.get(fields["size"])
|
| 122 |
+
if quoted:
|
| 123 |
+
price.update(monthly_usd=round(quoted["usd"] * nodes, 2), source=quoted["source"], read_at=cluster["read_at"])
|
| 124 |
+
else:
|
| 125 |
+
price["cause"] = f"Fixed price UNKNOWN for size {fields['size']}; no fixed price is configured."
|
| 126 |
+
except UsageReadError as exc:
|
| 127 |
+
cluster["cause"] = str(exc)
|
| 128 |
+
except Exception:
|
| 129 |
+
cluster["cause"] = "DigitalOcean cluster response is missing valid cluster details."
|
| 130 |
+
if not cluster["available"]:
|
| 131 |
+
price["cause"] = cluster["cause"]
|
| 132 |
+
|
| 133 |
+
try:
|
| 134 |
+
payload = _get("/customers/my/balance", token)
|
| 135 |
+
amounts = {key: _money(payload.get(key)) for key in
|
| 136 |
+
("month_to_date_usage", "account_balance", "month_to_date_balance")}
|
| 137 |
+
generated_at = payload.get("generated_at")
|
| 138 |
+
if amounts["month_to_date_usage"] < 0 or not _stamp(generated_at):
|
| 139 |
+
raise ValueError()
|
| 140 |
+
balance.update(amounts, generated_at=generated_at, available=True, read_at=_now())
|
| 141 |
+
except UsageReadError as exc:
|
| 142 |
+
balance["cause"] = str(exc)
|
| 143 |
+
except Exception:
|
| 144 |
+
balance["cause"] = "DigitalOcean balance response is missing valid amounts or its snapshot time."
|
| 145 |
+
|
| 146 |
+
try:
|
| 147 |
+
payload = _get("/customers/my/billing_history", token)
|
| 148 |
+
rows = payload.get("billing_history")
|
| 149 |
+
# A new account's empty history arrives as the bare list envelope {"links": {}, "meta": {}}
|
| 150 |
+
# with the key omitted (read 2026-09-25): an empty history, not an unreadable one.
|
| 151 |
+
if rows is None and isinstance(payload, dict) and "meta" in payload:
|
| 152 |
+
rows = []
|
| 153 |
+
if not isinstance(rows, list):
|
| 154 |
+
raise ValueError()
|
| 155 |
+
history = []
|
| 156 |
+
for row in rows:
|
| 157 |
+
date = row.get("date")
|
| 158 |
+
if not _stamp(date):
|
| 159 |
+
raise ValueError()
|
| 160 |
+
history.append({"date": date, "description": _text(row.get("description")),
|
| 161 |
+
"amount": _money(row.get("amount")), "type": _text(row.get("type"))})
|
| 162 |
+
# Report a bounded page honestly; never follow an API-provided URL with a bearer token.
|
| 163 |
+
pages = (payload.get("links") or {}).get("pages") or {}
|
| 164 |
+
billing.update(available=True, read_at=_now(), has_more=bool(pages.get("next")))
|
| 165 |
+
result["billing_history"] = history
|
| 166 |
+
except UsageReadError as exc:
|
| 167 |
+
billing["cause"] = str(exc)
|
| 168 |
+
except Exception:
|
| 169 |
+
billing["cause"] = "DigitalOcean billing history response could not be read."
|
| 170 |
+
result["available"] = any(block["available"] for block in (cluster, balance, billing))
|
| 171 |
+
if not result["available"]:
|
| 172 |
+
result["cause"] = " ".join(f"{name}: {block['cause']}" for name, block in
|
| 173 |
+
(("Cluster", cluster), ("Balance", balance), ("Billing history", billing)))
|
| 174 |
+
return result
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def evaluate_alarm(account):
|
| 178 |
+
"""Account-wide MTD against the sum of the configured cluster's fixed node prices."""
|
| 179 |
+
balance, price = account["balance"], account["fixed_price"]
|
| 180 |
+
if not balance["available"]:
|
| 181 |
+
return None, "Alert evaluation deferred. " + balance["cause"]
|
| 182 |
+
expected = price["monthly_usd"]
|
| 183 |
+
if expected is None:
|
| 184 |
+
return None, "Alert evaluation deferred. " + price["cause"]
|
| 185 |
+
override = os.environ.get("AIOS_DO_ALARM_USD")
|
| 186 |
+
try:
|
| 187 |
+
threshold = expected if override is None else _money(override)
|
| 188 |
+
except (ValueError, TypeError, OverflowError):
|
| 189 |
+
return None, "AIOS_DO_ALARM_USD is invalid; alert evaluation is deferred."
|
| 190 |
+
if balance["month_to_date_usage"] <= threshold:
|
| 191 |
+
return None, None
|
| 192 |
+
test = override is not None
|
| 193 |
+
cluster = account["cluster"]
|
| 194 |
+
message = ("TEST " if test else "") + (
|
| 195 |
+
f"DigitalOcean month-to-date spend USD {balance['month_to_date_usage']:.2f}, "
|
| 196 |
+
f"as of {balance['generated_at']} ({balance['source']}; read {balance['read_at']}). "
|
| 197 |
+
f"Expected fixed price USD {expected:.2f} a month ({price['source']}; read {price['read_at']}). "
|
| 198 |
+
f"Cluster {cluster['name']} ({cluster['size']}, {cluster['num_nodes']} "
|
| 199 |
+
f"node{'' if cluster['num_nodes'] == 1 else 's'}, {cluster['region']}).")
|
| 200 |
+
if test:
|
| 201 |
+
message += f" TEST threshold USD {threshold:.2f} (AIOS_DO_ALARM_USD; read {account['read_at']})."
|
| 202 |
+
return {"level": "red", "message": message, "test": test, "threshold": threshold,
|
| 203 |
+
"period": _stamp(balance["generated_at"]).strftime("%Y-%m")}, None
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
_lock = threading.RLock()
|
| 207 |
+
_sample_lock = threading.Lock()
|
| 208 |
+
_thread = None
|
| 209 |
+
_stop = threading.Event()
|
| 210 |
+
_samples = []
|
| 211 |
+
_alerted = {}
|
| 212 |
+
_loaded = False
|
| 213 |
+
_cluster_id = None
|
| 214 |
+
_status = {"every_s": 0, "last_at": None, "persisted": None,
|
| 215 |
+
"cause": "Scheduled sampling is disabled; AIOS_DO_SAMPLE_SECONDS is unset or zero."}
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def snapshot():
|
| 219 |
+
with _lock:
|
| 220 |
+
return copy.deepcopy(_samples), dict(_status), dict(_alerted)
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def report():
|
| 224 |
+
"""The route performs management GETs only; it never loads storage or sends alerts."""
|
| 225 |
+
account = read_account()
|
| 226 |
+
samples, status, alerted = snapshot()
|
| 227 |
+
alarm, cause = evaluate_alarm(account)
|
| 228 |
+
return {**account, "samples": samples, "sampler": status, "alarm": alarm,
|
| 229 |
+
"alarm_cause": cause, "alerted": alerted}
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _clean_samples(samples):
|
| 233 |
+
clean = {}
|
| 234 |
+
for row in samples if isinstance(samples, list) else []:
|
| 235 |
+
try:
|
| 236 |
+
if not _stamp(row.get("at")) or not _stamp(row.get("generated_at")):
|
| 237 |
+
continue
|
| 238 |
+
usage = _money(row.get("month_to_date_usage"))
|
| 239 |
+
if usage < 0:
|
| 240 |
+
continue
|
| 241 |
+
clean[row["at"]] = {"at": row["at"], "generated_at": row["generated_at"],
|
| 242 |
+
"month_to_date_usage": usage, "source": BALANCE_SOURCE}
|
| 243 |
+
except (AttributeError, TypeError, ValueError, OverflowError):
|
| 244 |
+
continue
|
| 245 |
+
return sorted(clean.values(), key=lambda row: _stamp(row["at"]))[-MAX_SAMPLES:]
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def _clean_alerted(alerted):
|
| 249 |
+
if not isinstance(alerted, dict):
|
| 250 |
+
raise ValueError()
|
| 251 |
+
for month, at in alerted.items():
|
| 252 |
+
if not isinstance(month, str) or not re.fullmatch(r"\d{4}-(0[1-9]|1[0-2])", month) or not _stamp(at):
|
| 253 |
+
raise ValueError()
|
| 254 |
+
return {key: alerted[key] for key in sorted(alerted)[-2:]}
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def _sample_once(load, persist, *, sender=None):
|
| 258 |
+
# Serialize manual and scheduled ticks as well as delivery marker changes.
|
| 259 |
+
with _sample_lock:
|
| 260 |
+
_sample_tick(load, persist, sender=sender)
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def _sample_tick(load, persist, *, sender=None):
|
| 264 |
+
global _loaded, _samples, _alerted, _cluster_id
|
| 265 |
+
account = read_account()
|
| 266 |
+
balance = account["balance"]
|
| 267 |
+
alarm, alarm_cause = evaluate_alarm(account)
|
| 268 |
+
if not balance["available"]:
|
| 269 |
+
with _lock:
|
| 270 |
+
_status["cause"] = alarm_cause
|
| 271 |
+
return
|
| 272 |
+
selected = os.environ.get("AIOS_DO_CLUSTER_ID", DEFAULT_CLUSTER_ID)
|
| 273 |
+
with _lock:
|
| 274 |
+
if _cluster_id != selected:
|
| 275 |
+
_cluster_id, _loaded, _samples, _alerted = selected, False, [], {}
|
| 276 |
+
_status["persisted"] = None
|
| 277 |
+
_samples = _clean_samples([*_samples, {"at": balance["read_at"],
|
| 278 |
+
"generated_at": balance["generated_at"], "month_to_date_usage": balance["month_to_date_usage"]}])
|
| 279 |
+
_status["last_at"] = balance["read_at"]
|
| 280 |
+
awake, cause = store_is_awake()
|
| 281 |
+
if not awake:
|
| 282 |
+
with _lock:
|
| 283 |
+
_status["cause"] = cause
|
| 284 |
+
return
|
| 285 |
+
try:
|
| 286 |
+
if not _loaded:
|
| 287 |
+
saved = load()
|
| 288 |
+
saved = {} if saved is None else saved
|
| 289 |
+
matching = saved.get("cluster_id") == selected
|
| 290 |
+
history = _clean_samples(saved.get("samples", [])) if matching else []
|
| 291 |
+
markers = _clean_alerted(saved.get("alerted", {})) if matching else {}
|
| 292 |
+
with _lock:
|
| 293 |
+
_samples = _clean_samples([*history, *_samples])
|
| 294 |
+
_alerted, _loaded = markers, True
|
| 295 |
+
except Exception:
|
| 296 |
+
with _lock:
|
| 297 |
+
_status["cause"] = "Sample history and alert markers could not be loaded; alerts deferred."
|
| 298 |
+
return
|
| 299 |
+
sent = False
|
| 300 |
+
try:
|
| 301 |
+
def save():
|
| 302 |
+
awake, cause = store_is_awake()
|
| 303 |
+
if not awake:
|
| 304 |
+
raise UsageReadError(cause)
|
| 305 |
+
samples, _, markers = snapshot()
|
| 306 |
+
persist({"cluster_id": selected, "samples": samples, "alerted": markers})
|
| 307 |
+
with _lock:
|
| 308 |
+
_status.update(persisted=_now(), cause=alarm_cause)
|
| 309 |
+
|
| 310 |
+
# Flush any successful delivery marker before considering another send.
|
| 311 |
+
save()
|
| 312 |
+
if not alarm or alarm["period"] in _alerted:
|
| 313 |
+
return
|
| 314 |
+
try:
|
| 315 |
+
ok, delivery = (sender or send_telegram)(alarm["message"])
|
| 316 |
+
except Exception:
|
| 317 |
+
ok, delivery = False, "Telegram alert could not be sent."
|
| 318 |
+
if not ok:
|
| 319 |
+
with _lock:
|
| 320 |
+
_status["cause"] = delivery
|
| 321 |
+
return
|
| 322 |
+
sent = True
|
| 323 |
+
with _lock:
|
| 324 |
+
_alerted = _clean_alerted({**_alerted, alarm["period"]: _now()})
|
| 325 |
+
save()
|
| 326 |
+
except Exception:
|
| 327 |
+
with _lock:
|
| 328 |
+
_status["cause"] = (
|
| 329 |
+
"Telegram alert sent; its marker remains pending in memory because storage is unavailable."
|
| 330 |
+
if sent else "Sample history could not be stored; alerts deferred until storage is available.")
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def start_sampler(*, load=None, persist=None, sender=None):
|
| 334 |
+
"""Off by default; at most one sampler per process, with a one-minute floor."""
|
| 335 |
+
global _thread
|
| 336 |
+
try:
|
| 337 |
+
every_s = int(os.environ.get("AIOS_DO_SAMPLE_SECONDS", "0"))
|
| 338 |
+
except ValueError:
|
| 339 |
+
every_s = 0
|
| 340 |
+
if every_s <= 0 or load is None or persist is None:
|
| 341 |
+
with _lock:
|
| 342 |
+
_status.update(every_s=0, cause="Scheduled sampling is disabled; AIOS_DO_SAMPLE_SECONDS is unset, invalid or zero, or storage callbacks are unavailable.")
|
| 343 |
+
return False
|
| 344 |
+
every_s = max(60, every_s)
|
| 345 |
+
with _lock:
|
| 346 |
+
if _thread is not None and _thread.is_alive():
|
| 347 |
+
return True
|
| 348 |
+
_status.update(every_s=every_s, cause="Waiting for the first scheduled sample.")
|
| 349 |
+
_stop.clear()
|
| 350 |
+
|
| 351 |
+
def run():
|
| 352 |
+
while not _stop.is_set():
|
| 353 |
+
try:
|
| 354 |
+
_sample_once(load, persist, sender=sender)
|
| 355 |
+
except Exception:
|
| 356 |
+
with _lock:
|
| 357 |
+
_status["cause"] = "The scheduled sample could not be completed; it will retry."
|
| 358 |
+
if _stop.wait(every_s):
|
| 359 |
+
break
|
| 360 |
+
|
| 361 |
+
_thread = threading.Thread(target=run, name="do-usage-sampler", daemon=True)
|
| 362 |
+
_thread.start()
|
| 363 |
+
return True
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
def stop_sampler():
|
| 367 |
+
_stop.set()
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def _selftest():
|
| 371 |
+
"""Run without credentials, sockets, runtimes, files or a real Telegram sender."""
|
| 372 |
+
import sys
|
| 373 |
+
from unittest.mock import patch
|
| 374 |
+
from core import neon_usage
|
| 375 |
+
|
| 376 |
+
module = sys.modules[__name__]
|
| 377 |
+
at = "2026-09-25T12:00:00Z"
|
| 378 |
+
snapshot_at = "2026-09-25T00:00:00Z"
|
| 379 |
+
database = {"id": DEFAULT_CLUSTER_ID, "name": "fixture-cluster", "size": "db-s-1vcpu-1gb",
|
| 380 |
+
"num_nodes": 1, "region": "nyc3", "status": "online", "version": "18",
|
| 381 |
+
"storage_size_mib": 10240, "connection": {"password": "fixture-secret"}}
|
| 382 |
+
responses = {
|
| 383 |
+
"/databases/" + DEFAULT_CLUSTER_ID: {"database": database},
|
| 384 |
+
"/customers/my/balance": {"month_to_date_usage": "20.00", "account_balance": "-1.00",
|
| 385 |
+
"month_to_date_balance": "19.00", "generated_at": snapshot_at},
|
| 386 |
+
"/customers/my/billing_history": {"billing_history": []},
|
| 387 |
+
}
|
| 388 |
+
calls, sends, documents = [], [], []
|
| 389 |
+
|
| 390 |
+
def fake_get(path, token):
|
| 391 |
+
calls.append(path)
|
| 392 |
+
return copy.deepcopy(responses[path])
|
| 393 |
+
|
| 394 |
+
def fake_sender(message):
|
| 395 |
+
sends.append(message)
|
| 396 |
+
return True, len(sends)
|
| 397 |
+
|
| 398 |
+
def persist(document):
|
| 399 |
+
documents.append(copy.deepcopy(document))
|
| 400 |
+
|
| 401 |
+
def reset():
|
| 402 |
+
global _loaded, _samples, _alerted, _cluster_id
|
| 403 |
+
_loaded, _samples, _alerted, _cluster_id = False, [], {}, None
|
| 404 |
+
sends.clear()
|
| 405 |
+
documents.clear()
|
| 406 |
+
|
| 407 |
+
# Double protection: all requests are fake, including if a test forgets sender=.
|
| 408 |
+
with patch.dict(os.environ, {}, clear=True), patch.object(module, "_get", side_effect=fake_get), \
|
| 409 |
+
patch.object(module, "send_telegram", side_effect=fake_sender), \
|
| 410 |
+
patch.object(neon_usage, "send_telegram", side_effect=fake_sender), \
|
| 411 |
+
patch.object(module, "_now", return_value=at):
|
| 412 |
+
missing = read_account()
|
| 413 |
+
assert not missing["available"] and missing["cause"] and not calls
|
| 414 |
+
assert missing["balance"]["month_to_date_usage"] is None
|
| 415 |
+
assert missing["fixed_price"]["monthly_usd"] is None
|
| 416 |
+
assert not start_sampler(load=lambda: {}, persist=persist, sender=fake_sender)
|
| 417 |
+
assert "disabled" in report()["sampler"]["cause"]
|
| 418 |
+
os.environ["DIGITALOCEAN_TOKEN"] = "fixture-fallback"
|
| 419 |
+
account = read_account()
|
| 420 |
+
assert account["fixed_price"]["monthly_usd"] == 15.15
|
| 421 |
+
assert account["balance"]["account_balance"] == -1
|
| 422 |
+
assert account["billing"]["available"] and account["billing_history"] == []
|
| 423 |
+
# The real empty history omits the key: {"links": {}, "meta": {}} is empty, not unreadable.
|
| 424 |
+
responses["/customers/my/billing_history"] = {"links": {}, "meta": {}}
|
| 425 |
+
enveloped = read_account()
|
| 426 |
+
assert enveloped["billing"]["available"] and enveloped["billing_history"] == []
|
| 427 |
+
responses["/customers/my/billing_history"] = {"links": {}}
|
| 428 |
+
assert not read_account()["billing"]["available"]
|
| 429 |
+
responses["/customers/my/billing_history"] = {"billing_history": []}
|
| 430 |
+
assert "fixture-secret" not in json.dumps(account)
|
| 431 |
+
assert account["balance"]["source"] == BALANCE_SOURCE and account["balance"]["generated_at"] == snapshot_at
|
| 432 |
+
os.environ["AIOS_DO_USAGE_TOKEN"] = "fixture-preferred"
|
| 433 |
+
def preferred_get(path, token):
|
| 434 |
+
assert token == "fixture-preferred"
|
| 435 |
+
return fake_get(path, token)
|
| 436 |
+
with patch.object(module, "_get", side_effect=preferred_get):
|
| 437 |
+
assert read_account()["available"]
|
| 438 |
+
|
| 439 |
+
for failed_path, failed_block in (("/databases/" + DEFAULT_CLUSTER_ID, "cluster"),
|
| 440 |
+
("/customers/my/balance", "balance"),
|
| 441 |
+
("/customers/my/billing_history", "billing")):
|
| 442 |
+
def fail_one(path, token):
|
| 443 |
+
if path == failed_path:
|
| 444 |
+
raise UsageReadError("DigitalOcean API returned HTTP 403.")
|
| 445 |
+
return fake_get(path, token)
|
| 446 |
+
with patch.object(module, "_get", side_effect=fail_one):
|
| 447 |
+
partial = read_account()
|
| 448 |
+
assert partial["available"] and not partial[failed_block]["available"]
|
| 449 |
+
assert "403" in partial[failed_block]["cause"]
|
| 450 |
+
assert all(partial[key]["available"] for key in ("cluster", "balance", "billing") if key != failed_block)
|
| 451 |
+
if failed_block == "balance":
|
| 452 |
+
assert partial["balance"]["month_to_date_usage"] is None
|
| 453 |
+
assert evaluate_alarm(partial)[0] is None and evaluate_alarm(partial)[1]
|
| 454 |
+
_sample_once(lambda: {}, persist, sender=fake_sender)
|
| 455 |
+
assert not sends and "403" in snapshot()[1]["cause"]
|
| 456 |
+
|
| 457 |
+
original_balance = copy.deepcopy(responses["/customers/my/balance"])
|
| 458 |
+
for invalid in (None, True, "NaN", "Infinity", "unreadable"):
|
| 459 |
+
responses["/customers/my/balance"]["month_to_date_usage"] = invalid
|
| 460 |
+
malformed = read_account()
|
| 461 |
+
assert not malformed["balance"]["available"] and malformed["balance"]["cause"]
|
| 462 |
+
assert malformed["balance"]["month_to_date_usage"] is None
|
| 463 |
+
assert evaluate_alarm(malformed)[0] is None
|
| 464 |
+
responses["/customers/my/balance"] = original_balance
|
| 465 |
+
|
| 466 |
+
database["size"] = "unknown-size"
|
| 467 |
+
unknown = read_account()
|
| 468 |
+
assert unknown["fixed_price"]["monthly_usd"] is None
|
| 469 |
+
assert "UNKNOWN" in unknown["fixed_price"]["cause"] and "unknown-size" in unknown["fixed_price"]["cause"]
|
| 470 |
+
assert evaluate_alarm(unknown)[0] is None
|
| 471 |
+
database["size"] = "db-s-1vcpu-1gb"
|
| 472 |
+
database["num_nodes"] = 2
|
| 473 |
+
assert read_account()["fixed_price"]["monthly_usd"] == 30.30
|
| 474 |
+
database["num_nodes"] = 1
|
| 475 |
+
|
| 476 |
+
reset()
|
| 477 |
+
os.environ["AIOS_DO_ALARM_USD"] = "1"
|
| 478 |
+
_sample_once(lambda: {}, persist, sender=fake_sender)
|
| 479 |
+
_sample_once(lambda: documents[-1], persist, sender=fake_sender)
|
| 480 |
+
assert len(sends) == 1 and sends[0].startswith("TEST ")
|
| 481 |
+
assert all(word in sends[0] for word in ("20.00", "15.15", snapshot_at, "fixture-cluster", "R28"))
|
| 482 |
+
saved = copy.deepcopy(documents[-1])
|
| 483 |
+
reset()
|
| 484 |
+
_sample_once(lambda: saved, persist, sender=fake_sender)
|
| 485 |
+
assert not sends and "2026-09" in documents[-1]["alerted"]
|
| 486 |
+
responses["/customers/my/balance"]["generated_at"] = "2026-10-01T00:00:00Z"
|
| 487 |
+
_sample_once(lambda: saved, persist, sender=fake_sender)
|
| 488 |
+
assert len(sends) == 1 # A new billing month permits one new delivery.
|
| 489 |
+
responses["/customers/my/balance"]["generated_at"] = snapshot_at
|
| 490 |
+
|
| 491 |
+
reset()
|
| 492 |
+
del os.environ["AIOS_DO_ALARM_USD"]
|
| 493 |
+
responses["/customers/my/balance"]["month_to_date_usage"] = "15.15"
|
| 494 |
+
_sample_once(lambda: {}, persist, sender=fake_sender)
|
| 495 |
+
assert not sends
|
| 496 |
+
responses["/customers/my/balance"]["month_to_date_usage"] = "0.00"
|
| 497 |
+
zero = report()
|
| 498 |
+
assert zero["balance"]["month_to_date_usage"] == 0 and not zero["alarm"]
|
| 499 |
+
_sample_once(lambda: {}, persist, sender=fake_sender)
|
| 500 |
+
assert not sends
|
| 501 |
+
os.environ["AIOS_DO_ALARM_USD"] = "-1"
|
| 502 |
+
_sample_once(lambda: {}, persist, sender=fake_sender)
|
| 503 |
+
assert len(sends) == 1 and sends[0].startswith("TEST ")
|
| 504 |
+
os.environ["AIOS_DO_ALARM_USD"] = "NaN"
|
| 505 |
+
assert report()["alarm"] is None and "invalid" in report()["alarm_cause"]
|
| 506 |
+
|
| 507 |
+
reset()
|
| 508 |
+
os.environ["AIOS_DO_ALARM_USD"] = "-1"
|
| 509 |
+
os.environ["STORE_BACKEND"] = "pg"
|
| 510 |
+
os.environ["DATABASE_URL"] = "postgresql://fixture@ep-fixture.neon.tech/app"
|
| 511 |
+
with patch.object(neon_usage, "read_project", return_value={"available": True, "id": "fixture"}), \
|
| 512 |
+
patch.object(neon_usage, "_get", return_value={"endpoints": [
|
| 513 |
+
{"id": "ep-fixture", "current_state": "idle"}]}):
|
| 514 |
+
def forbidden(*args):
|
| 515 |
+
raise AssertionError("Idle compute must never call storage")
|
| 516 |
+
_sample_once(forbidden, forbidden, sender=fake_sender)
|
| 517 |
+
assert not sends and "idle" in snapshot()[1]["cause"]
|
| 518 |
+
assert report()["balance"]["available"]
|
| 519 |
+
del os.environ["STORE_BACKEND"]
|
| 520 |
+
del os.environ["DATABASE_URL"]
|
| 521 |
+
with patch.object(neon_usage, "read_project", side_effect=AssertionError("Unexpected Neon read")):
|
| 522 |
+
assert store_is_awake(backend="pg", database_url="postgresql://fixture@fixture.db.ondigitalocean.com/app")[0]
|
| 523 |
+
assert not store_is_awake(backend="pg", database_url="")[0]
|
| 524 |
+
assert not store_is_awake({"available": False, "cause": "offline"}, backend="pg",
|
| 525 |
+
database_url="postgresql://fixture@ep-fixture.neon.tech/app")[0]
|
| 526 |
+
reset()
|
| 527 |
+
def failed_load():
|
| 528 |
+
raise RuntimeError("fixture-secret")
|
| 529 |
+
_sample_once(failed_load, persist, sender=fake_sender)
|
| 530 |
+
assert not sends and "loaded" in snapshot()[1]["cause"]
|
| 531 |
+
reset()
|
| 532 |
+
def failed_persist(document):
|
| 533 |
+
raise RuntimeError("fixture-secret")
|
| 534 |
+
_sample_once(lambda: {}, failed_persist, sender=fake_sender)
|
| 535 |
+
assert not sends and "stored" in snapshot()[1]["cause"]
|
| 536 |
+
assert "fixture-secret" not in json.dumps(report())
|
| 537 |
+
reset()
|
| 538 |
+
def fail_after_delivery(document):
|
| 539 |
+
if sends:
|
| 540 |
+
raise RuntimeError("fixture-secret")
|
| 541 |
+
persist(document)
|
| 542 |
+
_sample_once(lambda: {}, fail_after_delivery, sender=fake_sender)
|
| 543 |
+
assert len(sends) == 1 and "pending in memory" in snapshot()[1]["cause"]
|
| 544 |
+
_sample_once(lambda: {}, persist, sender=fake_sender)
|
| 545 |
+
assert len(sends) == 1 and "2026-09" in documents[-1]["alerted"]
|
| 546 |
+
rows = [{"at": f"2026-09-{1 + n // 24:02}T{n % 24:02}:00:00Z", "generated_at": snapshot_at,
|
| 547 |
+
"month_to_date_usage": n} for n in range(MAX_SAMPLES + 1)]
|
| 548 |
+
assert len(_clean_samples(rows)) == MAX_SAMPLES
|
| 549 |
+
|
| 550 |
+
# Exercise the actual HTTP boundary with a fake opener, including credential-bearing errors.
|
| 551 |
+
class Response:
|
| 552 |
+
def __enter__(self):
|
| 553 |
+
return self
|
| 554 |
+
def __exit__(self, *args):
|
| 555 |
+
pass
|
| 556 |
+
def read(self, limit):
|
| 557 |
+
return b'{"ok": true}'
|
| 558 |
+
|
| 559 |
+
class Opener:
|
| 560 |
+
def open(self, request, *, timeout):
|
| 561 |
+
assert request.get_method() == "GET" and timeout == 5
|
| 562 |
+
return Response()
|
| 563 |
+
|
| 564 |
+
with patch.object(module, "build_opener", return_value=Opener()) as opener:
|
| 565 |
+
assert _get("/fixture", "fixture-token") == {"ok": True}
|
| 566 |
+
assert isinstance(opener.call_args.args[0], _NoRedirect)
|
| 567 |
+
assert opener.call_args.args[0].redirect_request(None, None, 302, None, None, None) is None
|
| 568 |
+
for error in (HTTPError("fixture-private-url", 403, "fixture-token", None, None),
|
| 569 |
+
URLError("fixture-private-url"), TimeoutError("fixture-token"), ValueError("fixture-token")):
|
| 570 |
+
with patch.object(module, "build_opener", side_effect=error):
|
| 571 |
+
try:
|
| 572 |
+
_get("/fixture", "fixture-token")
|
| 573 |
+
raise AssertionError("The HTTP error must fail closed")
|
| 574 |
+
except UsageReadError as exc:
|
| 575 |
+
assert "fixture-" not in str(exc)
|
| 576 |
+
print("PASS DigitalOcean usage selftest: isolated API failures, pricing, snapshot provenance, monthly alerts, restart markers, idle-store guard and safe HTTP errors; fake senders only.")
|
| 577 |
+
|
| 578 |
+
|
| 579 |
+
if __name__ == "__main__":
|
| 580 |
+
import sys
|
| 581 |
+
if sys.argv[1:] != ["--selftest"]:
|
| 582 |
+
raise SystemExit("Usage: python -m core.do_usage --selftest")
|
| 583 |
+
_selftest()
|
platform/core/external_tools.py
CHANGED
|
@@ -9,7 +9,7 @@ import json
|
|
| 9 |
from dataclasses import dataclass
|
| 10 |
from typing import Callable, Optional
|
| 11 |
|
| 12 |
-
from core import sheets, sheet_structure, sheet_templates, write_execution
|
| 13 |
|
| 14 |
|
| 15 |
@dataclass(frozen=True)
|
|
@@ -42,12 +42,19 @@ _STRING = {"type": "string"}
|
|
| 42 |
_ADDRESS = {"type": "string", "description": "An A1 cell address on the named tab."}
|
| 43 |
_FORMAT = _object({"kind": _STRING, "code": _STRING, "decimals": {"type": "integer"},
|
| 44 |
"currency": _STRING, "thousands": {"type": "boolean"}}, ("kind",))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
_STYLE = _object({
|
| 46 |
**{key: {"type": "boolean"} for key in ("bold", "italic", "underline", "strike", "wrap")},
|
| 47 |
**{key: _STRING for key in ("fill", "color", "font", "align", "valign")},
|
| 48 |
"size": {"type": ["number", "string"]}, "indent": {"type": "integer"},
|
| 49 |
"textRotation": {"type": "number"}, "format": _FORMAT,
|
| 50 |
-
"border": _object({side:
|
| 51 |
})
|
| 52 |
_VALIDATION = _object({"list": {"type": "array", "items": _STRING}}, ("list",))
|
| 53 |
_FILTER = _object({"conj": {"enum": ["and", "or"]}, "nodes": {"type": "array", "items": _object({
|
|
@@ -62,15 +69,25 @@ _BINDING = _object({
|
|
| 62 |
"period": _object({"unit": _STRING, "start": _STRING}),
|
| 63 |
"ending": {"type": "object"},
|
| 64 |
}, ("database", "aggregate", "scope"))
|
| 65 |
-
_CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]},
|
| 66 |
-
"value": {}, "text": _STRING, "formula": _STRING, "state": _STRING,
|
| 67 |
-
"style": _STYLE, "validation": _VALIDATION, "binding": _BINDING}, ("kind",))
|
| 68 |
_RANGE = {"oneOf": [_STRING, _object({key: {"type": "integer", "minimum": 0}
|
| 69 |
for key in ("col0", "row0", "col1", "row1")},
|
| 70 |
("col0", "row0", "col1", "row1"))]}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
_RULE = _object({"id": _STRING, "range": _RANGE, "kind": _STRING, "op": _STRING,
|
| 72 |
"value": {}, "value2": {}, "formula": _STRING, "style": _STYLE,
|
| 73 |
-
"scale": {"
|
|
|
|
|
|
|
| 74 |
_CHART = _object({"id": _STRING, "kind": _STRING, "range": _RANGE, "title": _STRING,
|
| 75 |
"stacking": _STRING, "source": {"type": "object"},
|
| 76 |
"series": {"type": "array", "items": {"type": "object"}},
|
|
@@ -166,6 +183,16 @@ def _add_tab(ctx, args):
|
|
| 166 |
return _put(ctx, args, book, tab["id"])
|
| 167 |
|
| 168 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
def _save_template(ctx, args):
|
| 170 |
book = ctx.project_book(_book(ctx, args))
|
| 171 |
record = sheet_templates.save_sheet_template(
|
|
@@ -269,7 +296,7 @@ def _structure(ctx, args):
|
|
| 269 |
return _put(ctx, args, shifted, tab["id"])
|
| 270 |
|
| 271 |
|
| 272 |
-
def _tool(description, schema, handler, *, write=False, grants=("
|
| 273 |
return {"description": description, "inputSchema": schema, "write": write,
|
| 274 |
"grants": grants, "handler": handler}
|
| 275 |
|
|
@@ -328,6 +355,9 @@ TOOLS = {
|
|
| 328 |
_object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
|
| 329 |
"sheet_add_tab": _tool("Add a blank tab to a workbook.",
|
| 330 |
_schema({"name": _STRING}, ("name",), tab=False), _add_tab, write=True),
|
|
|
|
|
|
|
|
|
|
| 331 |
"sheet_read_cells": _tool("Read a tab with styles, geometry and figures under the maker's grants.",
|
| 332 |
_schema({"addresses": {"type": "array", "items": _ADDRESS}}), _read_cells),
|
| 333 |
"sheet_write_cells": _tool("Write cell documents with values or formulas; null deletes a cell.",
|
|
@@ -367,10 +397,10 @@ TOOLS = {
|
|
| 367 |
("workbook_id", "key", "name", "description"), additionalProperties=False),
|
| 368 |
_save_template, write=True),
|
| 369 |
"list_databases": _tool("List the databases permitted or visible to the key's maker.",
|
| 370 |
-
_object(additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("
|
| 371 |
"describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
|
| 372 |
_object({"database": _STRING}, ("database",), additionalProperties=False),
|
| 373 |
-
lambda ctx, args: ctx.describe_database(args["database"]), grants=("
|
| 374 |
"query_database": _tool(
|
| 375 |
"Query currently permitted records with bounded filters, stable sorts, cursor pagination and aggregates.",
|
| 376 |
_object({
|
|
@@ -389,7 +419,7 @@ TOOLS = {
|
|
| 389 |
}, ("database",), additionalProperties=False),
|
| 390 |
lambda ctx, args: ctx.query_database(args["database"], {
|
| 391 |
key: value for key, value in args.items() if key != "database"
|
| 392 |
-
}), grants=("
|
| 393 |
"view_upsert": _tool(
|
| 394 |
"Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
|
| 395 |
_object({"database": _STRING, "view": _object({
|
|
@@ -398,7 +428,7 @@ TOOLS = {
|
|
| 398 |
"config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"),
|
| 399 |
additionalProperties=False)},
|
| 400 |
("database", "view"), additionalProperties=False),
|
| 401 |
-
lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("
|
| 402 |
"view_share_preview": _tool(
|
| 403 |
"Preview a personal draft View's C8 audience and field inheritance without sharing or mutating it.",
|
| 404 |
_object({
|
|
@@ -408,21 +438,21 @@ TOOLS = {
|
|
| 408 |
"user": _STRING, "role": {"enum": ["view", "edit"]},
|
| 409 |
}, ("user", "role"), additionalProperties=False)},
|
| 410 |
}, ("database", "view_id", "entries"), additionalProperties=False),
|
| 411 |
-
lambda ctx, args: ctx.view_share_preview(args), grants=("
|
| 412 |
"view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
|
| 413 |
_object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
|
| 414 |
additionalProperties=False),
|
| 415 |
-
lambda ctx, args: ctx.view_event("view_delete", args), write=True, grants=("
|
| 416 |
"app_upsert": _tool(
|
| 417 |
"Create an App with name, type, compatibleDatabases and config, or update it by app_id. Identity fields are immutable.",
|
| 418 |
_object(_APPLICATION, additionalProperties=False,
|
| 419 |
anyOf=[{"required": ["app_id"]}, {"required": ["name", "type", "config"]},
|
| 420 |
{"required": ["name", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"]}]),
|
| 421 |
-
lambda ctx, args: ctx.app_upsert(args), write=True, grants=("
|
| 422 |
"app_delete": _tool("Delete an App. confirmName must match its current name exactly.",
|
| 423 |
_object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
|
| 424 |
additionalProperties=False),
|
| 425 |
-
lambda ctx, args: ctx.app_delete(args), write=True, grants=("
|
| 426 |
"database_object": _tool(
|
| 427 |
"Discover/query a database or preview-confirm record and field CRUD through its live UI walls.",
|
| 428 |
_object({
|
|
@@ -435,7 +465,7 @@ TOOLS = {
|
|
| 435 |
"query": {"type": "object"},
|
| 436 |
}, ("action",), additionalProperties=False),
|
| 437 |
lambda ctx, args: _callback(ctx, "database_object", args), write=True,
|
| 438 |
-
grants=("
|
| 439 |
"view_object": _tool(
|
| 440 |
"Read or preview-confirm a personal View operation through the ordinary grid writer.",
|
| 441 |
_object({
|
|
@@ -452,7 +482,7 @@ TOOLS = {
|
|
| 452 |
ctx.view_share_preview({"database": args["database"],
|
| 453 |
"view_id": args["view_id"],
|
| 454 |
"entries": args.get("entries") or []})
|
| 455 |
-
), write=True, grants=("
|
| 456 |
**{name: _object_tool(name, actions)
|
| 457 |
for name, actions in _SHEET_OBJECT_ACTIONS.items()},
|
| 458 |
"validated_batch": _tool(
|
|
@@ -464,12 +494,12 @@ TOOLS = {
|
|
| 464 |
additionalProperties=False)}},
|
| 465 |
("operations",), additionalProperties=False),
|
| 466 |
lambda ctx, args: _callback(ctx, "validated_batch", args), write=True,
|
| 467 |
-
grants=("
|
| 468 |
"confirm_change": _tool(
|
| 469 |
"Confirm one unexpired preview. Destructive/schema changes require its exact typed phrase.",
|
| 470 |
_object({"confirmation_token": _STRING, "confirmation_phrase": _STRING},
|
| 471 |
("confirmation_token",), additionalProperties=False),
|
| 472 |
-
lambda ctx, args: _callback(ctx, "confirm_change", args), grants=("
|
| 473 |
}
|
| 474 |
|
| 475 |
|
|
@@ -506,7 +536,12 @@ def dispatch(ctx, name, arguments):
|
|
| 506 |
if tool is None:
|
| 507 |
raise sheets.SheetRefusal("tool_not_found", "The requested tool is not available.")
|
| 508 |
for grant in tool["grants"]:
|
| 509 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 510 |
schema = tool["inputSchema"]
|
| 511 |
if not isinstance(arguments, dict):
|
| 512 |
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.")
|
|
|
|
| 9 |
from dataclasses import dataclass
|
| 10 |
from typing import Callable, Optional
|
| 11 |
|
| 12 |
+
from core import perm_scope, sheets, sheet_structure, sheet_templates, write_execution
|
| 13 |
|
| 14 |
|
| 15 |
@dataclass(frozen=True)
|
|
|
|
| 42 |
_ADDRESS = {"type": "string", "description": "An A1 cell address on the named tab."}
|
| 43 |
_FORMAT = _object({"kind": _STRING, "code": _STRING, "decimals": {"type": "integer"},
|
| 44 |
"currency": _STRING, "thousands": {"type": "boolean"}}, ("kind",))
|
| 45 |
+
# S67-T14 / contract C2: a border side is the legacy `true` (read as a thin black line) or
|
| 46 |
+
# `{style, color}` with one of Excel's seven line styles and a six digit hex colour; `false` or an
|
| 47 |
+
# absent side is no side. `sheets.py::_clean_border_side` refuses anything else by code, and this
|
| 48 |
+
# publishes the same shape so an AI client can draw a styled border at all.
|
| 49 |
+
_BORDER_SIDE = {"oneOf": [{"type": "boolean"}, _object({
|
| 50 |
+
"style": {"enum": list(sheets.CELL_BORDER_STYLES)},
|
| 51 |
+
"color": {"type": "string", "pattern": "^#[0-9A-Fa-f]{6}$"}}, ("style", "color"))]}
|
| 52 |
_STYLE = _object({
|
| 53 |
**{key: {"type": "boolean"} for key in ("bold", "italic", "underline", "strike", "wrap")},
|
| 54 |
**{key: _STRING for key in ("fill", "color", "font", "align", "valign")},
|
| 55 |
"size": {"type": ["number", "string"]}, "indent": {"type": "integer"},
|
| 56 |
"textRotation": {"type": "number"}, "format": _FORMAT,
|
| 57 |
+
"border": _object({side: _BORDER_SIDE for side in ("top", "right", "bottom", "left")}),
|
| 58 |
})
|
| 59 |
_VALIDATION = _object({"list": {"type": "array", "items": _STRING}}, ("list",))
|
| 60 |
_FILTER = _object({"conj": {"enum": ["and", "or"]}, "nodes": {"type": "array", "items": _object({
|
|
|
|
| 69 |
"period": _object({"unit": _STRING, "start": _STRING}),
|
| 70 |
"ending": {"type": "object"},
|
| 71 |
}, ("database", "aggregate", "scope"))
|
|
|
|
|
|
|
|
|
|
| 72 |
_RANGE = {"oneOf": [_STRING, _object({key: {"type": "integer", "minimum": 0}
|
| 73 |
for key in ("col0", "row0", "col1", "row1")},
|
| 74 |
("col0", "row0", "col1", "row1"))]}
|
| 75 |
+
# S67-T18: an in-cell sparkline, `{kind, range}` exactly as `sheets.py::_clean_sparkline` stores
|
| 76 |
+
# it; the range is a rectangle on the SAME tab (an A1 range such as "B2:M2", or the dict).
|
| 77 |
+
_SPARKLINE = _object({"kind": {"enum": list(sheets.SPARKLINE_KINDS)}, "range": _RANGE},
|
| 78 |
+
("kind", "range"))
|
| 79 |
+
_CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]},
|
| 80 |
+
"value": {}, "text": _STRING, "formula": _STRING, "state": _STRING,
|
| 81 |
+
"style": _STYLE, "validation": _VALIDATION, "binding": _BINDING,
|
| 82 |
+
"sparkline": _SPARKLINE}, ("kind",))
|
| 83 |
+
# S67-T13: a colour scale's two ends and what each end means. `sheets.py::_clean_rules` refuses
|
| 84 |
+
# a type outside `SHEET_RULE_SCALE_TYPES`; the enum publishes the same list to the caller.
|
| 85 |
+
_SCALE_TYPE = {"enum": list(sheets.SHEET_RULE_SCALE_TYPES)}
|
| 86 |
_RULE = _object({"id": _STRING, "range": _RANGE, "kind": _STRING, "op": _STRING,
|
| 87 |
"value": {}, "value2": {}, "formula": _STRING, "style": _STYLE,
|
| 88 |
+
"scale": _object({"low": _STRING, "high": _STRING,
|
| 89 |
+
"minType": _SCALE_TYPE, "maxType": _SCALE_TYPE})},
|
| 90 |
+
("range", "op"))
|
| 91 |
_CHART = _object({"id": _STRING, "kind": _STRING, "range": _RANGE, "title": _STRING,
|
| 92 |
"stacking": _STRING, "source": {"type": "object"},
|
| 93 |
"series": {"type": "array", "items": {"type": "object"}},
|
|
|
|
| 183 |
return _put(ctx, args, book, tab["id"])
|
| 184 |
|
| 185 |
|
| 186 |
+
def _delete_tab(ctx, args):
|
| 187 |
+
book = _book(ctx, args, write=True)
|
| 188 |
+
tab = sheets.find_tab(book, args["tab"])
|
| 189 |
+
if len(book["sheets"]) <= 1:
|
| 190 |
+
raise sheets.SheetRefusal(
|
| 191 |
+
"tab_last", "A workbook keeps at least one sheet, so this one cannot be deleted.")
|
| 192 |
+
book["sheets"] = [row for row in book["sheets"] if row["id"] != tab["id"]]
|
| 193 |
+
return _put(ctx, args, book)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
def _save_template(ctx, args):
|
| 197 |
book = ctx.project_book(_book(ctx, args))
|
| 198 |
record = sheet_templates.save_sheet_template(
|
|
|
|
| 296 |
return _put(ctx, args, shifted, tab["id"])
|
| 297 |
|
| 298 |
|
| 299 |
+
def _tool(description, schema, handler, *, write=False, grants=("ai_apps", "sheets")):
|
| 300 |
return {"description": description, "inputSchema": schema, "write": write,
|
| 301 |
"grants": grants, "handler": handler}
|
| 302 |
|
|
|
|
| 355 |
_object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
|
| 356 |
"sheet_add_tab": _tool("Add a blank tab to a workbook.",
|
| 357 |
_schema({"name": _STRING}, ("name",), tab=False), _add_tab, write=True),
|
| 358 |
+
"sheet_delete_tab": _tool(
|
| 359 |
+
"Preview deleting a tab by id or name; requires typed confirmation. The last tab cannot be deleted.",
|
| 360 |
+
_schema({}), _delete_tab, write=True),
|
| 361 |
"sheet_read_cells": _tool("Read a tab with styles, geometry and figures under the maker's grants.",
|
| 362 |
_schema({"addresses": {"type": "array", "items": _ADDRESS}}), _read_cells),
|
| 363 |
"sheet_write_cells": _tool("Write cell documents with values or formulas; null deletes a cell.",
|
|
|
|
| 397 |
("workbook_id", "key", "name", "description"), additionalProperties=False),
|
| 398 |
_save_template, write=True),
|
| 399 |
"list_databases": _tool("List the databases permitted or visible to the key's maker.",
|
| 400 |
+
_object(additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("ai_apps",)),
|
| 401 |
"describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
|
| 402 |
_object({"database": _STRING}, ("database",), additionalProperties=False),
|
| 403 |
+
lambda ctx, args: ctx.describe_database(args["database"]), grants=("ai_apps",)),
|
| 404 |
"query_database": _tool(
|
| 405 |
"Query currently permitted records with bounded filters, stable sorts, cursor pagination and aggregates.",
|
| 406 |
_object({
|
|
|
|
| 419 |
}, ("database",), additionalProperties=False),
|
| 420 |
lambda ctx, args: ctx.query_database(args["database"], {
|
| 421 |
key: value for key, value in args.items() if key != "database"
|
| 422 |
+
}), grants=("ai_apps",)),
|
| 423 |
"view_upsert": _tool(
|
| 424 |
"Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
|
| 425 |
_object({"database": _STRING, "view": _object({
|
|
|
|
| 428 |
"config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"),
|
| 429 |
additionalProperties=False)},
|
| 430 |
("database", "view"), additionalProperties=False),
|
| 431 |
+
lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("ai_apps",)),
|
| 432 |
"view_share_preview": _tool(
|
| 433 |
"Preview a personal draft View's C8 audience and field inheritance without sharing or mutating it.",
|
| 434 |
_object({
|
|
|
|
| 438 |
"user": _STRING, "role": {"enum": ["view", "edit"]},
|
| 439 |
}, ("user", "role"), additionalProperties=False)},
|
| 440 |
}, ("database", "view_id", "entries"), additionalProperties=False),
|
| 441 |
+
lambda ctx, args: ctx.view_share_preview(args), grants=("ai_apps",)),
|
| 442 |
"view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
|
| 443 |
_object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
|
| 444 |
additionalProperties=False),
|
| 445 |
+
lambda ctx, args: ctx.view_event("view_delete", args), write=True, grants=("ai_apps",)),
|
| 446 |
"app_upsert": _tool(
|
| 447 |
"Create an App with name, type, compatibleDatabases and config, or update it by app_id. Identity fields are immutable.",
|
| 448 |
_object(_APPLICATION, additionalProperties=False,
|
| 449 |
anyOf=[{"required": ["app_id"]}, {"required": ["name", "type", "config"]},
|
| 450 |
{"required": ["name", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"]}]),
|
| 451 |
+
lambda ctx, args: ctx.app_upsert(args), write=True, grants=("ai_apps",)),
|
| 452 |
"app_delete": _tool("Delete an App. confirmName must match its current name exactly.",
|
| 453 |
_object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
|
| 454 |
additionalProperties=False),
|
| 455 |
+
lambda ctx, args: ctx.app_delete(args), write=True, grants=("ai_apps",)),
|
| 456 |
"database_object": _tool(
|
| 457 |
"Discover/query a database or preview-confirm record and field CRUD through its live UI walls.",
|
| 458 |
_object({
|
|
|
|
| 465 |
"query": {"type": "object"},
|
| 466 |
}, ("action",), additionalProperties=False),
|
| 467 |
lambda ctx, args: _callback(ctx, "database_object", args), write=True,
|
| 468 |
+
grants=("ai_apps",)),
|
| 469 |
"view_object": _tool(
|
| 470 |
"Read or preview-confirm a personal View operation through the ordinary grid writer.",
|
| 471 |
_object({
|
|
|
|
| 482 |
ctx.view_share_preview({"database": args["database"],
|
| 483 |
"view_id": args["view_id"],
|
| 484 |
"entries": args.get("entries") or []})
|
| 485 |
+
), write=True, grants=("ai_apps",)),
|
| 486 |
**{name: _object_tool(name, actions)
|
| 487 |
for name, actions in _SHEET_OBJECT_ACTIONS.items()},
|
| 488 |
"validated_batch": _tool(
|
|
|
|
| 494 |
additionalProperties=False)}},
|
| 495 |
("operations",), additionalProperties=False),
|
| 496 |
lambda ctx, args: _callback(ctx, "validated_batch", args), write=True,
|
| 497 |
+
grants=("ai_apps",)),
|
| 498 |
"confirm_change": _tool(
|
| 499 |
"Confirm one unexpired preview. Destructive/schema changes require its exact typed phrase.",
|
| 500 |
_object({"confirmation_token": _STRING, "confirmation_phrase": _STRING},
|
| 501 |
("confirmation_token",), additionalProperties=False),
|
| 502 |
+
lambda ctx, args: _callback(ctx, "confirm_change", args), grants=("ai_apps",)),
|
| 503 |
}
|
| 504 |
|
| 505 |
|
|
|
|
| 536 |
if tool is None:
|
| 537 |
raise sheets.SheetRefusal("tool_not_found", "The requested tool is not available.")
|
| 538 |
for grant in tool["grants"]:
|
| 539 |
+
if grant == perm_scope.AI_APPS:
|
| 540 |
+
reason = perm_scope.ai_apps_refusal(ctx.session.user, ctx.session.runtime)
|
| 541 |
+
if reason:
|
| 542 |
+
raise sheets.SheetRefusal("ai_apps_off", reason)
|
| 543 |
+
else:
|
| 544 |
+
ctx.session.require(grant)
|
| 545 |
schema = tool["inputSchema"]
|
| 546 |
if not isinstance(arguments, dict):
|
| 547 |
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.")
|
platform/core/grid_events.py
CHANGED
|
@@ -632,6 +632,57 @@ def _grant_marker(ctx, ws):
|
|
| 632 |
return _has_grants
|
| 633 |
|
| 634 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 635 |
def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False):
|
| 636 |
"""Durable Airtable-style view/schema/overlay state, with an honest session fallback.
|
| 637 |
|
|
@@ -707,6 +758,8 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_do
|
|
| 707 |
merged.update(ws.get('views') or {}) # own views win the id collision
|
| 708 |
ws['views'] = {vid: scope_view(v, uname, allowed_pids)
|
| 709 |
for vid, v in merged.items()}
|
|
|
|
|
|
|
| 710 |
return ws
|
| 711 |
# Store down. Streamlit degrades to its captioned session workspace exactly as before; an
|
| 712 |
# API caller has no durable dict and must be told, not quietly served memory.
|
|
@@ -1584,7 +1637,13 @@ def handle_one(event, ctx):
|
|
| 1584 |
_refuse(ctx, 'invalid_application_ref',
|
| 1585 |
'that Application reference is not a valid typed identity', view_id)
|
| 1586 |
return False
|
| 1587 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1588 |
# Wave-6 item 10: a direct non-grid display mode rides the view config, validated
|
| 1589 |
# structurally (mode whitelist, refs must be this table's fields). A bound Application's
|
| 1590 |
# config remains in the registry and is resolved on read, so an offered clone is dropped.
|
|
|
|
| 632 |
return _has_grants
|
| 633 |
|
| 634 |
|
| 635 |
+
#: S67-T39 / R36: the Application registry's store key. READ here, never imported:
|
| 636 |
+
#: `application_registry` lives in the api layer and core never imports up (ARCHITECTURE).
|
| 637 |
+
_APPLICATIONS_KEY = 'applications_v1'
|
| 638 |
+
|
| 639 |
+
|
| 640 |
+
def _live_application_ids(ctx):
|
| 641 |
+
"""The set of App ids the tenant registry holds, or None when that is not KNOWN.
|
| 642 |
+
|
| 643 |
+
None (read failed, or no registry document with an `applications` map) means "strip
|
| 644 |
+
nothing": an unknown answer must never turn a live App binding into a plain view.
|
| 645 |
+
"""
|
| 646 |
+
try:
|
| 647 |
+
doc = _store_of(ctx).get(_APPLICATIONS_KEY)
|
| 648 |
+
except Exception as exc:
|
| 649 |
+
_tel.error('applications:live', exc)
|
| 650 |
+
return None
|
| 651 |
+
apps = doc.get('applications') if isinstance(doc, dict) else None
|
| 652 |
+
return {str(key) for key in apps} if isinstance(apps, dict) else None
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
def _application_ref_id(view):
|
| 656 |
+
config = view.get('config') if isinstance(view, dict) else None
|
| 657 |
+
ref = config.get('application') if isinstance(config, dict) else None
|
| 658 |
+
return str(ref.get('id') or '') if isinstance(ref, dict) else None
|
| 659 |
+
|
| 660 |
+
|
| 661 |
+
def _drop_dead_application_refs(ctx, views):
|
| 662 |
+
"""S67-T39 / R36 / D-2233: a View bound to a DELETED App is served as a plain view of its
|
| 663 |
+
database (the `application` member dropped, everything else kept), so it opens with no
|
| 664 |
+
error. Projection only, and it never raises: any failure serves the views unchanged.
|
| 665 |
+
The registry is read only when some View actually carries a reference.
|
| 666 |
+
"""
|
| 667 |
+
try:
|
| 668 |
+
if not any(_application_ref_id(v) is not None for v in (views or {}).values()):
|
| 669 |
+
return views
|
| 670 |
+
live = _live_application_ids(ctx)
|
| 671 |
+
if live is None:
|
| 672 |
+
return views
|
| 673 |
+
out = {}
|
| 674 |
+
for vid, view in views.items():
|
| 675 |
+
ref_id = _application_ref_id(view)
|
| 676 |
+
if ref_id is not None and ref_id not in live:
|
| 677 |
+
config = {k: v for k, v in view['config'].items() if k != 'application'}
|
| 678 |
+
view = {**view, 'config': config}
|
| 679 |
+
out[vid] = view
|
| 680 |
+
return out
|
| 681 |
+
except Exception as exc:
|
| 682 |
+
_tel.error('applications:dead_refs', exc)
|
| 683 |
+
return views
|
| 684 |
+
|
| 685 |
+
|
| 686 |
def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False):
|
| 687 |
"""Durable Airtable-style view/schema/overlay state, with an honest session fallback.
|
| 688 |
|
|
|
|
| 758 |
merged.update(ws.get('views') or {}) # own views win the id collision
|
| 759 |
ws['views'] = {vid: scope_view(v, uname, allowed_pids)
|
| 760 |
for vid, v in merged.items()}
|
| 761 |
+
if ws.get('views'):
|
| 762 |
+
ws['views'] = _drop_dead_application_refs(ctx, ws['views'])
|
| 763 |
return ws
|
| 764 |
# Store down. Streamlit degrades to its captioned session workspace exactly as before; an
|
| 765 |
# API caller has no durable dict and must be told, not quietly served memory.
|
|
|
|
| 1637 |
_refuse(ctx, 'invalid_application_ref',
|
| 1638 |
'that Application reference is not a valid typed identity', view_id)
|
| 1639 |
return False
|
| 1640 |
+
# S67-T39 / R36: a reference to a DELETED App is dropped, not stored, so the View
|
| 1641 |
+
# saves as a plain view of its database. Unknown registry state keeps the ref.
|
| 1642 |
+
_live = _live_application_ids(ctx)
|
| 1643 |
+
if _live is not None and application['id'] not in _live:
|
| 1644 |
+
application = None
|
| 1645 |
+
else:
|
| 1646 |
+
config['application'] = application
|
| 1647 |
# Wave-6 item 10: a direct non-grid display mode rides the view config, validated
|
| 1648 |
# structurally (mode whitelist, refs must be this table's fields). A bound Application's
|
| 1649 |
# config remains in the registry and is resolved on read, so an offered clone is dropped.
|
platform/core/neon_usage.py
ADDED
|
@@ -0,0 +1,590 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Neon management-plane usage. Importing or reading a card never opens a database.
|
| 2 |
+
|
| 3 |
+
Storage is injected by the API host and called only after an awake observation.
|
| 4 |
+
Unchanged counters are stale observations, not evidence of zero consumption.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import copy
|
| 9 |
+
import json
|
| 10 |
+
import math
|
| 11 |
+
import os
|
| 12 |
+
import re
|
| 13 |
+
import threading
|
| 14 |
+
from datetime import datetime, timezone
|
| 15 |
+
from urllib.error import HTTPError, URLError
|
| 16 |
+
from urllib.parse import urlencode, urlsplit
|
| 17 |
+
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
# Launch pricing: AutoWiki/research/neon-usage-samples/2026-09-25-sample1.json
|
| 21 |
+
# notes[2] and AutoWiki/research/neon-alternative-feasibility.md. The compute
|
| 22 |
+
# allowance is a meter/alert threshold; ALL CU-hours are billed, including it.
|
| 23 |
+
LAUNCH_PLAN = {
|
| 24 |
+
"allowance": {"transfer_gb": 500.0, "compute_hours": 300.0, "storage_gib": 10.0},
|
| 25 |
+
"usd": {"transfer_per_gb": 0.10, "compute_per_hour": 0.106,
|
| 26 |
+
"storage_per_gb_month": 0.35},
|
| 27 |
+
}
|
| 28 |
+
API_BASE = "https://console.neon.tech/api/v2"
|
| 29 |
+
HTTP_TIMEOUT = 5
|
| 30 |
+
DEFAULT_SAMPLE_SECONDS = 21600
|
| 31 |
+
MAX_SAMPLES = 240
|
| 32 |
+
DEFAULT_ALARM_SHARE = 0.70
|
| 33 |
+
STORE_KEY = "platform_neon_usage_samples"
|
| 34 |
+
_FIELDS = ("data_transfer_bytes", "compute_time_seconds", "synthetic_storage_size")
|
| 35 |
+
_METRIC_LABELS = {"transfer_gb": "Transfer", "compute_hours": "Compute", "storage_gib": "Storage"}
|
| 36 |
+
_ALERT_KEYS = {f"{metric}:{kind}" for metric in _METRIC_LABELS
|
| 37 |
+
for kind in ("share", "projected_overage")}
|
| 38 |
+
_guards = None
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def set_guards(guards):
|
| 42 |
+
"""The API host publishes its startup checks; this layer never imports the API."""
|
| 43 |
+
global _guards
|
| 44 |
+
_guards = copy.deepcopy(guards)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class UsageReadError(Exception):
|
| 48 |
+
"""Contains only a fixed, credential-free explanation suitable for the card."""
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class _NoRedirect(HTTPRedirectHandler):
|
| 52 |
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
| 53 |
+
return None
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _get(path, key):
|
| 57 |
+
request = Request(API_BASE + path, headers={"Authorization": "Bearer " + key}, method="GET")
|
| 58 |
+
try:
|
| 59 |
+
with build_opener(_NoRedirect()).open(request, timeout=HTTP_TIMEOUT) as response:
|
| 60 |
+
payload = json.loads(response.read(1_000_001))
|
| 61 |
+
if not isinstance(payload, dict):
|
| 62 |
+
raise ValueError()
|
| 63 |
+
return payload
|
| 64 |
+
except HTTPError as exc:
|
| 65 |
+
raise UsageReadError(f"Neon management API returned HTTP {exc.code}.") from None
|
| 66 |
+
except TimeoutError:
|
| 67 |
+
raise UsageReadError("Neon management API timed out.") from None
|
| 68 |
+
except URLError as exc:
|
| 69 |
+
cause = "timed out" if isinstance(exc.reason, TimeoutError) else "could not be reached"
|
| 70 |
+
raise UsageReadError(f"Neon management API {cause}.") from None
|
| 71 |
+
except (ValueError, OSError):
|
| 72 |
+
raise UsageReadError("Neon management API returned an unreadable response.") from None
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _identifier(value):
|
| 76 |
+
return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_-]{1,128}", value) is not None
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _pages(path, field, key, params=None):
|
| 80 |
+
"""Bound discovery, and refuse partial discovery rather than guessing uniqueness."""
|
| 81 |
+
params, seen = dict(params or {}), set()
|
| 82 |
+
for _ in range(10):
|
| 83 |
+
payload = _get(path + ("?" + urlencode(params) if params else ""), key)
|
| 84 |
+
rows = payload.get(field)
|
| 85 |
+
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
|
| 86 |
+
raise UsageReadError("Neon project discovery returned an unreadable response.")
|
| 87 |
+
# Neon echoes the last cursor even on the terminal empty project page.
|
| 88 |
+
if not rows:
|
| 89 |
+
return
|
| 90 |
+
yield from rows
|
| 91 |
+
cursor = (payload.get("pagination") or {}).get("cursor")
|
| 92 |
+
if not cursor:
|
| 93 |
+
return
|
| 94 |
+
if cursor in seen:
|
| 95 |
+
break
|
| 96 |
+
seen.add(cursor)
|
| 97 |
+
params["cursor"] = cursor
|
| 98 |
+
raise UsageReadError("Neon project discovery is incomplete; set NEON_PROJECT_ID.")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def resolve_project_id(api_key=None, project_id=None):
|
| 102 |
+
"""Explicit id first; otherwise exactly one project across the visible orgs."""
|
| 103 |
+
key = os.environ.get("NEON_API_KEY", "") if api_key is None else api_key
|
| 104 |
+
if not key:
|
| 105 |
+
raise UsageReadError("NEON_API_KEY is absent; Neon usage cannot be read.")
|
| 106 |
+
selected = os.environ.get("NEON_PROJECT_ID", "") if project_id is None else project_id
|
| 107 |
+
if selected:
|
| 108 |
+
if not _identifier(selected):
|
| 109 |
+
raise UsageReadError("NEON_PROJECT_ID is invalid.")
|
| 110 |
+
return selected
|
| 111 |
+
projects = set()
|
| 112 |
+
for org in _pages("/users/me/organizations", "organizations", key):
|
| 113 |
+
if not _identifier(org.get("id")):
|
| 114 |
+
raise UsageReadError("Neon organization discovery returned no usable id.")
|
| 115 |
+
for project in _pages("/projects", "projects", key, {"org_id": org["id"], "limit": 100}):
|
| 116 |
+
if not _identifier(project.get("id")):
|
| 117 |
+
raise UsageReadError("Neon project discovery returned no usable id.")
|
| 118 |
+
projects.add(project["id"])
|
| 119 |
+
if len(projects) > 1:
|
| 120 |
+
raise UsageReadError("Multiple Neon projects are visible; set NEON_PROJECT_ID.")
|
| 121 |
+
if len(projects) != 1:
|
| 122 |
+
raise UsageReadError("No Neon project is visible; set NEON_PROJECT_ID.")
|
| 123 |
+
return projects.pop()
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _now():
|
| 127 |
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _stamp(value):
|
| 131 |
+
try:
|
| 132 |
+
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
| 133 |
+
return parsed if parsed.tzinfo else None
|
| 134 |
+
except (ValueError, TypeError):
|
| 135 |
+
return None
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def _number(value):
|
| 139 |
+
return (isinstance(value, (int, float)) and not isinstance(value, bool)
|
| 140 |
+
and math.isfinite(value) and value >= 0)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def read_project(api_key=None, project_id=None):
|
| 144 |
+
"""Consumption comes ONLY from GET /projects/{id}, never branch sums or SQL."""
|
| 145 |
+
result = {"available": False, "cause": None, "read_at": _now(), "id": None, "name": None,
|
| 146 |
+
"consumption_period_start": None, "consumption_period_end": None,
|
| 147 |
+
**dict.fromkeys(_FIELDS)}
|
| 148 |
+
try:
|
| 149 |
+
key = os.environ.get("NEON_API_KEY", "") if api_key is None else api_key
|
| 150 |
+
selected = resolve_project_id(key, project_id)
|
| 151 |
+
project = _get("/projects/" + selected, key).get("project")
|
| 152 |
+
if not isinstance(project, dict) or any(not _number(project.get(k)) for k in _FIELDS):
|
| 153 |
+
raise UsageReadError("Neon project response is missing valid usage totals.")
|
| 154 |
+
start, end = (_stamp(project.get("consumption_period_" + k)) for k in ("start", "end"))
|
| 155 |
+
if not start or not end or end <= start:
|
| 156 |
+
raise UsageReadError("Neon project response is missing a valid billing period.")
|
| 157 |
+
if project.get("id") != selected:
|
| 158 |
+
raise UsageReadError("Neon project response did not match the requested project.")
|
| 159 |
+
result.update({k: project[k] for k in _FIELDS})
|
| 160 |
+
result.update(available=True, id=selected, name=project.get("name") or selected,
|
| 161 |
+
consumption_period_start=project["consumption_period_start"],
|
| 162 |
+
consumption_period_end=project["consumption_period_end"], read_at=_now())
|
| 163 |
+
except UsageReadError as exc:
|
| 164 |
+
result["cause"] = str(exc)
|
| 165 |
+
except Exception:
|
| 166 |
+
# Never return exception text: HTTP clients and credentials can occur in it.
|
| 167 |
+
result["cause"] = "Neon usage could not be read because of an unexpected response."
|
| 168 |
+
return result
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def _clean_samples(samples):
|
| 172 |
+
clean = {}
|
| 173 |
+
for item in samples if isinstance(samples, list) else []:
|
| 174 |
+
if not isinstance(item, dict) or not _stamp(item.get("at")) or not _stamp(item.get("period_start")):
|
| 175 |
+
continue
|
| 176 |
+
if any(not _number(item.get(k)) for k in ("transfer_bytes", "compute_seconds", "storage_bytes")):
|
| 177 |
+
continue
|
| 178 |
+
row = {k: item[k] for k in ("at", "transfer_bytes", "compute_seconds", "storage_bytes", "period_start")}
|
| 179 |
+
clean[(row["at"], row["period_start"])] = row
|
| 180 |
+
return sorted(clean.values(), key=lambda row: _stamp(row["at"]))[-MAX_SAMPLES:]
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def _rate(samples, field, divisor, *, gauge=False):
|
| 184 |
+
result = {"per_day": None, "cause": "Waiting for two samples with different values; Neon totals refresh slowly.",
|
| 185 |
+
"from_at": None, "to_at": None}
|
| 186 |
+
if len(samples) < 2:
|
| 187 |
+
return result
|
| 188 |
+
latest = samples[-1]
|
| 189 |
+
for prior in reversed(samples[:-1]):
|
| 190 |
+
days = (_stamp(latest["at"]) - _stamp(prior["at"])).total_seconds() / 86400
|
| 191 |
+
delta = latest[field] - prior[field]
|
| 192 |
+
if days <= 0 or delta == 0:
|
| 193 |
+
continue
|
| 194 |
+
if delta < 0 and not gauge:
|
| 195 |
+
result["cause"] = "The counter decreased; waiting for a new pair of increasing samples."
|
| 196 |
+
return result
|
| 197 |
+
return {"per_day": delta / divisor / days, "cause": None,
|
| 198 |
+
"from_at": prior["at"], "to_at": latest["at"]}
|
| 199 |
+
return result
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def _alarm_threshold():
|
| 203 |
+
raw = os.environ.get("AIOS_NEON_ALARM_SHARE")
|
| 204 |
+
if raw is None:
|
| 205 |
+
return DEFAULT_ALARM_SHARE, None
|
| 206 |
+
try:
|
| 207 |
+
threshold = float(raw)
|
| 208 |
+
if not math.isfinite(threshold) or not 0 < threshold <= 1.5:
|
| 209 |
+
raise ValueError()
|
| 210 |
+
return threshold, None
|
| 211 |
+
except ValueError:
|
| 212 |
+
return DEFAULT_ALARM_SHARE, "AIOS_NEON_ALARM_SHARE is invalid; using the default 70% threshold."
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _alarm_message(summary, crossings, test):
|
| 216 |
+
metrics = []
|
| 217 |
+
for metric, label in _METRIC_LABELS.items():
|
| 218 |
+
rows = [row for row in crossings if row["metric"] == metric]
|
| 219 |
+
if rows:
|
| 220 |
+
kinds = ", ".join("share limit" if row["kind"] == "share" else "projected overage"
|
| 221 |
+
for row in rows)
|
| 222 |
+
metrics.append(f"{label} {rows[0]['share']:.1%} used ({kinds})")
|
| 223 |
+
total = summary.get("projection", {}).get("total")
|
| 224 |
+
bill = f"USD {total:,.2f}" if _number(total) else "unavailable"
|
| 225 |
+
return ("TEST " if test else "") + "Neon usage: " + "; ".join(metrics) + f". Projected bill {bill}."
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def evaluate_alarm(summary, share_threshold=DEFAULT_ALARM_SHARE):
|
| 229 |
+
"""Pure threshold evaluation, independent of delivery or saved alert markers."""
|
| 230 |
+
if not summary.get("available"):
|
| 231 |
+
return None
|
| 232 |
+
if not _number(share_threshold) or not 0 < share_threshold <= 1.5:
|
| 233 |
+
share_threshold = DEFAULT_ALARM_SHARE
|
| 234 |
+
crossings = []
|
| 235 |
+
projection = summary.get("projection", {})
|
| 236 |
+
bill = projection.get("total")
|
| 237 |
+
for metric in _METRIC_LABELS:
|
| 238 |
+
share = summary.get("share", {}).get(metric)
|
| 239 |
+
if not _number(share):
|
| 240 |
+
continue
|
| 241 |
+
projected = projection.get("usage", {}).get(metric)
|
| 242 |
+
allowance = summary.get("allowance", {}).get(metric)
|
| 243 |
+
for kind, crossed in (
|
| 244 |
+
("share", share >= share_threshold),
|
| 245 |
+
("projected_overage", _number(projected) and _number(allowance) and projected > allowance),
|
| 246 |
+
):
|
| 247 |
+
if crossed:
|
| 248 |
+
crossings.append({"metric": metric, "kind": kind, "share": share,
|
| 249 |
+
"projected_bill": bill if _number(bill) else None})
|
| 250 |
+
if not crossings:
|
| 251 |
+
return None
|
| 252 |
+
test = share_threshold != DEFAULT_ALARM_SHARE
|
| 253 |
+
return {"level": "red", "message": _alarm_message(summary, crossings, test),
|
| 254 |
+
"crossings": crossings, "threshold": share_threshold, "test": test}
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def send_telegram(text):
|
| 258 |
+
"""Bounded POST; return only a message id or a fixed, credential-free cause."""
|
| 259 |
+
token = os.environ.get("TELEGRAM_BOT_TOKEN", "")
|
| 260 |
+
chat_id = os.environ.get("TELEGRAM_CHAT_ID", "")
|
| 261 |
+
missing = [name for name, value in (("TELEGRAM_BOT_TOKEN", token), ("TELEGRAM_CHAT_ID", chat_id))
|
| 262 |
+
if not value]
|
| 263 |
+
if missing:
|
| 264 |
+
return False, "Telegram alert cannot be sent: missing " + ", ".join(missing) + "."
|
| 265 |
+
try:
|
| 266 |
+
body = json.dumps({"chat_id": chat_id, "text": text[:4096]}).encode("utf-8")
|
| 267 |
+
request = Request("https://api.telegram.org/bot" + token + "/sendMessage", data=body,
|
| 268 |
+
headers={"Content-Type": "application/json"}, method="POST")
|
| 269 |
+
with build_opener(_NoRedirect()).open(request, timeout=HTTP_TIMEOUT) as response:
|
| 270 |
+
payload = json.loads(response.read(65_537))
|
| 271 |
+
if not isinstance(payload, dict):
|
| 272 |
+
return False, "Telegram returned an unreadable alert response."
|
| 273 |
+
result = payload.get("result")
|
| 274 |
+
if (payload.get("ok") is True and isinstance(result, dict)
|
| 275 |
+
and type(result.get("message_id")) is int):
|
| 276 |
+
return True, result["message_id"]
|
| 277 |
+
# Telegram descriptions can echo request data. Never return them.
|
| 278 |
+
code = payload.get("error_code")
|
| 279 |
+
suffix = f" (HTTP {code})" if type(code) is int and 100 <= code <= 599 else ""
|
| 280 |
+
return False, "Telegram rejected the alert" + suffix + "."
|
| 281 |
+
except HTTPError as exc:
|
| 282 |
+
code = exc.code
|
| 283 |
+
suffix = f" HTTP {code}" if type(code) is int and 100 <= code <= 599 else " an HTTP error"
|
| 284 |
+
return False, "Telegram returned" + suffix + "."
|
| 285 |
+
except TimeoutError:
|
| 286 |
+
return False, "Telegram alert timed out."
|
| 287 |
+
except URLError as exc:
|
| 288 |
+
cause = "timed out" if isinstance(exc.reason, TimeoutError) else "could not be reached"
|
| 289 |
+
return False, f"Telegram {cause}."
|
| 290 |
+
except (ValueError, OSError):
|
| 291 |
+
return False, "Telegram returned an unreadable alert response."
|
| 292 |
+
except Exception:
|
| 293 |
+
return False, "Telegram alert could not be sent."
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def summarise(project, samples, *, share_threshold=None):
|
| 297 |
+
"""Calculate usage and alarms; earlier periods remain in history, never in rates."""
|
| 298 |
+
if share_threshold is None:
|
| 299 |
+
share_threshold, _ = _alarm_threshold()
|
| 300 |
+
samples = _clean_samples(samples)
|
| 301 |
+
allowance = dict(LAUNCH_PLAN["allowance"])
|
| 302 |
+
result = {"available": project["available"], "cause": project.get("cause"),
|
| 303 |
+
"read_at": project["read_at"], "project": {k: project.get(k) for k in ("id", "name")},
|
| 304 |
+
"period": {k: project.get("consumption_period_" + k) for k in ("start", "end")},
|
| 305 |
+
"usage": dict.fromkeys(allowance), "allowance": allowance,
|
| 306 |
+
"share": dict.fromkeys(allowance), "rate": {},
|
| 307 |
+
"projection": {"transfer": None, "compute": None, "storage": None, "total": None,
|
| 308 |
+
"currency": "USD", "basis": {}, "cause": project.get("cause")},
|
| 309 |
+
"samples": samples, "alarm": None, "guards": copy.deepcopy(_guards)}
|
| 310 |
+
if not project["available"]:
|
| 311 |
+
for metric in allowance:
|
| 312 |
+
result["rate"][metric] = {"per_day": None, "cause": project.get("cause"),
|
| 313 |
+
"from_at": None, "to_at": None}
|
| 314 |
+
return result
|
| 315 |
+
usage = dict(zip(allowance, (project[_FIELDS[0]] / 1e9, project[_FIELDS[1]] / 3600,
|
| 316 |
+
project[_FIELDS[2]] / 2**30)))
|
| 317 |
+
result["usage"] = usage
|
| 318 |
+
result["share"] = {key: usage[key] / value for key, value in allowance.items()}
|
| 319 |
+
start, end, now = (_stamp(value) for value in (result["period"]["start"], result["period"]["end"],
|
| 320 |
+
result["read_at"]))
|
| 321 |
+
current = [row for row in samples if _stamp(row["period_start"]) == start
|
| 322 |
+
and start <= _stamp(row["at"]) <= min(now, end)]
|
| 323 |
+
rates = {"transfer_gb": _rate(current, "transfer_bytes", 1e9),
|
| 324 |
+
"compute_hours": _rate(current, "compute_seconds", 3600),
|
| 325 |
+
"storage_gib": _rate(current, "storage_bytes", 2**30, gauge=True)}
|
| 326 |
+
result["rate"] = rates
|
| 327 |
+
elapsed = (min(now, end) - start).total_seconds() / 86400
|
| 328 |
+
remaining = max(0, (end - now).total_seconds() / 86400)
|
| 329 |
+
period_days = (end - start).total_seconds() / 86400
|
| 330 |
+
if elapsed <= 0:
|
| 331 |
+
result["projection"]["cause"] = "The billing period has not started; an average is not available."
|
| 332 |
+
result["alarm"] = evaluate_alarm(result, share_threshold)
|
| 333 |
+
return result
|
| 334 |
+
projected, basis = {}, {}
|
| 335 |
+
for metric in ("transfer_gb", "compute_hours"):
|
| 336 |
+
rate = rates[metric]["per_day"]
|
| 337 |
+
basis[metric] = "Two changed samples" if rate is not None else "Period-to-date average"
|
| 338 |
+
projected[metric] = usage[metric] + remaining * (rate if rate is not None else usage[metric] / elapsed)
|
| 339 |
+
# Storage is a gauge, not a cumulative counter: dividing its size by elapsed
|
| 340 |
+
# days would invent growth. Integrate the observed sizes over the period,
|
| 341 |
+
# carrying the first size back to its start when no older observation exists.
|
| 342 |
+
cursor, size, size_days = start, (current[0]["storage_bytes"] / 2**30 if current else usage["storage_gib"]), 0.0
|
| 343 |
+
for row in current:
|
| 344 |
+
at = _stamp(row["at"])
|
| 345 |
+
size_days += size * (at - cursor).total_seconds() / 86400
|
| 346 |
+
cursor, size = at, row["storage_bytes"] / 2**30
|
| 347 |
+
size_days += size * (min(now, end) - cursor).total_seconds() / 86400
|
| 348 |
+
storage_average = size_days / elapsed
|
| 349 |
+
storage_rate = rates["storage_gib"]["per_day"]
|
| 350 |
+
projected["storage_gib"] = (max(0, usage["storage_gib"] + remaining * storage_rate)
|
| 351 |
+
if storage_rate is not None else storage_average)
|
| 352 |
+
future_average = ((usage["storage_gib"] + projected["storage_gib"]) / 2
|
| 353 |
+
if storage_rate is not None else storage_average)
|
| 354 |
+
billed_storage_gb = (size_days + remaining * future_average) / period_days * 2**30 / 1e9
|
| 355 |
+
basis["storage_gib"] = ("Two changed samples; estimated average storage" if storage_rate is not None
|
| 356 |
+
else "Period-to-date average storage size; held constant")
|
| 357 |
+
prices = LAUNCH_PLAN["usd"]
|
| 358 |
+
costs = {"transfer": max(0, projected["transfer_gb"] - allowance["transfer_gb"]) * prices["transfer_per_gb"],
|
| 359 |
+
"compute": projected["compute_hours"] * prices["compute_per_hour"],
|
| 360 |
+
"storage": billed_storage_gb * prices["storage_per_gb_month"]}
|
| 361 |
+
result["projection"] = {**costs, "total": sum(costs.values()), "currency": "USD",
|
| 362 |
+
"basis": basis, "usage": projected, "cause": None}
|
| 363 |
+
result["alarm"] = evaluate_alarm(result, share_threshold)
|
| 364 |
+
return result
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
def store_is_awake(project=None, database_url=None, backend=None):
|
| 368 |
+
"""No SQL probe. Unknown host/state fails closed, including history loading."""
|
| 369 |
+
backend = os.environ.get("STORE_BACKEND", "hf") if backend is None else backend
|
| 370 |
+
if backend != "pg":
|
| 371 |
+
return True, None
|
| 372 |
+
try:
|
| 373 |
+
host = urlsplit(os.environ.get("DATABASE_URL", "") if database_url is None else database_url).hostname
|
| 374 |
+
except ValueError:
|
| 375 |
+
host = None
|
| 376 |
+
if not host:
|
| 377 |
+
return False, "Sample history is pending: the store host is not configured."
|
| 378 |
+
if not (host.endswith(".neon.tech") or host.endswith(".neon.build")):
|
| 379 |
+
return True, None
|
| 380 |
+
# Other management-plane samplers share this guard without reading Neon
|
| 381 |
+
# at all when their history store is hosted elsewhere.
|
| 382 |
+
if project is None:
|
| 383 |
+
project = read_project()
|
| 384 |
+
if not project["available"]:
|
| 385 |
+
return False, project["cause"]
|
| 386 |
+
try:
|
| 387 |
+
endpoints = _get("/projects/" + project["id"] + "/endpoints", os.environ.get("NEON_API_KEY", ""))["endpoints"]
|
| 388 |
+
for endpoint in endpoints:
|
| 389 |
+
endpoint_id = endpoint.get("id")
|
| 390 |
+
if _identifier(endpoint_id) and (host.startswith(endpoint_id + ".") or host.startswith(endpoint_id + "-pooler.")):
|
| 391 |
+
if endpoint.get("current_state") == "active":
|
| 392 |
+
return True, None
|
| 393 |
+
return False, "Sample history is pending while the store compute is idle."
|
| 394 |
+
return False, "Sample history is pending: the store endpoint was not found in this project."
|
| 395 |
+
except UsageReadError as exc:
|
| 396 |
+
return False, str(exc)
|
| 397 |
+
except Exception:
|
| 398 |
+
return False, "Sample history is pending: the store compute state could not be read."
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
_lock = threading.RLock()
|
| 402 |
+
_thread = None
|
| 403 |
+
_stop = threading.Event()
|
| 404 |
+
_samples = []
|
| 405 |
+
_loaded = False
|
| 406 |
+
_project_id = None
|
| 407 |
+
_alerted = {}
|
| 408 |
+
_status = {"every_s": 0, "last_at": None, "pending": 0, "persisted": None,
|
| 409 |
+
"cause": "Scheduled sampling is disabled; AIOS_NEON_SAMPLE_SECONDS is unset or zero."}
|
| 410 |
+
|
| 411 |
+
|
| 412 |
+
def snapshot():
|
| 413 |
+
with _lock:
|
| 414 |
+
return copy.deepcopy(_samples), dict(_status)
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
def _sampler_cause(cause=None):
|
| 418 |
+
_, config_cause = _alarm_threshold()
|
| 419 |
+
if config_cause and config_cause not in (cause or ""):
|
| 420 |
+
return " ".join(item for item in (cause, config_cause) if item)
|
| 421 |
+
return cause or None
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def _period_key(value):
|
| 425 |
+
stamp = _stamp(value)
|
| 426 |
+
if stamp is None:
|
| 427 |
+
raise ValueError("Invalid alert period")
|
| 428 |
+
return stamp.astimezone(timezone.utc).isoformat(timespec="seconds")
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def _clean_alerted(alerted, period_start, samples):
|
| 432 |
+
"""At most six keys each for the current and previous monthly billing period."""
|
| 433 |
+
if not isinstance(alerted, dict):
|
| 434 |
+
raise ValueError("Unreadable alert markers")
|
| 435 |
+
current = _period_key(period_start)
|
| 436 |
+
clean = {}
|
| 437 |
+
for period, keys in alerted.items():
|
| 438 |
+
period = _period_key(period)
|
| 439 |
+
if not isinstance(keys, list) or any(not isinstance(key, str) or key not in _ALERT_KEYS for key in keys):
|
| 440 |
+
raise ValueError("Unreadable alert markers")
|
| 441 |
+
clean.setdefault(period, set()).update(keys)
|
| 442 |
+
periods = set(clean) | {_period_key(row["period_start"]) for row in samples}
|
| 443 |
+
start = _stamp(current)
|
| 444 |
+
month = start.year * 12 + start.month
|
| 445 |
+
previous = max((period for period in periods if period < current
|
| 446 |
+
and month - (_stamp(period).year * 12 + _stamp(period).month) <= 1), default=None)
|
| 447 |
+
return {period: sorted(clean.get(period, set())) for period in (previous, current) if period}
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def report():
|
| 451 |
+
"""Card path: management API plus memory only, never the store callbacks."""
|
| 452 |
+
project = read_project()
|
| 453 |
+
samples, status = snapshot()
|
| 454 |
+
status["cause"] = _sampler_cause(status["cause"])
|
| 455 |
+
return {**summarise(project, samples), "sampler": status}
|
| 456 |
+
|
| 457 |
+
|
| 458 |
+
def _sample_once(load, persist, *, sender=None):
|
| 459 |
+
global _loaded, _samples, _alerted, _project_id
|
| 460 |
+
project = read_project()
|
| 461 |
+
if not project["available"]:
|
| 462 |
+
with _lock:
|
| 463 |
+
_status["cause"] = _sampler_cause(project["cause"])
|
| 464 |
+
return
|
| 465 |
+
sample = {"at": project["read_at"], "transfer_bytes": project[_FIELDS[0]],
|
| 466 |
+
"compute_seconds": project[_FIELDS[1]], "storage_bytes": project[_FIELDS[2]],
|
| 467 |
+
"period_start": project["consumption_period_start"]}
|
| 468 |
+
with _lock:
|
| 469 |
+
if _project_id != project["id"]:
|
| 470 |
+
_project_id, _loaded, _samples, _alerted = project["id"], False, [], {}
|
| 471 |
+
_status.update(pending=0, persisted=None)
|
| 472 |
+
_samples = _clean_samples([*_samples, sample])
|
| 473 |
+
_status.update(last_at=sample["at"], pending=min(MAX_SAMPLES, _status["pending"] + 1))
|
| 474 |
+
awake, cause = store_is_awake(project)
|
| 475 |
+
if not awake:
|
| 476 |
+
with _lock:
|
| 477 |
+
_status["cause"] = _sampler_cause(cause)
|
| 478 |
+
return
|
| 479 |
+
try:
|
| 480 |
+
if not _loaded:
|
| 481 |
+
saved = load()
|
| 482 |
+
if saved is None:
|
| 483 |
+
saved = {}
|
| 484 |
+
with _lock:
|
| 485 |
+
# Never mix a saved ring from a different configured project.
|
| 486 |
+
matching = saved.get("project_id") == project["id"]
|
| 487 |
+
history = saved.get("samples", []) if matching else []
|
| 488 |
+
_samples = _clean_samples([*history, *_samples])
|
| 489 |
+
_alerted = _clean_alerted(saved.get("alerted", {}) if matching else {},
|
| 490 |
+
sample["period_start"], _samples)
|
| 491 |
+
_loaded = True
|
| 492 |
+
except Exception:
|
| 493 |
+
with _lock:
|
| 494 |
+
_status["cause"] = _sampler_cause("Sample history and alert markers could not be loaded; alerts deferred.")
|
| 495 |
+
return
|
| 496 |
+
sent = False
|
| 497 |
+
try:
|
| 498 |
+
# Loading/initializing the runtime may take time. Re-observe immediately
|
| 499 |
+
# before the write rather than relying on its earlier active state.
|
| 500 |
+
awake, cause = store_is_awake(project)
|
| 501 |
+
if not awake:
|
| 502 |
+
with _lock:
|
| 503 |
+
_status["cause"] = _sampler_cause(cause)
|
| 504 |
+
return
|
| 505 |
+
samples, _ = snapshot()
|
| 506 |
+
with _lock:
|
| 507 |
+
_alerted = _clean_alerted(_alerted, sample["period_start"], samples)
|
| 508 |
+
alerted = copy.deepcopy(_alerted)
|
| 509 |
+
# Confirm storage works before sending. This also flushes any markers
|
| 510 |
+
# left in memory by a successful send whose subsequent save failed.
|
| 511 |
+
persist({"project_id": project["id"], "samples": samples, "alerted": alerted})
|
| 512 |
+
with _lock:
|
| 513 |
+
_status.update(pending=0, persisted=_now(), cause=_sampler_cause())
|
| 514 |
+
summary = summarise(project, samples)
|
| 515 |
+
alarm = summary["alarm"]
|
| 516 |
+
period = _period_key(sample["period_start"])
|
| 517 |
+
crossings = [row for row in alarm["crossings"]
|
| 518 |
+
if f"{row['metric']}:{row['kind']}" not in alerted[period]] if alarm else []
|
| 519 |
+
if not crossings:
|
| 520 |
+
return
|
| 521 |
+
text = (_alarm_message(summary, crossings, alarm["test"])
|
| 522 |
+
+ f"\nPeriod {period} to {_period_key(summary['period']['end'])} (end exclusive).")
|
| 523 |
+
try:
|
| 524 |
+
ok, delivery = (sender or send_telegram)(text)
|
| 525 |
+
except Exception:
|
| 526 |
+
ok, delivery = False, "Telegram alert could not be sent."
|
| 527 |
+
if not ok:
|
| 528 |
+
with _lock:
|
| 529 |
+
_status["cause"] = _sampler_cause(delivery)
|
| 530 |
+
return
|
| 531 |
+
sent = True
|
| 532 |
+
with _lock:
|
| 533 |
+
_alerted[period] = sorted(set(_alerted[period])
|
| 534 |
+
| {f"{row['metric']}:{row['kind']}" for row in crossings})
|
| 535 |
+
alerted = copy.deepcopy(_alerted)
|
| 536 |
+
# Keep successful deliveries in memory even if this save fails. The
|
| 537 |
+
# next tick flushes them before considering another message.
|
| 538 |
+
awake, cause = store_is_awake(project)
|
| 539 |
+
if not awake:
|
| 540 |
+
with _lock:
|
| 541 |
+
_status["cause"] = _sampler_cause("Telegram alert sent; alert markers await storage. " + (cause or ""))
|
| 542 |
+
return
|
| 543 |
+
persist({"project_id": project["id"], "samples": samples, "alerted": alerted})
|
| 544 |
+
with _lock:
|
| 545 |
+
_status.update(persisted=_now(), cause=_sampler_cause())
|
| 546 |
+
except Exception:
|
| 547 |
+
with _lock:
|
| 548 |
+
_status["cause"] = _sampler_cause(
|
| 549 |
+
"Telegram alert sent; alert markers could not be stored and remain pending in memory." if sent
|
| 550 |
+
else "Sample history could not be stored; samples and alert markers remain pending in memory.")
|
| 551 |
+
|
| 552 |
+
|
| 553 |
+
def start_sampler(*, load=None, persist=None, sender=None):
|
| 554 |
+
"""One daemon per app process; unset/zero is OFF in imports, gates and tests.
|
| 555 |
+
|
| 556 |
+
Deploy explicitly installs DEFAULT_SAMPLE_SECONDS. Callbacks are mandatory
|
| 557 |
+
so this module never imports a database implementation or a tenant runtime.
|
| 558 |
+
"""
|
| 559 |
+
global _thread
|
| 560 |
+
try:
|
| 561 |
+
every_s = int(os.environ.get("AIOS_NEON_SAMPLE_SECONDS", "0"))
|
| 562 |
+
except ValueError:
|
| 563 |
+
every_s = 0
|
| 564 |
+
if every_s <= 0 or load is None or persist is None:
|
| 565 |
+
return False
|
| 566 |
+
# Never turn a typo into a tight management-API polling loop.
|
| 567 |
+
every_s = max(60, every_s)
|
| 568 |
+
with _lock:
|
| 569 |
+
if _thread is not None and _thread.is_alive():
|
| 570 |
+
return True
|
| 571 |
+
_status.update(every_s=every_s, cause=_sampler_cause("Waiting for the first scheduled sample."))
|
| 572 |
+
_stop.clear()
|
| 573 |
+
|
| 574 |
+
def run():
|
| 575 |
+
while not _stop.is_set():
|
| 576 |
+
try:
|
| 577 |
+
_sample_once(load, persist, sender=sender)
|
| 578 |
+
except Exception:
|
| 579 |
+
with _lock:
|
| 580 |
+
_status["cause"] = _sampler_cause("The scheduled sample could not be completed; it will retry.")
|
| 581 |
+
if _stop.wait(every_s):
|
| 582 |
+
break
|
| 583 |
+
|
| 584 |
+
_thread = threading.Thread(target=run, name="neon-usage-sampler", daemon=True)
|
| 585 |
+
_thread.start()
|
| 586 |
+
return True
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
def stop_sampler():
|
| 590 |
+
_stop.set()
|
platform/core/perm_scope.py
CHANGED
|
@@ -51,6 +51,9 @@ import core.perms as perms
|
|
| 51 |
_FORMULA_REF = re.compile(r"\{([^{}]*)\}")
|
| 52 |
|
| 53 |
PERMS_VERSION = 1
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
|
| 56 |
def _rec(user):
|
|
@@ -118,6 +121,32 @@ def may_metrics(user, module):
|
|
| 118 |
return not (isinstance(e, dict) and e.get('metrics') is False)
|
| 119 |
|
| 120 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
def nav_may_open(user, key, st=None):
|
| 122 |
"""May this account SEE `key` β in the nav, and at the route gate? The ADMISSION question.
|
| 123 |
|
|
@@ -136,6 +165,8 @@ def nav_may_open(user, key, st=None):
|
|
| 136 |
1. an admin sees everything (break-glass, as everywhere else);
|
| 137 |
2. an EXPLICIT `access: false` DENIES, on any key β an administrator unticking a box must
|
| 138 |
take the row off the nav, and before this it did not;
|
|
|
|
|
|
|
| 139 |
3. a `ut_*` key with no explicit deny defers to `user_tables.may_open` β creator, admin or
|
| 140 |
a `core.shares` grant β and **an `access: true` entry may NEVER widen past it**. The
|
| 141 |
editor must not become a way to hand somebody another user's private table;
|
|
@@ -149,11 +180,12 @@ def nav_may_open(user, key, st=None):
|
|
| 149 |
registry topic and fatal for exactly one key family. Every door that can be handed a
|
| 150 |
`ut_*` key now threads its session's own runtime: `deps.Session.require`,
|
| 151 |
`routes_charts._require_database`, `routes_nav.nav` and `routes_tables` (both lending
|
| 152 |
-
the rows-free projection) and `routes_admin.get_perms`. The
|
| 153 |
-
`st`
|
| 154 |
carrying no `ut_` key β `verify_perm_scope.py` section K pins that, so the fence can go
|
| 155 |
-
red rather than merely being asserted here)
|
| 156 |
-
|
|
|
|
| 157 |
|
| 158 |
4. a REGISTRY TOPIC with an explicit entry takes that entry, grant included. Here the
|
| 159 |
editor IS the authority: `_clean_perms` writes an entry for every governed topic on
|
|
@@ -169,8 +201,11 @@ def nav_may_open(user, key, st=None):
|
|
| 169 |
if perms.is_admin(user):
|
| 170 |
return True
|
| 171 |
e = entry(user, key)
|
| 172 |
-
if e is not None and
|
|
|
|
| 173 |
return False
|
|
|
|
|
|
|
| 174 |
if str(key or '').startswith(_ut().KEY_PREFIX):
|
| 175 |
return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st))
|
| 176 |
if e is not None:
|
|
@@ -186,12 +221,39 @@ def assistant_entry(user, module):
|
|
| 186 |
that reader must be able to name the migrated grant that admitted a database. In particular,
|
| 187 |
a store-outage admin identity with no ``perms`` document is not an unresolved permission that
|
| 188 |
may be widened into data access.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
"""
|
| 190 |
e = entry(user, module)
|
| 191 |
-
if (
|
| 192 |
-
|
| 193 |
-
return
|
| 194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 195 |
|
| 196 |
|
| 197 |
# ββ FIELDS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -1672,7 +1734,9 @@ def _read_user_table(table_key, st, want_rows=True):
|
|
| 1672 |
'reading a read-through database, or read it through the API'))
|
| 1673 |
keys = {f['key'] for f in fields if f.get('key')}
|
| 1674 |
return fields, list(_MIRROR_READER(table_key, keys, st) or ())
|
| 1675 |
-
|
|
|
|
|
|
|
| 1676 |
if whole is None:
|
| 1677 |
# Deleted between the wall and here. The same refusal, not an empty table.
|
| 1678 |
raise UnknownTable(f"no database named '{table_key}'")
|
|
|
|
| 51 |
_FORMULA_REF = re.compile(r"\{([^{}]*)\}")
|
| 52 |
|
| 53 |
PERMS_VERSION = 1
|
| 54 |
+
AI_APPS = "ai_apps"
|
| 55 |
+
AI_APPS_OFF = ("AI apps (MCP) is switched off for your account. "
|
| 56 |
+
"An administrator can switch it on in Settings > Manage users.")
|
| 57 |
|
| 58 |
|
| 59 |
def _rec(user):
|
|
|
|
| 121 |
return not (isinstance(e, dict) and e.get('metrics') is False)
|
| 122 |
|
| 123 |
|
| 124 |
+
def may_ai_apps(user):
|
| 125 |
+
"""R3/C6: AI apps are a separate capability, granted unless explicitly switched off.
|
| 126 |
+
|
| 127 |
+
Like may_metrics, this deliberately differs from may_access: no existing migrated or
|
| 128 |
+
legacy account could declare this new key. Absence must not revoke everybody's gateway
|
| 129 |
+
access. Only access: False denies; the in-app Assistant still uses analyst.
|
| 130 |
+
"""
|
| 131 |
+
if perms.is_admin(user):
|
| 132 |
+
return True
|
| 133 |
+
e = entry(user, AI_APPS)
|
| 134 |
+
return not (e is not None and e.get('access') is False)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def ai_apps_refusal(user, runtime):
|
| 138 |
+
"""Return the account or tenant refusal, or None. Shared by OAuth and every tool call.
|
| 139 |
+
|
| 140 |
+
Do not route this through Session.require: its literal tenant-list check cannot recognise
|
| 141 |
+
the existing analyst tenant entitlement that tenant_has_module preserves for this gateway.
|
| 142 |
+
"""
|
| 143 |
+
if not may_ai_apps(user):
|
| 144 |
+
return AI_APPS_OFF
|
| 145 |
+
if not perms.tenant_has_module(runtime, AI_APPS):
|
| 146 |
+
return "AI apps (MCP) is not available in this workspace."
|
| 147 |
+
return None
|
| 148 |
+
|
| 149 |
+
|
| 150 |
def nav_may_open(user, key, st=None):
|
| 151 |
"""May this account SEE `key` β in the nav, and at the route gate? The ADMISSION question.
|
| 152 |
|
|
|
|
| 165 |
1. an admin sees everything (break-glass, as everywhere else);
|
| 166 |
2. an EXPLICIT `access: false` DENIES, on any key β an administrator unticking a box must
|
| 167 |
take the row off the nav, and before this it did not;
|
| 168 |
+
R3/C6's ai_apps capability then grants by absence, including migrated and legacy-list
|
| 169 |
+
accounts. It is an editor surface, not a registry page or a rail entry;
|
| 170 |
3. a `ut_*` key with no explicit deny defers to `user_tables.may_open` β creator, admin or
|
| 171 |
a `core.shares` grant β and **an `access: true` entry may NEVER widen past it**. The
|
| 172 |
editor must not become a way to hand somebody another user's private table;
|
|
|
|
| 180 |
registry topic and fatal for exactly one key family. Every door that can be handed a
|
| 181 |
`ut_*` key now threads its session's own runtime: `deps.Session.require`,
|
| 182 |
`routes_charts._require_database`, `routes_nav.nav` and `routes_tables` (both lending
|
| 183 |
+
the rows-free projection) and `routes_admin.get_perms`. The caller that still omits
|
| 184 |
+
`st` is `perms.nav_pages`, which iterates `core.registry.REGISTRY` (a compiled literal
|
| 185 |
carrying no `ut_` key β `verify_perm_scope.py` section K pins that, so the fence can go
|
| 186 |
+
red rather than merely being asserted here). It cannot reach this leg. The editor's
|
| 187 |
+
surface default now also threads its runtime; its catalogue includes R3/C6's gateway
|
| 188 |
+
capability as well as registry pages;
|
| 189 |
|
| 190 |
4. a REGISTRY TOPIC with an explicit entry takes that entry, grant included. Here the
|
| 191 |
editor IS the authority: `_clean_perms` writes an entry for every governed topic on
|
|
|
|
| 201 |
if perms.is_admin(user):
|
| 202 |
return True
|
| 203 |
e = entry(user, key)
|
| 204 |
+
if e is not None and (e.get('access') is False if key == AI_APPS
|
| 205 |
+
else not bool(e.get('access', True))):
|
| 206 |
return False
|
| 207 |
+
if key == AI_APPS:
|
| 208 |
+
return True
|
| 209 |
if str(key or '').startswith(_ut().KEY_PREFIX):
|
| 210 |
return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st))
|
| 211 |
if e is not None:
|
|
|
|
| 221 |
that reader must be able to name the migrated grant that admitted a database. In particular,
|
| 222 |
a store-outage admin identity with no ``perms`` document is not an unresolved permission that
|
| 223 |
may be widened into data access.
|
| 224 |
+
|
| 225 |
+
β S67-T34 / D-276: ONE NAMED EXCEPTION, the grant the NAVIGATION uses. For the two registry
|
| 226 |
+
topics in ``ASSISTANT_NAV_TOPICS`` an administrator account is admitted unfiltered (unless
|
| 227 |
+
explicitly unticked), because ``nav_may_open`` leg 1 opens them to it; the AI list must equal
|
| 228 |
+
the nav (R33). See the comment at that leg for why the outage identity no longer reaches it.
|
| 229 |
"""
|
| 230 |
e = entry(user, module)
|
| 231 |
+
if (is_migrated(user) and isinstance(e, dict)
|
| 232 |
+
and bool(e.get('access', True)) and may_access(user, module)):
|
| 233 |
+
return e
|
| 234 |
+
# ββ S67-T34 / D-276 / R26: THE ADMINISTRATOR THE NAVIGATION ADMITS. `nav_may_open` leg 1
|
| 235 |
+
# opens every registry topic to an administrator, and `_clean_perms` writes no per-topic
|
| 236 |
+
# entry for one, so an administrator saw `customer_data` and `product_data` (Odoo products,
|
| 237 |
+
# R26's Inventory source) in the nav and had them refused here. MEASURED on staging
|
| 238 |
+
# 2026-09-28: aaron (admin, no `perms_v`) had both in the nav and neither in the AI list. The
|
| 239 |
+
# answer is the grant the nav already uses, admitted for exactly these topics: an account
|
| 240 |
+
# whose ROLE is admin, minus an explicit `access: false` (narrower than the nav, never wider).
|
| 241 |
+
# Unfiltered, as the nav's break-glass is.
|
| 242 |
+
# β THE STORE-OUTAGE IDENTITY the docstring names is no longer built by `deps._user_for`
|
| 243 |
+
# (it admits a stored record or the QA runner, nothing else), so the role is read off a
|
| 244 |
+
# stored account here. If a synthetic admin identity is ever reintroduced, it must be kept
|
| 245 |
+
# out of this leg.
|
| 246 |
+
# β NOT A MEMBER. A non-administrator still needs the explicit migrated grant above, so a
|
| 247 |
+
# user without it does not see the database (the NC).
|
| 248 |
+
if (module in ASSISTANT_NAV_TOPICS and perms.is_admin(user)
|
| 249 |
+
and not (isinstance(e, dict) and e.get('access') is False)):
|
| 250 |
+
return dict(e) if isinstance(e, dict) else {'access': True, 'filter': None}
|
| 251 |
+
return None
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
#: S67-T34: the registry topics the navigation opens to an administrator by break-glass, which
|
| 255 |
+
#: `assistant_entry` therefore admits for a migrated administrator without a per-topic entry.
|
| 256 |
+
ASSISTANT_NAV_TOPICS = frozenset({'customer_data', 'product_data'})
|
| 257 |
|
| 258 |
|
| 259 |
# ββ FIELDS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 1734 |
'reading a read-through database, or read it through the API'))
|
| 1735 |
keys = {f['key'] for f in fields if f.get('key')}
|
| 1736 |
return fields, list(_MIRROR_READER(table_key, keys, st) or ())
|
| 1737 |
+
# HOTFIX 2026-09-28: this table's entry alone where the store can (`get_one`), not a deep
|
| 1738 |
+
# copy of every table in the tenant. Amazon link cells read listings here on every page.
|
| 1739 |
+
whole = ut.get_one(table_key, st=st)
|
| 1740 |
if whole is None:
|
| 1741 |
# Deleted between the wall and here. The same refusal, not an empty table.
|
| 1742 |
raise UnknownTable(f"no database named '{table_key}'")
|
platform/core/perms.py
CHANGED
|
@@ -253,6 +253,10 @@ def tenant_has_module(runtime, key):
|
|
| 253 |
allowed = tenant_modules(runtime)
|
| 254 |
if allowed is None or key in allowed:
|
| 255 |
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
| 256 |
parent = registry.BY_KEY.get(key, {}).get('parent')
|
| 257 |
return parent in allowed if parent else False
|
| 258 |
|
|
@@ -336,17 +340,22 @@ def governable_surface_modules():
|
|
| 336 |
filter or hiddenFields against an empty vocabulary, so a client cannot store a rule nothing
|
| 337 |
would apply.
|
| 338 |
|
| 339 |
-
|
| 340 |
`topic` that `nav_pages` would offer (not `validate_only`, not `api_surface: False`, not
|
| 341 |
`settings`). Today that is `analyst` (Assistant) and `automation` (Agents); a surface added
|
| 342 |
to the registry tomorrow is governable with no edit here β the same rule
|
| 343 |
`tenant_governable_modules` states for databases, applied to the other kind of row.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 344 |
"""
|
| 345 |
return [{'key': m['key'], 'label': m['label']}
|
| 346 |
for m in registry.REGISTRY
|
| 347 |
if not m.get('archived') and not m.get('group_only') and not m.get('topic')
|
| 348 |
and not m.get('validate_only') and m.get('api_surface') is not False
|
| 349 |
-
and m['key'] != 'settings']
|
| 350 |
|
| 351 |
|
| 352 |
#: β `enforced_module_keys` IS DELETED (W36-T22 / C2), NOT NEUTERED TO "every governable key".
|
|
|
|
| 253 |
allowed = tenant_modules(runtime)
|
| 254 |
if allowed is None or key in allowed:
|
| 255 |
return True
|
| 256 |
+
# R3/C6: tenants provisioned with Assistant already had the external gateway. Its new
|
| 257 |
+
# account-level grant must not remove that entitlement from existing tenant catalogues.
|
| 258 |
+
if key == 'ai_apps':
|
| 259 |
+
return 'analyst' in allowed
|
| 260 |
parent = registry.BY_KEY.get(key, {}).get('parent')
|
| 261 |
return parent in allowed if parent else False
|
| 262 |
|
|
|
|
| 340 |
filter or hiddenFields against an empty vocabulary, so a client cannot store a rule nothing
|
| 341 |
would apply.
|
| 342 |
|
| 343 |
+
Page surfaces are DERIVED: every non-archived, non-`group_only` registry row with NO
|
| 344 |
`topic` that `nav_pages` would offer (not `validate_only`, not `api_surface: False`, not
|
| 345 |
`settings`). Today that is `analyst` (Assistant) and `automation` (Agents); a surface added
|
| 346 |
to the registry tomorrow is governable with no edit here β the same rule
|
| 347 |
`tenant_governable_modules` states for databases, applied to the other kind of row.
|
| 348 |
+
|
| 349 |
+
R3/C6 adds one listed capability: ai_apps, labelled AI apps (MCP). It governs the external
|
| 350 |
+
gateway, not a page, so it belongs in the account editor without entering REGISTRY or the
|
| 351 |
+
navigation rail. registry.py is outside this ticket's fence. Both consumers here are editor
|
| 352 |
+
helpers (the catalogue and empty field schema); neither turns this row into navigation.
|
| 353 |
"""
|
| 354 |
return [{'key': m['key'], 'label': m['label']}
|
| 355 |
for m in registry.REGISTRY
|
| 356 |
if not m.get('archived') and not m.get('group_only') and not m.get('topic')
|
| 357 |
and not m.get('validate_only') and m.get('api_surface') is not False
|
| 358 |
+
and m['key'] != 'settings'] + [{'key': 'ai_apps', 'label': 'AI apps (MCP)'}]
|
| 359 |
|
| 360 |
|
| 361 |
#: β `enforced_module_keys` IS DELETED (W36-T22 / C2), NOT NEUTERED TO "every governable key".
|
platform/core/registry.py
CHANGED
|
@@ -203,7 +203,12 @@ REGISTRY = [
|
|
| 203 |
'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does. Delete '
|
| 204 |
'them") made that literal: `validate_only` takes it out of the account menu too, '
|
| 205 |
'which is the last place it was still visible. The proof survives; the door does not.'},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
{'key': 'automation', 'label': 'Agents', 'brand': False, 'hq': True, 'validate': False,
|
|
|
|
| 207 |
'nav': True,
|
| 208 |
'note': 'Wave 18 (C-AUTONAV): scheduled jobs that create and refresh user databases '
|
| 209 |
'(website scrape-to-DB, the Instagram field). React-only surface (no PAGE_FUNCS '
|
|
@@ -280,6 +285,11 @@ def brand_filterable(key) -> bool:
|
|
| 280 |
return bool(BY_KEY.get(key, {}).get('brand'))
|
| 281 |
|
| 282 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 283 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 284 |
# ββ W33-T41 / item 12a β ONE SUBJECT, ONE DATABASE
|
| 285 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 203 |
'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does. Delete '
|
| 204 |
'them") made that literal: `validate_only` takes it out of the account menu too, '
|
| 205 |
'which is the last place it was still visible. The proof survives; the door does not.'},
|
| 206 |
+
# S67-T36 (owner instr 24, R25, 2026-09-28): Agents is TABLED. `archived` hides the UI only:
|
| 207 |
+
# no rail row, no Settings module toggle, no Home section, no agent templates. The
|
| 208 |
+
# `/automations` APIs, stored automations and their schedules are untouched and keep running.
|
| 209 |
+
# β RESTORING IS THIS ONE LINE: delete `'archived': True` below and every door comes back.
|
| 210 |
{'key': 'automation', 'label': 'Agents', 'brand': False, 'hq': True, 'validate': False,
|
| 211 |
+
'archived': True,
|
| 212 |
'nav': True,
|
| 213 |
'note': 'Wave 18 (C-AUTONAV): scheduled jobs that create and refresh user databases '
|
| 214 |
'(website scrape-to-DB, the Instagram field). React-only surface (no PAGE_FUNCS '
|
|
|
|
| 285 |
return bool(BY_KEY.get(key, {}).get('brand'))
|
| 286 |
|
| 287 |
|
| 288 |
+
def is_archived(key) -> bool:
|
| 289 |
+
"""True when the registry row `key` is archived (tabled for everyone). Unknown keys: False."""
|
| 290 |
+
return bool(BY_KEY.get(key, {}).get('archived'))
|
| 291 |
+
|
| 292 |
+
|
| 293 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 294 |
# ββ W33-T41 / item 12a β ONE SUBJECT, ONE DATABASE
|
| 295 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
platform/core/sheet_binding.py
CHANGED
|
@@ -163,11 +163,13 @@ class BindingMetadataRefusal(Exception):
|
|
| 163 |
|
| 164 |
|
| 165 |
def clean_related(raw):
|
| 166 |
-
"""
|
| 167 |
|
| 168 |
Absence is handled by the caller. An explicitly supplied null is malformed, as are
|
| 169 |
extra relationship bags and nested relationships. Operator support belongs to the
|
| 170 |
query compiler, which reports unpushable conditions rather than dropping metadata.
|
|
|
|
|
|
|
| 171 |
"""
|
| 172 |
def refuse():
|
| 173 |
raise BindingMetadataRefusal("related_invalid")
|
|
@@ -214,6 +216,63 @@ def clean_related(raw):
|
|
| 214 |
return {"via": via.strip(), "filter": deepcopy(tree)}
|
| 215 |
|
| 216 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 217 |
def clean_period(raw):
|
| 218 |
"""Return one canonical Sheet period, or ``None`` when no window was named."""
|
| 219 |
if not isinstance(raw, dict):
|
|
|
|
| 163 |
|
| 164 |
|
| 165 |
def clean_related(raw):
|
| 166 |
+
"""Read the legacy edge bag for its one-release migration to ordinary filter leaves.
|
| 167 |
|
| 168 |
Absence is handled by the caller. An explicitly supplied null is malformed, as are
|
| 169 |
extra relationship bags and nested relationships. Operator support belongs to the
|
| 170 |
query compiler, which reports unpushable conditions rather than dropping metadata.
|
| 171 |
+
The binding cleaner qualifies this copied tree and intersects it with the local scope;
|
| 172 |
+
neither this reader nor that rewrite mutates the saved document.
|
| 173 |
"""
|
| 174 |
def refuse():
|
| 175 |
raise BindingMetadataRefusal("related_invalid")
|
|
|
|
| 216 |
return {"via": via.strip(), "filter": deepcopy(tree)}
|
| 217 |
|
| 218 |
|
| 219 |
+
def rewrite_legacy_related(binding, *, linked=False):
|
| 220 |
+
"""Copy a binding, moving its legacy bag into the ordinary filter tree.
|
| 221 |
+
|
| 222 |
+
This is a lossless metadata rewrite, not a binding cleaner or a relationship lookup.
|
| 223 |
+
The scoped resolver still proves every named edge. Migration may also replace the
|
| 224 |
+
linked spelling with its explicit bridge identity; normal reads leave those aliases
|
| 225 |
+
to the API's permission-scoped declaration lookup.
|
| 226 |
+
"""
|
| 227 |
+
out = deepcopy(binding)
|
| 228 |
+
if "related" in out:
|
| 229 |
+
legacy = clean_related(out.pop("related"))
|
| 230 |
+
|
| 231 |
+
def parts(tree):
|
| 232 |
+
if isinstance(tree, dict):
|
| 233 |
+
nodes = tree.get("nodes")
|
| 234 |
+
return (nodes if isinstance(nodes, list) else []), (
|
| 235 |
+
"or" if tree.get("conj") == "or" else "and")
|
| 236 |
+
return (tree if isinstance(tree, list) else []), "and"
|
| 237 |
+
|
| 238 |
+
def qualify(node):
|
| 239 |
+
if isinstance(node.get("children"), list):
|
| 240 |
+
return {**node, "children": [qualify(child) for child in node["children"]]}
|
| 241 |
+
leaf = {**node, "colId": f"related:{legacy['via']}:{node['colId']}"}
|
| 242 |
+
if isinstance(node.get("rhs"), dict) and node["rhs"].get("colId"):
|
| 243 |
+
leaf["rhs"] = {**node["rhs"],
|
| 244 |
+
"colId": f"related:{legacy['via']}:{node['rhs']['colId']}"}
|
| 245 |
+
return leaf
|
| 246 |
+
|
| 247 |
+
scope = out.setdefault("scope", {})
|
| 248 |
+
if not isinstance(scope, dict):
|
| 249 |
+
raise BindingMetadataRefusal("related_invalid")
|
| 250 |
+
nodes, conj = parts(legacy["filter"])
|
| 251 |
+
original, original_conj = parts(scope.get("filter"))
|
| 252 |
+
original = ([{"conj": original_conj, "children": original}]
|
| 253 |
+
if original and original_conj == "or" else list(original))
|
| 254 |
+
scope["filter"] = {"conj": "and", "nodes": [*original,
|
| 255 |
+
{"conj": conj, "children": [qualify(node) for node in nodes]}]}
|
| 256 |
+
|
| 257 |
+
def rewrite_links(value):
|
| 258 |
+
if isinstance(value, dict):
|
| 259 |
+
key = value.get("colId")
|
| 260 |
+
if isinstance(key, str) and key.startswith("linked:"):
|
| 261 |
+
parts = key.split(":")
|
| 262 |
+
if len(parts) != 3 or not all(parts[1:]):
|
| 263 |
+
raise BindingMetadataRefusal("related_invalid", key)
|
| 264 |
+
value["colId"] = "related:" + key[len("linked:"):]
|
| 265 |
+
for child in value.values():
|
| 266 |
+
rewrite_links(child)
|
| 267 |
+
elif isinstance(value, list):
|
| 268 |
+
for child in value:
|
| 269 |
+
rewrite_links(child)
|
| 270 |
+
|
| 271 |
+
if linked:
|
| 272 |
+
rewrite_links(out)
|
| 273 |
+
return out
|
| 274 |
+
|
| 275 |
+
|
| 276 |
def clean_period(raw):
|
| 277 |
"""Return one canonical Sheet period, or ``None`` when no window was named."""
|
| 278 |
if not isinstance(raw, dict):
|
platform/core/sheet_import.py
CHANGED
|
@@ -43,9 +43,11 @@ from __future__ import annotations
|
|
| 43 |
import re
|
| 44 |
import zipfile
|
| 45 |
import hashlib
|
|
|
|
| 46 |
import json
|
| 47 |
import math
|
| 48 |
import os
|
|
|
|
| 49 |
import tempfile
|
| 50 |
import xml.etree.ElementTree as ET
|
| 51 |
from datetime import date, datetime, time, timezone
|
|
@@ -392,12 +394,11 @@ def style_of(cell, notes):
|
|
| 392 |
if border is not None:
|
| 393 |
sides = {}
|
| 394 |
for side in sheets._BORDER_SIDES:
|
| 395 |
-
|
| 396 |
-
if
|
| 397 |
-
sides[side] =
|
| 398 |
if sides:
|
| 399 |
out["border"] = sides
|
| 400 |
-
notes.add("a border keeps its side but not its weight, style or colour")
|
| 401 |
|
| 402 |
fmt, note = classify_format(getattr(cell, "number_format", ""))
|
| 403 |
if fmt:
|
|
@@ -407,6 +408,47 @@ def style_of(cell, notes):
|
|
| 407 |
return out
|
| 408 |
|
| 409 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 410 |
# ββ values βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 411 |
|
| 412 |
def json_value(value):
|
|
@@ -1037,6 +1079,134 @@ def guard_import_archive(path):
|
|
| 1037 |
"That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
|
| 1038 |
|
| 1039 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1040 |
def _print_of(ws):
|
| 1041 |
"""Store print setup and inclusive print areas; export executes these settings later."""
|
| 1042 |
setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
|
|
@@ -1148,6 +1318,7 @@ def import_workbook(path, *, source="", progress=None):
|
|
| 1148 |
|
| 1149 |
counts = {"cells": 0, "formulas": 0}
|
| 1150 |
book_notes = set()
|
|
|
|
| 1151 |
authored = _authored_chart_document(path, book_notes)
|
| 1152 |
chart_tabs = {tab["name"]: tab for tab in authored["sheets"]} if authored else {}
|
| 1153 |
if authored and len(chart_tabs) != len(authored["sheets"]):
|
|
@@ -1209,6 +1380,8 @@ def import_workbook(path, *, source="", progress=None):
|
|
| 1209 |
cells = _cells_of(ws, cached, notes, counts, excel)
|
| 1210 |
_say("formats", 0.7 + 0.25 * (index / total))
|
| 1211 |
_validations_of(ws, cells, notes)
|
|
|
|
|
|
|
| 1212 |
cols, rows, widths, freeze, hidden_rows, hidden_cols, heights = _geometry_of(ws, notes)
|
| 1213 |
tab = {"id": "", "name": name[:sheets._MAX_NAME], "cols": cols, "rows": rows,
|
| 1214 |
"cells": cells, "usedExtent": sheets._used_extent(cells, excel["blankStyles"]),
|
|
|
|
| 43 |
import re
|
| 44 |
import zipfile
|
| 45 |
import hashlib
|
| 46 |
+
import io
|
| 47 |
import json
|
| 48 |
import math
|
| 49 |
import os
|
| 50 |
+
import posixpath
|
| 51 |
import tempfile
|
| 52 |
import xml.etree.ElementTree as ET
|
| 53 |
from datetime import date, datetime, time, timezone
|
|
|
|
| 394 |
if border is not None:
|
| 395 |
sides = {}
|
| 396 |
for side in sheets._BORDER_SIDES:
|
| 397 |
+
line = border_line(getattr(border, side, None), notes)
|
| 398 |
+
if line is not None:
|
| 399 |
+
sides[side] = line
|
| 400 |
if sides:
|
| 401 |
out["border"] = sides
|
|
|
|
| 402 |
|
| 403 |
fmt, note = classify_format(getattr(cell, "number_format", ""))
|
| 404 |
if fmt:
|
|
|
|
| 408 |
return out
|
| 409 |
|
| 410 |
|
| 411 |
+
#: S67-T14: the OOXML line styles that are not one of C2's seven, each drawn as its nearest.
|
| 412 |
+
#: The dash and dot variants keep their PATTERN over their weight, because a patterned line drawn
|
| 413 |
+
#: solid reads as a different border while a medium dash drawn at 1px still reads as dashed.
|
| 414 |
+
_BORDER_NEAREST = {
|
| 415 |
+
"mediumDashed": "dashed", "dashDot": "dashed", "dashDotDot": "dashed",
|
| 416 |
+
"mediumDashDot": "dashed", "mediumDashDotDot": "dashed", "slantDashDot": "dashed",
|
| 417 |
+
}
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def border_line(edge, notes):
|
| 421 |
+
"""One openpyxl border side as C2's `{style, color}`, or None when the side draws nothing.
|
| 422 |
+
|
| 423 |
+
The seven Excel names C2 shares pass through unchanged; the dash dot variants become their
|
| 424 |
+
nearest with a note naming the approximation. The colour is the explicit RGB when there is one;
|
| 425 |
+
an automatic colour is black, which is what Excel draws; a theme or indexed colour is drawn in
|
| 426 |
+
black too, with a note, because a colour nobody chose is never guessed (see `argb_of`).
|
| 427 |
+
"""
|
| 428 |
+
style = str(getattr(edge, "style", None) or "") if edge is not None else ""
|
| 429 |
+
if not style or style == "none":
|
| 430 |
+
return None
|
| 431 |
+
if style not in sheets.CELL_BORDER_STYLES:
|
| 432 |
+
nearest = _BORDER_NEAREST.get(style, "thin")
|
| 433 |
+
notes.add("a " + style + " border is drawn as " + nearest)
|
| 434 |
+
style = nearest
|
| 435 |
+
colour = getattr(edge, "color", None)
|
| 436 |
+
argb = argb_of(colour)
|
| 437 |
+
if argb:
|
| 438 |
+
hex_colour = "#" + argb.lower()
|
| 439 |
+
else:
|
| 440 |
+
hex_colour = "#000000"
|
| 441 |
+
kind = getattr(colour, "type", "") if colour is not None else ""
|
| 442 |
+
# β `type`, never `.auto`: openpyxl answers an unset descriptor with a (truthy) error
|
| 443 |
+
# STRING, so `colour.auto` reads as set on every theme colour.
|
| 444 |
+
automatic = (colour is None or kind == "auto"
|
| 445 |
+
or (kind == "indexed" and getattr(colour, "indexed", None) == 64)
|
| 446 |
+
or (kind == "rgb" and str(getattr(colour, "rgb", "")).upper() == "00000000"))
|
| 447 |
+
if not automatic:
|
| 448 |
+
notes.add("a theme or indexed border colour was drawn in black")
|
| 449 |
+
return {"style": style, "color": hex_colour}
|
| 450 |
+
|
| 451 |
+
|
| 452 |
# ββ values βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 453 |
|
| 454 |
def json_value(value):
|
|
|
|
| 1079 |
"That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
|
| 1080 |
|
| 1081 |
|
| 1082 |
+
# ββ sparklines (S67-T18) βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1083 |
+
|
| 1084 |
+
#: β S67-T18: Excel keeps sparkline groups in the worksheet's x14 extension, which openpyxl drops,
|
| 1085 |
+
#: so they are read from the worksheet PART itself. A group with no `type` draws lines, `column`
|
| 1086 |
+
#: draws columns and `stacked` draws win/loss bars (Excel's name for `sheets.SPARKLINE_KINDS`'s
|
| 1087 |
+
#: `winloss`). The part was already admitted by `guard_import_archive`, so this second read adds
|
| 1088 |
+
#: no new exposure.
|
| 1089 |
+
_X14 = "{http://schemas.microsoft.com/office/spreadsheetml/2009/9/main}"
|
| 1090 |
+
_XM = "{http://schemas.microsoft.com/office/excel/2006/main}"
|
| 1091 |
+
_MAIN_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
|
| 1092 |
+
_DOC_REL_NS = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
|
| 1093 |
+
_PKG_REL_NS = "{http://schemas.openxmlformats.org/package/2006/relationships}"
|
| 1094 |
+
_SPARKLINE_KIND_OF = {"line": "line", "column": "column", "stacked": "winloss"}
|
| 1095 |
+
|
| 1096 |
+
|
| 1097 |
+
def _worksheet_parts(archive):
|
| 1098 |
+
"""`{tab name: worksheet part}` through `xl/workbook.xml` and its relationships, never by index."""
|
| 1099 |
+
names = set(archive.namelist())
|
| 1100 |
+
if "xl/workbook.xml" not in names or "xl/_rels/workbook.xml.rels" not in names:
|
| 1101 |
+
return {}
|
| 1102 |
+
targets = {}
|
| 1103 |
+
for rel in ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")).iter(_PKG_REL_NS + "Relationship"):
|
| 1104 |
+
target = str(rel.get("Target") or "")
|
| 1105 |
+
targets[rel.get("Id")] = (target.lstrip("/") if target.startswith("/")
|
| 1106 |
+
else posixpath.normpath("xl/" + target))
|
| 1107 |
+
out = {}
|
| 1108 |
+
for sheet in ET.fromstring(archive.read("xl/workbook.xml")).iter(_MAIN_NS + "sheet"):
|
| 1109 |
+
part = targets.get(sheet.get(_DOC_REL_NS + "id"))
|
| 1110 |
+
if part in names:
|
| 1111 |
+
out[str(sheet.get("name") or "")] = part
|
| 1112 |
+
return out
|
| 1113 |
+
|
| 1114 |
+
|
| 1115 |
+
def _sparkline_groups(stream):
|
| 1116 |
+
"""`[(type, [(formula, sqref), ...]), ...]` for one worksheet part's x14 sparkline groups."""
|
| 1117 |
+
groups = []
|
| 1118 |
+
inside = 0 # a group's subtree is kept until the group itself ends
|
| 1119 |
+
for event, element in ET.iterparse(stream, events=("start", "end")):
|
| 1120 |
+
if element.tag == _X14 + "sparklineGroup":
|
| 1121 |
+
if event == "start":
|
| 1122 |
+
inside += 1
|
| 1123 |
+
continue
|
| 1124 |
+
inside -= 1
|
| 1125 |
+
entries = [((spark.findtext(_XM + "f") or "").strip(),
|
| 1126 |
+
(spark.findtext(_XM + "sqref") or "").strip())
|
| 1127 |
+
for spark in element.iter(_X14 + "sparkline")]
|
| 1128 |
+
groups.append((str(element.get("type") or "line"), entries))
|
| 1129 |
+
element.clear()
|
| 1130 |
+
elif event == "end" and not inside:
|
| 1131 |
+
element.clear()
|
| 1132 |
+
return groups
|
| 1133 |
+
|
| 1134 |
+
|
| 1135 |
+
def _sparklines_of(path, notes):
|
| 1136 |
+
"""`{tab name: groups}` for every worksheet that carries x14 sparklines; `{}` for the rest.
|
| 1137 |
+
|
| 1138 |
+
A part whose bytes never mention a sparkline group is not parsed at all, so a workbook with
|
| 1139 |
+
no sparklines imports exactly as it did before S67-T18.
|
| 1140 |
+
"""
|
| 1141 |
+
out = {}
|
| 1142 |
+
try:
|
| 1143 |
+
with zipfile.ZipFile(path) as archive:
|
| 1144 |
+
for name, part in _worksheet_parts(archive).items():
|
| 1145 |
+
raw = archive.read(part)
|
| 1146 |
+
if b"sparklineGroup" not in raw:
|
| 1147 |
+
continue
|
| 1148 |
+
groups = _sparkline_groups(io.BytesIO(raw))
|
| 1149 |
+
if groups:
|
| 1150 |
+
out[name] = groups
|
| 1151 |
+
except (KeyError, OSError, ET.ParseError, zipfile.BadZipFile):
|
| 1152 |
+
notes.add("the workbook's sparklines could not be read, so none were imported")
|
| 1153 |
+
return {}
|
| 1154 |
+
return out
|
| 1155 |
+
|
| 1156 |
+
|
| 1157 |
+
def _count_phrase(count, one, many):
|
| 1158 |
+
return (one if count == 1 else str(count) + " " + many)
|
| 1159 |
+
|
| 1160 |
+
|
| 1161 |
+
def _apply_sparklines(tab_name, cells, groups, blank_styles, notes):
|
| 1162 |
+
"""Write `cell.sparkline {kind, range}` on each location cell of this tab's groups.
|
| 1163 |
+
|
| 1164 |
+
`sheets._clean_sparkline` stores a rectangle LOCAL to its tab, so a sparkline drawn from
|
| 1165 |
+
another tab's data cannot be carried: it is left out and NAMED, never dropped in silence
|
| 1166 |
+
(the module note). A location that is not one cell, or data that is not a range, is named too.
|
| 1167 |
+
A blank location becomes an `empty` cell holding only the sparkline, which is exactly what
|
| 1168 |
+
`clean_cell`'s empty branch keeps; its compact blank-style reference moves onto it.
|
| 1169 |
+
"""
|
| 1170 |
+
other_tab = unreadable = 0
|
| 1171 |
+
for kind_name, entries in groups:
|
| 1172 |
+
kind = _SPARKLINE_KIND_OF.get(kind_name)
|
| 1173 |
+
if kind is None:
|
| 1174 |
+
unreadable += len(entries)
|
| 1175 |
+
continue
|
| 1176 |
+
for formula, sqref in entries:
|
| 1177 |
+
source = sheet_of_ref(formula)
|
| 1178 |
+
if source and source.casefold() != tab_name.casefold():
|
| 1179 |
+
other_tab += 1
|
| 1180 |
+
continue
|
| 1181 |
+
rect = rect_of(formula)
|
| 1182 |
+
places = sqref.replace("$", "").upper().split()
|
| 1183 |
+
if (rect is None or len(places) != 1 or ":" in places[0]
|
| 1184 |
+
or not _address_in_document(places[0])):
|
| 1185 |
+
unreadable += 1
|
| 1186 |
+
continue
|
| 1187 |
+
addr = places[0]
|
| 1188 |
+
spark = {"kind": kind, "range": rect}
|
| 1189 |
+
cell = cells.get(addr)
|
| 1190 |
+
if cell is None:
|
| 1191 |
+
if len(cells) >= sheets._MAX_CELLS_PER_SHEET:
|
| 1192 |
+
unreadable += 1
|
| 1193 |
+
continue
|
| 1194 |
+
cell = {"kind": "empty", "sparkline": spark}
|
| 1195 |
+
if addr in blank_styles:
|
| 1196 |
+
cell["excel"] = {"styleId": blank_styles.pop(addr)}
|
| 1197 |
+
cells[addr] = cell
|
| 1198 |
+
else:
|
| 1199 |
+
cell["sparkline"] = spark
|
| 1200 |
+
if other_tab:
|
| 1201 |
+
notes.add(_count_phrase(other_tab, "a sparkline", "sparklines")
|
| 1202 |
+
+ " drawn from another tab's data " + ("was" if other_tab == 1 else "were")
|
| 1203 |
+
+ " left out")
|
| 1204 |
+
if unreadable:
|
| 1205 |
+
notes.add(_count_phrase(unreadable, "a sparkline", "sparklines")
|
| 1206 |
+
+ " with a location or data range that could not be read "
|
| 1207 |
+
+ ("was" if unreadable == 1 else "were") + " left out")
|
| 1208 |
+
|
| 1209 |
+
|
| 1210 |
def _print_of(ws):
|
| 1211 |
"""Store print setup and inclusive print areas; export executes these settings later."""
|
| 1212 |
setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
|
|
|
|
| 1318 |
|
| 1319 |
counts = {"cells": 0, "formulas": 0}
|
| 1320 |
book_notes = set()
|
| 1321 |
+
sparklines = _sparklines_of(path, book_notes) # S67-T18
|
| 1322 |
authored = _authored_chart_document(path, book_notes)
|
| 1323 |
chart_tabs = {tab["name"]: tab for tab in authored["sheets"]} if authored else {}
|
| 1324 |
if authored and len(chart_tabs) != len(authored["sheets"]):
|
|
|
|
| 1380 |
cells = _cells_of(ws, cached, notes, counts, excel)
|
| 1381 |
_say("formats", 0.7 + 0.25 * (index / total))
|
| 1382 |
_validations_of(ws, cells, notes)
|
| 1383 |
+
if name in sparklines: # S67-T18, before usedExtent
|
| 1384 |
+
_apply_sparklines(name, cells, sparklines[name], excel["blankStyles"], notes)
|
| 1385 |
cols, rows, widths, freeze, hidden_rows, hidden_cols, heights = _geometry_of(ws, notes)
|
| 1386 |
tab = {"id": "", "name": name[:sheets._MAX_NAME], "cols": cols, "rows": rows,
|
| 1387 |
"cells": cells, "usedExtent": sheets._used_extent(cells, excel["blankStyles"]),
|
platform/core/sheet_structure.py
CHANGED
|
@@ -11,7 +11,8 @@ import math
|
|
| 11 |
import re
|
| 12 |
from collections.abc import Callable
|
| 13 |
|
| 14 |
-
from core.sheets import
|
|
|
|
| 15 |
|
| 16 |
|
| 17 |
REF_SOURCE = r"(?:(?:'([^']{1,64})'|([A-Za-z_][A-Za-z0-9_]{0,63}))!)?(\$?)([A-Za-z]{1,3})(\$?)([1-9][0-9]{0,6})"
|
|
@@ -162,8 +163,7 @@ def shift_for_structure(text: str, shift: dict, own_sheet: str, *, bare: bool =
|
|
| 162 |
|
| 163 |
return format_ref(at_index(first, move(start))) + separator + format_ref(at_index(last, move(end)))
|
| 164 |
|
| 165 |
-
|
| 166 |
-
return rewrite_refs(text, rewrite, rewrite_range if shift["delta"] < 0 else None)
|
| 167 |
|
| 168 |
|
| 169 |
def _moved_index(value: int, at: int, delta: int) -> int | None:
|
|
@@ -185,6 +185,11 @@ def _reanchor_binding(value, shift: dict, own_sheet: str) -> bool:
|
|
| 185 |
if key in BINDING_REF_KEYS and isinstance(child, str) and parse_ref(child):
|
| 186 |
value[key] = shift_for_structure(child, shift, own_sheet, bare=True)
|
| 187 |
changed = value[key] != child or changed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 188 |
elif isinstance(child, (dict, list)):
|
| 189 |
changed = _reanchor_binding(child, shift, own_sheet) or changed
|
| 190 |
return changed
|
|
@@ -206,7 +211,6 @@ def _reanchor_cells(tab: dict, shift: dict) -> None:
|
|
| 206 |
href = link.get("href")
|
| 207 |
if isinstance(href, str) and parse_ref(href):
|
| 208 |
link["href"] = shift_for_structure(href, shift, tab["name"], bare=True)
|
| 209 |
-
changed = link["href"] != href or changed
|
| 210 |
if changed and cell.get("kind") in ("formula", "bound"):
|
| 211 |
cell["state"] = "stale"
|
| 212 |
|
|
@@ -239,6 +243,28 @@ def _move_tab(tab: dict, op: dict) -> None:
|
|
| 239 |
continue
|
| 240 |
moved = _moved_index(ref["row"] if axis == "row" else col_to_index(ref["col"]), at, delta)
|
| 241 |
if moved is not None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
ref[axis] = moved if axis == "row" else index_to_col(moved)
|
| 243 |
cells[format_ref(ref)] = cell
|
| 244 |
tab["cells"] = cells
|
|
@@ -262,26 +288,78 @@ def _move_tab(tab: dict, op: dict) -> None:
|
|
| 262 |
if "rules" in tab:
|
| 263 |
rules = []
|
| 264 |
for rule in tab["rules"]:
|
| 265 |
-
rect =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
if rect is not None:
|
| 267 |
rules.append({**rule, "range": rect})
|
| 268 |
tab["rules"] = rules
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
|
| 270 |
|
| 271 |
-
def _move_charts(tab: dict, tab_id: str, op: dict) -> None:
|
| 272 |
for chart in tab.get("charts", []):
|
| 273 |
source = chart.get("source") or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 274 |
if source.get("sheetId", tab["id"]) == tab_id:
|
| 275 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 276 |
if "ranges" in source:
|
| 277 |
-
source["ranges"] = [
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
|
| 286 |
|
| 287 |
def apply_structure(book: dict, tab_id: str, op: dict) -> dict:
|
|
@@ -308,9 +386,31 @@ def apply_structure(book: dict, tab_id: str, op: dict) -> dict:
|
|
| 308 |
if not op["delta"]:
|
| 309 |
return result
|
| 310 |
shift = {**op, "sheet": target["name"]}
|
|
|
|
|
|
|
| 311 |
for tab in result["sheets"]:
|
| 312 |
_reanchor_cells(tab, shift)
|
| 313 |
-
_move_charts(tab, tab_id, op)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
if tab["id"] == tab_id:
|
| 315 |
_move_tab(tab, op)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
return result
|
|
|
|
| 11 |
import re
|
| 12 |
from collections.abc import Callable
|
| 13 |
|
| 14 |
+
from core.sheets import (DEFAULT_SHEET_COLS, DEFAULT_SHEET_ROWS, MAX_SHEET_COLS,
|
| 15 |
+
MAX_SHEET_ROWS, SheetRefusal)
|
| 16 |
|
| 17 |
|
| 18 |
REF_SOURCE = r"(?:(?:'([^']{1,64})'|([A-Za-z_][A-Za-z0-9_]{0,63}))!)?(\$?)([A-Za-z]{1,3})(\$?)([1-9][0-9]{0,6})"
|
|
|
|
| 163 |
|
| 164 |
return format_ref(at_index(first, move(start))) + separator + format_ref(at_index(last, move(end)))
|
| 165 |
|
| 166 |
+
return rewrite_refs(text, rewrite, rewrite_range)
|
|
|
|
| 167 |
|
| 168 |
|
| 169 |
def _moved_index(value: int, at: int, delta: int) -> int | None:
|
|
|
|
| 185 |
if key in BINDING_REF_KEYS and isinstance(child, str) and parse_ref(child):
|
| 186 |
value[key] = shift_for_structure(child, shift, own_sheet, bare=True)
|
| 187 |
changed = value[key] != child or changed
|
| 188 |
+
elif key in BINDING_REF_KEYS and isinstance(child, str) and child.strip().startswith("="):
|
| 189 |
+
# HOTFIX 2026-09-29: a filter value slot may hold a FORMULA (`=DATE($E$24,1,1)`);
|
| 190 |
+
# its references follow a row or column edit exactly as a formula cell's do.
|
| 191 |
+
value[key] = shift_for_structure(child, shift, own_sheet)
|
| 192 |
+
changed = value[key] != child or changed
|
| 193 |
elif isinstance(child, (dict, list)):
|
| 194 |
changed = _reanchor_binding(child, shift, own_sheet) or changed
|
| 195 |
return changed
|
|
|
|
| 211 |
href = link.get("href")
|
| 212 |
if isinstance(href, str) and parse_ref(href):
|
| 213 |
link["href"] = shift_for_structure(href, shift, tab["name"], bare=True)
|
|
|
|
| 214 |
if changed and cell.get("kind") in ("formula", "bound"):
|
| 215 |
cell["state"] = "stale"
|
| 216 |
|
|
|
|
| 243 |
continue
|
| 244 |
moved = _moved_index(ref["row"] if axis == "row" else col_to_index(ref["col"]), at, delta)
|
| 245 |
if moved is not None:
|
| 246 |
+
binding = cell.get("binding") or {}
|
| 247 |
+
if (cell.get("kind") == "bound" and binding.get("mode") == "view"
|
| 248 |
+
and binding.get("viewType") != "grid" and isinstance(binding.get("block"), dict)):
|
| 249 |
+
block = binding["block"]
|
| 250 |
+
size = {key: value if type(value := block.get(key)) is int and value > 0 else fallback
|
| 251 |
+
for key, fallback in (("cols", 8), ("rows", 20))}
|
| 252 |
+
key = "rows" if axis == "row" else "cols"
|
| 253 |
+
value = ref["row"] if axis == "row" else col_to_index(ref["col"])
|
| 254 |
+
end = value + size[key]
|
| 255 |
+
change = (delta if value < at < end else 0) if delta > 0 else -max(
|
| 256 |
+
0, min(end, at - delta) - max(value, at))
|
| 257 |
+
if change:
|
| 258 |
+
binding["block"] = {**size, key: max(1, size[key] + change)}
|
| 259 |
+
child = cell.get("viewChild")
|
| 260 |
+
origin = parse_ref(child.get("origin")) if isinstance(child, dict) else None
|
| 261 |
+
if origin:
|
| 262 |
+
index = _moved_index(origin["row"] if axis == "row" else col_to_index(origin["col"]), at, delta)
|
| 263 |
+
if index is None:
|
| 264 |
+
cell.pop("viewChild", None)
|
| 265 |
+
else:
|
| 266 |
+
origin[axis] = index if axis == "row" else index_to_col(index)
|
| 267 |
+
child["origin"] = format_ref(origin)
|
| 268 |
ref[axis] = moved if axis == "row" else index_to_col(moved)
|
| 269 |
cells[format_ref(ref)] = cell
|
| 270 |
tab["cells"] = cells
|
|
|
|
| 288 |
if "rules" in tab:
|
| 289 |
rules = []
|
| 290 |
for rule in tab["rules"]:
|
| 291 |
+
rect = rule["range"]
|
| 292 |
+
if isinstance(rect, str):
|
| 293 |
+
rect = shift_for_structure(rect, {**op, "sheet": tab["name"]}, tab["name"])
|
| 294 |
+
if rect == "#REF!":
|
| 295 |
+
continue
|
| 296 |
+
else:
|
| 297 |
+
rect = _move_range(rect, op, drop_deleted=True)
|
| 298 |
if rect is not None:
|
| 299 |
rules.append({**rule, "range": rect})
|
| 300 |
tab["rules"] = rules
|
| 301 |
+
if axis == "row" and "rowGroups" in tab:
|
| 302 |
+
groups = []
|
| 303 |
+
for group in tab["rowGroups"]:
|
| 304 |
+
rect = _move_range({"row0": group["start"] - 1, "row1": group["end"] - 1},
|
| 305 |
+
op, drop_deleted=True)
|
| 306 |
+
if rect is not None:
|
| 307 |
+
groups.append({**group, "start": rect["row0"] + 1, "end": rect["row1"] + 1})
|
| 308 |
+
if groups:
|
| 309 |
+
tab["rowGroups"] = groups
|
| 310 |
+
else:
|
| 311 |
+
tab.pop("rowGroups", None)
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def _move_chart_range(rect: dict, op: dict, extent: dict) -> dict:
|
| 315 |
+
"""Match types.ts::chartRangeAfterStructure, including its viewport clamp."""
|
| 316 |
+
start, end = op["axis"] + "0", op["axis"] + "1"
|
| 317 |
+
lo, hi = max(0, min(rect[start], rect[end])), max(0, max(rect[start], rect[end]))
|
| 318 |
+
moved = _move_range({**rect, start: lo, end: hi}, op)
|
| 319 |
+
lo, hi = moved[start], moved[end]
|
| 320 |
+
ceiling = max(0, extent["rows" if op["axis"] == "row" else "cols"] - 1)
|
| 321 |
+
if hi > ceiling:
|
| 322 |
+
lo, hi = max(0, lo - (hi - ceiling)), ceiling
|
| 323 |
+
return {**moved, start: min(lo, ceiling), end: min(max(lo, hi), ceiling)}
|
| 324 |
|
| 325 |
|
| 326 |
+
def _move_charts(tab: dict, tab_id: str, op: dict, extent: dict) -> None:
|
| 327 |
for chart in tab.get("charts", []):
|
| 328 |
source = chart.get("source") or {}
|
| 329 |
+
previous = chart.get("geometry") or {}
|
| 330 |
+
geometry = {"col": previous.get("col", min(chart["range"]["col0"], chart["range"]["col1"])),
|
| 331 |
+
"row": previous.get("row", min(chart["range"]["row0"], chart["range"]["row1"])),
|
| 332 |
+
"width": previous.get("width", 640), "height": previous.get("height", 320)}
|
| 333 |
+
if tab["id"] == tab_id:
|
| 334 |
+
point = _move_chart_range({"col0": geometry["col"], "col1": geometry["col"],
|
| 335 |
+
"row0": geometry["row"], "row1": geometry["row"]}, op, extent)
|
| 336 |
+
if point["col0"] != geometry["col"] or point["row0"] != geometry["row"]:
|
| 337 |
+
chart["geometry"] = {**geometry, "col": point["col0"], "row": point["row0"]}
|
| 338 |
+
anchor = chart.get("anchor")
|
| 339 |
+
if isinstance(anchor, dict) and op["axis"] in anchor:
|
| 340 |
+
moved = shift_index(anchor[op["axis"]] + 1, op["at"], op["delta"])
|
| 341 |
+
anchor[op["axis"]] = (op["at"] if moved is None else moved) - 1
|
| 342 |
if source.get("sheetId", tab["id"]) == tab_id:
|
| 343 |
+
rect = _move_chart_range(chart["range"], op, extent)
|
| 344 |
+
if (tab["id"] != tab_id and "geometry" not in chart
|
| 345 |
+
and (rect["col0"] != chart["range"]["col0"] or rect["row0"] != chart["range"]["row0"])):
|
| 346 |
+
chart["geometry"] = geometry
|
| 347 |
+
chart["range"] = rect
|
| 348 |
if "ranges" in source:
|
| 349 |
+
source["ranges"] = [_move_chart_range(rect, op, extent) for rect in source["ranges"]]
|
| 350 |
+
presentation = chart.get("presentation") or {}
|
| 351 |
+
if "series" in presentation:
|
| 352 |
+
series = []
|
| 353 |
+
for setting in presentation["series"]:
|
| 354 |
+
match = re.fullmatch(r"(column|row):([A-Z]+|\d+)", setting["key"])
|
| 355 |
+
if match and ("col" if match[1] == "column" else "row") == op["axis"]:
|
| 356 |
+
index = col_to_index(match[2]) if match[1] == "column" else int(match[2])
|
| 357 |
+
moved = _moved_index(index, op["at"], op["delta"])
|
| 358 |
+
if moved is None:
|
| 359 |
+
continue
|
| 360 |
+
setting["key"] = match[1] + ":" + (index_to_col(moved) if match[1] == "column" else str(moved))
|
| 361 |
+
series.append(setting)
|
| 362 |
+
presentation["series"] = series
|
| 363 |
|
| 364 |
|
| 365 |
def apply_structure(book: dict, tab_id: str, op: dict) -> dict:
|
|
|
|
| 386 |
if not op["delta"]:
|
| 387 |
return result
|
| 388 |
shift = {**op, "sheet": target["name"]}
|
| 389 |
+
extent = {"cols": min(MAX_SHEET_COLS, max(DEFAULT_SHEET_COLS, target.get("cols", DEFAULT_SHEET_COLS))),
|
| 390 |
+
"rows": min(MAX_SHEET_ROWS, max(DEFAULT_SHEET_ROWS, target.get("rows", DEFAULT_SHEET_ROWS)))}
|
| 391 |
for tab in result["sheets"]:
|
| 392 |
_reanchor_cells(tab, shift)
|
| 393 |
+
_move_charts(tab, tab_id, op, extent)
|
| 394 |
+
for rule in tab.get("rules", []):
|
| 395 |
+
if rule.get("kind") == "formula":
|
| 396 |
+
for key in ("value", "value2"):
|
| 397 |
+
if isinstance(rule.get(key), str):
|
| 398 |
+
rule[key] = shift_for_structure(rule[key], shift, tab["name"])
|
| 399 |
if tab["id"] == tab_id:
|
| 400 |
_move_tab(tab, op)
|
| 401 |
+
names = result.get("namedRanges") or []
|
| 402 |
+
# The lossless document accepts the older name -> ref dictionary as well as named rows.
|
| 403 |
+
if isinstance(names, dict):
|
| 404 |
+
for name, value in names.items():
|
| 405 |
+
if isinstance(value, str):
|
| 406 |
+
names[name] = shift_for_structure(value, shift, result["sheets"][0]["name"])
|
| 407 |
+
names = [value for value in names.values() if isinstance(value, dict)]
|
| 408 |
+
for named_range in names:
|
| 409 |
+
scope = named_range.get("sheet")
|
| 410 |
+
own_sheet = next((tab["name"] for tab in result["sheets"]
|
| 411 |
+
if tab["id"] == scope or tab["name"] == scope),
|
| 412 |
+
scope if scope is not None else result["sheets"][0]["name"])
|
| 413 |
+
for key in ("ref", "base"):
|
| 414 |
+
if isinstance(named_range.get(key), str):
|
| 415 |
+
named_range[key] = shift_for_structure(named_range[key], shift, own_sheet)
|
| 416 |
return result
|
platform/core/sheets.py
CHANGED
|
@@ -152,6 +152,14 @@ MAX_FONT_PT = 96
|
|
| 152 |
_STYLE_MARKS = ("bold", "italic", "underline")
|
| 153 |
_BORDER_SIDES = ("top", "right", "bottom", "left")
|
| 154 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 155 |
#: R8's indent, in steps rather than pixels: a stored pixel count would be a layout decision that
|
| 156 |
#: outlives every layout change, exactly as a stored hex outlives a palette.
|
| 157 |
_MAX_INDENT = 8
|
|
@@ -204,6 +212,12 @@ SHEET_RULE_OPS = ("gt", "lt", "gte", "lte", "eq", "neq", "between", "contains",
|
|
| 204 |
#: every rule written before this ticket already is, so writing the word onto them would make a
|
| 205 |
#: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`.
|
| 206 |
SHEET_RULE_KINDS = ("cell", "scale", "bar", "formula")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
# The client compiler accepts 500 characters per condition.
|
| 208 |
MAX_RULE_FORMULA_LENGTH = 500
|
| 209 |
|
|
@@ -267,10 +281,8 @@ _MAX_UNSUPPORTED = 200
|
|
| 267 |
CELL_COLOUR_PATTERN = r"^(--lp-[a-z0-9-]{1,40}|#[0-9a-f]{6})$"
|
| 268 |
_CELL_COLOUR = re.compile(CELL_COLOUR_PATTERN)
|
| 269 |
|
| 270 |
-
#:
|
| 271 |
-
#: `
|
| 272 |
-
#: only. Its existing storage allowance is unchanged; a cell fill's custom door does not widen
|
| 273 |
-
#: the conditional-format palettes or make either guard dead.
|
| 274 |
_COLOUR = re.compile(r"^[A-Za-z0-9#_-]{1,40}$")
|
| 275 |
|
| 276 |
_MAX_DECIMALS = 10
|
|
@@ -654,13 +666,44 @@ def _clean_style(raw):
|
|
| 654 |
|
| 655 |
border = raw.get("border")
|
| 656 |
if isinstance(border, dict):
|
| 657 |
-
sides = {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 658 |
if sides:
|
| 659 |
out["border"] = sides
|
| 660 |
|
| 661 |
return out or None
|
| 662 |
|
| 663 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 664 |
# ββ the cell βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 665 |
|
| 666 |
def _clean_validation(raw):
|
|
@@ -1100,9 +1143,22 @@ def _clean_rule_style(raw):
|
|
| 1100 |
return {}
|
| 1101 |
out = {}
|
| 1102 |
for key in _RULE_STYLE_COLOURS:
|
| 1103 |
-
token =
|
| 1104 |
-
if token
|
| 1105 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1106 |
for mark in _RULE_STYLE_MARKS:
|
| 1107 |
if raw.get(mark) is True:
|
| 1108 |
out[mark] = True
|
|
@@ -1114,21 +1170,37 @@ def _clean_rule_scale(raw):
|
|
| 1114 |
|
| 1115 |
β BOTH ENDS OR NEITHER. One end alone names no ramp at all, and storing half of one would
|
| 1116 |
leave the client looking up a pair that cannot match and falling back to a default scale the
|
| 1117 |
-
person never chose. The tokens
|
| 1118 |
-
|
| 1119 |
|
| 1120 |
β THE SCALE'S BOUNDS ARE NOT HERE. They ride `value` and `value2` on the rule itself, which
|
| 1121 |
`_clean_rules` already persists for every op that takes operands, so a colour scale needed no
|
| 1122 |
new numeric slot at all.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1123 |
"""
|
| 1124 |
if not isinstance(raw, dict):
|
| 1125 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1126 |
out = {}
|
| 1127 |
for key in ("low", "high"):
|
| 1128 |
token = _text(raw.get(key), 40)
|
| 1129 |
if not token or not _COLOUR.match(token):
|
| 1130 |
return None
|
| 1131 |
out[key] = token
|
|
|
|
| 1132 |
return out
|
| 1133 |
|
| 1134 |
|
|
@@ -1699,10 +1771,31 @@ def _append_write_log(tab, rev, addrs, *, actor="", actor_name=""):
|
|
| 1699 |
tab["writes"] = writes[-WRITE_LOG_REVS:]
|
| 1700 |
|
| 1701 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1702 |
def get_workbook(st, workbook_id):
|
| 1703 |
"""One workbook, or a named refusal when the document is not there."""
|
| 1704 |
wid = _slug_id(workbook_id)
|
| 1705 |
-
doc =
|
| 1706 |
if not doc or not doc.get("id"):
|
| 1707 |
raise SheetRefusal("workbook_missing",
|
| 1708 |
"That workbook no longer exists.")
|
|
|
|
| 152 |
_STYLE_MARKS = ("bold", "italic", "underline")
|
| 153 |
_BORDER_SIDES = ("top", "right", "bottom", "left")
|
| 154 |
|
| 155 |
+
#: ββ S67-T14 / CONTRACT C2: Excel's seven border line styles, name for name. Mirrored in
|
| 156 |
+
#: `aios-web/web/src/sheet/types.ts::CELL_BORDER_STYLES`, and `verify_sheet.py` compares the two
|
| 157 |
+
#: lists. A side is either the legacy `True` (read as a thin black line and stored unchanged, so a
|
| 158 |
+
#: document written before S67 reads back byte for byte) or `{"style": <one of these>,
|
| 159 |
+
#: "color": "#rrggbb"}`.
|
| 160 |
+
CELL_BORDER_STYLES = ("thin", "medium", "thick", "dashed", "dotted", "double", "hair")
|
| 161 |
+
_BORDER_COLOUR = re.compile(r"^#[0-9a-f]{6}$")
|
| 162 |
+
|
| 163 |
#: R8's indent, in steps rather than pixels: a stored pixel count would be a layout decision that
|
| 164 |
#: outlives every layout change, exactly as a stored hex outlives a palette.
|
| 165 |
_MAX_INDENT = 8
|
|
|
|
| 212 |
#: every rule written before this ticket already is, so writing the word onto them would make a
|
| 213 |
#: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`.
|
| 214 |
SHEET_RULE_KINDS = ("cell", "scale", "bar", "formula")
|
| 215 |
+
#: β S67-T13: Excel's colour scale point types, one list for both ends. `lowest` / `highest` are
|
| 216 |
+
#: measured from the range at paint, so the scale re-spreads as the data changes; the other three
|
| 217 |
+
#: read their number from the rule's `value` (low end) or `value2` (high end). A type outside this
|
| 218 |
+
#: list is REFUSED by name, never dropped: a dropped type reads back as a different scale. Mirrored
|
| 219 |
+
#: in `web/src/sheet/types.ts::SheetRuleScaleType`.
|
| 220 |
+
SHEET_RULE_SCALE_TYPES = ("lowest", "highest", "number", "percent", "percentile")
|
| 221 |
# The client compiler accepts 500 characters per condition.
|
| 222 |
MAX_RULE_FORMULA_LENGTH = 500
|
| 223 |
|
|
|
|
| 281 |
CELL_COLOUR_PATTERN = r"^(--lp-[a-z0-9-]{1,40}|#[0-9a-f]{6})$"
|
| 282 |
_CELL_COLOUR = re.compile(CELL_COLOUR_PATTERN)
|
| 283 |
|
| 284 |
+
#: Cell and rule `color` / `fill` use CELL_COLOUR_PATTERN above. This older guard now serves
|
| 285 |
+
#: `_clean_rule_scale` only; the scale's existing storage allowance is unchanged.
|
|
|
|
|
|
|
| 286 |
_COLOUR = re.compile(r"^[A-Za-z0-9#_-]{1,40}$")
|
| 287 |
|
| 288 |
_MAX_DECIMALS = 10
|
|
|
|
| 666 |
|
| 667 |
border = raw.get("border")
|
| 668 |
if isinstance(border, dict):
|
| 669 |
+
sides = {}
|
| 670 |
+
for side in _BORDER_SIDES:
|
| 671 |
+
line = _clean_border_side(border.get(side))
|
| 672 |
+
if line is not None:
|
| 673 |
+
sides[side] = line
|
| 674 |
if sides:
|
| 675 |
out["border"] = sides
|
| 676 |
|
| 677 |
return out or None
|
| 678 |
|
| 679 |
|
| 680 |
+
def _clean_border_side(raw):
|
| 681 |
+
"""One C2 border side: `True`, a `{style, color}` dict, or None when the side is absent.
|
| 682 |
+
|
| 683 |
+
ββ THE SECOND DOCUMENTED EXCEPTION TO "UNKNOWN KEYS ARE DROPPED IN SILENCE". A border side
|
| 684 |
+
that names a line style outside the seven, or a colour that is not a six digit hex, is
|
| 685 |
+
REFUSED by name. Dropping it would store a cell with no line where the person drew one, and
|
| 686 |
+
the caller could not learn its request went nowhere: `_clean_format`'s reason, one key over.
|
| 687 |
+
A side that is `False`, None or absent is simply no side. A legacy `True` stays `True`.
|
| 688 |
+
"""
|
| 689 |
+
if raw is None or raw is False:
|
| 690 |
+
return None
|
| 691 |
+
if raw is True:
|
| 692 |
+
return True
|
| 693 |
+
if not isinstance(raw, dict):
|
| 694 |
+
return None
|
| 695 |
+
style = raw.get("style")
|
| 696 |
+
colour = raw.get("color")
|
| 697 |
+
if isinstance(colour, str):
|
| 698 |
+
colour = colour.strip().lower()
|
| 699 |
+
if (not isinstance(style, str) or style not in CELL_BORDER_STYLES
|
| 700 |
+
or not isinstance(colour, str) or not _BORDER_COLOUR.match(colour)):
|
| 701 |
+
raise SheetRefusal(
|
| 702 |
+
"cell_border_invalid",
|
| 703 |
+
"A border needs a line style such as thin and a six digit hex colour such as #000000.")
|
| 704 |
+
return {"style": style, "color": colour}
|
| 705 |
+
|
| 706 |
+
|
| 707 |
# ββ the cell βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 708 |
|
| 709 |
def _clean_validation(raw):
|
|
|
|
| 1143 |
return {}
|
| 1144 |
out = {}
|
| 1145 |
for key in _RULE_STYLE_COLOURS:
|
| 1146 |
+
token = raw.get(key)
|
| 1147 |
+
if token is None or token == "":
|
| 1148 |
+
continue
|
| 1149 |
+
if isinstance(token, str):
|
| 1150 |
+
token = token.strip()
|
| 1151 |
+
if not token:
|
| 1152 |
+
continue
|
| 1153 |
+
if token.startswith("#"):
|
| 1154 |
+
token = token.lower()
|
| 1155 |
+
# Validate the whole value before normalising: neither truncation nor the cell
|
| 1156 |
+
# cleaner's three digit import rescue may turn an invalid rule colour into a choice.
|
| 1157 |
+
if not isinstance(token, str) or not _CELL_COLOUR.fullmatch(token):
|
| 1158 |
+
raise SheetRefusal(
|
| 1159 |
+
"sheet_rule_colour_invalid",
|
| 1160 |
+
"A rule colour must be a palette colour or a six digit hex colour such as #c8a24b.")
|
| 1161 |
+
out[key] = _normalise_colour(token)
|
| 1162 |
for mark in _RULE_STYLE_MARKS:
|
| 1163 |
if raw.get(mark) is True:
|
| 1164 |
out[mark] = True
|
|
|
|
| 1170 |
|
| 1171 |
β BOTH ENDS OR NEITHER. One end alone names no ramp at all, and storing half of one would
|
| 1172 |
leave the client looking up a pair that cannot match and falling back to a default scale the
|
| 1173 |
+
person never chose. The tokens keep their existing `_COLOUR` guard, so nothing free form
|
| 1174 |
+
reaches a canvas `fillStyle` from a stored document.
|
| 1175 |
|
| 1176 |
β THE SCALE'S BOUNDS ARE NOT HERE. They ride `value` and `value2` on the rule itself, which
|
| 1177 |
`_clean_rules` already persists for every op that takes operands, so a colour scale needed no
|
| 1178 |
new numeric slot at all.
|
| 1179 |
+
|
| 1180 |
+
β S67-T13: `minType` / `maxType` say what each end MEANS (`SHEET_RULE_SCALE_TYPES`). Each is
|
| 1181 |
+
kept only when present and never injected, so a stored boundless scale reads back byte for
|
| 1182 |
+
byte, and input with no types returns exactly the `{low, high}` it always did. An unknown type
|
| 1183 |
+
is refused by name.
|
| 1184 |
"""
|
| 1185 |
if not isinstance(raw, dict):
|
| 1186 |
return None
|
| 1187 |
+
types = {}
|
| 1188 |
+
for key in ("minType", "maxType"):
|
| 1189 |
+
kind = raw.get(key)
|
| 1190 |
+
if kind is None or kind == "":
|
| 1191 |
+
continue
|
| 1192 |
+
if kind not in SHEET_RULE_SCALE_TYPES:
|
| 1193 |
+
raise SheetRefusal(
|
| 1194 |
+
"sheet_rule_scale_type_unknown",
|
| 1195 |
+
"A colour scale end must be one of " + ", ".join(SHEET_RULE_SCALE_TYPES) + ".")
|
| 1196 |
+
types[key] = kind
|
| 1197 |
out = {}
|
| 1198 |
for key in ("low", "high"):
|
| 1199 |
token = _text(raw.get(key), 40)
|
| 1200 |
if not token or not _COLOUR.match(token):
|
| 1201 |
return None
|
| 1202 |
out[key] = token
|
| 1203 |
+
out.update(types)
|
| 1204 |
return out
|
| 1205 |
|
| 1206 |
|
|
|
|
| 1771 |
tab["writes"] = writes[-WRITE_LOG_REVS:]
|
| 1772 |
|
| 1773 |
|
| 1774 |
+
def _read_workbook_doc(st, key):
|
| 1775 |
+
"""A workbook document, `{}` when it is absent, and a REFUSAL when the store could not answer.
|
| 1776 |
+
|
| 1777 |
+
HOTFIX 2026-09-29: `_read` swallows every store error into `{}`, so an overloaded database
|
| 1778 |
+
(staging right after a restart: a pool timeout, a 503 on the next request) was reported as
|
| 1779 |
+
"That workbook no longer exists." and the browser dropped the owner's open tab. A store that
|
| 1780 |
+
fails is not a workbook that is gone; the reader is told to try again and nothing is dropped.
|
| 1781 |
+
"""
|
| 1782 |
+
try:
|
| 1783 |
+
projected = getattr(st, "get_projection", None)
|
| 1784 |
+
if callable(projected):
|
| 1785 |
+
return dict(projected(key, drop=()) or {})
|
| 1786 |
+
return dict(st.get(key) or {})
|
| 1787 |
+
except (KeyError, FileNotFoundError):
|
| 1788 |
+
return {}
|
| 1789 |
+
except Exception as exc: # noqa: BLE001
|
| 1790 |
+
raise SheetRefusal("workbook_unavailable",
|
| 1791 |
+
"This workbook could not be read just now, so nothing was changed. "
|
| 1792 |
+
"Try again in a moment.") from exc
|
| 1793 |
+
|
| 1794 |
+
|
| 1795 |
def get_workbook(st, workbook_id):
|
| 1796 |
"""One workbook, or a named refusal when the document is not there."""
|
| 1797 |
wid = _slug_id(workbook_id)
|
| 1798 |
+
doc = _read_workbook_doc(st, workbook_key(wid))
|
| 1799 |
if not doc or not doc.get("id"):
|
| 1800 |
raise SheetRefusal("workbook_missing",
|
| 1801 |
"That workbook no longer exists.")
|
platform/core/store_pg.py
CHANGED
|
@@ -36,16 +36,25 @@ worth hiding: a connection failure RAISES from `get()` here. Callers that must n
|
|
| 36 |
should catch it β silently returning `{}` from a database that is simply unreachable is how an
|
| 37 |
empty dict gets written back over real data.
|
| 38 |
"""
|
|
|
|
|
|
|
|
|
|
| 39 |
import json
|
| 40 |
import os
|
|
|
|
|
|
|
| 41 |
import threading
|
|
|
|
|
|
|
| 42 |
from urllib.parse import urlparse
|
| 43 |
|
| 44 |
import core.data_binding as data_binding
|
|
|
|
| 45 |
|
| 46 |
_URL_ENV = 'DATABASE_URL'
|
| 47 |
_LOCK = threading.RLock()
|
| 48 |
_POOL = {'pool': None, 'url': None, 'schema': None}
|
|
|
|
| 49 |
|
| 50 |
|
| 51 |
def _url():
|
|
@@ -149,15 +158,25 @@ def sandbox_tenants():
|
|
| 149 |
|
| 150 |
|
| 151 |
def isolated_staging_database():
|
| 152 |
-
"""
|
| 153 |
if data_binding.deployment_id() != _STAGING_SPACE:
|
| 154 |
return False
|
| 155 |
-
|
| 156 |
try:
|
| 157 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
except ValueError:
|
| 159 |
return False
|
| 160 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
|
| 162 |
|
| 163 |
def write_refusal(tenant_slug=None):
|
|
@@ -172,7 +191,7 @@ def write_refusal(tenant_slug=None):
|
|
| 172 |
4. anything else -> REFUSED
|
| 173 |
|
| 174 |
Rule 2 permits one narrow exception: the exact staging Space, an explicitly handed tenant,
|
| 175 |
-
and a DATABASE_URL
|
| 176 |
"""
|
| 177 |
slug = _effective_slug(tenant_slug)
|
| 178 |
if data_binding.is_production_deployment():
|
|
@@ -202,6 +221,29 @@ def write_refusal(tenant_slug=None):
|
|
| 202 |
f'intended.')
|
| 203 |
|
| 204 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
def check_write(tenant_slug=None, operation='write'):
|
| 206 |
"""Raise `StoreWriteRefused` unless this deployment may write this tenant. The pg door.
|
| 207 |
|
|
@@ -212,12 +254,44 @@ def check_write(tenant_slug=None, operation='write'):
|
|
| 212 |
a session-scoped fallback and tell the user their change was saved. A new type would have had
|
| 213 |
to re-earn both properties, and would have failed the second one silently.
|
| 214 |
"""
|
| 215 |
-
reason = write_refusal(tenant_slug)
|
| 216 |
if reason:
|
| 217 |
raise data_binding.StoreWriteRefused(
|
| 218 |
f'{operation} to tenant {_effective_slug(tenant_slug)!r} refused: {reason}')
|
| 219 |
|
| 220 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
def _pool():
|
| 222 |
"""The psycopg connection pool, opened once per process.
|
| 223 |
|
|
@@ -229,7 +303,10 @@ def _pool():
|
|
| 229 |
if not url:
|
| 230 |
raise RuntimeError(
|
| 231 |
f'{_URL_ENV} is not set; the Postgres store backend has nothing to connect to.')
|
| 232 |
-
|
|
|
|
|
|
|
|
|
|
| 233 |
return _POOL['pool']
|
| 234 |
try:
|
| 235 |
from psycopg_pool import ConnectionPool
|
|
@@ -249,9 +326,11 @@ def _pool():
|
|
| 249 |
# real requests, not be held awake by this process's idle pool for psycopg's 10-minute
|
| 250 |
# default. Thirty seconds still reuses a normal request burst without creating a resident
|
| 251 |
# database client.
|
|
|
|
| 252 |
_POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0,
|
| 253 |
-
max_idle=30.0)
|
| 254 |
_POOL['url'] = url
|
|
|
|
| 255 |
return _POOL['pool']
|
| 256 |
|
| 257 |
|
|
@@ -389,6 +468,22 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
|
|
| 389 |
}
|
| 390 |
|
| 391 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 392 |
def _get_many(names, tenant_slug=None):
|
| 393 |
"""Several ordinary documents in one round trip, keyed by requested store key."""
|
| 394 |
keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
|
|
@@ -435,6 +530,32 @@ def exists(name, tenant_slug=None):
|
|
| 435 |
DOCUMENT_CEILING_BYTES = 128 * 1024 * 1024
|
| 436 |
|
| 437 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
class StoreDocumentTooLarge(RuntimeError):
|
| 439 |
"""A write refused because the document would grow past `DOCUMENT_CEILING_BYTES`."""
|
| 440 |
|
|
@@ -733,6 +854,10 @@ class PgStore:
|
|
| 733 |
"""Bound multi-key read used by request-scoped read snapshots."""
|
| 734 |
return _get_many(names, tenant_slug=self.tenant_slug)
|
| 735 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 736 |
def _read_strict(self, name):
|
| 737 |
"""Interface parity with the HF Store. There, `get` is lenient (swallows a transient
|
| 738 |
failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
|
|
|
|
| 36 |
should catch it β silently returning `{}` from a database that is simply unreachable is how an
|
| 37 |
empty dict gets written back over real data.
|
| 38 |
"""
|
| 39 |
+
import atexit
|
| 40 |
+
import base64
|
| 41 |
+
import binascii
|
| 42 |
import json
|
| 43 |
import os
|
| 44 |
+
import ssl
|
| 45 |
+
import tempfile
|
| 46 |
import threading
|
| 47 |
+
from functools import lru_cache
|
| 48 |
+
from pathlib import Path
|
| 49 |
from urllib.parse import urlparse
|
| 50 |
|
| 51 |
import core.data_binding as data_binding
|
| 52 |
+
from core.user_tables import enforce_byte_budget as _BYTE_BUDGET_ENFORCER
|
| 53 |
|
| 54 |
_URL_ENV = 'DATABASE_URL'
|
| 55 |
_LOCK = threading.RLock()
|
| 56 |
_POOL = {'pool': None, 'url': None, 'schema': None}
|
| 57 |
+
_DATABASE_CA_ENV = 'AIOS_DATABASE_CA_PEM_B64'
|
| 58 |
|
| 59 |
|
| 60 |
def _url():
|
|
|
|
| 158 |
|
| 159 |
|
| 160 |
def isolated_staging_database():
|
| 161 |
+
"""Match the exact staging database; legacy host stamps are valid only on port 5432."""
|
| 162 |
if data_binding.deployment_id() != _STAGING_SPACE:
|
| 163 |
return False
|
| 164 |
+
stamp = str(os.environ.get(_ISOLATED_DATABASE_HOST_VAR) or '').strip()
|
| 165 |
try:
|
| 166 |
+
current = _database_endpoint(_url() or '')
|
| 167 |
+
if not stamp or not current[0]:
|
| 168 |
+
return False
|
| 169 |
+
if '/' in stamp or ':' in stamp:
|
| 170 |
+
return current == _database_endpoint('postgresql://' + stamp)
|
| 171 |
+
return current[1] == 5432 and current[0] == stamp.lower()
|
| 172 |
except ValueError:
|
| 173 |
return False
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def _database_endpoint(url):
|
| 177 |
+
parsed = urlparse(url)
|
| 178 |
+
return ((parsed.hostname or '').lower(), parsed.port or 5432,
|
| 179 |
+
parsed.path.lstrip('/').split('/', 1)[0])
|
| 180 |
|
| 181 |
|
| 182 |
def write_refusal(tenant_slug=None):
|
|
|
|
| 191 |
4. anything else -> REFUSED
|
| 192 |
|
| 193 |
Rule 2 permits one narrow exception: the exact staging Space, an explicitly handed tenant,
|
| 194 |
+
and a DATABASE_URL identity matching the isolated database stamped by the deploy must agree.
|
| 195 |
"""
|
| 196 |
slug = _effective_slug(tenant_slug)
|
| 197 |
if data_binding.is_production_deployment():
|
|
|
|
| 221 |
f'intended.')
|
| 222 |
|
| 223 |
|
| 224 |
+
def gate_write_refusal(tenant_slug=None):
|
| 225 |
+
"""`None` unless a GATE process is writing a production tenant, else why not. W-219.
|
| 226 |
+
|
| 227 |
+
ββ CONSULTED BEFORE `write_refusal`, AND IT IS THE HALF THAT CLOSES W-219. A lane container is
|
| 228 |
+
the exact isolated staging posture `write_refusal` rule 2 ALLOWS for tenant #0, so a gate run
|
| 229 |
+
there wrote staging's `t_royal_imports` (SDLC 61: `odoo_connector_config` `syncEvery=manual`,
|
| 230 |
+
`ut_odoo_invoices` disabled, four fixture keychain credentials). A gate writes fixture tenants
|
| 231 |
+
(`qa-b`, `qa-pg`) or nothing, on every deployment, whatever the environment grants: see
|
| 232 |
+
`data_binding.gate_process` for who counts as a gate and why the environment cannot opt out.
|
| 233 |
+
`write_refusal` itself is unchanged, so the product rule stays testable on its own.
|
| 234 |
+
"""
|
| 235 |
+
who = data_binding.gate_process()
|
| 236 |
+
if not who:
|
| 237 |
+
return None
|
| 238 |
+
slug = _effective_slug(tenant_slug)
|
| 239 |
+
if slug not in PRODUCTION_TENANTS:
|
| 240 |
+
return None
|
| 241 |
+
return (f'this process is a verification gate ({who}), and a gate never writes tenant '
|
| 242 |
+
f'{slug!r} ({_schema_for(slug)}), which holds real customer data, not even on isolated '
|
| 243 |
+
f'staging. A gate writes fixture tenants only: run it with STORE_BACKEND=hf and no '
|
| 244 |
+
f'DATABASE_URL (W-219).')
|
| 245 |
+
|
| 246 |
+
|
| 247 |
def check_write(tenant_slug=None, operation='write'):
|
| 248 |
"""Raise `StoreWriteRefused` unless this deployment may write this tenant. The pg door.
|
| 249 |
|
|
|
|
| 254 |
a session-scoped fallback and tell the user their change was saved. A new type would have had
|
| 255 |
to re-earn both properties, and would have failed the second one silently.
|
| 256 |
"""
|
| 257 |
+
reason = gate_write_refusal(tenant_slug) or write_refusal(tenant_slug)
|
| 258 |
if reason:
|
| 259 |
raise data_binding.StoreWriteRefused(
|
| 260 |
f'{operation} to tenant {_effective_slug(tenant_slug)!r} refused: {reason}')
|
| 261 |
|
| 262 |
|
| 263 |
+
class DatabaseCAError(RuntimeError):
|
| 264 |
+
"""The configured database CA cannot be used; TLS must never silently fall back."""
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
@lru_cache(maxsize=1)
|
| 268 |
+
def _database_ca_file(encoded):
|
| 269 |
+
"""Decode a configured CA once per process and retain its private file for libpq."""
|
| 270 |
+
try:
|
| 271 |
+
pem = base64.b64decode(encoded, validate=True)
|
| 272 |
+
if not (pem.strip().startswith(b'-----BEGIN CERTIFICATE-----')
|
| 273 |
+
and pem.strip().endswith(b'-----END CERTIFICATE-----')):
|
| 274 |
+
raise ValueError
|
| 275 |
+
ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT).load_verify_locations(cadata=pem.decode('ascii'))
|
| 276 |
+
except (ValueError, binascii.Error, ssl.SSLError):
|
| 277 |
+
raise DatabaseCAError(
|
| 278 |
+
f'{_DATABASE_CA_ENV} must be base64-encoded PEM certificate data') from None
|
| 279 |
+
try:
|
| 280 |
+
fd, name = tempfile.mkstemp(prefix='aios-database-ca-', suffix='.crt')
|
| 281 |
+
path = Path(name)
|
| 282 |
+
try:
|
| 283 |
+
with os.fdopen(fd, 'wb') as target:
|
| 284 |
+
os.chmod(path, 0o600)
|
| 285 |
+
target.write(pem)
|
| 286 |
+
except BaseException:
|
| 287 |
+
path.unlink(missing_ok=True)
|
| 288 |
+
raise
|
| 289 |
+
except OSError:
|
| 290 |
+
raise DatabaseCAError(f'{_DATABASE_CA_ENV} could not create a private CA file') from None
|
| 291 |
+
atexit.register(path.unlink, missing_ok=True)
|
| 292 |
+
return str(path)
|
| 293 |
+
|
| 294 |
+
|
| 295 |
def _pool():
|
| 296 |
"""The psycopg connection pool, opened once per process.
|
| 297 |
|
|
|
|
| 303 |
if not url:
|
| 304 |
raise RuntimeError(
|
| 305 |
f'{_URL_ENV} is not set; the Postgres store backend has nothing to connect to.')
|
| 306 |
+
ca = os.environ.get('AIOS_DATABASE_CA_PEM_B64')
|
| 307 |
+
sslrootcert = _database_ca_file(ca) if ca is not None else None
|
| 308 |
+
if (_POOL['pool'] is not None and _POOL['url'] == url
|
| 309 |
+
and _POOL.get('sslrootcert') == sslrootcert):
|
| 310 |
return _POOL['pool']
|
| 311 |
try:
|
| 312 |
from psycopg_pool import ConnectionPool
|
|
|
|
| 326 |
# real requests, not be held awake by this process's idle pool for psycopg's 10-minute
|
| 327 |
# default. Thirty seconds still reuses a normal request burst without creating a resident
|
| 328 |
# database client.
|
| 329 |
+
options = {'kwargs': {'sslrootcert': sslrootcert}} if sslrootcert is not None else {}
|
| 330 |
_POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0,
|
| 331 |
+
max_idle=30.0, **options)
|
| 332 |
_POOL['url'] = url
|
| 333 |
+
_POOL['sslrootcert'] = sslrootcert
|
| 334 |
return _POOL['pool']
|
| 335 |
|
| 336 |
|
|
|
|
| 468 |
}
|
| 469 |
|
| 470 |
|
| 471 |
+
def _get_entry(name, entry, tenant_slug=None):
|
| 472 |
+
"""ONE top-level entry of document `name` (`value -> entry`), or None when either is absent.
|
| 473 |
+
|
| 474 |
+
HOTFIX 2026-09-28. Private for `_get_projections`' reason: an optimization offered by a bound
|
| 475 |
+
`PgStore` handle, never a store operation, so callers keep their whole-read fallback. A
|
| 476 |
+
stored `ut_*` table read this way crosses the wire as that table alone, not the whole
|
| 477 |
+
`user_tables` document (~60-100 MB on staging, ~3 s per read, holding a pool connection).
|
| 478 |
+
"""
|
| 479 |
+
with _pool().connection() as con:
|
| 480 |
+
row = con.execute(
|
| 481 |
+
_q('SELECT value -> %s FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
|
| 482 |
+
(str(entry), str(name))).fetchone()
|
| 483 |
+
value = row[0] if row else None
|
| 484 |
+
return dict(value) if isinstance(value, dict) else None
|
| 485 |
+
|
| 486 |
+
|
| 487 |
def _get_many(names, tenant_slug=None):
|
| 488 |
"""Several ordinary documents in one round trip, keyed by requested store key."""
|
| 489 |
keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
|
|
|
|
| 530 |
DOCUMENT_CEILING_BYTES = 128 * 1024 * 1024
|
| 531 |
|
| 532 |
|
| 533 |
+
def byte_budget_status():
|
| 534 |
+
"""Inspect the shipped limits and enforcer without reading or writing a store."""
|
| 535 |
+
from core import user_tables as ut
|
| 536 |
+
|
| 537 |
+
reasons = []
|
| 538 |
+
limits = (
|
| 539 |
+
("MAX_TABLE_BYTES", ut.MAX_TABLE_BYTES, 48 * 1024 * 1024),
|
| 540 |
+
("MAX_DOCUMENT_BYTES", ut.MAX_DOCUMENT_BYTES, 128 * 1024 * 1024),
|
| 541 |
+
("DOCUMENT_CEILING_BYTES", DOCUMENT_CEILING_BYTES, 128 * 1024 * 1024),
|
| 542 |
+
)
|
| 543 |
+
for name, value, shipped in limits:
|
| 544 |
+
# bool is an int subclass, but is not a byte limit.
|
| 545 |
+
if type(value) is not int or value <= 0:
|
| 546 |
+
reasons.append(f"{name} must be a positive integer")
|
| 547 |
+
elif value > shipped:
|
| 548 |
+
reasons.append(f"{name} exceeds the shipped {shipped // 1048576} MB limit")
|
| 549 |
+
if (ut.enforce_byte_budget is not _BYTE_BUDGET_ENFORCER
|
| 550 |
+
or getattr(_BYTE_BUDGET_ENFORCER, "__module__", None) != "core.user_tables"
|
| 551 |
+
or getattr(_BYTE_BUDGET_ENFORCER, "__name__", None) != "enforce_byte_budget"):
|
| 552 |
+
reasons.append("user_tables.enforce_byte_budget is not the original enforcer")
|
| 553 |
+
return {"armed": not reasons, "reasons": reasons,
|
| 554 |
+
"max_table_bytes": ut.MAX_TABLE_BYTES,
|
| 555 |
+
"max_document_bytes": ut.MAX_DOCUMENT_BYTES,
|
| 556 |
+
"document_ceiling_bytes": DOCUMENT_CEILING_BYTES}
|
| 557 |
+
|
| 558 |
+
|
| 559 |
class StoreDocumentTooLarge(RuntimeError):
|
| 560 |
"""A write refused because the document would grow past `DOCUMENT_CEILING_BYTES`."""
|
| 561 |
|
|
|
|
| 854 |
"""Bound multi-key read used by request-scoped read snapshots."""
|
| 855 |
return _get_many(names, tenant_slug=self.tenant_slug)
|
| 856 |
|
| 857 |
+
def get_entry(self, name, entry):
|
| 858 |
+
"""Bound single-entry read; None means absent, and the caller falls back to `get`."""
|
| 859 |
+
return _get_entry(name, entry, tenant_slug=self.tenant_slug)
|
| 860 |
+
|
| 861 |
def _read_strict(self, name):
|
| 862 |
"""Interface parity with the HF Store. There, `get` is lenient (swallows a transient
|
| 863 |
failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
|
platform/core/user_tables.py
CHANGED
|
@@ -46,6 +46,7 @@ inherits C1 by construction β there is no list to add it to, which is R6's act
|
|
| 46 |
"""
|
| 47 |
import datetime as _dt
|
| 48 |
import hashlib
|
|
|
|
| 49 |
import json as _json
|
| 50 |
import math
|
| 51 |
import re
|
|
@@ -278,6 +279,11 @@ def is_connected(table_key, st=None):
|
|
| 278 |
table = all_defs(st).get(key) or {}
|
| 279 |
if not table:
|
| 280 |
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
if table.get('connected') is True:
|
| 282 |
return True
|
| 283 |
return (any(key.startswith(p) for p in _CONNECTED_PREFIXES)
|
|
@@ -285,6 +291,26 @@ def is_connected(table_key, st=None):
|
|
| 285 |
and table.get('recordMode') == AUTOMATION_RECORD_MODE)
|
| 286 |
|
| 287 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
#: ββ W30-T31 (D-87) β table keys whose rows DO NOT LIVE HERE AT ALL.
|
| 289 |
#:
|
| 290 |
#: A subset of `_CONNECTED`, and the distinction is the whole ticket. *Connected* says where the
|
|
@@ -1572,6 +1598,28 @@ def get(table_key, st=None):
|
|
| 1572 |
return all_tables(st).get(str(table_key))
|
| 1573 |
|
| 1574 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1575 |
# --- DURABLE MAP ZONES -------------------------------------------------------
|
| 1576 |
# Zones intentionally do not live in a view config: a view is a person's presentation of rows,
|
| 1577 |
# while a named geographic boundary is tenant data used by several views and routes. Keeping a
|
|
@@ -1807,7 +1855,7 @@ def create(label, username, fields=None, source=None, st=None, *, managed_by=Non
|
|
| 1807 |
if not label:
|
| 1808 |
return None
|
| 1809 |
existing = all_tables(st)
|
| 1810 |
-
if
|
| 1811 |
return None
|
| 1812 |
key = KEY_PREFIX + _slug(label)
|
| 1813 |
from core import route_system
|
|
@@ -5177,6 +5225,23 @@ def unread_note_subjects(subjects, viewer, *, st=None):
|
|
| 5177 |
return out
|
| 5178 |
|
| 5179 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5180 |
def nav_entries(viewer=None, is_admin=False, st=None):
|
| 5181 |
"""Registry-SHAPED dicts so the nav can render user tables beside real modules with no
|
| 5182 |
special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
|
|
@@ -5206,8 +5271,7 @@ def nav_entries(viewer=None, is_admin=False, st=None):
|
|
| 5206 |
if route_system.is_managed(t) or route_system.is_managed(key):
|
| 5207 |
continue
|
| 5208 |
source = str(t.get('source') or BLANK_SOURCE)
|
| 5209 |
-
source_group = (
|
| 5210 |
-
else 'Custom' if source == BLANK_SOURCE else source)
|
| 5211 |
out.append({'key': key, 'label': t.get('label') or key,
|
| 5212 |
'source': source, 'sourceGroup': source_group,
|
| 5213 |
'user_table': True, 'nav': True, 'validate': False})
|
|
|
|
| 46 |
"""
|
| 47 |
import datetime as _dt
|
| 48 |
import hashlib
|
| 49 |
+
import inspect
|
| 50 |
import json as _json
|
| 51 |
import math
|
| 52 |
import re
|
|
|
|
| 279 |
table = all_defs(st).get(key) or {}
|
| 280 |
if not table:
|
| 281 |
return False
|
| 282 |
+
return _connected_definition(key, table)
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def _connected_definition(key, table):
|
| 286 |
+
"""`is_connected`'s answers 2 and 3, read off a definition the caller already holds."""
|
| 287 |
if table.get('connected') is True:
|
| 288 |
return True
|
| 289 |
return (any(key.startswith(p) for p in _CONNECTED_PREFIXES)
|
|
|
|
| 291 |
and table.get('recordMode') == AUTOMATION_RECORD_MODE)
|
| 292 |
|
| 293 |
|
| 294 |
+
def counts_toward_table_cap(table_key, definition=None):
|
| 295 |
+
"""Does this database take one of the tenant's `MAX_TABLES` slots?
|
| 296 |
+
|
| 297 |
+
ββ OWNER RULING 2026-09-27 (D-3284): CONNECTOR-OWNED DATABASES DO NOT. The cap bounds what
|
| 298 |
+
PEOPLE create. A connector spawns one database per report (Amazon alone about sixty), and
|
| 299 |
+
counting those hid ten Amazon databases behind the cap with their rows sitting unreachable in
|
| 300 |
+
the mirror. The same answers as `is_connected`, off a definition the caller holds, because a
|
| 301 |
+
spawn's updater holds the raw document rather than a store.
|
| 302 |
+
"""
|
| 303 |
+
key = str(table_key or '').strip()
|
| 304 |
+
if key in _CONNECTED:
|
| 305 |
+
return False
|
| 306 |
+
return not _connected_definition(key, definition if isinstance(definition, dict) else {})
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
def capped_table_count(tables):
|
| 310 |
+
"""How many `MAX_TABLES` slots `{key: definition}` uses. Connector databases take none."""
|
| 311 |
+
return sum(1 for k, d in (tables or {}).items() if counts_toward_table_cap(k, d))
|
| 312 |
+
|
| 313 |
+
|
| 314 |
#: ββ W30-T31 (D-87) β table keys whose rows DO NOT LIVE HERE AT ALL.
|
| 315 |
#:
|
| 316 |
#: A subset of `_CONNECTED`, and the distinction is the whole ticket. *Connected* says where the
|
|
|
|
| 1598 |
return all_tables(st).get(str(table_key))
|
| 1599 |
|
| 1600 |
|
| 1601 |
+
def get_one(table_key, st=None):
|
| 1602 |
+
"""`get`, reading ONLY this table's entry when the bound store can (HOTFIX 2026-09-28).
|
| 1603 |
+
|
| 1604 |
+
`get` deep-copies the whole tenant document to answer for one table. On a Postgres-backed
|
| 1605 |
+
runtime `get_entry` reads `value -> table_key` instead, so a 1,168-row stored table no longer
|
| 1606 |
+
costs a ~60-100 MB read. β ONLY A CLASS THAT DEFINES `get_entry` QUALIFIES (`getattr_static`),
|
| 1607 |
+
never one that FORWARDS it: `_Lent`, the external-API draft runtime and the other snapshot
|
| 1608 |
+
wrappers serve documents of their own and pass the rest through `__getattr__`, so a plain
|
| 1609 |
+
`getattr` would read the live store past the snapshot (or the drafted rows) they promised.
|
| 1610 |
+
Anything else (no capability, an absent entry, a failed read) falls back to `get` unchanged.
|
| 1611 |
+
"""
|
| 1612 |
+
if (st is not None and not isinstance(st, _Lent)
|
| 1613 |
+
and callable(inspect.getattr_static(type(st), 'get_entry', None))):
|
| 1614 |
+
try:
|
| 1615 |
+
one = st.get_entry(STORE_KEY, str(table_key))
|
| 1616 |
+
except Exception: # noqa: BLE001
|
| 1617 |
+
one = None
|
| 1618 |
+
if isinstance(one, dict):
|
| 1619 |
+
return one
|
| 1620 |
+
return get(table_key, st)
|
| 1621 |
+
|
| 1622 |
+
|
| 1623 |
# --- DURABLE MAP ZONES -------------------------------------------------------
|
| 1624 |
# Zones intentionally do not live in a view config: a view is a person's presentation of rows,
|
| 1625 |
# while a named geographic boundary is tenant data used by several views and routes. Keeping a
|
|
|
|
| 1855 |
if not label:
|
| 1856 |
return None
|
| 1857 |
existing = all_tables(st)
|
| 1858 |
+
if capped_table_count(existing) >= MAX_TABLES: # connector databases take no slot (D-3284)
|
| 1859 |
return None
|
| 1860 |
key = KEY_PREFIX + _slug(label)
|
| 1861 |
from core import route_system
|
|
|
|
| 5225 |
return out
|
| 5226 |
|
| 5227 |
|
| 5228 |
+
def database_source_group(key, source=None):
|
| 5229 |
+
"""Default origin folder; a user's placement is applied by the Data index consumer."""
|
| 5230 |
+
from core.connector_spec import connector_of
|
| 5231 |
+
|
| 5232 |
+
connector = connector_of(key)
|
| 5233 |
+
if connector is None:
|
| 5234 |
+
# Connector registries can lag newly synced tables. Monid's ut_amazon_ keys
|
| 5235 |
+
# are Loopable-made, so only the connector-owned ut_amzn_ prefix belongs here.
|
| 5236 |
+
connector = next((name for prefix, name in (
|
| 5237 |
+
('ut_amzn_', 'amazon'), ('ut_meta_', 'meta'), ('ut_odoo_', 'odoo'),
|
| 5238 |
+
) if key.startswith(prefix)), None)
|
| 5239 |
+
if connector is None and (key in {'customer_data', 'product_data'}
|
| 5240 |
+
or str(source or '').lower() == 'odoo'):
|
| 5241 |
+
connector = 'odoo'
|
| 5242 |
+
return {'amazon': 'Amazon', 'meta': 'Meta Ads', 'odoo': 'Odoo'}.get(connector, 'Loopable')
|
| 5243 |
+
|
| 5244 |
+
|
| 5245 |
def nav_entries(viewer=None, is_admin=False, st=None):
|
| 5246 |
"""Registry-SHAPED dicts so the nav can render user tables beside real modules with no
|
| 5247 |
special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
|
|
|
|
| 5271 |
if route_system.is_managed(t) or route_system.is_managed(key):
|
| 5272 |
continue
|
| 5273 |
source = str(t.get('source') or BLANK_SOURCE)
|
| 5274 |
+
source_group = database_source_group(key, source)
|
|
|
|
| 5275 |
out.append({'key': key, 'label': t.get('label') or key,
|
| 5276 |
'source': source, 'sourceGroup': source_group,
|
| 5277 |
'user_table': True, 'nav': True, 'validate': False})
|
platform/core/users.py
CHANGED
|
@@ -566,6 +566,15 @@ def registry():
|
|
| 566 |
return store.get('users')
|
| 567 |
|
| 568 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 569 |
def ensure_bootstrap():
|
| 570 |
"""Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
|
| 571 |
|
|
@@ -700,6 +709,7 @@ def verify(username, pw):
|
|
| 700 |
|
| 701 |
``APP_PASSWORD`` may have seeded the first admin record, but it is never consulted here. A
|
| 702 |
missing/unreachable registry therefore fails closed instead of minting an MFA-exempt identity.
|
|
|
|
| 703 |
"""
|
| 704 |
username = (username or '').strip().lower()
|
| 705 |
if not username or not pw:
|
|
@@ -707,8 +717,8 @@ def verify(username, pw):
|
|
| 707 |
try:
|
| 708 |
# read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
|
| 709 |
reg = store.get('users', fresh=True)
|
| 710 |
-
except Exception:
|
| 711 |
-
|
| 712 |
username, u = _resolve_identifier(reg, username)
|
| 713 |
# β KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
|
| 714 |
# R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
|
|
|
|
| 566 |
return store.get('users')
|
| 567 |
|
| 568 |
|
| 569 |
+
class RegistryUnavailable(RuntimeError):
|
| 570 |
+
"""The account registry could not be READ, which is not the same fact as "no such account".
|
| 571 |
+
|
| 572 |
+
HOTFIX 2026-09-28: `verify` used to fold a failed read into `{}`, so a busy store (a
|
| 573 |
+
`PoolTimeout` on staging) answered a CORRECT password with "that username and password do
|
| 574 |
+
not match" and charged the attempt to the lockout budget. Still fail-closed: the caller
|
| 575 |
+
answers 503, it never admits anyone."""
|
| 576 |
+
|
| 577 |
+
|
| 578 |
def ensure_bootstrap():
|
| 579 |
"""Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
|
| 580 |
|
|
|
|
| 709 |
|
| 710 |
``APP_PASSWORD`` may have seeded the first admin record, but it is never consulted here. A
|
| 711 |
missing/unreachable registry therefore fails closed instead of minting an MFA-exempt identity.
|
| 712 |
+
An unreadable registry raises `RegistryUnavailable` rather than reading as an unknown user.
|
| 713 |
"""
|
| 714 |
username = (username or '').strip().lower()
|
| 715 |
if not username or not pw:
|
|
|
|
| 717 |
try:
|
| 718 |
# read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
|
| 719 |
reg = store.get('users', fresh=True)
|
| 720 |
+
except Exception as exc:
|
| 721 |
+
raise RegistryUnavailable(f'{type(exc).__name__}: {exc}') from exc
|
| 722 |
username, u = _resolve_identifier(reg, username)
|
| 723 |
# β KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
|
| 724 |
# R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
|
platform/harness/amazon_client.py
CHANGED
|
@@ -21,19 +21,19 @@ disregards AWS SigV4 entirely and authorizes on the LWA access token alone: a pl
|
|
| 21 |
signing requests is describing a dead API.
|
| 22 |
Β· https://developer-docs.amazon.com/sp-api/changelog/sp-api-will-no-longer-require-aws-iam-or-aws-signature-version-4
|
| 23 |
|
| 24 |
-
β **READ-ONLY, ENFORCED HERE, WITH EXACTLY
|
| 25 |
GET before it builds a request. SP-API can create feeds, send buyer messages and adjust listings;
|
| 26 |
none of that is reachable from this module, and the refusal sits below every route and payload so no
|
| 27 |
caller can opt out.
|
| 28 |
|
| 29 |
-
β **The exceptions are `_create_report`
|
| 30 |
rather than hidden.** Asking Amazon to BUILD a report is a POST, and Product Pricing's
|
| 31 |
-
`getCompetitiveSummary`
|
| 32 |
their path as a module constant rather than a parameter, so no caller can redirect either one. They
|
| 33 |
cannot alter a listing, price, order or inventory level, and cannot send a buyer anything. Full
|
| 34 |
reasoning sits on their path constants.
|
| 35 |
|
| 36 |
-
β **
|
| 37 |
operation per selling-partner/application pair, and Amazon treats throttling as evidence of a wrong
|
| 38 |
call pattern rather than a wrong ceiling. Standing rule 1 forbids a silent cap on a connector's
|
| 39 |
records, and its second half is what applies: a limit that genuinely cannot be removed must be
|
|
@@ -47,8 +47,10 @@ from __future__ import annotations
|
|
| 47 |
import contextvars
|
| 48 |
import json
|
| 49 |
import random
|
|
|
|
| 50 |
import threading
|
| 51 |
import time
|
|
|
|
| 52 |
import urllib.error
|
| 53 |
import urllib.parse
|
| 54 |
import urllib.request
|
|
@@ -100,6 +102,62 @@ _report_pacer = {
|
|
| 100 |
_entity_pacers = {
|
| 101 |
"competitive_summary": {"last": None, "lock": threading.Lock()},
|
| 102 |
"awd_inventory": {"last": None, "lock": threading.Lock()},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
}
|
| 104 |
|
| 105 |
#: β MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
|
|
@@ -179,6 +237,9 @@ CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "seller_id",
|
|
| 179 |
#: production host, whereas a production credential wrongly marked sandbox would quietly read mocks.
|
| 180 |
OPTIONAL_FIELDS = ("sandbox", "region")
|
| 181 |
|
|
|
|
|
|
|
|
|
|
| 182 |
#: β THE SCHEMA SPINE (owner rulings R4 + R14: seven tables, and NOTHING ELSE). Each row names
|
| 183 |
#: the table, the bulk report
|
| 184 |
#: type that fills it, and the summary aggregate its `validate()` reconciles against.
|
|
@@ -230,6 +291,7 @@ REPORT_TYPES = {
|
|
| 230 |
},
|
| 231 |
"listings": {
|
| 232 |
"report": "GET_MERCHANT_LISTINGS_ALL_DATA",
|
|
|
|
| 233 |
"validates_against": "listing report row count vs our listing rows",
|
| 234 |
"role": "Product Listing",
|
| 235 |
},
|
|
@@ -254,6 +316,8 @@ REPORT_TYPES = {
|
|
| 254 |
},
|
| 255 |
"inventory": {
|
| 256 |
"report": "GET_LEDGER_SUMMARY_VIEW_DATA",
|
|
|
|
|
|
|
| 257 |
"entity_read": "AWD v2024-05-09 listInventory (all 200-row pages)",
|
| 258 |
"validates_against": "FBA ledger summary vs detail and AWD API items vs stored AWD rows",
|
| 259 |
"role": "Amazon Fulfillment / Amazon Warehousing and Distribution",
|
|
@@ -273,6 +337,7 @@ REPORT_TYPES = {
|
|
| 273 |
# page loop over a full catalogue is precisely what rule 1 forbids.
|
| 274 |
"pricing": {
|
| 275 |
"report": "GET_SALES_AND_TRAFFIC_REPORT",
|
|
|
|
| 276 |
"report_covers": "buy box SHARE only; PRICE has no bulk path",
|
| 277 |
"price_path": "Product Pricing v2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)",
|
| 278 |
"validates_against": "sales and traffic report totals and competitive ASIN responses",
|
|
@@ -283,11 +348,630 @@ REPORT_TYPES = {
|
|
| 283 |
# wave's one UNPROVEN ticket with its exit condition written in advance: returns >0 rows
|
| 284 |
# for any registered brand, else demote to a declared-but-unsynced stub.
|
| 285 |
"report": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT",
|
|
|
|
| 286 |
"validates_against": "report period totals vs our search-term rows",
|
| 287 |
"role": "Brand Analytics",
|
| 288 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
}
|
| 290 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 291 |
#: Eight of the ten roles Amazon approved on 2026-07-22. Buyer Communication and Buyer Solicitation
|
| 292 |
#: are REFUSED (owner ruling R7): both are write-shaped and PII-bearing, and it is precisely those
|
| 293 |
#: restricted roles that trigger Phase 1 verification, a Phase 2 data-security assessment and an
|
|
@@ -312,13 +996,14 @@ REPORT_ROLES = {
|
|
| 312 |
}
|
| 313 |
_REPORT_ROLE_INDEX = {"inventory": 0, "pricing": 1}
|
| 314 |
_BULK_REPORT_ROLES = tuple(dict.fromkeys(
|
| 315 |
-
roles[_REPORT_ROLE_INDEX.get(table, 0)] for table, roles in REPORT_ROLES.items()
|
|
|
|
| 316 |
_APPROVED_REPORT_TYPES = tuple(dict.fromkeys(
|
| 317 |
spec[key] for spec in REPORT_TYPES.values() for key in ("report", "report_incremental")
|
| 318 |
if spec.get(key)))
|
| 319 |
|
| 320 |
#: Exact method/path pairs. Braced ids match ONE opaque path segment, never a suffix, query,
|
| 321 |
-
#: escaped slash or traversal. The
|
| 322 |
#: LWA, presigned documents and the schema module's public GitHub refresh are not SP-API
|
| 323 |
#: operations. The gate enumerates those transports separately; they confer no operation scope.
|
| 324 |
OPERATIONS = {
|
|
@@ -353,6 +1038,22 @@ OPERATIONS = {
|
|
| 353 |
"name": "listInventory", "roles": ("Amazon Warehousing and Distribution",),
|
| 354 |
"method": "GET", "path": "awd/2024-05-09/inventory",
|
| 355 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 356 |
}
|
| 357 |
|
| 358 |
#: Answer "what does this approved role let this connector read?" directly from data.
|
|
@@ -444,6 +1145,26 @@ class AmazonError(RuntimeError):
|
|
| 444 |
"""Any refusal from LWA or SP-API, already stripped of anything secret."""
|
| 445 |
|
| 446 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 447 |
class AmazonOperationRefused(AmazonError):
|
| 448 |
"""A local scope refusal, never a throttle and never eligible for throttling retries."""
|
| 449 |
|
|
@@ -497,6 +1218,8 @@ def operation_for(method, path):
|
|
| 497 |
|
| 498 |
def require_operation(method, path, *, report_type=None):
|
| 499 |
"""Fail closed before constructing any URL or touching a transport."""
|
|
|
|
|
|
|
| 500 |
operation = operation_for(method, path)
|
| 501 |
if operation is None:
|
| 502 |
raise AmazonOperationRefused(f"{str(method).upper()} {_safe(path)}")
|
|
@@ -564,6 +1287,252 @@ def _pace_entity(name, interval, *, sleep=time.sleep, now=time.monotonic):
|
|
| 564 |
return wait
|
| 565 |
|
| 566 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 567 |
def _post_form(url, data, timeout=_TIMEOUT):
|
| 568 |
# LWA exchanges credentials at api.amazon.com, not an SP-API operation. Its sole caller fixes
|
| 569 |
# LWA_TOKEN_URL; the connector-wide gate checks that call separately from OPERATIONS.
|
|
@@ -850,6 +1819,9 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
|
|
| 850 |
is that this function can reach exactly one operation and a caller cannot redirect it.
|
| 851 |
"""
|
| 852 |
require_operation("POST", _REPORTS_CREATE_PATH, report_type=str(report_type))
|
|
|
|
|
|
|
|
|
|
| 853 |
|
| 854 |
def attempt():
|
| 855 |
"""One raw attempt at the declared report-creation POST."""
|
|
@@ -877,10 +1849,12 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
|
|
| 877 |
except urllib.error.HTTPError as e:
|
| 878 |
detail = ""
|
| 879 |
try:
|
| 880 |
-
|
| 881 |
-
|
|
|
|
|
|
|
| 882 |
except Exception: # noqa: BLE001
|
| 883 |
-
|
| 884 |
if e.code == 429:
|
| 885 |
raise AmazonThrottled(
|
| 886 |
f"{_safe(url)} was throttled (429) creating a {report_type} report. Report "
|
|
@@ -891,8 +1865,11 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
|
|
| 891 |
raise AmazonError(
|
| 892 |
f"{_safe(url)} refused a {report_type} report (403). That is a ROLE answer: this "
|
| 893 |
"report needs a role the application was not granted") from None
|
|
|
|
|
|
|
|
|
|
| 894 |
raise AmazonError(f"{_safe(url)} answered {e.code}"
|
| 895 |
-
+ (f": {detail}" if detail else "")) from None
|
| 896 |
except Exception as e: # noqa: BLE001
|
| 897 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
| 898 |
rid = str(got.get("reportId") or "")
|
|
@@ -954,7 +1931,41 @@ def report_document(fields, report_type, start=None, end=None, options=None,
|
|
| 954 |
doc_id = str(got.get("reportDocumentId") or "")
|
| 955 |
break
|
| 956 |
if state in _REPORT_DEAD:
|
| 957 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 958 |
time.sleep(poll_seconds)
|
| 959 |
waited += poll_seconds
|
| 960 |
# β The token is ~1 hour and a long report can outlive it. Re-mint rather than 403.
|
|
@@ -973,6 +1984,8 @@ def report_document(fields, report_type, start=None, end=None, options=None,
|
|
| 973 |
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
|
| 974 |
say(f" {report_type}: {len(text):,} chars")
|
| 975 |
return text, {"reportId": rid, "reportDocumentId": doc_id,
|
|
|
|
|
|
|
| 976 |
"compression": meta.get("compressionAlgorithm")}
|
| 977 |
|
| 978 |
|
|
@@ -1039,6 +2052,31 @@ def settlement_document(fields, document_id, *, timeout=_TIMEOUT):
|
|
| 1039 |
return _download_document(url, str(meta.get("compressionAlgorithm") or ""))
|
| 1040 |
|
| 1041 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1042 |
def _download_document(url, compression="", timeout=120):
|
| 1043 |
"""Fetch one presigned report document and decode it. -> text.
|
| 1044 |
|
|
@@ -1057,12 +2095,20 @@ def _download_document(url, compression="", timeout=120):
|
|
| 1057 |
raw = r.read()
|
| 1058 |
except Exception as e: # noqa: BLE001
|
| 1059 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1060 |
if str(compression).upper() == "GZIP":
|
| 1061 |
import gzip
|
| 1062 |
try:
|
| 1063 |
raw = gzip.decompress(raw)
|
| 1064 |
except Exception as e: # noqa: BLE001
|
| 1065 |
raise AmazonError(f"the report document did not decompress: {type(e).__name__}") from None
|
|
|
|
|
|
|
| 1066 |
for enc in ("utf-8", "cp1252"):
|
| 1067 |
try:
|
| 1068 |
return raw.decode(enc)
|
|
|
|
| 21 |
signing requests is describing a dead API.
|
| 22 |
Β· https://developer-docs.amazon.com/sp-api/changelog/sp-api-will-no-longer-require-aws-iam-or-aws-signature-version-4
|
| 23 |
|
| 24 |
+
β **READ-ONLY, ENFORCED HERE, WITH EXACTLY THREE NAMED EXCEPTIONS.** `_request` refuses any method but
|
| 25 |
GET before it builds a request. SP-API can create feeds, send buyer messages and adjust listings;
|
| 26 |
none of that is reachable from this module, and the refusal sits below every route and payload so no
|
| 27 |
caller can opt out.
|
| 28 |
|
| 29 |
+
β **The exceptions are `_create_report`, `_competitive_summary_batch` and `_create_query`, all declared
|
| 30 |
rather than hidden.** Asking Amazon to BUILD a report is a POST, and Product Pricing's
|
| 31 |
+
`getCompetitiveSummary` wraps read-only GETs, and Data Kiosk generates a fixed economics query. These functions have
|
| 32 |
their path as a module constant rather than a parameter, so no caller can redirect either one. They
|
| 33 |
cannot alter a listing, price, order or inventory level, and cannot send a buyer anything. Full
|
| 34 |
reasoning sits on their path constants.
|
| 35 |
|
| 36 |
+
β **REPORTS AND DATA KIOSK ARE THE BULK PATHS; FINANCES FOLLOWS EVERY PAGE.** SP-API is token-bucket rate limited per
|
| 37 |
operation per selling-partner/application pair, and Amazon treats throttling as evidence of a wrong
|
| 38 |
call pattern rather than a wrong ceiling. Standing rule 1 forbids a silent cap on a connector's
|
| 39 |
records, and its second half is what applies: a limit that genuinely cannot be removed must be
|
|
|
|
| 47 |
import contextvars
|
| 48 |
import json
|
| 49 |
import random
|
| 50 |
+
import re
|
| 51 |
import threading
|
| 52 |
import time
|
| 53 |
+
from datetime import date, datetime, timezone, timedelta
|
| 54 |
import urllib.error
|
| 55 |
import urllib.parse
|
| 56 |
import urllib.request
|
|
|
|
| 102 |
_entity_pacers = {
|
| 103 |
"competitive_summary": {"last": None, "lock": threading.Lock()},
|
| 104 |
"awd_inventory": {"last": None, "lock": threading.Lock()},
|
| 105 |
+
"transactions": {"last": None, "lock": threading.Lock()},
|
| 106 |
+
"getOrderMetrics": {"last": None, "lock": threading.Lock()},
|
| 107 |
+
"getInventorySummaries": {"last": None, "lock": threading.Lock()},
|
| 108 |
+
"searchListingsItems": {"last": None, "lock": threading.Lock()},
|
| 109 |
+
"searchCatalogItems": {"last": None, "lock": threading.Lock()},
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
# Data Kiosk has its own quota, independent of createReport.
|
| 113 |
+
QUERY_BURST = 15
|
| 114 |
+
QUERY_SUSTAINED_SECONDS = 60.0
|
| 115 |
+
_query_pacer = {"tokens": float(QUERY_BURST), "updated": None, "lock": threading.Lock()}
|
| 116 |
+
_QUERIES_CREATE_PATH = "dataKiosk/2023-11-15/queries"
|
| 117 |
+
_TRANSACTIONS_PATH = "finances/2024-06-19/transactions"
|
| 118 |
+
TRANSACTIONS_INTERVAL = 2.0
|
| 119 |
+
|
| 120 |
+
# Selection pinned to Amazon's published analytics_economics_2024_03_15.graphql,
|
| 121 |
+
# read 2026-09-25. RANGE avoids MONTH's automatic rounding of partial months.
|
| 122 |
+
# Only dates, a validated marketplace ID and one of two fixed grains are substituted.
|
| 123 |
+
_ECONOMICS_QUERY = '''query {
|
| 124 |
+
analytics_economics_2024_03_15 {
|
| 125 |
+
economics(startDate: "__START__", endDate: "__END__",
|
| 126 |
+
marketplaceIds: ["__MARKETPLACE__"], aggregateBy: {date: __DATE__, productId: __PRODUCT__}) {
|
| 127 |
+
startDate endDate marketplaceId msku fnsku childAsin parentAsin
|
| 128 |
+
sales {
|
| 129 |
+
orderedProductSales { amount currencyCode } unitsOrdered
|
| 130 |
+
refundedProductSales { amount currencyCode } unitsRefunded
|
| 131 |
+
netProductSales { amount currencyCode } netUnitsSold
|
| 132 |
+
averageSellingPrice { amount currencyCode }
|
| 133 |
+
}
|
| 134 |
+
fees { feeTypeName charges { startDate endDate identifier aggregatedDetail {
|
| 135 |
+
amount { amount currencyCode } totalAmount { amount currencyCode }
|
| 136 |
+
amountPerUnit { amount currencyCode } amountPerUnitDelta { amount currencyCode }
|
| 137 |
+
promotionAmount { amount currencyCode } taxAmount { amount currencyCode } quantity
|
| 138 |
+
} } }
|
| 139 |
+
ads { adTypeName charge {
|
| 140 |
+
amount { amount currencyCode } totalAmount { amount currencyCode }
|
| 141 |
+
amountPerUnit { amount currencyCode } promotionAmount { amount currencyCode }
|
| 142 |
+
taxAmount { amount currencyCode } quantity
|
| 143 |
+
} }
|
| 144 |
+
netProceeds { total { amount currencyCode } perUnit { amount currencyCode } }
|
| 145 |
+
cost {
|
| 146 |
+
costOfGoodsSold { amount currencyCode } miscellaneousCost { amount currencyCode }
|
| 147 |
+
fbaCost { shippingToAmazonCost { amount currencyCode } }
|
| 148 |
+
mfnCost { fulfillmentCost { amount currencyCode } storageCost { amount currencyCode } }
|
| 149 |
+
}
|
| 150 |
+
}
|
| 151 |
+
}
|
| 152 |
+
}'''
|
| 153 |
+
_ECONOMICS_TEMPLATES = {
|
| 154 |
+
False: _ECONOMICS_QUERY.replace("__DATE__", "DAY").replace("__PRODUCT__", "MSKU"),
|
| 155 |
+
# S66-T40 (lead, measured live 2026-09-25, window 2026-09-20..24): RANGE x MSKU equals the
|
| 156 |
+
# sum of our DAY x MSKU rows to the cent (fees 38,009.94, net 38,474.335), while ANY
|
| 157 |
+
# PARENT_ASIN grain carries 37.75 of fees Amazon attributes to no MSKU. The reconciliation
|
| 158 |
+
# checks OUR date roll-up at the SAME product grain; comparing across grains refused a
|
| 159 |
+
# correct transform.
|
| 160 |
+
True: _ECONOMICS_QUERY.replace("__DATE__", "RANGE").replace("__PRODUCT__", "MSKU"),
|
| 161 |
}
|
| 162 |
|
| 163 |
#: β MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
|
|
|
|
| 237 |
#: production host, whereas a production credential wrongly marked sandbox would quietly read mocks.
|
| 238 |
OPTIONAL_FIELDS = ("sandbox", "region")
|
| 239 |
|
| 240 |
+
SQP_ASIN_OPTION_MAX_CHARS = 200
|
| 241 |
+
SQP_PERIOD = "MONTH"
|
| 242 |
+
|
| 243 |
#: β THE SCHEMA SPINE (owner rulings R4 + R14: seven tables, and NOTHING ELSE). Each row names
|
| 244 |
#: the table, the bulk report
|
| 245 |
#: type that fills it, and the summary aggregate its `validate()` reconciles against.
|
|
|
|
| 291 |
},
|
| 292 |
"listings": {
|
| 293 |
"report": "GET_MERCHANT_LISTINGS_ALL_DATA",
|
| 294 |
+
"snapshot": True,
|
| 295 |
"validates_against": "listing report row count vs our listing rows",
|
| 296 |
"role": "Product Listing",
|
| 297 |
},
|
|
|
|
| 316 |
},
|
| 317 |
"inventory": {
|
| 318 |
"report": "GET_LEDGER_SUMMARY_VIEW_DATA",
|
| 319 |
+
"snapshot": True,
|
| 320 |
+
"options": {"aggregateByTimePeriod": "DAILY", "aggregateByLocation": "COUNTRY"},
|
| 321 |
"entity_read": "AWD v2024-05-09 listInventory (all 200-row pages)",
|
| 322 |
"validates_against": "FBA ledger summary vs detail and AWD API items vs stored AWD rows",
|
| 323 |
"role": "Amazon Fulfillment / Amazon Warehousing and Distribution",
|
|
|
|
| 337 |
# page loop over a full catalogue is precisely what rule 1 forbids.
|
| 338 |
"pricing": {
|
| 339 |
"report": "GET_SALES_AND_TRAFFIC_REPORT",
|
| 340 |
+
"json_root": "salesAndTrafficByAsin",
|
| 341 |
"report_covers": "buy box SHARE only; PRICE has no bulk path",
|
| 342 |
"price_path": "Product Pricing v2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)",
|
| 343 |
"validates_against": "sales and traffic report totals and competitive ASIN responses",
|
|
|
|
| 348 |
# wave's one UNPROVEN ticket with its exit condition written in advance: returns >0 rows
|
| 349 |
# for any registered brand, else demote to a declared-but-unsynced stub.
|
| 350 |
"report": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT",
|
| 351 |
+
"json_root": "dataByDepartmentAndSearchTerm",
|
| 352 |
"validates_against": "report period totals vs our search-term rows",
|
| 353 |
"role": "Brand Analytics",
|
| 354 |
},
|
| 355 |
+
"economics": {
|
| 356 |
+
"api": "createQuery",
|
| 357 |
+
"validates_against": "DAY x MSKU sales, fees and net proceeds vs Amazon RANGE x MSKU totals per currency",
|
| 358 |
+
"role": "Selling Partner Insights",
|
| 359 |
+
},
|
| 360 |
+
"transactions": {
|
| 361 |
+
"api": "listTransactions",
|
| 362 |
+
"validates_against": "Amazon transaction count and totalAmount per currency vs stored items, and breakdowns vs transaction totals",
|
| 363 |
+
"role": "Finance and Accounting",
|
| 364 |
+
},
|
| 365 |
+
# snapshot forbids dataStartTime/dataEndTime; min_interval_s is a successful
|
| 366 |
+
# refresh floor (also enforced on manual sync). Monthly history is separate.
|
| 367 |
+
"fba_fee_estimates": {
|
| 368 |
+
"report": "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA", "snapshot": True,
|
| 369 |
+
"min_interval_s": 86400,
|
| 370 |
+
"validates_against": "Amazon document data-row count vs stored rows for its latest snapshot date",
|
| 371 |
+
"role": "Amazon Fulfillment",
|
| 372 |
+
},
|
| 373 |
+
"referral_fees": {
|
| 374 |
+
"report": "GET_REFERRAL_FEE_PREVIEW_REPORT", "snapshot": True,
|
| 375 |
+
"min_interval_s": 86400,
|
| 376 |
+
"validates_against": "Amazon document data-row count vs stored rows for its latest snapshot date",
|
| 377 |
+
"role": "Finance and Accounting",
|
| 378 |
+
},
|
| 379 |
+
"storage_fees": {
|
| 380 |
+
"report": "GET_FBA_STORAGE_FEE_CHARGES_DATA", "monthly": True,
|
| 381 |
+
"min_interval_s": 30 * 86400,
|
| 382 |
+
"validates_against": "Amazon document data-row count vs stored rows for its latest calendar month",
|
| 383 |
+
"role": "Amazon Fulfillment",
|
| 384 |
+
},
|
| 385 |
+
"search_query_performance": {
|
| 386 |
+
"report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
|
| 387 |
+
"role": "Brand Analytics",
|
| 388 |
+
"validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
|
| 389 |
+
"json_root": "dataByAsin", "monthly": True,
|
| 390 |
+
"period": SQP_PERIOD, "asin_option_max_chars": SQP_ASIN_OPTION_MAX_CHARS,
|
| 391 |
+
},
|
| 392 |
+
}
|
| 393 |
+
|
| 394 |
+
# S66-T42: inventory sections 2.2 and 2.3. Fields are measured at sync, including
|
| 395 |
+
# JSON documents. Row roots were read from amzn/selling-partner-api-models/schemas/reports
|
| 396 |
+
# on 2026-09-25 at commit 713565ff394d136629a342120872e65cb073d162.
|
| 397 |
+
# json_schema names the public source, without changing the pinned proto.
|
| 398 |
+
GROUP_A_TABLES = {
|
| 399 |
+
"long_term_storage_fees": {
|
| 400 |
+
"label": "Amazon long term storage fees",
|
| 401 |
+
"report": "GET_FBA_FULFILLMENT_LONGTERM_STORAGE_FEE_CHARGES_DATA",
|
| 402 |
+
"role": "Amazon Fulfillment",
|
| 403 |
+
"min_interval_s": 14400,
|
| 404 |
+
"cadence_note": "Complete calendar months; refresh no sooner than 14400 seconds.",
|
| 405 |
+
"monthly": True,
|
| 406 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 407 |
+
},
|
| 408 |
+
"storage_overage_fees": {
|
| 409 |
+
"label": "Amazon storage overage fees",
|
| 410 |
+
"report": "GET_FBA_OVERAGE_FEE_CHARGES_DATA",
|
| 411 |
+
"role": "Amazon Fulfillment",
|
| 412 |
+
"min_interval_s": 14400,
|
| 413 |
+
"cadence_note": "Complete calendar months; refresh no sooner than 14400 seconds.",
|
| 414 |
+
"monthly": True,
|
| 415 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 416 |
+
},
|
| 417 |
+
"reimbursements": {
|
| 418 |
+
"label": "Amazon reimbursements",
|
| 419 |
+
"report": "GET_FBA_REIMBURSEMENTS_DATA",
|
| 420 |
+
"role": "Amazon Fulfillment",
|
| 421 |
+
"min_interval_s": 14400,
|
| 422 |
+
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 423 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 424 |
+
},
|
| 425 |
+
"fba_customer_returns": {
|
| 426 |
+
"label": "Amazon FBA customer returns",
|
| 427 |
+
"report": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
|
| 428 |
+
"role": "Amazon Fulfillment",
|
| 429 |
+
"min_interval_s": 14400,
|
| 430 |
+
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 431 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 432 |
+
},
|
| 433 |
+
"replacements": {
|
| 434 |
+
"label": "Amazon replacements",
|
| 435 |
+
"report": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA",
|
| 436 |
+
"role": "Amazon Fulfillment",
|
| 437 |
+
"min_interval_s": 14400,
|
| 438 |
+
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 439 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 440 |
+
},
|
| 441 |
+
"fba_shipped_sales": {
|
| 442 |
+
"label": "Amazon FBA shipped sales",
|
| 443 |
+
"report": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_SALES_DATA",
|
| 444 |
+
"role": "Amazon Fulfillment",
|
| 445 |
+
"min_interval_s": 1800,
|
| 446 |
+
"cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
|
| 447 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 448 |
+
},
|
| 449 |
+
"fba_shipment_promotions": {
|
| 450 |
+
"label": "Amazon FBA shipment promotions",
|
| 451 |
+
"report": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA",
|
| 452 |
+
"role": "Amazon Fulfillment",
|
| 453 |
+
"min_interval_s": 14400,
|
| 454 |
+
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 455 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 456 |
+
},
|
| 457 |
+
"inventory_ledger_detail": {
|
| 458 |
+
"label": "Amazon inventory ledger detail",
|
| 459 |
+
"report": "GET_LEDGER_DETAIL_VIEW_DATA",
|
| 460 |
+
"role": "Amazon Fulfillment",
|
| 461 |
+
"min_interval_s": 1800,
|
| 462 |
+
"cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
|
| 463 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 464 |
+
},
|
| 465 |
+
"inventory_planning": {
|
| 466 |
+
"label": "Amazon inventory planning",
|
| 467 |
+
"report": "GET_FBA_INVENTORY_PLANNING_DATA",
|
| 468 |
+
"role": "Amazon Fulfillment",
|
| 469 |
+
"min_interval_s": 14400,
|
| 470 |
+
"cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
|
| 471 |
+
"snapshot": True,
|
| 472 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 473 |
+
},
|
| 474 |
+
"manage_inventory": {
|
| 475 |
+
"label": "Amazon manage inventory",
|
| 476 |
+
"report": "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA",
|
| 477 |
+
"role": "Amazon Fulfillment",
|
| 478 |
+
"min_interval_s": 1800,
|
| 479 |
+
"cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
|
| 480 |
+
"snapshot": True,
|
| 481 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 482 |
+
},
|
| 483 |
+
"reserved_inventory": {
|
| 484 |
+
"label": "Amazon reserved inventory",
|
| 485 |
+
"report": "GET_RESERVED_INVENTORY_DATA",
|
| 486 |
+
"role": "Amazon Fulfillment",
|
| 487 |
+
"min_interval_s": 1800,
|
| 488 |
+
"cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
|
| 489 |
+
"snapshot": True,
|
| 490 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 491 |
+
},
|
| 492 |
+
"restock_recommendations": {
|
| 493 |
+
"label": "Amazon restock recommendations",
|
| 494 |
+
"report": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT",
|
| 495 |
+
"role": "Amazon Fulfillment",
|
| 496 |
+
"min_interval_s": 1800,
|
| 497 |
+
"cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
|
| 498 |
+
"snapshot": True,
|
| 499 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 500 |
+
},
|
| 501 |
+
"stranded_inventory": {
|
| 502 |
+
"label": "Amazon stranded inventory",
|
| 503 |
+
"report": "GET_STRANDED_INVENTORY_UI_DATA",
|
| 504 |
+
"role": "Amazon Fulfillment",
|
| 505 |
+
"min_interval_s": 1800,
|
| 506 |
+
"cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
|
| 507 |
+
"snapshot": True,
|
| 508 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 509 |
+
},
|
| 510 |
+
"inbound_noncompliance": {
|
| 511 |
+
"label": "Amazon inbound noncompliance",
|
| 512 |
+
"report": "GET_FBA_FULFILLMENT_INBOUND_NONCOMPLIANCE_DATA",
|
| 513 |
+
"role": "Amazon Fulfillment",
|
| 514 |
+
"min_interval_s": 14400,
|
| 515 |
+
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 516 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 517 |
+
},
|
| 518 |
+
"inventory_expiration_dates": {
|
| 519 |
+
"label": "Amazon inventory expiration dates",
|
| 520 |
+
"report": "GET_FBA_FULFILLMENT_INVENTORY_EXPIRATION_DATE_DATA",
|
| 521 |
+
"role": "Amazon Fulfillment",
|
| 522 |
+
"min_interval_s": 14400,
|
| 523 |
+
"cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
|
| 524 |
+
"snapshot": True,
|
| 525 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 526 |
+
},
|
| 527 |
+
"recommended_removals": {
|
| 528 |
+
"label": "Amazon recommended removals",
|
| 529 |
+
"report": "GET_FBA_RECOMMENDED_REMOVAL_DATA",
|
| 530 |
+
"role": "Amazon Fulfillment",
|
| 531 |
+
"min_interval_s": 14400,
|
| 532 |
+
"cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
|
| 533 |
+
"snapshot": True,
|
| 534 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 535 |
+
},
|
| 536 |
+
"removal_orders": {
|
| 537 |
+
"label": "Amazon removal orders",
|
| 538 |
+
"report": "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
|
| 539 |
+
"role": "Amazon Fulfillment",
|
| 540 |
+
"min_interval_s": 1800,
|
| 541 |
+
"cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
|
| 542 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 543 |
+
},
|
| 544 |
+
"removal_shipments": {
|
| 545 |
+
"label": "Amazon removal shipments",
|
| 546 |
+
"report": "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
|
| 547 |
+
"role": "Amazon Fulfillment",
|
| 548 |
+
"min_interval_s": 1800,
|
| 549 |
+
"cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
|
| 550 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 551 |
+
},
|
| 552 |
+
"subscribe_save_forecast": {
|
| 553 |
+
"label": "Amazon Subscribe and Save forecast",
|
| 554 |
+
"report": "GET_FBA_SNS_FORECAST_DATA",
|
| 555 |
+
"role": "Amazon Fulfillment",
|
| 556 |
+
"min_interval_s": 14400,
|
| 557 |
+
"cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
|
| 558 |
+
"snapshot": True,
|
| 559 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 560 |
+
},
|
| 561 |
+
"subscribe_save_performance": {
|
| 562 |
+
"label": "Amazon Subscribe and Save performance",
|
| 563 |
+
"report": "GET_FBA_SNS_PERFORMANCE_DATA",
|
| 564 |
+
"role": "Amazon Fulfillment",
|
| 565 |
+
"min_interval_s": 14400,
|
| 566 |
+
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 567 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 568 |
+
},
|
| 569 |
+
"remote_fulfillment_eligibility": {
|
| 570 |
+
"label": "Amazon remote fulfillment eligibility",
|
| 571 |
+
"report": "GET_REMOTE_FULFILLMENT_ELIGIBILITY",
|
| 572 |
+
"role": "Amazon Fulfillment",
|
| 573 |
+
"min_interval_s": 86400,
|
| 574 |
+
"cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
|
| 575 |
+
"snapshot": True,
|
| 576 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 577 |
+
},
|
| 578 |
+
"search_catalog_performance": {
|
| 579 |
+
"label": "Amazon search catalog performance",
|
| 580 |
+
"report": "GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT",
|
| 581 |
+
"role": "Brand Analytics",
|
| 582 |
+
"min_interval_s": 86400,
|
| 583 |
+
"cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
|
| 584 |
+
"monthly": True,
|
| 585 |
+
"json_root": "dataByAsin",
|
| 586 |
+
"json_schema": "sellingPartnerSearchCatalogPerformanceReport.json",
|
| 587 |
+
"period": "MONTH",
|
| 588 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 589 |
+
},
|
| 590 |
+
"market_basket": {
|
| 591 |
+
"label": "Amazon market basket",
|
| 592 |
+
"report": "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT",
|
| 593 |
+
"role": "Brand Analytics",
|
| 594 |
+
"min_interval_s": 86400,
|
| 595 |
+
"cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
|
| 596 |
+
"monthly": True,
|
| 597 |
+
"json_root": "dataByAsin",
|
| 598 |
+
"json_schema": "sellingPartnerMarketBasketAnalysisReport.json",
|
| 599 |
+
"period": "MONTH",
|
| 600 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 601 |
+
},
|
| 602 |
+
"repeat_purchase": {
|
| 603 |
+
"label": "Amazon repeat purchase",
|
| 604 |
+
"report": "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT",
|
| 605 |
+
"role": "Brand Analytics",
|
| 606 |
+
"min_interval_s": 86400,
|
| 607 |
+
"cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
|
| 608 |
+
"monthly": True,
|
| 609 |
+
"json_root": "dataByAsin",
|
| 610 |
+
"json_schema": "sellingPartnerRepeatPurchaseReport.json",
|
| 611 |
+
"period": "MONTH",
|
| 612 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 613 |
+
},
|
| 614 |
+
"promotion_performance": {
|
| 615 |
+
"label": "Amazon promotion performance",
|
| 616 |
+
"report": "GET_PROMOTION_PERFORMANCE_REPORT",
|
| 617 |
+
"role": "Selling Partner Insights",
|
| 618 |
+
"min_interval_s": 86400,
|
| 619 |
+
"cadence_note": "Date windows; refresh no sooner than 86400 seconds.",
|
| 620 |
+
"json_root": "promotions",
|
| 621 |
+
"json_schema": "promotionReport.json",
|
| 622 |
+
"date_options": ["promotionStartDateFrom","promotionStartDateTo"],
|
| 623 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 624 |
+
},
|
| 625 |
+
"coupon_performance": {
|
| 626 |
+
"label": "Amazon coupon performance",
|
| 627 |
+
"report": "GET_COUPON_PERFORMANCE_REPORT",
|
| 628 |
+
"role": "Brand Analytics",
|
| 629 |
+
"min_interval_s": 86400,
|
| 630 |
+
"cadence_note": "Date windows; refresh no sooner than 86400 seconds.",
|
| 631 |
+
"json_root": "coupons",
|
| 632 |
+
"json_schema": "sellerCouponReport.json",
|
| 633 |
+
"date_options": ["couponStartDateFrom","couponStartDateTo"],
|
| 634 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 635 |
+
},
|
| 636 |
+
"account_health": {
|
| 637 |
+
"label": "Amazon account health",
|
| 638 |
+
"report": "GET_V2_SELLER_PERFORMANCE_REPORT",
|
| 639 |
+
"role": "Selling Partner Insights",
|
| 640 |
+
"min_interval_s": 86400,
|
| 641 |
+
"cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
|
| 642 |
+
"snapshot": True,
|
| 643 |
+
"json_root": "performanceMetrics",
|
| 644 |
+
"json_schema": "accountHealthReport-2020-11-18.json",
|
| 645 |
+
"json_extra_roots": ["accountStatuses"],
|
| 646 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 647 |
+
},
|
| 648 |
+
"inactive_listings": {
|
| 649 |
+
"label": "Amazon inactive listings",
|
| 650 |
+
"report": "GET_MERCHANT_LISTINGS_INACTIVE_DATA",
|
| 651 |
+
"role": "Inventory and Order Tracking",
|
| 652 |
+
"min_interval_s": 14400,
|
| 653 |
+
"cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
|
| 654 |
+
"snapshot": True,
|
| 655 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 656 |
+
},
|
| 657 |
+
"cancelled_listings": {
|
| 658 |
+
"label": "Amazon cancelled listings",
|
| 659 |
+
"report": "GET_MERCHANT_CANCELLED_LISTINGS_DATA",
|
| 660 |
+
"role": "Inventory and Order Tracking",
|
| 661 |
+
"min_interval_s": 14400,
|
| 662 |
+
"cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
|
| 663 |
+
"snapshot": True,
|
| 664 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 665 |
+
},
|
| 666 |
+
"suppressed_listings": {
|
| 667 |
+
"label": "Amazon suppressed listings",
|
| 668 |
+
"report": "GET_MERCHANTS_LISTINGS_FYP_REPORT",
|
| 669 |
+
"role": "Product Listing",
|
| 670 |
+
"min_interval_s": 14400,
|
| 671 |
+
"cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
|
| 672 |
+
"snapshot": True,
|
| 673 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 674 |
+
},
|
| 675 |
+
"listing_defects": {
|
| 676 |
+
"label": "Amazon listing defects",
|
| 677 |
+
"report": "GET_MERCHANT_LISTINGS_DEFECT_DATA",
|
| 678 |
+
"role": "Inventory and Order Tracking",
|
| 679 |
+
"min_interval_s": 14400,
|
| 680 |
+
"cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
|
| 681 |
+
"snapshot": True,
|
| 682 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 683 |
+
},
|
| 684 |
+
"sold_listings": {
|
| 685 |
+
"label": "Amazon sold listings",
|
| 686 |
+
"report": "GET_CONVERGED_FLAT_FILE_SOLD_LISTINGS_DATA",
|
| 687 |
+
"role": "Inventory and Order Tracking",
|
| 688 |
+
"min_interval_s": 14400,
|
| 689 |
+
"cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
|
| 690 |
+
"snapshot": True,
|
| 691 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 692 |
+
},
|
| 693 |
+
"mfn_returns": {
|
| 694 |
+
"label": "Amazon MFN returns",
|
| 695 |
+
"report": "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
|
| 696 |
+
"role": "Inventory and Order Tracking",
|
| 697 |
+
"min_interval_s": 14400,
|
| 698 |
+
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
|
| 699 |
+
"max_window_days": 60,
|
| 700 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 701 |
+
},
|
| 702 |
+
"b2b_recommended": {
|
| 703 |
+
"label": "Amazon B2B product opportunities",
|
| 704 |
+
"report": "GET_B2B_PRODUCT_OPPORTUNITIES_RECOMMENDED_FOR_YOU",
|
| 705 |
+
"role": "Product Listing",
|
| 706 |
+
"min_interval_s": 604800,
|
| 707 |
+
"cadence_note": "Current snapshot; refresh no sooner than 604800 seconds.",
|
| 708 |
+
"snapshot": True,
|
| 709 |
+
"json_root": "recommendations",
|
| 710 |
+
"json_schema": "b2bProductOpportunitiesRecommendedForYouReport-2020-11-19.json",
|
| 711 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 712 |
+
},
|
| 713 |
+
"b2b_not_yet_on_amazon": {
|
| 714 |
+
"label": "Amazon B2B products not yet on Amazon",
|
| 715 |
+
"report": "GET_B2B_PRODUCT_OPPORTUNITIES_NOT_YET_ON_AMAZON",
|
| 716 |
+
"role": "Product Listing",
|
| 717 |
+
"min_interval_s": 604800,
|
| 718 |
+
"cadence_note": "Current snapshot; refresh no sooner than 604800 seconds.",
|
| 719 |
+
"snapshot": True,
|
| 720 |
+
"json_root": "recommendations",
|
| 721 |
+
"json_schema": "b2bProductOpportunitiesNotYetOnAmazonReport-2020-11-19.json",
|
| 722 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 723 |
+
},
|
| 724 |
+
"geo_opportunities": {
|
| 725 |
+
"label": "Amazon geo opportunities",
|
| 726 |
+
"report": "GET_FLAT_FILE_GEO_OPPORTUNITIES",
|
| 727 |
+
"role": "Inventory and Order Tracking",
|
| 728 |
+
"min_interval_s": 86400,
|
| 729 |
+
"cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
|
| 730 |
+
"snapshot": True,
|
| 731 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 732 |
+
},
|
| 733 |
+
"out_of_stock_mfn": {
|
| 734 |
+
"label": "Amazon out of stock MFN listings",
|
| 735 |
+
"report": "GET_OUT_OF_STOCK_MFN_INFORMATION",
|
| 736 |
+
"role": "Inventory and Order Tracking",
|
| 737 |
+
"min_interval_s": 14400,
|
| 738 |
+
"cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
|
| 739 |
+
"snapshot": True,
|
| 740 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 741 |
+
},
|
| 742 |
+
"b2b_quotes": {
|
| 743 |
+
"label": "Amazon B2B quantity discounts",
|
| 744 |
+
"report": "RFQD_BULK_DOWNLOAD",
|
| 745 |
+
"role": "Product Listing",
|
| 746 |
+
"min_interval_s": 86400,
|
| 747 |
+
"cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
|
| 748 |
+
"snapshot": True,
|
| 749 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 750 |
+
},
|
| 751 |
+
"fee_discounts": {
|
| 752 |
+
"label": "Amazon fee discounts",
|
| 753 |
+
"report": "FEE_DISCOUNTS_REPORT",
|
| 754 |
+
"role": "Product Listing",
|
| 755 |
+
"min_interval_s": 86400,
|
| 756 |
+
"cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
|
| 757 |
+
"snapshot": True,
|
| 758 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 759 |
+
},
|
| 760 |
+
"order_metrics": {
|
| 761 |
+
"label": "Amazon order metrics",
|
| 762 |
+
"api": "getOrderMetrics",
|
| 763 |
+
"role": "Inventory and Order Tracking",
|
| 764 |
+
"min_interval_s": 1800,
|
| 765 |
+
"cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
|
| 766 |
+
"validates_against": "Amazon daily order, unit and sales totals vs stored metrics; order counts independently reconcile orders",
|
| 767 |
+
},
|
| 768 |
+
"fba_inventory": {
|
| 769 |
+
"label": "Amazon FBA inventory summaries",
|
| 770 |
+
"api": "getInventorySummaries",
|
| 771 |
+
"role": "Amazon Fulfillment",
|
| 772 |
+
"min_interval_s": 1800,
|
| 773 |
+
"cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
|
| 774 |
+
"snapshot": True,
|
| 775 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 776 |
+
},
|
| 777 |
+
"listing_items": {
|
| 778 |
+
"label": "Amazon listing status and issues",
|
| 779 |
+
"api": "searchListingsItems",
|
| 780 |
+
"role": "Product Listing",
|
| 781 |
+
"min_interval_s": 1800,
|
| 782 |
+
"cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
|
| 783 |
+
"snapshot": True,
|
| 784 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 785 |
+
},
|
| 786 |
+
"catalog_items": {
|
| 787 |
+
"label": "Amazon catalog attributes",
|
| 788 |
+
"api": "searchCatalogItems",
|
| 789 |
+
"role": "Product Listing",
|
| 790 |
+
"min_interval_s": 1800,
|
| 791 |
+
"cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
|
| 792 |
+
"snapshot": True,
|
| 793 |
+
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
|
| 794 |
+
},
|
| 795 |
+
}
|
| 796 |
+
REPORT_TYPES.update(GROUP_A_TABLES)
|
| 797 |
+
|
| 798 |
+
# These are visible declarations, never enablement keys or request allow-list entries.
|
| 799 |
+
DECLARED_DATASETS = {
|
| 800 |
+
"manage_inventory_variants": {
|
| 801 |
+
"label": "Amazon manage inventory alternatives",
|
| 802 |
+
"report_types": ["GET_FBA_MYI_ALL_INVENTORY_DATA","GET_AFN_INVENTORY_DATA"],
|
| 803 |
+
"role": "Amazon Fulfillment",
|
| 804 |
+
"cadence_note": "Not requested by this connector.",
|
| 805 |
+
"reason": "The connector uses the unsuppressed manage inventory report for this dataset.",
|
| 806 |
+
},
|
| 807 |
+
"stranded_inventory_loader": {
|
| 808 |
+
"label": "Amazon stranded inventory loader",
|
| 809 |
+
"report_types": ["GET_STRANDED_INVENTORY_LOADER_DATA"],
|
| 810 |
+
"role": "Amazon Fulfillment",
|
| 811 |
+
"cadence_note": "Not requested by this connector.",
|
| 812 |
+
"reason": "The connector uses the stranded inventory UI report for this dataset.",
|
| 813 |
+
},
|
| 814 |
+
"listing_variants": {
|
| 815 |
+
"label": "Amazon listing report variants",
|
| 816 |
+
"report_types": ["GET_MERCHANT_LISTINGS_DATA","GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT","GET_MERCHANT_LISTINGS_DATA_LITE","GET_MERCHANT_LISTINGS_DATA_LITER","GET_FLAT_FILE_OPEN_LISTINGS_DATA"],
|
| 817 |
+
"role": "Product Listing",
|
| 818 |
+
"cadence_note": "Not requested by this connector.",
|
| 819 |
+
"reason": "These listing variants overlap Amazon listings, which reads the all listings report.",
|
| 820 |
+
},
|
| 821 |
+
"pending_orders": {
|
| 822 |
+
"label": "Amazon pending order reports",
|
| 823 |
+
"report_types": ["GET_FLAT_FILE_PENDING_ORDERS_DATA","GET_PENDING_ORDERS_DATA","GET_CONVERGED_FLAT_FILE_PENDING_ORDERS_DATA"],
|
| 824 |
+
"role": "Inventory and Order Tracking",
|
| 825 |
+
"cadence_note": "Not requested by this connector.",
|
| 826 |
+
"reason": "Pending orders are covered by Amazon orders.",
|
| 827 |
+
},
|
| 828 |
+
"archived_orders": {
|
| 829 |
+
"label": "Amazon archived orders",
|
| 830 |
+
"report_types": ["GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE"],
|
| 831 |
+
"role": "Inventory and Order Tracking",
|
| 832 |
+
"cadence_note": "Not requested by this connector.",
|
| 833 |
+
"reason": "Archived orders before the normal history window are not built in this release.",
|
| 834 |
+
},
|
| 835 |
+
"xml_orders": {
|
| 836 |
+
"label": "Amazon XML order reports",
|
| 837 |
+
"report_types": ["GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL","GET_XML_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL"],
|
| 838 |
+
"role": "Inventory and Order Tracking",
|
| 839 |
+
"cadence_note": "Not requested by this connector.",
|
| 840 |
+
"reason": "The XML order reports duplicate the flat file reports used by Amazon orders.",
|
| 841 |
+
},
|
| 842 |
+
"mfn_return_variants": {
|
| 843 |
+
"label": "Amazon MFN return report variants",
|
| 844 |
+
"report_types": ["GET_CSV_MFN_PRIME_RETURNS_REPORT","GET_XML_MFN_PRIME_RETURNS_REPORT","GET_FLAT_FILE_MFN_SKU_RETURN_ATTRIBUTES_REPORT","GET_XML_MFN_SKU_RETURN_ATTRIBUTES_REPORT"],
|
| 845 |
+
"role": "Inventory and Order Tracking",
|
| 846 |
+
"cadence_note": "Not requested by this connector.",
|
| 847 |
+
"reason": "Prime return, return attribute and attribute feed record variants are not built; Amazon MFN returns reads returns by return date.",
|
| 848 |
+
},
|
| 849 |
+
"browse_tree": {
|
| 850 |
+
"label": "Amazon browse tree",
|
| 851 |
+
"report_types": ["GET_XML_BROWSE_TREE_DATA"],
|
| 852 |
+
"role": "Product Listing",
|
| 853 |
+
"cadence_note": "Not requested by this connector.",
|
| 854 |
+
"reason": "The XML browse tree is category reference data and is not built in this release.",
|
| 855 |
+
},
|
| 856 |
+
"sales_traffic_kiosk": {
|
| 857 |
+
"label": "Amazon sales and traffic query",
|
| 858 |
+
"operations": ["analytics_salesAndTraffic_2024_04_24"],
|
| 859 |
+
"role": "Brand Analytics",
|
| 860 |
+
"cadence_note": "Not requested by this connector.",
|
| 861 |
+
"reason": "Same rows as Amazon pricing and Buy Box, which reads the report twin.",
|
| 862 |
+
},
|
| 863 |
+
"financial_events": {
|
| 864 |
+
"label": "Amazon legacy financial events",
|
| 865 |
+
"operations": ["listFinancialEvents","listFinancialEventGroups"],
|
| 866 |
+
"role": "Finance and Accounting",
|
| 867 |
+
"cadence_note": "Not requested by this connector.",
|
| 868 |
+
"reason": "Superseded by Amazon financial transactions.",
|
| 869 |
+
},
|
| 870 |
+
"payouts": {
|
| 871 |
+
"label": "Amazon payouts",
|
| 872 |
+
"operations": ["listPayouts","listExpectedPayouts"],
|
| 873 |
+
"role": "Finance and Accounting",
|
| 874 |
+
"cadence_note": "Not requested by this connector.",
|
| 875 |
+
"reason": "Payout timing from the Transfers API is not built in this release.",
|
| 876 |
+
},
|
| 877 |
+
"fee_estimates_batch": {
|
| 878 |
+
"label": "Amazon batch fee estimates",
|
| 879 |
+
"operations": ["getMyFeesEstimates"],
|
| 880 |
+
"role": "Pricing",
|
| 881 |
+
"cadence_note": "Not requested by this connector.",
|
| 882 |
+
"reason": "Current price fee estimates are covered by Amazon FBA fee estimates and referral fee preview; hypothetical prices are not built.",
|
| 883 |
+
},
|
| 884 |
+
"offer_prices": {
|
| 885 |
+
"label": "Amazon offer price alternatives",
|
| 886 |
+
"operations": ["getItemOffersBatch","getListingOffersBatch","getFeaturedOfferExpectedPriceBatch"],
|
| 887 |
+
"role": "Pricing",
|
| 888 |
+
"cadence_note": "Not requested by this connector.",
|
| 889 |
+
"reason": "Amazon pricing and Buy Box reads competitive summaries; other offer price operations are not built in this release.",
|
| 890 |
+
},
|
| 891 |
+
"inbound_shipments": {
|
| 892 |
+
"label": "Amazon inbound shipments",
|
| 893 |
+
"operations": ["listInboundPlans","listShipmentItems","listInboundShipments"],
|
| 894 |
+
"role": "Amazon Fulfillment",
|
| 895 |
+
"cadence_note": "Not requested by this connector.",
|
| 896 |
+
"reason": "FBA and AWD inbound shipment plans and items are not built in this release.",
|
| 897 |
+
},
|
| 898 |
+
"replenishment": {
|
| 899 |
+
"label": "Amazon replenishment metrics",
|
| 900 |
+
"operations": ["listOfferMetrics","getSellingPartnerMetrics"],
|
| 901 |
+
"role": "Brand Analytics",
|
| 902 |
+
"cadence_note": "Not requested by this connector.",
|
| 903 |
+
"reason": "Only returns rows for Subscribe and Save offers; API offer metrics are not built in this release.",
|
| 904 |
+
},
|
| 905 |
+
"review_topics": {
|
| 906 |
+
"label": "Amazon review topics and trends",
|
| 907 |
+
"operations": ["getItemReviewTopics","getItemReviewTrends","getBrowseNodeReturnTopics","getBrowseNodeReturnTrends"],
|
| 908 |
+
"role": "Selling Partner Insights",
|
| 909 |
+
"cadence_note": "Not requested by this connector.",
|
| 910 |
+
"reason": "Per ASIN review and return topics are not built in this release; the limit is Amazon's per ASIN request rate.",
|
| 911 |
+
},
|
| 912 |
+
"promotions": {
|
| 913 |
+
"label": "Amazon promotion calendar",
|
| 914 |
+
"operations": ["searchPromotions","getPromotion"],
|
| 915 |
+
"role": "Pricing",
|
| 916 |
+
"cadence_note": "Not requested by this connector.",
|
| 917 |
+
"reason": "The Promotions API calendar is not built in this release; promotion results are in Amazon promotion performance.",
|
| 918 |
+
},
|
| 919 |
+
"aplus_content": {
|
| 920 |
+
"label": "Amazon A plus content",
|
| 921 |
+
"operations": ["searchContentDocuments"],
|
| 922 |
+
"role": "Brand Analytics",
|
| 923 |
+
"cadence_note": "Not requested by this connector.",
|
| 924 |
+
"reason": "A plus content reference data is not built in this release.",
|
| 925 |
+
},
|
| 926 |
+
"listing_restrictions": {
|
| 927 |
+
"label": "Amazon listing restrictions",
|
| 928 |
+
"operations": ["getListingsRestrictions"],
|
| 929 |
+
"role": "Product Listing",
|
| 930 |
+
"cadence_note": "Not requested by this connector.",
|
| 931 |
+
"reason": "Listing restriction reference data is not built in this release.",
|
| 932 |
+
},
|
| 933 |
+
"product_types": {
|
| 934 |
+
"label": "Amazon product type definitions",
|
| 935 |
+
"operations": ["searchDefinitionsProductTypes","getDefinitionsProductType"],
|
| 936 |
+
"role": "Product Listing",
|
| 937 |
+
"cadence_note": "Not requested by this connector.",
|
| 938 |
+
"reason": "Product type definition reference data is not built in this release.",
|
| 939 |
+
},
|
| 940 |
+
"supply_sources": {
|
| 941 |
+
"label": "Amazon supply sources",
|
| 942 |
+
"operations": ["getSupplySources"],
|
| 943 |
+
"role": "Selling Partner Insights",
|
| 944 |
+
"cadence_note": "Not requested by this connector.",
|
| 945 |
+
"reason": "Supply source reference data is not built in this release.",
|
| 946 |
+
},
|
| 947 |
}
|
| 948 |
|
| 949 |
+
# Include the existing seven base tables and T40/T41 datasets in the same inventory.
|
| 950 |
+
GROUP_A = {
|
| 951 |
+
key: {"key": key, "label": spec.get("label") or "Amazon " + key.replace("_", " "),
|
| 952 |
+
"report_types": tuple(spec[k] for k in ("report", "report_incremental", "report_discover") if spec.get(k)),
|
| 953 |
+
"operation": spec.get("api"), "role": spec["role"],
|
| 954 |
+
"cadence_note": spec.get("cadence_note") or "Uses the existing connector cadence."}
|
| 955 |
+
for key, spec in REPORT_TYPES.items()
|
| 956 |
+
}
|
| 957 |
+
GROUP_A.update({key: {"key": key, **spec} for key, spec in DECLARED_DATASETS.items()})
|
| 958 |
+
|
| 959 |
+
|
| 960 |
+
# Inventory section 2.5: immutable refusal even if a future report roster admits one.
|
| 961 |
+
EXCLUDED_REPORT_TYPES = frozenset({
|
| 962 |
+
"GET_ORDER_REPORT_DATA_SHIPPING", "GET_ORDER_REPORT_DATA_TAX", "GET_ORDER_REPORT_DATA_ALL",
|
| 963 |
+
"GET_FLAT_FILE_ORDER_REPORT_DATA_SHIPPING", "GET_FLAT_FILE_ORDER_REPORT_DATA_TAX",
|
| 964 |
+
"GET_FLAT_FILE_ORDERS_RECONCILIATION_DATA_SHIPPING", "GET_FLAT_FILE_ORDERS_RECONCILIATION_DATA_TAX",
|
| 965 |
+
"GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING", "GET_ORDERS_DATA",
|
| 966 |
+
"GET_XML_ALL_ORDERS_DATA_BY_LAST_UPDATE_ALL", "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_TAX",
|
| 967 |
+
"GET_FLAT_FILE_SALES_TAX_DATA", "GET_FLAT_FILE_VAT_INVOICE_DATA_REPORT",
|
| 968 |
+
"GET_XML_VAT_INVOICE_DATA_REPORT", "GET_VAT_TRANSACTION_DATA",
|
| 969 |
+
"GET_GST_MTR_B2B_CUSTOM", "GET_GST_MTR_B2C_CUSTOM", "GET_EASYSHIP_DOCUMENTS",
|
| 970 |
+
"GET_EASYSHIP_DOCUMENTS_PICKEDUP", "GET_EASYSHIP_DOCUMENTS_WAITING_FOR_PICKUP",
|
| 971 |
+
"GET_MONTHLY_WARRANTIES", "WARRANTY_BASE_ITEM_INFO",
|
| 972 |
+
"GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL", "GET_SELLER_FEEDBACK_DATA",
|
| 973 |
+
})
|
| 974 |
+
|
| 975 |
#: Eight of the ten roles Amazon approved on 2026-07-22. Buyer Communication and Buyer Solicitation
|
| 976 |
#: are REFUSED (owner ruling R7): both are write-shaped and PII-bearing, and it is precisely those
|
| 977 |
#: restricted roles that trigger Phase 1 verification, a Phase 2 data-security assessment and an
|
|
|
|
| 996 |
}
|
| 997 |
_REPORT_ROLE_INDEX = {"inventory": 0, "pricing": 1}
|
| 998 |
_BULK_REPORT_ROLES = tuple(dict.fromkeys(
|
| 999 |
+
roles[_REPORT_ROLE_INDEX.get(table, 0)] for table, roles in REPORT_ROLES.items()
|
| 1000 |
+
if REPORT_TYPES[table].get("report") or REPORT_TYPES[table].get("report_discover")))
|
| 1001 |
_APPROVED_REPORT_TYPES = tuple(dict.fromkeys(
|
| 1002 |
spec[key] for spec in REPORT_TYPES.values() for key in ("report", "report_incremental")
|
| 1003 |
if spec.get(key)))
|
| 1004 |
|
| 1005 |
#: Exact method/path pairs. Braced ids match ONE opaque path segment, never a suffix, query,
|
| 1006 |
+
#: escaped slash or traversal. The three POSTs generate reports/queries or wrap fixed read-only GETs.
|
| 1007 |
#: LWA, presigned documents and the schema module's public GitHub refresh are not SP-API
|
| 1008 |
#: operations. The gate enumerates those transports separately; they confer no operation scope.
|
| 1009 |
OPERATIONS = {
|
|
|
|
| 1038 |
"name": "listInventory", "roles": ("Amazon Warehousing and Distribution",),
|
| 1039 |
"method": "GET", "path": "awd/2024-05-09/inventory",
|
| 1040 |
},
|
| 1041 |
+
"createQuery": {"name": "createQuery", "roles": ("Selling Partner Insights",),
|
| 1042 |
+
"method": "POST", "path": _QUERIES_CREATE_PATH},
|
| 1043 |
+
"getQuery": {"name": "getQuery", "roles": ("Selling Partner Insights",),
|
| 1044 |
+
"method": "GET", "path": "dataKiosk/2023-11-15/queries/{queryId}"},
|
| 1045 |
+
"getDocument": {"name": "getDocument", "roles": ("Selling Partner Insights",),
|
| 1046 |
+
"method": "GET", "path": "dataKiosk/2023-11-15/documents/{documentId}"},
|
| 1047 |
+
"listTransactions": {"name": "listTransactions", "roles": ("Finance and Accounting",),
|
| 1048 |
+
"method": "GET", "path": _TRANSACTIONS_PATH},
|
| 1049 |
+
"getOrderMetrics": {"name": "getOrderMetrics", "roles": ("Inventory and Order Tracking",),
|
| 1050 |
+
"method": "GET", "path": "sales/v1/orderMetrics"},
|
| 1051 |
+
"getInventorySummaries": {"name": "getInventorySummaries", "roles": ("Amazon Fulfillment",),
|
| 1052 |
+
"method": "GET", "path": "fba/inventory/v1/summaries"},
|
| 1053 |
+
"searchListingsItems": {"name": "searchListingsItems", "roles": ("Product Listing",),
|
| 1054 |
+
"method": "GET", "path": "listings/2021-08-01/items/{sellerId}"},
|
| 1055 |
+
"searchCatalogItems": {"name": "searchCatalogItems", "roles": ("Product Listing",),
|
| 1056 |
+
"method": "GET", "path": "catalog/2022-04-01/items"},
|
| 1057 |
}
|
| 1058 |
|
| 1059 |
#: Answer "what does this approved role let this connector read?" directly from data.
|
|
|
|
| 1145 |
"""Any refusal from LWA or SP-API, already stripped of anything secret."""
|
| 1146 |
|
| 1147 |
|
| 1148 |
+
class AmazonReportError(AmazonError):
|
| 1149 |
+
"""Vendor report refusal, with its complete reason available to the batch coordinator."""
|
| 1150 |
+
|
| 1151 |
+
def __init__(self, message, reason, status):
|
| 1152 |
+
self.vendor_reason = str(reason).replace("\u2014", "; ").replace("\u2013", " to ")
|
| 1153 |
+
self.status = status
|
| 1154 |
+
super().__init__(f"{message}: {self.vendor_reason}")
|
| 1155 |
+
|
| 1156 |
+
|
| 1157 |
+
def sqp_refused_asins(error, asins):
|
| 1158 |
+
"""Only Amazon's named brand refusals permit removing an ASIN from one retry."""
|
| 1159 |
+
if not isinstance(error, AmazonReportError) or error.status not in (400, "FATAL"):
|
| 1160 |
+
return []
|
| 1161 |
+
reason = error.vendor_reason
|
| 1162 |
+
if not re.search(r"not\s+(?:part|associated|owned|eligible|authorized|belong|registered)|invalid\s+asin|outside.*brand", reason, re.I):
|
| 1163 |
+
return []
|
| 1164 |
+
named = set(re.findall(r"\b[A-Z0-9]{10}\b", reason.upper()))
|
| 1165 |
+
return [asin for asin in asins if asin in named]
|
| 1166 |
+
|
| 1167 |
+
|
| 1168 |
class AmazonOperationRefused(AmazonError):
|
| 1169 |
"""A local scope refusal, never a throttle and never eligible for throttling retries."""
|
| 1170 |
|
|
|
|
| 1218 |
|
| 1219 |
def require_operation(method, path, *, report_type=None):
|
| 1220 |
"""Fail closed before constructing any URL or touching a transport."""
|
| 1221 |
+
if report_type in EXCLUDED_REPORT_TYPES:
|
| 1222 |
+
raise AmazonOperationRefused(f"excluded report {report_type}")
|
| 1223 |
operation = operation_for(method, path)
|
| 1224 |
if operation is None:
|
| 1225 |
raise AmazonOperationRefused(f"{str(method).upper()} {_safe(path)}")
|
|
|
|
| 1287 |
return wait
|
| 1288 |
|
| 1289 |
|
| 1290 |
+
def _pace_query_create(*, sleep=time.sleep, now=time.monotonic):
|
| 1291 |
+
"""Reserve from Data Kiosk's independent burst-15, one-per-minute bucket."""
|
| 1292 |
+
with _query_pacer["lock"]:
|
| 1293 |
+
stamp = now()
|
| 1294 |
+
updated = _query_pacer["updated"]
|
| 1295 |
+
elapsed = max(0.0, stamp - updated) if updated is not None else 0.0
|
| 1296 |
+
tokens = min(QUERY_BURST, _query_pacer["tokens"] + elapsed / QUERY_SUSTAINED_SECONDS)
|
| 1297 |
+
wait = max(0.0, (1.0 - tokens) * QUERY_SUSTAINED_SECONDS)
|
| 1298 |
+
if wait:
|
| 1299 |
+
sleep(wait)
|
| 1300 |
+
issued = max(stamp + wait, now())
|
| 1301 |
+
_query_pacer.update(tokens=min(QUERY_BURST, tokens +
|
| 1302 |
+
(issued - stamp) / QUERY_SUSTAINED_SECONDS) - 1.0, updated=issued)
|
| 1303 |
+
return wait
|
| 1304 |
+
|
| 1305 |
+
|
| 1306 |
+
def economics_query(start, end, marketplace_id, *, summary=False):
|
| 1307 |
+
"""Build only the pinned economics query; never accept caller GraphQL."""
|
| 1308 |
+
for value in (start, end):
|
| 1309 |
+
if not isinstance(value, str) or not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value):
|
| 1310 |
+
raise AmazonError("Economics dates must be YYYY-MM-DD")
|
| 1311 |
+
try:
|
| 1312 |
+
date.fromisoformat(value)
|
| 1313 |
+
except ValueError:
|
| 1314 |
+
raise AmazonError("Economics dates must be valid YYYY-MM-DD dates") from None
|
| 1315 |
+
if start > end or type(summary) is not bool:
|
| 1316 |
+
raise AmazonError("Economics needs an ordered date range and a boolean summary selection")
|
| 1317 |
+
if not isinstance(marketplace_id, str) or not re.fullmatch(r"[A-Z0-9]+", marketplace_id):
|
| 1318 |
+
raise AmazonError("Economics needs an alphanumeric marketplace ID")
|
| 1319 |
+
return (_ECONOMICS_TEMPLATES[summary].replace("__START__", start)
|
| 1320 |
+
.replace("__END__", end).replace("__MARKETPLACE__", marketplace_id))
|
| 1321 |
+
|
| 1322 |
+
|
| 1323 |
+
def _create_query(token, start, end, marketplace_id, *, summary=False,
|
| 1324 |
+
region=DEFAULT_REGION, timeout=_TIMEOUT, sandbox=False):
|
| 1325 |
+
"""The third declared POST wrapper, fixed to a read-only economics query."""
|
| 1326 |
+
require_operation("POST", _QUERIES_CREATE_PATH)
|
| 1327 |
+
body = {"query": economics_query(start, end, marketplace_id, summary=summary)}
|
| 1328 |
+
|
| 1329 |
+
def attempt():
|
| 1330 |
+
base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(str(region).lower())
|
| 1331 |
+
if not base:
|
| 1332 |
+
raise AmazonError(f"unknown region `{region}`")
|
| 1333 |
+
url = f"{base}/{_QUERIES_CREATE_PATH}"
|
| 1334 |
+
req = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST")
|
| 1335 |
+
req.add_header("x-amz-access-token", token)
|
| 1336 |
+
req.add_header("Content-Type", "application/json")
|
| 1337 |
+
req.add_header("User-Agent", _UA)
|
| 1338 |
+
try:
|
| 1339 |
+
with urllib.request.urlopen(req, timeout=timeout) as response:
|
| 1340 |
+
got = json.loads(response.read().decode("utf-8") or "{}")
|
| 1341 |
+
except urllib.error.HTTPError as exc:
|
| 1342 |
+
if exc.code == 429:
|
| 1343 |
+
raise AmazonThrottled("createQuery throttled (429); Data Kiosk permits a burst of "
|
| 1344 |
+
"15, then one query per 60 seconds",
|
| 1345 |
+
retry_after=(exc.headers or {}).get("Retry-After")) from None
|
| 1346 |
+
if exc.code == 403:
|
| 1347 |
+
raise AmazonError("createQuery answered 403: needs the Selling Partner Insights role") from None
|
| 1348 |
+
raise AmazonError(f"createQuery answered {exc.code}") from None
|
| 1349 |
+
except Exception as exc:
|
| 1350 |
+
raise AmazonError(f"createQuery unreachable: {type(exc).__name__}") from None
|
| 1351 |
+
if not got.get("queryId"):
|
| 1352 |
+
raise AmazonError("createQuery returned no queryId")
|
| 1353 |
+
return str(got["queryId"])
|
| 1354 |
+
|
| 1355 |
+
_pace_query_create()
|
| 1356 |
+
return _retry(attempt, _QUERIES_CREATE_PATH,
|
| 1357 |
+
exhausted_fix="reduce Data Kiosk query frequency and retry the window")
|
| 1358 |
+
|
| 1359 |
+
|
| 1360 |
+
def _kiosk_document(token, document_id, *, region, sandbox, timeout):
|
| 1361 |
+
meta = _request(f"dataKiosk/2023-11-15/documents/{document_id}", token,
|
| 1362 |
+
region=region, sandbox=sandbox, timeout=timeout)
|
| 1363 |
+
url = str(meta.get("documentUrl") or "")
|
| 1364 |
+
if not url:
|
| 1365 |
+
raise AmazonError("Data Kiosk document returned no documentUrl")
|
| 1366 |
+
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
|
| 1367 |
+
return text
|
| 1368 |
+
|
| 1369 |
+
|
| 1370 |
+
def economics_document(fields, start, end, *, summary=False, max_wait=900,
|
| 1371 |
+
poll_seconds=15, timeout=_TIMEOUT, sleep=time.sleep):
|
| 1372 |
+
"""Request, poll and download JSONL, including the vendor error document on failure."""
|
| 1373 |
+
marketplace = str(fields.get("marketplace_id") or US_MARKETPLACE_ID)
|
| 1374 |
+
economics_query(start, end, marketplace, summary=summary) # Validate before LWA.
|
| 1375 |
+
token = access_token(fields, timeout=timeout)
|
| 1376 |
+
region, sandbox = str(fields.get("region") or DEFAULT_REGION), is_sandbox(fields)
|
| 1377 |
+
query_id = _create_query(token, start, end, marketplace, summary=summary,
|
| 1378 |
+
region=region, sandbox=sandbox, timeout=timeout)
|
| 1379 |
+
waited = 0
|
| 1380 |
+
while waited <= max_wait:
|
| 1381 |
+
got = _request(f"dataKiosk/2023-11-15/queries/{query_id}", token,
|
| 1382 |
+
region=region, sandbox=sandbox, timeout=timeout)
|
| 1383 |
+
state = got.get("processingStatus")
|
| 1384 |
+
if got.get("errorDocumentId"):
|
| 1385 |
+
detail = _kiosk_document(token, got["errorDocumentId"], region=region,
|
| 1386 |
+
sandbox=sandbox, timeout=timeout)
|
| 1387 |
+
detail = detail.replace("\u2014", "; ").replace("\u2013", " to ")
|
| 1388 |
+
raise AmazonError(f"Economics query {query_id} {state}: {detail[:1200]}")
|
| 1389 |
+
if state == "DONE":
|
| 1390 |
+
doc_id = got.get("dataDocumentId")
|
| 1391 |
+
if not doc_id:
|
| 1392 |
+
raise AmazonError(f"Economics query {query_id} returned no dataDocumentId")
|
| 1393 |
+
return (_kiosk_document(token, doc_id, region=region, sandbox=sandbox, timeout=timeout),
|
| 1394 |
+
{"queryId": query_id, "reportDocumentId": doc_id})
|
| 1395 |
+
if state in ("FATAL", "CANCELLED"):
|
| 1396 |
+
raise AmazonError(f"Economics query {query_id} ended as {state}")
|
| 1397 |
+
sleep(poll_seconds)
|
| 1398 |
+
waited += poll_seconds
|
| 1399 |
+
if waited and waited % 1800 == 0:
|
| 1400 |
+
token = access_token(fields, timeout=timeout)
|
| 1401 |
+
raise AmazonError(f"Economics query {query_id} still pending after {max_wait}s; "
|
| 1402 |
+
"increase the query wait before retrying the window")
|
| 1403 |
+
|
| 1404 |
+
|
| 1405 |
+
def transactions(fields, start, end, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
|
| 1406 |
+
"""Follow every Finances page at 0.5/s, including empty pages carrying a nextToken."""
|
| 1407 |
+
token, issued = access_token(fields, timeout=timeout), now()
|
| 1408 |
+
following, seen, rows = None, set(), []
|
| 1409 |
+
while True:
|
| 1410 |
+
_pace_entity("transactions", TRANSACTIONS_INTERVAL, sleep=sleep, now=now)
|
| 1411 |
+
if now() - issued >= 3000:
|
| 1412 |
+
token, issued = access_token(fields, timeout=timeout), now()
|
| 1413 |
+
got = _request(_TRANSACTIONS_PATH, token,
|
| 1414 |
+
region=str(fields.get("region") or DEFAULT_REGION),
|
| 1415 |
+
sandbox=is_sandbox(fields), timeout=timeout,
|
| 1416 |
+
postedAfter=start, postedBefore=end,
|
| 1417 |
+
marketplaceId=fields.get("marketplace_id") or US_MARKETPLACE_ID,
|
| 1418 |
+
nextToken=following,
|
| 1419 |
+
throttle_fix="resume the same Finances window at its 0.5 requests/second rate")
|
| 1420 |
+
payload = got.get("payload") or {}
|
| 1421 |
+
page = payload.get("transactions")
|
| 1422 |
+
if not isinstance(page, list):
|
| 1423 |
+
raise AmazonError("listTransactions returned no transactions list")
|
| 1424 |
+
rows.extend(page)
|
| 1425 |
+
following = payload.get("nextToken")
|
| 1426 |
+
if not following:
|
| 1427 |
+
return rows
|
| 1428 |
+
if following in seen:
|
| 1429 |
+
raise AmazonError("listTransactions repeated a nextToken; cannot prove complete pagination; "
|
| 1430 |
+
"retry the window after Amazon corrects the cursor")
|
| 1431 |
+
seen.add(following)
|
| 1432 |
+
|
| 1433 |
+
|
| 1434 |
+
ENTITY_INTERVALS = {"getOrderMetrics": 2.0, "getInventorySummaries": 0.5,
|
| 1435 |
+
"searchListingsItems": 0.2, "searchCatalogItems": 0.5}
|
| 1436 |
+
|
| 1437 |
+
|
| 1438 |
+
def group_a_records(fields, operation, *, start=None, end=None, asins=(),
|
| 1439 |
+
timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
|
| 1440 |
+
"""Fixed GET readers, every page and every own-ASIN batch; no record ceiling."""
|
| 1441 |
+
if operation not in ENTITY_INTERVALS:
|
| 1442 |
+
raise AmazonOperationRefused("unapproved group (a) reader")
|
| 1443 |
+
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 1444 |
+
path = OPERATIONS[operation]["path"]
|
| 1445 |
+
params = {"marketplaceIds": marketplace}
|
| 1446 |
+
if operation == "getOrderMetrics":
|
| 1447 |
+
params.update(interval=f"{start}--{end}", granularity="Day", granularityTimeZone="UTC")
|
| 1448 |
+
root, payload_key, next_key = None, "payload", None
|
| 1449 |
+
elif operation == "getInventorySummaries":
|
| 1450 |
+
params.update(details="true", granularityType="Marketplace", granularityId=marketplace)
|
| 1451 |
+
root, payload_key, next_key = "inventorySummaries", "payload", "nextToken"
|
| 1452 |
+
elif operation == "searchListingsItems":
|
| 1453 |
+
seller = str(fields.get("seller_id") or "")
|
| 1454 |
+
if not re.fullmatch(r"[A-Za-z0-9]+", seller):
|
| 1455 |
+
raise AmazonError("Listing status needs the seller ID saved with the connection")
|
| 1456 |
+
path = path.replace("{sellerId}", seller)
|
| 1457 |
+
params.update(includedData="summaries,issues,offers,fulfillmentAvailability,attributes", pageSize=20)
|
| 1458 |
+
root, payload_key, next_key = "items", None, "pageToken"
|
| 1459 |
+
else:
|
| 1460 |
+
# `relationships` (S67-T29, O23): same Catalog Items endpoint and Product Listing role;
|
| 1461 |
+
# it carries VARIATION parentAsins so the mirror can map every own ASIN to its parent.
|
| 1462 |
+
params.update(identifiersType="ASIN",
|
| 1463 |
+
includedData="summaries,salesRanks,dimensions,attributes,relationships",
|
| 1464 |
+
pageSize=20)
|
| 1465 |
+
root, payload_key, next_key = "items", None, "pageToken"
|
| 1466 |
+
batches = [None]
|
| 1467 |
+
if operation == "searchCatalogItems":
|
| 1468 |
+
identifiers = sorted(set(asins))
|
| 1469 |
+
if any(not re.fullmatch(r"[A-Z0-9]{10}", str(a)) for a in identifiers):
|
| 1470 |
+
raise AmazonError("Catalog contains an invalid ASIN; correct the listings or pricing mirror and retry")
|
| 1471 |
+
batches = [identifiers[i:i + 20] for i in range(0, len(identifiers), 20)]
|
| 1472 |
+
if not batches:
|
| 1473 |
+
raise AmazonError("Catalog attributes are waiting for our ASINs; sync Amazon listings first")
|
| 1474 |
+
token, issued = access_token(fields, timeout=timeout), now()
|
| 1475 |
+
rows = []
|
| 1476 |
+
listing_end = datetime.now(timezone.utc)
|
| 1477 |
+
# Date partitions bypass Listings search's 1,000-item pagination ceiling.
|
| 1478 |
+
# Inclusive boundaries are deduplicated by SKU after every partition has finished.
|
| 1479 |
+
if operation == "searchListingsItems":
|
| 1480 |
+
batches = [(datetime(1970, 1, 1, tzinfo=timezone.utc), listing_end)]
|
| 1481 |
+
for batch in batches:
|
| 1482 |
+
query, seen = dict(params), set()
|
| 1483 |
+
if operation == "searchListingsItems":
|
| 1484 |
+
lower, upper = batch
|
| 1485 |
+
query.update(createdAfter=lower.isoformat(), createdBefore=upper.isoformat())
|
| 1486 |
+
elif batch is not None:
|
| 1487 |
+
query["identifiers"] = ",".join(batch)
|
| 1488 |
+
page_rows, expected = [], None
|
| 1489 |
+
while True:
|
| 1490 |
+
_pace_entity(operation, ENTITY_INTERVALS[operation], sleep=sleep, now=now)
|
| 1491 |
+
if now() - issued >= 3000:
|
| 1492 |
+
token, issued = access_token(fields, timeout=timeout), now()
|
| 1493 |
+
got = _request(path, token, region=str(fields.get("region") or DEFAULT_REGION),
|
| 1494 |
+
sandbox=is_sandbox(fields), timeout=timeout, **query,
|
| 1495 |
+
throttle_fix=f"continue every page at Amazon's {1 / ENTITY_INTERVALS[operation]:g} requests per second")
|
| 1496 |
+
body = got.get(payload_key) if payload_key else got
|
| 1497 |
+
if operation == "searchListingsItems" and isinstance(body, dict):
|
| 1498 |
+
count = body.get("numberOfResults")
|
| 1499 |
+
if isinstance(count, int) and count > 1000:
|
| 1500 |
+
midpoint = lower + (upper - lower) / 2
|
| 1501 |
+
if (upper - lower) <= timedelta(milliseconds=1):
|
| 1502 |
+
raise AmazonError("Amazon Listings search exceeds its 1,000 SKU limit within one millisecond. "
|
| 1503 |
+
"No partial snapshot was stored. Ask Amazon to raise this search limit or use a SKU-filtered export.")
|
| 1504 |
+
batches.extend(((lower, midpoint), (midpoint, upper)))
|
| 1505 |
+
page_rows = []
|
| 1506 |
+
break
|
| 1507 |
+
page = body.get(root) if root and isinstance(body, dict) else body
|
| 1508 |
+
if not isinstance(page, list) or any(not isinstance(row, dict) for row in page):
|
| 1509 |
+
raise AmazonError(f"{operation} returned no valid row list; retry after Amazon restores its response")
|
| 1510 |
+
page_rows.extend(page)
|
| 1511 |
+
if isinstance(body, dict) and type(body.get("numberOfResults")) is int:
|
| 1512 |
+
expected = max(expected or 0, body["numberOfResults"])
|
| 1513 |
+
following = (got.get("pagination") or {}).get("nextToken")
|
| 1514 |
+
if isinstance(body, dict):
|
| 1515 |
+
following = following or (body.get("pagination") or {}).get("nextToken") or body.get("nextToken")
|
| 1516 |
+
if not following:
|
| 1517 |
+
if expected is not None and len(page_rows) != expected:
|
| 1518 |
+
raise AmazonError(f"{operation} reported {expected} rows but returned {len(page_rows)}. "
|
| 1519 |
+
"No partial snapshot was stored; retry after Amazon restores complete pagination.")
|
| 1520 |
+
break
|
| 1521 |
+
if not next_key or following in seen:
|
| 1522 |
+
raise AmazonError(f"{operation} repeated or cannot follow its nextToken; retry after Amazon fixes pagination")
|
| 1523 |
+
seen.add(following)
|
| 1524 |
+
query[next_key] = following
|
| 1525 |
+
rows.extend(page_rows)
|
| 1526 |
+
if operation == "searchListingsItems":
|
| 1527 |
+
unique = {}
|
| 1528 |
+
for row in rows:
|
| 1529 |
+
if not row.get("sku"):
|
| 1530 |
+
raise AmazonError("Amazon listing search omitted a SKU; cannot reconcile date partition boundaries")
|
| 1531 |
+
unique[row["sku"]] = row
|
| 1532 |
+
rows = list(unique.values())
|
| 1533 |
+
return rows
|
| 1534 |
+
|
| 1535 |
+
|
| 1536 |
def _post_form(url, data, timeout=_TIMEOUT):
|
| 1537 |
# LWA exchanges credentials at api.amazon.com, not an SP-API operation. Its sole caller fixes
|
| 1538 |
# LWA_TOKEN_URL; the connector-wide gate checks that call separately from OPERATIONS.
|
|
|
|
| 1819 |
is that this function can reach exactly one operation and a caller cannot redirect it.
|
| 1820 |
"""
|
| 1821 |
require_operation("POST", _REPORTS_CREATE_PATH, report_type=str(report_type))
|
| 1822 |
+
if any(s.get("snapshot") and s.get("report") == report_type for s in REPORT_TYPES.values()):
|
| 1823 |
+
if start is not None or end is not None:
|
| 1824 |
+
raise AmazonError("Snapshot reports cannot take a data window")
|
| 1825 |
|
| 1826 |
def attempt():
|
| 1827 |
"""One raw attempt at the declared report-creation POST."""
|
|
|
|
| 1849 |
except urllib.error.HTTPError as e:
|
| 1850 |
detail = ""
|
| 1851 |
try:
|
| 1852 |
+
detail = (e.read() or b"").decode("utf-8", errors="replace")
|
| 1853 |
+
errs = json.loads(detail or "{}").get("errors") or []
|
| 1854 |
+
detail = " ".join(str(err.get(part) or "") for err in errs
|
| 1855 |
+
for part in ("message", "details") if err.get(part)) or detail
|
| 1856 |
except Exception: # noqa: BLE001
|
| 1857 |
+
pass
|
| 1858 |
if e.code == 429:
|
| 1859 |
raise AmazonThrottled(
|
| 1860 |
f"{_safe(url)} was throttled (429) creating a {report_type} report. Report "
|
|
|
|
| 1865 |
raise AmazonError(
|
| 1866 |
f"{_safe(url)} refused a {report_type} report (403). That is a ROLE answer: this "
|
| 1867 |
"report needs a role the application was not granted") from None
|
| 1868 |
+
if e.code == 400 and report_type == REPORT_TYPES["search_query_performance"]["report"]:
|
| 1869 |
+
raise AmazonReportError("Amazon refused Search Query Performance (400)",
|
| 1870 |
+
detail, 400) from None
|
| 1871 |
raise AmazonError(f"{_safe(url)} answered {e.code}"
|
| 1872 |
+
+ (f": {detail[:200]}" if detail else "")) from None
|
| 1873 |
except Exception as e: # noqa: BLE001
|
| 1874 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
| 1875 |
rid = str(got.get("reportId") or "")
|
|
|
|
| 1931 |
doc_id = str(got.get("reportDocumentId") or "")
|
| 1932 |
break
|
| 1933 |
if state in _REPORT_DEAD:
|
| 1934 |
+
if state == "FATAL" and report_type == REPORT_TYPES["search_query_performance"]["report"]:
|
| 1935 |
+
error_id = got.get("reportDocumentId") or got.get("errorDocumentId")
|
| 1936 |
+
reason = str(got.get("errorMessage") or "No error document was supplied by Amazon")
|
| 1937 |
+
if error_id:
|
| 1938 |
+
meta = _request(f"reports/2021-06-30/documents/{error_id}", token,
|
| 1939 |
+
region=region, timeout=timeout, sandbox=sbx)
|
| 1940 |
+
if meta.get("url"):
|
| 1941 |
+
reason = _download_document(meta["url"], str(meta.get("compressionAlgorithm") or ""))
|
| 1942 |
+
raise AmazonReportError(f"Amazon ended Search Query Performance as FATAL (reportId {rid})",
|
| 1943 |
+
reason, state)
|
| 1944 |
+
if state == "CANCELLED":
|
| 1945 |
+
# S66-T42 (lead, observed live 2026-09-25): Amazon documents CANCELLED as also
|
| 1946 |
+
# meaning "the report had no data to return". Raising turned an empty period into
|
| 1947 |
+
# a role-shaped refusal on screen and left the window to retry forever. An empty
|
| 1948 |
+
# document lets the window complete and the connector detail name the empty period.
|
| 1949 |
+
say(f" {report_type}: Amazon cancelled report {rid}; no data for the requested period")
|
| 1950 |
+
return "", {"reportId": rid, "reportDocumentId": None, "cancelled": True,
|
| 1951 |
+
"dataStartTime": got.get("dataStartTime"),
|
| 1952 |
+
"dataEndTime": got.get("dataEndTime"),
|
| 1953 |
+
"createdTime": got.get("createdTime"), "compression": None}
|
| 1954 |
+
# Every FATAL report now carries Amazon's OWN reason when it supplied an error document,
|
| 1955 |
+
# so the connector detail names why, never only a generic role hint.
|
| 1956 |
+
detail = ""
|
| 1957 |
+
error_id = got.get("reportDocumentId") or got.get("errorDocumentId")
|
| 1958 |
+
if error_id:
|
| 1959 |
+
try:
|
| 1960 |
+
meta = _request(f"reports/2021-06-30/documents/{error_id}", token,
|
| 1961 |
+
region=region, timeout=timeout, sandbox=sbx)
|
| 1962 |
+
if meta.get("url"):
|
| 1963 |
+
detail = _download_document(meta["url"], str(meta.get("compressionAlgorithm") or ""))
|
| 1964 |
+
except AmazonError:
|
| 1965 |
+
detail = ""
|
| 1966 |
+
detail = " ".join(str(detail).replace("\u2014", "; ").replace("\u2013", " to ").split())[:240]
|
| 1967 |
+
raise AmazonError(f"Amazon ended the {report_type} report as {state} (reportId {rid})"
|
| 1968 |
+
+ (f": {detail}" if detail else ""))
|
| 1969 |
time.sleep(poll_seconds)
|
| 1970 |
waited += poll_seconds
|
| 1971 |
# β The token is ~1 hour and a long report can outlive it. Re-mint rather than 403.
|
|
|
|
| 1984 |
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
|
| 1985 |
say(f" {report_type}: {len(text):,} chars")
|
| 1986 |
return text, {"reportId": rid, "reportDocumentId": doc_id,
|
| 1987 |
+
"dataStartTime": got.get("dataStartTime"),
|
| 1988 |
+
"dataEndTime": got.get("dataEndTime"), "createdTime": got.get("createdTime"),
|
| 1989 |
"compression": meta.get("compressionAlgorithm")}
|
| 1990 |
|
| 1991 |
|
|
|
|
| 2052 |
return _download_document(url, str(meta.get("compressionAlgorithm") or ""))
|
| 2053 |
|
| 2054 |
|
| 2055 |
+
def _workbook_as_tsv(raw):
|
| 2056 |
+
"""An Excel workbook document, as the tab-separated text every flat-file parser reads.
|
| 2057 |
+
|
| 2058 |
+
S66-T42 (lead, observed live 2026-09-25): Amazon delivers some reports (FEE_DISCOUNTS_REPORT,
|
| 2059 |
+
GET_REMOTE_FULFILLMENT_ELIGIBILITY) as an .xlsx workbook, a ZIP container. Decoded as text it
|
| 2060 |
+
became garbage column names carrying NUL bytes, and Postgres refused the whole Amazon write.
|
| 2061 |
+
The first sheet is the report; its first row is the header, like a flat file. Tabs and line
|
| 2062 |
+
breaks inside a cell become spaces so the TSV shape holds.
|
| 2063 |
+
"""
|
| 2064 |
+
import io
|
| 2065 |
+
try:
|
| 2066 |
+
import openpyxl
|
| 2067 |
+
book = openpyxl.load_workbook(io.BytesIO(raw), read_only=True, data_only=True)
|
| 2068 |
+
except Exception as e: # noqa: BLE001
|
| 2069 |
+
raise AmazonError(f"the report document is a workbook that did not open: {type(e).__name__}") from None
|
| 2070 |
+
sheet = book.worksheets[0] if book.worksheets else None
|
| 2071 |
+
lines = []
|
| 2072 |
+
for values in (sheet.iter_rows(values_only=True) if sheet is not None else ()):
|
| 2073 |
+
cells = ["" if v is None else " ".join(str(v).replace("\x00", "").split()) for v in values]
|
| 2074 |
+
if any(cells):
|
| 2075 |
+
lines.append("\t".join(cells))
|
| 2076 |
+
book.close()
|
| 2077 |
+
return "\n".join(lines)
|
| 2078 |
+
|
| 2079 |
+
|
| 2080 |
def _download_document(url, compression="", timeout=120):
|
| 2081 |
"""Fetch one presigned report document and decode it. -> text.
|
| 2082 |
|
|
|
|
| 2095 |
raw = r.read()
|
| 2096 |
except Exception as e: # noqa: BLE001
|
| 2097 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
| 2098 |
+
# S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
|
| 2099 |
+
# `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
|
| 2100 |
+
# algorithm the bytes were decoded as text and every economics window failed to parse. The
|
| 2101 |
+
# gzip magic number is unambiguous (no text report starts with 0x1f 0x8b), so it decides.
|
| 2102 |
+
if not compression and raw[:2] == b"\x1f\x8b":
|
| 2103 |
+
compression = "GZIP"
|
| 2104 |
if str(compression).upper() == "GZIP":
|
| 2105 |
import gzip
|
| 2106 |
try:
|
| 2107 |
raw = gzip.decompress(raw)
|
| 2108 |
except Exception as e: # noqa: BLE001
|
| 2109 |
raise AmazonError(f"the report document did not decompress: {type(e).__name__}") from None
|
| 2110 |
+
if raw[:4] == b"PK\x03\x04":
|
| 2111 |
+
return _workbook_as_tsv(raw)
|
| 2112 |
for enc in ("utf-8", "cp1252"):
|
| 2113 |
try:
|
| 2114 |
return raw.decode(enc)
|
platform/harness/amazon_schema.py
CHANGED
|
@@ -53,6 +53,9 @@ from pathlib import Path
|
|
| 53 |
|
| 54 |
_HERE = Path(__file__).resolve().parent
|
| 55 |
CATALOGUE = _HERE / "proto" / "amazon-report-schemas.json"
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
#: The vendor repository, pinned. β `main` is what `--refresh` reads, and the resulting file
|
| 58 |
#: records the exact commit and per-file blob shas it saw, so a later refresh that changes a column
|
|
@@ -81,14 +84,24 @@ PROVENANCE = {
|
|
| 81 |
"inventory": "measured_at_sync",
|
| 82 |
"pricing": "schema_artifact",
|
| 83 |
"brand_analytics": "schema_artifact",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
}
|
| 85 |
|
|
|
|
|
|
|
|
|
|
| 86 |
#: Human copy for the disclosure screen, per provenance. β NO EM DASH OR EN DASH: this reaches a
|
| 87 |
#: screen and standing rule 2 governs it.
|
| 88 |
PROVENANCE_COPY = {
|
|
|
|
| 89 |
"schema_artifact": "Columns come from Amazon's published report schema, pinned below.",
|
| 90 |
-
"measured_at_sync": ("
|
| 91 |
-
"
|
| 92 |
}
|
| 93 |
|
| 94 |
_JSON_SCALAR = {"string": "VARCHAR", "integer": "BIGINT", "number": "DOUBLE", "boolean": "VARCHAR"}
|
|
@@ -254,9 +267,23 @@ def refresh(log=print):
|
|
| 254 |
def catalogue():
|
| 255 |
"""The pinned catalogue off disk, or an empty shell when it has never been refreshed."""
|
| 256 |
try:
|
| 257 |
-
|
| 258 |
except Exception: # noqa: BLE001
|
| 259 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
|
| 261 |
|
| 262 |
def columns_for(table):
|
|
@@ -282,7 +309,7 @@ def _table_spec(table, identity, relationships=(), *, grain="entity", line_discr
|
|
| 282 |
"provenance": PROVENANCE[table],
|
| 283 |
"assumption": (
|
| 284 |
"A-QI14: This wave declares from REPORT_TYPES and the pinned artifacts. "
|
| 285 |
-
"
|
| 286 |
"production sync during the deferred QI13 hotfix (intake RDA6)."),
|
| 287 |
},
|
| 288 |
"row_identity": identity,
|
|
@@ -358,9 +385,122 @@ TABLE_SPECS = {
|
|
| 358 |
"brand_analytics": _table_spec("brand_analytics", (
|
| 359 |
"department_name", "search_term", "clicked_asin", "click_share_rank",
|
| 360 |
)),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 361 |
}
|
| 362 |
|
| 363 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 364 |
def row_identity_for(table, row):
|
| 365 |
"""Evaluate the declared full composite. Missing lineage is an error, never a parent ID.
|
| 366 |
|
|
@@ -469,7 +609,8 @@ def selftest(log=print):
|
|
| 469 |
ck("NC-AS6 every schema_artifact table has an artifact", not orphan, orphan)
|
| 470 |
|
| 471 |
# NC-AS7 β the seven tables of R4, no more and no fewer.
|
| 472 |
-
ck("NC-AS7
|
|
|
|
| 473 |
|
| 474 |
# NC-AS8 β standing rule 2 governs PROVENANCE_COPY: it reaches the disclosure screen.
|
| 475 |
dashed = [k for k, v in PROVENANCE_COPY.items() if "β" in v or "β" in v]
|
|
@@ -477,8 +618,8 @@ def selftest(log=print):
|
|
| 477 |
|
| 478 |
# NC-AS9 β disclosure() names every table and carries the sync stamp key even when unpinned.
|
| 479 |
d = disclosure()
|
| 480 |
-
ck("NC-AS9 disclosure covers all
|
| 481 |
-
len(d["tables"]) ==
|
| 482 |
for r in d["tables"]))
|
| 483 |
|
| 484 |
# NC-AS10 β a measured_at_sync table must ship ZERO columns. An invented column is the defect.
|
|
|
|
| 53 |
|
| 54 |
_HERE = Path(__file__).resolve().parent
|
| 55 |
CATALOGUE = _HERE / "proto" / "amazon-report-schemas.json"
|
| 56 |
+
if str(_HERE.parent) not in sys.path:
|
| 57 |
+
sys.path.insert(0, str(_HERE.parent))
|
| 58 |
+
from harness.amazon_client import GROUP_A_TABLES, DECLARED_DATASETS
|
| 59 |
|
| 60 |
#: The vendor repository, pinned. β `main` is what `--refresh` reads, and the resulting file
|
| 61 |
#: records the exact commit and per-file blob shas it saw, so a later refresh that changes a column
|
|
|
|
| 84 |
"inventory": "measured_at_sync",
|
| 85 |
"pricing": "schema_artifact",
|
| 86 |
"brand_analytics": "schema_artifact",
|
| 87 |
+
"economics": "measured_at_sync",
|
| 88 |
+
"transactions": "measured_at_sync",
|
| 89 |
+
"fba_fee_estimates": "measured_at_sync",
|
| 90 |
+
"referral_fees": "measured_at_sync",
|
| 91 |
+
"storage_fees": "measured_at_sync",
|
| 92 |
+
"search_query_performance": "measured_at_sync",
|
| 93 |
}
|
| 94 |
|
| 95 |
+
PROVENANCE.update(dict.fromkeys(GROUP_A_TABLES, "measured_at_sync"))
|
| 96 |
+
PROVENANCE.update(dict.fromkeys(DECLARED_DATASETS, "declared"))
|
| 97 |
+
|
| 98 |
#: Human copy for the disclosure screen, per provenance. β NO EM DASH OR EN DASH: this reaches a
|
| 99 |
#: screen and standing rule 2 governs it.
|
| 100 |
PROVENANCE_COPY = {
|
| 101 |
+
"declared": "This dataset is listed for completeness; its named reason explains why no rows are loaded.",
|
| 102 |
"schema_artifact": "Columns come from Amazon's published report schema, pinned below.",
|
| 103 |
+
"measured_at_sync": ("Columns are measured from the document header or JSON record keys "
|
| 104 |
+
"when this database syncs; no column artifact is pinned here."),
|
| 105 |
}
|
| 106 |
|
| 107 |
_JSON_SCALAR = {"string": "VARCHAR", "integer": "BIGINT", "number": "DOUBLE", "boolean": "VARCHAR"}
|
|
|
|
| 267 |
def catalogue():
|
| 268 |
"""The pinned catalogue off disk, or an empty shell when it has never been refreshed."""
|
| 269 |
try:
|
| 270 |
+
result = json.loads(CATALOGUE.read_text(encoding="utf-8"))
|
| 271 |
except Exception: # noqa: BLE001
|
| 272 |
+
result = {"repo": REPO, "commit": None, "fetched_utc": None, "tables": {}}
|
| 273 |
+
from core.connector_spec import AMAZON_TABLES
|
| 274 |
+
descriptors = {row.key: row for row in AMAZON_TABLES}
|
| 275 |
+
for name, provenance in PROVENANCE.items():
|
| 276 |
+
key = "ut_amzn_" + name
|
| 277 |
+
table = result.setdefault("tables", {}).setdefault(name, {})
|
| 278 |
+
table["provenance"] = provenance
|
| 279 |
+
if name in DECLARED_DATASETS:
|
| 280 |
+
declared = DECLARED_DATASETS[name]
|
| 281 |
+
label = declared["label"]
|
| 282 |
+
native = ", ".join(declared.get("report_types") or declared.get("operations") or ())
|
| 283 |
+
else:
|
| 284 |
+
label, native = descriptors[key].label, descriptors[key].native
|
| 285 |
+
table[key] = {"label": label, "native": native}
|
| 286 |
+
return result
|
| 287 |
|
| 288 |
|
| 289 |
def columns_for(table):
|
|
|
|
| 309 |
"provenance": PROVENANCE[table],
|
| 310 |
"assumption": (
|
| 311 |
"A-QI14: This wave declares from REPORT_TYPES and the pinned artifacts. "
|
| 312 |
+
"Measured-at-sync columns are re-measured against the first "
|
| 313 |
"production sync during the deferred QI13 hotfix (intake RDA6)."),
|
| 314 |
},
|
| 315 |
"row_identity": identity,
|
|
|
|
| 385 |
"brand_analytics": _table_spec("brand_analytics", (
|
| 386 |
"department_name", "search_term", "clicked_asin", "click_share_rank",
|
| 387 |
)),
|
| 388 |
+
"economics": _table_spec("economics", ("start_date", "msku", "marketplace_id"), (
|
| 389 |
+
("msku", "listings", "seller_sku", "many_to_one"),
|
| 390 |
+
("child_asin", "listings", "asin1", "many_to_one"),
|
| 391 |
+
), grain="day_msku"),
|
| 392 |
+
"transactions": _table_spec("transactions", ("transaction_id", "item_position"), (
|
| 393 |
+
("amazon_order_id", "orders", "amazon_order_id", "many_to_one"),
|
| 394 |
+
("sku", "listings", "seller_sku", "many_to_one"),
|
| 395 |
+
), grain="transaction_item"),
|
| 396 |
+
# S66-T40 (lead, measured live): the fee estimate snapshot has one row per SKU per Amazon
|
| 397 |
+
# store, and the referral fee preview spells its SKU `seller_sku`.
|
| 398 |
+
"fba_fee_estimates": _table_spec("fba_fee_estimates", ("_snapshot_date", "sku", "amazon_store"), (
|
| 399 |
+
("sku", "listings", "seller_sku", "many_to_one"),
|
| 400 |
+
), grain="snapshot_sku"),
|
| 401 |
+
"referral_fees": _table_spec("referral_fees", ("_snapshot_date", "seller_sku"), (
|
| 402 |
+
("seller_sku", "listings", "seller_sku", "many_to_one"),
|
| 403 |
+
), grain="snapshot_sku"),
|
| 404 |
+
"storage_fees": _table_spec("storage_fees", ("month", "asin", "fnsku", "fulfillment_center"), (
|
| 405 |
+
("asin", "listings", "asin1", "many_to_one"),
|
| 406 |
+
), grain="month_product_center"),
|
| 407 |
+
"search_query_performance": _table_spec("search_query_performance", (
|
| 408 |
+
"start_date", "end_date", "asin", "search_query_data_search_query"), (
|
| 409 |
+
("asin", "listings", "asin1", "many_to_one"),
|
| 410 |
+
), grain="month_asin_search_query"),
|
| 411 |
}
|
| 412 |
|
| 413 |
|
| 414 |
+
TABLE_SPECS.update({
|
| 415 |
+
"long_term_storage_fees": _table_spec("long_term_storage_fees", ("_period_start", "sku", "fnsku", "asin", "_report_row_number"),
|
| 416 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
|
| 417 |
+
"storage_overage_fees": _table_spec("storage_overage_fees", ("_period_start", "storage_type", "_report_row_number"),
|
| 418 |
+
(), grain="month_row"),
|
| 419 |
+
"reimbursements": _table_spec("reimbursements", ("_period_start", "reimbursement_id", "sku", "asin", "amazon_order_id", "_report_row_number"),
|
| 420 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"), ("amazon_order_id", "orders", "amazon_order_id", "many_to_one"),), grain="window_row"),
|
| 421 |
+
"fba_customer_returns": _table_spec("fba_customer_returns", ("_period_start", "order_id", "sku", "fnsku", "asin", "_report_row_number"),
|
| 422 |
+
(("order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
|
| 423 |
+
"replacements": _table_spec("replacements", ("_period_start", "replacement_amazon_order_id", "original_amazon_order_id", "sku", "asin", "_report_row_number"),
|
| 424 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
|
| 425 |
+
"fba_shipped_sales": _table_spec("fba_shipped_sales", ("_period_start", "amazon_order_id", "sku", "fnsku", "asin", "_report_row_number"),
|
| 426 |
+
(("amazon_order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
|
| 427 |
+
"fba_shipment_promotions": _table_spec("fba_shipment_promotions", ("_period_start", "amazon_order_id", "sku", "_report_row_number"),
|
| 428 |
+
(("amazon_order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"),), grain="window_row"),
|
| 429 |
+
"inventory_ledger_detail": _table_spec("inventory_ledger_detail", ("_period_start", "msku", "fnsku", "asin", "_report_row_number"),
|
| 430 |
+
(("msku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
|
| 431 |
+
"inventory_planning": _table_spec("inventory_planning", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 432 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 433 |
+
"manage_inventory": _table_spec("manage_inventory", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 434 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 435 |
+
"reserved_inventory": _table_spec("reserved_inventory", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 436 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 437 |
+
"restock_recommendations": _table_spec("restock_recommendations", ("_snapshot_date", "merchant_sku", "fnsku", "asin", "_report_row_number"),
|
| 438 |
+
(("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 439 |
+
"stranded_inventory": _table_spec("stranded_inventory", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 440 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 441 |
+
"inbound_noncompliance": _table_spec("inbound_noncompliance", ("_period_start", "shipment_id", "sku", "asin", "_report_row_number"),
|
| 442 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
|
| 443 |
+
"inventory_expiration_dates": _table_spec("inventory_expiration_dates", ("_snapshot_date", "sku", "fnsku", "_report_row_number"),
|
| 444 |
+
(("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
|
| 445 |
+
"recommended_removals": _table_spec("recommended_removals", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
|
| 446 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 447 |
+
"removal_orders": _table_spec("removal_orders", ("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
|
| 448 |
+
(("sku", "listings", "seller_sku", "many_to_one"),), grain="window_row"),
|
| 449 |
+
"removal_shipments": _table_spec("removal_shipments", ("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
|
| 450 |
+
(("sku", "listings", "seller_sku", "many_to_one"),), grain="window_row"),
|
| 451 |
+
"subscribe_save_forecast": _table_spec("subscribe_save_forecast", ("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 452 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 453 |
+
"subscribe_save_performance": _table_spec("subscribe_save_performance", ("_period_start", "sku", "asin", "_report_row_number"),
|
| 454 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
|
| 455 |
+
"remote_fulfillment_eligibility": _table_spec("remote_fulfillment_eligibility", ("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 456 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 457 |
+
"search_catalog_performance": _table_spec("search_catalog_performance", ("_period_start", "asin", "_report_row_number"),
|
| 458 |
+
(("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
|
| 459 |
+
"market_basket": _table_spec("market_basket", ("_period_start", "asin", "purchased_with_asin", "_report_row_number"),
|
| 460 |
+
(("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
|
| 461 |
+
"repeat_purchase": _table_spec("repeat_purchase", ("_period_start", "asin", "_report_row_number"),
|
| 462 |
+
(("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
|
| 463 |
+
"promotion_performance": _table_spec("promotion_performance", ("_period_start", "promotion_id", "_report_row_number"),
|
| 464 |
+
(), grain="window_row"),
|
| 465 |
+
"coupon_performance": _table_spec("coupon_performance", ("_period_start", "coupon_id", "_report_row_number"),
|
| 466 |
+
(), grain="window_row"),
|
| 467 |
+
"account_health": _table_spec("account_health", ("_snapshot_date", "marketplace_id", "_json_root", "_report_row_number"),
|
| 468 |
+
(), grain="snapshot_row"),
|
| 469 |
+
"inactive_listings": _table_spec("inactive_listings", ("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
|
| 470 |
+
(("seller_sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
|
| 471 |
+
"cancelled_listings": _table_spec("cancelled_listings", ("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
|
| 472 |
+
(("seller_sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
|
| 473 |
+
"suppressed_listings": _table_spec("suppressed_listings", ("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 474 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 475 |
+
"listing_defects": _table_spec("listing_defects", ("_snapshot_date", "sku", "_report_row_number"),
|
| 476 |
+
(("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
|
| 477 |
+
"sold_listings": _table_spec("sold_listings", ("_snapshot_date", "sku", "_report_row_number"),
|
| 478 |
+
(("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
|
| 479 |
+
"mfn_returns": _table_spec("mfn_returns", ("_period_start", "order_id", "sku", "asin", "_report_row_number"),
|
| 480 |
+
(("order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
|
| 481 |
+
"b2b_recommended": _table_spec("b2b_recommended", ("_snapshot_date", "asin", "_report_row_number"),
|
| 482 |
+
(("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 483 |
+
"b2b_not_yet_on_amazon": _table_spec("b2b_not_yet_on_amazon", ("_snapshot_date", "ean", "upc", "part_number", "_report_row_number"),
|
| 484 |
+
(), grain="snapshot_row"),
|
| 485 |
+
"geo_opportunities": _table_spec("geo_opportunities", ("_snapshot_date", "asin", "_report_row_number"),
|
| 486 |
+
(("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 487 |
+
"out_of_stock_mfn": _table_spec("out_of_stock_mfn", ("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 488 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 489 |
+
"b2b_quotes": _table_spec("b2b_quotes", ("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 490 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 491 |
+
"fee_discounts": _table_spec("fee_discounts", ("_snapshot_date", "sku", "asin", "_report_row_number"),
|
| 492 |
+
(("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 493 |
+
"order_metrics": _table_spec("order_metrics", ("_period_start", "interval", "_report_row_number"),
|
| 494 |
+
(), grain="window_row"),
|
| 495 |
+
"fba_inventory": _table_spec("fba_inventory", ("_snapshot_date", "seller_sku", "fn_sku", "asin", "_report_row_number"),
|
| 496 |
+
(("seller_sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 497 |
+
"listing_items": _table_spec("listing_items", ("_snapshot_date", "sku", "_report_row_number"),
|
| 498 |
+
(("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
|
| 499 |
+
"catalog_items": _table_spec("catalog_items", ("_snapshot_date", "asin", "_report_row_number"),
|
| 500 |
+
(("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
|
| 501 |
+
})
|
| 502 |
+
|
| 503 |
+
|
| 504 |
def row_identity_for(table, row):
|
| 505 |
"""Evaluate the declared full composite. Missing lineage is an error, never a parent ID.
|
| 506 |
|
|
|
|
| 609 |
ck("NC-AS6 every schema_artifact table has an artifact", not orphan, orphan)
|
| 610 |
|
| 611 |
# NC-AS7 β the seven tables of R4, no more and no fewer.
|
| 612 |
+
ck("NC-AS7 table and declared dataset keys cover group (a)",
|
| 613 |
+
set(PROVENANCE) == set(TABLE_SPECS) | set(DECLARED_DATASETS), sorted(PROVENANCE))
|
| 614 |
|
| 615 |
# NC-AS8 β standing rule 2 governs PROVENANCE_COPY: it reaches the disclosure screen.
|
| 616 |
dashed = [k for k, v in PROVENANCE_COPY.items() if "β" in v or "β" in v]
|
|
|
|
| 618 |
|
| 619 |
# NC-AS9 β disclosure() names every table and carries the sync stamp key even when unpinned.
|
| 620 |
d = disclosure()
|
| 621 |
+
ck("NC-AS9 disclosure covers all datasets and carries a synced field",
|
| 622 |
+
len(d["tables"]) == len(PROVENANCE) and "synced" in d and all(r["table"].startswith("ut_amzn_")
|
| 623 |
for r in d["tables"]))
|
| 624 |
|
| 625 |
# NC-AS10 β a measured_at_sync table must ship ZERO columns. An invented column is the defect.
|
platform/harness/amazon_store.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
platform/harness/datastore.py
CHANGED
|
@@ -797,13 +797,63 @@ def _published_content_sha(tok, data=None):
|
|
| 797 |
("sha256", "contentSha", "content_sha", "sha"))
|
| 798 |
|
| 799 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 800 |
def ensure_seed():
|
| 801 |
"""Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store
|
| 802 |
from the PRIVATE dataset's seed snapshot β Analyst-ready in ~a minute instead of a 20-40 min
|
| 803 |
XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores
|
| 804 |
the pilot views/routines when absent. Fail-quiet: no seed/token β normal backfill."""
|
| 805 |
import shutil
|
| 806 |
-
if
|
| 807 |
return False
|
| 808 |
tok = os.environ.get("HF_TOKEN")
|
| 809 |
if not tok:
|
|
@@ -812,11 +862,10 @@ def ensure_seed():
|
|
| 812 |
from huggingface_hub import hf_hub_download
|
| 813 |
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 814 |
p = hf_hub_download(SEED_DATASET, SEED_FILE, repo_type="dataset", token=tok)
|
| 815 |
-
# Atomic install: copy to a temp path then
|
| 816 |
-
# DB_PATH.exists() must never bind a half-written file into the shared instance.
|
| 817 |
tmp = DB_PATH.with_name(DB_PATH.name + ".tmp")
|
| 818 |
shutil.copyfile(p, tmp)
|
| 819 |
-
|
| 820 |
# β W50-T29 β RECORD WHICH SEED THIS IS, because nothing did and the status surface could
|
| 821 |
# therefore only ever say "there is a mirror". Three fields: the dataset, the commit sha of
|
| 822 |
# the exact file downloaded, and the date the DATA was cut, which comes from the seed's own
|
|
|
|
| 797 |
("sha256", "contentSha", "content_sha", "sha"))
|
| 798 |
|
| 799 |
|
| 800 |
+
def _seedable():
|
| 801 |
+
"""Is there no usable mirror to keep? True for a missing file, or for one an early reader
|
| 802 |
+
CREATED empty (`_instance()` makes the file) before the boot seed landed: tables none but
|
| 803 |
+
`_sync_state`, and that one without a row. A file holding anything else is kept."""
|
| 804 |
+
if not DB_PATH.exists():
|
| 805 |
+
return True
|
| 806 |
+
try:
|
| 807 |
+
cur = ro_cursor()
|
| 808 |
+
try:
|
| 809 |
+
names = {r[0] for r in cur.execute(
|
| 810 |
+
"SELECT table_name FROM information_schema.tables "
|
| 811 |
+
"WHERE table_schema = 'main'").fetchall()}
|
| 812 |
+
rows = (cur.execute("SELECT count(*) FROM _sync_state").fetchone()[0]
|
| 813 |
+
if "_sync_state" in names else 0)
|
| 814 |
+
finally:
|
| 815 |
+
cur.close()
|
| 816 |
+
except Exception: # noqa: BLE001
|
| 817 |
+
return False # cannot tell: keep the file, the pre-hotfix behaviour
|
| 818 |
+
return not (names - {"_sync_state"}) and not rows
|
| 819 |
+
|
| 820 |
+
|
| 821 |
+
def _install_seed(tmp):
|
| 822 |
+
"""Swap the downloaded seed in, closing any instance a reader opened on the old file.
|
| 823 |
+
|
| 824 |
+
ββ HOTFIX 2026-09-27 (D-3030). A reader that reached `_instance()` during the seed download
|
| 825 |
+
CREATED an empty royal.duckdb and kept it open; a bare `os.replace` then orphaned that handle
|
| 826 |
+
(Linux keeps the unlinked inode), so the whole process read an EMPTY mirror, 0 of 8 Odoo
|
| 827 |
+
phases, while the seed on disk had all 8 `live`. Staging's Odoo cadence is manual, so nothing
|
| 828 |
+
ever refilled it: every Odoo grid, measure column and Sheet cell refused "first sync".
|
| 829 |
+
Under `_INSTANCE_LOCK` (a plain Lock, so `use_path` is inlined, not called) no reader can
|
| 830 |
+
reopen the old file between the close and the swap; the close checkpoints its WAL, and any
|
| 831 |
+
leftover WAL is removed so it cannot be replayed onto the seed.
|
| 832 |
+
"""
|
| 833 |
+
with _INSTANCE_LOCK:
|
| 834 |
+
con = _INSTANCE.get("con")
|
| 835 |
+
if con is not None:
|
| 836 |
+
try:
|
| 837 |
+
con.close()
|
| 838 |
+
except Exception: # noqa: BLE001
|
| 839 |
+
pass
|
| 840 |
+
_INSTANCE["con"] = None
|
| 841 |
+
wal = DB_PATH.with_name(DB_PATH.name + ".wal")
|
| 842 |
+
if wal.exists():
|
| 843 |
+
wal.unlink()
|
| 844 |
+
os.replace(tmp, DB_PATH)
|
| 845 |
+
_RO_TLS.__dict__.pop("cur", None)
|
| 846 |
+
_READY["ok"] = False
|
| 847 |
+
_GENERATION[0] += 1 # every other thread's cached cursor is now stale
|
| 848 |
+
|
| 849 |
+
|
| 850 |
def ensure_seed():
|
| 851 |
"""Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store
|
| 852 |
from the PRIVATE dataset's seed snapshot β Analyst-ready in ~a minute instead of a 20-40 min
|
| 853 |
XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores
|
| 854 |
the pilot views/routines when absent. Fail-quiet: no seed/token β normal backfill."""
|
| 855 |
import shutil
|
| 856 |
+
if not _seedable():
|
| 857 |
return False
|
| 858 |
tok = os.environ.get("HF_TOKEN")
|
| 859 |
if not tok:
|
|
|
|
| 862 |
from huggingface_hub import hf_hub_download
|
| 863 |
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 864 |
p = hf_hub_download(SEED_DATASET, SEED_FILE, repo_type="dataset", token=tok)
|
| 865 |
+
# Atomic install: copy to a temp path then swap it in under the instance lock.
|
|
|
|
| 866 |
tmp = DB_PATH.with_name(DB_PATH.name + ".tmp")
|
| 867 |
shutil.copyfile(p, tmp)
|
| 868 |
+
_install_seed(tmp)
|
| 869 |
# β W50-T29 β RECORD WHICH SEED THIS IS, because nothing did and the status surface could
|
| 870 |
# therefore only ever say "there is a mirror". Three fields: the dataset, the commit sha of
|
| 871 |
# the exact file downloaded, and the date the DATA was cut, which comes from the seed's own
|
platform/harness/runtime.py
CHANGED
|
@@ -204,6 +204,14 @@ class TenantRuntime:
|
|
| 204 |
answer = reader([store_key for _name, store_key in keyed])
|
| 205 |
return {name: answer.get(store_key, {}) for name, store_key in keyed}
|
| 206 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
def revision(self, name):
|
| 208 |
"""Return the tiny durable change token for this tenant's addressed bucket."""
|
| 209 |
return self._store().revision(self.store_key(name))
|
|
|
|
| 204 |
answer = reader([store_key for _name, store_key in keyed])
|
| 205 |
return {name: answer.get(store_key, {}) for name, store_key in keyed}
|
| 206 |
|
| 207 |
+
def get_entry(self, name, entry):
|
| 208 |
+
"""One top-level entry of `name` when the bound store offers it, else None (HOTFIX
|
| 209 |
+
2026-09-28). None also means absent; either way the caller falls back to `get`."""
|
| 210 |
+
reader = getattr(self._store(), "get_entry", None)
|
| 211 |
+
if not callable(reader):
|
| 212 |
+
return None
|
| 213 |
+
return reader(self.store_key(name), entry)
|
| 214 |
+
|
| 215 |
def revision(self, name):
|
| 216 |
"""Return the tiny durable change token for this tenant's addressed bucket."""
|
| 217 |
return self._store().revision(self.store_key(name))
|
platform/harness/telemetry.py
CHANGED
|
@@ -70,8 +70,13 @@ def error(where, exc=None, **fields):
|
|
| 70 |
ERR_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 71 |
with ERR_PATH.open("a", encoding="utf-8") as f:
|
| 72 |
f.write(json.dumps(rec, default=str) + "\n")
|
| 73 |
-
|
| 74 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
except Exception:
|
| 76 |
pass
|
| 77 |
|
|
|
|
| 70 |
ERR_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 71 |
with ERR_PATH.open("a", encoding="utf-8") as f:
|
| 72 |
f.write(json.dumps(rec, default=str) + "\n")
|
| 73 |
+
# SDLC 67 sweep: an exc-less call (routes_activity forwarding a client error) left
|
| 74 |
+
# `type`/`msg` unset, so this mirror printed an empty ": " on every one -- the JSONL line
|
| 75 |
+
# kept the real `name`/`message`/`source` fields, only the container-log mirror lost them.
|
| 76 |
+
detail = f"{rec.get('type', '')} {rec.get('msg', '')}".strip()
|
| 77 |
+
if not detail:
|
| 78 |
+
detail = " ".join(f"{k}={v}" for k, v in fields.items() if v) or "(no detail)"
|
| 79 |
+
print(f"[app-error] {rec['ts']} {rec['where']}: {detail}", file=sys.stderr, flush=True)
|
| 80 |
except Exception:
|
| 81 |
pass
|
| 82 |
|