Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- RELEASES.json +1 -1
- VERSION +1 -1
- api/application_registry.py +193 -9
- api/automation_engine.py +96 -13
- api/connectors_ig.py +30 -0
- api/connectors_tt.py +43 -3
- api/deps.py +29 -0
- api/formula_runtime.py +51 -9
- api/main.py +329 -83
- api/odoo_relational.py +444 -20
- api/providers.py +344 -0
- api/routes_activity.py +392 -0
- api/routes_admin.py +8 -1
- api/routes_applications.py +258 -0
- api/routes_automation.py +16 -4
- api/routes_connected_tables.py +4 -1
- api/routes_connectors.py +1180 -38
- api/routes_customers.py +73 -0
- api/routes_external.py +0 -0
- api/routes_geo.py +52 -0
- api/routes_module_index.py +25 -2
- api/routes_odoo_tables.py +351 -35
- api/routes_shares.py +173 -308
- api/routes_sheet.py +600 -216
- api/routes_slack.py +55 -9
- api/routes_templates.py +36 -1
- api/test_s63_integration.py +240 -0
- api/test_sheet_commit_events.py +162 -0
- platform/aios_grid.py +5 -2
- platform/aios_grid_fields.json +8 -0
- platform/core/audit.py +29 -0
- platform/core/daily_sales_repair.py +977 -0
- platform/core/external_access.py +137 -5
- platform/core/external_oauth.py +723 -0
- platform/core/external_tools.py +171 -8
- platform/core/field_permissions.py +384 -0
- platform/core/grid_events.py +97 -0
- platform/core/registry.py +47 -0
- platform/core/sheet_binding.py +178 -5
- platform/core/sheet_compat.py +875 -0
- platform/core/sheet_events.py +101 -0
- platform/core/sheet_excel.py +65 -0
- platform/core/sheet_import.py +289 -27
- platform/core/sheet_templates.py +2 -2
- platform/core/sheets.py +204 -13
- platform/core/table_store.py +5 -4
- platform/core/user_tables.py +84 -0
- platform/core/write_execution.py +45 -0
- platform/harness/filter_sql.py +7 -0
- platform/harness/mirror_tokens.py +64 -0
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v75",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "df68553 (df68553)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v75",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
df68553 (df68553)
|
api/application_registry.py
CHANGED
|
@@ -2,12 +2,20 @@
|
|
| 2 |
from __future__ import annotations
|
| 3 |
|
| 4 |
import copy
|
| 5 |
-
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
|
| 9 |
APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
_KEY = "applications_v1"
|
| 12 |
_MAX = 500
|
| 13 |
|
|
@@ -48,6 +56,15 @@ _TEMPLATE_CONTRACTS = {
|
|
| 48 |
],
|
| 49 |
"defaultPlacement": {"section": "app", "order": 0},
|
| 50 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
}
|
| 52 |
|
| 53 |
|
|
@@ -57,7 +74,130 @@ class ApplicationError(ValueError):
|
|
| 57 |
|
| 58 |
def template_contracts():
|
| 59 |
"""The trusted renderer contracts offered by the Application creation flow."""
|
| 60 |
-
return [copy.deepcopy(_TEMPLATE_CONTRACTS[kind]) for kind in
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
|
| 62 |
|
| 63 |
def blank_doc():
|
|
@@ -205,6 +345,24 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract
|
|
| 205 |
template_kind = str(payload.get("templateKind") or "").strip().lower()
|
| 206 |
if template_kind != kind:
|
| 207 |
raise ApplicationError(f"{kind} applications require templateKind {kind!r}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 208 |
source = str(payload.get("sourceDatabase") or "").strip()[:120]
|
| 209 |
if not source:
|
| 210 |
raise ApplicationError("sourceDatabase is required")
|
|
@@ -305,6 +463,13 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
|
|
| 305 |
"default":bool(is_admin)}
|
| 306 |
row["starred"] = user in (app.get("starredBy") or ())
|
| 307 |
row["isDefault"] = bool(is_default)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
|
| 309 |
row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
|
| 310 |
"database":target_database}
|
|
@@ -318,7 +483,8 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 318 |
for app in doc["applications"].values():
|
| 319 |
if not may_read(app,user,is_admin): continue
|
| 320 |
if database and database not in (app.get("compatibleDatabases") or ()): continue
|
| 321 |
-
if
|
|
|
|
| 322 |
if kind and app.get("type") != kind: continue
|
| 323 |
if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
|
| 324 |
default_key = f"{database}::{app.get('type')}" if database else ""
|
|
@@ -328,6 +494,22 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 328 |
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
|
| 329 |
|
| 330 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
def create(runtime, tenant, user, payload, *, semantic_records=None, field_contracts=None,
|
| 332 |
allowed_databases=None):
|
| 333 |
payload=payload if isinstance(payload,dict) else {}
|
|
@@ -339,8 +521,9 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, field_contr
|
|
| 339 |
allowed_databases=allowed_databases)
|
| 340 |
name=" ".join(str(payload.get("name") or "").split())[:120]
|
| 341 |
if not name: raise ApplicationError("application name is required")
|
| 342 |
-
app_id=f"app_{
|
| 343 |
-
compatible = ([
|
|
|
|
| 344 |
else _compat(payload.get("compatibleDatabases")))
|
| 345 |
if _allowed(compatible, allowed_databases) != set(compatible):
|
| 346 |
raise PermissionError("an Application cannot bind a database you cannot open")
|
|
@@ -368,7 +551,8 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None,
|
|
| 368 |
allowed_databases=None, target_database=None):
|
| 369 |
patch=patch if isinstance(patch,dict) else {}; landed={}
|
| 370 |
immutable = sorted(set(patch) & {
|
| 371 |
-
"id", "tenant", "type", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"
|
|
|
|
| 372 |
})
|
| 373 |
if immutable:
|
| 374 |
raise ApplicationError("Application identity and template definition are immutable: "
|
|
@@ -409,7 +593,7 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None,
|
|
| 409 |
stars=set(app.get("starredBy") or ())
|
| 410 |
(stars.add if patch["starred"] else stars.discard)(user); app["starredBy"]=sorted(stars)
|
| 411 |
app["revision"]=int(app.get("revision") or 0)+1
|
| 412 |
-
app["updatedAt"]=
|
| 413 |
return doc
|
| 414 |
runtime.update(_KEY,apply)
|
| 415 |
return _public(landed,user,is_admin,allowed_databases=allowed_databases,
|
|
|
|
| 2 |
from __future__ import annotations
|
| 3 |
|
| 4 |
import copy
|
| 5 |
+
from core import write_execution
|
| 6 |
+
|
| 7 |
+
from modules.finance_apps import (finance_application as _finance_application,
|
| 8 |
+
finance_applications as _finance_applications,
|
| 9 |
+
finance_group as _finance_group)
|
| 10 |
+
from modules.finance_spine import fact_specs as _finance_fact_specs
|
| 11 |
|
| 12 |
|
| 13 |
APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
|
| 14 |
+
# The wizard binds a database and fields. Finance templates use the catalogue POST instead;
|
| 15 |
+
# validation accepts both creation doors without offering Finance to the five-step wizard.
|
| 16 |
+
APPLICATION_WIZARD_TEMPLATE_KINDS = ("route", "catalog")
|
| 17 |
+
FINANCE_TEMPLATE_KIND = "finance"
|
| 18 |
+
APPLICATION_TEMPLATE_KINDS = APPLICATION_WIZARD_TEMPLATE_KINDS + (FINANCE_TEMPLATE_KIND,)
|
| 19 |
_KEY = "applications_v1"
|
| 20 |
_MAX = 500
|
| 21 |
|
|
|
|
| 56 |
],
|
| 57 |
"defaultPlacement": {"section": "app", "order": 0},
|
| 58 |
},
|
| 59 |
+
"finance": {
|
| 60 |
+
"kind": "finance",
|
| 61 |
+
"label": "Finance",
|
| 62 |
+
# Like Catalog, templateKind selects a trusted surface without growing APPLICATION_TYPES.
|
| 63 |
+
"applicationType": "chart",
|
| 64 |
+
"displayMode": "chart",
|
| 65 |
+
"fieldRoles": [],
|
| 66 |
+
"defaultPlacement": {"section": "app", "order": 0},
|
| 67 |
+
},
|
| 68 |
}
|
| 69 |
|
| 70 |
|
|
|
|
| 74 |
|
| 75 |
def template_contracts():
|
| 76 |
"""The trusted renderer contracts offered by the Application creation flow."""
|
| 77 |
+
return [copy.deepcopy(_TEMPLATE_CONTRACTS[kind]) for kind in APPLICATION_WIZARD_TEMPLATE_KINDS]
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def finance_group():
|
| 81 |
+
"""The one immutable Finance group consumed by navigation and direct-link routes."""
|
| 82 |
+
return _finance_group()
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def finance_applications(may_open=None):
|
| 86 |
+
"""Six separately addressable Finance application descriptors."""
|
| 87 |
+
return _finance_applications(may_open=may_open)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# R6: Finance records are read-only projections of finance_apps.py, never validated, created,
|
| 91 |
+
# mutated, or persisted. Adding "finance" to APPLICATION_TYPES buys nothing and weakens
|
| 92 |
+
# validate_application's governed type vocabulary.
|
| 93 |
+
# This remains true for the SEEDED six. Users may now persist their own Finance applications
|
| 94 |
+
# through templateKind, pointing to the same trusted surface without modifying those projections.
|
| 95 |
+
def finance_application_records(may_open=None):
|
| 96 |
+
"""Project the grant-filtered Finance catalogue as seeded Application records."""
|
| 97 |
+
return [
|
| 98 |
+
{"id": f"finance_{app['key']}",
|
| 99 |
+
"key": app["key"],
|
| 100 |
+
"name": app["navLabel"],
|
| 101 |
+
"type": "finance",
|
| 102 |
+
"templateKind": None,
|
| 103 |
+
"origin": "finance",
|
| 104 |
+
"permissionKey": app["permissionKey"],
|
| 105 |
+
"mode": app["mode"],
|
| 106 |
+
"question": app["question"],
|
| 107 |
+
"deepLink": app["deepLink"],
|
| 108 |
+
"compatibleDatabases": [],
|
| 109 |
+
"navPlacement": {"section": "finance", "order": index},
|
| 110 |
+
"openTarget": {"kind": "finance", "id": f"finance_{app['key']}", "type": "finance",
|
| 111 |
+
"database": None},
|
| 112 |
+
"capabilities": {"read": True, "edit": False, "share": False, "delete": False,
|
| 113 |
+
"duplicate": False, "star": False, "default": False},
|
| 114 |
+
"starred": False,
|
| 115 |
+
"isDefault": False}
|
| 116 |
+
for index, app in enumerate(finance_applications(may_open=may_open))
|
| 117 |
+
]
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
_FINANCE_SCOPES = {"business_unit", "hq_consolidated", "all_channel_inventory"}
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _require_unavailable(value, context):
|
| 124 |
+
if not isinstance(value, dict) or value.get("state") != "unavailable":
|
| 125 |
+
raise RuntimeError(f"{context} must be explicitly unavailable")
|
| 126 |
+
if "value" in value or not value.get("missingSource"):
|
| 127 |
+
raise RuntimeError(f"{context} cannot fabricate a missing value")
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def validate_finance_application_contract(app):
|
| 131 |
+
"""Fail closed when an app bypasses its declared workflow/dashboard or fact boundary."""
|
| 132 |
+
key = str(app.get("key") or "unknown")
|
| 133 |
+
mode = app.get("mode")
|
| 134 |
+
worklist = app.get("worklist")
|
| 135 |
+
dashboard = app.get("dashboard")
|
| 136 |
+
fact_keys = set(app.get("factKeys") or ())
|
| 137 |
+
if mode not in {"workflow", "dashboard"}:
|
| 138 |
+
raise RuntimeError(f"invalid Finance application mode for {key}")
|
| 139 |
+
if mode == "workflow":
|
| 140 |
+
if not isinstance(worklist, dict) or dashboard:
|
| 141 |
+
raise RuntimeError(f"invalid Finance worklist contract for {key}")
|
| 142 |
+
source_fact = worklist.get("sourceFact")
|
| 143 |
+
if source_fact:
|
| 144 |
+
if source_fact not in fact_keys:
|
| 145 |
+
raise RuntimeError(f"invalid Finance worklist source for {key}")
|
| 146 |
+
required = ("ownerField", "statusField", "dueDateField", "nextAction")
|
| 147 |
+
if not all(worklist.get(name) for name in required):
|
| 148 |
+
raise RuntimeError(f"incomplete Finance worklist fields for {key}")
|
| 149 |
+
elif worklist.get("kind") == "unavailable_requirements":
|
| 150 |
+
signals = worklist.get("signals")
|
| 151 |
+
if not isinstance(signals, list) or not signals:
|
| 152 |
+
raise RuntimeError(f"Finance requirements worklist {key} has no signals")
|
| 153 |
+
for signal in signals:
|
| 154 |
+
_require_unavailable(signal, f"Finance work signal in {key}")
|
| 155 |
+
if signal.get("scope") not in _FINANCE_SCOPES:
|
| 156 |
+
raise RuntimeError(f"Finance work signal in {key} has no governed scope")
|
| 157 |
+
if not all(signal.get(name) for name in (
|
| 158 |
+
"key", "label", "basisLabel", "sourceModule", "validationMethod",
|
| 159 |
+
"ownerRole", "status", "nextAction")):
|
| 160 |
+
raise RuntimeError(f"Finance work signal in {key} is ungrounded")
|
| 161 |
+
if signal.get("status") != "unavailable":
|
| 162 |
+
raise RuntimeError(f"Finance work signal in {key} claims executable state")
|
| 163 |
+
else:
|
| 164 |
+
raise RuntimeError(f"Finance worklist {key} has no governed source")
|
| 165 |
+
elif worklist or not isinstance(dashboard, dict):
|
| 166 |
+
raise RuntimeError(f"invalid Finance dashboard contract for {key}")
|
| 167 |
+
|
| 168 |
+
for section in (dashboard or {}).get("sections", []):
|
| 169 |
+
for metric in section.get("metrics", []):
|
| 170 |
+
if metric.get("sourceFact") and metric["sourceFact"] not in fact_keys:
|
| 171 |
+
raise RuntimeError(f"dashboard {key} references an ungoverned source fact")
|
| 172 |
+
if metric.get("state") == "unavailable":
|
| 173 |
+
_require_unavailable(metric, f"dashboard metric in {key}")
|
| 174 |
+
if metric.get("scope") not in _FINANCE_SCOPES:
|
| 175 |
+
raise RuntimeError(f"dashboard metric in {key} has no governed scope")
|
| 176 |
+
if not metric.get("basisLabel") or not metric.get("validationMethod"):
|
| 177 |
+
raise RuntimeError(f"dashboard metric in {key} is ungrounded")
|
| 178 |
+
for row in section.get("rows", []):
|
| 179 |
+
for column in ("actual", "budget", "prior"):
|
| 180 |
+
cell = row.get(column) or {}
|
| 181 |
+
if cell.get("state") == "fact" and cell.get("factKey") not in fact_keys:
|
| 182 |
+
raise RuntimeError(f"dashboard {key} references an ungoverned fact")
|
| 183 |
+
for row in section.get("planningRows", []):
|
| 184 |
+
if row.get("scope") not in _FINANCE_SCOPES or not row.get("basisLabel"):
|
| 185 |
+
raise RuntimeError(f"planning row in {key} is ungrounded")
|
| 186 |
+
actual = row.get("actual") or {}
|
| 187 |
+
if actual.get("state") != "fact" or actual.get("factKey") not in fact_keys:
|
| 188 |
+
raise RuntimeError(f"planning row in {key} references an ungoverned actual")
|
| 189 |
+
for column in ("plan", "variance", "driver"):
|
| 190 |
+
_require_unavailable(row.get(column), f"planning {column} in {key}")
|
| 191 |
+
return True
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def finance_application(key, may_open=None):
|
| 195 |
+
"""Resolve one Finance application without constructing a Finance-home payload."""
|
| 196 |
+
app = _finance_application(key, may_open=may_open)
|
| 197 |
+
validate_finance_application_contract(app)
|
| 198 |
+
app["facts"] = _finance_fact_specs(app["factKeys"])
|
| 199 |
+
app["kpis"] = _finance_fact_specs(app["kpiKeys"])
|
| 200 |
+
return app
|
| 201 |
|
| 202 |
|
| 203 |
def blank_doc():
|
|
|
|
| 345 |
template_kind = str(payload.get("templateKind") or "").strip().lower()
|
| 346 |
if template_kind != kind:
|
| 347 |
raise ApplicationError(f"{kind} applications require templateKind {kind!r}")
|
| 348 |
+
if kind == FINANCE_TEMPLATE_KIND:
|
| 349 |
+
finance_key = str(payload.get("financeKey") or "").strip()
|
| 350 |
+
try:
|
| 351 |
+
app = _finance_application(finance_key)
|
| 352 |
+
except KeyError:
|
| 353 |
+
raise ApplicationError(f"Finance application {finance_key!r} does not exist.")
|
| 354 |
+
if payload.get("sourceDatabase") or payload.get("fieldMap") or payload.get("compatibleDatabases"):
|
| 355 |
+
raise ApplicationError("Finance applications do not bind a database or fields.")
|
| 356 |
+
raw_placement = payload.get("navPlacement")
|
| 357 |
+
if not isinstance(raw_placement, dict) or set(raw_placement) != {"section", "order"}:
|
| 358 |
+
raise ApplicationError("navPlacement must contain section and order")
|
| 359 |
+
if raw_placement.get("section") != "app":
|
| 360 |
+
raise ApplicationError("Finance applications must be placed in App.")
|
| 361 |
+
order = raw_placement.get("order")
|
| 362 |
+
if type(order) is not int or not 0 <= order <= 999:
|
| 363 |
+
raise ApplicationError("navPlacement order must be an integer from 0 to 999")
|
| 364 |
+
return {"templateKind": kind, "financeKey": finance_key, "deepLink": app["deepLink"],
|
| 365 |
+
"navPlacement": {"section": "app", "order": order}}
|
| 366 |
source = str(payload.get("sourceDatabase") or "").strip()[:120]
|
| 367 |
if not source:
|
| 368 |
raise ApplicationError("sourceDatabase is required")
|
|
|
|
| 463 |
"default":bool(is_admin)}
|
| 464 |
row["starred"] = user in (app.get("starredBy") or ())
|
| 465 |
row["isDefault"] = bool(is_default)
|
| 466 |
+
if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
|
| 467 |
+
# The database-bound mutation doors cannot edit Finance definitions. Honour that in
|
| 468 |
+
# every card rather than displaying controls that can only refuse.
|
| 469 |
+
row["capabilities"].update({key: False for key in (
|
| 470 |
+
"edit", "share", "delete", "duplicate", "star", "default")})
|
| 471 |
+
row["starred"] = False
|
| 472 |
+
row["isDefault"] = False
|
| 473 |
target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
|
| 474 |
row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
|
| 475 |
"database":target_database}
|
|
|
|
| 483 |
for app in doc["applications"].values():
|
| 484 |
if not may_read(app,user,is_admin): continue
|
| 485 |
if database and database not in (app.get("compatibleDatabases") or ()): continue
|
| 486 |
+
if (app.get("templateKind") != FINANCE_TEMPLATE_KIND
|
| 487 |
+
and not _allowed(app.get("compatibleDatabases"), allowed_databases)): continue
|
| 488 |
if kind and app.get("type") != kind: continue
|
| 489 |
if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
|
| 490 |
default_key = f"{database}::{app.get('type')}" if database else ""
|
|
|
|
| 494 |
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
|
| 495 |
|
| 496 |
|
| 497 |
+
def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_databases=None):
|
| 498 |
+
"""The ONE generic navigation read for stored and seeded Application records (R6)."""
|
| 499 |
+
rows = [{"id": app["id"], "name": app.get("name"),
|
| 500 |
+
"type": app.get("type"), "templateKind": app.get("templateKind"),
|
| 501 |
+
"deepLink": app.get("deepLink"),
|
| 502 |
+
"origin": "tenant",
|
| 503 |
+
"compatibleDatabases": app.get("compatibleDatabases") or [],
|
| 504 |
+
"openTarget": app.get("openTarget"),
|
| 505 |
+
"navPlacement": app.get("navPlacement") or {"section": "app", "order": index},
|
| 506 |
+
"capabilities": app.get("capabilities"), "starred": app.get("starred"),
|
| 507 |
+
"isDefault": app.get("isDefault")}
|
| 508 |
+
for index, app in enumerate(list_apps(runtime, user, is_admin,
|
| 509 |
+
allowed_databases=allowed_databases))]
|
| 510 |
+
return rows + finance_application_records(may_open=may_open)
|
| 511 |
+
|
| 512 |
+
|
| 513 |
def create(runtime, tenant, user, payload, *, semantic_records=None, field_contracts=None,
|
| 514 |
allowed_databases=None):
|
| 515 |
payload=payload if isinstance(payload,dict) else {}
|
|
|
|
| 521 |
allowed_databases=allowed_databases)
|
| 522 |
name=" ".join(str(payload.get("name") or "").split())[:120]
|
| 523 |
if not name: raise ApplicationError("application name is required")
|
| 524 |
+
app_id=f"app_{write_execution.new_id()[:16]}"; now=write_execution.utcnow().isoformat()
|
| 525 |
+
compatible = ([] if template_kind == FINANCE_TEMPLATE_KIND
|
| 526 |
+
else [definition["sourceDatabase"]] if definition
|
| 527 |
else _compat(payload.get("compatibleDatabases")))
|
| 528 |
if _allowed(compatible, allowed_databases) != set(compatible):
|
| 529 |
raise PermissionError("an Application cannot bind a database you cannot open")
|
|
|
|
| 551 |
allowed_databases=None, target_database=None):
|
| 552 |
patch=patch if isinstance(patch,dict) else {}; landed={}
|
| 553 |
immutable = sorted(set(patch) & {
|
| 554 |
+
"id", "tenant", "type", "templateKind", "sourceDatabase", "fieldMap", "navPlacement",
|
| 555 |
+
"financeKey", "deepLink"
|
| 556 |
})
|
| 557 |
if immutable:
|
| 558 |
raise ApplicationError("Application identity and template definition are immutable: "
|
|
|
|
| 593 |
stars=set(app.get("starredBy") or ())
|
| 594 |
(stars.add if patch["starred"] else stars.discard)(user); app["starredBy"]=sorted(stars)
|
| 595 |
app["revision"]=int(app.get("revision") or 0)+1
|
| 596 |
+
app["updatedAt"]=write_execution.utcnow().isoformat(); landed.update(app)
|
| 597 |
return doc
|
| 598 |
runtime.update(_KEY,apply)
|
| 599 |
return _public(landed,user,is_admin,allowed_databases=allowed_databases,
|
api/automation_engine.py
CHANGED
|
@@ -34,7 +34,7 @@ import re
|
|
| 34 |
import socket
|
| 35 |
import threading
|
| 36 |
import time
|
| 37 |
-
from urllib.parse import urljoin, urlparse
|
| 38 |
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
| 39 |
|
| 40 |
import requests
|
|
@@ -61,6 +61,25 @@ STORE_KEY = "automations"
|
|
| 61 |
AGENT_CONTRACT_KEY = "agent_contracts"
|
| 62 |
AGENT_EVENT_TYPES = ("transcript", "message", "gathered_data", "run", "consent")
|
| 63 |
AGENT_ACTIONS = ("read", "create", "update", "overwrite", "delete")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
#: ⭐⭐ W47-T21 / R3 — THE THREE NAMED REASONS AN AGENT'S ROW FILTER REFUSES WITH.
|
| 65 |
#:
|
| 66 |
#: R3, verbatim: an agent's row filter is *"its whole world: it may neither read nor write a
|
|
@@ -2218,13 +2237,18 @@ def clean_definition(raw, previous=None, username="", notes=None, rt=None):
|
|
| 2218 |
"""Whole-definition validation. Returns `(defn, error)`."""
|
| 2219 |
raw = raw if isinstance(raw, dict) else {}
|
| 2220 |
prev = previous or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2221 |
# ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing
|
| 2222 |
# that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second
|
| 2223 |
# reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys
|
| 2224 |
# exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload
|
| 2225 |
# invalid in BOTH its trigger and its config now answers with the trigger's sentence.
|
| 2226 |
-
|
| 2227 |
-
|
| 2228 |
if terr:
|
| 2229 |
return None, terr
|
| 2230 |
# C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this
|
|
@@ -2619,6 +2643,56 @@ def retire_automation_board_state(rt, tables=None):
|
|
| 2619 |
return {"definitions": len(plan), **fields}
|
| 2620 |
|
| 2621 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2622 |
#: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe.
|
| 2623 |
#: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which
|
| 2624 |
#: exactly one caller uses (`grid_hook`).
|
|
@@ -2696,7 +2770,8 @@ def all_definitions(rt, cached=False):
|
|
| 2696 |
raw = dict(rt.get(STORE_KEY) or {})
|
| 2697 |
except Exception:
|
| 2698 |
return {}
|
| 2699 |
-
out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()
|
|
|
|
| 2700 |
if cached:
|
| 2701 |
_DEFS_MEMO[str(getattr(rt, "key", "") or "")] = (time.monotonic(), out)
|
| 2702 |
return out
|
|
@@ -2813,7 +2888,11 @@ def agent_contract_dto(rt, agent_id):
|
|
| 2813 |
|
| 2814 |
|
| 2815 |
def authorize_agent_principal(rt, agent_id, actor, is_admin=False):
|
| 2816 |
-
rec = _agent_contracts(rt).get(str(agent_id or ""))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2817 |
sharing = rec.get("sharing") or {}
|
| 2818 |
if is_admin or rec.get("createdBy") == str(actor or "") or str(actor or "") in set(sharing.get("sharedWith") or []):
|
| 2819 |
return True
|
|
@@ -3353,7 +3432,7 @@ def patch(rt, auto_id, raw, username="", notes=None):
|
|
| 3353 |
# DISCARDED — the client shows a saved flow that the store never received, and nothing goes
|
| 3354 |
# red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only
|
| 3355 |
# ordering that never has a window where the bug exists.
|
| 3356 |
-
for k in ("name", "config", "schedule", "kind", "trigger", "flow"):
|
| 3357 |
if k in (raw or {}):
|
| 3358 |
merged[k] = raw[k]
|
| 3359 |
# 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the
|
|
@@ -9958,8 +10037,6 @@ ACTION_CATALOG = [
|
|
| 9958 |
"ready": True, "menu": False,
|
| 9959 |
"detail": "Fill an AI column for the records this flow walks. Configured on the column, not "
|
| 9960 |
"here"},
|
| 9961 |
-
{"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
|
| 9962 |
-
"detail": "Needs the Slack connector"},
|
| 9963 |
# ── 4. ADVANCED LOGIC ────────────────────────────────────────────────────────────────────
|
| 9964 |
# `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism,
|
| 9965 |
# and R8 made it a FORK with lettered branches, which is a thing people already have a name
|
|
@@ -15492,7 +15569,10 @@ def _row_gate(rt, defn, trg):
|
|
| 15492 |
"""
|
| 15493 |
key = trg.get("key")
|
| 15494 |
if key == "enters_view":
|
| 15495 |
-
|
|
|
|
|
|
|
|
|
|
| 15496 |
if problem:
|
| 15497 |
return None, problem
|
| 15498 |
import harness.filter_eval as filter_eval
|
|
@@ -15527,7 +15607,7 @@ def _settle_eval(rt, tenant, auto_id, log=print):
|
|
| 15527 |
return
|
| 15528 |
fired = []
|
| 15529 |
if gate is not None:
|
| 15530 |
-
rows = (ut_get(rt, trg.get("table") or
|
| 15531 |
disarmed = set((d.get("state") or {}).get("eventDisarmed") or [])
|
| 15532 |
nxt = set(disarmed)
|
| 15533 |
for rid in touched:
|
|
@@ -15572,7 +15652,7 @@ def _seed_event_state(rt, defn):
|
|
| 15572 |
gate, problem = _row_gate(rt, defn, trg)
|
| 15573 |
if gate is None or problem:
|
| 15574 |
return # no match gate ⇒ nothing to arm; a broken gate seeds nothing
|
| 15575 |
-
rows = (ut_get(rt, trg.get("table") or
|
| 15576 |
matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {}))
|
| 15577 |
set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]})
|
| 15578 |
|
|
@@ -15600,10 +15680,11 @@ def grid_hook(evt):
|
|
| 15600 |
for aid, d in defs.items():
|
| 15601 |
trg = (d or {}).get("trigger") or {}
|
| 15602 |
key = str(trg.get("key") or "")
|
|
|
|
| 15603 |
if trg.get("paused") or not trg.get("enabled", True) \
|
| 15604 |
or not trg.get("configured", True) \
|
| 15605 |
or key not in TRIGGER_ROW_KEYS \
|
| 15606 |
-
or
|
| 15607 |
continue
|
| 15608 |
if kind == "record_created" and key == "record_created":
|
| 15609 |
hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0
|
|
@@ -16060,7 +16141,9 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None):
|
|
| 16060 |
# inside `dispatch_authorized_slack_automation` before its reply sentence is composed;
|
| 16061 |
# notification trouble must never rewrite the truthful outcome of work that already ran.
|
| 16062 |
try:
|
| 16063 |
-
dispatch_authorized_slack_automation(
|
|
|
|
|
|
|
| 16064 |
except Exception as e: # noqa: BLE001
|
| 16065 |
log(f"[aios-auto] Slack notification for {auto_id} failed: {type(e).__name__}: {e}")
|
| 16066 |
return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected,
|
|
|
|
| 34 |
import socket
|
| 35 |
import threading
|
| 36 |
import time
|
| 37 |
+
from urllib.parse import quote, urljoin, urlparse
|
| 38 |
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
| 39 |
|
| 40 |
import requests
|
|
|
|
| 61 |
AGENT_CONTRACT_KEY = "agent_contracts"
|
| 62 |
AGENT_EVENT_TYPES = ("transcript", "message", "gathered_data", "run", "consent")
|
| 63 |
AGENT_ACTIONS = ("read", "create", "update", "overwrite", "delete")
|
| 64 |
+
|
| 65 |
+
#: MIGRATION NOTE (SDLC 64 / R3, 2026-09-21) — the only surviving occurrence of the
|
| 66 |
+
#: removed preset name. D-295 records why this must remain: a person's stored automation
|
| 67 |
+
#: must be dropped with a named reason, not refused on every save forever. Every read
|
| 68 |
+
#: drops these definitions; `retire_preset_automations` also sweeps them from the store.
|
| 69 |
+
RETIRED_PRESETS = {
|
| 70 |
+
"customer_intelligence": (
|
| 71 |
+
"Customer intelligence has been removed from Loopable. This automation's trigger and "
|
| 72 |
+
"its insight action no longer exist, so it can no longer run and has been removed."),
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def retired_preset_reason(definition):
|
| 77 |
+
"""The sentence to show for a stored definition whose preset is gone, else ``""``."""
|
| 78 |
+
if not isinstance(definition, dict):
|
| 79 |
+
return ""
|
| 80 |
+
return RETIRED_PRESETS.get(str(definition.get("preset") or "").strip(), "")
|
| 81 |
+
|
| 82 |
+
|
| 83 |
#: ⭐⭐ W47-T21 / R3 — THE THREE NAMED REASONS AN AGENT'S ROW FILTER REFUSES WITH.
|
| 84 |
#:
|
| 85 |
#: R3, verbatim: an agent's row filter is *"its whole world: it may neither read nor write a
|
|
|
|
| 2237 |
"""Whole-definition validation. Returns `(defn, error)`."""
|
| 2238 |
raw = raw if isinstance(raw, dict) else {}
|
| 2239 |
prev = previous or {}
|
| 2240 |
+
preset = _s(raw.get("preset") if "preset" in raw else prev.get("preset"), 60).strip()
|
| 2241 |
+
if preset in RETIRED_PRESETS:
|
| 2242 |
+
return None, RETIRED_PRESETS[preset]
|
| 2243 |
+
if preset:
|
| 2244 |
+
return None, f"unknown automation preset {preset!r}"
|
| 2245 |
# ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing
|
| 2246 |
# that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second
|
| 2247 |
# reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys
|
| 2248 |
# exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload
|
| 2249 |
# invalid in BOTH its trigger and its config now answers with the trigger's sentence.
|
| 2250 |
+
trigger_raw = raw.get("trigger") if "trigger" in raw else prev.get("trigger")
|
| 2251 |
+
trigger, terr = clean_trigger(trigger_raw, prev.get("trigger"))
|
| 2252 |
if terr:
|
| 2253 |
return None, terr
|
| 2254 |
# C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this
|
|
|
|
| 2643 |
return {"definitions": len(plan), **fields}
|
| 2644 |
|
| 2645 |
|
| 2646 |
+
def _no_preset_retirement():
|
| 2647 |
+
"""The receipt when nothing was retired: the SAME keys the success path answers with.
|
| 2648 |
+
|
| 2649 |
+
A FUNCTION rather than a module constant, because the receipt is handed to a caller and a
|
| 2650 |
+
shared literal would let one caller's edit reach the next one.
|
| 2651 |
+
"""
|
| 2652 |
+
return {"definitions": 0, "ids": [], "reasons": {}, "removed": []}
|
| 2653 |
+
|
| 2654 |
+
|
| 2655 |
+
def retire_preset_automations(rt):
|
| 2656 |
+
"""Persist retired-definition removal and name each removed automation in the log.
|
| 2657 |
+
|
| 2658 |
+
⭐⭐ THE RECEIPT CARRIES `removed` SO THE CALLER NEEDS NO SECOND READ. `list_automations` has
|
| 2659 |
+
to tell the reader WHICH automation went and why, and the obvious way to get the names — read
|
| 2660 |
+
the bucket again beside the sweep — is the defect the WAVE 30 · T12 comment block above that
|
| 2661 |
+
call site is the tombstone of: a whole-tenant document re-serialised under the store lock on
|
| 2662 |
+
every single request, to report something that is empty on all but one of them. This function
|
| 2663 |
+
has already read every row it removed, so it hands them back.
|
| 2664 |
+
"""
|
| 2665 |
+
try:
|
| 2666 |
+
stored = dict(rt.get(STORE_KEY) or {})
|
| 2667 |
+
except Exception:
|
| 2668 |
+
return _no_preset_retirement()
|
| 2669 |
+
plan = {aid for aid, definition in stored.items() if retired_preset_reason(definition)}
|
| 2670 |
+
if not plan:
|
| 2671 |
+
return _no_preset_retirement()
|
| 2672 |
+
removed = {}
|
| 2673 |
+
|
| 2674 |
+
def _up(cur):
|
| 2675 |
+
cur = cur if isinstance(cur, dict) else {}
|
| 2676 |
+
# A store retry replays this callback. Report only the successful attempt, and
|
| 2677 |
+
# recheck each row so an intervening edit to a live definition is preserved.
|
| 2678 |
+
removed.clear()
|
| 2679 |
+
for aid in sorted(plan):
|
| 2680 |
+
definition = cur.get(aid)
|
| 2681 |
+
reason = retired_preset_reason(definition)
|
| 2682 |
+
if reason:
|
| 2683 |
+
removed[str(aid)] = (str(definition.get("name") or "Unnamed automation"), reason)
|
| 2684 |
+
cur.pop(aid)
|
| 2685 |
+
return cur
|
| 2686 |
+
|
| 2687 |
+
_store_update(rt, _up, flush="sync")
|
| 2688 |
+
for aid, (name, reason) in removed.items():
|
| 2689 |
+
print(f"[aios-auto] removed automation {aid!r} ({name!r}): {reason}")
|
| 2690 |
+
return {"definitions": len(removed), "ids": list(removed),
|
| 2691 |
+
"reasons": {aid: reason for aid, (_name, reason) in removed.items()},
|
| 2692 |
+
"removed": [{"id": aid, "name": name, "reason": reason}
|
| 2693 |
+
for aid, (name, reason) in removed.items()]}
|
| 2694 |
+
|
| 2695 |
+
|
| 2696 |
#: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe.
|
| 2697 |
#: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which
|
| 2698 |
#: exactly one caller uses (`grid_hook`).
|
|
|
|
| 2770 |
raw = dict(rt.get(STORE_KEY) or {})
|
| 2771 |
except Exception:
|
| 2772 |
return {}
|
| 2773 |
+
out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()
|
| 2774 |
+
if not retired_preset_reason(defn)}
|
| 2775 |
if cached:
|
| 2776 |
_DEFS_MEMO[str(getattr(rt, "key", "") or "")] = (time.monotonic(), out)
|
| 2777 |
return out
|
|
|
|
| 2888 |
|
| 2889 |
|
| 2890 |
def authorize_agent_principal(rt, agent_id, actor, is_admin=False):
|
| 2891 |
+
rec = _agent_contracts(rt).get(str(agent_id or ""))
|
| 2892 |
+
# An admin may read a live private Agent; that role cannot resurrect a deleted Agent
|
| 2893 |
+
# or replace the user identity required for authorization.
|
| 2894 |
+
if not str(actor or "").strip() or not isinstance(rec, dict) or not rec:
|
| 2895 |
+
raise PermissionError("this Agent or its principal is no longer available")
|
| 2896 |
sharing = rec.get("sharing") or {}
|
| 2897 |
if is_admin or rec.get("createdBy") == str(actor or "") or str(actor or "") in set(sharing.get("sharedWith") or []):
|
| 2898 |
return True
|
|
|
|
| 3432 |
# DISCARDED — the client shows a saved flow that the store never received, and nothing goes
|
| 3433 |
# red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only
|
| 3434 |
# ordering that never has a window where the bug exists.
|
| 3435 |
+
for k in ("name", "config", "schedule", "kind", "trigger", "flow", "preset"):
|
| 3436 |
if k in (raw or {}):
|
| 3437 |
merged[k] = raw[k]
|
| 3438 |
# 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the
|
|
|
|
| 10037 |
"ready": True, "menu": False,
|
| 10038 |
"detail": "Fill an AI column for the records this flow walks. Configured on the column, not "
|
| 10039 |
"here"},
|
|
|
|
|
|
|
| 10040 |
# ── 4. ADVANCED LOGIC ────────────────────────────────────────────────────────────────────
|
| 10041 |
# `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism,
|
| 10042 |
# and R8 made it a FORK with lettered branches, which is a thing people already have a name
|
|
|
|
| 15569 |
"""
|
| 15570 |
key = trg.get("key")
|
| 15571 |
if key == "enters_view":
|
| 15572 |
+
# Native View intelligence keeps the C2 trigger exact (`kind` + `viewId`); its table is
|
| 15573 |
+
# the definition's source binding rather than a second, client-editable trigger field.
|
| 15574 |
+
tree, fields, problem = view_filter(
|
| 15575 |
+
rt, trg.get("table") or _flow_table(defn), trg.get("viewId"))
|
| 15576 |
if problem:
|
| 15577 |
return None, problem
|
| 15578 |
import harness.filter_eval as filter_eval
|
|
|
|
| 15607 |
return
|
| 15608 |
fired = []
|
| 15609 |
if gate is not None:
|
| 15610 |
+
rows = (ut_get(rt, trg.get("table") or _flow_table(d)) or {}).get("rows") or {}
|
| 15611 |
disarmed = set((d.get("state") or {}).get("eventDisarmed") or [])
|
| 15612 |
nxt = set(disarmed)
|
| 15613 |
for rid in touched:
|
|
|
|
| 15652 |
gate, problem = _row_gate(rt, defn, trg)
|
| 15653 |
if gate is None or problem:
|
| 15654 |
return # no match gate ⇒ nothing to arm; a broken gate seeds nothing
|
| 15655 |
+
rows = (ut_get(rt, trg.get("table") or _flow_table(defn)) or {}).get("rows") or {}
|
| 15656 |
matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {}))
|
| 15657 |
set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]})
|
| 15658 |
|
|
|
|
| 15680 |
for aid, d in defs.items():
|
| 15681 |
trg = (d or {}).get("trigger") or {}
|
| 15682 |
key = str(trg.get("key") or "")
|
| 15683 |
+
source_table = str(trg.get("table") or _flow_table(d) or "")
|
| 15684 |
if trg.get("paused") or not trg.get("enabled", True) \
|
| 15685 |
or not trg.get("configured", True) \
|
| 15686 |
or key not in TRIGGER_ROW_KEYS \
|
| 15687 |
+
or source_table != table:
|
| 15688 |
continue
|
| 15689 |
if kind == "record_created" and key == "record_created":
|
| 15690 |
hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0
|
|
|
|
| 16141 |
# inside `dispatch_authorized_slack_automation` before its reply sentence is composed;
|
| 16142 |
# notification trouble must never rewrite the truthful outcome of work that already ran.
|
| 16143 |
try:
|
| 16144 |
+
dispatch_authorized_slack_automation(
|
| 16145 |
+
rt, defn, {"state": state, "summary": summary,
|
| 16146 |
+
"rowIds": affected or []})
|
| 16147 |
except Exception as e: # noqa: BLE001
|
| 16148 |
log(f"[aios-auto] Slack notification for {auto_id} failed: {type(e).__name__}: {e}")
|
| 16149 |
return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected,
|
api/connectors_ig.py
CHANGED
|
@@ -1652,3 +1652,33 @@ def pull_profile(url, max_posts=None, log=print,
|
|
| 1652 |
"no provider could read this profile "
|
| 1653 |
f"({'; '.join(attempts) or 'blocked'})")}
|
| 1654 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1652 |
"no provider could read this profile "
|
| 1653 |
f"({'; '.join(attempts) or 'blocked'})")}
|
| 1654 |
|
| 1655 |
+
|
| 1656 |
+
def pull_ig_recipe(seed, limit, post_type="video", log=print):
|
| 1657 |
+
"""Run one saved Instagram database recipe through the established paid runner.
|
| 1658 |
+
|
| 1659 |
+
This deliberately stays a very small adapter over :func:`pull_profile`. The database route
|
| 1660 |
+
owns persistence and the explicit ``Run now`` boundary; this module owns the vendor's native
|
| 1661 |
+
last-N/type vocabulary. Passing ``post_groups`` is load-bearing: it reaches the existing
|
| 1662 |
+
reels-only specialization (Apify when configured, otherwise Bright Data's native
|
| 1663 |
+
``post_type: reel`` request) instead of buying a mixed window and filtering it afterwards.
|
| 1664 |
+
"""
|
| 1665 |
+
try:
|
| 1666 |
+
n = int(limit)
|
| 1667 |
+
except (TypeError, ValueError):
|
| 1668 |
+
return {"state": "error", "profile": {}, "posts": [], "comments": [],
|
| 1669 |
+
"via": "", "note": "the post limit must be a whole number"}
|
| 1670 |
+
kind = str(post_type or "").strip().lower()
|
| 1671 |
+
if kind not in ("video", "image", "carousel"):
|
| 1672 |
+
return {"state": "error", "profile": {}, "posts": [], "comments": [],
|
| 1673 |
+
"via": "", "note": "choose video, image, or carousel posts"}
|
| 1674 |
+
if n < 1 or n > MAX_POST_WINDOW:
|
| 1675 |
+
return {"state": "error", "profile": {}, "posts": [], "comments": [],
|
| 1676 |
+
"via": "", "note": f"the post limit must be between 1 and {MAX_POST_WINDOW}"}
|
| 1677 |
+
return pull_profile(
|
| 1678 |
+
seed,
|
| 1679 |
+
max_posts=n,
|
| 1680 |
+
post_metrics=True,
|
| 1681 |
+
comment_metrics=False,
|
| 1682 |
+
post_groups=[{"type": kind, "limit": n}],
|
| 1683 |
+
log=log,
|
| 1684 |
+
)
|
api/connectors_tt.py
CHANGED
|
@@ -337,7 +337,7 @@ def tt_handle(url):
|
|
| 337 |
return s if s and all(c.isalnum() or c in "._" for c in s) else ""
|
| 338 |
|
| 339 |
|
| 340 |
-
def tt_post_urls(node, limit=0):
|
| 341 |
"""⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
|
| 342 |
|
| 343 |
⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
|
|
@@ -363,6 +363,7 @@ def tt_post_urls(node, limit=0):
|
|
| 363 |
not a number this function may invent.
|
| 364 |
"""
|
| 365 |
raw = (node or {}).get("top_videos")
|
|
|
|
| 366 |
out = []
|
| 367 |
for item in raw if isinstance(raw, list) else []:
|
| 368 |
# ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
|
|
@@ -389,8 +390,17 @@ def tt_post_urls(node, limit=0):
|
|
| 389 |
# whichever array happened to be longer is how one creator's window silently differs from
|
| 390 |
# another's, and that reasoning is unchanged by this correction.
|
| 391 |
if isinstance(item, dict):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 392 |
u = str(item.get("video_url") or item.get("url") or "").strip()
|
| 393 |
else:
|
|
|
|
|
|
|
| 394 |
u = str(item or "").strip()
|
| 395 |
if u.lower().startswith("http") and u not in out:
|
| 396 |
out.append(u)
|
|
@@ -458,7 +468,8 @@ def _media_deferrals(deferred):
|
|
| 458 |
|
| 459 |
|
| 460 |
def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
|
| 461 |
-
max_posts=0, post_metrics=False, comment_metrics=False
|
|
|
|
| 462 |
"""ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
|
| 463 |
|
| 464 |
`{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
|
|
@@ -529,7 +540,7 @@ def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
|
|
| 529 |
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 530 |
"via": "brightdata",
|
| 531 |
"note": note or "profile read; post capture is off for this step"}
|
| 532 |
-
urls = tt_post_urls(node, limit=max_posts)
|
| 533 |
if not urls:
|
| 534 |
# ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
|
| 535 |
# saying so is what stops the next run paying to be told the same thing.
|
|
@@ -566,3 +577,32 @@ def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
|
|
| 566 |
return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
|
| 567 |
"via": "brightdata", "deferredMedia": deferred_media,
|
| 568 |
"note": c_note or note or ""}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
return s if s and all(c.isalnum() or c in "._" for c in s) else ""
|
| 338 |
|
| 339 |
|
| 340 |
+
def tt_post_urls(node, limit=0, post_types=None):
|
| 341 |
"""⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
|
| 342 |
|
| 343 |
⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
|
|
|
|
| 363 |
not a number this function may invent.
|
| 364 |
"""
|
| 365 |
raw = (node or {}).get("top_videos")
|
| 366 |
+
wanted = {str(value or "").strip().lower() for value in (post_types or ()) if str(value or "").strip()}
|
| 367 |
out = []
|
| 368 |
for item in raw if isinstance(raw, list) else []:
|
| 369 |
# ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
|
|
|
|
| 390 |
# whichever array happened to be longer is how one creator's window silently differs from
|
| 391 |
# another's, and that reasoning is unchanged by this correction.
|
| 392 |
if isinstance(item, dict):
|
| 393 |
+
# ``top_videos`` is the vendor's native video window. Keep the type selection at
|
| 394 |
+
# this boundary so an excluded type never reaches the paid Posts dataset. A future
|
| 395 |
+
# vendor row that starts declaring photo-mode entries can use its own ``post_type``;
|
| 396 |
+
# today's measured rows have no such key and are videos by construction.
|
| 397 |
+
delivered_type = str(item.get("post_type") or "video").strip().lower()
|
| 398 |
+
if wanted and delivered_type not in wanted:
|
| 399 |
+
continue
|
| 400 |
u = str(item.get("video_url") or item.get("url") or "").strip()
|
| 401 |
else:
|
| 402 |
+
if wanted and "video" not in wanted:
|
| 403 |
+
continue
|
| 404 |
u = str(item or "").strip()
|
| 405 |
if u.lower().startswith("http") and u not in out:
|
| 406 |
out.append(u)
|
|
|
|
| 468 |
|
| 469 |
|
| 470 |
def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
|
| 471 |
+
max_posts=0, post_metrics=False, comment_metrics=False,
|
| 472 |
+
post_types=None):
|
| 473 |
"""ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
|
| 474 |
|
| 475 |
`{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
|
|
|
|
| 540 |
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 541 |
"via": "brightdata",
|
| 542 |
"note": note or "profile read; post capture is off for this step"}
|
| 543 |
+
urls = tt_post_urls(node, limit=max_posts, post_types=post_types)
|
| 544 |
if not urls:
|
| 545 |
# ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
|
| 546 |
# saying so is what stops the next run paying to be told the same thing.
|
|
|
|
| 577 |
return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
|
| 578 |
"via": "brightdata", "deferredMedia": deferred_media,
|
| 579 |
"note": c_note or note or ""}
|
| 580 |
+
|
| 581 |
+
|
| 582 |
+
def pull_tt_recipe(seed, limit, post_type="video", log=print):
|
| 583 |
+
"""Run one saved TikTok database recipe through the established paid runner.
|
| 584 |
+
|
| 585 |
+
TikTok's measured profile payload exposes a native ``top_videos`` window, so ``limit`` and
|
| 586 |
+
``post_types`` are applied before the paid Posts scrape. Keeping this adapter beside the
|
| 587 |
+
vendor runner makes the route unable to silently regress to an after-the-fact UI filter.
|
| 588 |
+
"""
|
| 589 |
+
try:
|
| 590 |
+
n = int(limit)
|
| 591 |
+
except (TypeError, ValueError):
|
| 592 |
+
return {"state": "error", "profile": {}, "posts": [], "comments": [],
|
| 593 |
+
"via": "", "note": "the post limit must be a whole number"}
|
| 594 |
+
kind = str(post_type or "").strip().lower()
|
| 595 |
+
if kind != "video":
|
| 596 |
+
return {"state": "error", "profile": {}, "posts": [], "comments": [],
|
| 597 |
+
"via": "", "note": "TikTok's native profile window supports video posts"}
|
| 598 |
+
if n < 1 or n > 30:
|
| 599 |
+
return {"state": "error", "profile": {}, "posts": [], "comments": [],
|
| 600 |
+
"via": "", "note": "the post limit must be between 1 and 30"}
|
| 601 |
+
return pull_profile_tt(
|
| 602 |
+
seed,
|
| 603 |
+
max_posts=n,
|
| 604 |
+
post_metrics=True,
|
| 605 |
+
comment_metrics=False,
|
| 606 |
+
post_types=(kind,),
|
| 607 |
+
log=log,
|
| 608 |
+
)
|
api/deps.py
CHANGED
|
@@ -37,6 +37,35 @@ def err(status, code, message):
|
|
| 37 |
return HTTPException(status_code=status, detail={"error": {"code": code, "message": message}})
|
| 38 |
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
@dataclass
|
| 41 |
class Session:
|
| 42 |
"""A verified request identity. `user` is the PUBLIC record (never a hash or a salt)."""
|
|
|
|
| 37 |
return HTTPException(status_code=status, detail={"error": {"code": code, "message": message}})
|
| 38 |
|
| 39 |
|
| 40 |
+
|
| 41 |
+
def running_main():
|
| 42 |
+
"""The `main` module THIS PROCESS IS ACTUALLY RUNNING, or None. Never an `import`.
|
| 43 |
+
|
| 44 |
+
⛔⛔ D-781 — `import main` IN A ROUTE MODULE STARTED A SECOND COPY OF EVERY RESIDENT DAEMON.
|
| 45 |
+
The Space runs `uvicorn api.main:app`, so the module executing is `api.main`. Python keys
|
| 46 |
+
`sys.modules` by the imported NAME, so a plain `import main` from a route file is a DIFFERENT
|
| 47 |
+
module object and **re-executes main.py's whole body** — including its module-level
|
| 48 |
+
`threading.Thread(target=_prewarm, …).start()` and `threading.Thread(target=_store_resync_loop,
|
| 49 |
+
…).start()`. Two resync loops, two prewarms, in one single-worker container.
|
| 50 |
+
|
| 51 |
+
⛔ AND IT COST REAL MONEY. Staging's own log showed the tell twice over: two `prewarm done`
|
| 52 |
+
lines from ONE boot (46.3s and 110.3s, 2026-09-21T11:35:24) and two
|
| 53 |
+
`odoo relational rebuild required`/`done` pairs per resync pass, every pass — double the
|
| 54 |
+
whole-document Neon reads, on a bill whose entire overage was read egress.
|
| 55 |
+
|
| 56 |
+
⛔⛔ AND THE STATUS ROUTE WAS READING THE SHADOW. `_resync_state` pulled `RESYNC_STATE` from the
|
| 57 |
+
duplicate module — a second, separately-populated dict belonging to the second loop — so the
|
| 58 |
+
operator surface answering *"is the mirror advancing"* was describing the copy nobody meant to
|
| 59 |
+
start. That is the worse half: the cost was invisible and the disclosure was wrong.
|
| 60 |
+
|
| 61 |
+
⚠ IT RESOLVES, IT NEVER IMPORTS, and that restraint is the whole fix: falling back to
|
| 62 |
+
`import main` when neither name is loaded would reintroduce exactly the defect. `api.main`
|
| 63 |
+
first (uvicorn), then `main` (the gates, which import it under that name) — and None when this
|
| 64 |
+
process is running neither, which each caller already handles.
|
| 65 |
+
"""
|
| 66 |
+
return sys.modules.get("api.main") or sys.modules.get("main")
|
| 67 |
+
|
| 68 |
+
|
| 69 |
@dataclass
|
| 70 |
class Session:
|
| 71 |
"""A verified request identity. `user` is the PUBLIC record (never a hash or a salt)."""
|
api/formula_runtime.py
CHANGED
|
@@ -84,7 +84,10 @@ FN_ARITY = {
|
|
| 84 |
"OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
|
| 85 |
"FILTER": (2, 3), "SORT": (1, 4),
|
| 86 |
}
|
| 87 |
-
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
#: ⭐ THE TRANSPORT SPELLING OF A DOTTED EXCEL NAME, AND IT IS NOT AN ALIAS.
|
| 90 |
#:
|
|
@@ -1755,6 +1758,26 @@ def _eval_value(node, row, today, diagnostics=None):
|
|
| 1755 |
if isinstance(row, _SheetInputs):
|
| 1756 |
left, right = 0 if left is None else left, 0 if right is None else right
|
| 1757 |
left, right = int(left) if isinstance(left, bool) else left, int(right) if isinstance(right, bool) else right
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1758 |
a, b = _num(left), _num(right)
|
| 1759 |
if a is None or b is None: return None
|
| 1760 |
if node["op"] == "/" and b == 0:
|
|
@@ -3257,9 +3280,17 @@ class SheetCalculation:
|
|
| 3257 |
token = id(node)
|
| 3258 |
if token not in self._volatile_values:
|
| 3259 |
now = dt.datetime.fromisoformat(str(self.today)) if self.today else self._clock.replace(tzinfo=None)
|
| 3260 |
-
|
| 3261 |
-
|
| 3262 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3263 |
return self._volatile_values[token]
|
| 3264 |
|
| 3265 |
def _connect(self, key):
|
|
@@ -3612,7 +3643,7 @@ def _sheet_error_message(code):
|
|
| 3612 |
}.get(code, "This formula could not be calculated.")
|
| 3613 |
|
| 3614 |
|
| 3615 |
-
def recompute_document(book, *, cache_key=None, changes=None):
|
| 3616 |
"""Recalculate a C7 document in place; return C8 values/errors/provenance/work counters.
|
| 3617 |
|
| 3618 |
Only a trusted (tenant, user, workbook) tuple can opt into this process-local bounded LRU.
|
|
@@ -3622,12 +3653,18 @@ def recompute_document(book, *, cache_key=None, changes=None):
|
|
| 3622 |
formula values; it never scans a sheet's cell map on a valid delta.
|
| 3623 |
"""
|
| 3624 |
with _document_calculations_lock:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3625 |
tabs = [tab for tab in book.get("sheets", ()) if isinstance(tab, dict)]
|
| 3626 |
names = _document_scopes(tabs)
|
| 3627 |
tab_by_scope = {name.upper(): tab for tab, name in zip(tabs, names)}
|
| 3628 |
signature = json.dumps([[(tab.get("id"), name) for tab, name in zip(tabs, names)],
|
| 3629 |
book.get("namedRanges"), book.get("dateSystem"),
|
| 3630 |
-
|
| 3631 |
revision = book.get("rev")
|
| 3632 |
valid_revision = type(revision) is int and revision >= 0
|
| 3633 |
trusted = (isinstance(cache_key, tuple) and len(cache_key) == 3
|
|
@@ -3650,7 +3687,8 @@ def recompute_document(book, *, cache_key=None, changes=None):
|
|
| 3650 |
else:
|
| 3651 |
formula_keys.discard(key)
|
| 3652 |
if incremental:
|
| 3653 |
-
engine.update(delta, revision=revision
|
|
|
|
| 3654 |
if not incremental:
|
| 3655 |
specs, formula_keys = {}, set()
|
| 3656 |
for tab, name in zip(tabs, names):
|
|
@@ -3661,8 +3699,12 @@ def recompute_document(book, *, cache_key=None, changes=None):
|
|
| 3661 |
specs[key] = _document_spec(cell, name)
|
| 3662 |
if cell.get("kind") == "formula":
|
| 3663 |
formula_keys.add(key)
|
| 3664 |
-
engine = SheetCalculation(
|
| 3665 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3666 |
projected = 0
|
| 3667 |
for key in formula_keys:
|
| 3668 |
scope, _, addr = key.rpartition("!")
|
|
|
|
| 84 |
"OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
|
| 85 |
"FILTER": (2, 3), "SORT": (1, 4),
|
| 86 |
}
|
| 87 |
+
# Excel workbooks in the Daily Sales lineage use both spellings. Keep the authored formula text
|
| 88 |
+
# untouched in the workbook; normalization maps AVG to the same AST/function the catalogue's
|
| 89 |
+
# Average action emits, so quoted-sheet references and cached formula text still round-trip.
|
| 90 |
+
ALIASES = {"CONCAT": "CONCATENATE", "AVG": "AVERAGE"}
|
| 91 |
|
| 92 |
#: ⭐ THE TRANSPORT SPELLING OF A DOTTED EXCEL NAME, AND IT IS NOT AN ALIAS.
|
| 93 |
#:
|
|
|
|
| 1758 |
if isinstance(row, _SheetInputs):
|
| 1759 |
left, right = 0 if left is None else left, 0 if right is None else right
|
| 1760 |
left, right = int(left) if isinstance(left, bool) else left, int(right) if isinstance(right, bool) else right
|
| 1761 |
+
# ISO is the one date value shape shared by import, rendering and database filters. Give
|
| 1762 |
+
# that same value Excel's day arithmetic here instead of forcing a caller to replace it
|
| 1763 |
+
# with a serial number that the date filter cannot read. Date-date subtraction answers a
|
| 1764 |
+
# day count; date +/- number answers another ISO date. Other operators remain numeric.
|
| 1765 |
+
if node["op"] in ("+", "-"):
|
| 1766 |
+
left_day, right_day = _date(left), _date(right)
|
| 1767 |
+
left_num, right_num = _num(left), _num(right)
|
| 1768 |
+
try:
|
| 1769 |
+
if left_day is not None and right_day is not None and node["op"] == "-":
|
| 1770 |
+
return (left_day - right_day).days
|
| 1771 |
+
if left_day is not None and right_num is not None:
|
| 1772 |
+
shift = right_num if node["op"] == "+" else -right_num
|
| 1773 |
+
if int(shift) == shift:
|
| 1774 |
+
return (left_day + dt.timedelta(days=int(shift))).isoformat()
|
| 1775 |
+
if right_day is not None and left_num is not None and node["op"] == "+":
|
| 1776 |
+
if int(left_num) == left_num:
|
| 1777 |
+
return (right_day + dt.timedelta(days=int(left_num))).isoformat()
|
| 1778 |
+
except (OverflowError, ValueError):
|
| 1779 |
+
_diagnose(diagnostics, "bad_value")
|
| 1780 |
+
return None
|
| 1781 |
a, b = _num(left), _num(right)
|
| 1782 |
if a is None or b is None: return None
|
| 1783 |
if node["op"] == "/" and b == 0:
|
|
|
|
| 3280 |
token = id(node)
|
| 3281 |
if token not in self._volatile_values:
|
| 3282 |
now = dt.datetime.fromisoformat(str(self.today)) if self.today else self._clock.replace(tzinfo=None)
|
| 3283 |
+
# Formula dates have one representation in this runtime: ISO, the same representation
|
| 3284 |
+
# imported date literals, DATE/DATEVALUE, filters and the cell display path consume.
|
| 3285 |
+
# The old document graph special-cased volatile dates as Excel serials while the
|
| 3286 |
+
# compatibility evaluator returned ISO, so TODAY() could render as 46284 while
|
| 3287 |
+
# DATE(...) beside it rendered 2026-09-19. dateSystem still governs imported Excel
|
| 3288 |
+
# serial conversion at the input boundary; it must not create a second stored value
|
| 3289 |
+
# type after that conversion.
|
| 3290 |
+
self._volatile_values[token] = (
|
| 3291 |
+
random.random() if fn == "RAND" else
|
| 3292 |
+
now.strftime("%Y-%m-%d %H:%M:%S") if fn == "NOW" else
|
| 3293 |
+
now.date().isoformat())
|
| 3294 |
return self._volatile_values[token]
|
| 3295 |
|
| 3296 |
def _connect(self, key):
|
|
|
|
| 3643 |
}.get(code, "This formula could not be calculated.")
|
| 3644 |
|
| 3645 |
|
| 3646 |
+
def recompute_document(book, *, cache_key=None, changes=None, today=None):
|
| 3647 |
"""Recalculate a C7 document in place; return C8 values/errors/provenance/work counters.
|
| 3648 |
|
| 3649 |
Only a trusted (tenant, user, workbook) tuple can opt into this process-local bounded LRU.
|
|
|
|
| 3653 |
formula values; it never scans a sheet's cell map on a valid delta.
|
| 3654 |
"""
|
| 3655 |
with _document_calculations_lock:
|
| 3656 |
+
explicit_today = today is not None
|
| 3657 |
+
calculation_day = str(today or dt.date.today().isoformat())[:10]
|
| 3658 |
+
try:
|
| 3659 |
+
calculation_day = dt.date.fromisoformat(calculation_day).isoformat()
|
| 3660 |
+
except ValueError as exc:
|
| 3661 |
+
raise ValueError("document calculation today must be an ISO date") from exc
|
| 3662 |
tabs = [tab for tab in book.get("sheets", ()) if isinstance(tab, dict)]
|
| 3663 |
names = _document_scopes(tabs)
|
| 3664 |
tab_by_scope = {name.upper(): tab for tab, name in zip(tabs, names)}
|
| 3665 |
signature = json.dumps([[(tab.get("id"), name) for tab, name in zip(tabs, names)],
|
| 3666 |
book.get("namedRanges"), book.get("dateSystem"),
|
| 3667 |
+
calculation_day], sort_keys=True)
|
| 3668 |
revision = book.get("rev")
|
| 3669 |
valid_revision = type(revision) is int and revision >= 0
|
| 3670 |
trusted = (isinstance(cache_key, tuple) and len(cache_key) == 3
|
|
|
|
| 3687 |
else:
|
| 3688 |
formula_keys.discard(key)
|
| 3689 |
if incremental:
|
| 3690 |
+
engine.update(delta, revision=revision,
|
| 3691 |
+
today=calculation_day if explicit_today else None)
|
| 3692 |
if not incremental:
|
| 3693 |
specs, formula_keys = {}, set()
|
| 3694 |
for tab, name in zip(tabs, names):
|
|
|
|
| 3699 |
specs[key] = _document_spec(cell, name)
|
| 3700 |
if cell.get("kind") == "formula":
|
| 3701 |
formula_keys.add(key)
|
| 3702 |
+
engine = SheetCalculation(
|
| 3703 |
+
specs, revision=revision if valid_revision else 0, sheets=names,
|
| 3704 |
+
named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"),
|
| 3705 |
+
# A supplied day makes an exceptional repair deterministic. Ordinary workbook
|
| 3706 |
+
# recalculation keeps the live clock, so NOW() does not silently become midnight.
|
| 3707 |
+
today=calculation_day if explicit_today else None)
|
| 3708 |
projected = 0
|
| 3709 |
for key in formula_keys:
|
| 3710 |
scope, _, addr = key.rpartition("!")
|
api/main.py
CHANGED
|
@@ -13,7 +13,8 @@ WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bear
|
|
| 13 |
user RECORD. `/api/health` stays unauthenticated (liveness only).
|
| 14 |
* **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
|
| 15 |
for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
|
| 16 |
-
* **
|
|
|
|
| 17 |
|
| 18 |
⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
|
| 19 |
session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
|
|
@@ -25,18 +26,6 @@ import os
|
|
| 25 |
import sys
|
| 26 |
from pathlib import Path
|
| 27 |
|
| 28 |
-
# --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
|
| 29 |
-
# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
|
| 30 |
-
# unverified default context; the HF Space and the container are unaffected. This used to run
|
| 31 |
-
# unconditionally at import — i.e. the shipped container disabled TLS verification for every
|
| 32 |
-
# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
|
| 33 |
-
# man-in-the-middle away from being someone else's data. It is now opt-in, must be set
|
| 34 |
-
# deliberately, and is never on by default.
|
| 35 |
-
# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
|
| 36 |
-
if os.environ.get("AIOS_INSECURE_SSL") == "1":
|
| 37 |
-
import ssl
|
| 38 |
-
ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
|
| 39 |
-
|
| 40 |
# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
|
| 41 |
# (where the layout differs from the local sibling-dir default). ---
|
| 42 |
_HERE = Path(__file__).resolve()
|
|
@@ -93,6 +82,7 @@ import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operato
|
|
| 93 |
import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router)
|
| 94 |
import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router)
|
| 95 |
from core import grid_events # noqa: E402
|
|
|
|
| 96 |
# D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below.
|
| 97 |
# ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
|
| 98 |
# session fallback must NOT degrade a refusal, or the user is told a change was saved that was not.
|
|
@@ -324,6 +314,8 @@ app.include_router(routes_nav.router)
|
|
| 324 |
app.include_router(routes_customers.router)
|
| 325 |
app.include_router(routes_products.router)
|
| 326 |
app.include_router(routes_assets.router)
|
|
|
|
|
|
|
| 327 |
app.include_router(routes_tables.router)
|
| 328 |
app.include_router(routes_grid.router)
|
| 329 |
app.include_router(routes_records.router)
|
|
@@ -541,9 +533,6 @@ def _startup_notes():
|
|
| 541 |
print("[aios-api] AIOS_SESSION_SECRET is not set. Signing with a random per-process "
|
| 542 |
"key. Sessions will not survive a restart and will not work across workers. "
|
| 543 |
"Set it in production.")
|
| 544 |
-
if os.environ.get("AIOS_INSECURE_SSL") == "1":
|
| 545 |
-
print("[aios-api] AIOS_INSECURE_SSL=1. TLS verification is DISABLED for outbound "
|
| 546 |
-
"requests. Local development only; never in a deployed environment.")
|
| 547 |
|
| 548 |
|
| 549 |
_assert_contract_floor()
|
|
@@ -782,49 +771,93 @@ class _SyncNowRefused(Exception):
|
|
| 782 |
"""An application-authored, credential-free sentence for a Sync now receipt."""
|
| 783 |
|
| 784 |
|
| 785 |
-
def
|
| 786 |
"""Classify actual provider failures without returning vendor text or exception reprs."""
|
| 787 |
import re
|
| 788 |
-
|
|
|
|
| 789 |
causes = (
|
| 790 |
-
(("reduce the amount of data", "response too large"),
|
| 791 |
"The provider refused the pull because the response contains too much data."),
|
| 792 |
-
(("no ad accounts",), "The Meta token cannot access any ad accounts."),
|
| 793 |
-
(("max_pages", "page cap"), "The pull stopped at the provider page limit."),
|
| 794 |
-
(
|
| 795 |
-
|
| 796 |
-
(
|
|
|
|
|
|
|
| 797 |
"This tenant has no usable credential for the connector."),
|
| 798 |
-
(("enablement document", "capability unavailable"),
|
| 799 |
"The connector database settings could not be read."),
|
| 800 |
-
(("throttl", "rate limit", "too many requests"),
|
| 801 |
"The provider rate limit stopped the sync. Retry after the provider delay."),
|
| 802 |
-
(("timed out", "timeout"), "The sync failed because the provider request timed out."),
|
| 803 |
-
(("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
|
| 804 |
"expired", "authentication"), "The provider refused the connector credential or its permissions."),
|
| 805 |
-
(("duplicate key", "primary key", "constraint"),
|
| 806 |
"The mirror rejected rows because they violate a database constraint."),
|
| 807 |
-
(("connection", "network", "name resolution"),
|
| 808 |
"The sync could not connect to the provider or tenant database."),
|
| 809 |
# A-META / D-201: a Meta pull on a container whose mirror never held `meta_*` raises
|
| 810 |
# DuckDB's CatalogException. Standing rule 1 wants the CAUSE, not "unexpected error":
|
| 811 |
# "the loader is broken" and "this server has never held these tables" are different
|
| 812 |
# operator actions, and only the second one is true here.
|
| 813 |
-
(("does not exist", "no such table", "catalog error"),
|
| 814 |
"This server's mirror does not hold this connector's tables yet, so there was nothing "
|
| 815 |
"to update. Run the connector's first load on the server that owns this tenant."),
|
| 816 |
)
|
| 817 |
-
for needles, sentence in causes:
|
| 818 |
if any(needle in text for needle in needles):
|
| 819 |
-
return sentence
|
| 820 |
status = re.search(r"\bhttp\s+(\d{3})\b", text)
|
| 821 |
if status:
|
| 822 |
-
return f"The provider refused the sync request with HTTP status {status.group(1)}."
|
| 823 |
if isinstance(cause, TypeError):
|
| 824 |
-
return "The connector failed because its provider call used incompatible arguments."
|
| 825 |
if isinstance(cause, (KeyError, ValueError)):
|
| 826 |
-
return "The connector could not read an expected value in the sync response."
|
| 827 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 828 |
|
| 829 |
|
| 830 |
def _run_sync_requests():
|
|
@@ -854,28 +887,47 @@ def _run_sync_requests():
|
|
| 854 |
if provider == "odoo":
|
| 855 |
import core.odoo as odoo
|
| 856 |
import odoo_relational as rel
|
|
|
|
| 857 |
if rt.odoo_paused() or rel.frozen(rt):
|
| 858 |
raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.")
|
| 859 |
-
|
| 860 |
-
|
| 861 |
-
|
| 862 |
-
|
| 863 |
-
|
| 864 |
-
|
| 865 |
-
|
| 866 |
-
|
| 867 |
-
|
| 868 |
-
|
| 869 |
-
|
| 870 |
-
|
| 871 |
-
|
| 872 |
-
|
| 873 |
-
|
| 874 |
-
|
| 875 |
-
|
| 876 |
-
|
| 877 |
-
|
| 878 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 879 |
elif provider == "amazon_sp":
|
| 880 |
from harness import amazon_store as amazon
|
| 881 |
if not amazon.credential(tenant, rt=rt):
|
|
@@ -911,13 +963,13 @@ def _run_sync_requests():
|
|
| 911 |
if problems:
|
| 912 |
detail += " " + _sync_now_failure(problems[0])
|
| 913 |
elif problems:
|
| 914 |
-
detail =
|
| 915 |
else:
|
| 916 |
ok, detail = True, "Sync finished."
|
| 917 |
except _SyncNowRefused as exc:
|
| 918 |
detail = str(exc)
|
| 919 |
except Exception as exc: # noqa: BLE001
|
| 920 |
-
detail =
|
| 921 |
if first_load_progress is not None:
|
| 922 |
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail,
|
| 923 |
first_load=first_load_progress)
|
|
@@ -1117,7 +1169,8 @@ def _sweep_automation_schemas(why):
|
|
| 1117 |
print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)")
|
| 1118 |
|
| 1119 |
|
| 1120 |
-
def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None
|
|
|
|
| 1121 |
"""Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
|
| 1122 |
'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
|
| 1123 |
could have asked" are different diagnoses.
|
|
@@ -1127,21 +1180,38 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1127 |
and was fine" from "the rebuild was never reached". Three failure markers and no success
|
| 1128 |
marker means silence is ambiguous; now it is not.
|
| 1129 |
|
| 1130 |
-
|
| 1131 |
-
|
| 1132 |
-
|
| 1133 |
-
straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME
|
| 1134 |
-
process-global, and rebinding it in a daemon thread under live readers can serve one tenant's
|
| 1135 |
-
rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify.
|
| 1136 |
"""
|
| 1137 |
try:
|
|
|
|
| 1138 |
import odoo_relational as _rel
|
| 1139 |
from harness import runtime as _runtime
|
| 1140 |
-
|
| 1141 |
-
|
| 1142 |
-
_rt = _runtime.get_runtime("royal-imports")
|
| 1143 |
from core import user_tables as _ut
|
| 1144 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1145 |
# ⭐⭐ W54-T46 / R4 / R7 — EVERY Odoo CONNECTION, not just the first. `all_connections`
|
| 1146 |
# delegates to `odoo_relational.refresh_all`, which walks this tenant's Odoo
|
| 1147 |
# connections in slot order and records a failure against the connection it belongs
|
|
@@ -1156,7 +1226,7 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1156 |
# because a gate that counts RAW occurrences of a call would count the one in this
|
| 1157 |
# comment too. That gate happens to strip comments first; the next one may not.
|
| 1158 |
# See `odoo_relational.refresh` and its `all_connections` arm.
|
| 1159 |
-
counts = _rel.refresh(_rt,
|
| 1160 |
raw_models, schema_errors = _rel.authoritative_schema_snapshot()
|
| 1161 |
_ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
|
| 1162 |
errors=schema_errors, st=_rt)
|
|
@@ -1183,19 +1253,134 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1183 |
except Exception as e: # noqa: BLE001
|
| 1184 |
rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
|
| 1185 |
print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1186 |
try:
|
| 1187 |
import rollup_sql as _rollup
|
|
|
|
| 1188 |
for _b, _key, _l, _f in _rel.TABLES:
|
| 1189 |
-
_rollup.compute(_rt, _key)
|
| 1190 |
except Exception as e: # noqa: BLE001
|
| 1191 |
rebuild_errors.append(f"rollups: {type(e).__name__}: {e}")
|
| 1192 |
print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}")
|
| 1193 |
print(f"[aios-api] odoo relational rebuild done ({why}): "
|
| 1194 |
+ ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items())))
|
| 1195 |
-
|
| 1196 |
-
|
| 1197 |
-
|
| 1198 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1199 |
meta_state = _rebuild_meta_relational(
|
| 1200 |
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
|
| 1201 |
if isinstance(meta_state_out, dict):
|
|
@@ -1207,7 +1392,7 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1207 |
# The isolated cells/rollups/meta handlers above are deliberately allowed to report and
|
| 1208 |
# continue. This return value answers the narrower resync-policy question: did the Odoo
|
| 1209 |
# table refresh itself complete, so a confirmed zero-change pass may be skipped?
|
| 1210 |
-
return
|
| 1211 |
except Exception as e: # noqa: BLE001
|
| 1212 |
# ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only
|
| 1213 |
# `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a
|
|
@@ -1217,7 +1402,14 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1217 |
try:
|
| 1218 |
if '_rt' in locals():
|
| 1219 |
from core import user_tables as _ut
|
| 1220 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1221 |
except Exception:
|
| 1222 |
pass
|
| 1223 |
return False
|
|
@@ -1297,8 +1489,10 @@ RESYNC_STATE = {
|
|
| 1297 |
"disagreement": None,
|
| 1298 |
"passes": 0,
|
| 1299 |
"lastPassAt": None,
|
|
|
|
| 1300 |
"lastPassOk": None,
|
| 1301 |
"lastError": None,
|
|
|
|
| 1302 |
"consecutiveFailures": 0,
|
| 1303 |
"bootSyncError": None,
|
| 1304 |
}
|
|
@@ -1467,15 +1661,29 @@ def _classify_resync_error(exc):
|
|
| 1467 |
⚠ NOT the whole of D-314: naming the offending TABLE and ROW ID belongs to the DuckDB writer
|
| 1468 |
that raised, which is not this loop.
|
| 1469 |
"""
|
|
|
|
| 1470 |
text = f"{type(exc).__name__}: {exc}"
|
| 1471 |
low = text.lower()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1472 |
persistent = any(t in low for t in
|
| 1473 |
("duplicate key", "primary key", "unique constraint", "constraint error",
|
| 1474 |
"binder", "conversion error"))
|
| 1475 |
-
|
| 1476 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1477 |
"meaning": ("this failure repeats until someone repairs the mirror; retrying the same "
|
| 1478 |
-
"pass will not clear it" if persistent else
|
| 1479 |
"this failure usually clears on the next pass")}
|
| 1480 |
|
| 1481 |
|
|
@@ -1543,6 +1751,14 @@ def _store_resync_loop():
|
|
| 1543 |
"tenantSeconds": _cad["tenantSeconds"], "envSeconds": _cad["envSeconds"],
|
| 1544 |
"resolvedSeconds": _every, "manual": _manual,
|
| 1545 |
"disagreement": _cad["disagreement"]})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1546 |
# ⚠ ON CHANGE, NOT EVERY CYCLE. An hourly warning is a skimmed warning.
|
| 1547 |
if _cad["disagreement"] != _RESYNC_NOTICE[0]:
|
| 1548 |
_RESYNC_NOTICE[0] = _cad["disagreement"]
|
|
@@ -1566,6 +1782,8 @@ def _store_resync_loop():
|
|
| 1566 |
RESYNC_STATE["lastError"] = _err
|
| 1567 |
if _err is None:
|
| 1568 |
RESYNC_STATE["consecutiveFailures"] = 0
|
|
|
|
|
|
|
| 1569 |
else:
|
| 1570 |
RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
|
| 1571 |
# D-314: the TYPE and the persistent/transient reading, both in the line and in the
|
|
@@ -1619,7 +1837,17 @@ def _store_resync_loop():
|
|
| 1619 |
_meta_state = {}
|
| 1620 |
if _reason is not None:
|
| 1621 |
print(f"[aios-api] odoo relational rebuild required ({_reason})")
|
| 1622 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1623 |
last_relational_success_at = _t.time()
|
| 1624 |
else:
|
| 1625 |
# A prior successful projection no longer proves the changed mirror was applied.
|
|
@@ -1797,7 +2025,7 @@ if os.environ.get("AIOS_PREWARM") == "1":
|
|
| 1797 |
# ⭐⭐ W50-T29 / R8 — AND THE DEPLOYMENT NOW OPTS IN, ON STAGING AND ON LIVE. The owner's ruling is
|
| 1798 |
# that the Space resyncs its own mirror hourly rather than waiting for someone to visit; without
|
| 1799 |
# it a Space serves whatever the seed was cut with, forever, and August reads zero while Odoo has
|
| 1800 |
-
# 723 orders. `deploy_web.py` pushes `AIOS_STORE_RESYNC=1` plus `AIOS_STORE_RESYNC_SECONDS=
|
| 1801 |
#
|
| 1802 |
# ⛔ THE OLD REASON FOR KEEPING IT OFF WAS "so a Pg-backed Live Space can scale its Neon compute to
|
| 1803 |
# zero between real requests", AND THAT REASON IS ONLY HALF SATISFIED. `_resync_mirror_pass` is
|
|
@@ -1806,6 +2034,24 @@ if os.environ.get("AIOS_PREWARM") == "1":
|
|
| 1806 |
# `_rebuild_meta_relational` write through `core.user_tables`, which is the tenant product store,
|
| 1807 |
# and under `STORE_BACKEND=pg` that is Neon. Named here rather than left for the next reader to
|
| 1808 |
# discover from a bill, and carried as a register row by this ticket's handoff.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1809 |
if os.environ.get("AIOS_STORE_RESYNC") == "1":
|
| 1810 |
import threading as _threading_resync
|
| 1811 |
_threading_resync.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
|
|
|
|
| 13 |
user RECORD. `/api/health` stays unauthenticated (liveness only).
|
| 14 |
* **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
|
| 15 |
for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
|
| 16 |
+
* **Outbound TLS always uses the verified default context.** A certificate failure is a source
|
| 17 |
+
failure, never permission to accept an untrusted connection.
|
| 18 |
|
| 19 |
⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
|
| 20 |
session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
|
|
|
|
| 26 |
import sys
|
| 27 |
from pathlib import Path
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
|
| 30 |
# (where the layout differs from the local sibling-dir default). ---
|
| 31 |
_HERE = Path(__file__).resolve()
|
|
|
|
| 82 |
import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router)
|
| 83 |
import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router)
|
| 84 |
from core import grid_events # noqa: E402
|
| 85 |
+
|
| 86 |
# D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below.
|
| 87 |
# ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
|
| 88 |
# session fallback must NOT degrade a refusal, or the user is told a change was saved that was not.
|
|
|
|
| 314 |
app.include_router(routes_customers.router)
|
| 315 |
app.include_router(routes_products.router)
|
| 316 |
app.include_router(routes_assets.router)
|
| 317 |
+
import routes_applications
|
| 318 |
+
app.include_router(routes_applications.router)
|
| 319 |
app.include_router(routes_tables.router)
|
| 320 |
app.include_router(routes_grid.router)
|
| 321 |
app.include_router(routes_records.router)
|
|
|
|
| 533 |
print("[aios-api] AIOS_SESSION_SECRET is not set. Signing with a random per-process "
|
| 534 |
"key. Sessions will not survive a restart and will not work across workers. "
|
| 535 |
"Set it in production.")
|
|
|
|
|
|
|
|
|
|
| 536 |
|
| 537 |
|
| 538 |
_assert_contract_floor()
|
|
|
|
| 771 |
"""An application-authored, credential-free sentence for a Sync now receipt."""
|
| 772 |
|
| 773 |
|
| 774 |
+
def _sync_now_cause(cause):
|
| 775 |
"""Classify actual provider failures without returning vendor text or exception reprs."""
|
| 776 |
import re
|
| 777 |
+
raw = str(cause)
|
| 778 |
+
text = raw.lower()
|
| 779 |
causes = (
|
| 780 |
+
("response_too_large", ("reduce the amount of data", "response too large"),
|
| 781 |
"The provider refused the pull because the response contains too much data."),
|
| 782 |
+
("meta_no_ad_accounts", ("no ad accounts",), "The Meta token cannot access any ad accounts."),
|
| 783 |
+
("page_cap", ("max_pages", "page cap"), "The pull stopped at the provider page limit."),
|
| 784 |
+
("sandbox_credential", ("sandbox",),
|
| 785 |
+
"Amazon sync requires a production credential; this credential is for the sandbox."),
|
| 786 |
+
("deletion_outstanding", ("deletion", "retention"),
|
| 787 |
+
"Amazon sync is blocked by an outstanding data deletion requirement."),
|
| 788 |
+
("no_credential", ("no stored", "no odoo credentials", "access_token is not set"),
|
| 789 |
"This tenant has no usable credential for the connector."),
|
| 790 |
+
("enablement_unreadable", ("enablement document", "capability unavailable"),
|
| 791 |
"The connector database settings could not be read."),
|
| 792 |
+
("rate_limited", ("throttl", "rate limit", "too many requests"),
|
| 793 |
"The provider rate limit stopped the sync. Retry after the provider delay."),
|
| 794 |
+
("timeout", ("timed out", "timeout"), "The sync failed because the provider request timed out."),
|
| 795 |
+
("credential_refused", ("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
|
| 796 |
"expired", "authentication"), "The provider refused the connector credential or its permissions."),
|
| 797 |
+
("mirror_constraint", ("duplicate key", "primary key", "constraint"),
|
| 798 |
"The mirror rejected rows because they violate a database constraint."),
|
| 799 |
+
("connect_failed", ("connection", "network", "name resolution"),
|
| 800 |
"The sync could not connect to the provider or tenant database."),
|
| 801 |
# A-META / D-201: a Meta pull on a container whose mirror never held `meta_*` raises
|
| 802 |
# DuckDB's CatalogException. Standing rule 1 wants the CAUSE, not "unexpected error":
|
| 803 |
# "the loader is broken" and "this server has never held these tables" are different
|
| 804 |
# operator actions, and only the second one is true here.
|
| 805 |
+
("mirror_missing_tables", ("does not exist", "no such table", "catalog error"),
|
| 806 |
"This server's mirror does not hold this connector's tables yet, so there was nothing "
|
| 807 |
"to update. Run the connector's first load on the server that owns this tenant."),
|
| 808 |
)
|
| 809 |
+
for classifier, needles, sentence in causes:
|
| 810 |
if any(needle in text for needle in needles):
|
| 811 |
+
return classifier, sentence
|
| 812 |
status = re.search(r"\bhttp\s+(\d{3})\b", text)
|
| 813 |
if status:
|
| 814 |
+
return "http_status", f"The provider refused the sync request with HTTP status {status.group(1)}."
|
| 815 |
if isinstance(cause, TypeError):
|
| 816 |
+
return "bad_arguments", "The connector failed because its provider call used incompatible arguments."
|
| 817 |
if isinstance(cause, (KeyError, ValueError)):
|
| 818 |
+
return "unreadable_value", "The connector could not read an expected value in the sync response."
|
| 819 |
+
if isinstance(cause, str):
|
| 820 |
+
# amazon_store._sync_problem appends the application-authored fix LAST. The
|
| 821 |
+
# exception message may itself contain delimiters; none of it is display text.
|
| 822 |
+
problem, separator, fix = raw.rpartition("; fix: ")
|
| 823 |
+
header = re.match(
|
| 824 |
+
r"(?P<table>[A-Za-z_][A-Za-z0-9_.-]*): "
|
| 825 |
+
r"(?P<operation>[A-Za-z_][A-Za-z0-9_.-]*) (?P<state>[A-Z]+): "
|
| 826 |
+
r"(?P<exception>[A-Za-z_][A-Za-z0-9_]*): ", problem)
|
| 827 |
+
if separator and header and fix.strip():
|
| 828 |
+
# Keep the authored action, with sentence punctuation suitable for the UI.
|
| 829 |
+
action = fix.strip().replace("\u2014", ". ").replace("\u2013", " to ")
|
| 830 |
+
sentence = (
|
| 831 |
+
f"The Amazon table {header['table']} could not be read. "
|
| 832 |
+
f"The {header['operation']} call was {header['state'].lower()} "
|
| 833 |
+
f"with {header['exception']}. What to do: {action}"
|
| 834 |
+
)
|
| 835 |
+
return "connector_reported", sentence if sentence.endswith(".") else sentence + "."
|
| 836 |
+
return "unclassified", (
|
| 837 |
+
"The connector reported an unclassified plain-text failure. "
|
| 838 |
+
"Ask an administrator to inspect the connector's sync report and retry.")
|
| 839 |
+
return "unclassified", (
|
| 840 |
+
f"The connector failed with an unclassified {type(cause).__name__}. "
|
| 841 |
+
"Ask an administrator to inspect the connector's sync report and retry.")
|
| 842 |
+
|
| 843 |
+
|
| 844 |
+
def _sync_now_failure(cause):
|
| 845 |
+
"""Classify actual provider failures without returning vendor text or exception reprs."""
|
| 846 |
+
return _sync_now_cause(cause)[1]
|
| 847 |
+
|
| 848 |
+
|
| 849 |
+
def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
|
| 850 |
+
"""Record one classified failure; leave successful receipt text untouched."""
|
| 851 |
+
if ok:
|
| 852 |
+
return detail
|
| 853 |
+
classifier, sentence = _sync_now_cause(cause)
|
| 854 |
+
print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
|
| 855 |
+
suffix = f" (cause code: {classifier})"
|
| 856 |
+
# finish_sync_request stores only 300 characters. Reserve the suffix before it clips.
|
| 857 |
+
budget = 300 - len(suffix)
|
| 858 |
+
if len(sentence) > budget:
|
| 859 |
+
sentence = sentence[:budget - 3].rstrip() + "..."
|
| 860 |
+
return sentence + suffix
|
| 861 |
|
| 862 |
|
| 863 |
def _run_sync_requests():
|
|
|
|
| 887 |
if provider == "odoo":
|
| 888 |
import core.odoo as odoo
|
| 889 |
import odoo_relational as rel
|
| 890 |
+
import routes_connectors as _sync_routes
|
| 891 |
if rt.odoo_paused() or rel.frozen(rt):
|
| 892 |
raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.")
|
| 893 |
+
requested_cid = (_sync_routes.sync_request_connection(provider, tenant)
|
| 894 |
+
or _sync_routes._odoo_connection_slot(rt, None))
|
| 895 |
+
default_cid = _sync_routes._odoo_connection_slot(rt, None)
|
| 896 |
+
if requested_cid != default_cid:
|
| 897 |
+
# Named connections have their own provider reader and must never be pulled
|
| 898 |
+
# through the process-global default mirror under another connection's id.
|
| 899 |
+
from types import SimpleNamespace
|
| 900 |
+
report = _sync_routes._odoo_sync_runner(
|
| 901 |
+
SimpleNamespace(runtime=rt, tenant=tenant,
|
| 902 |
+
uname=receipt.get("requestedBy") or "automation"),
|
| 903 |
+
requested_cid)
|
| 904 |
+
problems = list(report.get("errors") or ())
|
| 905 |
+
if report.get("state") in ("error", "partial") and not problems:
|
| 906 |
+
problems.append("The named Odoo connection did not finish cleanly.")
|
| 907 |
+
else:
|
| 908 |
+
source = rt.odoo_source()
|
| 909 |
+
if source is None:
|
| 910 |
+
raise _SyncNowRefused(
|
| 911 |
+
"This tenant has no Odoo credential. Connect Odoo before syncing.")
|
| 912 |
+
odoo_requested = True
|
| 913 |
+
previous_client = getattr(odoo._tlocal, "client", None)
|
| 914 |
+
previous_context = getattr(odoo._tlocal, "tenant_ctx", None)
|
| 915 |
+
try:
|
| 916 |
+
# Raw mirror pulls use the resolved tenant credential and a dedicated
|
| 917 |
+
# transport. Do not borrow a request's cached XML-RPC client.
|
| 918 |
+
odoo._tlocal.client = odoo.OdooClient(creds=source._creds)
|
| 919 |
+
odoo._tlocal.tenant_ctx = None
|
| 920 |
+
report = datastore.sync_all(log=lambda *a, **k: None)
|
| 921 |
+
finally:
|
| 922 |
+
odoo._tlocal.client = previous_client
|
| 923 |
+
odoo._tlocal.tenant_ctx = previous_context
|
| 924 |
+
if not report:
|
| 925 |
+
raise _SyncNowRefused(
|
| 926 |
+
"Odoo did not run a sync. Check whether the connection is paused.")
|
| 927 |
+
if any(row.get("phase") != "live" for row in report.values()):
|
| 928 |
+
raise _SyncNowRefused(
|
| 929 |
+
"Odoo backfill is incomplete. Run Sync now again to continue it.")
|
| 930 |
+
problems = []
|
| 931 |
elif provider == "amazon_sp":
|
| 932 |
from harness import amazon_store as amazon
|
| 933 |
if not amazon.credential(tenant, rt=rt):
|
|
|
|
| 963 |
if problems:
|
| 964 |
detail += " " + _sync_now_failure(problems[0])
|
| 965 |
elif problems:
|
| 966 |
+
detail = _sync_now_detail(provider, tenant, ok=False, cause=problems[0])
|
| 967 |
else:
|
| 968 |
ok, detail = True, "Sync finished."
|
| 969 |
except _SyncNowRefused as exc:
|
| 970 |
detail = str(exc)
|
| 971 |
except Exception as exc: # noqa: BLE001
|
| 972 |
+
detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
|
| 973 |
if first_load_progress is not None:
|
| 974 |
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail,
|
| 975 |
first_load=first_load_progress)
|
|
|
|
| 1169 |
print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)")
|
| 1170 |
|
| 1171 |
|
| 1172 |
+
def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
|
| 1173 |
+
tenant=None, cadence_seconds=None):
|
| 1174 |
"""Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
|
| 1175 |
'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
|
| 1176 |
could have asked" are different diagnoses.
|
|
|
|
| 1180 |
and was fine" from "the rebuild was never reached". Three failure markers and no success
|
| 1181 |
marker means silence is ambiguous; now it is not.
|
| 1182 |
|
| 1183 |
+
The tenant is the owner of the process-global mirror resolved by `_resync_tenant`; this
|
| 1184 |
+
function never loops or rebinds datastore paths. That preserves D-29's isolation rule while
|
| 1185 |
+
removing the old hard-coded tenant/status path.
|
|
|
|
|
|
|
|
|
|
| 1186 |
"""
|
| 1187 |
try:
|
| 1188 |
+
import time as _t
|
| 1189 |
import odoo_relational as _rel
|
| 1190 |
from harness import runtime as _runtime
|
| 1191 |
+
tenant_key = str(tenant or _resync_tenant()[0])
|
| 1192 |
+
_rt = _runtime.get_runtime(tenant_key)
|
|
|
|
| 1193 |
from core import user_tables as _ut
|
| 1194 |
+
default_cid = routes_connectors._odoo_connection_slot(_rt, None)
|
| 1195 |
+
connection_ids = [default_cid]
|
| 1196 |
+
try:
|
| 1197 |
+
import core.keychain as _keychain
|
| 1198 |
+
connection_ids.extend(str(row.get("connection_id") or row.get("id"))
|
| 1199 |
+
for row in _keychain.list_entries(_rt)
|
| 1200 |
+
if row.get("type") == "odoo")
|
| 1201 |
+
except Exception: # noqa: BLE001
|
| 1202 |
+
pass
|
| 1203 |
+
connection_ids = list(dict.fromkeys(cid for cid in connection_ids if cid))
|
| 1204 |
+
|
| 1205 |
+
def _health(cid, stage, state, **kwargs):
|
| 1206 |
+
try:
|
| 1207 |
+
routes_connectors.record_odoo_stage(
|
| 1208 |
+
_rt, cid, stage, state, cadence_seconds=cadence_seconds, **kwargs)
|
| 1209 |
+
except Exception: # noqa: BLE001
|
| 1210 |
+
pass # The established lifecycle receipt below remains independently visible.
|
| 1211 |
+
|
| 1212 |
+
for cid in connection_ids:
|
| 1213 |
+
_ut.record_odoo_sync_status(phase="syncing", st=_rt, connection_id=cid)
|
| 1214 |
+
_health(cid, "materialization", "running")
|
| 1215 |
# ⭐⭐ W54-T46 / R4 / R7 — EVERY Odoo CONNECTION, not just the first. `all_connections`
|
| 1216 |
# delegates to `odoo_relational.refresh_all`, which walks this tenant's Odoo
|
| 1217 |
# connections in slot order and records a failure against the connection it belongs
|
|
|
|
| 1226 |
# because a gate that counts RAW occurrences of a call would count the one in this
|
| 1227 |
# comment too. That gate happens to strip comments first; the next one may not.
|
| 1228 |
# See `odoo_relational.refresh` and its `all_connections` arm.
|
| 1229 |
+
counts = _rel.refresh(_rt, tenant_key, all_connections=True)
|
| 1230 |
raw_models, schema_errors = _rel.authoritative_schema_snapshot()
|
| 1231 |
_ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
|
| 1232 |
errors=schema_errors, st=_rt)
|
|
|
|
| 1253 |
except Exception as e: # noqa: BLE001
|
| 1254 |
rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
|
| 1255 |
print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
|
| 1256 |
+
# ⭐⭐ 2026-09-21 — ONE DOCUMENT READ FOR ALL EIGHT TABLES, NOT EIGHT. This loop used to
|
| 1257 |
+
# call `compute(_rt, _key)` with no `tables=`, and that argument is the whole cost of the
|
| 1258 |
+
# pass. Without it `rollup_sql.compute` does `rt.get(ut.STORE_KEY)` ITSELF, once per table
|
| 1259 |
+
# — and under `STORE_BACKEND=pg` the store is key -> whole-JSONB-document, so each of
|
| 1260 |
+
# those is the ENTIRE tenant workspace (~28 MB for tenant #0) coming back through Neon's
|
| 1261 |
+
# public proxy. Eight tables meant EIGHT full reads per rebuild before a single cell was
|
| 1262 |
+
# computed, and up to sixteen in the worst case, because each table that HAS source
|
| 1263 |
+
# rollups and finds a changed cell re-reads the document in its `persist` write.
|
| 1264 |
+
# ⚠ EIGHT IS THE MEASURED FLOOR AND SIXTEEN IS A CEILING, NOT AN OBSERVATION: `compute()`
|
| 1265 |
+
# takes its read BEFORE it checks `source_fields(tdef)`, so every table pays the read even
|
| 1266 |
+
# when it returns immediately — but only a table WITH source rollups can reach the write.
|
| 1267 |
+
# Two of the eight (`order_lines`, `gl_lines`) are read-through grains with no stored rows.
|
| 1268 |
+
#
|
| 1269 |
+
# ⛔ AND EGRESS IS THE HALF THAT IS BILLED. Neon's public network transfer is defined as
|
| 1270 |
+
# *"data sent from your databases through the Neon proxy to clients"* — outbound only.
|
| 1271 |
+
# Writes are free; these READS are the 823.47 GB that took September past a 500 GB
|
| 1272 |
+
# allowance. Skipping redundant WRITES would have saved nothing, which is why this is the
|
| 1273 |
+
# change that was made and that one was not.
|
| 1274 |
+
#
|
| 1275 |
+
# ⭐ `tables=`/`persist=` ARE NOT NEW AND THIS IS NOT A NEW PATTERN — it is the calling
|
| 1276 |
+
# convention `automation_engine._refresh_source_rollups` has always used
|
| 1277 |
+
# (`rollup_sql.compute(rt, table_key, tables=snapshot, persist=True)`). The `owned` branch
|
| 1278 |
+
# applies every table's plan against the snapshot in MEMORY and only opens a write when a
|
| 1279 |
+
# cell actually changed, so a quiet pass now costs ONE read instead of eight.
|
| 1280 |
+
# ⚠ THE SHARED SNAPSHOT IS DELIBERATE, not an accident of reuse: table N sees the cells
|
| 1281 |
+
# table N-1 just computed, which is one consistent view of the workspace rather than eight
|
| 1282 |
+
# reads of a document that other writers can move underneath the loop.
|
| 1283 |
+
# ⚠ `persist=True` IS LOAD-BEARING — the `owned` branch returns WITHOUT writing when it is
|
| 1284 |
+
# false, so dropping it would turn this into an expensive no-op that still reports counts.
|
| 1285 |
+
# ⛔ THE SCOPE STAYS `_rel.TABLES`, not `snapshot.items()` like the automation helper: this
|
| 1286 |
+
# is the Odoo rebuild and it must not start computing rollups for unrelated tables.
|
| 1287 |
try:
|
| 1288 |
import rollup_sql as _rollup
|
| 1289 |
+
_rollup_blob = _rt.get(_ut.STORE_KEY) or {}
|
| 1290 |
for _b, _key, _l, _f in _rel.TABLES:
|
| 1291 |
+
_rollup.compute(_rt, _key, tables=_rollup_blob, persist=True)
|
| 1292 |
except Exception as e: # noqa: BLE001
|
| 1293 |
rebuild_errors.append(f"rollups: {type(e).__name__}: {e}")
|
| 1294 |
print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}")
|
| 1295 |
print(f"[aios-api] odoo relational rebuild done ({why}): "
|
| 1296 |
+ ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items())))
|
| 1297 |
+
reports = ((counts or {}).get("connections")
|
| 1298 |
+
if isinstance((counts or {}).get("connections"), dict) else {})
|
| 1299 |
+
if not reports:
|
| 1300 |
+
reports = {default_cid: counts or {}}
|
| 1301 |
+
any_connection_failed = False
|
| 1302 |
+
from datetime import datetime as _datetime, timezone as _timezone
|
| 1303 |
+
refreshed_at = _datetime.now(_timezone.utc).isoformat()
|
| 1304 |
+
for cid, report in reports.items():
|
| 1305 |
+
cid = str(cid or default_cid)
|
| 1306 |
+
report = report if isinstance(report, dict) else {}
|
| 1307 |
+
failed = report.get("failed")
|
| 1308 |
+
database_failures, unmatched_failures = (
|
| 1309 |
+
routes_connectors._odoo_result_database_failures(_rt, cid, report))
|
| 1310 |
+
failure = (routes_connectors._odoo_failure(failed, "materialization")
|
| 1311 |
+
if failed else None)
|
| 1312 |
+
if not failure and (database_failures or unmatched_failures):
|
| 1313 |
+
failure = routes_connectors._odoo_failure(
|
| 1314 |
+
"one or more databases failed", "materialization")
|
| 1315 |
+
tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
|
| 1316 |
+
database_receipts = report.get("databases")
|
| 1317 |
+
database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
|
| 1318 |
+
canonical_receipts = report.get("canonicalDatabases")
|
| 1319 |
+
canonical_receipts = canonical_receipts if isinstance(canonical_receipts, dict) else {}
|
| 1320 |
+
all_receipts = {**database_receipts, **canonical_receipts}
|
| 1321 |
+
if failure:
|
| 1322 |
+
any_connection_failed = True
|
| 1323 |
+
_health(cid, "materialization", "failed", failure=failure)
|
| 1324 |
+
_health(cid, "freshness", "failed",
|
| 1325 |
+
failure={**failure, "stage": "freshness"})
|
| 1326 |
+
else:
|
| 1327 |
+
_health(cid, "source", "ready")
|
| 1328 |
+
_health(cid, "materialization", "ready")
|
| 1329 |
+
# A receipt is the authoritative per-database result. ``tables`` alone omits
|
| 1330 |
+
# read-through grains and the two compiled canonical databases, which previously
|
| 1331 |
+
# made their rich reconciliation evidence disappear at this scheduler boundary.
|
| 1332 |
+
database_keys = list(dict.fromkeys([*all_receipts, *tables]))
|
| 1333 |
+
for key in database_keys:
|
| 1334 |
+
table_counts = tables.get(key)
|
| 1335 |
+
if key in database_failures:
|
| 1336 |
+
continue
|
| 1337 |
+
receipt = all_receipts.get(key)
|
| 1338 |
+
receipt = receipt if isinstance(receipt, dict) else None
|
| 1339 |
+
nrows = (receipt.get("servingCount") if receipt is not None
|
| 1340 |
+
else table_counts.get("rows") if isinstance(table_counts, dict)
|
| 1341 |
+
else None)
|
| 1342 |
+
_health(cid, "materialization", "ready", database_key=key,
|
| 1343 |
+
row_count=nrows, receipt=receipt)
|
| 1344 |
+
if rebuild_errors:
|
| 1345 |
+
recon_failure = routes_connectors._odoo_failure(
|
| 1346 |
+
"relation or rollup pass failed", "reconciliation")
|
| 1347 |
+
_health(cid, "reconciliation", "failed", database_key=key,
|
| 1348 |
+
row_count=nrows, failure=recon_failure, receipt=receipt)
|
| 1349 |
+
_health(cid, "freshness", "failed", database_key=key,
|
| 1350 |
+
row_count=nrows, failure={**recon_failure, "stage": "freshness"},
|
| 1351 |
+
receipt=receipt)
|
| 1352 |
+
else:
|
| 1353 |
+
_health(cid, "reconciliation", "ready", database_key=key,
|
| 1354 |
+
row_count=nrows, receipt=receipt)
|
| 1355 |
+
_health(cid, "freshness", "ready", database_key=key,
|
| 1356 |
+
row_count=nrows, refreshed_at=refreshed_at, receipt=receipt)
|
| 1357 |
+
for key, db_failure in database_failures.items():
|
| 1358 |
+
receipt = all_receipts.get(key)
|
| 1359 |
+
receipt = receipt if isinstance(receipt, dict) else None
|
| 1360 |
+
failed_stage = str(db_failure.get("stage") or "materialization")
|
| 1361 |
+
failed_stage = (failed_stage if failed_stage in
|
| 1362 |
+
routes_connectors._ODOO_HEALTH_STAGES else "materialization")
|
| 1363 |
+
_health(cid, failed_stage, "failed", database_key=key,
|
| 1364 |
+
failure=db_failure, receipt=receipt)
|
| 1365 |
+
_health(cid, "freshness", "failed", database_key=key,
|
| 1366 |
+
failure={**db_failure, "stage": "freshness"}, receipt=receipt)
|
| 1367 |
+
if rebuild_errors:
|
| 1368 |
+
recon_failure = routes_connectors._odoo_failure(
|
| 1369 |
+
"relation or rollup pass failed", "reconciliation")
|
| 1370 |
+
_health(cid, "reconciliation", "failed", failure=recon_failure)
|
| 1371 |
+
_health(cid, "freshness", "failed",
|
| 1372 |
+
failure={**recon_failure, "stage": "freshness"})
|
| 1373 |
+
elif not failure:
|
| 1374 |
+
_health(cid, "reconciliation", "ready")
|
| 1375 |
+
_health(cid, "freshness", "ready")
|
| 1376 |
+
safe_errors = []
|
| 1377 |
+
if failure:
|
| 1378 |
+
safe_errors.append(f"{failure['code']}: {failure['message']}")
|
| 1379 |
+
if rebuild_errors:
|
| 1380 |
+
safe_errors.append("reconciliation_failed: Odoo reconciliation did not complete.")
|
| 1381 |
+
_ut.record_odoo_sync_status(
|
| 1382 |
+
phase="error" if safe_errors else "ok", errors=safe_errors,
|
| 1383 |
+
st=_rt, connection_id=cid)
|
| 1384 |
meta_state = _rebuild_meta_relational(
|
| 1385 |
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
|
| 1386 |
if isinstance(meta_state_out, dict):
|
|
|
|
| 1392 |
# The isolated cells/rollups/meta handlers above are deliberately allowed to report and
|
| 1393 |
# continue. This return value answers the narrower resync-policy question: did the Odoo
|
| 1394 |
# table refresh itself complete, so a confirmed zero-change pass may be skipped?
|
| 1395 |
+
return not any_connection_failed and not rebuild_errors
|
| 1396 |
except Exception as e: # noqa: BLE001
|
| 1397 |
# ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only
|
| 1398 |
# `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a
|
|
|
|
| 1402 |
try:
|
| 1403 |
if '_rt' in locals():
|
| 1404 |
from core import user_tables as _ut
|
| 1405 |
+
failure = routes_connectors._odoo_failure(e, "materialization")
|
| 1406 |
+
cid = locals().get("default_cid") or "default"
|
| 1407 |
+
routes_connectors.record_odoo_stage(
|
| 1408 |
+
_rt, cid, failure["stage"], "failed", failure=failure,
|
| 1409 |
+
cadence_seconds=cadence_seconds)
|
| 1410 |
+
_ut.record_odoo_sync_status(
|
| 1411 |
+
phase="error", errors=[f"{failure['code']}: {failure['message']}"],
|
| 1412 |
+
st=_rt, connection_id=cid)
|
| 1413 |
except Exception:
|
| 1414 |
pass
|
| 1415 |
return False
|
|
|
|
| 1489 |
"disagreement": None,
|
| 1490 |
"passes": 0,
|
| 1491 |
"lastPassAt": None,
|
| 1492 |
+
"lastSuccessAt": None,
|
| 1493 |
"lastPassOk": None,
|
| 1494 |
"lastError": None,
|
| 1495 |
+
"connectionId": "",
|
| 1496 |
"consecutiveFailures": 0,
|
| 1497 |
"bootSyncError": None,
|
| 1498 |
}
|
|
|
|
| 1661 |
⚠ NOT the whole of D-314: naming the offending TABLE and ROW ID belongs to the DuckDB writer
|
| 1662 |
that raised, which is not this loop.
|
| 1663 |
"""
|
| 1664 |
+
import ssl as _ssl
|
| 1665 |
text = f"{type(exc).__name__}: {exc}"
|
| 1666 |
low = text.lower()
|
| 1667 |
+
tls = (isinstance(exc, _ssl.SSLCertVerificationError)
|
| 1668 |
+
or "certificate_verify_failed" in low
|
| 1669 |
+
or "certificate verify failed" in low
|
| 1670 |
+
or "certificate has expired" in low
|
| 1671 |
+
or "self signed certificate" in low)
|
| 1672 |
persistent = any(t in low for t in
|
| 1673 |
("duplicate key", "primary key", "unique constraint", "constraint error",
|
| 1674 |
"binder", "conversion error"))
|
| 1675 |
+
code = ("tls_certificate_invalid" if tls else
|
| 1676 |
+
"mirror_integrity_failure" if persistent else "mirror_pull_failed")
|
| 1677 |
+
message = ("Odoo TLS certificate verification failed because the certificate is expired or "
|
| 1678 |
+
"untrusted. No source data was accepted." if tls else
|
| 1679 |
+
"The analytical mirror failed an integrity check and was not marked fresh."
|
| 1680 |
+
if persistent else
|
| 1681 |
+
"The Odoo mirror pull failed and freshness was not advanced.")
|
| 1682 |
+
return {"stage": "source" if tls else "mirrorPull", "code": code,
|
| 1683 |
+
"type": type(exc).__name__, "message": message,
|
| 1684 |
+
"kind": "persistent" if persistent or tls else "transient",
|
| 1685 |
"meaning": ("this failure repeats until someone repairs the mirror; retrying the same "
|
| 1686 |
+
"pass will not clear it" if persistent or tls else
|
| 1687 |
"this failure usually clears on the next pass")}
|
| 1688 |
|
| 1689 |
|
|
|
|
| 1751 |
"tenantSeconds": _cad["tenantSeconds"], "envSeconds": _cad["envSeconds"],
|
| 1752 |
"resolvedSeconds": _every, "manual": _manual,
|
| 1753 |
"disagreement": _cad["disagreement"]})
|
| 1754 |
+
try:
|
| 1755 |
+
from harness import runtime as _health_runtime
|
| 1756 |
+
import odoo_relational as _health_rel
|
| 1757 |
+
RESYNC_STATE["connectionId"] = str(
|
| 1758 |
+
_health_rel.default_connection_id(
|
| 1759 |
+
_health_runtime.get_runtime(_cad["tenant"])) or "default")
|
| 1760 |
+
except Exception: # noqa: BLE001
|
| 1761 |
+
RESYNC_STATE["connectionId"] = "default"
|
| 1762 |
# ⚠ ON CHANGE, NOT EVERY CYCLE. An hourly warning is a skimmed warning.
|
| 1763 |
if _cad["disagreement"] != _RESYNC_NOTICE[0]:
|
| 1764 |
_RESYNC_NOTICE[0] = _cad["disagreement"]
|
|
|
|
| 1782 |
RESYNC_STATE["lastError"] = _err
|
| 1783 |
if _err is None:
|
| 1784 |
RESYNC_STATE["consecutiveFailures"] = 0
|
| 1785 |
+
if not _manual:
|
| 1786 |
+
RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
|
| 1787 |
else:
|
| 1788 |
RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
|
| 1789 |
# D-314: the TYPE and the persistent/transient reading, both in the line and in the
|
|
|
|
| 1837 |
_meta_state = {}
|
| 1838 |
if _reason is not None:
|
| 1839 |
print(f"[aios-api] odoo relational rebuild required ({_reason})")
|
| 1840 |
+
# Test/recovery harnesses replace the rebuild seam with its historical three-argument
|
| 1841 |
+
# callable. Bind tenant/cadence whenever the active implementation accepts them,
|
| 1842 |
+
# without turning an internal TypeError into a second materialization attempt.
|
| 1843 |
+
import inspect as _inspect
|
| 1844 |
+
_rebuild_params = _inspect.signature(_rebuild_odoo_relational).parameters
|
| 1845 |
+
_rebuild_variadic = any(
|
| 1846 |
+
p.kind == _inspect.Parameter.VAR_KEYWORD for p in _rebuild_params.values())
|
| 1847 |
+
_rebuild_scope = ({"tenant": _cad["tenant"], "cadence_seconds": _every}
|
| 1848 |
+
if _rebuild_variadic or "tenant" in _rebuild_params else {})
|
| 1849 |
+
if _rebuild_odoo_relational(
|
| 1850 |
+
"resync", _meta_previous, _meta_state, **_rebuild_scope):
|
| 1851 |
last_relational_success_at = _t.time()
|
| 1852 |
else:
|
| 1853 |
# A prior successful projection no longer proves the changed mirror was applied.
|
|
|
|
| 2025 |
# ⭐⭐ W50-T29 / R8 — AND THE DEPLOYMENT NOW OPTS IN, ON STAGING AND ON LIVE. The owner's ruling is
|
| 2026 |
# that the Space resyncs its own mirror hourly rather than waiting for someone to visit; without
|
| 2027 |
# it a Space serves whatever the seed was cut with, forever, and August reads zero while Odoo has
|
| 2028 |
+
# 723 orders. `deploy_web.py` pushes `AIOS_STORE_RESYNC=1` plus `AIOS_STORE_RESYNC_SECONDS=14400`.
|
| 2029 |
#
|
| 2030 |
# ⛔ THE OLD REASON FOR KEEPING IT OFF WAS "so a Pg-backed Live Space can scale its Neon compute to
|
| 2031 |
# zero between real requests", AND THAT REASON IS ONLY HALF SATISFIED. `_resync_mirror_pass` is
|
|
|
|
| 2034 |
# `_rebuild_meta_relational` write through `core.user_tables`, which is the tenant product store,
|
| 2035 |
# and under `STORE_BACKEND=pg` that is Neon. Named here rather than left for the next reader to
|
| 2036 |
# discover from a bill, and carried as a register row by this ticket's handoff.
|
| 2037 |
+
#
|
| 2038 |
+
# ⛔⛔ AND IT WAS LEFT FOR THE NEXT READER TO DISCOVER FROM A BILL — 2026-09-21, THIRTY-ONE DAYS
|
| 2039 |
+
# LATER. The register row never became work, and the paragraph above was the only thing in the
|
| 2040 |
+
# codebase that knew. September's Neon transfer reached 823.47 GB against a 500 GB allowance by
|
| 2041 |
+
# day 21 (compute 165 of 300 CU-hours, storage 96 MB of 10 GiB — both fine), which is an
|
| 2042 |
+
# over-$50 bill on a $19 plan, pacing to ~$87. The two Spaces' own run logs showed 61 of 61
|
| 2043 |
+
# hourly passes on live and two loops an hour on staging: 72 full rebuilds a day at ~545 MB each,
|
| 2044 |
+
# because `core.store_pg.update()` is a whole-JSONB-document read-modify-write and one pass makes
|
| 2045 |
+
# about ten of them against tenant #0's ~28.6 MB document.
|
| 2046 |
+
#
|
| 2047 |
+
# ⭐ THE HOTFIX MOVED THE CADENCE ONLY, 3600 -> 14400 (R11's existing four-hourly preset), which
|
| 2048 |
+
# is safe at any value because the wait below is `sync_wake_event().wait(_every)` and a Sync now
|
| 2049 |
+
# SETS it. TWO STRUCTURAL CAUSES REMAIN OPEN AS REGISTER ROWS, and neither is a hotfix:
|
| 2050 |
+
# * `_resync_rebuild_reason` treats `pulled > 0` as a change, but `pulled` counts rows FETCHED,
|
| 2051 |
+
# not rows CHANGED — live rebuilt on passes its own report scored `added/updated/removed: 0`.
|
| 2052 |
+
# Narrowing it is what would make an hourly cadence nearly free.
|
| 2053 |
+
# * the whole-document store write itself. A partial-JSONB write is a wave.
|
| 2054 |
+
# ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
|
| 2055 |
if os.environ.get("AIOS_STORE_RESYNC") == "1":
|
| 2056 |
import threading as _threading_resync
|
| 2057 |
_threading_resync.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
|
api/odoo_relational.py
CHANGED
|
@@ -1546,13 +1546,15 @@ def read_customers(cur, excluded=None):
|
|
| 1546 |
The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale
|
| 1547 |
order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer.
|
| 1548 |
|
| 1549 |
-
⭐⭐ THE THIRD IS `customer_rank > 0
|
|
|
|
| 1550 |
the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old
|
| 1551 |
docstring said partners with no document are *"left out on purpose — a row that can never
|
| 1552 |
appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same
|
| 1553 |
join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records**
|
| 1554 |
(MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer
|
| 1555 |
-
a salesperson has not sold to yet is exactly the row a prospecting view needs.
|
|
|
|
| 1556 |
|
| 1557 |
⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the
|
| 1558 |
document legs for the rank leg would drop **16 partners that hold posted documents** (7
|
|
@@ -1585,18 +1587,18 @@ def read_customers(cur, excluded=None):
|
|
| 1585 |
agent = ("ag.name" if "agent_id" in have else "NULL")
|
| 1586 |
agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL")
|
| 1587 |
join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "")
|
| 1588 |
-
#
|
| 1589 |
-
#
|
| 1590 |
-
#
|
| 1591 |
-
rank_leg = (" OR (p.customer_rank > 0
|
| 1592 |
-
if {"customer_rank", "active"} <= have else "")
|
| 1593 |
# ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like
|
| 1594 |
# every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL
|
| 1595 |
# to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the
|
| 1596 |
# live 500 that rule was written after).
|
| 1597 |
sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
|
| 1598 |
f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, "
|
| 1599 |
-
f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')} "
|
|
|
|
| 1600 |
"FROM res_partner p "
|
| 1601 |
f"{join}"
|
| 1602 |
"WHERE p.id IN ("
|
|
@@ -1607,7 +1609,7 @@ def read_customers(cur, excluded=None):
|
|
| 1607 |
f"{rank_leg}")
|
| 1608 |
out = []
|
| 1609 |
for r in cur.execute(sql).fetchall():
|
| 1610 |
-
(pid, name, city, state, country, agent, agent_id, street, street2, zipc) = r
|
| 1611 |
out.append({
|
| 1612 |
"_id": str(pid),
|
| 1613 |
"customer": str(name or ""),
|
|
@@ -1620,6 +1622,7 @@ def read_customers(cur, excluded=None):
|
|
| 1620 |
"country": str(country or ""),
|
| 1621 |
"agent": str(agent or ""),
|
| 1622 |
AGENT_JOIN_KEY: int(agent_id) if agent_id else "",
|
|
|
|
| 1623 |
"wholesale_scope": _in_scope(pid, excluded),
|
| 1624 |
})
|
| 1625 |
return out
|
|
@@ -2771,6 +2774,292 @@ TABLE_SUBJECTS = {
|
|
| 2771 |
GL_LINES_KEY: "odoo:account.move.line",
|
| 2772 |
}
|
| 2773 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2774 |
#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS.
|
| 2775 |
#:
|
| 2776 |
#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the
|
|
@@ -3999,6 +4288,119 @@ def available(rt=None, tenant_key=None, connection_id=None):
|
|
| 3999 |
return False
|
| 4000 |
|
| 4001 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4002 |
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
|
| 4003 |
all_connections=False, table_keys=None, on_apply=None):
|
| 4004 |
"""THE entry point — the store-resync path and the route both call this.
|
|
@@ -4105,18 +4507,36 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
|
|
| 4105 |
if table_key_for(key, ns) in requested}
|
| 4106 |
skipped = sorted(table_key_for(key, ns)
|
| 4107 |
for bucket, key, _l, _f in TABLES if bucket not in keep)
|
| 4108 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4109 |
# ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing.
|
| 4110 |
# Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no
|
| 4111 |
# longer built) SAYS SO, with the keys.
|
| 4112 |
-
plan_report = {}
|
| 4113 |
-
if on_apply is not None:
|
| 4114 |
-
try:
|
| 4115 |
-
on_apply()
|
| 4116 |
-
except Exception: # noqa: BLE001
|
| 4117 |
-
pass # Optional status reporting cannot fail a valid plan's publication.
|
| 4118 |
-
written = apply_plan(rt, built, username=username, today=today, report=plan_report,
|
| 4119 |
-
connection_id=ns, retire=requested is None)
|
| 4120 |
# The fixed mirror roster above and the discovered long tail deliberately use separate
|
| 4121 |
# materialisers. Only an explicit long-tail enablement reaches this read-only path; disabled
|
| 4122 |
# databases are not passed an empty set and therefore cannot be erased by reconciliation.
|
|
@@ -4131,7 +4551,9 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
|
|
| 4131 |
# FROZEN workspace returned before reaching here, so a disconnected connector is not
|
| 4132 |
# re-recorded. See `_refresh_declared_schema` for why it writes the declared half only.
|
| 4133 |
declared_report = _refresh_declared_schema(rt) if requested is None else None
|
| 4134 |
-
out = {"tables": written,
|
|
|
|
|
|
|
| 4135 |
**({"schemaDeclared": declared_report} if declared_report else {}),
|
| 4136 |
**({"discovered": discovered_report}
|
| 4137 |
if discovered_report and (discovered_report.get("tables")
|
|
@@ -4143,6 +4565,8 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
|
|
| 4143 |
# ⚠ THE DEFAULT CONNECTION'S RETURN SHAPE DOES NOT MOVE BY ONE KEY. The two additions below
|
| 4144 |
# are reachable only on the namespaced path, because `verify_odoo_relational` asserts on this
|
| 4145 |
# dict as it stands for the default.
|
|
|
|
|
|
|
| 4146 |
if ns is not None:
|
| 4147 |
out["connection"] = cid
|
| 4148 |
if built.get("limits"):
|
|
@@ -4191,7 +4615,7 @@ def refresh_all(rt, tenant, username="automation", cur=None, today=None):
|
|
| 4191 |
cur=(cur if named is None else None),
|
| 4192 |
today=today, connection_id=named)
|
| 4193 |
except Exception as exc: # noqa: BLE001
|
| 4194 |
-
one = {"tables": {}, "failed": f"{type(exc).__name__}:
|
| 4195 |
reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
|
| 4196 |
if named is None and primary is None:
|
| 4197 |
primary = one
|
|
|
|
| 1546 |
The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale
|
| 1547 |
order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer.
|
| 1548 |
|
| 1549 |
+
⭐⭐ THE THIRD IS `customer_rank > 0` across both active states (wave 29, item 22 / R12
|
| 1550 |
+
via finding F2, widened by S63-T14 —
|
| 1551 |
the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old
|
| 1552 |
docstring said partners with no document are *"left out on purpose — a row that can never
|
| 1553 |
appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same
|
| 1554 |
join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records**
|
| 1555 |
(MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer
|
| 1556 |
+
a salesperson has not sold to yet is exactly the row a prospecting view needs. Archived is a
|
| 1557 |
+
projected boolean; the default View hides it with an ordinary removable condition.
|
| 1558 |
|
| 1559 |
⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the
|
| 1560 |
document legs for the rank leg would drop **16 partners that hold posted documents** (7
|
|
|
|
| 1587 |
agent = ("ag.name" if "agent_id" in have else "NULL")
|
| 1588 |
agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL")
|
| 1589 |
join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "")
|
| 1590 |
+
# The rank leg intentionally includes active and archived accounts. An absent rank column
|
| 1591 |
+
# still drops the leg whole; an absent active column keeps row arity with a conservative live
|
| 1592 |
+
# default while the health gate reports the schema gap separately.
|
| 1593 |
+
rank_leg = (" OR (p.customer_rank > 0) " if "customer_rank" in have else "")
|
|
|
|
| 1594 |
# ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like
|
| 1595 |
# every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL
|
| 1596 |
# to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the
|
| 1597 |
# live 500 that rule was written after).
|
| 1598 |
sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
|
| 1599 |
f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, "
|
| 1600 |
+
f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')}, "
|
| 1601 |
+
f" {_col(have, 'p.active', 'TRUE')} "
|
| 1602 |
"FROM res_partner p "
|
| 1603 |
f"{join}"
|
| 1604 |
"WHERE p.id IN ("
|
|
|
|
| 1609 |
f"{rank_leg}")
|
| 1610 |
out = []
|
| 1611 |
for r in cur.execute(sql).fetchall():
|
| 1612 |
+
(pid, name, city, state, country, agent, agent_id, street, street2, zipc, active) = r
|
| 1613 |
out.append({
|
| 1614 |
"_id": str(pid),
|
| 1615 |
"customer": str(name or ""),
|
|
|
|
| 1622 |
"country": str(country or ""),
|
| 1623 |
"agent": str(agent or ""),
|
| 1624 |
AGENT_JOIN_KEY: int(agent_id) if agent_id else "",
|
| 1625 |
+
"archived": not bool(active),
|
| 1626 |
"wholesale_scope": _in_scope(pid, excluded),
|
| 1627 |
})
|
| 1628 |
return out
|
|
|
|
| 2774 |
GL_LINES_KEY: "odoo:account.move.line",
|
| 2775 |
}
|
| 2776 |
|
| 2777 |
+
# One receipt identity per curated database. The model is not enough: invoices/bills share
|
| 2778 |
+
# ``account.move`` and agents/vendors/customers share ``res.partner`` while presenting different
|
| 2779 |
+
# governed populations. These keys are public metadata only; no source row enters a receipt.
|
| 2780 |
+
TABLE_TOPICS = {
|
| 2781 |
+
INVOICES_KEY: "odoo_invoices",
|
| 2782 |
+
ORDERS_KEY: "odoo_orders",
|
| 2783 |
+
AGENTS_KEY: "odoo_agents",
|
| 2784 |
+
ACCOUNTS_KEY: "odoo_accounts",
|
| 2785 |
+
VENDORS_KEY: "odoo_vendors",
|
| 2786 |
+
BILLS_KEY: "odoo_bills",
|
| 2787 |
+
ORDER_LINES_KEY: "sales_lines",
|
| 2788 |
+
GL_LINES_KEY: "gl_lines",
|
| 2789 |
+
}
|
| 2790 |
+
|
| 2791 |
+
_SOURCE_COUNT_SPECS = {
|
| 2792 |
+
"invoices": {"model": "account.move", "mode": "count", "domain": [
|
| 2793 |
+
("state", "=", "posted"), ("move_type", "in", ["out_invoice", "out_refund"]),
|
| 2794 |
+
("partner_id", "!=", False)]},
|
| 2795 |
+
"orders": {"model": "sale.order", "mode": "count", "domain": [
|
| 2796 |
+
("state", "in", ["sale", "done"]), ("partner_id", "!=", False)]},
|
| 2797 |
+
"agents": {"model": "res.partner", "mode": "agent_union", "domain": [
|
| 2798 |
+
("agent", "=", True)]},
|
| 2799 |
+
"accounts": {"model": "account.account", "mode": "count", "domain": []},
|
| 2800 |
+
"vendors": {"model": "account.move", "mode": "distinct_partner", "domain": [
|
| 2801 |
+
("state", "=", "posted"), ("move_type", "in", ["in_invoice", "in_refund"]),
|
| 2802 |
+
("partner_id", "!=", False)]},
|
| 2803 |
+
"bills": {"model": "account.move", "mode": "count", "domain": [
|
| 2804 |
+
("state", "=", "posted"), ("move_type", "in", ["in_invoice", "in_refund"]),
|
| 2805 |
+
("partner_id", "!=", False)]},
|
| 2806 |
+
"order_lines": {"model": "sale.order.line", "mode": "count", "domain": [
|
| 2807 |
+
("order_id.state", "in", ["sale", "done"])]},
|
| 2808 |
+
"gl_lines": {"model": "account.move.line", "mode": "count", "domain": []},
|
| 2809 |
+
}
|
| 2810 |
+
|
| 2811 |
+
# The two canonical registry databases replaced the retired ``ut_odoo_customers`` and
|
| 2812 |
+
# ``ut_odoo_products`` twins. They still need an identity receipt after every full default-
|
| 2813 |
+
# connection sync; otherwise the eight curated-table receipts can all be green while either of
|
| 2814 |
+
# the two databases people actually use is short. These are public schema/domain facts only.
|
| 2815 |
+
CANONICAL_RECEIPT_SPECS = {
|
| 2816 |
+
"customer_data": {
|
| 2817 |
+
"topicKey": "odoo_customers", "domainKey": "odoo:res.partner",
|
| 2818 |
+
"model": "res.partner", "selection": "customer eligibility union",
|
| 2819 |
+
},
|
| 2820 |
+
"product_data": {
|
| 2821 |
+
"topicKey": "odoo_products", "domainKey": "odoo:product.product",
|
| 2822 |
+
"model": "product.product", "selection": "active catalogue by business key",
|
| 2823 |
+
},
|
| 2824 |
+
}
|
| 2825 |
+
|
| 2826 |
+
|
| 2827 |
+
def _relation_id(value):
|
| 2828 |
+
"""Return the id from an Odoo many2one/group value without retaining its label."""
|
| 2829 |
+
if isinstance(value, (list, tuple)) and value:
|
| 2830 |
+
value = value[0]
|
| 2831 |
+
try:
|
| 2832 |
+
return int(value) if value not in (None, False, "") else None
|
| 2833 |
+
except (TypeError, ValueError):
|
| 2834 |
+
return None
|
| 2835 |
+
|
| 2836 |
+
|
| 2837 |
+
def _source_domain_count(client, bucket):
|
| 2838 |
+
"""Exact source population for one curated database domain, plus its counting method."""
|
| 2839 |
+
spec = _SOURCE_COUNT_SPECS[bucket]
|
| 2840 |
+
if spec["mode"] == "count":
|
| 2841 |
+
return int(client.search_count(spec["model"], spec["domain"])), "search_count"
|
| 2842 |
+
if spec["mode"] == "distinct_partner":
|
| 2843 |
+
groups = client.read_group(spec["model"], spec["domain"], ["partner_id"],
|
| 2844 |
+
["partner_id"], lazy=False)
|
| 2845 |
+
ids = {_relation_id(row.get("partner_id")) for row in groups}
|
| 2846 |
+
ids.discard(None)
|
| 2847 |
+
return len(ids), "distinct_partner"
|
| 2848 |
+
if spec["mode"] == "agent_union":
|
| 2849 |
+
flagged = client.search_read("res.partner", spec["domain"], ["id"], limit=100000)
|
| 2850 |
+
commissioned = client.read_group(
|
| 2851 |
+
"account.invoice.line.agent", [("agent_id", "!=", False)], ["agent_id"],
|
| 2852 |
+
["agent_id"], lazy=False)
|
| 2853 |
+
ids = {_relation_id(row.get("id")) for row in flagged}
|
| 2854 |
+
ids.update(_relation_id(row.get("agent_id")) for row in commissioned)
|
| 2855 |
+
ids.discard(None)
|
| 2856 |
+
return len(ids), "union_distinct_agent"
|
| 2857 |
+
raise Refused("the database source-count mode is not supported")
|
| 2858 |
+
|
| 2859 |
+
|
| 2860 |
+
def _group_identity_ids(client, model, domain, field):
|
| 2861 |
+
"""Return a read-only grouped identity set without retaining relation labels."""
|
| 2862 |
+
groups = client.read_group(model, domain, [field], [field], lazy=False)
|
| 2863 |
+
ids = {_relation_id(row.get(field)) for row in groups}
|
| 2864 |
+
ids.discard(None)
|
| 2865 |
+
return ids
|
| 2866 |
+
|
| 2867 |
+
|
| 2868 |
+
def _canonical_source_population(client, database_key):
|
| 2869 |
+
"""Return the exact source identities used by a canonical registry database.
|
| 2870 |
+
|
| 2871 |
+
Customer eligibility is deliberately a union: rank-positive partners in either active state,
|
| 2872 |
+
confirmed-order partners, and posted customer-document partners. Products are the active
|
| 2873 |
+
catalogue. Every operation is a verified read method; a capped read is refused rather than
|
| 2874 |
+
reported as a complete population.
|
| 2875 |
+
"""
|
| 2876 |
+
if database_key == "customer_data":
|
| 2877 |
+
rank_domain = [("customer_rank", ">", 0), ("active", "in", [True, False])]
|
| 2878 |
+
ranked = client.search_read("res.partner", rank_domain, ["id"], limit=200000)
|
| 2879 |
+
ranked_count = int(client.search_count("res.partner", rank_domain))
|
| 2880 |
+
if len(ranked) != ranked_count:
|
| 2881 |
+
raise Refused("the customer identity read was incomplete")
|
| 2882 |
+
ids = {_relation_id(row.get("id")) for row in ranked}
|
| 2883 |
+
ids.update(_group_identity_ids(
|
| 2884 |
+
client, "sale.order",
|
| 2885 |
+
[("state", "in", ["sale", "done"]), ("partner_id", "!=", False)],
|
| 2886 |
+
"partner_id"))
|
| 2887 |
+
ids.update(_group_identity_ids(
|
| 2888 |
+
client, "account.move",
|
| 2889 |
+
[("state", "=", "posted"),
|
| 2890 |
+
("move_type", "in", ["out_invoice", "out_refund"]),
|
| 2891 |
+
("partner_id", "!=", False)], "partner_id"))
|
| 2892 |
+
ids.discard(None)
|
| 2893 |
+
return [{"id": value} for value in sorted(ids)]
|
| 2894 |
+
if database_key == "product_data":
|
| 2895 |
+
domain = [("active", "=", True)]
|
| 2896 |
+
rows = client.search_read(
|
| 2897 |
+
"product.product", domain, ["id", "default_code"], limit=100000)
|
| 2898 |
+
count = int(client.search_count("product.product", domain))
|
| 2899 |
+
if len(rows) != count:
|
| 2900 |
+
raise Refused("the product identity read was incomplete")
|
| 2901 |
+
return [{"id": _relation_id(row.get("id")),
|
| 2902 |
+
"code": str(row.get("default_code") or "")}
|
| 2903 |
+
for row in rows]
|
| 2904 |
+
raise Refused("the canonical database identity is not supported")
|
| 2905 |
+
|
| 2906 |
+
|
| 2907 |
+
def _identity_counts(values):
|
| 2908 |
+
"""Return a normalized identity list, set, and duplicate count."""
|
| 2909 |
+
normalized = [value for value in values if value not in (None, False, "")]
|
| 2910 |
+
return normalized, set(normalized), len(normalized) - len(set(normalized))
|
| 2911 |
+
|
| 2912 |
+
|
| 2913 |
+
def canonical_reconciliation_receipts(cur, source_client=None, captured_at=None):
|
| 2914 |
+
"""Data-free source -> mirror -> canonical receipts for Customers and Products.
|
| 2915 |
+
|
| 2916 |
+
The canonical grids are compiled registry databases, not ``ut_*`` tables. Customer rows keep
|
| 2917 |
+
the Odoo partner id. Product rows are SKU-grained, so multiple active product records sharing
|
| 2918 |
+
one business key are an explained merge rather than an unexplained missing row. Only counts
|
| 2919 |
+
and public schema/domain metadata leave this function; source ids and SKU values never do.
|
| 2920 |
+
"""
|
| 2921 |
+
captured = str(captured_at or _dt.datetime.now(_dt.timezone.utc).isoformat())
|
| 2922 |
+
receipts = {}
|
| 2923 |
+
for database_key, spec in CANONICAL_RECEIPT_SPECS.items():
|
| 2924 |
+
identity = {
|
| 2925 |
+
"databaseKey": database_key, "schemaKey": database_key,
|
| 2926 |
+
"topicKey": spec["topicKey"], "domainKey": spec["domainKey"],
|
| 2927 |
+
"model": spec["model"],
|
| 2928 |
+
"selection": {"rule": spec["selection"], "enabled": True},
|
| 2929 |
+
}
|
| 2930 |
+
source_rows, source_failure = None, None
|
| 2931 |
+
if source_client is not None:
|
| 2932 |
+
try:
|
| 2933 |
+
source_rows = _canonical_source_population(source_client, database_key)
|
| 2934 |
+
except Exception as exc: # noqa: BLE001
|
| 2935 |
+
source_failure = {
|
| 2936 |
+
"code": "source_reconciliation_failed", "type": type(exc).__name__,
|
| 2937 |
+
"message": "This canonical Odoo population could not be reconciled.",
|
| 2938 |
+
}
|
| 2939 |
+
try:
|
| 2940 |
+
if database_key == "customer_data":
|
| 2941 |
+
mirror_rows = read_customers(cur)
|
| 2942 |
+
mirror_values = [row.get(JOIN_KEY) for row in mirror_rows]
|
| 2943 |
+
mirror_canonical_values = list(mirror_values)
|
| 2944 |
+
if source_client is not None:
|
| 2945 |
+
import modules.customer_data as customer_data
|
| 2946 |
+
# This is the same identity evaluator consumed by ``_pool_build``. Reading
|
| 2947 |
+
# the mirror again here would let a dropped canonical row certify itself.
|
| 2948 |
+
canonical_values = list(customer_data.identity_pids())
|
| 2949 |
+
else:
|
| 2950 |
+
canonical_values = None
|
| 2951 |
+
source_values = ([row.get("id") for row in source_rows]
|
| 2952 |
+
if source_rows is not None else None)
|
| 2953 |
+
source_canonical_values = source_values
|
| 2954 |
+
explained_merges = 0
|
| 2955 |
+
else:
|
| 2956 |
+
mirror_rows = [row for row in read_products(cur) if row.get("active") == "1"]
|
| 2957 |
+
mirror_values = [row.get(PRODUCT_JOIN_KEY) for row in mirror_rows]
|
| 2958 |
+
# product_data is one row per business key. An uncoded product has a stable,
|
| 2959 |
+
# per-record fallback, matching modules.products.catalogue().
|
| 2960 |
+
mirror_canonical_values = [
|
| 2961 |
+
(str(row.get("code") or "").strip()
|
| 2962 |
+
or f"pid:{row.get(PRODUCT_JOIN_KEY)}") for row in mirror_rows]
|
| 2963 |
+
if source_client is not None:
|
| 2964 |
+
import modules.product_data as product_data
|
| 2965 |
+
# ``identity_rows`` uses the catalogue seam and collision guard that the
|
| 2966 |
+
# shipped product pool consumes. It intentionally omits expensive cells.
|
| 2967 |
+
canonical_values = [row.get("code") for row in product_data.identity_rows()]
|
| 2968 |
+
else:
|
| 2969 |
+
canonical_values = None
|
| 2970 |
+
source_values = ([row.get("id") for row in source_rows]
|
| 2971 |
+
if source_rows is not None else None)
|
| 2972 |
+
source_canonical_values = ([
|
| 2973 |
+
(str(row.get("code") or "").strip() or f"pid:{row.get('id')}")
|
| 2974 |
+
for row in source_rows] if source_rows is not None else None)
|
| 2975 |
+
explained_merges = (len(mirror_canonical_values)
|
| 2976 |
+
- len(set(mirror_canonical_values)))
|
| 2977 |
+
|
| 2978 |
+
mirror_list, mirror_set, mirror_duplicates = _identity_counts(mirror_values)
|
| 2979 |
+
_mirror_canonical_list, mirror_canonical_set, mirror_key_duplicates = (
|
| 2980 |
+
_identity_counts(mirror_canonical_values))
|
| 2981 |
+
if canonical_values is None:
|
| 2982 |
+
canonical_list, canonical_set, canonical_duplicates = None, None, None
|
| 2983 |
+
else:
|
| 2984 |
+
canonical_list, canonical_set, canonical_duplicates = _identity_counts(
|
| 2985 |
+
canonical_values)
|
| 2986 |
+
if source_values is None:
|
| 2987 |
+
source_list, source_set, source_duplicates = None, None, None
|
| 2988 |
+
source_to_mirror = "unconfirmed"
|
| 2989 |
+
missing, extra = None, None
|
| 2990 |
+
canonical_missing, canonical_extra = None, None
|
| 2991 |
+
else:
|
| 2992 |
+
source_list, source_set, source_duplicates = _identity_counts(source_values)
|
| 2993 |
+
missing = len(source_set - mirror_set)
|
| 2994 |
+
extra = len(mirror_set - source_set)
|
| 2995 |
+
_source_canonical_list, source_canonical_set, _source_canonical_duplicates = (
|
| 2996 |
+
_identity_counts(source_canonical_values))
|
| 2997 |
+
canonical_missing = (len(source_canonical_set - canonical_set)
|
| 2998 |
+
if canonical_set is not None else None)
|
| 2999 |
+
canonical_extra = (len(canonical_set - source_canonical_set)
|
| 3000 |
+
if canonical_set is not None else None)
|
| 3001 |
+
source_to_mirror = (
|
| 3002 |
+
"matched" if not missing and not extra and not source_duplicates
|
| 3003 |
+
and not mirror_duplicates else "mismatch")
|
| 3004 |
+
if canonical_set is None:
|
| 3005 |
+
mirror_to_canonical = "unconfirmed"
|
| 3006 |
+
elif (mirror_canonical_set == canonical_set and not mirror_duplicates
|
| 3007 |
+
and not canonical_duplicates):
|
| 3008 |
+
mirror_to_canonical = (
|
| 3009 |
+
"matched_with_explained_merges"
|
| 3010 |
+
if database_key == "product_data" and explained_merges else "matched")
|
| 3011 |
+
else:
|
| 3012 |
+
mirror_to_canonical = "mismatch"
|
| 3013 |
+
# ID and canonical-key gaps are two views of the same records, so use the larger
|
| 3014 |
+
# layer gap rather than double-counting one absent row twice.
|
| 3015 |
+
unexplained = (max((missing or 0) + (extra or 0),
|
| 3016 |
+
(canonical_missing or 0) + (canonical_extra or 0))
|
| 3017 |
+
+ (source_duplicates or 0)
|
| 3018 |
+
+ mirror_duplicates
|
| 3019 |
+
+ (canonical_duplicates or 0)
|
| 3020 |
+
+ (0 if database_key == "product_data"
|
| 3021 |
+
else mirror_key_duplicates))
|
| 3022 |
+
partial = (source_to_mirror != "matched" or mirror_to_canonical not in {
|
| 3023 |
+
"matched", "matched_with_explained_merges"}
|
| 3024 |
+
or unexplained > 0 or source_failure is not None)
|
| 3025 |
+
receipts[database_key] = {
|
| 3026 |
+
**identity, "state": "partial" if partial else "succeeded",
|
| 3027 |
+
"sourceCount": len(source_list) if source_list is not None else None,
|
| 3028 |
+
"sourceCountMode": "identity_set" if source_list is not None else "unavailable",
|
| 3029 |
+
"sourceReconciliation": source_to_mirror,
|
| 3030 |
+
"mirrorCount": len(mirror_list), "materializedCount": None,
|
| 3031 |
+
"canonicalCount": (len(canonical_set) if canonical_set is not None else None),
|
| 3032 |
+
"servingCount": (len(canonical_set) if canonical_set is not None else None),
|
| 3033 |
+
"readThrough": True,
|
| 3034 |
+
"duplicateCount": canonical_duplicates,
|
| 3035 |
+
"explainedMergeCount": explained_merges,
|
| 3036 |
+
"unexplainedMissingCount": missing,
|
| 3037 |
+
"unexplainedExtraCount": extra,
|
| 3038 |
+
"unexplainedCanonicalMissingCount": canonical_missing,
|
| 3039 |
+
"unexplainedCanonicalExtraCount": canonical_extra,
|
| 3040 |
+
"unexplainedCount": unexplained,
|
| 3041 |
+
"reconciliation": {"sourceToMirror": source_to_mirror,
|
| 3042 |
+
"mirrorToCanonical": mirror_to_canonical},
|
| 3043 |
+
"freshness": {"state": "partial" if partial else "current",
|
| 3044 |
+
"capturedAt": captured},
|
| 3045 |
+
**({"sourceError": source_failure} if source_failure else {}),
|
| 3046 |
+
}
|
| 3047 |
+
except Exception as exc: # noqa: BLE001
|
| 3048 |
+
failure = {"code": "canonical_reconciliation_failed", "type": type(exc).__name__,
|
| 3049 |
+
"message": "This canonical Odoo database could not be reconciled."}
|
| 3050 |
+
receipts[database_key] = {
|
| 3051 |
+
**identity, "state": "failed", "sourceCount": None,
|
| 3052 |
+
"sourceCountMode": "unavailable", "sourceReconciliation": "unconfirmed",
|
| 3053 |
+
"mirrorCount": None, "materializedCount": None, "canonicalCount": None,
|
| 3054 |
+
"servingCount": None, "readThrough": True, "duplicateCount": None,
|
| 3055 |
+
"explainedMergeCount": None, "unexplainedMissingCount": None,
|
| 3056 |
+
"unexplainedExtraCount": None, "unexplainedCanonicalMissingCount": None,
|
| 3057 |
+
"unexplainedCanonicalExtraCount": None, "unexplainedCount": None,
|
| 3058 |
+
"freshness": {"state": "failed", "capturedAt": captured}, "error": failure,
|
| 3059 |
+
**({"sourceError": source_failure} if source_failure else {}),
|
| 3060 |
+
}
|
| 3061 |
+
return receipts
|
| 3062 |
+
|
| 3063 |
#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS.
|
| 3064 |
#:
|
| 3065 |
#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the
|
|
|
|
| 4288 |
return False
|
| 4289 |
|
| 4290 |
|
| 4291 |
+
def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
|
| 4292 |
+
today=None, retire=True, on_apply=None, source_client=None):
|
| 4293 |
+
"""Refresh each enabled database independently; retain failed databases unchanged.
|
| 4294 |
+
|
| 4295 |
+
Counts describe the database's own domain, never its shared Odoo model. When a read-only
|
| 4296 |
+
source client is available, each receipt asks Odoo independently for that exact population;
|
| 4297 |
+
a failed source/count or mirror reader marks only that database and cannot stale siblings.
|
| 4298 |
+
"""
|
| 4299 |
+
from copy import deepcopy
|
| 4300 |
+
built, written, errors, receipts, report = {}, {}, {}, {}, {}
|
| 4301 |
+
captured_at = _dt.datetime.now(_dt.timezone.utc).isoformat()
|
| 4302 |
+
for bucket, base_key, _label, _fields in TABLES:
|
| 4303 |
+
if bucket not in buckets:
|
| 4304 |
+
continue
|
| 4305 |
+
key = table_key_for(base_key, connection_id)
|
| 4306 |
+
source_count = None
|
| 4307 |
+
source_mode = "unavailable"
|
| 4308 |
+
source_failure = None
|
| 4309 |
+
if source_client is not None:
|
| 4310 |
+
try:
|
| 4311 |
+
source_count, source_mode = _source_domain_count(source_client, bucket)
|
| 4312 |
+
except Exception as exc: # noqa: BLE001
|
| 4313 |
+
source_failure = {
|
| 4314 |
+
"code": "source_reconciliation_failed", "type": type(exc).__name__,
|
| 4315 |
+
"message": "This Odoo database source population could not be counted.",
|
| 4316 |
+
}
|
| 4317 |
+
identity = {
|
| 4318 |
+
"databaseKey": key,
|
| 4319 |
+
"schemaKey": key,
|
| 4320 |
+
"topicKey": TABLE_TOPICS[base_key],
|
| 4321 |
+
"domainKey": TABLE_SUBJECTS[base_key],
|
| 4322 |
+
"model": ODOO_TABLE_MODELS.get(base_key),
|
| 4323 |
+
"countModel": _SOURCE_COUNT_SPECS[bucket]["model"],
|
| 4324 |
+
"domain": deepcopy(_SOURCE_COUNT_SPECS[bucket]["domain"]),
|
| 4325 |
+
"selection": {"bucket": bucket, "enabled": True},
|
| 4326 |
+
}
|
| 4327 |
+
try:
|
| 4328 |
+
one = plan(cur, rt=rt, connection_id=connection_id, buckets={bucket})
|
| 4329 |
+
rows = one[bucket]
|
| 4330 |
+
ids = [row.get("_id") for row in rows]
|
| 4331 |
+
if len(ids) != len(set(ids)) or any(value is None for value in ids):
|
| 4332 |
+
raise Refused("the database reader returned duplicate or missing record identities")
|
| 4333 |
+
if one.get("problems"):
|
| 4334 |
+
raise Refused("; ".join(one["problems"]))
|
| 4335 |
+
read_through = is_read_through_key(key) or _ut().row_limit(key, st=rt) == 0
|
| 4336 |
+
count = len(rows)
|
| 4337 |
+
if read_through and cur is not None:
|
| 4338 |
+
from routes_odoo_tables import population
|
| 4339 |
+
count = population(key, cur=cur, rt=rt)
|
| 4340 |
+
if count is None:
|
| 4341 |
+
raise Refused("the database has no readable mirror binding")
|
| 4342 |
+
if on_apply is not None:
|
| 4343 |
+
on_apply()
|
| 4344 |
+
# Each callback receives a detached document so even a store double or an
|
| 4345 |
+
# in-memory backend cannot leak a partially failed database update.
|
| 4346 |
+
class DetachedRuntime:
|
| 4347 |
+
def __getattr__(self, name):
|
| 4348 |
+
return getattr(rt, name)
|
| 4349 |
+
|
| 4350 |
+
def update(self, name, fn, **kwargs):
|
| 4351 |
+
return rt.update(name, lambda value: fn(deepcopy(value)), **kwargs)
|
| 4352 |
+
|
| 4353 |
+
result = apply_plan(DetachedRuntime(), one, username=username, today=today,
|
| 4354 |
+
connection_id=connection_id, retire=False)
|
| 4355 |
+
written.update(result)
|
| 4356 |
+
built[bucket] = rows
|
| 4357 |
+
if one.get("limits"):
|
| 4358 |
+
built.setdefault("limits", {}).update(one["limits"])
|
| 4359 |
+
materialized_count = 0 if read_through else result[key]["rows"]
|
| 4360 |
+
source_reconciliation = (
|
| 4361 |
+
"unconfirmed" if source_count is None else
|
| 4362 |
+
"matched" if source_count == count else "mismatch")
|
| 4363 |
+
mirror_reconciliation = (
|
| 4364 |
+
"served_read_through" if read_through else
|
| 4365 |
+
"matched" if count == materialized_count else "mismatch")
|
| 4366 |
+
partial = bool(one.get("limits") or source_failure
|
| 4367 |
+
or source_reconciliation == "mismatch"
|
| 4368 |
+
or mirror_reconciliation == "mismatch")
|
| 4369 |
+
receipts[key] = {
|
| 4370 |
+
**identity, "state": "partial" if partial else "succeeded",
|
| 4371 |
+
"sourceCount": source_count, "sourceCountMode": source_mode,
|
| 4372 |
+
"sourceReconciliation": source_reconciliation,
|
| 4373 |
+
"mirrorCount": count, "materializedCount": materialized_count,
|
| 4374 |
+
"servingCount": count, "readThrough": read_through, "duplicateCount": 0,
|
| 4375 |
+
"reconciliation": {"sourceToMirror": source_reconciliation,
|
| 4376 |
+
"mirrorToMaterialized": mirror_reconciliation},
|
| 4377 |
+
"freshness": {"state": "partial" if partial else "current",
|
| 4378 |
+
"capturedAt": captured_at},
|
| 4379 |
+
**({"sourceError": source_failure} if source_failure else {}),
|
| 4380 |
+
}
|
| 4381 |
+
if source_failure:
|
| 4382 |
+
errors[key] = source_failure
|
| 4383 |
+
except Exception as exc:
|
| 4384 |
+
# Exception prose may contain connection or record values. Keep only the
|
| 4385 |
+
# structural class and a fixed product message in returned/persisted evidence.
|
| 4386 |
+
failure = {"code": "materialization_failed", "type": type(exc).__name__,
|
| 4387 |
+
"message": "This Odoo database could not be materialized."}
|
| 4388 |
+
errors[key] = failure
|
| 4389 |
+
receipts[key] = {
|
| 4390 |
+
**identity, "state": "failed", "sourceCount": source_count,
|
| 4391 |
+
"sourceCountMode": source_mode, "sourceReconciliation": "unconfirmed",
|
| 4392 |
+
"mirrorCount": None, "materializedCount": None, "servingCount": None,
|
| 4393 |
+
"readThrough": is_read_through_key(key), "duplicateCount": None,
|
| 4394 |
+
"freshness": {"state": "failed", "capturedAt": captured_at},
|
| 4395 |
+
"error": failure,
|
| 4396 |
+
**({"sourceError": source_failure} if source_failure else {}),
|
| 4397 |
+
}
|
| 4398 |
+
if retire:
|
| 4399 |
+
apply_plan(rt, {}, username=username, today=today, report=report,
|
| 4400 |
+
connection_id=connection_id, retire=True)
|
| 4401 |
+
return built, written, errors, receipts, report
|
| 4402 |
+
|
| 4403 |
+
|
| 4404 |
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
|
| 4405 |
all_connections=False, table_keys=None, on_apply=None):
|
| 4406 |
"""THE entry point — the store-resync path and the route both call this.
|
|
|
|
| 4507 |
if table_key_for(key, ns) in requested}
|
| 4508 |
skipped = sorted(table_key_for(key, ns)
|
| 4509 |
for bucket, key, _l, _f in TABLES if bucket not in keep)
|
| 4510 |
+
source = None
|
| 4511 |
+
try:
|
| 4512 |
+
source = (rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id))
|
| 4513 |
+
except Exception: # noqa: BLE001
|
| 4514 |
+
source = None
|
| 4515 |
+
canonical_receipts = {}
|
| 4516 |
+
if source is None:
|
| 4517 |
+
built, written, database_errors, database_receipts, plan_report = refresh_curated(
|
| 4518 |
+
rt, cur, keep, connection_id=ns, username=username, today=today,
|
| 4519 |
+
retire=requested is None, on_apply=on_apply)
|
| 4520 |
+
if ns is None and requested is None and cur is not None:
|
| 4521 |
+
canonical_receipts = canonical_reconciliation_receipts(cur)
|
| 4522 |
+
else:
|
| 4523 |
+
# The binding selects this connection's verified read-only client. Authentication failure
|
| 4524 |
+
# leaves source counts unconfirmed but does not discard a usable mirror materialization.
|
| 4525 |
+
with source._bound(): # tenant credential + scope
|
| 4526 |
+
try:
|
| 4527 |
+
from core.odoo import get_odoo
|
| 4528 |
+
source_client = get_odoo()
|
| 4529 |
+
except Exception: # noqa: BLE001
|
| 4530 |
+
source_client = None
|
| 4531 |
+
built, written, database_errors, database_receipts, plan_report = refresh_curated(
|
| 4532 |
+
rt, cur, keep, connection_id=ns, username=username, today=today,
|
| 4533 |
+
retire=requested is None, on_apply=on_apply, source_client=source_client)
|
| 4534 |
+
if ns is None and requested is None and cur is not None:
|
| 4535 |
+
canonical_receipts = canonical_reconciliation_receipts(
|
| 4536 |
+
cur, source_client=source_client)
|
| 4537 |
# ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing.
|
| 4538 |
# Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no
|
| 4539 |
# longer built) SAYS SO, with the keys.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4540 |
# The fixed mirror roster above and the discovered long tail deliberately use separate
|
| 4541 |
# materialisers. Only an explicit long-tail enablement reaches this read-only path; disabled
|
| 4542 |
# databases are not passed an empty set and therefore cannot be erased by reconciliation.
|
|
|
|
| 4551 |
# FROZEN workspace returned before reaching here, so a disconnected connector is not
|
| 4552 |
# re-recorded. See `_refresh_declared_schema` for why it writes the declared half only.
|
| 4553 |
declared_report = _refresh_declared_schema(rt) if requested is None else None
|
| 4554 |
+
out = {"tables": written, "databases": database_receipts,
|
| 4555 |
+
**({"canonicalDatabases": canonical_receipts} if canonical_receipts else {}),
|
| 4556 |
+
**({"databaseErrors": database_errors} if database_errors else {}),
|
| 4557 |
**({"schemaDeclared": declared_report} if declared_report else {}),
|
| 4558 |
**({"discovered": discovered_report}
|
| 4559 |
if discovered_report and (discovered_report.get("tables")
|
|
|
|
| 4565 |
# ⚠ THE DEFAULT CONNECTION'S RETURN SHAPE DOES NOT MOVE BY ONE KEY. The two additions below
|
| 4566 |
# are reachable only on the namespaced path, because `verify_odoo_relational` asserts on this
|
| 4567 |
# dict as it stands for the default.
|
| 4568 |
+
if built.get("limits"):
|
| 4569 |
+
out["limits"] = built["limits"]
|
| 4570 |
if ns is not None:
|
| 4571 |
out["connection"] = cid
|
| 4572 |
if built.get("limits"):
|
|
|
|
| 4615 |
cur=(cur if named is None else None),
|
| 4616 |
today=today, connection_id=named)
|
| 4617 |
except Exception as exc: # noqa: BLE001
|
| 4618 |
+
one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"}
|
| 4619 |
reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
|
| 4620 |
if named is None and primary is None:
|
| 4621 |
primary = one
|
api/providers.py
CHANGED
|
@@ -37,6 +37,7 @@ import json
|
|
| 37 |
import os
|
| 38 |
import time
|
| 39 |
from dataclasses import dataclass, field
|
|
|
|
| 40 |
from typing import Callable
|
| 41 |
|
| 42 |
# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in
|
|
@@ -58,6 +59,32 @@ import requests
|
|
| 58 |
#: reads a live price — a modelled figure that says so is honest; one that pretends is not.
|
| 59 |
_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015)
|
| 60 |
_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
|
| 62 |
|
| 63 |
@dataclass(frozen=True)
|
|
@@ -135,6 +162,9 @@ PROVIDERS: dict[str, Provider] = {
|
|
| 135 |
"apify": Provider(
|
| 136 |
key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
|
| 137 |
caps={
|
|
|
|
|
|
|
|
|
|
| 138 |
# ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
|
| 139 |
# 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
|
| 140 |
"ig_post_views": Capability(True, _COST_APIFY,
|
|
@@ -146,6 +176,7 @@ PROVIDERS: dict[str, Provider] = {
|
|
| 146 |
|
| 147 |
#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`.
|
| 148 |
DEFAULT_CHAINS: dict[str, tuple] = {
|
|
|
|
| 149 |
"ig_profile": ("brightdata", "apify"),
|
| 150 |
"ig_post_metrics": ("brightdata", "apify"),
|
| 151 |
# Only one entry, and that is the point: Bright Data is declared incapable, so listing it here
|
|
@@ -239,6 +270,11 @@ def run(capability, work, satisfied=None, log=None):
|
|
| 239 |
started = time.time()
|
| 240 |
try:
|
| 241 |
result, note = work(provider)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
except Exception as e: # noqa: BLE001
|
| 243 |
# ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is
|
| 244 |
# one refactor away from being a query param; this line must not be what leaks it.
|
|
@@ -264,6 +300,8 @@ def _count(result):
|
|
| 264 |
if isinstance(result, (list, tuple, set)):
|
| 265 |
return len(result)
|
| 266 |
if isinstance(result, dict):
|
|
|
|
|
|
|
| 267 |
return len(result)
|
| 268 |
return 1 if result else 0
|
| 269 |
|
|
@@ -888,6 +926,312 @@ def normalize_profile_apify(row):
|
|
| 888 |
return {k: v for k, v in out.items() if v is not None and v != ""}
|
| 889 |
|
| 890 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 891 |
def canonical_gaps(row, want=("views",)):
|
| 892 |
"""Which requested canonical fields this row does NOT carry — the `satisfied` input.
|
| 893 |
|
|
|
|
| 37 |
import os
|
| 38 |
import time
|
| 39 |
from dataclasses import dataclass, field
|
| 40 |
+
from datetime import datetime, timezone
|
| 41 |
from typing import Callable
|
| 42 |
|
| 43 |
# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in
|
|
|
|
| 59 |
#: reads a live price — a modelled figure that says so is honest; one that pretends is not.
|
| 60 |
_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015)
|
| 61 |
_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027)
|
| 62 |
+
#: Google Maps store discovery is a different Apify actor and therefore has its own modelled
|
| 63 |
+
#: list rate. This is a planning/receipt rate, not an invoice; deployments may replace it without
|
| 64 |
+
#: a release when the actor's published price changes. Keeping it separate from Instagram avoids
|
| 65 |
+
#: making a Maps estimate silently inherit another actor's commercial terms.
|
| 66 |
+
_COST_APIFY_MAPS = float(os.environ.get("AIOS_COST_APIFY_MAPS") or 0.004)
|
| 67 |
+
|
| 68 |
+
#: One actor run produces one dataset. Pagination reads that dataset; it never launches a second
|
| 69 |
+
#: paid run. The result cap remains the hard provider-record and modelled-spend ceiling.
|
| 70 |
+
APIFY_BASE = "https://api.apify.com/v2"
|
| 71 |
+
APIFY_MAPS_ACTOR = os.environ.get("AIOS_APIFY_MAPS_ACTOR") or "compass~crawler-google-places"
|
| 72 |
+
APIFY_MAPS_WAIT = float(os.environ.get("AIOS_APIFY_MAPS_WAIT") or 240)
|
| 73 |
+
APIFY_MAPS_PAGE_SIZE = 50
|
| 74 |
+
APIFY_MAPS_MAX_RECORDS = 500
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
class ProviderRefusal(RuntimeError):
|
| 78 |
+
"""A safe, named provider refusal whose message may cross the API boundary."""
|
| 79 |
+
|
| 80 |
+
def __init__(self, code: str, public_message: str):
|
| 81 |
+
super().__init__(public_message)
|
| 82 |
+
self.code = str(code)
|
| 83 |
+
self.public_message = str(public_message)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class MapsProviderRefusal(ProviderRefusal):
|
| 87 |
+
"""Named Google Maps refusal; never contains a credential, URL, or vendor response body."""
|
| 88 |
|
| 89 |
|
| 90 |
@dataclass(frozen=True)
|
|
|
|
| 162 |
"apify": Provider(
|
| 163 |
key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
|
| 164 |
caps={
|
| 165 |
+
"google_maps_stores": Capability(
|
| 166 |
+
True, _COST_APIFY_MAPS,
|
| 167 |
+
"Google Maps places by category/search and geography; modelled per place"),
|
| 168 |
# ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
|
| 169 |
# 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
|
| 170 |
"ig_post_views": Capability(True, _COST_APIFY,
|
|
|
|
| 176 |
|
| 177 |
#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`.
|
| 178 |
DEFAULT_CHAINS: dict[str, tuple] = {
|
| 179 |
+
"google_maps_stores": ("apify",),
|
| 180 |
"ig_profile": ("brightdata", "apify"),
|
| 181 |
"ig_post_metrics": ("brightdata", "apify"),
|
| 182 |
# Only one entry, and that is the point: Bright Data is declared incapable, so listing it here
|
|
|
|
| 270 |
started = time.time()
|
| 271 |
try:
|
| 272 |
result, note = work(provider)
|
| 273 |
+
except ProviderRefusal:
|
| 274 |
+
# A named refusal is part of the product contract, not a fallback signal. In
|
| 275 |
+
# particular, retrying after Maps schema drift can buy the same unusable dataset
|
| 276 |
+
# twice and replace the actionable error code with a generic provider failure.
|
| 277 |
+
raise
|
| 278 |
except Exception as e: # noqa: BLE001
|
| 279 |
# ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is
|
| 280 |
# one refactor away from being a query param; this line must not be what leaks it.
|
|
|
|
| 300 |
if isinstance(result, (list, tuple, set)):
|
| 301 |
return len(result)
|
| 302 |
if isinstance(result, dict):
|
| 303 |
+
if isinstance(result.get("rows"), list):
|
| 304 |
+
return len(result["rows"])
|
| 305 |
return len(result)
|
| 306 |
return 1 if result else 0
|
| 307 |
|
|
|
|
| 926 |
return {k: v for k, v in out.items() if v is not None and v != ""}
|
| 927 |
|
| 928 |
|
| 929 |
+
def _maps_json(raw, *, subject: str):
|
| 930 |
+
"""Decode one Apify Maps envelope without echoing its body into a refusal."""
|
| 931 |
+
try:
|
| 932 |
+
return json.loads(bytes(raw or b"").decode("utf-8", "replace"))
|
| 933 |
+
except Exception as exc:
|
| 934 |
+
raise MapsProviderRefusal(
|
| 935 |
+
"maps_schema_drift",
|
| 936 |
+
f"Apify's Google Maps {subject} was not readable JSON. No result was written.") from exc
|
| 937 |
+
|
| 938 |
+
|
| 939 |
+
def _apify_maps_page_source(provider, search, geography, cap):
|
| 940 |
+
"""Start one bounded actor run and return a reader for its one resulting dataset.
|
| 941 |
+
|
| 942 |
+
The actor is launched once. The returned closure pages the resulting dataset by offset, so
|
| 943 |
+
walking a 500-place result never re-buys the search. Credentials remain in request URLs and
|
| 944 |
+
are consequently never copied into an exception or receipt.
|
| 945 |
+
"""
|
| 946 |
+
from automation_engine import Refused, fetch, fetch_json # lazy: only explicit Run now pays
|
| 947 |
+
|
| 948 |
+
key = (os.environ.get(provider.key_env) or "").strip()
|
| 949 |
+
if not key:
|
| 950 |
+
raise MapsProviderRefusal(
|
| 951 |
+
"maps_credential_missing",
|
| 952 |
+
"Add the Apify credential in Keychain before choosing Run now.")
|
| 953 |
+
token = requests.utils.quote(key, safe="")
|
| 954 |
+
actor = requests.utils.quote(APIFY_MAPS_ACTOR, safe="~")
|
| 955 |
+
start_url = (f"{APIFY_BASE}/acts/{actor}/runs?token={token}"
|
| 956 |
+
f"&waitForFinish={max(1, min(60, int(APIFY_MAPS_WAIT)))}")
|
| 957 |
+
body = {
|
| 958 |
+
"searchStringsArray": [str(search)],
|
| 959 |
+
"locationQuery": str(geography),
|
| 960 |
+
"maxCrawledPlacesPerSearch": int(cap),
|
| 961 |
+
"maxCrawledPlaces": int(cap),
|
| 962 |
+
# Details required by the stable schema, without unrelated review/image payloads.
|
| 963 |
+
"maxReviews": 0,
|
| 964 |
+
"maxImages": 0,
|
| 965 |
+
"scrapePlaceDetailPage": True,
|
| 966 |
+
}
|
| 967 |
+
try:
|
| 968 |
+
status, raw = fetch_json(start_url, body, timeout=min(APIFY_MAPS_WAIT, 60) + 30,
|
| 969 |
+
max_kb=1024)
|
| 970 |
+
except Refused as exc:
|
| 971 |
+
raise MapsProviderRefusal(
|
| 972 |
+
"maps_vendor_error",
|
| 973 |
+
"The Apify Google Maps address was refused by the provider safety rail. "
|
| 974 |
+
"No result was written.") from exc
|
| 975 |
+
except Exception as exc: # noqa: BLE001
|
| 976 |
+
raise MapsProviderRefusal(
|
| 977 |
+
"maps_vendor_error",
|
| 978 |
+
f"Apify Google Maps did not answer ({type(exc).__name__}). No result was written.") from exc
|
| 979 |
+
if not 200 <= int(status or 0) < 300:
|
| 980 |
+
raise MapsProviderRefusal(
|
| 981 |
+
"maps_vendor_error",
|
| 982 |
+
"Apify Google Maps refused the bounded run. No result was written.")
|
| 983 |
+
envelope = _maps_json(raw, subject="run envelope")
|
| 984 |
+
data = envelope.get("data") if isinstance(envelope, dict) else None
|
| 985 |
+
if not isinstance(data, dict):
|
| 986 |
+
raise MapsProviderRefusal(
|
| 987 |
+
"maps_schema_drift",
|
| 988 |
+
"Apify's Google Maps run envelope changed. No result was written.")
|
| 989 |
+
|
| 990 |
+
deadline = time.monotonic() + max(1.0, float(APIFY_MAPS_WAIT))
|
| 991 |
+
terminal = {"SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT"}
|
| 992 |
+
run_id = str(data.get("id") or "").strip()
|
| 993 |
+
run_status = str(data.get("status") or "").strip().upper()
|
| 994 |
+
while run_status not in terminal:
|
| 995 |
+
remaining = deadline - time.monotonic()
|
| 996 |
+
if remaining <= 0 or not run_id:
|
| 997 |
+
raise MapsProviderRefusal(
|
| 998 |
+
"maps_vendor_error",
|
| 999 |
+
"Apify Google Maps did not finish within the bounded wait. No result was written.")
|
| 1000 |
+
wait_seconds = max(1, min(60, int(remaining)))
|
| 1001 |
+
run_url = (f"{APIFY_BASE}/actor-runs/{requests.utils.quote(run_id, safe='')}"
|
| 1002 |
+
f"?token={token}&waitForFinish={wait_seconds}")
|
| 1003 |
+
try:
|
| 1004 |
+
poll_status, _final, poll_raw = fetch(
|
| 1005 |
+
run_url, timeout=wait_seconds + 30, max_kb=1024)
|
| 1006 |
+
except Exception as exc: # noqa: BLE001
|
| 1007 |
+
raise MapsProviderRefusal(
|
| 1008 |
+
"maps_vendor_error",
|
| 1009 |
+
f"Apify Google Maps status did not answer ({type(exc).__name__}). "
|
| 1010 |
+
"No result was written.") from exc
|
| 1011 |
+
if not 200 <= int(poll_status or 0) < 300:
|
| 1012 |
+
raise MapsProviderRefusal(
|
| 1013 |
+
"maps_vendor_error",
|
| 1014 |
+
"Apify Google Maps status could not be read. No result was written.")
|
| 1015 |
+
polled = _maps_json(poll_raw, subject="status envelope")
|
| 1016 |
+
data = polled.get("data") if isinstance(polled, dict) else None
|
| 1017 |
+
if not isinstance(data, dict):
|
| 1018 |
+
raise MapsProviderRefusal(
|
| 1019 |
+
"maps_schema_drift",
|
| 1020 |
+
"Apify's Google Maps status envelope changed. No result was written.")
|
| 1021 |
+
run_status = str(data.get("status") or "").strip().upper()
|
| 1022 |
+
|
| 1023 |
+
if run_status != "SUCCEEDED":
|
| 1024 |
+
raise MapsProviderRefusal(
|
| 1025 |
+
"maps_vendor_error",
|
| 1026 |
+
f"Apify Google Maps ended in {run_status.lower() or 'an error'}. No result was written.")
|
| 1027 |
+
dataset_id = str(data.get("defaultDatasetId") or "").strip()
|
| 1028 |
+
if not dataset_id:
|
| 1029 |
+
raise MapsProviderRefusal(
|
| 1030 |
+
"maps_schema_drift",
|
| 1031 |
+
"Apify's Google Maps result did not name its dataset. No result was written.")
|
| 1032 |
+
|
| 1033 |
+
def _page(offset, limit):
|
| 1034 |
+
dataset = requests.utils.quote(dataset_id, safe="")
|
| 1035 |
+
page_url = (f"{APIFY_BASE}/datasets/{dataset}/items?token={token}&clean=true&format=json"
|
| 1036 |
+
f"&offset={int(offset)}&limit={int(limit)}")
|
| 1037 |
+
try:
|
| 1038 |
+
page_status, _final, page_raw = fetch(page_url, timeout=60, max_kb=16384)
|
| 1039 |
+
except Exception as exc: # noqa: BLE001
|
| 1040 |
+
raise MapsProviderRefusal(
|
| 1041 |
+
"maps_vendor_error",
|
| 1042 |
+
f"Apify Google Maps results did not answer ({type(exc).__name__}). "
|
| 1043 |
+
"No result was written.") from exc
|
| 1044 |
+
if not 200 <= int(page_status or 0) < 300:
|
| 1045 |
+
raise MapsProviderRefusal(
|
| 1046 |
+
"maps_vendor_error",
|
| 1047 |
+
"Apify Google Maps results could not be read. No result was written.")
|
| 1048 |
+
page = _maps_json(page_raw, subject="dataset page")
|
| 1049 |
+
if not isinstance(page, list):
|
| 1050 |
+
raise MapsProviderRefusal(
|
| 1051 |
+
"maps_schema_drift",
|
| 1052 |
+
"Apify's Google Maps dataset page changed shape. No result was written.")
|
| 1053 |
+
return page
|
| 1054 |
+
|
| 1055 |
+
return _page
|
| 1056 |
+
|
| 1057 |
+
|
| 1058 |
+
def _maps_coordinate(value, name):
|
| 1059 |
+
try:
|
| 1060 |
+
number = float(value)
|
| 1061 |
+
except (TypeError, ValueError) as exc:
|
| 1062 |
+
raise MapsProviderRefusal(
|
| 1063 |
+
"maps_schema_drift",
|
| 1064 |
+
f"Apify's Google Maps result has no usable {name}. No result was written.") from exc
|
| 1065 |
+
if name == "latitude" and not -90 <= number <= 90:
|
| 1066 |
+
raise MapsProviderRefusal(
|
| 1067 |
+
"maps_schema_drift",
|
| 1068 |
+
"Apify's Google Maps result has an invalid latitude. No result was written.")
|
| 1069 |
+
if name == "longitude" and not -180 <= number <= 180:
|
| 1070 |
+
raise MapsProviderRefusal(
|
| 1071 |
+
"maps_schema_drift",
|
| 1072 |
+
"Apify's Google Maps result has an invalid longitude. No result was written.")
|
| 1073 |
+
return number
|
| 1074 |
+
|
| 1075 |
+
|
| 1076 |
+
def normalize_google_maps_store(row, *, captured_at=""):
|
| 1077 |
+
"""One Apify Google Maps place -> the stable store-discovery database schema.
|
| 1078 |
+
|
| 1079 |
+
Identity, name, physical address, category and coordinates are the structural minimum. Phone
|
| 1080 |
+
and website are optional facts about a real place, not schema failures. The capture timestamp
|
| 1081 |
+
falls back to the run timestamp so every observation says when its source was read.
|
| 1082 |
+
"""
|
| 1083 |
+
if not isinstance(row, dict):
|
| 1084 |
+
raise MapsProviderRefusal(
|
| 1085 |
+
"maps_schema_drift",
|
| 1086 |
+
"Apify's Google Maps result is not a place object. No result was written.")
|
| 1087 |
+
location = row.get("location") if isinstance(row.get("location"), dict) else {}
|
| 1088 |
+
place_id = str(row.get("placeId") or row.get("place_id") or "").strip()
|
| 1089 |
+
name = str(row.get("title") or row.get("name") or "").strip()
|
| 1090 |
+
address = str(row.get("address") or row.get("completeAddress") or "").strip()
|
| 1091 |
+
category = str(row.get("categoryName") or row.get("category") or "").strip()
|
| 1092 |
+
if not category and isinstance(row.get("categories"), list):
|
| 1093 |
+
category = str(next((item for item in row["categories"] if item), "")).strip()
|
| 1094 |
+
missing = [label for label, value in (
|
| 1095 |
+
("placeId", place_id), ("title", name), ("address", address),
|
| 1096 |
+
("categoryName", category)) if not value]
|
| 1097 |
+
if missing:
|
| 1098 |
+
raise MapsProviderRefusal(
|
| 1099 |
+
"maps_schema_drift",
|
| 1100 |
+
"Apify's Google Maps result schema changed; required fields are missing: "
|
| 1101 |
+
f"{', '.join(missing)}. No result was written.")
|
| 1102 |
+
latitude = _maps_coordinate(location.get("lat", row.get("latitude")), "latitude")
|
| 1103 |
+
longitude = _maps_coordinate(location.get("lng", row.get("longitude")), "longitude")
|
| 1104 |
+
source_timestamp = str(
|
| 1105 |
+
row.get("scrapedAt") or row.get("scraped_at") or row.get("sourceTimestamp")
|
| 1106 |
+
or captured_at or datetime.now(timezone.utc).isoformat()).strip()
|
| 1107 |
+
out = {
|
| 1108 |
+
"place_id": place_id,
|
| 1109 |
+
"name": name,
|
| 1110 |
+
"category": category,
|
| 1111 |
+
"address": address,
|
| 1112 |
+
"city": str(row.get("city") or "").strip(),
|
| 1113 |
+
"region": str(row.get("state") or row.get("region") or "").strip(),
|
| 1114 |
+
"postal_code": str(row.get("postalCode") or row.get("postal_code") or "").strip(),
|
| 1115 |
+
"country": str(row.get("countryCode") or row.get("country") or "").strip(),
|
| 1116 |
+
"phone": str(row.get("phone") or row.get("phoneUnformatted") or "").strip(),
|
| 1117 |
+
"website": str(row.get("website") or row.get("webSite") or "").strip(),
|
| 1118 |
+
"latitude": latitude,
|
| 1119 |
+
"longitude": longitude,
|
| 1120 |
+
"rating": row.get("totalScore", row.get("rating")),
|
| 1121 |
+
"reviews": _int_or_none(row.get("reviewsCount", row.get("reviews"))),
|
| 1122 |
+
"source_timestamp": source_timestamp,
|
| 1123 |
+
"source_payload": json.dumps(row, default=str, separators=(",", ":"))[:32_000],
|
| 1124 |
+
}
|
| 1125 |
+
return {key: value for key, value in out.items() if value is not None and value != ""}
|
| 1126 |
+
|
| 1127 |
+
|
| 1128 |
+
def discover_google_maps_stores(search, geography, cap, *, page_source=None,
|
| 1129 |
+
page_size=APIFY_MAPS_PAGE_SIZE, captured_at=""):
|
| 1130 |
+
"""Run one bounded Maps discovery through the provider chain and return rows + receipts.
|
| 1131 |
+
|
| 1132 |
+
`page_source(provider, search, geography, cap) -> page(offset, limit)` is the network seam used
|
| 1133 |
+
by the recorded gate fixture. Production uses one Apify actor run and pages its dataset.
|
| 1134 |
+
Provider rows, including duplicates, consume the explicit cap; the runner never buys extra
|
| 1135 |
+
rows merely to replace duplicate place IDs.
|
| 1136 |
+
"""
|
| 1137 |
+
query = str(search or "").strip()
|
| 1138 |
+
area = str(geography or "").strip()
|
| 1139 |
+
if not query or not area:
|
| 1140 |
+
raise MapsProviderRefusal(
|
| 1141 |
+
"maps_invalid_search",
|
| 1142 |
+
"Google Maps discovery needs a category or search term and a geography.")
|
| 1143 |
+
try:
|
| 1144 |
+
limit = int(cap)
|
| 1145 |
+
except (TypeError, ValueError) as exc:
|
| 1146 |
+
raise MapsProviderRefusal(
|
| 1147 |
+
"maps_unbounded_request",
|
| 1148 |
+
"Google Maps discovery requires an explicit result cap between 1 and 500.") from exc
|
| 1149 |
+
if limit < 1 or limit > APIFY_MAPS_MAX_RECORDS:
|
| 1150 |
+
raise MapsProviderRefusal(
|
| 1151 |
+
"maps_unbounded_request",
|
| 1152 |
+
"Google Maps discovery requires an explicit result cap between 1 and 500.")
|
| 1153 |
+
try:
|
| 1154 |
+
chunk = max(1, min(int(page_size), APIFY_MAPS_PAGE_SIZE, limit))
|
| 1155 |
+
except (TypeError, ValueError):
|
| 1156 |
+
chunk = min(APIFY_MAPS_PAGE_SIZE, limit)
|
| 1157 |
+
if not chain("google_maps_stores"):
|
| 1158 |
+
raise MapsProviderRefusal(
|
| 1159 |
+
"maps_credential_missing",
|
| 1160 |
+
"Add the Apify credential in Keychain before choosing Run now.")
|
| 1161 |
+
captured = str(captured_at or datetime.now(timezone.utc).isoformat())
|
| 1162 |
+
factory = page_source or _apify_maps_page_source
|
| 1163 |
+
|
| 1164 |
+
def _work(provider):
|
| 1165 |
+
page = factory(provider, query, area, limit)
|
| 1166 |
+
if not callable(page):
|
| 1167 |
+
raise MapsProviderRefusal(
|
| 1168 |
+
"maps_schema_drift",
|
| 1169 |
+
"The Google Maps provider did not supply a page reader. No result was written.")
|
| 1170 |
+
offset = 0
|
| 1171 |
+
pages = 0
|
| 1172 |
+
raw_count = 0
|
| 1173 |
+
duplicates = 0
|
| 1174 |
+
seen = set()
|
| 1175 |
+
rows = []
|
| 1176 |
+
while raw_count < limit:
|
| 1177 |
+
ask = min(chunk, limit - raw_count)
|
| 1178 |
+
batch = page(offset, ask)
|
| 1179 |
+
pages += 1
|
| 1180 |
+
if not isinstance(batch, list):
|
| 1181 |
+
raise MapsProviderRefusal(
|
| 1182 |
+
"maps_schema_drift",
|
| 1183 |
+
"The Google Maps provider returned a non-list dataset page. "
|
| 1184 |
+
"No result was written.")
|
| 1185 |
+
if len(batch) > ask:
|
| 1186 |
+
raise MapsProviderRefusal(
|
| 1187 |
+
"maps_vendor_error",
|
| 1188 |
+
"The Google Maps provider exceeded the requested page bound. "
|
| 1189 |
+
"No result was written.")
|
| 1190 |
+
if not batch:
|
| 1191 |
+
break
|
| 1192 |
+
for vendor_row in batch:
|
| 1193 |
+
normal = normalize_google_maps_store(vendor_row, captured_at=captured)
|
| 1194 |
+
raw_count += 1
|
| 1195 |
+
place_id = normal["place_id"]
|
| 1196 |
+
if place_id in seen:
|
| 1197 |
+
duplicates += 1
|
| 1198 |
+
continue
|
| 1199 |
+
seen.add(place_id)
|
| 1200 |
+
rows.append(normal)
|
| 1201 |
+
offset += len(batch)
|
| 1202 |
+
if len(batch) < ask:
|
| 1203 |
+
break
|
| 1204 |
+
rate = float((provider.cap("google_maps_stores") or Capability()).cost_per_record or 0.0)
|
| 1205 |
+
cost_cap = round(rate * limit, 4)
|
| 1206 |
+
payload = {
|
| 1207 |
+
"state": "ok",
|
| 1208 |
+
"via": provider.key,
|
| 1209 |
+
"rows": rows,
|
| 1210 |
+
"counts": {"cap": limit, "pages": pages, "raw": raw_count,
|
| 1211 |
+
"normalized": raw_count, "duplicates": duplicates,
|
| 1212 |
+
"returned": len(rows)},
|
| 1213 |
+
"cost": {"currency": "USD", "basis": "modelled per provider record",
|
| 1214 |
+
"ratePerRecordUsd": rate, "estimatedUsd": cost_cap,
|
| 1215 |
+
"costCapUsd": cost_cap,
|
| 1216 |
+
"incurredUsd": round(rate * raw_count, 4)},
|
| 1217 |
+
}
|
| 1218 |
+
return payload, ""
|
| 1219 |
+
|
| 1220 |
+
result, attempts = run(
|
| 1221 |
+
"google_maps_stores", _work, satisfied=lambda answer: isinstance(answer, dict))
|
| 1222 |
+
if not isinstance(result, dict):
|
| 1223 |
+
raise MapsProviderRefusal(
|
| 1224 |
+
"maps_vendor_error",
|
| 1225 |
+
"Apify Google Maps could not complete the bounded run. No result was written.")
|
| 1226 |
+
result["attempts"] = [
|
| 1227 |
+
{"provider": attempt.provider, "ok": attempt.ok,
|
| 1228 |
+
"records": (len(result.get("rows") or []) if attempt.ok else 0),
|
| 1229 |
+
"seconds": attempt.seconds}
|
| 1230 |
+
for attempt in attempts
|
| 1231 |
+
]
|
| 1232 |
+
return result
|
| 1233 |
+
|
| 1234 |
+
|
| 1235 |
def canonical_gaps(row, want=("views",)):
|
| 1236 |
"""Which requested canonical fields this row does NOT carry — the `satisfied` input.
|
| 1237 |
|
api/routes_activity.py
ADDED
|
@@ -0,0 +1,392 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Value-free activity history, persisted only through the tenant store (S65-T16)."""
|
| 2 |
+
import json
|
| 3 |
+
import math
|
| 4 |
+
import re
|
| 5 |
+
import time
|
| 6 |
+
import uuid
|
| 7 |
+
from urllib.parse import urlsplit
|
| 8 |
+
|
| 9 |
+
from fastapi import APIRouter, Depends, Query, Request
|
| 10 |
+
from starlette.concurrency import run_in_threadpool
|
| 11 |
+
|
| 12 |
+
from deps import Session, err, require_session
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
RETENTION_DAYS = 30
|
| 16 |
+
MAX_SESSIONS = 200
|
| 17 |
+
MAX_EVENTS_PER_SESSION = 2000
|
| 18 |
+
MAX_MOVEMENT_SAMPLES = 500
|
| 19 |
+
MOVEMENT_WINDOW_MS = 30_000
|
| 20 |
+
MAX_BATCH_EVENTS = 100
|
| 21 |
+
MAX_PAYLOAD_BYTES = 512 * 1024
|
| 22 |
+
MAX_STRING_LENGTH = 256
|
| 23 |
+
MAX_CLASS_LENGTH = 128
|
| 24 |
+
MAX_DATA_ATTRIBUTES = 16
|
| 25 |
+
MAX_PAGE_EVENTS = 100
|
| 26 |
+
BUCKET = "activity_log"
|
| 27 |
+
|
| 28 |
+
# The entire client field vocabulary, including nested objects, is here. Each
|
| 29 |
+
# shape below selects only its own subset; no arbitrary recursive object survives.
|
| 30 |
+
ACTIVITY_ALLOWED_KEYS = frozenset({
|
| 31 |
+
"sessionId", "events", "dropped", "kind", "ts", "route", "from", "to", "ms",
|
| 32 |
+
"target", "method", "url", "status", "message", "source", "line", "name",
|
| 33 |
+
"movement", "movementDropped", "x", "y", "key", "tagName", "role", "type",
|
| 34 |
+
"id", "data", "className", "aria-label", "path",
|
| 35 |
+
})
|
| 36 |
+
SECRET_KEYS = frozenset({
|
| 37 |
+
"value", "defaultvalue", "textcontent", "innertext", "innerhtml", "placeholder",
|
| 38 |
+
"password", "token", "apikey", "authorization", "cookie", "secret", "credential",
|
| 39 |
+
"accesstoken", "refreshtoken", "clientsecret", "clipboard", "body", "query",
|
| 40 |
+
"cellvalue", "celldata", "text", "content", "payload",
|
| 41 |
+
})
|
| 42 |
+
_BEARER = re.compile(r"\bbearer\s+\S+", re.I)
|
| 43 |
+
_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
|
| 44 |
+
_CREDENTIAL = re.compile(r"\blp[bcor]1\.[A-Za-z0-9._~-]+")
|
| 45 |
+
_TOKEN = re.compile(r"^[a-zA-Z][a-zA-Z0-9_-]{0,79}$")
|
| 46 |
+
_CLASS = re.compile(r"^(?:set|shell|sheet|cg|pg|activity|lp|is)-[a-z][a-z0-9-]*$")
|
| 47 |
+
_SID = re.compile(r"^[a-f0-9-]{32,36}$")
|
| 48 |
+
_TAGS = set("a button input textarea select option div span svg path rect circle canvas td th tr table body html main section nav header footer label form ul ol li h1 h2 h3 h4 p img dialog details summary video audio".split())
|
| 49 |
+
_ROLES = set("button link textbox checkbox radio switch combobox listbox option grid gridcell cell row columnheader rowheader dialog tab tabpanel menu menuitem navigation region slider spinbutton searchbox".split())
|
| 50 |
+
_TYPES = set("button submit reset text password email number search tel url date time checkbox radio range file hidden color".split())
|
| 51 |
+
_ERROR_NAMES = {"Error", "TypeError", "ReferenceError", "SyntaxError", "RangeError", "URIError", "EvalError", "AggregateError", "NetworkError", "HttpError", "ResourceError"}
|
| 52 |
+
_MESSAGES = {"Application error.", "Unhandled rejection.", "Script error.", "A request failed.", "A network request failed.", "A resource failed to load."}
|
| 53 |
+
_LABELS = {"Open", "Close", "Save", "Cancel", "Delete", "Edit", "Search", "Settings", "Close settings", "Activity log", "Refresh", "Previous", "Next", "Add", "Remove", "Menu", "More", "Retry"}
|
| 54 |
+
# Attribute values may hold tenant data even when their names look harmless.
|
| 55 |
+
# Retain only structural enums; all other safe data-* names record presence.
|
| 56 |
+
_DATA_ENUMS = {
|
| 57 |
+
"data-section": {"account", "scope", "security", "shortcuts", "activity", "users", "database", "keychains", "connectors", "padmin"},
|
| 58 |
+
"data-state": {"open", "closed", "active", "inactive", "loading", "ready"},
|
| 59 |
+
"data-view-kind": {"sheet", "grid", "table", "chart", "timeline"},
|
| 60 |
+
}
|
| 61 |
+
# Routes describe the surface, not record ids, searches, or cell text embedded in
|
| 62 |
+
# a URL. Unknown segments become :item on BOTH sides of the wire.
|
| 63 |
+
_ROUTE_SEGMENTS = set("api v1 auth me login logout refresh nav workspace settings account scope security shortcuts activity sessions users database databases grid sheet sheets view views table tables records rows columns fields cells query sort filter data applications finance facts orders inventory customers products sales collections procurement agents agent automation automations connectors keychains external oauth connections admin operator subscriptions billing health status run runs events logs actions export import search schema preferences folders pages charts catalog reports reconciliation permissions detail assets".split())
|
| 64 |
+
|
| 65 |
+
# ⛔ NO `prefix=` HERE, AND THAT IS DELIBERATE. This router is included INSIDE
|
| 66 |
+
# `routes_applications.router` (main.py belongs to another session this wave), and a child
|
| 67 |
+
# router that carries its own `prefix=` has that prefix baked into its own `.routes` at
|
| 68 |
+
# DECLARATION time. `verify_api.py::_mounted_paths` then adds `nested.prefix` a SECOND time
|
| 69 |
+
# while walking the nested include, so `/api/v1/activity` is derived as
|
| 70 |
+
# `/api/v1/api/v1/activity` and D-48 reports a client call with no route -- a false RED over a
|
| 71 |
+
# route the app genuinely serves (`app.openapi()` proves it). Spelling the full path on each
|
| 72 |
+
# decorator makes the declared path the served path, which is the honest reading either way.
|
| 73 |
+
router = APIRouter(tags=["activity"])
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
class ActivityRefused(ValueError):
|
| 77 |
+
"""Never put a refused value into an exception, traceback, or response."""
|
| 78 |
+
|
| 79 |
+
def __init__(self, code, message):
|
| 80 |
+
self.code = code
|
| 81 |
+
self.message = message
|
| 82 |
+
super().__init__(message)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _text(value, maximum=MAX_STRING_LENGTH):
|
| 86 |
+
if not isinstance(value, str):
|
| 87 |
+
return ""
|
| 88 |
+
if (_BEARER.search(value) or _JWT.search(value) or _CREDENTIAL.search(value)
|
| 89 |
+
or value.startswith("Atzr|")):
|
| 90 |
+
return ""
|
| 91 |
+
return re.sub(r"[\x00-\x1f\x7f\u2013\u2014]", " ", value)[:maximum]
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _shape(value):
|
| 95 |
+
if not isinstance(value, dict):
|
| 96 |
+
return {}
|
| 97 |
+
return {key: val for key, val in value.items() if key in ACTIVITY_ALLOWED_KEYS}
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _number(value, default=0, maximum=10**15):
|
| 101 |
+
if type(value) not in (int, float) or (isinstance(value, float) and not math.isfinite(value)):
|
| 102 |
+
return default
|
| 103 |
+
return min(maximum, max(0, round(value)))
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _path(value):
|
| 107 |
+
value = _text(value)
|
| 108 |
+
if not value:
|
| 109 |
+
return ""
|
| 110 |
+
try:
|
| 111 |
+
# Hash routes are paths too, but hash query parameters never survive.
|
| 112 |
+
raw = value[1:] if value.startswith("#") else urlsplit(value).path
|
| 113 |
+
raw = raw.split("?", 1)[0].split("#", 1)[0]
|
| 114 |
+
segments = raw.strip("/").split("/")
|
| 115 |
+
clean = [part if part in _ROUTE_SEGMENTS or re.fullmatch(r"[a-zA-Z0-9_-]+\.(?:js|css|tsx|ts)", part) else ":item" for part in segments if part]
|
| 116 |
+
return ("/" + "/".join(clean))[:MAX_STRING_LENGTH]
|
| 117 |
+
except ValueError:
|
| 118 |
+
return ""
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _classes(value):
|
| 122 |
+
return " ".join(part for part in _text(value).split() if _CLASS.fullmatch(part))[:MAX_CLASS_LENGTH]
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def _element(raw):
|
| 126 |
+
raw = _shape(raw)
|
| 127 |
+
tag = _text(raw.get("tagName")).lower()
|
| 128 |
+
role = _text(raw.get("role"))
|
| 129 |
+
result = {"tagName": tag if tag in _TAGS else "element"}
|
| 130 |
+
if role in _ROLES:
|
| 131 |
+
result["role"] = role
|
| 132 |
+
element_type = _text(raw.get("type"), 32)
|
| 133 |
+
if element_type in _TYPES:
|
| 134 |
+
result["type"] = element_type
|
| 135 |
+
# The collector marks descendants of a grid/editor with role=gridcell/textbox.
|
| 136 |
+
# Defend direct forged input too; never accept rich metadata on these shapes.
|
| 137 |
+
protected = (tag in {"input", "textarea", "option", "select", "canvas", "td", "th"}
|
| 138 |
+
or role in {"grid", "gridcell", "cell", "textbox", "searchbox", "option"}
|
| 139 |
+
or "sheet-cell" in str(raw.get("className", ""))
|
| 140 |
+
or "cg-cell" in str(raw.get("className", "")))
|
| 141 |
+
hops = raw.get("path", [])
|
| 142 |
+
if isinstance(hops, list):
|
| 143 |
+
protected = protected or any(
|
| 144 |
+
_text(hop).split(".")[0] in {"td", "th", "input", "textarea", "option", "canvas"}
|
| 145 |
+
or re.search(r"\.(?:sheet|cg)-cell(?:\.|$)", _text(hop))
|
| 146 |
+
for hop in hops[:5])
|
| 147 |
+
path = []
|
| 148 |
+
for hop in hops[:5]:
|
| 149 |
+
parts = _text(hop).split(".")
|
| 150 |
+
if parts[0] in _TAGS:
|
| 151 |
+
classes = _classes(" ".join(parts[1:])) if not protected else ""
|
| 152 |
+
path.append(parts[0] + ("." + classes.replace(" ", ".") if classes else ""))
|
| 153 |
+
result["path"] = path
|
| 154 |
+
if protected:
|
| 155 |
+
return result
|
| 156 |
+
classes = _classes(raw.get("className"))
|
| 157 |
+
if classes:
|
| 158 |
+
result["className"] = classes
|
| 159 |
+
for key in ("id", "name"):
|
| 160 |
+
val = _text(raw.get(key), 80)
|
| 161 |
+
if val == "root" or (_TOKEN.fullmatch(val) and _CLASS.fullmatch(val)):
|
| 162 |
+
result[key] = val
|
| 163 |
+
label = _text(raw.get("aria-label"))
|
| 164 |
+
if label in _LABELS:
|
| 165 |
+
result["aria-label"] = label
|
| 166 |
+
data = raw.get("data")
|
| 167 |
+
if isinstance(data, dict):
|
| 168 |
+
safe = {}
|
| 169 |
+
for key, val in list(data.items())[:MAX_DATA_ATTRIBUTES]:
|
| 170 |
+
if not isinstance(key, str) or not re.fullmatch(r"data-[a-z][a-z-]{0,47}", key):
|
| 171 |
+
continue
|
| 172 |
+
normalized = re.sub(r"[^a-z]", "", key[5:])
|
| 173 |
+
if any(word in normalized for word in SECRET_KEYS):
|
| 174 |
+
continue
|
| 175 |
+
val = _text(val)
|
| 176 |
+
if not val:
|
| 177 |
+
continue
|
| 178 |
+
safe[key] = val if val in _DATA_ENUMS.get(key, set()) else "present"
|
| 179 |
+
if safe:
|
| 180 |
+
result["data"] = safe
|
| 181 |
+
return result
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def _key(value, target):
|
| 185 |
+
value = _text(value, 64)
|
| 186 |
+
parts = value.split("+")
|
| 187 |
+
key = parts[-1]
|
| 188 |
+
if any(part not in {"Control", "Alt", "Shift", "Meta"} for part in parts[:-1]):
|
| 189 |
+
return "Key"
|
| 190 |
+
named = {"Enter", "Escape", "Tab", "Backspace", "Delete", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "Home", "End", "PageUp", "PageDown", "Control", "Alt", "Shift", "Meta", "CapsLock", "Space", "Character"}
|
| 191 |
+
if key not in named and not re.fullmatch(r"(?:[a-zA-Z0-9]|F(?:[1-9]|1[0-2]))", key):
|
| 192 |
+
return "Key"
|
| 193 |
+
editable = target.get("tagName") in {"input", "textarea", "option", "select"} or target.get("role") in {"textbox", "searchbox", "gridcell", "cell", "grid"}
|
| 194 |
+
shortcut = any(part in {"Control", "Meta"} for part in parts[:-1]) and "Alt" not in parts[:-1]
|
| 195 |
+
if editable and (len(key) == 1 or key == "Space") and not shortcut:
|
| 196 |
+
return "Character"
|
| 197 |
+
return value
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def project_event(raw, now_ms, *, movement=False):
|
| 201 |
+
raw = _shape(raw)
|
| 202 |
+
kind = raw.get("kind")
|
| 203 |
+
allowed = {"hover", "scroll", "focus", "keypress", "pointer"} if movement else {"route", "click", "api", "error"}
|
| 204 |
+
if not isinstance(kind, str) or kind not in allowed:
|
| 205 |
+
return None
|
| 206 |
+
ts = _number(raw.get("ts"), now_ms, now_ms)
|
| 207 |
+
if ts < now_ms - RETENTION_DAYS * 86400_000:
|
| 208 |
+
return None
|
| 209 |
+
event = {"kind": kind, "ts": ts}
|
| 210 |
+
if movement or kind == "click":
|
| 211 |
+
event["target"] = _element(raw.get("target"))
|
| 212 |
+
if movement:
|
| 213 |
+
if kind in {"pointer", "scroll"}:
|
| 214 |
+
event.update({key: _number(raw.get(key), maximum=10**7) for key in ("x", "y")})
|
| 215 |
+
if kind == "keypress":
|
| 216 |
+
event["key"] = _key(raw.get("key"), event["target"])
|
| 217 |
+
return event
|
| 218 |
+
event["route"] = _path(raw.get("route"))
|
| 219 |
+
if kind == "route":
|
| 220 |
+
event.update({"from": _path(raw.get("from")), "to": _path(raw.get("to")), "ms": _number(raw.get("ms"))})
|
| 221 |
+
elif kind == "api":
|
| 222 |
+
method = _text(raw.get("method"), 16).upper()
|
| 223 |
+
event.update({"method": method if method in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"} else "OTHER",
|
| 224 |
+
"url": _path(raw.get("url")), "status": _number(raw.get("status"), maximum=599), "ms": _number(raw.get("ms"))})
|
| 225 |
+
elif kind == "error":
|
| 226 |
+
name = _text(raw.get("name"), 64)
|
| 227 |
+
message = _text(raw.get("message"))
|
| 228 |
+
event.update({"name": name if name in _ERROR_NAMES else "Error",
|
| 229 |
+
# Raw exception text often interpolates the offending cell.
|
| 230 |
+
"message": message if message in _MESSAGES else "Application error.",
|
| 231 |
+
"source": _path(raw.get("source")), "line": _number(raw.get("line"), maximum=10**7)})
|
| 232 |
+
samples = raw.get("movement")
|
| 233 |
+
samples = samples if isinstance(samples, list) else []
|
| 234 |
+
trail = [project_event(sample, now_ms, movement=True) for sample in samples[-MAX_MOVEMENT_SAMPLES:]]
|
| 235 |
+
event["movement"] = sorted([sample for sample in trail if sample and ts - MOVEMENT_WINDOW_MS <= sample["ts"] <= ts], key=lambda sample: sample["ts"])
|
| 236 |
+
event["movementDropped"] = _number(raw.get("movementDropped")) + len(samples) - len(event["movement"])
|
| 237 |
+
return event
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def _limits():
|
| 241 |
+
return {"retentionDays": RETENTION_DAYS, "maxSessions": MAX_SESSIONS,
|
| 242 |
+
"maxEventsPerSession": MAX_EVENTS_PER_SESSION, "maxMovementSamples": MAX_MOVEMENT_SAMPLES,
|
| 243 |
+
"movementWindowSeconds": MOVEMENT_WINDOW_MS // 1000, "pageSize": MAX_PAGE_EVENTS}
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def _retained(doc, tenant, now_ms):
|
| 247 |
+
if not isinstance(doc, dict) or not isinstance(doc.get("sessions"), list):
|
| 248 |
+
return []
|
| 249 |
+
cutoff = now_ms - RETENTION_DAYS * 86400_000
|
| 250 |
+
rows = [row for row in doc["sessions"] if isinstance(row, dict) and row.get("tenant") == tenant
|
| 251 |
+
and isinstance(row.get("events"), list) and _number(row.get("endedAt")) >= cutoff]
|
| 252 |
+
retained = []
|
| 253 |
+
for row in rows:
|
| 254 |
+
events = [event for event in row["events"] if isinstance(event, dict) and _number(event.get("ts")) >= cutoff]
|
| 255 |
+
if events:
|
| 256 |
+
retained.append({**row, "events": events[-MAX_EVENTS_PER_SESSION:]})
|
| 257 |
+
return sorted(retained, key=lambda row: row["endedAt"])[-MAX_SESSIONS:]
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _record(session, session_id, events, dropped, now_ms):
|
| 261 |
+
from harness import runtime
|
| 262 |
+
|
| 263 |
+
def _append(doc):
|
| 264 |
+
rows = _retained(doc, session.tenant, now_ms)
|
| 265 |
+
row = next((item for item in rows if item["sessionId"] == session_id and item["user"] == session.uname), None)
|
| 266 |
+
if row is None:
|
| 267 |
+
row = {"sessionId": session_id, "user": session.uname, "tenant": session.tenant,
|
| 268 |
+
"startedAt": min(event["ts"] for event in events), "endedAt": now_ms,
|
| 269 |
+
"route": next((event.get("to") or event.get("route") for event in events if event.get("to") or event.get("route")), "/"),
|
| 270 |
+
"events": [], "dropped": 0, "prunedEvents": 0}
|
| 271 |
+
rows.append(row)
|
| 272 |
+
merged = sorted(row["events"] + events, key=lambda event: event["ts"])
|
| 273 |
+
if merged[0]["ts"] < row["startedAt"]:
|
| 274 |
+
row["route"] = merged[0].get("to") or merged[0].get("route") or "/"
|
| 275 |
+
row["startedAt"] = min(row["startedAt"], merged[0]["ts"])
|
| 276 |
+
row["endedAt"] = merged[-1]["ts"]
|
| 277 |
+
row["prunedEvents"] += max(0, len(merged) - MAX_EVENTS_PER_SESSION)
|
| 278 |
+
row["dropped"] += dropped
|
| 279 |
+
row["events"] = merged[-MAX_EVENTS_PER_SESSION:]
|
| 280 |
+
return {"sessions": sorted(rows, key=lambda item: item["endedAt"])[-MAX_SESSIONS:]}
|
| 281 |
+
|
| 282 |
+
# Async coalescing avoids blocking uploads and one HF commit per batch.
|
| 283 |
+
# A container loss before the worker flushes may lose cached telemetry.
|
| 284 |
+
runtime.get_runtime(str(session.tenant)).update(BUCKET, _append, flush="async")
|
| 285 |
+
# platform/ is available after session resolution, not necessarily at import.
|
| 286 |
+
try:
|
| 287 |
+
from harness.telemetry import error as telemetry_error
|
| 288 |
+
except ImportError:
|
| 289 |
+
return
|
| 290 |
+
for event in events:
|
| 291 |
+
if event["kind"] == "error":
|
| 292 |
+
try:
|
| 293 |
+
# This is the existing diagnostic sink, not a second activity store.
|
| 294 |
+
# Never forward exc, the incoming payload, or the movement snapshot.
|
| 295 |
+
telemetry_error("activity", name=event["name"], message=event["message"],
|
| 296 |
+
source=event["source"], line=event["line"])
|
| 297 |
+
except Exception:
|
| 298 |
+
pass
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
def _batch(body):
|
| 302 |
+
events = body.get("events")
|
| 303 |
+
session_id = _text(body.get("sessionId"), 80)
|
| 304 |
+
if not _SID.fullmatch(session_id) or not isinstance(events, list):
|
| 305 |
+
raise ActivityRefused("activity_invalid", "Activity requires a tab session and an event list.")
|
| 306 |
+
if len(events) > MAX_BATCH_EVENTS:
|
| 307 |
+
raise ActivityRefused("activity_batch_too_large", f"Activity batches must contain at most {MAX_BATCH_EVENTS} events.")
|
| 308 |
+
return session_id, events
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
@router.post("/api/v1/activity")
|
| 312 |
+
async def ingest_activity(request: Request, session: Session = Depends(require_session)):
|
| 313 |
+
data = bytearray()
|
| 314 |
+
async for chunk in request.stream():
|
| 315 |
+
if len(data) + len(chunk) > MAX_PAYLOAD_BYTES:
|
| 316 |
+
raise err(400, "activity_payload_too_large", f"Activity batches must be at most {MAX_PAYLOAD_BYTES} bytes.")
|
| 317 |
+
data.extend(chunk)
|
| 318 |
+
try:
|
| 319 |
+
body = _shape(json.loads(data))
|
| 320 |
+
except (ValueError, UnicodeError, RecursionError):
|
| 321 |
+
raise err(400, "activity_invalid", "Activity requires a valid JSON object.") from None
|
| 322 |
+
try:
|
| 323 |
+
session_id, events = _batch(body)
|
| 324 |
+
except ActivityRefused as exc:
|
| 325 |
+
raise err(400, exc.code, exc.message) from None
|
| 326 |
+
now_ms = round(time.time() * 1000)
|
| 327 |
+
clean = []
|
| 328 |
+
for raw in events:
|
| 329 |
+
event = project_event(raw, now_ms)
|
| 330 |
+
if event is not None:
|
| 331 |
+
clean.append({**event, "eventId": uuid.uuid4().hex})
|
| 332 |
+
dropped = len(events) - len(clean)
|
| 333 |
+
clean.sort(key=lambda event: event["ts"])
|
| 334 |
+
if clean:
|
| 335 |
+
try:
|
| 336 |
+
await run_in_threadpool(_record, session, session_id, clean,
|
| 337 |
+
_number(body.get("dropped")) + dropped, now_ms)
|
| 338 |
+
except Exception:
|
| 339 |
+
raise err(503, "activity_unavailable", "Activity could not be saved.") from None
|
| 340 |
+
return {"accepted": len(clean), "dropped": dropped}
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def _user(session, requested):
|
| 344 |
+
if requested is not None and requested != session.uname and not session.admin:
|
| 345 |
+
raise err(403, "activity_forbidden", "You can only view your own activity.")
|
| 346 |
+
return session.uname if requested is None else requested
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def _read(session):
|
| 350 |
+
from harness import runtime
|
| 351 |
+
try:
|
| 352 |
+
return _retained(runtime.get_runtime(str(session.tenant)).get(BUCKET), session.tenant, round(time.time() * 1000))
|
| 353 |
+
except Exception:
|
| 354 |
+
raise err(503, "activity_unavailable", "Activity could not be loaded.") from None
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
@router.get("/api/v1/activity/sessions")
|
| 358 |
+
def activity_sessions(user: str | None = None, session: Session = Depends(require_session)):
|
| 359 |
+
owner = _user(session, user)
|
| 360 |
+
rows = []
|
| 361 |
+
for row in reversed(_read(session)):
|
| 362 |
+
if row["user"] == owner:
|
| 363 |
+
rows.append({key: row[key] for key in ("sessionId", "startedAt", "endedAt", "route", "dropped", "prunedEvents")})
|
| 364 |
+
rows[-1].update(events=len(row["events"]), errors=sum(event["kind"] == "error" for event in row["events"]))
|
| 365 |
+
return {"sessions": rows, "limits": _limits()}
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
@router.get("/api/v1/activity/sessions/{session_id}")
|
| 369 |
+
def activity_session(session_id: str, user: str | None = None,
|
| 370 |
+
limit: int = Query(MAX_PAGE_EVENTS, ge=1, le=MAX_PAGE_EVENTS),
|
| 371 |
+
offset: int = Query(0, ge=0), session: Session = Depends(require_session)):
|
| 372 |
+
owner = _user(session, user)
|
| 373 |
+
row = next((item for item in _read(session) if item["user"] == owner and item["sessionId"] == session_id), None)
|
| 374 |
+
if row is None:
|
| 375 |
+
raise err(404, "activity_session_missing", "This activity session is no longer available.")
|
| 376 |
+
# Re-project the displayed slice as defense against a legacy or corrupt document.
|
| 377 |
+
now_ms = round(time.time() * 1000)
|
| 378 |
+
events = []
|
| 379 |
+
for raw in row["events"][offset:offset + limit]:
|
| 380 |
+
clean = project_event(raw, now_ms)
|
| 381 |
+
if clean:
|
| 382 |
+
events.append({**clean, "eventId": _text(raw.get("eventId"), 36)})
|
| 383 |
+
return {"sessionId": session_id, "events": events, "total": len(row["events"]),
|
| 384 |
+
"offset": offset, "limit": limit, "dropped": row["dropped"],
|
| 385 |
+
"prunedEvents": row["prunedEvents"], "limits": _limits()}
|
| 386 |
+
|
| 387 |
+
|
| 388 |
+
@router.get("/api/v1/activity/users")
|
| 389 |
+
def activity_users(session: Session = Depends(require_session)):
|
| 390 |
+
if not session.admin:
|
| 391 |
+
raise err(403, "activity_forbidden", "Only administrators can view other users' activity.")
|
| 392 |
+
return {"users": sorted({row["user"] for row in _read(session)})}
|
api/routes_admin.py
CHANGED
|
@@ -2091,8 +2091,13 @@ def _confines_to_one_bu(tree):
|
|
| 2091 |
return any(kids) if node.get("conj") == "or" else all(kids)
|
| 2092 |
if node.get("colId") != "dba":
|
| 2093 |
return True # every other condition: assume satisfied
|
| 2094 |
-
want = str(node.get("value") or "").strip().lower()
|
| 2095 |
op = node.get("op")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2096 |
if op == "eq":
|
| 2097 |
return brand.lower() == want
|
| 2098 |
if op == "neq":
|
|
@@ -2107,6 +2112,8 @@ def _confines_to_one_bu(tree):
|
|
| 2107 |
return True
|
| 2108 |
return True # an op that cannot judge a brand does not confine
|
| 2109 |
|
|
|
|
|
|
|
| 2110 |
nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and")
|
| 2111 |
root = {"conj": conj, "children": nodes}
|
| 2112 |
admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)}
|
|
|
|
| 2091 |
return any(kids) if node.get("conj") == "or" else all(kids)
|
| 2092 |
if node.get("colId") != "dba":
|
| 2093 |
return True # every other condition: assume satisfied
|
|
|
|
| 2094 |
op = node.get("op")
|
| 2095 |
+
if op in fe.MULTI_VALUE_OPS:
|
| 2096 |
+
# HOTFIX 2026-09-23: `dba is any of [Royal]` is the shape the condition builder writes
|
| 2097 |
+
# now, and `perm_scope` already reads it. Falling to "cannot judge" here made a save
|
| 2098 |
+
# that KEPT the BU condition read as one that removed it, and refused it as bu_widening.
|
| 2099 |
+
return fe._match_multi_value(op, brand, fe.parse_filter_values(node.get("values")))
|
| 2100 |
+
want = str(node.get("value") or "").strip().lower()
|
| 2101 |
if op == "eq":
|
| 2102 |
return brand.lower() == want
|
| 2103 |
if op == "neq":
|
|
|
|
| 2112 |
return True
|
| 2113 |
return True # an op that cannot judge a brand does not confine
|
| 2114 |
|
| 2115 |
+
from harness import filter_eval as fe
|
| 2116 |
+
|
| 2117 |
nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and")
|
| 2118 |
root = {"conj": conj, "children": nodes}
|
| 2119 |
admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)}
|
api/routes_applications.py
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Permissioned, directly addressable Finance application contracts (S63-T16)."""
|
| 2 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 3 |
+
|
| 4 |
+
import application_registry
|
| 5 |
+
from deps import Session, err, require_session
|
| 6 |
+
from harness import runtime as mirror_runtime
|
| 7 |
+
from modules import finance_spine
|
| 8 |
+
from pages import resolve_bu
|
| 9 |
+
from routes_activity import router as activity_router
|
| 10 |
+
from routes_statements import _gate as statement_gate
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
finance_router = APIRouter(prefix="/api/v1/finance", tags=["finance"])
|
| 14 |
+
nav_router = APIRouter(prefix="/api/v1", tags=["applications"])
|
| 15 |
+
# ⛔ NO `prefix=` HERE, for the same reason S65-T16 spelled full paths on `routes_activity`:
|
| 16 |
+
# this router is included INSIDE this module's own `router`, and a child router carrying
|
| 17 |
+
# `prefix=` has that prefix baked into its `.routes` at DECLARATION time, so
|
| 18 |
+
# `verify_api.py::_mounted_paths` adds it a SECOND time and D-48 reads a route the app really
|
| 19 |
+
# serves as one it does not. The full path on each decorator is also the honest reading.
|
| 20 |
+
collection_router = APIRouter(tags=["collection"], dependencies=[Depends(require_session)])
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _may_open(session, permission_key):
|
| 24 |
+
try:
|
| 25 |
+
session.require(permission_key)
|
| 26 |
+
return True
|
| 27 |
+
except Exception:
|
| 28 |
+
return False
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _require_app(session, key):
|
| 32 |
+
try:
|
| 33 |
+
return application_registry.finance_application(
|
| 34 |
+
key, may_open=lambda permission: _may_open(session, permission))
|
| 35 |
+
except PermissionError:
|
| 36 |
+
raise err(403, "finance_forbidden", "your account does not have access to this Finance application")
|
| 37 |
+
except KeyError:
|
| 38 |
+
raise err(404, "finance_app_not_found", "Finance application not found")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _require_fact(session, key):
|
| 42 |
+
candidates = [app for app in application_registry.finance_applications()
|
| 43 |
+
if key in app["factKeys"]]
|
| 44 |
+
if not candidates:
|
| 45 |
+
raise err(404, "finance_fact_not_found", "Finance fact not found")
|
| 46 |
+
if not any(_may_open(session, app["permissionKey"]) for app in candidates):
|
| 47 |
+
raise err(403, "finance_forbidden", "your account does not have access to this Finance fact")
|
| 48 |
+
return finance_spine.fact_spec(key)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@finance_router.get("/applications")
|
| 52 |
+
def list_finance_applications(session: Session = Depends(require_session)):
|
| 53 |
+
"""One Finance group containing only the applications this session may open."""
|
| 54 |
+
apps = application_registry.finance_applications(
|
| 55 |
+
may_open=lambda permission: _may_open(session, permission))
|
| 56 |
+
apps = [{**app, "id": app["key"], "name": app["navLabel"],
|
| 57 |
+
"navPlacement": {"section": "finance", "order": index}}
|
| 58 |
+
for index, app in enumerate(apps)]
|
| 59 |
+
return {"group": application_registry.finance_group(), "applications": apps}
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@finance_router.get("/applications/{app_key}")
|
| 63 |
+
def get_finance_application(app_key: str, bu: str | None = None,
|
| 64 |
+
date_from: str | None = None, date_to: str | None = None,
|
| 65 |
+
session: Session = Depends(require_session)):
|
| 66 |
+
"""Render one deep-linked app; this endpoint deliberately has no Finance-home aggregate."""
|
| 67 |
+
app = _require_app(session, app_key)
|
| 68 |
+
team_id = resolve_bu(session, bu)
|
| 69 |
+
try:
|
| 70 |
+
cursor = mirror_runtime.mirror_cursor(session.runtime)
|
| 71 |
+
except mirror_runtime.DatastoreMismatch:
|
| 72 |
+
raise err(503, "finance_store_mismatch", "the Finance mirror belongs to a different workspace")
|
| 73 |
+
except RuntimeError:
|
| 74 |
+
facts = [{**finance_spine.fact_spec(key), "state": "warming", "value": None}
|
| 75 |
+
for key in app["factKeys"]]
|
| 76 |
+
return {"group": application_registry.finance_group(), "application": app,
|
| 77 |
+
"businessUnit": team_id, "state": "warming",
|
| 78 |
+
"facts": facts,
|
| 79 |
+
"kpis": [fact for fact in facts if fact["key"] in app["kpiKeys"]]}
|
| 80 |
+
facts = [finance_spine.observe(key, cursor, date_from=date_from, date_to=date_to,
|
| 81 |
+
team_id=team_id) for key in app["factKeys"]]
|
| 82 |
+
kpis = [fact for fact in facts if fact["key"] in app["kpiKeys"]]
|
| 83 |
+
state = "ready" if all(fact["state"] in {"ready", "requires_sku"} for fact in facts) else "partial"
|
| 84 |
+
return {"group": application_registry.finance_group(), "application": app,
|
| 85 |
+
"businessUnit": team_id, "state": state, "facts": facts, "kpis": kpis}
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
@finance_router.get("/facts/{fact_key}/records")
|
| 89 |
+
def get_finance_fact_records(fact_key: str, bu: str | None = None,
|
| 90 |
+
date_from: str | None = None, date_to: str | None = None,
|
| 91 |
+
limit: int = Query(100, ge=1, le=500), offset: int = Query(0, ge=0),
|
| 92 |
+
session: Session = Depends(require_session)):
|
| 93 |
+
"""Source-record drill-through under the same permission and tenant mirror walls."""
|
| 94 |
+
spec = _require_fact(session, fact_key)
|
| 95 |
+
team_id = resolve_bu(session, bu)
|
| 96 |
+
try:
|
| 97 |
+
finance_spine.effective_team_id(spec, team_id)
|
| 98 |
+
mirror_runtime.mirror_cursor(session.runtime)
|
| 99 |
+
return finance_spine.source_records(
|
| 100 |
+
fact_key, date_from=date_from, date_to=date_to, team_id=team_id,
|
| 101 |
+
limit=limit, offset=offset)
|
| 102 |
+
except PermissionError:
|
| 103 |
+
raise err(403, "finance_scope_forbidden", "this fact requires consolidated Finance access")
|
| 104 |
+
except ValueError as exc:
|
| 105 |
+
raise err(400, "finance_scope_invalid", str(exc))
|
| 106 |
+
except mirror_runtime.DatastoreMismatch:
|
| 107 |
+
raise err(503, "finance_store_mismatch", "the Finance mirror belongs to a different workspace")
|
| 108 |
+
except RuntimeError:
|
| 109 |
+
return {"fact": spec, "state": "warming", "rows": [], "row_count": None,
|
| 110 |
+
"total_count": None, "aggregates": {}}
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
@finance_router.get("/facts/{fact_key}/reconciliation")
|
| 114 |
+
def reconcile_finance_fact(fact_key: str, bu: str | None = None,
|
| 115 |
+
date_from: str | None = None, date_to: str | None = None,
|
| 116 |
+
session: Session = Depends(require_session)):
|
| 117 |
+
"""Explicit independent Odoo read; ordinary page loads never issue live source reads."""
|
| 118 |
+
spec = _require_fact(session, fact_key)
|
| 119 |
+
team_id = resolve_bu(session, bu)
|
| 120 |
+
try:
|
| 121 |
+
finance_spine.effective_team_id(spec, team_id)
|
| 122 |
+
mirror_runtime.mirror_cursor(session.runtime)
|
| 123 |
+
except PermissionError:
|
| 124 |
+
raise err(403, "finance_scope_forbidden", "this fact requires consolidated Finance access")
|
| 125 |
+
except ValueError as exc:
|
| 126 |
+
raise err(400, "finance_scope_invalid", str(exc))
|
| 127 |
+
except mirror_runtime.DatastoreMismatch:
|
| 128 |
+
raise err(503, "finance_store_mismatch", "the Finance mirror belongs to a different workspace")
|
| 129 |
+
except RuntimeError:
|
| 130 |
+
return {"state": "warming", "message": "The tenant mirror is still warming."}
|
| 131 |
+
return finance_spine.reconcile(
|
| 132 |
+
fact_key, date_from=date_from, date_to=date_to, team_id=team_id)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
@nav_router.get("/nav/applications")
|
| 136 |
+
def list_nav_applications(session: Session = Depends(require_session)):
|
| 137 |
+
"""Every Application this session may open, in ONE generic navigation shape (R6).
|
| 138 |
+
|
| 139 |
+
Finance's six arrive here as seeded Application records rather than as new template kinds,
|
| 140 |
+
so the App module reaches them by the same path Routing and Catalog ride.
|
| 141 |
+
"""
|
| 142 |
+
if not session.runtime.available():
|
| 143 |
+
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 144 |
+
return {"applications": application_registry.nav_applications(
|
| 145 |
+
session.runtime, session.uname, session.admin,
|
| 146 |
+
may_open=lambda permission: _may_open(session, permission))}
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def _collection_ids(body, *, confirm=False):
|
| 150 |
+
body = body or {}
|
| 151 |
+
if confirm and body.get("confirm") is not True:
|
| 152 |
+
raise err(400, "collection_confirmation_required", "Confirm the whole batch before sending.")
|
| 153 |
+
allowed = {"partnerIds", "confirm"} if confirm else {"partnerIds"}
|
| 154 |
+
if set(body) - allowed:
|
| 155 |
+
raise err(400, "collection_request_invalid", "Only customer identifiers are accepted.")
|
| 156 |
+
ids = body.get("partnerIds")
|
| 157 |
+
if not isinstance(ids, list) or not ids:
|
| 158 |
+
raise err(400, "collection_selection_required", "Select at least one customer.")
|
| 159 |
+
if any(type(pid) is not int or pid <= 0 for pid in ids):
|
| 160 |
+
raise err(400, "collection_selection_invalid", "Customer identifiers must be positive integers.")
|
| 161 |
+
return list(dict.fromkeys(ids))
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _collection_source(cs):
|
| 165 |
+
try:
|
| 166 |
+
odoo = cs.Odoo()
|
| 167 |
+
return odoo, cs.load_collection_list(odoo)
|
| 168 |
+
except Exception:
|
| 169 |
+
raise err(502, "collection_odoo_unavailable", "The Collection worklist could not be read from Odoo.")
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _collection_selection(rows, ids):
|
| 173 |
+
by_id = {row["partner_id"]: row for row in rows}
|
| 174 |
+
selected = [by_id[pid] for pid in ids if pid in by_id]
|
| 175 |
+
refused = [{"customer": f"Customer {pid}", "partnerId": pid,
|
| 176 |
+
"reason": "This customer is no longer on the Collection worklist."}
|
| 177 |
+
for pid in ids if pid not in by_id]
|
| 178 |
+
return selected, refused
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def _collection_refusal(cs, row):
|
| 182 |
+
"""Preview the existing data-layer guard; the sender still enforces it independently."""
|
| 183 |
+
if not row.get("Email"):
|
| 184 |
+
return "No email address is saved for this customer."
|
| 185 |
+
if not cs.safe_recipient_ok(row["Email"]):
|
| 186 |
+
return "Safe mode does not allow this recipient address."
|
| 187 |
+
return None
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
@collection_router.get("/api/v1/apps/collection/worklist")
|
| 191 |
+
def collection_worklist(session: Session = Depends(statement_gate)):
|
| 192 |
+
import modules.collections_send as cs
|
| 193 |
+
_, rows = _collection_source(cs)
|
| 194 |
+
public = []
|
| 195 |
+
for row in rows:
|
| 196 |
+
docs = row.get("_docs") or []
|
| 197 |
+
public.append({**{key: value for key, value in row.items() if key != "_docs"},
|
| 198 |
+
"_docs": len(docs),
|
| 199 |
+
"documents": [{"name": doc["name"],
|
| 200 |
+
"date": doc.get("invoice_date") or None,
|
| 201 |
+
"dueDate": doc.get("invoice_date_due") or None,
|
| 202 |
+
"daysOverdue": doc["days_overdue"],
|
| 203 |
+
"openBalance": doc["open_signed"]} for doc in docs],
|
| 204 |
+
"sendRefusal": _collection_refusal(cs, row)})
|
| 205 |
+
return {"rows": public, "safeMode": bool(cs.SAFE_MODE),
|
| 206 |
+
"safeRecipients": sorted(cs.SAFE_RECIPIENTS),
|
| 207 |
+
"sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, "replyTo": cs.REPLY_TO}}
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
@collection_router.post("/api/v1/apps/collection/preview")
|
| 211 |
+
def collection_preview(body: dict = Body(default=None), session: Session = Depends(statement_gate)):
|
| 212 |
+
"""Read-only rendered statements, using the same defaults the batch will queue."""
|
| 213 |
+
import datetime as dt
|
| 214 |
+
import modules.collections_send as cs
|
| 215 |
+
ids = _collection_ids(body)
|
| 216 |
+
_, rows = _collection_source(cs)
|
| 217 |
+
selected, refused = _collection_selection(rows, ids)
|
| 218 |
+
statements = []
|
| 219 |
+
for row in selected:
|
| 220 |
+
reason = _collection_refusal(cs, row)
|
| 221 |
+
if reason:
|
| 222 |
+
refused.append({"customer": row["Customer"], "reason": reason})
|
| 223 |
+
statements.append({"partnerId": row["partner_id"], "customer": row["Customer"],
|
| 224 |
+
"to": row["Email"], "refusal": reason,
|
| 225 |
+
"subject": cs.DEFAULT_SUBJECT.format(
|
| 226 |
+
customer=row["Customer"], company=cs.COMPANY,
|
| 227 |
+
month=dt.date.today().strftime("%B %Y")),
|
| 228 |
+
"html": cs.render_statement_html(row)})
|
| 229 |
+
return {"statements": statements, "refused": refused, "requested": len(ids),
|
| 230 |
+
"safeMode": bool(cs.SAFE_MODE), "safeRecipients": sorted(cs.SAFE_RECIPIENTS),
|
| 231 |
+
"sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, "replyTo": cs.REPLY_TO}}
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
@collection_router.post("/api/v1/apps/collection/send")
|
| 235 |
+
def collection_send(body: dict = Body(default=None), session: Session = Depends(statement_gate)):
|
| 236 |
+
import modules.collections_send as cs
|
| 237 |
+
ids = _collection_ids(body, confirm=True)
|
| 238 |
+
odoo, rows = _collection_source(cs)
|
| 239 |
+
selected, refused = _collection_selection(rows, ids)
|
| 240 |
+
try:
|
| 241 |
+
result = cs.queue_statement_batch(odoo, selected)
|
| 242 |
+
except Exception:
|
| 243 |
+
# An interrupted response may follow a successful create. Never invite a blind retry.
|
| 244 |
+
raise err(502, "collection_send_unconfirmed",
|
| 245 |
+
"Odoo did not confirm the batch result. Some statements may already be in the "
|
| 246 |
+
"outbox. Check Odoo before trying again.")
|
| 247 |
+
return {"created": len(result["created"]), "refused": refused + result["refused"],
|
| 248 |
+
"safeMode": result["safe_mode"]}
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
# main.py includes exactly one object named router. These sub-routers keep
|
| 252 |
+
# /nav/applications outside the Finance prefix while every Finance URL stays unchanged.
|
| 253 |
+
router = APIRouter()
|
| 254 |
+
router.include_router(finance_router)
|
| 255 |
+
router.include_router(nav_router)
|
| 256 |
+
# main.py belongs to another session this wave; this router is already mounted there.
|
| 257 |
+
router.include_router(activity_router)
|
| 258 |
+
router.include_router(collection_router)
|
api/routes_automation.py
CHANGED
|
@@ -433,6 +433,7 @@ def _wire(defn, tenant, rt=None):
|
|
| 433 |
"config": defn.get("config") or {}, "schedule": sched, "status": status,
|
| 434 |
"runs": list(defn.get("runs") or [])[:engine.MAX_RUNS],
|
| 435 |
"created": defn.get("created"), "createdBy": defn.get("createdBy"),
|
|
|
|
| 436 |
"nextRunAt": nxt.strftime("%Y-%m-%d %H:%M") if nxt else "",
|
| 437 |
"running": bool(live),
|
| 438 |
# ⭐ WAVE 27 (contract C6) — DEBT D-70: "a paid search is already outstanding at the
|
|
@@ -1210,8 +1211,9 @@ def list_automations(session: Session = Depends(_GATE)):
|
|
| 1210 |
#
|
| 1211 |
# ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent":
|
| 1212 |
# `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s
|
| 1213 |
-
# `_without_retired_board`)
|
| 1214 |
-
#
|
|
|
|
| 1215 |
# wave 26's R6 deleted the board that made them.
|
| 1216 |
# ⭐⭐ WAVE 30 · T12 (owner items 4/5, the complaint that has now survived TWO waves).
|
| 1217 |
# ⛔ W29-T01 GUARDED THE SCAN AND NOT THE READ, and that is the whole of what was left. The
|
|
@@ -1221,6 +1223,13 @@ def list_automations(session: Session = Depends(_GATE)):
|
|
| 1221 |
# behind each other) — while on calls 2..N the result was DISCARDED: `tables` had exactly two
|
| 1222 |
# consumers, the `_BOARD_RETIRED` scan (skipped after call 1) and a picker DEFAULT STRING.
|
| 1223 |
# A whole-tenant document, per request, to render a rail showing a name and a toggle.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1224 |
tables = None
|
| 1225 |
if session.runtime.available() and session.tenant not in _BOARD_RETIRED:
|
| 1226 |
tables = engine.ut_all(session.runtime)
|
|
@@ -1228,6 +1237,7 @@ def list_automations(session: Session = Depends(_GATE)):
|
|
| 1228 |
# Idempotent Board retirement removes only engine-marked stage fields and legacy Board
|
| 1229 |
# state. User-created Status/Stage columns remain intact.
|
| 1230 |
engine.retire_automation_board_state(session.runtime, tables=tables)
|
|
|
|
| 1231 |
# ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY
|
| 1232 |
# ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds
|
| 1233 |
# it present and returns immediately. See `engine.seed_statements_agent` for why this one is
|
|
@@ -1254,6 +1264,7 @@ def list_automations(session: Session = Depends(_GATE)):
|
|
| 1254 |
items = sorted(items + _field_agent_rows(session) + _sys_rows,
|
| 1255 |
key=lambda r: str(r.get("name") or "").lower())
|
| 1256 |
return {"automations": items,
|
|
|
|
| 1257 |
# ⭐ WAVE 24 — DERIVED from the engine's public `PICKER_KINDS`, not a hand-written trio. It was
|
| 1258 |
# three literals that happened to match, which is a second copy of a server
|
| 1259 |
# vocabulary; R6 has just made two of them uncreatable and `plain` has joined, so a
|
|
@@ -2114,7 +2125,7 @@ def create_automation(body: dict = Body(default=None), session: Session = Depend
|
|
| 2114 |
# ⚠ It rides the SAVE's own response, beside `unconfigured`, because that is the moment the
|
| 2115 |
# person is looking — a note in a log they never open is the same silence with extra steps.
|
| 2116 |
_notes = []
|
| 2117 |
-
defn, error = engine.create(session.runtime, body
|
| 2118 |
notes=_notes)
|
| 2119 |
if error:
|
| 2120 |
raise err(400, "invalid_automation", error)
|
|
@@ -2159,8 +2170,9 @@ def patch_automation(auto_id: str, body: dict = Body(default=None),
|
|
| 2159 |
# attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the
|
| 2160 |
# step and permits an ordinary save of a flow that merely contains one.
|
| 2161 |
_agent_owned_guard(session, auto_id, body or {})
|
|
|
|
| 2162 |
_notes = []
|
| 2163 |
-
defn, error = engine.patch(session.runtime, auto_id, body
|
| 2164 |
notes=_notes)
|
| 2165 |
if error:
|
| 2166 |
raise err(400 if error != "no such automation" else 404,
|
|
|
|
| 433 |
"config": defn.get("config") or {}, "schedule": sched, "status": status,
|
| 434 |
"runs": list(defn.get("runs") or [])[:engine.MAX_RUNS],
|
| 435 |
"created": defn.get("created"), "createdBy": defn.get("createdBy"),
|
| 436 |
+
**({"preset": str(defn.get("preset"))} if defn.get("preset") else {}),
|
| 437 |
"nextRunAt": nxt.strftime("%Y-%m-%d %H:%M") if nxt else "",
|
| 438 |
"running": bool(live),
|
| 439 |
# ⭐ WAVE 27 (contract C6) — DEBT D-70: "a paid search is already outstanding at the
|
|
|
|
| 1211 |
#
|
| 1212 |
# ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent":
|
| 1213 |
# `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s
|
| 1214 |
+
# `_without_retired_board`) and drops retired presets, so persistence is hygiene, not
|
| 1215 |
+
# correctness. No client can see state this skip left behind. Nothing writes new
|
| 1216 |
+
# `stage_auto_` cells either —
|
| 1217 |
# wave 26's R6 deleted the board that made them.
|
| 1218 |
# ⭐⭐ WAVE 30 · T12 (owner items 4/5, the complaint that has now survived TWO waves).
|
| 1219 |
# ⛔ W29-T01 GUARDED THE SCAN AND NOT THE READ, and that is the whole of what was left. The
|
|
|
|
| 1223 |
# behind each other) — while on calls 2..N the result was DISCARDED: `tables` had exactly two
|
| 1224 |
# consumers, the `_BOARD_RETIRED` scan (skipped after call 1) and a picker DEFAULT STRING.
|
| 1225 |
# A whole-tenant document, per request, to render a rail showing a name and a toggle.
|
| 1226 |
+
# ⛔⛔ AND THE REMOVAL NOTICE OBEYS THE SAME RULE, which is why it is built HERE and not above.
|
| 1227 |
+
# The first draft of this read the whole `automations` document on EVERY request to name the
|
| 1228 |
+
# retired rows, and then a SECOND time to fetch their names — the defect this very comment
|
| 1229 |
+
# block is the tombstone of, re-introduced two lines under it. The sweep has already read
|
| 1230 |
+
# every row it removed, so its receipt carries them: calls 2..N report nothing and read
|
| 1231 |
+
# nothing, which is correct as well as cheap, because after the sweep no retired row exists.
|
| 1232 |
+
retired = []
|
| 1233 |
tables = None
|
| 1234 |
if session.runtime.available() and session.tenant not in _BOARD_RETIRED:
|
| 1235 |
tables = engine.ut_all(session.runtime)
|
|
|
|
| 1237 |
# Idempotent Board retirement removes only engine-marked stage fields and legacy Board
|
| 1238 |
# state. User-created Status/Stage columns remain intact.
|
| 1239 |
engine.retire_automation_board_state(session.runtime, tables=tables)
|
| 1240 |
+
retired = engine.retire_preset_automations(session.runtime)["removed"]
|
| 1241 |
# ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY
|
| 1242 |
# ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds
|
| 1243 |
# it present and returns immediately. See `engine.seed_statements_agent` for why this one is
|
|
|
|
| 1264 |
items = sorted(items + _field_agent_rows(session) + _sys_rows,
|
| 1265 |
key=lambda r: str(r.get("name") or "").lower())
|
| 1266 |
return {"automations": items,
|
| 1267 |
+
"retired": retired,
|
| 1268 |
# ⭐ WAVE 24 — DERIVED from the engine's public `PICKER_KINDS`, not a hand-written trio. It was
|
| 1269 |
# three literals that happened to match, which is a second copy of a server
|
| 1270 |
# vocabulary; R6 has just made two of them uncreatable and `plain` has joined, so a
|
|
|
|
| 2125 |
# ⚠ It rides the SAVE's own response, beside `unconfigured`, because that is the moment the
|
| 2126 |
# person is looking — a note in a log they never open is the same silence with extra steps.
|
| 2127 |
_notes = []
|
| 2128 |
+
defn, error = engine.create(session.runtime, body, username=session.uname,
|
| 2129 |
notes=_notes)
|
| 2130 |
if error:
|
| 2131 |
raise err(400, "invalid_automation", error)
|
|
|
|
| 2170 |
# attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the
|
| 2171 |
# step and permits an ordinary save of a flow that merely contains one.
|
| 2172 |
_agent_owned_guard(session, auto_id, body or {})
|
| 2173 |
+
previous = engine.all_definitions(session.runtime).get(str(auto_id))
|
| 2174 |
_notes = []
|
| 2175 |
+
defn, error = engine.patch(session.runtime, auto_id, body, username=session.uname,
|
| 2176 |
notes=_notes)
|
| 2177 |
if error:
|
| 2178 |
raise err(400 if error != "no such automation" else 404,
|
api/routes_connected_tables.py
CHANGED
|
@@ -62,6 +62,8 @@ def connected_table_rows(table_key: str,
|
|
| 62 |
sorts: str = Query(default=None),
|
| 63 |
search: str = Query(default=None),
|
| 64 |
related: str = Query(default=None),
|
|
|
|
|
|
|
| 65 |
session: Session = Depends(require_session)):
|
| 66 |
"""ONE WINDOW over a connected grid — `{fields, rows, total, totalUnfiltered, offset, limit,
|
| 67 |
limits, identity, recordsMutable}`, whichever connector the rows come from.
|
|
@@ -89,4 +91,5 @@ def connected_table_rows(table_key: str,
|
|
| 89 |
import routes_odoo_tables
|
| 90 |
return routes_odoo_tables.odoo_table_rows(
|
| 91 |
table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
|
| 92 |
-
sorts=sorts, search=search, related=related,
|
|
|
|
|
|
| 62 |
sorts: str = Query(default=None),
|
| 63 |
search: str = Query(default=None),
|
| 64 |
related: str = Query(default=None),
|
| 65 |
+
groupBy: str = Query(default=None),
|
| 66 |
+
groupKey: str = Query(default=None),
|
| 67 |
session: Session = Depends(require_session)):
|
| 68 |
"""ONE WINDOW over a connected grid — `{fields, rows, total, totalUnfiltered, offset, limit,
|
| 69 |
limits, identity, recordsMutable}`, whichever connector the rows come from.
|
|
|
|
| 91 |
import routes_odoo_tables
|
| 92 |
return routes_odoo_tables.odoo_table_rows(
|
| 93 |
table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
|
| 94 |
+
sorts=sorts, search=search, related=related, groupBy=groupBy, groupKey=groupKey,
|
| 95 |
+
session=session)
|
api/routes_connectors.py
CHANGED
|
@@ -21,6 +21,7 @@ that reported another user's Gmail as "connected" would be both wrong and a smal
|
|
| 21 |
from __future__ import annotations
|
| 22 |
|
| 23 |
import os
|
|
|
|
| 24 |
import time
|
| 25 |
import threading
|
| 26 |
import xmlrpc.client
|
|
@@ -30,7 +31,7 @@ from datetime import datetime, timezone
|
|
| 30 |
from fastapi import APIRouter, Body, Depends
|
| 31 |
|
| 32 |
import oauth_connect
|
| 33 |
-
from deps import Session, err, require_session
|
| 34 |
#: ⭐⭐ W49-T18 / R27 / C4 — THE TAXONOMY LIVES IN THE DATA LAYER, NOT HERE. `core` is already on
|
| 35 |
#: the path by the time this module loads (`deps` imports three of its modules), so this is the
|
| 36 |
#: same import shape the rest of the API uses. It is in `platform/core/` rather than beside this
|
|
@@ -82,6 +83,61 @@ from routes_monid import router as monid_router
|
|
| 82 |
router = APIRouter(prefix="/api/v1")
|
| 83 |
|
| 84 |
CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
|
| 87 |
# Measured reads take 1.34 s cold, 0.01 s warm, and above 8 s under boot contention.
|
|
@@ -150,13 +206,16 @@ PLANNED = [
|
|
| 150 |
#: 'Scraper' … would I see the option to choose either Instagram OR TikTok."* So the directory
|
| 151 |
#: mirrors the automation menus exactly: ONE card for the capability, and the platforms are behind
|
| 152 |
#: it rather than beside it.
|
| 153 |
-
#: ⚠ These are not connectors and must never become rows: they have no state
|
| 154 |
-
#:
|
|
|
|
| 155 |
SCRAPER_PLATFORMS = [
|
| 156 |
{"key": "instagram", "label": "Instagram",
|
| 157 |
"desc": "Profiles, posts and comments, by search or by handle."},
|
| 158 |
{"key": "tiktok", "label": "TikTok",
|
| 159 |
"desc": "The same three, on TikTok's own schema."},
|
|
|
|
|
|
|
| 160 |
]
|
| 161 |
|
| 162 |
#: Token connectors an admin can wire up TODAY by pasting a key — no OAuth client, no app review.
|
|
@@ -1063,13 +1122,16 @@ def _scraper_row():
|
|
| 1063 |
# ⭐ WAVE 30 · R4 — the sentence names the CAPABILITY; the platforms are the drill below.
|
| 1064 |
# It used to say "Instagram profile and post capture", which stopped being true the day
|
| 1065 |
# TikTok discovery shipped and would have to be re-edited for every platform after it.
|
| 1066 |
-
"desc": "
|
| 1067 |
"state": "connected" if ready else "unconfigured",
|
| 1068 |
"needs": "" if ready else "AIOS_BRIGHTDATA_KEY is not set on this deployment",
|
| 1069 |
"manage": "",
|
| 1070 |
-
#
|
| 1071 |
-
#
|
| 1072 |
-
|
|
|
|
|
|
|
|
|
|
| 1073 |
"platforms": [dict(p) for p in SCRAPER_PLATFORMS],
|
| 1074 |
}
|
| 1075 |
|
|
@@ -1129,6 +1191,8 @@ def connectors_directory(session: Session = Depends(require_session)):
|
|
| 1129 |
# trap; `chip_counts` carries the argument).
|
| 1130 |
served = connector_directory.to_client_rows(rows)
|
| 1131 |
for row in served:
|
|
|
|
|
|
|
| 1132 |
_safe_connector_status(row.get("key"), row)
|
| 1133 |
_safe_connector_status("amazon_sp", {"incomplete": incomplete})
|
| 1134 |
return {"connectors": served,
|
|
@@ -1162,6 +1226,357 @@ def connectors_directory(session: Session = Depends(require_session)):
|
|
| 1162 |
|
| 1163 |
_ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements"
|
| 1164 |
_ODOO_SYNC_STATUS = "odoo_sync_status"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1165 |
|
| 1166 |
|
| 1167 |
def _odoo_admin(session):
|
|
@@ -1178,8 +1593,7 @@ def _connection_jobs(rt, provider, connection_id):
|
|
| 1178 |
"""A status endpoint selects one connection; the jobs inventory may enumerate them all."""
|
| 1179 |
cid = str(connection_id or "").strip() or "default"
|
| 1180 |
if provider == "odoo" and cid == "default":
|
| 1181 |
-
|
| 1182 |
-
cid = odoo_relational.default_connection_id(rt) or "default"
|
| 1183 |
return connector_sync.jobs(rt, provider, cid)
|
| 1184 |
|
| 1185 |
|
|
@@ -1200,17 +1614,86 @@ def _sync_payload(connector_key, connection_id, row=None):
|
|
| 1200 |
return _safe_connector_status(connector_key, answer)
|
| 1201 |
|
| 1202 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1203 |
def _odoo_sync_runner(session, connection_id):
|
| 1204 |
"""Run Odoo through C6 without moving provider imports into the data layer."""
|
| 1205 |
-
|
|
|
|
|
|
|
|
|
|
| 1206 |
import odoo_relational
|
| 1207 |
-
|
| 1208 |
-
|
| 1209 |
-
|
| 1210 |
-
|
| 1211 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1212 |
tables = result.get("tables") if isinstance(result, dict) else {}
|
| 1213 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1214 |
sync_errors = []
|
| 1215 |
if isinstance(result, dict):
|
| 1216 |
if result.get("failed"):
|
|
@@ -1220,12 +1703,57 @@ def _odoo_sync_runner(session, connection_id):
|
|
| 1220 |
sync_errors.extend(str(value) for value in (discovered.get("errors") or ()))
|
| 1221 |
discovered_tables = discovered.get("tables")
|
| 1222 |
if isinstance(discovered_tables, dict):
|
|
|
|
| 1223 |
databases.extend(str(key) for key in discovered_tables)
|
| 1224 |
if result.get("frozen"):
|
| 1225 |
sync_errors.append(str(result.get("note") or "the connector is frozen"))
|
| 1226 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1227 |
return connector_sync.record(
|
| 1228 |
-
session.runtime, "odoo",
|
| 1229 |
detail=str(result.get("note") or "") if isinstance(result, dict) else "",
|
| 1230 |
errors=sync_errors, databases=sorted(set(databases)),
|
| 1231 |
)
|
|
@@ -1339,6 +1867,9 @@ def _run_database_job(rt, candidate):
|
|
| 1339 |
return
|
| 1340 |
fenced = _MaterializationRuntime(rt, job)
|
| 1341 |
key = job["databaseKey"]
|
|
|
|
|
|
|
|
|
|
| 1342 |
def finish(success=False, cause=""):
|
| 1343 |
# The pre-claim check cannot cover a disconnect during source I/O or publication.
|
| 1344 |
# Only the current lease may requeue; disable and a newer Save still win.
|
|
@@ -1398,22 +1929,64 @@ def _run_database_job(rt, candidate):
|
|
| 1398 |
fenced, job, rows=rows, total=total))}
|
| 1399 |
discovered = result.get("discovered") or {}
|
| 1400 |
published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
|
|
|
|
|
|
|
|
|
|
| 1401 |
success = (key in published and not result.get("failed") and not result.get("frozen")
|
| 1402 |
-
and not result.get("limits") and not discovered.get("errors")
|
| 1403 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1404 |
except _SupersededMaterialization:
|
| 1405 |
# Disable/new Save already invalidated the table marker; never restore it here.
|
| 1406 |
finish()
|
| 1407 |
except rel.Refused as exc:
|
| 1408 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1409 |
except Exception as exc:
|
| 1410 |
-
# Raw provider exceptions may carry credentials or source values. Store only
|
| 1411 |
-
|
|
|
|
| 1412 |
if _odoo_read_access_refused(exc):
|
| 1413 |
-
|
| 1414 |
-
|
| 1415 |
-
|
| 1416 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1417 |
finally:
|
| 1418 |
stopped.set()
|
| 1419 |
if heartbeat_thread:
|
|
@@ -1644,17 +2217,20 @@ def _odoo_field_key(rt, database_key, model, raw_name, connection_id=...):
|
|
| 1644 |
def odoo_status(connection_id: str = None, session: Session = Depends(require_session)):
|
| 1645 |
_odoo_admin(session)
|
| 1646 |
import odoo_relational
|
|
|
|
| 1647 |
cfg = odoo_relational.read_config(session.runtime)
|
| 1648 |
stored = session.runtime.get(_ODOO_SYNC_STATUS) or {}
|
| 1649 |
errors = stored.get("errors") if isinstance(stored.get("errors"), list) else []
|
| 1650 |
phase = str(stored.get("phase") or "unknown")
|
| 1651 |
-
lifecycle = connector_sync.read(session.runtime, "odoo",
|
| 1652 |
durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id))
|
|
|
|
| 1653 |
if durable:
|
| 1654 |
return {"schedule": cfg.get("syncEvery"),
|
| 1655 |
"health": "paused" if cfg.get("frozen") else durable["state"],
|
| 1656 |
"lastRun": durable["at"], "frozen": cfg.get("frozen") is True,
|
| 1657 |
-
**durable, "stateAt": durable["at"], "connectionId":
|
|
|
|
| 1658 |
# The old keys stay the old default-connection answer. A caller that names a connection gets
|
| 1659 |
# that slot's lifecycle instead of the tenant-wide legacy document that caused the C6 defect.
|
| 1660 |
named = connection_id is not None
|
|
@@ -1678,7 +2254,8 @@ def odoo_status(connection_id: str = None, session: Session = Depends(require_se
|
|
| 1678 |
"retryable": state in ("error", "partial"),
|
| 1679 |
"errorsTotal": int(errors_total or 0),
|
| 1680 |
"connectionId": (lifecycle or {}).get("connectionId")
|
| 1681 |
-
or
|
|
|
|
| 1682 |
|
| 1683 |
|
| 1684 |
@router.get("/connectors/{connector_key}/sync")
|
|
@@ -1694,13 +2271,25 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
|
|
| 1694 |
|
| 1695 |
|
| 1696 |
@router.post("/connectors/{connector_key}/sync/now")
|
| 1697 |
-
def connector_sync_now(connector_key: str,
|
|
|
|
| 1698 |
_sync_admin(session)
|
| 1699 |
if connector_key not in connector_sync.providers():
|
| 1700 |
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
|
| 1701 |
try:
|
| 1702 |
-
|
| 1703 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1704 |
except ValueError:
|
| 1705 |
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None
|
| 1706 |
|
|
@@ -1710,7 +2299,11 @@ def connector_sync_now_status(connector_key: str, session: Session = Depends(req
|
|
| 1710 |
_sync_admin(session)
|
| 1711 |
if connector_key not in connector_sync.providers():
|
| 1712 |
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
|
| 1713 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1714 |
"provider": connector_key, "tenant": session.runtime.key, "state": "idle",
|
| 1715 |
"requestedAt": None, "startedAt": None, "finishedAt": None,
|
| 1716 |
"requestedBy": "", "detail": "", "accepted": False,
|
|
@@ -1760,8 +2353,11 @@ def connector_database_jobs(connector_key: str, session: Session = Depends(requi
|
|
| 1760 |
_sync_admin(session)
|
| 1761 |
if connector_key not in connector_sync.providers():
|
| 1762 |
raise err(404, "sync_provider_unknown", "This connector has no database sync runner.")
|
| 1763 |
-
|
| 1764 |
-
session.runtime, connector_key, connection_id)}
|
|
|
|
|
|
|
|
|
|
| 1765 |
|
| 1766 |
|
| 1767 |
@router.get("/connectors/odoo/schema")
|
|
@@ -2241,6 +2837,114 @@ def _safe_connector_status(connector_key, payload):
|
|
| 2241 |
return payload
|
| 2242 |
|
| 2243 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2244 |
def _detail_connect(entry_type, is_admin):
|
| 2245 |
"""C4's `keychain.connect` half: what the Connect form must ask for, and what it may store.
|
| 2246 |
|
|
@@ -2624,6 +3328,173 @@ def _named_databases(rt, is_admin, uname, rows):
|
|
| 2624 |
return named
|
| 2625 |
|
| 2626 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2627 |
def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None):
|
| 2628 |
"""C4's `databases` half: what this connector materialises, and the fields behind each one.
|
| 2629 |
|
|
@@ -2652,6 +3523,8 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
|
|
| 2652 |
reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
|
| 2653 |
is walled, per row, by `canRename`.
|
| 2654 |
"""
|
|
|
|
|
|
|
| 2655 |
if not connector_directory.is_introspectable(connector_key):
|
| 2656 |
return []
|
| 2657 |
spec = _connector_spec_tables(connector_key)
|
|
@@ -2668,6 +3541,9 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
|
|
| 2668 |
census = odoo_relational.population_census(rt, namespace)
|
| 2669 |
except Exception: # noqa: BLE001
|
| 2670 |
return []
|
|
|
|
|
|
|
|
|
|
| 2671 |
census_at = str(census.get("capturedAt") or "")
|
| 2672 |
counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
|
| 2673 |
catalog = {}
|
|
@@ -2712,6 +3588,12 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
|
|
| 2712 |
# Counts are model-level facts. Databases backed by the same model deliberately show the
|
| 2713 |
# same sourceRows number; inventing a per-database domain here would make the number lie.
|
| 2714 |
source_rows = counts.get(model) if census_at and model in counts else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2715 |
out.append({"key": key,
|
| 2716 |
"identity": connector_sync.database_identity(
|
| 2717 |
"odoo", resolved_connection_id, key, model),
|
|
@@ -2721,6 +3603,12 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
|
|
| 2721 |
"enabled": bool(choice.get("enabled")),
|
| 2722 |
"sourceRows": source_rows,
|
| 2723 |
"populated": None if source_rows is None else source_rows > 0,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2724 |
"fields": scoped_fields,
|
| 2725 |
"excluded": [dict(item) for item in excluded],
|
| 2726 |
"sharesFieldsWith": []})
|
|
@@ -2811,6 +3699,8 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 2811 |
# what `native` carries: the vendor's own name for the thing, never one we invented. It is
|
| 2812 |
# empty for a table whose report type is unknown, and the tab says so rather than guessing.
|
| 2813 |
declared = (spec or {}).get(key) or {}
|
|
|
|
|
|
|
| 2814 |
out.append({"key": key,
|
| 2815 |
"label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
|
| 2816 |
"defaultLabel": str(declared.get("label")
|
|
@@ -2819,8 +3709,9 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 2819 |
# W55-T19 / R7 — AMAZON'S OWN tenant-local answer. This is deliberately not
|
| 2820 |
# read from Odoo's grid config, and disabling it leaves every stored row and
|
| 2821 |
# definition untouched: only this provider config document changes.
|
| 2822 |
-
"enabled":
|
| 2823 |
-
**({"sync":
|
|
|
|
| 2824 |
"fields": fields,
|
| 2825 |
"excluded": excluded,
|
| 2826 |
"sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
|
|
@@ -2888,6 +3779,14 @@ def _detail_introspection(rt, is_admin, row):
|
|
| 2888 |
stops "administrator only" being rendered as "this database has no fields".
|
| 2889 |
"""
|
| 2890 |
key = str(row.get("key") or "")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2891 |
if not connector_directory.is_introspectable(key):
|
| 2892 |
reason = ("Nothing is connected behind this listing yet, so there is no schema to read."
|
| 2893 |
if str(row.get("state") or "") == connector_directory.CATALOGUE_STATE
|
|
@@ -3008,6 +3907,229 @@ def _detail_enablement(rt, is_admin, row):
|
|
| 3008 |
"updatePath": "", "preservesData": True, "available": False}
|
| 3009 |
|
| 3010 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3011 |
def connector_detail(rt, user, uname, is_admin, connector_key):
|
| 3012 |
"""The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
|
| 3013 |
|
|
@@ -3106,6 +4228,26 @@ def amazon_database_enablement(database_key: str, body: dict = Body(default=None
|
|
| 3106 |
return _set_amazon_database_enablement(database_key, body, session)
|
| 3107 |
|
| 3108 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3109 |
# ⚠ DECLARED AFTER THE LITERAL `/connectors/odoo/*` ROUTES. FastAPI matches in declaration order,
|
| 3110 |
# and while none of those ends in `/detail` — so no shadowing is possible today — the ordering is
|
| 3111 |
# the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
|
|
|
|
| 21 |
from __future__ import annotations
|
| 22 |
|
| 23 |
import os
|
| 24 |
+
import ssl
|
| 25 |
import time
|
| 26 |
import threading
|
| 27 |
import xmlrpc.client
|
|
|
|
| 31 |
from fastapi import APIRouter, Body, Depends
|
| 32 |
|
| 33 |
import oauth_connect
|
| 34 |
+
from deps import Session, err, require_session, running_main
|
| 35 |
#: ⭐⭐ W49-T18 / R27 / C4 — THE TAXONOMY LIVES IN THE DATA LAYER, NOT HERE. `core` is already on
|
| 36 |
#: the path by the time this module loads (`deps` imports three of its modules), so this is the
|
| 37 |
#: same import shape the rest of the API uses. It is in `platform/core/` rather than beside this
|
|
|
|
| 83 |
router = APIRouter(prefix="/api/v1")
|
| 84 |
|
| 85 |
CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
|
| 86 |
+
SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps"))
|
| 87 |
+
SCRAPER_RECIPE_MAX_ROWS = 30
|
| 88 |
+
SCRAPER_RUN_HISTORY_LIMIT = 20
|
| 89 |
+
DATABASE_SYNC_STATES = frozenset(
|
| 90 |
+
("never_run", "queued", "running", "succeeded", "partial", "failed", "disabled")
|
| 91 |
+
)
|
| 92 |
+
_DATABASE_SAFE_ERRORS = {
|
| 93 |
+
"tls_certificate_invalid": ("Odoo TLS certificate verification failed because the certificate "
|
| 94 |
+
"is expired or untrusted. No source data was accepted."),
|
| 95 |
+
"source_read_refused": "Odoo refused the configured account's read access.",
|
| 96 |
+
"source_auth_failed": "Odoo rejected the configured read-only credential.",
|
| 97 |
+
"source_unavailable": "Odoo could not be reached for this read-only sync.",
|
| 98 |
+
"source_read_failed": "Odoo source validation failed.",
|
| 99 |
+
"mirror_pull_failed": "The Odoo mirror pull failed.",
|
| 100 |
+
"materialization_failed": "The database could not be materialized.",
|
| 101 |
+
"reconciliation_failed": "The database reconciliation failed.",
|
| 102 |
+
"freshness_unconfirmed": "Freshness could not be confirmed for this database.",
|
| 103 |
+
"sync_status_unavailable": "Sync status is unavailable for this database.",
|
| 104 |
+
"sync_stalled": "This database sync is stalled.",
|
| 105 |
+
"sync_state_unreadable": "This database's saved sync state could not be read.",
|
| 106 |
+
"maps_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
|
| 107 |
+
"scraper_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
|
| 108 |
+
"maps_unbounded_request": "Google Maps discovery requires an explicit bounded result cap.",
|
| 109 |
+
"maps_invalid_search": "Google Maps discovery needs a valid category or search.",
|
| 110 |
+
"maps_vendor_error": "The Maps provider could not complete this run.",
|
| 111 |
+
"maps_schema_drift": "The Maps provider returned an unreadable result shape.",
|
| 112 |
+
"scraper_run_failed": "The provider could not complete this run. No result was written.",
|
| 113 |
+
"provider_result_failed": "The provider returned no usable result for this database.",
|
| 114 |
+
"scraper_result_not_saved": "The provider answered, but the result could not be saved.",
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
# T11 established the saved database contract; T12's provider normalises every Apify answer into
|
| 118 |
+
# exactly this schema. Vendor field names never reach a saved View or downstream automation.
|
| 119 |
+
_MAPS_RECIPE_FIELDS = [
|
| 120 |
+
{"key": "place_id", "label": "Place id", "type": "text", "source": "automation"},
|
| 121 |
+
{"key": "name", "label": "Name", "type": "text", "source": "automation"},
|
| 122 |
+
{"key": "category", "label": "Category", "type": "text", "source": "automation"},
|
| 123 |
+
{"key": "address", "label": "Address", "type": "text", "source": "automation"},
|
| 124 |
+
{"key": "city", "label": "City", "type": "text", "source": "automation"},
|
| 125 |
+
{"key": "region", "label": "Region", "type": "text", "source": "automation"},
|
| 126 |
+
{"key": "postal_code", "label": "Postal code", "type": "text", "source": "automation"},
|
| 127 |
+
{"key": "country", "label": "Country", "type": "text", "source": "automation"},
|
| 128 |
+
{"key": "phone", "label": "Phone", "type": "phone", "source": "automation"},
|
| 129 |
+
{"key": "website", "label": "Website", "type": "url", "source": "automation"},
|
| 130 |
+
# Coordinates remain text because `number` is not a user-table field kind; the Map binding
|
| 131 |
+
# parses the canonical scalar value. Declaring an unsupported kind would silently drop both
|
| 132 |
+
# columns at database creation.
|
| 133 |
+
{"key": "latitude", "label": "Latitude", "type": "text", "source": "automation"},
|
| 134 |
+
{"key": "longitude", "label": "Longitude", "type": "text", "source": "automation"},
|
| 135 |
+
{"key": "rating", "label": "Rating", "type": "rating", "source": "automation"},
|
| 136 |
+
{"key": "reviews", "label": "Reviews", "type": "int", "source": "automation"},
|
| 137 |
+
{"key": "source_timestamp", "label": "Source timestamp", "type": "date",
|
| 138 |
+
"source": "automation"},
|
| 139 |
+
{"key": "source_payload", "label": "Source data", "type": "json", "source": "automation"},
|
| 140 |
+
]
|
| 141 |
|
| 142 |
|
| 143 |
# Measured reads take 1.34 s cold, 0.01 s warm, and above 8 s under boot contention.
|
|
|
|
| 206 |
#: 'Scraper' … would I see the option to choose either Instagram OR TikTok."* So the directory
|
| 207 |
#: mirrors the automation menus exactly: ONE card for the capability, and the platforms are behind
|
| 208 |
#: it rather than beside it.
|
| 209 |
+
#: ⚠ These are not connectors and must never become rows: they have no state or manage door of
|
| 210 |
+
#: their own. Instagram and TikTok share the Scraper credential; Maps uses the Apify credential
|
| 211 |
+
#: through the same governed provider registry.
|
| 212 |
SCRAPER_PLATFORMS = [
|
| 213 |
{"key": "instagram", "label": "Instagram",
|
| 214 |
"desc": "Profiles, posts and comments, by search or by handle."},
|
| 215 |
{"key": "tiktok", "label": "TikTok",
|
| 216 |
"desc": "The same three, on TikTok's own schema."},
|
| 217 |
+
{"key": "google_maps", "label": "Google Maps",
|
| 218 |
+
"desc": "Places by category or search and geography, through a bounded Apify run."},
|
| 219 |
]
|
| 220 |
|
| 221 |
#: Token connectors an admin can wire up TODAY by pasting a key — no OAuth client, no app review.
|
|
|
|
| 1122 |
# ⭐ WAVE 30 · R4 — the sentence names the CAPABILITY; the platforms are the drill below.
|
| 1123 |
# It used to say "Instagram profile and post capture", which stopped being true the day
|
| 1124 |
# TikTok discovery shipped and would have to be re-edited for every platform after it.
|
| 1125 |
+
"desc": "Create saved Instagram, TikTok and Google Maps database recipes.",
|
| 1126 |
"state": "connected" if ready else "unconfigured",
|
| 1127 |
"needs": "" if ready else "AIOS_BRIGHTDATA_KEY is not set on this deployment",
|
| 1128 |
"manage": "",
|
| 1129 |
+
# The directory taxonomy normally derives this flag from connected table prefixes.
|
| 1130 |
+
# Scraper recipes are user databases, so the route stamps it again after normalisation.
|
| 1131 |
+
"spawns_tables": True,
|
| 1132 |
+
# Platforms remain a drill behind one recipe/database capability, never directory rows.
|
| 1133 |
+
# `bd_ready()` describes today's social runners only; Maps' own sentence names that its
|
| 1134 |
+
# execution provider is intentionally absent until T12.
|
| 1135 |
"platforms": [dict(p) for p in SCRAPER_PLATFORMS],
|
| 1136 |
}
|
| 1137 |
|
|
|
|
| 1191 |
# trap; `chip_counts` carries the argument).
|
| 1192 |
served = connector_directory.to_client_rows(rows)
|
| 1193 |
for row in served:
|
| 1194 |
+
if str(row.get("key") or "") == "scraper":
|
| 1195 |
+
row["spawns_tables"] = True
|
| 1196 |
_safe_connector_status(row.get("key"), row)
|
| 1197 |
_safe_connector_status("amazon_sp", {"incomplete": incomplete})
|
| 1198 |
return {"connectors": served,
|
|
|
|
| 1226 |
|
| 1227 |
_ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements"
|
| 1228 |
_ODOO_SYNC_STATUS = "odoo_sync_status"
|
| 1229 |
+
_ODOO_HEALTH_STATUS = "odoo_health_status_v1"
|
| 1230 |
+
_ODOO_HEALTH_STAGES = ("source", "mirrorPull", "materialization", "reconciliation", "freshness")
|
| 1231 |
+
_SYNC_REQUEST_CONNECTIONS = {}
|
| 1232 |
+
_SYNC_REQUEST_CONNECTION_LOCK = threading.Lock()
|
| 1233 |
+
|
| 1234 |
+
|
| 1235 |
+
def _odoo_connection_slot(rt, connection_id=None):
|
| 1236 |
+
"""Return the durable Odoo connection id used by jobs, status, and health receipts."""
|
| 1237 |
+
requested = str(connection_id or "").strip()
|
| 1238 |
+
if requested and requested != "default":
|
| 1239 |
+
return requested
|
| 1240 |
+
try:
|
| 1241 |
+
import odoo_relational
|
| 1242 |
+
return str(odoo_relational.default_connection_id(rt) or "default")
|
| 1243 |
+
except Exception: # noqa: BLE001
|
| 1244 |
+
return "default"
|
| 1245 |
+
|
| 1246 |
+
|
| 1247 |
+
def sync_request_connection(provider, tenant):
|
| 1248 |
+
"""Return the connection bound to a process-local Sync now intent."""
|
| 1249 |
+
with _SYNC_REQUEST_CONNECTION_LOCK:
|
| 1250 |
+
return _SYNC_REQUEST_CONNECTIONS.get((str(provider), str(tenant)))
|
| 1251 |
+
|
| 1252 |
+
|
| 1253 |
+
def _odoo_failure(exc, stage="source"):
|
| 1254 |
+
"""Classify a source/database exception without persisting its potentially sensitive text."""
|
| 1255 |
+
stage = stage if stage in _ODOO_HEALTH_STAGES else "source"
|
| 1256 |
+
text = str(exc or "").casefold()
|
| 1257 |
+
exc_type = type(exc).__name__ if isinstance(exc, BaseException) else "SourceError"
|
| 1258 |
+
if (isinstance(exc, ssl.SSLCertVerificationError)
|
| 1259 |
+
or "certificate_verify_failed" in text
|
| 1260 |
+
or "certificate verify failed" in text
|
| 1261 |
+
or "certificate has expired" in text
|
| 1262 |
+
or "self signed certificate" in text):
|
| 1263 |
+
return {"stage": "source", "code": "tls_certificate_invalid",
|
| 1264 |
+
"type": exc_type,
|
| 1265 |
+
"message": ("Odoo TLS certificate verification failed because the certificate "
|
| 1266 |
+
"is expired or untrusted. No source data was accepted.")}
|
| 1267 |
+
if isinstance(exc, xmlrpc.client.Fault) and any(
|
| 1268 |
+
token in text for token in ("accesserror", "access denied", "security restrictions")):
|
| 1269 |
+
return {"stage": "source", "code": "source_read_refused", "type": exc_type,
|
| 1270 |
+
"message": "Odoo refused the configured account's read access."}
|
| 1271 |
+
if any(token in text for token in ("authentication", "access denied", "invalid api key")):
|
| 1272 |
+
return {"stage": "source", "code": "source_auth_failed", "type": exc_type,
|
| 1273 |
+
"message": "Odoo rejected the configured read-only credential."}
|
| 1274 |
+
if isinstance(exc, (TimeoutError, ConnectionError)) or any(
|
| 1275 |
+
token in text for token in ("timed out", "timeout", "name resolution", "connection refused")):
|
| 1276 |
+
return {"stage": "source", "code": "source_unavailable", "type": exc_type,
|
| 1277 |
+
"message": "Odoo could not be reached for this read-only sync."}
|
| 1278 |
+
labels = {
|
| 1279 |
+
"source": ("source_read_failed", "Odoo source validation failed."),
|
| 1280 |
+
"mirrorPull": ("mirror_pull_failed", "The Odoo mirror pull failed."),
|
| 1281 |
+
"materialization": ("materialization_failed", "The Odoo database could not be materialized."),
|
| 1282 |
+
"reconciliation": ("reconciliation_failed", "The Odoo database reconciliation failed."),
|
| 1283 |
+
"freshness": ("freshness_unconfirmed", "Freshness could not be confirmed for this database."),
|
| 1284 |
+
}
|
| 1285 |
+
code, message = labels[stage]
|
| 1286 |
+
return {"stage": stage, "code": code, "type": exc_type, "message": message}
|
| 1287 |
+
|
| 1288 |
+
|
| 1289 |
+
def _clean_odoo_failure(failure, stage):
|
| 1290 |
+
if failure is None:
|
| 1291 |
+
return None
|
| 1292 |
+
if not isinstance(failure, dict):
|
| 1293 |
+
return _odoo_failure(failure, stage)
|
| 1294 |
+
return {"stage": str(failure.get("stage") or stage),
|
| 1295 |
+
"code": str(failure.get("code") or f"{stage}_failed"),
|
| 1296 |
+
"type": str(failure.get("type") or "SourceError")[:80],
|
| 1297 |
+
"message": str(failure.get("message") or "Odoo sync failed.")[:300]}
|
| 1298 |
+
|
| 1299 |
+
|
| 1300 |
+
def _odoo_stage_failure(stages):
|
| 1301 |
+
"""Return the earliest unresolved lifecycle failure, preserving its named cause."""
|
| 1302 |
+
return next((stages.get(name, {}).get("failure") for name in _ODOO_HEALTH_STAGES
|
| 1303 |
+
if stages.get(name, {}).get("state") == "failed"
|
| 1304 |
+
and stages.get(name, {}).get("failure")), None)
|
| 1305 |
+
|
| 1306 |
+
|
| 1307 |
+
def _clean_odoo_database_receipt(receipt, database_key):
|
| 1308 |
+
"""Allow-list one data-free reconciliation receipt for durable health storage."""
|
| 1309 |
+
if not isinstance(receipt, dict):
|
| 1310 |
+
return None
|
| 1311 |
+
key = str(database_key or "").strip()
|
| 1312 |
+
if str(receipt.get("databaseKey") or "").strip() != key:
|
| 1313 |
+
raise ValueError("the reconciliation receipt belongs to a different database")
|
| 1314 |
+
out = {"databaseKey": key}
|
| 1315 |
+
for name in ("schemaKey", "topicKey", "domainKey", "model", "countModel", "state",
|
| 1316 |
+
"sourceCountMode", "sourceReconciliation"):
|
| 1317 |
+
value = receipt.get(name)
|
| 1318 |
+
if value is not None:
|
| 1319 |
+
out[name] = str(value)[:160]
|
| 1320 |
+
for name in ("sourceCount", "mirrorCount", "materializedCount", "canonicalCount",
|
| 1321 |
+
"servingCount", "duplicateCount", "explainedMergeCount",
|
| 1322 |
+
"unexplainedMissingCount", "unexplainedExtraCount",
|
| 1323 |
+
"unexplainedCanonicalMissingCount", "unexplainedCanonicalExtraCount",
|
| 1324 |
+
"unexplainedCount"):
|
| 1325 |
+
value = receipt.get(name)
|
| 1326 |
+
out[name] = None if value is None else max(0, int(value))
|
| 1327 |
+
if "readThrough" in receipt:
|
| 1328 |
+
out["readThrough"] = bool(receipt.get("readThrough"))
|
| 1329 |
+
selection = receipt.get("selection")
|
| 1330 |
+
if isinstance(selection, dict):
|
| 1331 |
+
out["selection"] = {
|
| 1332 |
+
name: (bool(selection.get(name)) if name == "enabled"
|
| 1333 |
+
else str(selection.get(name) or "")[:160])
|
| 1334 |
+
for name in ("bucket", "rule", "enabled") if name in selection}
|
| 1335 |
+
domain = receipt.get("domain")
|
| 1336 |
+
if isinstance(domain, list):
|
| 1337 |
+
# Curated domains contain public field/operator/value predicates. Refuse nested objects,
|
| 1338 |
+
# which could otherwise turn this metadata seam into durable source-row storage.
|
| 1339 |
+
clean_domain = []
|
| 1340 |
+
for predicate in domain:
|
| 1341 |
+
if not isinstance(predicate, (list, tuple)) or len(predicate) != 3:
|
| 1342 |
+
continue
|
| 1343 |
+
field, operator, value = predicate
|
| 1344 |
+
if isinstance(value, (list, tuple)):
|
| 1345 |
+
clean_value = [item for item in value
|
| 1346 |
+
if isinstance(item, (str, int, float, bool))][:50]
|
| 1347 |
+
elif isinstance(value, (str, int, float, bool)) or value is None:
|
| 1348 |
+
clean_value = value
|
| 1349 |
+
else:
|
| 1350 |
+
continue
|
| 1351 |
+
clean_domain.append([str(field)[:100], str(operator)[:40], clean_value])
|
| 1352 |
+
out["domain"] = clean_domain
|
| 1353 |
+
reconciliation = receipt.get("reconciliation")
|
| 1354 |
+
if isinstance(reconciliation, dict):
|
| 1355 |
+
out["reconciliation"] = {
|
| 1356 |
+
name: str(reconciliation.get(name) or "")[:80]
|
| 1357 |
+
for name in ("sourceToMirror", "mirrorToMaterialized", "mirrorToCanonical")
|
| 1358 |
+
if name in reconciliation}
|
| 1359 |
+
freshness = receipt.get("freshness")
|
| 1360 |
+
if isinstance(freshness, dict):
|
| 1361 |
+
out["freshness"] = {
|
| 1362 |
+
name: str(freshness.get(name) or "")[:100]
|
| 1363 |
+
for name in ("state", "capturedAt") if name in freshness}
|
| 1364 |
+
for name in ("sourceError", "error"):
|
| 1365 |
+
if receipt.get(name) is not None:
|
| 1366 |
+
out[name] = _clean_odoo_failure(receipt.get(name), "reconciliation")
|
| 1367 |
+
return out
|
| 1368 |
+
|
| 1369 |
+
|
| 1370 |
+
def record_odoo_stage(rt, connection_id, stage, state, *, database_key=None, row_count=None,
|
| 1371 |
+
refreshed_at=None, cadence_seconds=None, failure=None, receipt=None,
|
| 1372 |
+
at=None):
|
| 1373 |
+
"""Persist one data-free Odoo stage transition for one connection/database.
|
| 1374 |
+
|
| 1375 |
+
This is additive observability over the established ``connector_sync`` and materialization
|
| 1376 |
+
job documents. It stores identifiers, counts, timestamps, and classified failures only;
|
| 1377 |
+
credentials and source rows never cross this seam.
|
| 1378 |
+
"""
|
| 1379 |
+
if stage not in _ODOO_HEALTH_STAGES:
|
| 1380 |
+
raise ValueError(f"unknown Odoo health stage: {stage}")
|
| 1381 |
+
if state not in ("unknown", "queued", "running", "ready", "failed", "paused"):
|
| 1382 |
+
raise ValueError(f"unknown Odoo health state: {state}")
|
| 1383 |
+
cid = _odoo_connection_slot(rt, connection_id)
|
| 1384 |
+
stamp = str(at or datetime.now(timezone.utc).isoformat())
|
| 1385 |
+
failed = _clean_odoo_failure(failure, stage) if state == "failed" else None
|
| 1386 |
+
db_key = str(database_key or "").strip()
|
| 1387 |
+
safe_receipt = _clean_odoo_database_receipt(receipt, db_key) if receipt is not None else None
|
| 1388 |
+
|
| 1389 |
+
def _update(current):
|
| 1390 |
+
document = dict(current or {})
|
| 1391 |
+
connections = dict(document.get("connections") or {})
|
| 1392 |
+
connection = dict(connections.get(cid) or {})
|
| 1393 |
+
stages = dict(connection.get("stages") or {})
|
| 1394 |
+
target_stages = stages
|
| 1395 |
+
database = None
|
| 1396 |
+
databases = None
|
| 1397 |
+
if db_key:
|
| 1398 |
+
databases = dict(connection.get("databases") or {})
|
| 1399 |
+
database = dict(databases.get(db_key) or {"databaseKey": db_key})
|
| 1400 |
+
target_stages = dict(database.get("stages") or {})
|
| 1401 |
+
previous = dict(target_stages.get(stage) or {})
|
| 1402 |
+
stage_row = {**previous, "state": state, "at": stamp, "failure": failed}
|
| 1403 |
+
if state == "running":
|
| 1404 |
+
stage_row["startedAt"] = stamp
|
| 1405 |
+
stage_row["finishedAt"] = None
|
| 1406 |
+
elif state in ("ready", "failed", "paused"):
|
| 1407 |
+
stage_row["finishedAt"] = stamp
|
| 1408 |
+
if state == "ready":
|
| 1409 |
+
stage_row["lastSuccess"] = stamp
|
| 1410 |
+
target_stages[stage] = stage_row
|
| 1411 |
+
connection.update({"connectionId": cid, "stages": stages, "lastAttempt": stamp})
|
| 1412 |
+
if cadence_seconds is not None:
|
| 1413 |
+
connection["cadenceSeconds"] = int(cadence_seconds)
|
| 1414 |
+
|
| 1415 |
+
if db_key:
|
| 1416 |
+
database.update({"databaseKey": db_key, "stages": target_stages,
|
| 1417 |
+
"lastAttempt": stamp})
|
| 1418 |
+
if row_count is not None:
|
| 1419 |
+
database["rowCount"] = max(0, int(row_count))
|
| 1420 |
+
if refreshed_at:
|
| 1421 |
+
database["recordRefreshed"] = str(refreshed_at)
|
| 1422 |
+
if safe_receipt is not None:
|
| 1423 |
+
database["reconciliationReceipt"] = safe_receipt
|
| 1424 |
+
db_failure = _odoo_stage_failure(target_stages)
|
| 1425 |
+
if db_failure:
|
| 1426 |
+
database["failure"] = db_failure
|
| 1427 |
+
else:
|
| 1428 |
+
database.pop("failure", None)
|
| 1429 |
+
if state == "ready" and stage == "freshness":
|
| 1430 |
+
database["lastSuccess"] = stamp
|
| 1431 |
+
databases[db_key] = database
|
| 1432 |
+
connection["databases"] = databases
|
| 1433 |
+
else:
|
| 1434 |
+
stages[stage] = stage_row
|
| 1435 |
+
connection["stages"] = stages
|
| 1436 |
+
connection_failure = _odoo_stage_failure(stages)
|
| 1437 |
+
if connection_failure:
|
| 1438 |
+
connection["failure"] = connection_failure
|
| 1439 |
+
else:
|
| 1440 |
+
connection.pop("failure", None)
|
| 1441 |
+
if state == "ready" and stage == "freshness":
|
| 1442 |
+
connection["lastSuccess"] = stamp
|
| 1443 |
+
connections[cid] = connection
|
| 1444 |
+
document["connections"] = connections
|
| 1445 |
+
return document
|
| 1446 |
+
|
| 1447 |
+
rt.update(_ODOO_HEALTH_STATUS, _update, flush="sync")
|
| 1448 |
+
return odoo_health_receipt(rt, cid)
|
| 1449 |
+
|
| 1450 |
+
|
| 1451 |
+
def _lag_seconds(stamp, now=None):
|
| 1452 |
+
if not stamp:
|
| 1453 |
+
return None
|
| 1454 |
+
try:
|
| 1455 |
+
parsed = datetime.fromisoformat(str(stamp).replace("Z", "+00:00"))
|
| 1456 |
+
if parsed.tzinfo is None:
|
| 1457 |
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
| 1458 |
+
current = now or datetime.now(timezone.utc)
|
| 1459 |
+
return max(0, int((current - parsed.astimezone(timezone.utc)).total_seconds()))
|
| 1460 |
+
except (TypeError, ValueError):
|
| 1461 |
+
return None
|
| 1462 |
+
|
| 1463 |
+
|
| 1464 |
+
def _empty_odoo_stages():
|
| 1465 |
+
return {stage: {"state": "unknown", "at": None, "startedAt": None,
|
| 1466 |
+
"finishedAt": None, "lastSuccess": None, "failure": None}
|
| 1467 |
+
for stage in _ODOO_HEALTH_STAGES}
|
| 1468 |
+
|
| 1469 |
+
|
| 1470 |
+
def odoo_health_receipt(rt, connection_id=None, *, now=None):
|
| 1471 |
+
"""Project the truthful Odoo receipt for one connection and each of its databases."""
|
| 1472 |
+
cid = _odoo_connection_slot(rt, connection_id)
|
| 1473 |
+
try:
|
| 1474 |
+
document = rt.get(_ODOO_HEALTH_STATUS, fresh=True) or {}
|
| 1475 |
+
except TypeError:
|
| 1476 |
+
document = rt.get(_ODOO_HEALTH_STATUS) or {}
|
| 1477 |
+
except Exception: # noqa: BLE001
|
| 1478 |
+
document = {}
|
| 1479 |
+
stored = dict(((document.get("connections") or {}).get(cid)) or {})
|
| 1480 |
+
stages = _empty_odoo_stages()
|
| 1481 |
+
stages.update({key: dict(value) for key, value in (stored.get("stages") or {}).items()
|
| 1482 |
+
if key in stages and isinstance(value, dict)})
|
| 1483 |
+
databases = {str(key): dict(value) for key, value in (stored.get("databases") or {}).items()
|
| 1484 |
+
if isinstance(value, dict)}
|
| 1485 |
+
|
| 1486 |
+
# Durable materialization receipts are the authority for database work and survive restarts.
|
| 1487 |
+
for job in connector_sync.jobs(rt, "odoo", cid):
|
| 1488 |
+
key = str(job.get("databaseKey") or "")
|
| 1489 |
+
if not key:
|
| 1490 |
+
continue
|
| 1491 |
+
database = dict(databases.get(key) or {"databaseKey": key})
|
| 1492 |
+
db_stages = _empty_odoo_stages()
|
| 1493 |
+
db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items()
|
| 1494 |
+
if name in db_stages and isinstance(value, dict)})
|
| 1495 |
+
job_state = str(job.get("state") or "unknown")
|
| 1496 |
+
material_state = {"configured": "queued", "catalogue-ready": "queued",
|
| 1497 |
+
"queued": "queued", "syncing": "running", "ready": "ready",
|
| 1498 |
+
"failed": "failed"}.get(job_state, "unknown")
|
| 1499 |
+
job_at = None
|
| 1500 |
+
if job.get("at") is not None:
|
| 1501 |
+
try:
|
| 1502 |
+
job_at = datetime.fromtimestamp(float(job["at"]), timezone.utc).isoformat()
|
| 1503 |
+
except (TypeError, ValueError, OSError):
|
| 1504 |
+
job_at = None
|
| 1505 |
+
job_failure = (_odoo_failure(job.get("error"), "materialization")
|
| 1506 |
+
if material_state == "failed" else None)
|
| 1507 |
+
db_stages["materialization"] = {
|
| 1508 |
+
**db_stages["materialization"], "state": material_state,
|
| 1509 |
+
"at": job_at, "finishedAt": job_at if material_state in ("ready", "failed") else None,
|
| 1510 |
+
"lastSuccess": (job_at if material_state == "ready" else
|
| 1511 |
+
db_stages["materialization"].get("lastSuccess")),
|
| 1512 |
+
"failure": job_failure,
|
| 1513 |
+
}
|
| 1514 |
+
progress = job.get("progress") if isinstance(job.get("progress"), dict) else {}
|
| 1515 |
+
if progress.get("rows") is not None:
|
| 1516 |
+
database["rowCount"] = max(0, int(progress["rows"]))
|
| 1517 |
+
database.update({"databaseKey": key, "model": str(job.get("model") or ""),
|
| 1518 |
+
"state": material_state, "stages": db_stages})
|
| 1519 |
+
persisted_failure = database.get("failure") or _odoo_stage_failure(db_stages)
|
| 1520 |
+
if persisted_failure or job_failure:
|
| 1521 |
+
database["failure"] = persisted_failure or job_failure
|
| 1522 |
+
elif material_state == "ready" and not database.get("failure"):
|
| 1523 |
+
database["lastSuccess"] = database.get("lastSuccess") or job_at
|
| 1524 |
+
databases[key] = database
|
| 1525 |
+
|
| 1526 |
+
# The resident mirror state is process-local by design; overlay it only on the connection
|
| 1527 |
+
# that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
|
| 1528 |
+
cadence = stored.get("cadenceSeconds")
|
| 1529 |
+
try:
|
| 1530 |
+
_main = running_main() # D-781: resolve, never import
|
| 1531 |
+
resident = dict(_main.RESYNC_STATE)
|
| 1532 |
+
resident_cid = str(resident.get("connectionId") or "")
|
| 1533 |
+
if resident_cid == cid:
|
| 1534 |
+
cadence = resident.get("resolvedSeconds") or cadence
|
| 1535 |
+
mirror_failure = resident.get("lastError")
|
| 1536 |
+
mirror_state = ("paused" if resident.get("manual") else
|
| 1537 |
+
"failed" if resident.get("lastPassOk") is False else
|
| 1538 |
+
"ready" if resident.get("lastPassOk") is True else "unknown")
|
| 1539 |
+
stages["mirrorPull"] = {**stages["mirrorPull"], "state": mirror_state,
|
| 1540 |
+
"at": resident.get("lastPassAt"),
|
| 1541 |
+
"finishedAt": resident.get("lastPassAt"),
|
| 1542 |
+
"lastSuccess": resident.get("lastSuccessAt"),
|
| 1543 |
+
"failure": (_clean_odoo_failure(mirror_failure, "mirrorPull")
|
| 1544 |
+
if mirror_failure else None)}
|
| 1545 |
+
except Exception: # noqa: BLE001
|
| 1546 |
+
pass
|
| 1547 |
+
|
| 1548 |
+
for database in databases.values():
|
| 1549 |
+
db_stages = _empty_odoo_stages()
|
| 1550 |
+
db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items()
|
| 1551 |
+
if name in db_stages and isinstance(value, dict)})
|
| 1552 |
+
# Connection source/mirror/reconciliation facts apply to each database, while its own
|
| 1553 |
+
# materialization/freshness facts remain independent.
|
| 1554 |
+
for stage in ("source", "mirrorPull", "reconciliation"):
|
| 1555 |
+
if db_stages[stage].get("state") == "unknown":
|
| 1556 |
+
db_stages[stage] = dict(stages[stage])
|
| 1557 |
+
database["stages"] = db_stages
|
| 1558 |
+
success = database.get("lastSuccess") or db_stages["freshness"].get("lastSuccess")
|
| 1559 |
+
database["lastSuccess"] = success
|
| 1560 |
+
database["currentLagSeconds"] = _lag_seconds(success, now)
|
| 1561 |
+
database.setdefault("rowCount", None)
|
| 1562 |
+
database.setdefault("recordRefreshed", None)
|
| 1563 |
+
stage_failure = _odoo_stage_failure(db_stages)
|
| 1564 |
+
if stage_failure:
|
| 1565 |
+
database["failure"] = stage_failure
|
| 1566 |
+
elif not database.get("failure"):
|
| 1567 |
+
database["failure"] = None
|
| 1568 |
+
|
| 1569 |
+
successes = [row.get("lastSuccess") for row in databases.values() if row.get("lastSuccess")]
|
| 1570 |
+
last_success = max(successes, default=stored.get("lastSuccess"))
|
| 1571 |
+
row_counts = [row.get("rowCount") for row in databases.values()
|
| 1572 |
+
if row.get("rowCount") is not None]
|
| 1573 |
+
failure = (_odoo_stage_failure(stages) or stored.get("failure") or next(
|
| 1574 |
+
(row.get("failure") for row in databases.values() if row.get("failure")), None))
|
| 1575 |
+
return {"connectionId": cid, "cadenceSeconds": cadence,
|
| 1576 |
+
"lastSuccess": last_success, "currentLagSeconds": _lag_seconds(last_success, now),
|
| 1577 |
+
"rowCount": sum(row_counts) if row_counts else None,
|
| 1578 |
+
"failure": failure, "stages": stages,
|
| 1579 |
+
"databases": [databases[key] for key in sorted(databases)]}
|
| 1580 |
|
| 1581 |
|
| 1582 |
def _odoo_admin(session):
|
|
|
|
| 1593 |
"""A status endpoint selects one connection; the jobs inventory may enumerate them all."""
|
| 1594 |
cid = str(connection_id or "").strip() or "default"
|
| 1595 |
if provider == "odoo" and cid == "default":
|
| 1596 |
+
cid = _odoo_connection_slot(rt, connection_id)
|
|
|
|
| 1597 |
return connector_sync.jobs(rt, provider, cid)
|
| 1598 |
|
| 1599 |
|
|
|
|
| 1614 |
return _safe_connector_status(connector_key, answer)
|
| 1615 |
|
| 1616 |
|
| 1617 |
+
def _odoo_result_database_failures(rt, connection_id, result):
|
| 1618 |
+
"""Map provider-reported model/table failures to their durable database keys."""
|
| 1619 |
+
import odoo_relational as rel
|
| 1620 |
+
try:
|
| 1621 |
+
choices = rel.grid_choices(rt, connection_id)
|
| 1622 |
+
except Exception: # noqa: BLE001
|
| 1623 |
+
choices = []
|
| 1624 |
+
by_model = {str(row.get("model") or ""): str(row.get("key") or "")
|
| 1625 |
+
for row in choices if row.get("model") and row.get("key")}
|
| 1626 |
+
failures = {str(key): _clean_odoo_failure(value, "materialization")
|
| 1627 |
+
for key, value in (result.get("databaseErrors") or {}).items()}
|
| 1628 |
+
receipt_groups = []
|
| 1629 |
+
for name in ("databases", "canonicalDatabases"):
|
| 1630 |
+
group = result.get(name) if isinstance(result, dict) else None
|
| 1631 |
+
if isinstance(group, dict):
|
| 1632 |
+
receipt_groups.append(group)
|
| 1633 |
+
for group in receipt_groups:
|
| 1634 |
+
for raw_key, receipt in group.items():
|
| 1635 |
+
if not isinstance(receipt, dict) or receipt.get("state") == "succeeded":
|
| 1636 |
+
continue
|
| 1637 |
+
key = str(raw_key or "")
|
| 1638 |
+
if not key or key in failures:
|
| 1639 |
+
continue
|
| 1640 |
+
raw_failure = receipt.get("error") or receipt.get("sourceError") or {
|
| 1641 |
+
"stage": "reconciliation", "code": "reconciliation_incomplete",
|
| 1642 |
+
"type": "ReconciliationError",
|
| 1643 |
+
"message": "The Odoo database reconciliation is incomplete.",
|
| 1644 |
+
}
|
| 1645 |
+
failures[key] = _clean_odoo_failure(raw_failure, "reconciliation")
|
| 1646 |
+
unmatched = []
|
| 1647 |
+
discovered = result.get("discovered") if isinstance(result, dict) else {}
|
| 1648 |
+
for raw in ((discovered or {}).get("errors") or ()):
|
| 1649 |
+
model = str(raw).split(":", 1)[0].strip()
|
| 1650 |
+
key = by_model.get(model)
|
| 1651 |
+
if key:
|
| 1652 |
+
failures[key] = _odoo_failure(raw, "materialization")
|
| 1653 |
+
else:
|
| 1654 |
+
unmatched.append(str(raw))
|
| 1655 |
+
limits = result.get("limits") if isinstance(result, dict) else None
|
| 1656 |
+
if isinstance(limits, dict):
|
| 1657 |
+
for raw_key, raw in limits.items():
|
| 1658 |
+
key = by_model.get(str(raw_key or ""), str(raw_key or ""))
|
| 1659 |
+
if key:
|
| 1660 |
+
failures[key] = _odoo_failure(raw, "materialization")
|
| 1661 |
+
elif limits:
|
| 1662 |
+
unmatched.append(str(limits))
|
| 1663 |
+
return failures, unmatched
|
| 1664 |
+
|
| 1665 |
+
|
| 1666 |
def _odoo_sync_runner(session, connection_id):
|
| 1667 |
"""Run Odoo through C6 without moving provider imports into the data layer."""
|
| 1668 |
+
cid = _odoo_connection_slot(session.runtime, connection_id)
|
| 1669 |
+
connector_sync.record(session.runtime, "odoo", cid, "syncing")
|
| 1670 |
+
record_odoo_stage(session.runtime, cid, "source", "running")
|
| 1671 |
+
record_odoo_stage(session.runtime, cid, "materialization", "queued")
|
| 1672 |
import odoo_relational
|
| 1673 |
+
def _materializing():
|
| 1674 |
+
connector_sync.record(session.runtime, "odoo", cid, "materializing")
|
| 1675 |
+
record_odoo_stage(session.runtime, cid, "materialization", "running")
|
| 1676 |
+
try:
|
| 1677 |
+
result = odoo_relational.refresh(session.runtime, session.tenant,
|
| 1678 |
+
username=session.uname or "automation",
|
| 1679 |
+
connection_id=connection_id,
|
| 1680 |
+
on_apply=_materializing)
|
| 1681 |
+
except Exception as exc:
|
| 1682 |
+
failure = _odoo_failure(exc, "source")
|
| 1683 |
+
record_odoo_stage(session.runtime, cid, failure["stage"], "failed", failure=failure)
|
| 1684 |
+
record_odoo_stage(session.runtime, cid, "freshness", "failed",
|
| 1685 |
+
failure={**failure, "stage": "freshness"})
|
| 1686 |
+
connector_sync.record(session.runtime, "odoo", cid, "error",
|
| 1687 |
+
errors=[f"{failure['code']}: {failure['message']}"])
|
| 1688 |
+
raise
|
| 1689 |
tables = result.get("tables") if isinstance(result, dict) else {}
|
| 1690 |
+
published = dict(tables or {})
|
| 1691 |
+
database_receipts = result.get("databases") if isinstance(result, dict) else {}
|
| 1692 |
+
database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
|
| 1693 |
+
canonical_receipts = result.get("canonicalDatabases") if isinstance(result, dict) else {}
|
| 1694 |
+
canonical_receipts = canonical_receipts if isinstance(canonical_receipts, dict) else {}
|
| 1695 |
+
reconciliation_receipts = {**database_receipts, **canonical_receipts}
|
| 1696 |
+
databases = sorted(str(key) for key in published)
|
| 1697 |
sync_errors = []
|
| 1698 |
if isinstance(result, dict):
|
| 1699 |
if result.get("failed"):
|
|
|
|
| 1703 |
sync_errors.extend(str(value) for value in (discovered.get("errors") or ()))
|
| 1704 |
discovered_tables = discovered.get("tables")
|
| 1705 |
if isinstance(discovered_tables, dict):
|
| 1706 |
+
published.update(discovered_tables)
|
| 1707 |
databases.extend(str(key) for key in discovered_tables)
|
| 1708 |
if result.get("frozen"):
|
| 1709 |
sync_errors.append(str(result.get("note") or "the connector is frozen"))
|
| 1710 |
+
database_failures, unmatched_errors = _odoo_result_database_failures(
|
| 1711 |
+
session.runtime, connection_id, result if isinstance(result, dict) else {})
|
| 1712 |
+
databases.extend(reconciliation_receipts)
|
| 1713 |
+
databases.extend(database_failures)
|
| 1714 |
+
sync_errors.extend(f"{key}: {failure['code']}" for key, failure in database_failures.items())
|
| 1715 |
+
sync_errors.extend(raw for raw in unmatched_errors if raw not in sync_errors)
|
| 1716 |
+
state = ("partial" if published else "error") if sync_errors else "ready"
|
| 1717 |
+
record_odoo_stage(session.runtime, cid, "source", "ready")
|
| 1718 |
+
refreshed_at = datetime.now(timezone.utc).isoformat()
|
| 1719 |
+
database_keys = list(dict.fromkeys([*reconciliation_receipts, *published]))
|
| 1720 |
+
for key in database_keys:
|
| 1721 |
+
if key in database_failures:
|
| 1722 |
+
continue
|
| 1723 |
+
counts = published.get(key)
|
| 1724 |
+
receipt = reconciliation_receipts.get(key)
|
| 1725 |
+
receipt = receipt if isinstance(receipt, dict) else None
|
| 1726 |
+
nrows = (receipt.get("servingCount") if receipt is not None
|
| 1727 |
+
else counts.get("rows") if isinstance(counts, dict) else None)
|
| 1728 |
+
record_odoo_stage(session.runtime, cid, "source", "ready", database_key=key,
|
| 1729 |
+
receipt=receipt)
|
| 1730 |
+
record_odoo_stage(session.runtime, cid, "materialization", "ready",
|
| 1731 |
+
database_key=key, row_count=nrows, receipt=receipt)
|
| 1732 |
+
record_odoo_stage(session.runtime, cid, "reconciliation", "ready",
|
| 1733 |
+
database_key=key, row_count=nrows, receipt=receipt)
|
| 1734 |
+
record_odoo_stage(session.runtime, cid, "freshness", "ready",
|
| 1735 |
+
database_key=key, row_count=nrows, refreshed_at=refreshed_at,
|
| 1736 |
+
receipt=receipt)
|
| 1737 |
+
for key, failure in database_failures.items():
|
| 1738 |
+
receipt = reconciliation_receipts.get(key)
|
| 1739 |
+
receipt = receipt if isinstance(receipt, dict) else None
|
| 1740 |
+
failed_stage = str(failure.get("stage") or "materialization")
|
| 1741 |
+
failed_stage = failed_stage if failed_stage in _ODOO_HEALTH_STAGES else "materialization"
|
| 1742 |
+
record_odoo_stage(session.runtime, cid, failed_stage, "failed",
|
| 1743 |
+
database_key=key, failure=failure, receipt=receipt)
|
| 1744 |
+
record_odoo_stage(session.runtime, cid, "freshness", "failed", database_key=key,
|
| 1745 |
+
failure={**failure, "stage": "freshness"}, receipt=receipt)
|
| 1746 |
+
if sync_errors:
|
| 1747 |
+
failure = _odoo_failure("provider result contained errors", "materialization")
|
| 1748 |
+
record_odoo_stage(session.runtime, cid, "materialization", "failed", failure=failure)
|
| 1749 |
+
record_odoo_stage(session.runtime, cid, "freshness", "failed",
|
| 1750 |
+
failure={**failure, "stage": "freshness"})
|
| 1751 |
+
else:
|
| 1752 |
+
record_odoo_stage(session.runtime, cid, "materialization", "ready")
|
| 1753 |
+
record_odoo_stage(session.runtime, cid, "reconciliation", "ready")
|
| 1754 |
+
record_odoo_stage(session.runtime, cid, "freshness", "ready")
|
| 1755 |
return connector_sync.record(
|
| 1756 |
+
session.runtime, "odoo", cid, state,
|
| 1757 |
detail=str(result.get("note") or "") if isinstance(result, dict) else "",
|
| 1758 |
errors=sync_errors, databases=sorted(set(databases)),
|
| 1759 |
)
|
|
|
|
| 1867 |
return
|
| 1868 |
fenced = _MaterializationRuntime(rt, job)
|
| 1869 |
key = job["databaseKey"]
|
| 1870 |
+
cid = job["connectionId"]
|
| 1871 |
+
record_odoo_stage(rt, cid, "source", "running", database_key=key)
|
| 1872 |
+
record_odoo_stage(rt, cid, "materialization", "running", database_key=key)
|
| 1873 |
def finish(success=False, cause=""):
|
| 1874 |
# The pre-claim check cannot cover a disconnect during source I/O or publication.
|
| 1875 |
# Only the current lease may requeue; disable and a newer Save still win.
|
|
|
|
| 1929 |
fenced, job, rows=rows, total=total))}
|
| 1930 |
discovered = result.get("discovered") or {}
|
| 1931 |
published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
|
| 1932 |
+
receipt = (result.get("databases") or {}).get(key)
|
| 1933 |
+
receipt = receipt if isinstance(receipt, dict) else None
|
| 1934 |
+
database_errors = result.get("databaseErrors") or {}
|
| 1935 |
success = (key in published and not result.get("failed") and not result.get("frozen")
|
| 1936 |
+
and not result.get("limits") and not discovered.get("errors")
|
| 1937 |
+
and key not in database_errors
|
| 1938 |
+
and (receipt is None or receipt.get("state") == "succeeded"))
|
| 1939 |
+
counts = published.get(key) if isinstance(published.get(key), dict) else {}
|
| 1940 |
+
nrows = (receipt.get("servingCount") if receipt is not None
|
| 1941 |
+
else counts.get("rows") if isinstance(counts, dict) else None)
|
| 1942 |
+
if success:
|
| 1943 |
+
refreshed_at = datetime.now(timezone.utc).isoformat()
|
| 1944 |
+
record_odoo_stage(rt, cid, "source", "ready", database_key=key,
|
| 1945 |
+
receipt=receipt)
|
| 1946 |
+
record_odoo_stage(rt, cid, "materialization", "ready",
|
| 1947 |
+
database_key=key, row_count=nrows, receipt=receipt)
|
| 1948 |
+
record_odoo_stage(rt, cid, "reconciliation", "ready",
|
| 1949 |
+
database_key=key, row_count=nrows, receipt=receipt)
|
| 1950 |
+
record_odoo_stage(rt, cid, "freshness", "ready", database_key=key,
|
| 1951 |
+
row_count=nrows, refreshed_at=refreshed_at, receipt=receipt)
|
| 1952 |
+
finish(success=True)
|
| 1953 |
+
else:
|
| 1954 |
+
raw_failure = ((database_errors.get(key) if isinstance(database_errors, dict) else None)
|
| 1955 |
+
or (receipt or {}).get("error") or (receipt or {}).get("sourceError"))
|
| 1956 |
+
failure = (_clean_odoo_failure(raw_failure, "reconciliation") if raw_failure
|
| 1957 |
+
else _odoo_failure("provider result was incomplete", "materialization"))
|
| 1958 |
+
failed_stage = str(failure.get("stage") or "materialization")
|
| 1959 |
+
failed_stage = (failed_stage if failed_stage in _ODOO_HEALTH_STAGES
|
| 1960 |
+
else "materialization")
|
| 1961 |
+
record_odoo_stage(rt, cid, failed_stage, "failed",
|
| 1962 |
+
database_key=key, failure=failure, receipt=receipt)
|
| 1963 |
+
record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
|
| 1964 |
+
failure={**failure, "stage": "freshness"}, receipt=receipt)
|
| 1965 |
+
finish(cause=f"{failure['code']}: {failure['message']}")
|
| 1966 |
except _SupersededMaterialization:
|
| 1967 |
# Disable/new Save already invalidated the table marker; never restore it here.
|
| 1968 |
finish()
|
| 1969 |
except rel.Refused as exc:
|
| 1970 |
+
failure = _odoo_failure(exc, "materialization")
|
| 1971 |
+
record_odoo_stage(rt, cid, failure["stage"], "failed",
|
| 1972 |
+
database_key=key, failure=failure)
|
| 1973 |
+
record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
|
| 1974 |
+
failure={**failure, "stage": "freshness"})
|
| 1975 |
+
finish(cause=f"{failure['code']}: {failure['message']}")
|
| 1976 |
except Exception as exc:
|
| 1977 |
+
# Raw provider exceptions may carry credentials or source values. Store only a named,
|
| 1978 |
+
# data-free classification, including the fail-closed TLS case.
|
| 1979 |
+
failure = _odoo_failure(exc, "source")
|
| 1980 |
if _odoo_read_access_refused(exc):
|
| 1981 |
+
failure = {"stage": "source", "code": "source_read_refused",
|
| 1982 |
+
"type": type(exc).__name__,
|
| 1983 |
+
"message": (f"Odoo refused read access to {job['model']}. Give the "
|
| 1984 |
+
"connected account read access, or switch this database off.")}
|
| 1985 |
+
record_odoo_stage(rt, cid, failure["stage"], "failed",
|
| 1986 |
+
database_key=key, failure=failure)
|
| 1987 |
+
record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
|
| 1988 |
+
failure={**failure, "stage": "freshness"})
|
| 1989 |
+
finish(cause=f"{failure['code']}: {failure['message']}")
|
| 1990 |
finally:
|
| 1991 |
stopped.set()
|
| 1992 |
if heartbeat_thread:
|
|
|
|
| 2217 |
def odoo_status(connection_id: str = None, session: Session = Depends(require_session)):
|
| 2218 |
_odoo_admin(session)
|
| 2219 |
import odoo_relational
|
| 2220 |
+
resolved_connection = _odoo_connection_slot(session.runtime, connection_id)
|
| 2221 |
cfg = odoo_relational.read_config(session.runtime)
|
| 2222 |
stored = session.runtime.get(_ODOO_SYNC_STATUS) or {}
|
| 2223 |
errors = stored.get("errors") if isinstance(stored.get("errors"), list) else []
|
| 2224 |
phase = str(stored.get("phase") or "unknown")
|
| 2225 |
+
lifecycle = connector_sync.read(session.runtime, "odoo", resolved_connection)
|
| 2226 |
durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id))
|
| 2227 |
+
receipt = odoo_health_receipt(session.runtime, resolved_connection)
|
| 2228 |
if durable:
|
| 2229 |
return {"schedule": cfg.get("syncEvery"),
|
| 2230 |
"health": "paused" if cfg.get("frozen") else durable["state"],
|
| 2231 |
"lastRun": durable["at"], "frozen": cfg.get("frozen") is True,
|
| 2232 |
+
**durable, "stateAt": durable["at"], "connectionId": resolved_connection,
|
| 2233 |
+
"receipt": receipt}
|
| 2234 |
# The old keys stay the old default-connection answer. A caller that names a connection gets
|
| 2235 |
# that slot's lifecycle instead of the tenant-wide legacy document that caused the C6 defect.
|
| 2236 |
named = connection_id is not None
|
|
|
|
| 2254 |
"retryable": state in ("error", "partial"),
|
| 2255 |
"errorsTotal": int(errors_total or 0),
|
| 2256 |
"connectionId": (lifecycle or {}).get("connectionId")
|
| 2257 |
+
or resolved_connection,
|
| 2258 |
+
"receipt": receipt}
|
| 2259 |
|
| 2260 |
|
| 2261 |
@router.get("/connectors/{connector_key}/sync")
|
|
|
|
| 2271 |
|
| 2272 |
|
| 2273 |
@router.post("/connectors/{connector_key}/sync/now")
|
| 2274 |
+
def connector_sync_now(connector_key: str, connection_id: str = None,
|
| 2275 |
+
session: Session = Depends(require_session)):
|
| 2276 |
_sync_admin(session)
|
| 2277 |
if connector_key not in connector_sync.providers():
|
| 2278 |
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
|
| 2279 |
try:
|
| 2280 |
+
requested_connection = (_odoo_connection_slot(session.runtime, connection_id)
|
| 2281 |
+
if connector_key == "odoo" else
|
| 2282 |
+
str(connection_id or "default").strip() or "default")
|
| 2283 |
+
key = (connector_key, session.runtime.key)
|
| 2284 |
+
with _SYNC_REQUEST_CONNECTION_LOCK:
|
| 2285 |
+
# Bind under one API-side lock so a duplicate request cannot race ahead of the
|
| 2286 |
+
# accepted request and attach its different connection id to the queued intent.
|
| 2287 |
+
receipt = connector_sync.request_sync(connector_key, session.runtime.key,
|
| 2288 |
+
requested_by=session.uname)
|
| 2289 |
+
if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS:
|
| 2290 |
+
_SYNC_REQUEST_CONNECTIONS[key] = requested_connection
|
| 2291 |
+
receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key]
|
| 2292 |
+
return receipt
|
| 2293 |
except ValueError:
|
| 2294 |
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None
|
| 2295 |
|
|
|
|
| 2299 |
_sync_admin(session)
|
| 2300 |
if connector_key not in connector_sync.providers():
|
| 2301 |
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
|
| 2302 |
+
receipt = connector_sync.sync_request(connector_key, session.runtime.key)
|
| 2303 |
+
if receipt is not None:
|
| 2304 |
+
receipt["connectionId"] = sync_request_connection(connector_key, session.runtime.key)
|
| 2305 |
+
return receipt
|
| 2306 |
+
return {
|
| 2307 |
"provider": connector_key, "tenant": session.runtime.key, "state": "idle",
|
| 2308 |
"requestedAt": None, "startedAt": None, "finishedAt": None,
|
| 2309 |
"requestedBy": "", "detail": "", "accepted": False,
|
|
|
|
| 2353 |
_sync_admin(session)
|
| 2354 |
if connector_key not in connector_sync.providers():
|
| 2355 |
raise err(404, "sync_provider_unknown", "This connector has no database sync runner.")
|
| 2356 |
+
payload = {"provider": connector_key, "jobs": connector_sync.jobs(
|
| 2357 |
+
session.runtime, connector_key, connection_id)}
|
| 2358 |
+
if connector_key == "odoo":
|
| 2359 |
+
payload["receipt"] = odoo_health_receipt(session.runtime, connection_id)
|
| 2360 |
+
return _safe_connector_status(connector_key, payload)
|
| 2361 |
|
| 2362 |
|
| 2363 |
@router.get("/connectors/odoo/schema")
|
|
|
|
| 2837 |
return payload
|
| 2838 |
|
| 2839 |
|
| 2840 |
+
def _sync_stamp(value):
|
| 2841 |
+
"""Return a parseable public timestamp, never a creation-time freshness substitute."""
|
| 2842 |
+
text = str(value or "").strip()
|
| 2843 |
+
if not text:
|
| 2844 |
+
return None
|
| 2845 |
+
try:
|
| 2846 |
+
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
| 2847 |
+
if parsed.tzinfo is None:
|
| 2848 |
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
| 2849 |
+
return parsed.astimezone(timezone.utc).isoformat()
|
| 2850 |
+
except (TypeError, ValueError, OverflowError):
|
| 2851 |
+
return None
|
| 2852 |
+
|
| 2853 |
+
|
| 2854 |
+
def _database_sync_health(*, enabled=True, state="never_run", last_success=None,
|
| 2855 |
+
source_timestamp=None, materialized_timestamp=None, row_count=None,
|
| 2856 |
+
error=None, now=None):
|
| 2857 |
+
"""Normalize one database's durable facts into S63-T34's shared public vocabulary."""
|
| 2858 |
+
clean_state = str(state or "never_run")
|
| 2859 |
+
if clean_state not in DATABASE_SYNC_STATES:
|
| 2860 |
+
clean_state = "failed"
|
| 2861 |
+
error = {"code": "sync_state_unreadable",
|
| 2862 |
+
"message": "This database's saved sync state could not be read."}
|
| 2863 |
+
if not enabled:
|
| 2864 |
+
clean_state = "disabled"
|
| 2865 |
+
count = row_count if type(row_count) is int and row_count >= 0 else None
|
| 2866 |
+
success = _sync_stamp(last_success)
|
| 2867 |
+
source = _sync_stamp(source_timestamp)
|
| 2868 |
+
materialized = _sync_stamp(materialized_timestamp)
|
| 2869 |
+
safe_error = None
|
| 2870 |
+
if error is not None:
|
| 2871 |
+
raw = error if isinstance(error, dict) else {}
|
| 2872 |
+
requested_code = str(raw.get("code") or "")
|
| 2873 |
+
code = requested_code if requested_code in _DATABASE_SAFE_ERRORS else "connector_sync_failed"
|
| 2874 |
+
safe_error = {
|
| 2875 |
+
"code": code,
|
| 2876 |
+
# Raw provider and exception text never crosses this boundary.
|
| 2877 |
+
"message": _DATABASE_SAFE_ERRORS.get(code, "This database could not be synced."),
|
| 2878 |
+
}
|
| 2879 |
+
# A current failure may retain the prior success timestamp, but it can never look current.
|
| 2880 |
+
lag = _lag_seconds(success, now) if success else None
|
| 2881 |
+
return {"state": clean_state, "lastSuccess": success,
|
| 2882 |
+
"sourceTimestamp": source, "materializedTimestamp": materialized,
|
| 2883 |
+
"rowCount": count, "lagSeconds": lag, "error": safe_error}
|
| 2884 |
+
|
| 2885 |
+
|
| 2886 |
+
def _odoo_database_sync_health(raw, enabled=True, *, now=None):
|
| 2887 |
+
row = raw if isinstance(raw, dict) else {}
|
| 2888 |
+
stages = row.get("stages") if isinstance(row.get("stages"), dict) else {}
|
| 2889 |
+
failure = row.get("failure") if isinstance(row.get("failure"), dict) else None
|
| 2890 |
+
raw_state = str(row.get("state") or "unknown")
|
| 2891 |
+
stage_states = {str((value or {}).get("state") or "unknown")
|
| 2892 |
+
for value in stages.values() if isinstance(value, dict)}
|
| 2893 |
+
last_success = row.get("lastSuccess")
|
| 2894 |
+
if failure or "failed" in stage_states or raw_state == "failed":
|
| 2895 |
+
state = "partial" if last_success else "failed"
|
| 2896 |
+
elif raw_state == "running" or "running" in stage_states:
|
| 2897 |
+
state = "running"
|
| 2898 |
+
elif raw_state == "queued" or "queued" in stage_states:
|
| 2899 |
+
state = "queued"
|
| 2900 |
+
elif last_success or raw_state == "ready":
|
| 2901 |
+
state = "succeeded"
|
| 2902 |
+
else:
|
| 2903 |
+
state = "never_run"
|
| 2904 |
+
source_stage = stages.get("source") if isinstance(stages.get("source"), dict) else {}
|
| 2905 |
+
material_stage = (stages.get("materialization")
|
| 2906 |
+
if isinstance(stages.get("materialization"), dict) else {})
|
| 2907 |
+
return _database_sync_health(
|
| 2908 |
+
enabled=enabled, state=state, last_success=last_success,
|
| 2909 |
+
source_timestamp=source_stage.get("lastSuccess"),
|
| 2910 |
+
materialized_timestamp=material_stage.get("lastSuccess"),
|
| 2911 |
+
row_count=row.get("rowCount"), error=failure, now=now)
|
| 2912 |
+
|
| 2913 |
+
|
| 2914 |
+
def _amazon_database_sync_health(raw, enabled=True, *, now=None):
|
| 2915 |
+
if not enabled:
|
| 2916 |
+
return _database_sync_health(enabled=False)
|
| 2917 |
+
if not isinstance(raw, dict):
|
| 2918 |
+
return _database_sync_health(
|
| 2919 |
+
state="failed", error={"code": "sync_status_unavailable",
|
| 2920 |
+
"message": "Sync status is unavailable for this database."})
|
| 2921 |
+
success = raw.get("lastSuccess")
|
| 2922 |
+
if raw.get("stuck") is True:
|
| 2923 |
+
state = "partial" if success else "failed"
|
| 2924 |
+
error = {"code": "sync_stalled",
|
| 2925 |
+
"message": "This database sync is stalled."}
|
| 2926 |
+
elif raw.get("inProgress") is True:
|
| 2927 |
+
state, error = "running", None
|
| 2928 |
+
elif success:
|
| 2929 |
+
state, error = "succeeded", None
|
| 2930 |
+
elif raw.get("deferred"):
|
| 2931 |
+
state, error = "queued", None
|
| 2932 |
+
else:
|
| 2933 |
+
state, error = "never_run", None
|
| 2934 |
+
return _database_sync_health(
|
| 2935 |
+
state=state, last_success=success, materialized_timestamp=success,
|
| 2936 |
+
row_count=raw.get("rowsLoaded"), error=error, now=now)
|
| 2937 |
+
|
| 2938 |
+
|
| 2939 |
+
def _stored_database_sync_health(raw, enabled=True, *, now=None):
|
| 2940 |
+
row = raw if isinstance(raw, dict) else {}
|
| 2941 |
+
return _database_sync_health(
|
| 2942 |
+
enabled=enabled, state=row.get("state") or "never_run",
|
| 2943 |
+
last_success=row.get("lastSuccess"), source_timestamp=row.get("sourceTimestamp"),
|
| 2944 |
+
materialized_timestamp=row.get("materializedTimestamp"), row_count=row.get("rowCount"),
|
| 2945 |
+
error=row.get("error"), now=now)
|
| 2946 |
+
|
| 2947 |
+
|
| 2948 |
def _detail_connect(entry_type, is_admin):
|
| 2949 |
"""C4's `keychain.connect` half: what the Connect form must ask for, and what it may store.
|
| 2950 |
|
|
|
|
| 3328 |
return named
|
| 3329 |
|
| 3330 |
|
| 3331 |
+
def _clean_scraper_recipe(raw):
|
| 3332 |
+
"""Return T11's persisted recipe shape and explicit structural validation errors.
|
| 3333 |
+
|
| 3334 |
+
This function is intentionally pure. Preview and creation both call it, so opening a saved
|
| 3335 |
+
recipe cannot reinterpret it differently, and validation cannot accidentally become a paid
|
| 3336 |
+
provider probe.
|
| 3337 |
+
"""
|
| 3338 |
+
value = raw if isinstance(raw, dict) else {}
|
| 3339 |
+
kind = str(value.get("kind") or "").strip().lower()
|
| 3340 |
+
errors = []
|
| 3341 |
+
if kind not in SCRAPER_RECIPE_KINDS:
|
| 3342 |
+
errors.append("Choose Instagram, TikTok, or Google Maps.")
|
| 3343 |
+
return {}, errors
|
| 3344 |
+
|
| 3345 |
+
limits = value.get("limits") if isinstance(value.get("limits"), dict) else {}
|
| 3346 |
+
schedule = value.get("schedule") if isinstance(value.get("schedule"), dict) else {}
|
| 3347 |
+
schedule_mode = str(schedule.get("mode") or "manual").strip().lower()
|
| 3348 |
+
if schedule_mode != "manual":
|
| 3349 |
+
errors.append("This recipe must use the manual Run now schedule.")
|
| 3350 |
+
|
| 3351 |
+
if kind == "google_maps":
|
| 3352 |
+
seed = value.get("seed") if isinstance(value.get("seed"), dict) else {}
|
| 3353 |
+
search = str(seed.get("search") or value.get("search") or "").strip()
|
| 3354 |
+
geography = str(seed.get("geography") or value.get("geography") or "").strip()
|
| 3355 |
+
raw_cap = limits.get("records", value.get("limit"))
|
| 3356 |
+
if not search:
|
| 3357 |
+
errors.append("Enter a category or search term.")
|
| 3358 |
+
if not geography:
|
| 3359 |
+
errors.append("Enter a geography.")
|
| 3360 |
+
if raw_cap in (None, ""):
|
| 3361 |
+
errors.append("Enter a result cap.")
|
| 3362 |
+
cap = 0
|
| 3363 |
+
else:
|
| 3364 |
+
try:
|
| 3365 |
+
cap = int(raw_cap)
|
| 3366 |
+
except (TypeError, ValueError):
|
| 3367 |
+
cap = 0
|
| 3368 |
+
errors.append("The result cap must be a whole number.")
|
| 3369 |
+
if raw_cap not in (None, "") and (cap < 1 or cap > 500):
|
| 3370 |
+
errors.append("The result cap must be between 1 and 500.")
|
| 3371 |
+
return ({"kind": kind, "source": "places",
|
| 3372 |
+
"seed": {"search": search, "geography": geography},
|
| 3373 |
+
"limits": {"records": cap}, "schedule": {"mode": "manual"}}, errors)
|
| 3374 |
+
|
| 3375 |
+
seed = str(value.get("seed") or "").strip()
|
| 3376 |
+
raw_limit = limits.get("posts", value.get("limit"))
|
| 3377 |
+
post_type = str(limits.get("postType") or value.get("postType") or "video").strip().lower()
|
| 3378 |
+
if not seed:
|
| 3379 |
+
errors.append(f"Enter an {kind.title()} profile seed.")
|
| 3380 |
+
if raw_limit in (None, ""):
|
| 3381 |
+
errors.append("Enter a last-N post limit.")
|
| 3382 |
+
limit = 0
|
| 3383 |
+
else:
|
| 3384 |
+
try:
|
| 3385 |
+
limit = int(raw_limit)
|
| 3386 |
+
except (TypeError, ValueError):
|
| 3387 |
+
limit = 0
|
| 3388 |
+
errors.append("The post limit must be a whole number.")
|
| 3389 |
+
if raw_limit not in (None, "") and (limit < 1 or limit > SCRAPER_RECIPE_MAX_ROWS):
|
| 3390 |
+
errors.append(f"The post limit must be between 1 and {SCRAPER_RECIPE_MAX_ROWS}.")
|
| 3391 |
+
allowed_types = ("video", "image", "carousel") if kind == "instagram" else ("video",)
|
| 3392 |
+
if post_type not in allowed_types:
|
| 3393 |
+
errors.append("Choose video, image, or carousel posts."
|
| 3394 |
+
if kind == "instagram" else
|
| 3395 |
+
"TikTok's native profile window supports video posts only.")
|
| 3396 |
+
return ({"kind": kind, "source": "profile", "seed": seed,
|
| 3397 |
+
"limits": {"posts": limit, "postType": post_type},
|
| 3398 |
+
"schedule": {"mode": "manual"}}, errors)
|
| 3399 |
+
|
| 3400 |
+
|
| 3401 |
+
def _scraper_recipe_fields(kind):
|
| 3402 |
+
"""Stable user-table schema for one recipe, derived from the shipped normalisers."""
|
| 3403 |
+
if kind == "google_maps":
|
| 3404 |
+
return [dict(field) for field in _MAPS_RECIPE_FIELDS]
|
| 3405 |
+
import automation_engine as engine # noqa: PLC0415 - existing schema owner
|
| 3406 |
+
source = engine.POST_FIELDS if kind == "instagram" else engine.TT_POST_FIELDS
|
| 3407 |
+
# Links and rollups target the shared automation tables. A recipe database is independent,
|
| 3408 |
+
# so copying those references would create a plausible but broken relationship graph.
|
| 3409 |
+
return [dict(field) for field in source
|
| 3410 |
+
if isinstance(field, dict) and field.get("type") not in ("link", "rollup")]
|
| 3411 |
+
|
| 3412 |
+
|
| 3413 |
+
def _scraper_plan(recipe):
|
| 3414 |
+
"""Network-free setup preview: declared provider, modelled spend, cap, and credential state."""
|
| 3415 |
+
kind = str((recipe or {}).get("kind") or "")
|
| 3416 |
+
limit_key = "records" if kind == "google_maps" else "posts"
|
| 3417 |
+
limit = int(((recipe or {}).get("limits") or {}).get(limit_key) or 0)
|
| 3418 |
+
if kind == "google_maps":
|
| 3419 |
+
import providers # noqa: PLC0415 - registry owns cost/capability
|
| 3420 |
+
apify = providers.PROVIDERS["apify"]
|
| 3421 |
+
capability = apify.cap("google_maps_stores")
|
| 3422 |
+
per_record = float(capability.cost_per_record or 0.0) if capability else 0.0
|
| 3423 |
+
amount = round(per_record * limit, 4)
|
| 3424 |
+
return {"provider": apify.label, "credentialReady": apify.configured(),
|
| 3425 |
+
"estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit,
|
| 3426 |
+
"canRun": True,
|
| 3427 |
+
"note": "Modelled from the declared per-place rate; the provider receipt "
|
| 3428 |
+
"reconciles the bounded result count."}
|
| 3429 |
+
|
| 3430 |
+
import providers # noqa: PLC0415 - registry is the authority
|
| 3431 |
+
bright = providers.PROVIDERS["brightdata"]
|
| 3432 |
+
apify = providers.PROVIDERS["apify"]
|
| 3433 |
+
if kind == "tiktok":
|
| 3434 |
+
per_record = float(bright.cap("tt_post_metrics").cost_per_record or 0.0)
|
| 3435 |
+
profile_cost = float(bright.cap("tt_profile").cost_per_record or 0.0)
|
| 3436 |
+
estimate_usd = profile_cost + (per_record * limit)
|
| 3437 |
+
provider_label = bright.label
|
| 3438 |
+
credential_ready = bright.configured()
|
| 3439 |
+
else:
|
| 3440 |
+
# The established Instagram runner can use Bright Data and its Apify fallback. The cap
|
| 3441 |
+
# includes both post metrics and view enrichment, so it never hides the worst declared
|
| 3442 |
+
# cost from the person authorising Run now.
|
| 3443 |
+
bd_cost = float(bright.cap("ig_post_metrics").cost_per_record or 0.0)
|
| 3444 |
+
profile_cost = float(bright.cap("ig_profile").cost_per_record or 0.0)
|
| 3445 |
+
view_cost = float(apify.cap("ig_post_views").cost_per_record or 0.0)
|
| 3446 |
+
estimate_usd = profile_cost + ((bd_cost + view_cost) * limit)
|
| 3447 |
+
provider_label = (bright.label if bright.configured() or not apify.configured()
|
| 3448 |
+
else apify.label)
|
| 3449 |
+
credential_ready = bright.configured() or apify.configured()
|
| 3450 |
+
amount = round(estimate_usd, 4)
|
| 3451 |
+
return {"provider": provider_label, "credentialReady": bool(credential_ready),
|
| 3452 |
+
"estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit,
|
| 3453 |
+
"canRun": True, "note": "Modelled from declared per-record list rates."}
|
| 3454 |
+
|
| 3455 |
+
|
| 3456 |
+
def _scraper_databases(rt, is_admin, uname):
|
| 3457 |
+
"""Saved recipe databases this reader may open, with no row payloads or secrets."""
|
| 3458 |
+
try:
|
| 3459 |
+
import core.user_tables as user_tables
|
| 3460 |
+
definitions = user_tables.all_defs(rt)
|
| 3461 |
+
lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
|
| 3462 |
+
except Exception: # noqa: BLE001
|
| 3463 |
+
return []
|
| 3464 |
+
rows = []
|
| 3465 |
+
for key, definition in (definitions or {}).items():
|
| 3466 |
+
recipe = definition.get("scraperRecipe") if isinstance(definition, dict) else None
|
| 3467 |
+
if not isinstance(recipe, dict):
|
| 3468 |
+
continue
|
| 3469 |
+
try:
|
| 3470 |
+
if not user_tables.may_open(key, uname, bool(is_admin), st=lent):
|
| 3471 |
+
continue
|
| 3472 |
+
except Exception: # noqa: BLE001
|
| 3473 |
+
continue
|
| 3474 |
+
cleaned, errors = _clean_scraper_recipe(recipe)
|
| 3475 |
+
if errors:
|
| 3476 |
+
continue
|
| 3477 |
+
plan = _scraper_plan(cleaned)
|
| 3478 |
+
# Recipe existence alone is not enablement. The creation door stamps both ownership
|
| 3479 |
+
# markers; a legacy/malformed row without them is disclosed as disabled and cannot run.
|
| 3480 |
+
enabled = (definition.get("connectorKey") == "scraper"
|
| 3481 |
+
and definition.get("recordMode") == "automation")
|
| 3482 |
+
rows.append({"key": str(key), "label": str(definition.get("label") or key),
|
| 3483 |
+
"defaultLabel": str(definition.get("label") or key),
|
| 3484 |
+
"native": cleaned["kind"], "enabled": enabled, "exists": True,
|
| 3485 |
+
"canRename": True, "sourceRows": None, "populated": None,
|
| 3486 |
+
"fields": [dict(field) for field in (definition.get("fields") or ())
|
| 3487 |
+
if isinstance(field, dict)],
|
| 3488 |
+
"excluded": [], "sharesFieldsWith": [], "scraperRecipe": cleaned,
|
| 3489 |
+
"syncHealth": _stored_database_sync_health(
|
| 3490 |
+
definition.get(user_tables.CONNECTOR_HEALTH_FIELD), enabled),
|
| 3491 |
+
"scraperRuns": [dict(item) for item in (definition.get("scraperRuns") or ())
|
| 3492 |
+
if isinstance(item, dict)],
|
| 3493 |
+
"runPath": f"/connectors/scraper/databases/{key}/run",
|
| 3494 |
+
"canRun": enabled and plan["canRun"] is True})
|
| 3495 |
+
return _named_databases(rt, is_admin, uname, rows)
|
| 3496 |
+
|
| 3497 |
+
|
| 3498 |
def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None):
|
| 3499 |
"""C4's `databases` half: what this connector materialises, and the fields behind each one.
|
| 3500 |
|
|
|
|
| 3523 |
reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
|
| 3524 |
is walled, per row, by `canRename`.
|
| 3525 |
"""
|
| 3526 |
+
if str(connector_key) == "scraper":
|
| 3527 |
+
return _scraper_databases(rt, is_admin, uname)
|
| 3528 |
if not connector_directory.is_introspectable(connector_key):
|
| 3529 |
return []
|
| 3530 |
spec = _connector_spec_tables(connector_key)
|
|
|
|
| 3541 |
census = odoo_relational.population_census(rt, namespace)
|
| 3542 |
except Exception: # noqa: BLE001
|
| 3543 |
return []
|
| 3544 |
+
health = odoo_health_receipt(rt, connection_id)
|
| 3545 |
+
health_by_key = {str(item.get("databaseKey") or ""): item
|
| 3546 |
+
for item in (health.get("databases") or ())}
|
| 3547 |
census_at = str(census.get("capturedAt") or "")
|
| 3548 |
counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
|
| 3549 |
catalog = {}
|
|
|
|
| 3588 |
# Counts are model-level facts. Databases backed by the same model deliberately show the
|
| 3589 |
# same sourceRows number; inventing a per-database domain here would make the number lie.
|
| 3590 |
source_rows = counts.get(model) if census_at and model in counts else None
|
| 3591 |
+
database_health = health_by_key.get(key) or {
|
| 3592 |
+
# Source rows is a model census. It is not this governed database's materialized
|
| 3593 |
+
# row count, especially when two curated databases share one source model.
|
| 3594 |
+
"databaseKey": key, "state": "unknown", "rowCount": None,
|
| 3595 |
+
"lastSuccess": None, "currentLagSeconds": None, "recordRefreshed": None,
|
| 3596 |
+
"failure": None, "stages": _empty_odoo_stages()}
|
| 3597 |
out.append({"key": key,
|
| 3598 |
"identity": connector_sync.database_identity(
|
| 3599 |
"odoo", resolved_connection_id, key, model),
|
|
|
|
| 3603 |
"enabled": bool(choice.get("enabled")),
|
| 3604 |
"sourceRows": source_rows,
|
| 3605 |
"populated": None if source_rows is None else source_rows > 0,
|
| 3606 |
+
# Data-free client receipt: source/mirror/materialization/reconciliation/
|
| 3607 |
+
# freshness are distinct, and one database's failure cannot make a sibling
|
| 3608 |
+
# look newly successful.
|
| 3609 |
+
"sync": database_health,
|
| 3610 |
+
"syncHealth": _odoo_database_sync_health(
|
| 3611 |
+
database_health, bool(choice.get("enabled"))),
|
| 3612 |
"fields": scoped_fields,
|
| 3613 |
"excluded": [dict(item) for item in excluded],
|
| 3614 |
"sharesFieldsWith": []})
|
|
|
|
| 3699 |
# what `native` carries: the vendor's own name for the thing, never one we invented. It is
|
| 3700 |
# empty for a table whose report type is unknown, and the tab says so rather than guessing.
|
| 3701 |
declared = (spec or {}).get(key) or {}
|
| 3702 |
+
raw_sync = (states or {}).get(name) if is_admin else None
|
| 3703 |
+
is_enabled = enabled.get(key) is True
|
| 3704 |
out.append({"key": key,
|
| 3705 |
"label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
|
| 3706 |
"defaultLabel": str(declared.get("label")
|
|
|
|
| 3709 |
# W55-T19 / R7 — AMAZON'S OWN tenant-local answer. This is deliberately not
|
| 3710 |
# read from Odoo's grid config, and disabling it leaves every stored row and
|
| 3711 |
# definition untouched: only this provider config document changes.
|
| 3712 |
+
"enabled": is_enabled,
|
| 3713 |
+
**({"sync": raw_sync} if is_admin else {}),
|
| 3714 |
+
"syncHealth": _amazon_database_sync_health(raw_sync, is_enabled),
|
| 3715 |
"fields": fields,
|
| 3716 |
"excluded": excluded,
|
| 3717 |
"sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
|
|
|
|
| 3779 |
stops "administrator only" being rendered as "this database has no fields".
|
| 3780 |
"""
|
| 3781 |
key = str(row.get("key") or "")
|
| 3782 |
+
if key == "scraper":
|
| 3783 |
+
# Recipe databases publish their saved, stable schema to every reader who may open the
|
| 3784 |
+
# database. They have no generic schema-toggle route, so no write capability is implied.
|
| 3785 |
+
return {"introspectable": True, "reason": "", "fieldsVisible": True,
|
| 3786 |
+
"canToggleFields": False, "canToggleTables": False,
|
| 3787 |
+
"catalogueComplete": True, "capturedAt": "", "excludedBasis": "",
|
| 3788 |
+
"censusAt": "", "censusComplete": False, "censusDurationMs": 0,
|
| 3789 |
+
"censusModelsCounted": 0, "censusModelsWithRows": 0}
|
| 3790 |
if not connector_directory.is_introspectable(key):
|
| 3791 |
reason = ("Nothing is connected behind this listing yet, so there is no schema to read."
|
| 3792 |
if str(row.get("state") or "") == connector_directory.CATALOGUE_STATE
|
|
|
|
| 3907 |
"updatePath": "", "preservesData": True, "available": False}
|
| 3908 |
|
| 3909 |
|
| 3910 |
+
def _scraper_preview(body):
|
| 3911 |
+
request = body if isinstance(body, dict) else {}
|
| 3912 |
+
raw_recipe = request.get("recipe") if isinstance(request.get("recipe"), dict) else request
|
| 3913 |
+
recipe, errors = _clean_scraper_recipe(raw_recipe)
|
| 3914 |
+
if errors:
|
| 3915 |
+
raise err(400, "invalid_scraper_recipe", " ".join(errors))
|
| 3916 |
+
plan = _scraper_plan(recipe)
|
| 3917 |
+
missing = [] if plan["credentialReady"] else ["credential"]
|
| 3918 |
+
return {"recipe": recipe, "inputs": recipe["seed"], "provider": plan["provider"],
|
| 3919 |
+
"credentialReady": plan["credentialReady"], "estimatedUsd": plan["estimatedUsd"],
|
| 3920 |
+
"costCapUsd": plan["costCapUsd"], "recordsCap": plan["recordsCap"],
|
| 3921 |
+
"canRun": plan["canRun"], "missing": missing, "note": plan["note"],
|
| 3922 |
+
# This is a contract field, not a measurement. A preview and a create never call a
|
| 3923 |
+
# provider, so both truthfully report zero incurred spend.
|
| 3924 |
+
"incurredUsd": 0.0}
|
| 3925 |
+
|
| 3926 |
+
|
| 3927 |
+
def _create_scraper_database(rt, uname, body):
|
| 3928 |
+
"""Persist one recipe database. Contains no provider import or runner call by construction."""
|
| 3929 |
+
if not rt.available():
|
| 3930 |
+
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 3931 |
+
request = body if isinstance(body, dict) else {}
|
| 3932 |
+
label = str(request.get("label") or "").strip()
|
| 3933 |
+
if not label:
|
| 3934 |
+
raise err(400, "missing_database_name", "Enter a database name.")
|
| 3935 |
+
preview = _scraper_preview(request)
|
| 3936 |
+
recipe = preview["recipe"]
|
| 3937 |
+
import core.user_tables as user_tables # noqa: PLC0415 - existing create door
|
| 3938 |
+
key = user_tables.create(label, uname, fields=_scraper_recipe_fields(recipe["kind"]),
|
| 3939 |
+
source=user_tables.AUTOMATION_SOURCE, st=rt)
|
| 3940 |
+
if not key:
|
| 3941 |
+
raise err(409, "database_not_created",
|
| 3942 |
+
"The database name is unavailable or this workspace reached its database limit.")
|
| 3943 |
+
|
| 3944 |
+
def _attach(cur):
|
| 3945 |
+
current = dict(cur or {})
|
| 3946 |
+
definition = dict(current.get(key) or {})
|
| 3947 |
+
if not definition:
|
| 3948 |
+
raise RuntimeError("the new database disappeared before its recipe was attached")
|
| 3949 |
+
definition.update({"connectorKey": "scraper", "recordMode": "automation",
|
| 3950 |
+
"scraperRecipe": recipe, "scraperRuns": [],
|
| 3951 |
+
user_tables.CONNECTOR_HEALTH_FIELD:
|
| 3952 |
+
user_tables.connector_health_receipt("never_run")})
|
| 3953 |
+
current[key] = definition
|
| 3954 |
+
return current
|
| 3955 |
+
|
| 3956 |
+
try:
|
| 3957 |
+
rt.update(user_tables.STORE_KEY, _attach, flush="sync")
|
| 3958 |
+
except Exception: # noqa: BLE001
|
| 3959 |
+
user_tables.delete(key, st=rt)
|
| 3960 |
+
raise err(503, "recipe_not_saved",
|
| 3961 |
+
"The recipe could not be saved, so the new database was removed.")
|
| 3962 |
+
return {"key": key, "label": label, "recipe": recipe, "preview": preview,
|
| 3963 |
+
"createdWithoutProviderCall": True, "incurredUsd": 0.0}
|
| 3964 |
+
|
| 3965 |
+
|
| 3966 |
+
def _scraper_database_run(rt, uname, is_admin, database_key):
|
| 3967 |
+
"""The one explicit paid-work boundary for any saved scraper recipe."""
|
| 3968 |
+
if not rt.available():
|
| 3969 |
+
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 3970 |
+
import core.user_tables as user_tables
|
| 3971 |
+
key = str(database_key or "").strip()
|
| 3972 |
+
definitions = user_tables.lend_defs(rt)
|
| 3973 |
+
definition = user_tables.get(key, st=definitions) or {}
|
| 3974 |
+
if not definition:
|
| 3975 |
+
raise err(404, "unknown_scraper_database", "This saved scraper database was not found.")
|
| 3976 |
+
raw_recipe = definition.get("scraperRecipe")
|
| 3977 |
+
recipe, errors = _clean_scraper_recipe(raw_recipe)
|
| 3978 |
+
if errors:
|
| 3979 |
+
raw_kind = str((raw_recipe or {}).get("kind") or "") if isinstance(raw_recipe, dict) else ""
|
| 3980 |
+
raw_limits = ((raw_recipe or {}).get("limits") or {}) if isinstance(raw_recipe, dict) else {}
|
| 3981 |
+
raw_cap = raw_limits.get("records") if isinstance(raw_limits, dict) else None
|
| 3982 |
+
try:
|
| 3983 |
+
bounded = 1 <= int(raw_cap) <= 500
|
| 3984 |
+
except (TypeError, ValueError):
|
| 3985 |
+
bounded = False
|
| 3986 |
+
if raw_kind == "google_maps" and not bounded:
|
| 3987 |
+
raise err(400, "maps_unbounded_request",
|
| 3988 |
+
"Google Maps discovery requires an explicit result cap between 1 and 500.")
|
| 3989 |
+
raise err(409, "invalid_saved_scraper_recipe",
|
| 3990 |
+
"This saved scraper recipe is no longer valid. Review its settings before running it.")
|
| 3991 |
+
# Run now both spends against a provider account and mutates this database. A viewer grant
|
| 3992 |
+
# may expose the rows without granting either power, so the ordinary open wall is too broad.
|
| 3993 |
+
# Reuse the database editor authority that every other mutation surface already honours.
|
| 3994 |
+
if not user_tables.may_edit_database(key, uname, bool(is_admin), st=definitions):
|
| 3995 |
+
raise err(403, "database_forbidden", "You cannot run this database recipe.")
|
| 3996 |
+
plan = _scraper_plan(recipe)
|
| 3997 |
+
previous_health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD)
|
| 3998 |
+
previous_health = previous_health if isinstance(previous_health, dict) else {}
|
| 3999 |
+
|
| 4000 |
+
def record_attempt(state, error=None):
|
| 4001 |
+
try:
|
| 4002 |
+
return user_tables.record_connector_health(
|
| 4003 |
+
key, state, st=rt,
|
| 4004 |
+
last_success=previous_health.get("lastSuccess"),
|
| 4005 |
+
source_timestamp=previous_health.get("sourceTimestamp"),
|
| 4006 |
+
materialized_timestamp=previous_health.get("materializedTimestamp"),
|
| 4007 |
+
row_count=previous_health.get("rowCount"), error=error)
|
| 4008 |
+
except Exception: # noqa: BLE001
|
| 4009 |
+
raise err(503, "sync_status_not_saved",
|
| 4010 |
+
"The run did not start because its durable status could not be saved.") from None
|
| 4011 |
+
|
| 4012 |
+
if not plan["credentialReady"]:
|
| 4013 |
+
code = "maps_credential_missing" if recipe["kind"] == "google_maps" else \
|
| 4014 |
+
"scraper_credential_missing"
|
| 4015 |
+
message = "Add the scraper credential in Keychain before choosing Run now."
|
| 4016 |
+
record_attempt("failed", {"code": code, "message": message})
|
| 4017 |
+
raise err(409, code, "Add the scraper credential in Keychain before choosing Run now.")
|
| 4018 |
+
|
| 4019 |
+
# Imports and provider calls occur only below the explicit Run-now endpoint. Preview, create,
|
| 4020 |
+
# detail, and page load never reach either runner.
|
| 4021 |
+
started_at = datetime.now(timezone.utc).isoformat()
|
| 4022 |
+
record_attempt("running")
|
| 4023 |
+
key_field = "shortcode"
|
| 4024 |
+
if recipe["kind"] == "google_maps":
|
| 4025 |
+
import providers
|
| 4026 |
+
try:
|
| 4027 |
+
result = providers.discover_google_maps_stores(
|
| 4028 |
+
recipe["seed"]["search"], recipe["seed"]["geography"],
|
| 4029 |
+
recipe["limits"]["records"], captured_at=started_at)
|
| 4030 |
+
except providers.MapsProviderRefusal as exc:
|
| 4031 |
+
record_attempt("failed", {"code": exc.code, "message": exc.public_message})
|
| 4032 |
+
status = 409 if exc.code == "maps_credential_missing" else \
|
| 4033 |
+
(400 if exc.code in ("maps_unbounded_request", "maps_invalid_search") else 502)
|
| 4034 |
+
raise err(status, exc.code, exc.public_message)
|
| 4035 |
+
incoming_key = "rows"
|
| 4036 |
+
key_field = "place_id"
|
| 4037 |
+
else:
|
| 4038 |
+
limit = recipe["limits"]["posts"]
|
| 4039 |
+
post_type = recipe["limits"]["postType"]
|
| 4040 |
+
try:
|
| 4041 |
+
if recipe["kind"] == "instagram":
|
| 4042 |
+
import connectors_ig as runner
|
| 4043 |
+
run_recipe = runner.pull_ig_recipe
|
| 4044 |
+
else:
|
| 4045 |
+
import connectors_tt as runner
|
| 4046 |
+
run_recipe = runner.pull_tt_recipe
|
| 4047 |
+
result = run_recipe(recipe["seed"], limit, post_type=post_type,
|
| 4048 |
+
log=lambda *_args: None)
|
| 4049 |
+
except Exception: # noqa: BLE001
|
| 4050 |
+
record_attempt("failed", {
|
| 4051 |
+
"code": "scraper_run_failed",
|
| 4052 |
+
"message": "The provider could not complete this run. No result was written.",
|
| 4053 |
+
})
|
| 4054 |
+
raise err(502, "scraper_run_failed",
|
| 4055 |
+
"The provider could not complete this run. No result was written.")
|
| 4056 |
+
incoming_key = "posts"
|
| 4057 |
+
result = result if isinstance(result, dict) else {}
|
| 4058 |
+
incoming = [dict(row) for row in (result.get(incoming_key) or ()) if isinstance(row, dict)]
|
| 4059 |
+
state = str(result.get("state") or "error")
|
| 4060 |
+
if state not in ("ok", "partial"):
|
| 4061 |
+
incoming = []
|
| 4062 |
+
finished_at = datetime.now(timezone.utc).isoformat()
|
| 4063 |
+
source_stamps = [_sync_stamp(row.get("source_timestamp")) for row in incoming]
|
| 4064 |
+
source_timestamp = max((stamp for stamp in source_stamps if stamp), default=None)
|
| 4065 |
+
provider_counts = dict(result.get("counts") or {})
|
| 4066 |
+
provider_counts.setdefault("returned", len(incoming))
|
| 4067 |
+
provider_cost = dict(result.get("cost") or {})
|
| 4068 |
+
provider_cost.setdefault("currency", "USD")
|
| 4069 |
+
provider_cost.setdefault("estimatedUsd", plan["estimatedUsd"])
|
| 4070 |
+
provider_cost.setdefault("costCapUsd", plan["costCapUsd"])
|
| 4071 |
+
receipt = {"id": finished_at, "state": state,
|
| 4072 |
+
"provider": str(result.get("via") or plan["provider"]),
|
| 4073 |
+
"rows": len(incoming), "startedAt": started_at, "finishedAt": finished_at,
|
| 4074 |
+
"estimatedUsd": plan["estimatedUsd"], "costCapUsd": plan["costCapUsd"],
|
| 4075 |
+
"cost": provider_cost, "providerCounts": provider_counts,
|
| 4076 |
+
"attempts": [dict(item) for item in (result.get("attempts") or ())
|
| 4077 |
+
if isinstance(item, dict)],
|
| 4078 |
+
# Provider notes can echo a submitted handle. Run history stores operational
|
| 4079 |
+
# metadata only; the saved recipe already owns the input and no customer value is
|
| 4080 |
+
# duplicated into an audit-like receipt.
|
| 4081 |
+
"note": f"Provider returned {state}."}
|
| 4082 |
+
counts = {"inserted": 0, "updated": 0, "unchanged": 0, "skipped": 0,
|
| 4083 |
+
"duplicates": 0, "orphans": 0, "capped": 0}
|
| 4084 |
+
|
| 4085 |
+
def _write(cur):
|
| 4086 |
+
nonlocal counts
|
| 4087 |
+
current = dict(cur or {})
|
| 4088 |
+
stored = dict(current.get(key) or {})
|
| 4089 |
+
if not stored:
|
| 4090 |
+
raise RuntimeError("the database disappeared before the run could be written")
|
| 4091 |
+
if incoming:
|
| 4092 |
+
import automation_engine as engine
|
| 4093 |
+
rows, counts = engine.upsert_rows(stored.get("rows") or {}, incoming, key_field)
|
| 4094 |
+
stored["rows"] = rows
|
| 4095 |
+
processed = sum(int(counts.get(name) or 0)
|
| 4096 |
+
for name in ("inserted", "updated", "unchanged", "skipped", "capped"))
|
| 4097 |
+
receipt["writeCounts"] = dict(counts)
|
| 4098 |
+
receipt["countReconciled"] = (
|
| 4099 |
+
int(provider_counts.get("returned") or 0) == len(incoming) == processed)
|
| 4100 |
+
history = [dict(item) for item in (stored.get("scraperRuns") or ())
|
| 4101 |
+
if isinstance(item, dict)]
|
| 4102 |
+
stored["scraperRuns"] = [receipt, *history][:SCRAPER_RUN_HISTORY_LIMIT]
|
| 4103 |
+
final_state = "succeeded" if state == "ok" else "partial" if state == "partial" else "failed"
|
| 4104 |
+
final_error = None if final_state != "failed" else {
|
| 4105 |
+
"code": "provider_result_failed",
|
| 4106 |
+
"message": "The provider returned no usable result for this database.",
|
| 4107 |
+
}
|
| 4108 |
+
stored[user_tables.CONNECTOR_HEALTH_FIELD] = user_tables.connector_health_receipt(
|
| 4109 |
+
final_state,
|
| 4110 |
+
last_success=finished_at if final_state in ("succeeded", "partial")
|
| 4111 |
+
else previous_health.get("lastSuccess"),
|
| 4112 |
+
source_timestamp=source_timestamp,
|
| 4113 |
+
materialized_timestamp=finished_at if final_state in ("succeeded", "partial") else None,
|
| 4114 |
+
row_count=len(stored.get("rows") or {}), error=final_error)
|
| 4115 |
+
current[key] = stored
|
| 4116 |
+
return current
|
| 4117 |
+
|
| 4118 |
+
try:
|
| 4119 |
+
rt.update(user_tables.STORE_KEY, _write, flush="sync")
|
| 4120 |
+
except Exception: # noqa: BLE001
|
| 4121 |
+
try:
|
| 4122 |
+
record_attempt("failed", {
|
| 4123 |
+
"code": "scraper_result_not_saved",
|
| 4124 |
+
"message": "The provider answered, but the result could not be saved.",
|
| 4125 |
+
})
|
| 4126 |
+
except Exception: # noqa: BLE001
|
| 4127 |
+
pass
|
| 4128 |
+
raise err(503, "scraper_result_not_saved",
|
| 4129 |
+
"The provider answered, but the result could not be saved. Reload before retrying.")
|
| 4130 |
+
return {"key": key, "state": state, "counts": counts, "receipt": receipt}
|
| 4131 |
+
|
| 4132 |
+
|
| 4133 |
def connector_detail(rt, user, uname, is_admin, connector_key):
|
| 4134 |
"""The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
|
| 4135 |
|
|
|
|
| 4228 |
return _set_amazon_database_enablement(database_key, body, session)
|
| 4229 |
|
| 4230 |
|
| 4231 |
+
@router.post("/connectors/scraper/databases/preview")
|
| 4232 |
+
def scraper_database_preview(body: dict = Body(default=None),
|
| 4233 |
+
session: Session = Depends(require_session)):
|
| 4234 |
+
"""Validate and price a recipe locally. This endpoint makes zero provider calls."""
|
| 4235 |
+
return _scraper_preview(body)
|
| 4236 |
+
|
| 4237 |
+
|
| 4238 |
+
@router.post("/connectors/scraper/databases")
|
| 4239 |
+
def scraper_database_create(body: dict = Body(default=None),
|
| 4240 |
+
session: Session = Depends(require_session)):
|
| 4241 |
+
"""Create a saved recipe database for $0; paid work begins only at its Run-now door."""
|
| 4242 |
+
return _create_scraper_database(session.runtime, session.uname, body)
|
| 4243 |
+
|
| 4244 |
+
|
| 4245 |
+
@router.post("/connectors/scraper/databases/{database_key}/run")
|
| 4246 |
+
def scraper_database_run(database_key: str, session: Session = Depends(require_session)):
|
| 4247 |
+
"""Explicitly execute one bounded saved scraper recipe through its provider seam."""
|
| 4248 |
+
return _scraper_database_run(session.runtime, session.uname, bool(session.admin), database_key)
|
| 4249 |
+
|
| 4250 |
+
|
| 4251 |
# ⚠ DECLARED AFTER THE LITERAL `/connectors/odoo/*` ROUTES. FastAPI matches in declaration order,
|
| 4252 |
# and while none of those ends in `/detail` — so no shadowing is possible today — the ordering is
|
| 4253 |
# the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
|
api/routes_customers.py
CHANGED
|
@@ -22,8 +22,10 @@ owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtim
|
|
| 22 |
proof is single-process and would report green on exactly the thing that is still broken.
|
| 23 |
"""
|
| 24 |
import datetime as _dt
|
|
|
|
| 25 |
import hashlib
|
| 26 |
import json
|
|
|
|
| 27 |
import time
|
| 28 |
|
| 29 |
from fastapi import APIRouter, Body, Depends
|
|
@@ -39,6 +41,14 @@ MODULE = "customer_data"
|
|
| 39 |
|
| 40 |
_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
|
| 43 |
def _pool_rows(session: Session):
|
| 44 |
"""The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that
|
|
@@ -304,6 +314,65 @@ def _mark_offerability(fields):
|
|
| 304 |
|
| 305 |
def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
|
| 306 |
consume_corrections: bool = True):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
"""ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
|
| 308 |
events route (2026-07-31 — the standalone measure gap, owner item 1).
|
| 309 |
|
|
@@ -330,6 +399,10 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 330 |
|
| 331 |
import core.perm_scope as perm_scope
|
| 332 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 333 |
rt = session.runtime
|
| 334 |
team_id, agent = _team_agent(session)
|
| 335 |
rows_src = _pool_for(rt, team_id, agent)
|
|
|
|
| 22 |
proof is single-process and would report green on exactly the thing that is still broken.
|
| 23 |
"""
|
| 24 |
import datetime as _dt
|
| 25 |
+
from copy import deepcopy
|
| 26 |
import hashlib
|
| 27 |
import json
|
| 28 |
+
import threading
|
| 29 |
import time
|
| 30 |
|
| 31 |
from fastapi import APIRouter, Body, Depends
|
|
|
|
| 41 |
|
| 42 |
_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
|
| 43 |
|
| 44 |
+
# A slow producer must not indefinitely hold another threadpool request; timeout recomputes.
|
| 45 |
+
_GRID_ASSEMBLY_WAIT_SECONDS = 10.0
|
| 46 |
+
_grid_assembly_lock = threading.Lock()
|
| 47 |
+
_grid_assembly_inflight = {}
|
| 48 |
+
_grid_assembly_enabled = True
|
| 49 |
+
grid_assembly_computed = 0
|
| 50 |
+
grid_assembly_shared = 0
|
| 51 |
+
|
| 52 |
|
| 53 |
def _pool_rows(session: Session):
|
| 54 |
"""The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that
|
|
|
|
| 314 |
|
| 315 |
def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
|
| 316 |
consume_corrections: bool = True):
|
| 317 |
+
"""Share only overlapping assemblies of the same user's grid, never completed results."""
|
| 318 |
+
global grid_assembly_shared
|
| 319 |
+
rt = session.runtime
|
| 320 |
+
team_id, agent = _team_agent(session)
|
| 321 |
+
key = (session.tenant, session.uname, session.admin, team_id, agent, scope,
|
| 322 |
+
consume_corrections, _pool_stamp(rt, team_id, agent),
|
| 323 |
+
# A changed permission record must not borrow an assembly from the old record.
|
| 324 |
+
# ⛔ `default=str` IS NOT TIDINESS. This runs on every `/customers` and `/workspace`,
|
| 325 |
+
# the product's main grid; a `TypeError` from an unserialisable value in a stored user
|
| 326 |
+
# record would 500 BOTH doors, which is a far worse failure than the duplicated work
|
| 327 |
+
# this key exists to remove. The record round-trips JSON by construction today, so the
|
| 328 |
+
# fallback should never fire -- and a key that cannot raise is the only kind worth
|
| 329 |
+
# putting in front of a hot path.
|
| 330 |
+
json.dumps(session.user, sort_keys=True, separators=(",", ":"), default=str))
|
| 331 |
+
|
| 332 |
+
def own_copy(result):
|
| 333 |
+
# The pool is already shared by scope_cache and rows_from_pool never mutates it.
|
| 334 |
+
# Copy the rest together to preserve internal aliases without lending mutable state.
|
| 335 |
+
out = deepcopy({k: v for k, v in result.items() if k != "rows_src"})
|
| 336 |
+
out["rows_src"] = result["rows_src"]
|
| 337 |
+
out["workspace"]["storageKey"] = storage_key
|
| 338 |
+
return out
|
| 339 |
+
|
| 340 |
+
if not _grid_assembly_enabled:
|
| 341 |
+
return own_copy(_compute_grid_assembly(session, scope, storage_key, consume_corrections))
|
| 342 |
+
|
| 343 |
+
with _grid_assembly_lock:
|
| 344 |
+
entry = _grid_assembly_inflight.get(key)
|
| 345 |
+
producer = entry is None
|
| 346 |
+
if producer:
|
| 347 |
+
entry = {"event": threading.Event(), "result": None, "error": None}
|
| 348 |
+
_grid_assembly_inflight[key] = entry
|
| 349 |
+
|
| 350 |
+
if not producer:
|
| 351 |
+
if entry["event"].wait(_GRID_ASSEMBLY_WAIT_SECONDS):
|
| 352 |
+
if entry["error"] is not None:
|
| 353 |
+
raise entry["error"]
|
| 354 |
+
result = own_copy(entry["result"])
|
| 355 |
+
with _grid_assembly_lock:
|
| 356 |
+
grid_assembly_shared += 1
|
| 357 |
+
return result
|
| 358 |
+
# A timeout changes neither the caller's answer nor the producer's ownership.
|
| 359 |
+
return own_copy(_compute_grid_assembly(session, scope, storage_key, consume_corrections))
|
| 360 |
+
|
| 361 |
+
try:
|
| 362 |
+
entry["result"] = _compute_grid_assembly(session, scope, storage_key, consume_corrections)
|
| 363 |
+
except BaseException as exc:
|
| 364 |
+
entry["error"] = exc
|
| 365 |
+
raise
|
| 366 |
+
finally:
|
| 367 |
+
with _grid_assembly_lock:
|
| 368 |
+
if _grid_assembly_inflight.get(key) is entry:
|
| 369 |
+
del _grid_assembly_inflight[key]
|
| 370 |
+
entry["event"].set()
|
| 371 |
+
return own_copy(entry["result"])
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def _compute_grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
|
| 375 |
+
consume_corrections: bool = True):
|
| 376 |
"""ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
|
| 377 |
events route (2026-07-31 — the standalone measure gap, owner item 1).
|
| 378 |
|
|
|
|
| 399 |
|
| 400 |
import core.perm_scope as perm_scope
|
| 401 |
|
| 402 |
+
global grid_assembly_computed
|
| 403 |
+
with _grid_assembly_lock:
|
| 404 |
+
grid_assembly_computed += 1
|
| 405 |
+
|
| 406 |
rt = session.runtime
|
| 407 |
team_id, agent = _team_agent(session)
|
| 408 |
rows_src = _pool_for(rt, team_id, agent)
|
api/routes_external.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_geo.py
CHANGED
|
@@ -998,6 +998,53 @@ def _route_endpoint_index(pids, raw, label):
|
|
| 998 |
return matches[0]
|
| 999 |
|
| 1000 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1001 |
@router.post("/geo/route")
|
| 1002 |
def geo_route(body: dict = Body(...), session: Session = Depends(require_session)):
|
| 1003 |
"""Road distance, duration and the drawn line, for the order given AND for a better one.
|
|
@@ -1015,6 +1062,8 @@ def geo_route(body: dict = Body(...), session: Session = Depends(require_session
|
|
| 1015 |
|
| 1016 |
Request body:
|
| 1017 |
`stops` [{lat, lon, pid?}] the order to measure, at least two.
|
|
|
|
|
|
|
| 1018 |
`roundTrip` bool. The server closes the measured route at its selected start. A legacy
|
| 1019 |
closing duplicate is accepted too and never becomes a visitable stop.
|
| 1020 |
`startPid` optional selected start among the submitted `pid` values.
|
|
@@ -1067,6 +1116,7 @@ def geo_route(body: dict = Body(...), session: Session = Depends(require_session
|
|
| 1067 |
round_trip = bool(body.get("roundTrip")) or has_closing_stop
|
| 1068 |
core = points[:-1] if has_closing_stop else points
|
| 1069 |
core_pids = pids[:-1] if has_closing_stop else pids
|
|
|
|
| 1070 |
start_index = _route_endpoint_index(core_pids, body.get("startPid"), "start")
|
| 1071 |
end_index = _route_endpoint_index(core_pids, body.get("endPid"), "end")
|
| 1072 |
if round_trip and end_index is not None:
|
|
@@ -1139,6 +1189,8 @@ def geo_route(body: dict = Body(...), session: Session = Depends(require_session
|
|
| 1139 |
"attribution": credit,
|
| 1140 |
# ---- C2, for the order as submitted
|
| 1141 |
"ordered": core_pids,
|
|
|
|
|
|
|
| 1142 |
"meters": current["meters"],
|
| 1143 |
"seconds": current["seconds"],
|
| 1144 |
"source": current["source"],
|
|
|
|
| 998 |
return matches[0]
|
| 999 |
|
| 1000 |
|
| 1001 |
+
def _route_location_partition(body, located_pids):
|
| 1002 |
+
"""Validate C1's explicit client partition without trusting labels as coordinates.
|
| 1003 |
+
|
| 1004 |
+
Older map callers submit only ``stops`` and keep their established response shape. RouteNav
|
| 1005 |
+
submits both keys together; once either is present the pair is atomic and every pid must occur
|
| 1006 |
+
exactly once across the two halves.
|
| 1007 |
+
"""
|
| 1008 |
+
has_located = "locatedPids" in body
|
| 1009 |
+
has_unlocated = "unlocated" in body
|
| 1010 |
+
if not has_located and not has_unlocated:
|
| 1011 |
+
return list(located_pids), []
|
| 1012 |
+
if not has_located or not has_unlocated:
|
| 1013 |
+
raise err(400, "invalid_route_partition",
|
| 1014 |
+
"Route locations must identify both located and unlocated stops.")
|
| 1015 |
+
raw_located, raw_unlocated = body.get("locatedPids"), body.get("unlocated")
|
| 1016 |
+
if not isinstance(raw_located, list) or not isinstance(raw_unlocated, list):
|
| 1017 |
+
raise err(400, "invalid_route_partition",
|
| 1018 |
+
"Route locations must be lists of located and unlocated stops.")
|
| 1019 |
+
try:
|
| 1020 |
+
claimed_located = [int(pid) for pid in raw_located]
|
| 1021 |
+
except (TypeError, ValueError):
|
| 1022 |
+
raise err(400, "invalid_route_partition", "Every located Route stop needs a record id.")
|
| 1023 |
+
if claimed_located != list(located_pids) or len(set(claimed_located)) != len(claimed_located):
|
| 1024 |
+
raise err(400, "invalid_route_partition",
|
| 1025 |
+
"The located Route stop ids must match the submitted coordinates in order.")
|
| 1026 |
+
cleaned, unlocated_ids = [], set()
|
| 1027 |
+
for item in raw_unlocated:
|
| 1028 |
+
if not isinstance(item, dict):
|
| 1029 |
+
raise err(400, "invalid_route_partition",
|
| 1030 |
+
"Every unlocated Route stop needs a record id, label, and reason.")
|
| 1031 |
+
try:
|
| 1032 |
+
pid = int(item.get("pid"))
|
| 1033 |
+
except (TypeError, ValueError):
|
| 1034 |
+
raise err(400, "invalid_route_partition", "Every unlocated Route stop needs a record id.")
|
| 1035 |
+
label = " ".join(str(item.get("label") or "").split())[:160]
|
| 1036 |
+
reason = " ".join(str(item.get("reason") or "").split())[:240]
|
| 1037 |
+
if pid in unlocated_ids or pid in claimed_located or not label or not reason:
|
| 1038 |
+
raise err(400, "invalid_route_partition",
|
| 1039 |
+
"Every Route stop must occur once with a non-empty label and location reason.")
|
| 1040 |
+
unlocated_ids.add(pid)
|
| 1041 |
+
cleaned.append({"pid": pid, "label": label, "reason": reason})
|
| 1042 |
+
if len(cleaned) > MAX_ROUTE_STOPS:
|
| 1043 |
+
raise err(400, "too_many_stops",
|
| 1044 |
+
f"Identify at most {MAX_ROUTE_STOPS} unlocated Route stops at once.")
|
| 1045 |
+
return claimed_located, cleaned
|
| 1046 |
+
|
| 1047 |
+
|
| 1048 |
@router.post("/geo/route")
|
| 1049 |
def geo_route(body: dict = Body(...), session: Session = Depends(require_session)):
|
| 1050 |
"""Road distance, duration and the drawn line, for the order given AND for a better one.
|
|
|
|
| 1062 |
|
| 1063 |
Request body:
|
| 1064 |
`stops` [{lat, lon, pid?}] the order to measure, at least two.
|
| 1065 |
+
`locatedPids` [pid] matching ``stops`` in order when RouteNav submits C1's partition.
|
| 1066 |
+
`unlocated` [{pid,label,reason}] named fixed anchors excluded from both measurements.
|
| 1067 |
`roundTrip` bool. The server closes the measured route at its selected start. A legacy
|
| 1068 |
closing duplicate is accepted too and never becomes a visitable stop.
|
| 1069 |
`startPid` optional selected start among the submitted `pid` values.
|
|
|
|
| 1116 |
round_trip = bool(body.get("roundTrip")) or has_closing_stop
|
| 1117 |
core = points[:-1] if has_closing_stop else points
|
| 1118 |
core_pids = pids[:-1] if has_closing_stop else pids
|
| 1119 |
+
_claimed_located, unlocated = _route_location_partition(body, core_pids)
|
| 1120 |
start_index = _route_endpoint_index(core_pids, body.get("startPid"), "start")
|
| 1121 |
end_index = _route_endpoint_index(core_pids, body.get("endPid"), "end")
|
| 1122 |
if round_trip and end_index is not None:
|
|
|
|
| 1189 |
"attribution": credit,
|
| 1190 |
# ---- C2, for the order as submitted
|
| 1191 |
"ordered": core_pids,
|
| 1192 |
+
"locatedPids": core_pids,
|
| 1193 |
+
"unlocated": unlocated,
|
| 1194 |
"meters": current["meters"],
|
| 1195 |
"seconds": current["seconds"],
|
| 1196 |
"source": current["source"],
|
api/routes_module_index.py
CHANGED
|
@@ -65,10 +65,28 @@ def _readable_ids(module, session):
|
|
| 65 |
voice._store(session.runtime)["voice_agent"].values() if isinstance(row, dict)})
|
| 66 |
|
| 67 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
def _project(bucket, ids):
|
| 69 |
folders, placed, _ = validate_folder_tree(bucket.get("folders"), bucket.get("itemFolders"))
|
| 70 |
-
|
| 71 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
|
| 73 |
|
| 74 |
def _available(session):
|
|
@@ -111,6 +129,7 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
|
|
| 111 |
raise SheetFolderRefusal("module_index_item_forbidden",
|
| 112 |
"A folder placement names an item you cannot read in this module.")
|
| 113 |
folders, placed, _ = validate_folder_tree(raw["folders"], placements)
|
|
|
|
| 114 |
saved = {}
|
| 115 |
|
| 116 |
def apply(current):
|
|
@@ -122,6 +141,10 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
|
|
| 122 |
raise SheetFolderRefusal("index_revision_stale",
|
| 123 |
"Folders changed somewhere else. Reload folders and try again.")
|
| 124 |
saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
users[owner] = dict(saved)
|
| 126 |
state["schema"] = 1
|
| 127 |
return state
|
|
|
|
| 65 |
voice._store(session.runtime)["voice_agent"].values() if isinstance(row, dict)})
|
| 66 |
|
| 67 |
|
| 68 |
+
def _collapsed_groups(raw):
|
| 69 |
+
"""The module-index document owns per-user App chrome disclosure preferences.
|
| 70 |
+
|
| 71 |
+
Finance is the only registry group mounted in chrome today. Refuse unknown identifiers rather
|
| 72 |
+
than turning this preference bucket into unbounded client storage.
|
| 73 |
+
"""
|
| 74 |
+
if not isinstance(raw, list) or any(not isinstance(value, str) for value in raw):
|
| 75 |
+
raise SheetFolderRefusal("module_index_invalid", "collapsedGroups must be a list of supported group names.")
|
| 76 |
+
if any(value != "finance" for value in raw):
|
| 77 |
+
raise SheetFolderRefusal("module_index_invalid", "collapsedGroups names an unsupported App group.")
|
| 78 |
+
return list(dict.fromkeys(raw))
|
| 79 |
+
|
| 80 |
+
|
| 81 |
def _project(bucket, ids):
|
| 82 |
folders, placed, _ = validate_folder_tree(bucket.get("folders"), bucket.get("itemFolders"))
|
| 83 |
+
projected = {"folders": folders, "itemFolders": {k: v for k, v in placed.items() if k in ids},
|
| 84 |
+
"rev": bucket.get("rev", 0)}
|
| 85 |
+
# Additive and presence-preserving: old buckets retain their byte-for-byte response shape,
|
| 86 |
+
# while a user who explicitly opens Finance can persist an empty list as a real preference.
|
| 87 |
+
if "collapsedGroups" in bucket:
|
| 88 |
+
projected["collapsedGroups"] = _collapsed_groups(bucket.get("collapsedGroups"))
|
| 89 |
+
return projected
|
| 90 |
|
| 91 |
|
| 92 |
def _available(session):
|
|
|
|
| 129 |
raise SheetFolderRefusal("module_index_item_forbidden",
|
| 130 |
"A folder placement names an item you cannot read in this module.")
|
| 131 |
folders, placed, _ = validate_folder_tree(raw["folders"], placements)
|
| 132 |
+
collapsed = _collapsed_groups(raw.get("collapsedGroups")) if "collapsedGroups" in raw else None
|
| 133 |
saved = {}
|
| 134 |
|
| 135 |
def apply(current):
|
|
|
|
| 141 |
raise SheetFolderRefusal("index_revision_stale",
|
| 142 |
"Folders changed somewhere else. Reload folders and try again.")
|
| 143 |
saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
|
| 144 |
+
if collapsed is not None:
|
| 145 |
+
saved["collapsedGroups"] = collapsed
|
| 146 |
+
elif "collapsedGroups" in (users.get(owner) or {}):
|
| 147 |
+
saved["collapsedGroups"] = _collapsed_groups(users[owner].get("collapsedGroups"))
|
| 148 |
users[owner] = dict(saved)
|
| 149 |
state["schema"] = 1
|
| 150 |
return state
|
api/routes_odoo_tables.py
CHANGED
|
@@ -22,13 +22,15 @@ import base64
|
|
| 22 |
import hashlib
|
| 23 |
import hmac
|
| 24 |
import json
|
|
|
|
| 25 |
import time
|
| 26 |
from contextlib import contextmanager
|
| 27 |
|
| 28 |
from fastapi import Body, Depends, HTTPException
|
| 29 |
from fastapi import APIRouter, Query
|
| 30 |
|
| 31 |
-
from deps import (Session, err, module_cache_rows, module_cache_source,
|
|
|
|
| 32 |
|
| 33 |
router = APIRouter(prefix="/api/v1")
|
| 34 |
|
|
@@ -120,13 +122,14 @@ def refresh_odoo_tables(session: Session = Depends(require_session)):
|
|
| 120 |
# section and its negative control.
|
| 121 |
if not session.admin:
|
| 122 |
raise err(403, "forbidden", "Refreshing the Odoo databases is an admin action.")
|
| 123 |
-
if not rel.
|
| 124 |
# A plain 400 with the reason: this tenant has no Odoo mirror, and spawning two empty
|
| 125 |
# locked databases it can never fill would be worse than refusing.
|
| 126 |
raise err(400, "not_applicable",
|
| 127 |
-
"
|
| 128 |
try:
|
| 129 |
-
out = rel.refresh(session.runtime, session.tenant, username=session.
|
|
|
|
| 130 |
# ⭐⭐ W30-T31 — AND THEN THE ROWS THAT SHOULD NOT BE HERE LEAVE AGAIN.
|
| 131 |
#
|
| 132 |
# The spawn writes every bucket's rows straight into the tenant document by mutating it
|
|
@@ -209,7 +212,19 @@ def odoo_tables_status(brief: bool = Query(False),
|
|
| 209 |
# while those were the only two; the 2026-08-09 widening added products and orders, and a
|
| 210 |
# hard-coded list here would have reported "everything is fine" over two databases it had
|
| 211 |
# stopped knowing about ([[gate-answers-the-wrong-question]] — the hard-coded count).
|
| 212 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 213 |
table = tables.get(key) or {}
|
| 214 |
rows = (table.get("rows") or {})
|
| 215 |
stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()]
|
|
@@ -220,6 +235,8 @@ def odoo_tables_status(brief: bool = Query(False),
|
|
| 220 |
# says a worklist that quietly stopped updating must be legible without a refresh. So a
|
| 221 |
# read-through table is counted from the MIRROR and says where its count came from.
|
| 222 |
materialised = user_tables.materialises(key, defn=table)
|
|
|
|
|
|
|
| 223 |
if not materialised:
|
| 224 |
try:
|
| 225 |
rows = {}
|
|
@@ -233,11 +250,36 @@ def odoo_tables_status(brief: bool = Query(False),
|
|
| 233 |
order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
|
| 234 |
else:
|
| 235 |
n = -1 # unknown, and never reported as zero
|
|
|
|
|
|
|
|
|
|
| 236 |
except Exception: # noqa: BLE001
|
| 237 |
n = -1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
out[key] = {
|
| 239 |
"exists": bool(table),
|
| 240 |
"rows": (len(rows) if materialised else n),
|
|
|
|
| 241 |
# ⭐ W31-T45: a refused mirror says `refused`, never `mirror` — a size that came from
|
| 242 |
# nowhere must not be labelled with the source it did not come from.
|
| 243 |
"rowsFrom": ("document" if materialised
|
|
@@ -245,8 +287,14 @@ def odoo_tables_status(brief: bool = Query(False),
|
|
| 245 |
"materialised": materialised,
|
| 246 |
# why this grid still keeps its rows in the tenant document, when it does not have to
|
| 247 |
"materialisedBecause": (why_not.get(key, "") if materialised else ""),
|
| 248 |
-
"refreshed":
|
| 249 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 250 |
"locked": table.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE,
|
| 251 |
# W30/R7: whether this database is served THROUGH the mirror rather than from the copy
|
| 252 |
# above. Reported per table, because they convert one at a time and an operator
|
|
@@ -316,8 +364,8 @@ def _seed_state():
|
|
| 316 |
from harness import datastore as _ds
|
| 317 |
block["present"] = bool(_ds.DB_PATH.exists())
|
| 318 |
if not block["present"]:
|
| 319 |
-
|
| 320 |
-
state = dict(_main
|
| 321 |
block["manifest"] = "absent"
|
| 322 |
block["cause"] = state.get("cause") or "this container has no analytical mirror yet"
|
| 323 |
block["recommendation"] = state.get("recommendation") or (
|
|
@@ -456,10 +504,11 @@ def _resync_state(cur):
|
|
| 456 |
block = {"armed": False, "tenant": "", "tenantSource": "", "tenantSeconds": None,
|
| 457 |
"envSeconds": None, "resolvedSeconds": None, "manual": False,
|
| 458 |
"disagreement": None, "passes": 0, "lastPassAt": None, "lastPassOk": None,
|
| 459 |
-
"
|
|
|
|
| 460 |
"entitiesCause": "", "stuck": []}
|
| 461 |
try:
|
| 462 |
-
|
| 463 |
block.update({k: v for k, v in dict(_main.RESYNC_STATE).items() if k in block})
|
| 464 |
cadence = block.get("resolvedSeconds") or _main._RESYNC_DEFAULT_SECONDS
|
| 465 |
except Exception: # noqa: BLE001
|
|
@@ -545,6 +594,7 @@ GRID_SOURCES = {}
|
|
| 545 |
#: ⚠ A provider registers a CALLABLE, not a dict, so its module is not imported until the first
|
| 546 |
#: read — the same lazy rule `_sources` already follows for `odoo_relational`.
|
| 547 |
_SOURCE_PROVIDERS = []
|
|
|
|
| 548 |
|
| 549 |
|
| 550 |
def register_source_provider(fn):
|
|
@@ -558,6 +608,25 @@ def register_source_provider(fn):
|
|
| 558 |
return len(_SOURCE_PROVIDERS)
|
| 559 |
|
| 560 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 561 |
def _sources():
|
| 562 |
"""Build `GRID_SOURCES` lazily by asking EVERY registered provider — Odoo is one of them.
|
| 563 |
|
|
@@ -929,10 +998,24 @@ def _odoo_sources():
|
|
| 929 |
return GRID_SOURCES
|
| 930 |
|
| 931 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 932 |
#: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers
|
| 933 |
#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
|
| 934 |
#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
|
| 935 |
register_source_provider(_odoo_sources)
|
|
|
|
| 936 |
|
| 937 |
|
| 938 |
def sync_read_through(cur=None):
|
|
@@ -942,7 +1025,7 @@ def sync_read_through(cur=None):
|
|
| 942 |
registration is additive and `strip_materialised` is a no-op once a table is empty.
|
| 943 |
|
| 944 |
⛔ IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`, because the eligibility
|
| 945 |
-
question needs BOTH halves —
|
| 946 |
the mirror (for the size) — and `core` may import neither.
|
| 947 |
"""
|
| 948 |
eligible, reasons = _unmaterialisable(cur)
|
|
@@ -1034,7 +1117,7 @@ def _unmaterialisable(cur=None):
|
|
| 1034 |
|
| 1035 |
⛔ SIZE FORCES READ-THROUGH; IT NEVER BLOCKS IT — and getting that backwards was a real bug in
|
| 1036 |
the first cut of this function. A table too big for the tenant document has NO materialised
|
| 1037 |
-
option at all, so making it ineligible would have handed
|
| 1038 |
of None and invited it to build 963,783 python dicts. The window ceiling is a different, much
|
| 1039 |
softer thing: it only limits what the whole-table CLIENT door can serve today.
|
| 1040 |
|
|
@@ -1046,19 +1129,42 @@ def _unmaterialisable(cur=None):
|
|
| 1046 |
⚠ Without a mirror cursor the size questions cannot be asked, so only a grid that was never
|
| 1047 |
part of the materialised spawn is eligible — a table is never freed by a question we skipped.
|
| 1048 |
"""
|
| 1049 |
-
|
| 1050 |
-
folds, eligible, reasons = _fold_reasons(rel), {}, {}
|
| 1051 |
-
# ⛔ "WAS THIS PART OF THE MATERIALISED SPAWN?", and being in `TABLES` STOPPED ANSWERING IT.
|
| 1052 |
-
# W30-T35 declared the two line grains, which have no python row builder anywhere — their
|
| 1053 |
-
# absence from `_READERS` IS that statement — so a table the spawn could never materialise
|
| 1054 |
-
# started reading as spawned, and on a mirror-less deployment fell through to "its size could
|
| 1055 |
-
# not be read" and came back INELIGIBLE. That inverts this function's own first law (size
|
| 1056 |
-
# forces read-through; it never blocks it). Ask the question through the reader map, which is
|
| 1057 |
-
# what actually decides whether a row could ever have been built.
|
| 1058 |
-
spawned = {key for _b, key, _l, _f in rel.TABLES if _b in getattr(rel, "_READERS", {})}
|
| 1059 |
from harness import datastore
|
| 1060 |
ut = _ut()
|
| 1061 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1062 |
total = None
|
| 1063 |
if cur is not None:
|
| 1064 |
try:
|
|
@@ -1068,7 +1174,7 @@ def _unmaterialisable(cur=None):
|
|
| 1068 |
order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
|
| 1069 |
except Exception: # noqa: BLE001
|
| 1070 |
total = None
|
| 1071 |
-
if total is not None and total >
|
| 1072 |
eligible[key] = True # no other home exists; this is not a choice
|
| 1073 |
continue
|
| 1074 |
if total is None and key not in spawned:
|
|
@@ -1079,8 +1185,8 @@ def _unmaterialisable(cur=None):
|
|
| 1079 |
why.append("its size could not be read from the mirror on this deployment")
|
| 1080 |
if key in folds:
|
| 1081 |
why.append(folds[key])
|
| 1082 |
-
if total is not None and total >
|
| 1083 |
-
why.append(f"its {total:,} rows exceed the {
|
| 1084 |
f"the grid client still asks for whole tables, so un-materialising it "
|
| 1085 |
f"today could only truncate")
|
| 1086 |
eligible[key] = not why
|
|
@@ -2197,6 +2303,155 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
|
|
| 2197 |
return tree, sets
|
| 2198 |
|
| 2199 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2200 |
def _may_see_view(session, vid, store):
|
| 2201 |
"""May this caller open that saved view? The same predicate the view rail asks.
|
| 2202 |
|
|
@@ -2437,8 +2692,12 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 2437 |
# the reader is held to, exactly as `viewSets` resolves inside the rows the client holds) and
|
| 2438 |
# before `_leaf_cols` below, so the leaf never reaches the sentence that explains a column with
|
| 2439 |
# no mirror expression. One place learns it; `/rows` and `/aggregates` share this compiler.
|
|
|
|
| 2440 |
tree, cohort_sets = _views_as_cohorts(tree, table_key, session, spec, cols,
|
| 2441 |
base_where, wall_params, cur, limits)
|
|
|
|
|
|
|
|
|
|
| 2442 |
|
| 2443 |
# ── the predicate, pushed down ────────────────────────────────────────────────────────────
|
| 2444 |
# ⛔ `_PSEUDO_COLS` is subtracted BEFORE the missing-column test. `__cohort__` and `__view__`
|
|
@@ -2563,8 +2822,12 @@ def _materialised_read(table_key, session, query, related):
|
|
| 2563 |
"cols": bindings, "where": None}
|
| 2564 |
cols = _filter_columns(spec, fields)
|
| 2565 |
limits = list(payload.get("limits") or [])
|
|
|
|
| 2566 |
tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
|
| 2567 |
None, [], cur, limits)
|
|
|
|
|
|
|
|
|
|
| 2568 |
missing = _leaf_cols(tree) - _PSEUDO_COLS - set(cols)
|
| 2569 |
if missing:
|
| 2570 |
raise err(400, "filter_unsupported", "A filter names a column unavailable to this account.")
|
|
@@ -2585,7 +2848,42 @@ def _materialised_read(table_key, session, query, related):
|
|
| 2585 |
"frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload}
|
| 2586 |
|
| 2587 |
|
| 2588 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2589 |
from harness import datastore
|
| 2590 |
|
| 2591 |
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
|
|
@@ -2593,8 +2891,13 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
|
| 2593 |
with _materialised_read(table_key, session, query, related) as read:
|
| 2594 |
spec = read["spec"]
|
| 2595 |
order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
|
| 2596 |
-
|
| 2597 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2598 |
offset=offset, limit=limit or None, cur=read["cur"], **read["frm"])
|
| 2599 |
payload = read["payload"]
|
| 2600 |
by_id = {int(row["pid"]): row for row in payload["rows"]}
|
|
@@ -2617,6 +2920,8 @@ def odoo_table_rows(table_key: str,
|
|
| 2617 |
sorts: str = Query(default=None),
|
| 2618 |
search: str = Query(default=None),
|
| 2619 |
related: str = Query(default=None),
|
|
|
|
|
|
|
| 2620 |
session: Session = Depends(require_session)):
|
| 2621 |
"""ONE WINDOW over a connected grid, read straight from the mirror (contract C2).
|
| 2622 |
|
|
@@ -2647,7 +2952,7 @@ def odoo_table_rows(table_key: str,
|
|
| 2647 |
|
| 2648 |
if _materialised_table(table_key):
|
| 2649 |
return _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
| 2650 |
-
sorts, search, related)
|
| 2651 |
|
| 2652 |
# ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
|
| 2653 |
# hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
|
|
@@ -2661,6 +2966,10 @@ def odoo_table_rows(table_key: str,
|
|
| 2661 |
cols, limits, frm = read["cols"], read["limits"], read["frm"]
|
| 2662 |
base_where, wall_params = read["base_where"], read["wall_params"]
|
| 2663 |
where, params, sort_spec = read["where"], read["params"], read["sort_spec"]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2664 |
|
| 2665 |
# ── the order, made TOTAL ─────────────────────────────────────────────────────────────────
|
| 2666 |
# ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one
|
|
@@ -2699,7 +3008,7 @@ def odoo_table_rows(table_key: str,
|
|
| 2699 |
# ⭐ D-155 — the freshness stamp rides the SAME builder, so both read-through doors answer the
|
| 2700 |
# `refreshed` column identically instead of one of them leaving it blank.
|
| 2701 |
rows_src = _pool_from_window(spec, win,
|
| 2702 |
-
stamp=mirror_stamp(spec, cur, table_key,
|
| 2703 |
session.runtime))
|
| 2704 |
overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
|
| 2705 |
|
|
@@ -3063,7 +3372,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 3063 |
return body_out
|
| 3064 |
|
| 3065 |
|
| 3066 |
-
def mirror_stamp(spec, cur=None, table_key="", rt=None):
|
| 3067 |
"""⭐ D-155 — WHEN THIS READ-THROUGH GRID'S DATA WAS LAST RECONCILED AGAINST ODOO, as a date.
|
| 3068 |
|
| 3069 |
⛔ THE COLUMN EXISTED AND THE BINDING DID NOT, which is the whole of D-155. Every Odoo table
|
|
@@ -3108,9 +3417,12 @@ def mirror_stamp(spec, cur=None, table_key="", rt=None):
|
|
| 3108 |
# the materialised carve-out above, made structural. ⚠ Fail-QUIET to `""`: when we cannot
|
| 3109 |
# tell whether this grid stores rows, the safe answer is the behaviour that shipped (blank),
|
| 3110 |
# never a stamp that might overwrite a per-row one.
|
|
|
|
|
|
|
|
|
|
| 3111 |
if table_key:
|
| 3112 |
try:
|
| 3113 |
-
if _ut().materialises(str(table_key), st=rt):
|
| 3114 |
return ""
|
| 3115 |
except Exception: # noqa: BLE001
|
| 3116 |
return ""
|
|
@@ -3227,7 +3539,11 @@ def whole_pool(table_key, cur=None, rt=None):
|
|
| 3227 |
f"{table_key}: {win['total']:,} rows is more than one {datastore.WINDOW_MAX:,}-row "
|
| 3228 |
f"window, and this database no longer stores rows in the tenant document; it can "
|
| 3229 |
f"only be read a page at a time (`/odoo-tables/{table_key}/rows`)")
|
| 3230 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3231 |
|
| 3232 |
|
| 3233 |
def _fe():
|
|
|
|
| 22 |
import hashlib
|
| 23 |
import hmac
|
| 24 |
import json
|
| 25 |
+
import math
|
| 26 |
import time
|
| 27 |
from contextlib import contextmanager
|
| 28 |
|
| 29 |
from fastapi import Body, Depends, HTTPException
|
| 30 |
from fastapi import APIRouter, Query
|
| 31 |
|
| 32 |
+
from deps import (Session, err, module_cache_rows, module_cache_source,
|
| 33 |
+
require_session, running_main)
|
| 34 |
|
| 35 |
router = APIRouter(prefix="/api/v1")
|
| 36 |
|
|
|
|
| 122 |
# section and its negative control.
|
| 123 |
if not session.admin:
|
| 124 |
raise err(403, "forbidden", "Refreshing the Odoo databases is an admin action.")
|
| 125 |
+
if not rel.available(session.runtime) and not rel.frozen(session.runtime):
|
| 126 |
# A plain 400 with the reason: this tenant has no Odoo mirror, and spawning two empty
|
| 127 |
# locked databases it can never fill would be worse than refusing.
|
| 128 |
raise err(400, "not_applicable",
|
| 129 |
+
"Connect an Odoo source before refreshing its databases.")
|
| 130 |
try:
|
| 131 |
+
out = rel.refresh(session.runtime, session.tenant, username=session.uname,
|
| 132 |
+
all_connections=True)
|
| 133 |
# ⭐⭐ W30-T31 — AND THEN THE ROWS THAT SHOULD NOT BE HERE LEAVE AGAIN.
|
| 134 |
#
|
| 135 |
# The spawn writes every bucket's rows straight into the tenant document by mutating it
|
|
|
|
| 212 |
# while those were the only two; the 2026-08-09 widening added products and orders, and a
|
| 213 |
# hard-coded list here would have reported "everything is fine" over two databases it had
|
| 214 |
# stopped knowing about ([[gate-answers-the-wrong-question]] — the hard-coded count).
|
| 215 |
+
declared = list(rel.TABLES)
|
| 216 |
+
declared_keys = {key for _bucket, key, _label, _fields in declared}
|
| 217 |
+
# Include every namespaced/discovered Odoo database already present in this tenant. The old
|
| 218 |
+
# compiled-only loop made connection two invisible and therefore falsely healthy by omission.
|
| 219 |
+
for key, table in tables.items():
|
| 220 |
+
connector = table.get("connector") if isinstance(table, dict) else None
|
| 221 |
+
if (key not in declared_keys and isinstance(connector, dict)
|
| 222 |
+
and connector.get("key") == "odoo"):
|
| 223 |
+
declared.append(("stored", key, str(table.get("label") or key), lambda: ()))
|
| 224 |
+
declared_keys.add(key)
|
| 225 |
+
|
| 226 |
+
health_cache = {}
|
| 227 |
+
for _bucket, key, _label, _fields in declared:
|
| 228 |
table = tables.get(key) or {}
|
| 229 |
rows = (table.get("rows") or {})
|
| 230 |
stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()]
|
|
|
|
| 235 |
# says a worklist that quietly stopped updating must be legible without a refresh. So a
|
| 236 |
# read-through table is counted from the MIRROR and says where its count came from.
|
| 237 |
materialised = user_tables.materialises(key, defn=table)
|
| 238 |
+
spec = None
|
| 239 |
+
mirror_refreshed = ""
|
| 240 |
if not materialised:
|
| 241 |
try:
|
| 242 |
rows = {}
|
|
|
|
| 250 |
order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
|
| 251 |
else:
|
| 252 |
n = -1 # unknown, and never reported as zero
|
| 253 |
+
mirror_refreshed = mirror_stamp(
|
| 254 |
+
spec or {}, cur=cur, table_key=key, rt=session.runtime,
|
| 255 |
+
defn=(table if isinstance(table, dict) and table else None))
|
| 256 |
except Exception: # noqa: BLE001
|
| 257 |
n = -1
|
| 258 |
+
connection_id = str(((table.get("connector") or {}).get("connectionId")
|
| 259 |
+
if isinstance(table, dict) else "") or "")
|
| 260 |
+
if not connection_id:
|
| 261 |
+
try:
|
| 262 |
+
connection_id = str(rel.connection_for_table_key(key, session.runtime) or "default")
|
| 263 |
+
except Exception: # noqa: BLE001
|
| 264 |
+
connection_id = "default"
|
| 265 |
+
if connection_id not in health_cache:
|
| 266 |
+
try:
|
| 267 |
+
import routes_connectors as _connectors
|
| 268 |
+
receipt = _connectors.odoo_health_receipt(session.runtime, connection_id)
|
| 269 |
+
health_cache[connection_id] = {
|
| 270 |
+
str(item.get("databaseKey") or ""): item
|
| 271 |
+
for item in (receipt.get("databases") or ())}
|
| 272 |
+
except Exception: # noqa: BLE001
|
| 273 |
+
health_cache[connection_id] = {}
|
| 274 |
+
db_health = health_cache[connection_id].get(key) or {
|
| 275 |
+
"databaseKey": key, "state": "unknown", "lastSuccess": None,
|
| 276 |
+
"currentLagSeconds": None, "failure": None, "stages": {}}
|
| 277 |
+
refreshed = (max([s for s in stamps if s], default="") if materialised
|
| 278 |
+
else mirror_refreshed)
|
| 279 |
out[key] = {
|
| 280 |
"exists": bool(table),
|
| 281 |
"rows": (len(rows) if materialised else n),
|
| 282 |
+
"rowCount": (len(rows) if materialised else n),
|
| 283 |
# ⭐ W31-T45: a refused mirror says `refused`, never `mirror` — a size that came from
|
| 284 |
# nowhere must not be labelled with the source it did not come from.
|
| 285 |
"rowsFrom": ("document" if materialised
|
|
|
|
| 287 |
"materialised": materialised,
|
| 288 |
# why this grid still keeps its rows in the tenant document, when it does not have to
|
| 289 |
"materialisedBecause": (why_not.get(key, "") if materialised else ""),
|
| 290 |
+
"refreshed": refreshed,
|
| 291 |
+
# The row-level receipt carries an exact instant; mirror metadata may be date-only.
|
| 292 |
+
"recordRefreshed": db_health.get("recordRefreshed") or refreshed,
|
| 293 |
+
"lastSuccess": db_health.get("lastSuccess"),
|
| 294 |
+
"currentLagSeconds": db_health.get("currentLagSeconds"),
|
| 295 |
+
"failure": db_health.get("failure"),
|
| 296 |
+
"stages": db_health.get("stages") or {},
|
| 297 |
+
"connectionId": connection_id,
|
| 298 |
"locked": table.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE,
|
| 299 |
# W30/R7: whether this database is served THROUGH the mirror rather than from the copy
|
| 300 |
# above. Reported per table, because they convert one at a time and an operator
|
|
|
|
| 364 |
from harness import datastore as _ds
|
| 365 |
block["present"] = bool(_ds.DB_PATH.exists())
|
| 366 |
if not block["present"]:
|
| 367 |
+
_main = running_main() # D-781: resolve, never import
|
| 368 |
+
state = dict(getattr(_main, "MIRROR_SEED", None) or {})
|
| 369 |
block["manifest"] = "absent"
|
| 370 |
block["cause"] = state.get("cause") or "this container has no analytical mirror yet"
|
| 371 |
block["recommendation"] = state.get("recommendation") or (
|
|
|
|
| 504 |
block = {"armed": False, "tenant": "", "tenantSource": "", "tenantSeconds": None,
|
| 505 |
"envSeconds": None, "resolvedSeconds": None, "manual": False,
|
| 506 |
"disagreement": None, "passes": 0, "lastPassAt": None, "lastPassOk": None,
|
| 507 |
+
"lastSuccessAt": None, "lastError": None, "bootSyncError": None,
|
| 508 |
+
"connectionId": "", "entities": None,
|
| 509 |
"entitiesCause": "", "stuck": []}
|
| 510 |
try:
|
| 511 |
+
_main = running_main() # D-781: resolve, never import
|
| 512 |
block.update({k: v for k, v in dict(_main.RESYNC_STATE).items() if k in block})
|
| 513 |
cadence = block.get("resolvedSeconds") or _main._RESYNC_DEFAULT_SECONDS
|
| 514 |
except Exception: # noqa: BLE001
|
|
|
|
| 594 |
#: ⚠ A provider registers a CALLABLE, not a dict, so its module is not imported until the first
|
| 595 |
#: read — the same lazy rule `_sources` already follows for `odoo_relational`.
|
| 596 |
_SOURCE_PROVIDERS = []
|
| 597 |
+
_SOURCE_ANALYSES = {}
|
| 598 |
|
| 599 |
|
| 600 |
def register_source_provider(fn):
|
|
|
|
| 608 |
return len(_SOURCE_PROVIDERS)
|
| 609 |
|
| 610 |
|
| 611 |
+
def register_source_analysis(source_key, *, folds, spawned):
|
| 612 |
+
"""Register lazy `() -> {table_key: reason}` and `() -> set[table_key]` analyses.
|
| 613 |
+
|
| 614 |
+
Source ids come from core.registry's policies. Re-registering replaces that source's pair
|
| 615 |
+
without adding another entry. Returns the number of sources whose analysis is now known.
|
| 616 |
+
"""
|
| 617 |
+
if callable(folds) and callable(spawned):
|
| 618 |
+
_SOURCE_ANALYSES[source_key] = {"folds": folds, "spawned": spawned}
|
| 619 |
+
return len(_SOURCE_ANALYSES)
|
| 620 |
+
|
| 621 |
+
|
| 622 |
+
def _source_analysis(source_key):
|
| 623 |
+
"""Resolve one source's analysis lazily; an unregistered source has two empty results."""
|
| 624 |
+
analysis = _SOURCE_ANALYSES.get(source_key)
|
| 625 |
+
if analysis is None:
|
| 626 |
+
return {}, set()
|
| 627 |
+
return analysis["folds"](), analysis["spawned"]()
|
| 628 |
+
|
| 629 |
+
|
| 630 |
def _sources():
|
| 631 |
"""Build `GRID_SOURCES` lazily by asking EVERY registered provider — Odoo is one of them.
|
| 632 |
|
|
|
|
| 998 |
return GRID_SOURCES
|
| 999 |
|
| 1000 |
|
| 1001 |
+
def _odoo_spawned():
|
| 1002 |
+
"""The Odoo grains that have a materialised row builder."""
|
| 1003 |
+
rel = _rel()
|
| 1004 |
+
# ⛔ "WAS THIS PART OF THE MATERIALISED SPAWN?", and being in `TABLES` STOPPED ANSWERING IT.
|
| 1005 |
+
# W30-T35 declared the two line grains, which have no python row builder anywhere — their
|
| 1006 |
+
# absence from `_READERS` IS that statement — so a table the spawn could never materialise
|
| 1007 |
+
# started reading as spawned, and on a mirror-less deployment fell through to "its size could
|
| 1008 |
+
# not be read" and came back INELIGIBLE. That inverts eligibility's first law (size
|
| 1009 |
+
# forces read-through; it never blocks it). Ask the question through the reader map, which is
|
| 1010 |
+
# what actually decides whether a row could ever have been built.
|
| 1011 |
+
return {key for _b, key, _l, _f in rel.TABLES if _b in getattr(rel, "_READERS", {})}
|
| 1012 |
+
|
| 1013 |
+
|
| 1014 |
#: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers
|
| 1015 |
#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
|
| 1016 |
#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
|
| 1017 |
register_source_provider(_odoo_sources)
|
| 1018 |
+
register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned)
|
| 1019 |
|
| 1020 |
|
| 1021 |
def sync_read_through(cur=None):
|
|
|
|
| 1025 |
registration is additive and `strip_materialised` is a no-op once a table is empty.
|
| 1026 |
|
| 1027 |
⛔ IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`, because the eligibility
|
| 1028 |
+
question needs BOTH halves — the source's registered analysis (for the fold matrix) and
|
| 1029 |
the mirror (for the size) — and `core` may import neither.
|
| 1030 |
"""
|
| 1031 |
eligible, reasons = _unmaterialisable(cur)
|
|
|
|
| 1117 |
|
| 1118 |
⛔ SIZE FORCES READ-THROUGH; IT NEVER BLOCKS IT — and getting that backwards was a real bug in
|
| 1119 |
the first cut of this function. A table too big for the tenant document has NO materialised
|
| 1120 |
+
option at all, so making it ineligible would have handed the materialisation plan a `row_limit`
|
| 1121 |
of None and invited it to build 963,783 python dicts. The window ceiling is a different, much
|
| 1122 |
softer thing: it only limits what the whole-table CLIENT door can serve today.
|
| 1123 |
|
|
|
|
| 1129 |
⚠ Without a mirror cursor the size questions cannot be asked, so only a grid that was never
|
| 1130 |
part of the materialised spawn is eligible — a table is never freed by a question we skipped.
|
| 1131 |
"""
|
| 1132 |
+
from core import registry
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1133 |
from harness import datastore
|
| 1134 |
ut = _ut()
|
| 1135 |
+
analyses, eligible, reasons = {}, {}, {}
|
| 1136 |
+
candidates = dict(_sources())
|
| 1137 |
+
if any(policy["candidates"] == "prefix" for policy in registry.source_policies()):
|
| 1138 |
+
# ⚠ Every declared policy is "declared" today, so this walk does not run. It is here so a
|
| 1139 |
+
# source CAN opt in without editing this branch, which is the ticket. The store read is
|
| 1140 |
+
# guarded because a policy is a declaration and a missing store is not its problem.
|
| 1141 |
+
try:
|
| 1142 |
+
owned = list(ut.all_defs() or {})
|
| 1143 |
+
except Exception: # noqa: BLE001
|
| 1144 |
+
owned = []
|
| 1145 |
+
for key in owned:
|
| 1146 |
+
if registry.source_policy(key)["candidates"] == "prefix":
|
| 1147 |
+
candidates.setdefault(key, {})
|
| 1148 |
+
for key, spec in candidates.items():
|
| 1149 |
+
policy = registry.source_policy(key)
|
| 1150 |
+
max_rows = policy["max_rows"] if policy["max_rows"] is not None else ut.MAX_ROWS
|
| 1151 |
+
window_max = (policy["window_max"] if policy["window_max"] is not None
|
| 1152 |
+
else datastore.WINDOW_MAX)
|
| 1153 |
+
source_key = policy["key"]
|
| 1154 |
+
if source_key not in analyses:
|
| 1155 |
+
analyses[source_key] = _source_analysis(source_key)
|
| 1156 |
+
folds, spawned = analyses[source_key]
|
| 1157 |
+
# ⛔ A CANDIDATE WITH NO SPEC CANNOT BE JUDGED, AND MUST THEREFORE BE REFUSED, NOT FREED.
|
| 1158 |
+
# A `candidates: "prefix"` source contributes keys its provider never described, so the
|
| 1159 |
+
# size question below has nothing to ask. Falling through would reach the `total is None
|
| 1160 |
+
# and key not in spawned` arm and hand back ELIGIBLE, un-materialising a database whose
|
| 1161 |
+
# size was never read - the exact inversion of this function's first law, arrived at by
|
| 1162 |
+
# an absence rather than a decision. [S65-T13]
|
| 1163 |
+
if not (spec.get("table") or spec.get("from_sql")):
|
| 1164 |
+
eligible[key] = False
|
| 1165 |
+
reasons[key] = ("its source declares no read-through binding, so its size cannot be "
|
| 1166 |
+
"read and un-materialising it would be a guess")
|
| 1167 |
+
continue
|
| 1168 |
total = None
|
| 1169 |
if cur is not None:
|
| 1170 |
try:
|
|
|
|
| 1174 |
order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
|
| 1175 |
except Exception: # noqa: BLE001
|
| 1176 |
total = None
|
| 1177 |
+
if total is not None and total > max_rows:
|
| 1178 |
eligible[key] = True # no other home exists; this is not a choice
|
| 1179 |
continue
|
| 1180 |
if total is None and key not in spawned:
|
|
|
|
| 1185 |
why.append("its size could not be read from the mirror on this deployment")
|
| 1186 |
if key in folds:
|
| 1187 |
why.append(folds[key])
|
| 1188 |
+
if total is not None and total > window_max:
|
| 1189 |
+
why.append(f"its {total:,} rows exceed the {window_max:,}-row window and "
|
| 1190 |
f"the grid client still asks for whole tables, so un-materialising it "
|
| 1191 |
f"today could only truncate")
|
| 1192 |
eligible[key] = not why
|
|
|
|
| 2303 |
return tree, sets
|
| 2304 |
|
| 2305 |
|
| 2306 |
+
def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, cohort_sets,
|
| 2307 |
+
had_cohort, today, limits=None):
|
| 2308 |
+
"""Resolve every ACTIVE rank leaf (Top N, quartile, above average ...) into a synthetic
|
| 2309 |
+
`__cohort__` leaf over the ids it selects. `(tree, cohort_sets)`.
|
| 2310 |
+
|
| 2311 |
+
A rank is a property of the SET, so `filter_sql` refuses it at row grain. The set is answered
|
| 2312 |
+
here the way the client answers it: the residue (every non-rank condition) under the wall is
|
| 2313 |
+
the domain, and `filter_eval.resolve_rank_sets` (the Python port of the client's set pass,
|
| 2314 |
+
ties by ascending pid) picks the members. Only the id and the ranked columns are read, uncapped.
|
| 2315 |
+
"""
|
| 2316 |
+
fe = _fe()
|
| 2317 |
+
leaves = fe._rank_leaves(tree or [], cols)
|
| 2318 |
+
if not leaves:
|
| 2319 |
+
return tree, cohort_sets
|
| 2320 |
+
if had_cohort:
|
| 2321 |
+
raise err(400, "filter_unsupported",
|
| 2322 |
+
"a condition on cohort membership cannot be evaluated on this database's live "
|
| 2323 |
+
"connection, because a cohort is resolved from this tenant's own records rather "
|
| 2324 |
+
"than from the connection. Filter on a saved view instead, or open this "
|
| 2325 |
+
"database's stored copy")
|
| 2326 |
+
residue = fe._strip_rank_leaves(tree)
|
| 2327 |
+
try:
|
| 2328 |
+
pred = _fs().compile_filter_tree(residue, conj=conj, columns=cols, id_sql=spec["id"],
|
| 2329 |
+
today=today, cohort_sets=cohort_sets)
|
| 2330 |
+
except ValueError as exc:
|
| 2331 |
+
raise err(400, "filter_unsupported", str(exc))
|
| 2332 |
+
if pred is not None and pred.uses_aggregate:
|
| 2333 |
+
raise err(400, "filter_unsupported",
|
| 2334 |
+
"This condition cannot be evaluated for individual rows.")
|
| 2335 |
+
where = base_where
|
| 2336 |
+
params = list(wall_params)
|
| 2337 |
+
if pred is not None:
|
| 2338 |
+
where = f"({where}) AND {pred.sql}" if where else pred.sql
|
| 2339 |
+
params.extend(pred.params)
|
| 2340 |
+
frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
|
| 2341 |
+
answers = {}
|
| 2342 |
+
python_leaves = []
|
| 2343 |
+
for n in leaves:
|
| 2344 |
+
got = _rank_ids_sql(n, cols, spec["id"], frm_body, where, params, cur)
|
| 2345 |
+
if got is None:
|
| 2346 |
+
python_leaves.append(n)
|
| 2347 |
+
else:
|
| 2348 |
+
answers[id(n)] = got
|
| 2349 |
+
if python_leaves:
|
| 2350 |
+
# A GROUP-scoped rank (rare on this door) keeps the reference resolver: read the id and
|
| 2351 |
+
# the ranked columns once, uncapped, and let `resolve_rank_sets` answer it.
|
| 2352 |
+
ranked = sorted({str(n.get("colId")) for n in python_leaves if n.get("colId") in cols})
|
| 2353 |
+
select = ", ".join([spec["id"]] + [cols[c]["sql"] for c in ranked])
|
| 2354 |
+
sql = f"SELECT {select} FROM {frm_body}" + (f" WHERE {where}" if where else "")
|
| 2355 |
+
rows = []
|
| 2356 |
+
for r in cur.execute(sql, params).fetchall():
|
| 2357 |
+
if not r or r[0] is None:
|
| 2358 |
+
continue
|
| 2359 |
+
row = {"pid": int(r[0])}
|
| 2360 |
+
for i, c in enumerate(ranked, start=1):
|
| 2361 |
+
row[c] = r[i]
|
| 2362 |
+
rows.append(row)
|
| 2363 |
+
resolved = fe.resolve_rank_sets({"nodes": python_leaves, "conj": "and"}, rows, cols)
|
| 2364 |
+
for n in python_leaves:
|
| 2365 |
+
answers[id(n)] = sorted(resolved.sets.get(fe._rank_key(n)) or ())
|
| 2366 |
+
sets = dict(cohort_sets or {})
|
| 2367 |
+
counter = [0]
|
| 2368 |
+
wanted = {id(n) for n in leaves}
|
| 2369 |
+
|
| 2370 |
+
def rewrite(nodes):
|
| 2371 |
+
out = []
|
| 2372 |
+
for n in (nodes or []):
|
| 2373 |
+
if isinstance(n, dict) and isinstance(n.get("children"), list):
|
| 2374 |
+
out.append(dict(n, children=rewrite(n["children"])))
|
| 2375 |
+
elif isinstance(n, dict) and id(n) in wanted:
|
| 2376 |
+
counter[0] += 1
|
| 2377 |
+
key = "__rank_%d__" % counter[0]
|
| 2378 |
+
got = answers.get(id(n))
|
| 2379 |
+
sets[key] = got if got is not None else []
|
| 2380 |
+
if (limits is not None and isinstance(got, list)
|
| 2381 |
+
and len(got) >= _PIN_REPORT_AT):
|
| 2382 |
+
limits.append({
|
| 2383 |
+
"subject": str(n.get("colId") or ""), "effect": "slow",
|
| 2384 |
+
"cause": f"this ranking condition selects {len(got):,} rows, and "
|
| 2385 |
+
f"each page of this database names every one of them in the "
|
| 2386 |
+
f"query that narrows it",
|
| 2387 |
+
"recommendation": "rank a smaller slice (a lower Top N or percentage), "
|
| 2388 |
+
"or add a condition beside it"})
|
| 2389 |
+
out.append({"colId": "__cohort__", "op": "anyOf", "value": key})
|
| 2390 |
+
else:
|
| 2391 |
+
out.append(n)
|
| 2392 |
+
return out
|
| 2393 |
+
|
| 2394 |
+
return rewrite(tree), sets
|
| 2395 |
+
|
| 2396 |
+
|
| 2397 |
+
def _rank_ids_sql(leaf, cols, id_sql, frm_body, where, params, cur):
|
| 2398 |
+
"""One GLOBAL rank leaf as SQL: a `SqlMembership` selecting its ids, or None for the resolver.
|
| 2399 |
+
|
| 2400 |
+
Measured 2026-09-23: `resolve_rank_sets` over 256,542 order lines took 3.6 s in Python, and a
|
| 2401 |
+
paged grid pays it on every page; as a SQL window the same answer is tens of milliseconds,
|
| 2402 |
+
and as a subquery it never becomes a six-figure IN list. Semantics are the resolver's, line
|
| 2403 |
+
for line (parity-tested over all eight ops with ties and blanks): blank or non-finite values
|
| 2404 |
+
are ABSENT, ties break by ascending id in both directions, a bound outside the builder's
|
| 2405 |
+
whole-number range is unanswerable (empty), and a quartile or decile of fewer rows than
|
| 2406 |
+
buckets is empty. The domain's count and mean are read once here and inlined as literals.
|
| 2407 |
+
"""
|
| 2408 |
+
fe = _fe()
|
| 2409 |
+
Sql = _fs().SqlMembership
|
| 2410 |
+
empty = Sql("SELECT NULL WHERE FALSE", [])
|
| 2411 |
+
scope, _field = fe._rank_scope(leaf, None)
|
| 2412 |
+
if scope != "global":
|
| 2413 |
+
return None
|
| 2414 |
+
col = cols.get(leaf.get("colId")) or {}
|
| 2415 |
+
if col.get("type") not in fe.NUMERIC_TYPES:
|
| 2416 |
+
return empty
|
| 2417 |
+
v = f"TRY_CAST({col['sql']} AS DOUBLE)"
|
| 2418 |
+
dom = (f"SELECT {id_sql} AS pid, {v} AS v FROM {frm_body} WHERE "
|
| 2419 |
+
+ (f"({where}) AND " if where else "") + f"{v} IS NOT NULL AND isfinite({v})")
|
| 2420 |
+
params = list(params)
|
| 2421 |
+
op = leaf.get("op")
|
| 2422 |
+
if op in ("topN", "bottomN"):
|
| 2423 |
+
bound = fe._rank_bound(leaf.get("value"), 1, 10000)
|
| 2424 |
+
if bound is None:
|
| 2425 |
+
return empty
|
| 2426 |
+
direction = "DESC" if op == "topN" else "ASC"
|
| 2427 |
+
return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {bound}",
|
| 2428 |
+
params)
|
| 2429 |
+
count, total = cur.execute(f"SELECT count(*), sum(v) FROM ({dom}) d", params).fetchone()
|
| 2430 |
+
count = int(count or 0)
|
| 2431 |
+
if not count:
|
| 2432 |
+
return empty
|
| 2433 |
+
if op in ("inTopPct", "inBottomPct"):
|
| 2434 |
+
pct = fe._rank_bound(leaf.get("value"), 1, 100)
|
| 2435 |
+
if pct is None:
|
| 2436 |
+
return empty
|
| 2437 |
+
take = min(count, max(1, math.ceil(count * pct / 100)))
|
| 2438 |
+
direction = "DESC" if op == "inTopPct" else "ASC"
|
| 2439 |
+
return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {take}",
|
| 2440 |
+
params)
|
| 2441 |
+
if op in ("aboveAvg", "belowAvg"):
|
| 2442 |
+
mean = float(total) / count
|
| 2443 |
+
cmp = ">" if op == "aboveAvg" else "<"
|
| 2444 |
+
return Sql(f"SELECT pid FROM ({dom}) d WHERE v {cmp} {mean!r}", params)
|
| 2445 |
+
if op in ("inQuartile", "inDecile"):
|
| 2446 |
+
buckets = 4 if op == "inQuartile" else 10
|
| 2447 |
+
wanted = fe._rank_bound(leaf.get("value"), 1, buckets)
|
| 2448 |
+
if wanted is None or count < buckets:
|
| 2449 |
+
return empty
|
| 2450 |
+
return Sql(f"SELECT pid FROM (SELECT pid, row_number() OVER (ORDER BY v ASC, pid ASC) - 1"
|
| 2451 |
+
f" AS idx FROM ({dom}) d) r WHERE least({buckets}, (idx * {buckets}) // {count}"
|
| 2452 |
+
f" + 1) = {int(wanted)}", params)
|
| 2453 |
+
return None
|
| 2454 |
+
|
| 2455 |
def _may_see_view(session, vid, store):
|
| 2456 |
"""May this caller open that saved view? The same predicate the view rail asks.
|
| 2457 |
|
|
|
|
| 2692 |
# the reader is held to, exactly as `viewSets` resolves inside the rows the client holds) and
|
| 2693 |
# before `_leaf_cols` below, so the leaf never reaches the sentence that explains a column with
|
| 2694 |
# no mirror expression. One place learns it; `/rows` and `/aggregates` share this compiler.
|
| 2695 |
+
_had_cohort = bool(_active_cohort_leaves(tree))
|
| 2696 |
tree, cohort_sets = _views_as_cohorts(tree, table_key, session, spec, cols,
|
| 2697 |
base_where, wall_params, cur, limits)
|
| 2698 |
+
tree, cohort_sets = _ranks_as_cohorts(
|
| 2699 |
+
tree, (filterConj if filterConj in ("and", "or") else "and"), spec, cols, base_where,
|
| 2700 |
+
wall_params, cur, cohort_sets, _had_cohort, _today(), limits)
|
| 2701 |
|
| 2702 |
# ── the predicate, pushed down ────────────────────────────────────────────────────────────
|
| 2703 |
# ⛔ `_PSEUDO_COLS` is subtracted BEFORE the missing-column test. `__cohort__` and `__view__`
|
|
|
|
| 2822 |
"cols": bindings, "where": None}
|
| 2823 |
cols = _filter_columns(spec, fields)
|
| 2824 |
limits = list(payload.get("limits") or [])
|
| 2825 |
+
_had_cohort = bool(_active_cohort_leaves(query["filters"]))
|
| 2826 |
tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
|
| 2827 |
None, [], cur, limits)
|
| 2828 |
+
tree, cohorts = _ranks_as_cohorts(tree, query["filterConj"], spec, cols, None, [], cur,
|
| 2829 |
+
cohorts, _had_cohort,
|
| 2830 |
+
payload.get("today") or _today(), limits)
|
| 2831 |
missing = _leaf_cols(tree) - _PSEUDO_COLS - set(cols)
|
| 2832 |
if missing:
|
| 2833 |
raise err(400, "filter_unsupported", "A filter names a column unavailable to this account.")
|
|
|
|
| 2848 |
"frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload}
|
| 2849 |
|
| 2850 |
|
| 2851 |
+
def _group_pin(cols, group_by, group_key):
|
| 2852 |
+
"""`(sql, params)` narrowing a window to ONE group of `/aggregates`' grouping, or `(None, [])`.
|
| 2853 |
+
|
| 2854 |
+
HOTFIX 2026-09-23 (owner: "whatever is GROUPED we can add pagination per group"). A group's
|
| 2855 |
+
records page through this door, so the narrowing is the SAME expression `_aggregate_over`
|
| 2856 |
+
groups by (`filter_sql.group_sql`), never a client-built `eq` leaf: `_value_sql` lowercases
|
| 2857 |
+
text and rounds numbers, which would make a group's pages disagree with its count.
|
| 2858 |
+
`group_key` is the JSON of the key `/aggregates` put on the wire (`null` = the blank bucket).
|
| 2859 |
+
"""
|
| 2860 |
+
# A plain-Python caller that omits the argument hands us FastAPI's `Query` sentinel, not
|
| 2861 |
+
# None (the `related` scar in `routes_connected_tables`), so anything but a string is "none".
|
| 2862 |
+
gcol = group_by.strip() if isinstance(group_by, str) else ""
|
| 2863 |
+
if not gcol:
|
| 2864 |
+
return None, []
|
| 2865 |
+
group_key = group_key if isinstance(group_key, str) else None
|
| 2866 |
+
gspec = cols.get(gcol)
|
| 2867 |
+
if gspec is None:
|
| 2868 |
+
raise err(400, "bad_group", "this column cannot be grouped on this database's live "
|
| 2869 |
+
"connection, or it is not visible to your account")
|
| 2870 |
+
try:
|
| 2871 |
+
key = json.loads(group_key) if group_key not in (None, "") else None
|
| 2872 |
+
except ValueError:
|
| 2873 |
+
raise err(400, "bad_group", "groupKey must be the JSON of a group key")
|
| 2874 |
+
gexpr = _fs().group_sql(gspec)
|
| 2875 |
+
if key is None:
|
| 2876 |
+
return f"({gexpr}) IS NULL", []
|
| 2877 |
+
if gspec["type"] in _fs().NUMERIC_TYPES:
|
| 2878 |
+
try:
|
| 2879 |
+
return f"({gexpr}) = ?", [float(key)]
|
| 2880 |
+
except (TypeError, ValueError):
|
| 2881 |
+
raise err(400, "bad_group", "groupKey must be a number for a number column")
|
| 2882 |
+
return f"({gexpr}) = ?", [str(key)]
|
| 2883 |
+
|
| 2884 |
+
|
| 2885 |
+
def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
|
| 2886 |
+
groupBy=None, groupKey=None):
|
| 2887 |
from harness import datastore
|
| 2888 |
|
| 2889 |
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
|
|
|
|
| 2891 |
with _materialised_read(table_key, session, query, related) as read:
|
| 2892 |
spec = read["spec"]
|
| 2893 |
order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
|
| 2894 |
+
where, params = read["where"], list(read["params"])
|
| 2895 |
+
gsql, gparams = _group_pin(read["cols"], groupBy, groupKey)
|
| 2896 |
+
if gsql:
|
| 2897 |
+
where = f"({where}) AND {gsql}" if where else gsql
|
| 2898 |
+
params.extend(gparams)
|
| 2899 |
+
win = datastore.window(select=f'{spec["id"]} AS pid', where=where,
|
| 2900 |
+
params=tuple(params), order_by=order or spec["id"],
|
| 2901 |
offset=offset, limit=limit or None, cur=read["cur"], **read["frm"])
|
| 2902 |
payload = read["payload"]
|
| 2903 |
by_id = {int(row["pid"]): row for row in payload["rows"]}
|
|
|
|
| 2920 |
sorts: str = Query(default=None),
|
| 2921 |
search: str = Query(default=None),
|
| 2922 |
related: str = Query(default=None),
|
| 2923 |
+
groupBy: str = Query(default=None),
|
| 2924 |
+
groupKey: str = Query(default=None),
|
| 2925 |
session: Session = Depends(require_session)):
|
| 2926 |
"""ONE WINDOW over a connected grid, read straight from the mirror (contract C2).
|
| 2927 |
|
|
|
|
| 2952 |
|
| 2953 |
if _materialised_table(table_key):
|
| 2954 |
return _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
| 2955 |
+
sorts, search, related, groupBy, groupKey)
|
| 2956 |
|
| 2957 |
# ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
|
| 2958 |
# hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
|
|
|
|
| 2966 |
cols, limits, frm = read["cols"], read["limits"], read["frm"]
|
| 2967 |
base_where, wall_params = read["base_where"], read["wall_params"]
|
| 2968 |
where, params, sort_spec = read["where"], read["params"], read["sort_spec"]
|
| 2969 |
+
_gsql, _gparams = _group_pin(cols, groupBy, groupKey)
|
| 2970 |
+
if _gsql:
|
| 2971 |
+
where = f"({where}) AND {_gsql}" if where else _gsql
|
| 2972 |
+
params = list(params) + list(_gparams)
|
| 2973 |
|
| 2974 |
# ── the order, made TOTAL ─────────────────────────────────────────────────────────────────
|
| 2975 |
# ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one
|
|
|
|
| 3008 |
# ⭐ D-155 — the freshness stamp rides the SAME builder, so both read-through doors answer the
|
| 3009 |
# `refreshed` column identically instead of one of them leaving it blank.
|
| 3010 |
rows_src = _pool_from_window(spec, win,
|
| 3011 |
+
stamp=mirror_stamp(spec, cur, table_key, defn=defn, rt=
|
| 3012 |
session.runtime))
|
| 3013 |
overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
|
| 3014 |
|
|
|
|
| 3372 |
return body_out
|
| 3373 |
|
| 3374 |
|
| 3375 |
+
def mirror_stamp(spec, cur=None, table_key="", rt=None, defn=None):
|
| 3376 |
"""⭐ D-155 — WHEN THIS READ-THROUGH GRID'S DATA WAS LAST RECONCILED AGAINST ODOO, as a date.
|
| 3377 |
|
| 3378 |
⛔ THE COLUMN EXISTED AND THE BINDING DID NOT, which is the whole of D-155. Every Odoo table
|
|
|
|
| 3417 |
# the materialised carve-out above, made structural. ⚠ Fail-QUIET to `""`: when we cannot
|
| 3418 |
# tell whether this grid stores rows, the safe answer is the behaviour that shipped (blank),
|
| 3419 |
# never a stamp that might overwrite a per-row one.
|
| 3420 |
+
# HOTFIX 2026-09-23: `defn` is LENT by a caller already holding the definition. Without it
|
| 3421 |
+
# `materialises` falls through to a whole-document `user_tables` read, measured on staging
|
| 3422 |
+
# at ~3 s of every 50-row page of order lines (the `timing` phase `pool_and_stamp`).
|
| 3423 |
if table_key:
|
| 3424 |
try:
|
| 3425 |
+
if _ut().materialises(str(table_key), st=rt, defn=defn):
|
| 3426 |
return ""
|
| 3427 |
except Exception: # noqa: BLE001
|
| 3428 |
return ""
|
|
|
|
| 3539 |
f"{table_key}: {win['total']:,} rows is more than one {datastore.WINDOW_MAX:,}-row "
|
| 3540 |
f"window, and this database no longer stores rows in the tenant document; it can "
|
| 3541 |
f"only be read a page at a time (`/odoo-tables/{table_key}/rows`)")
|
| 3542 |
+
# HOTFIX 2026-09-23: this door serves only a table whose rows have LEFT the tenant document
|
| 3543 |
+
# (its docstring), so that fact is lent rather than re-derived: without `defn`,
|
| 3544 |
+
# `materialises` re-reads the whole `user_tables` document, ~3 s on the pg store.
|
| 3545 |
+
return _pool_from_window(spec, win, stamp=mirror_stamp(spec, cur, table_key, rt,
|
| 3546 |
+
defn={"readThrough": True}))
|
| 3547 |
|
| 3548 |
|
| 3549 |
def _fe():
|
api/routes_shares.py
CHANGED
|
@@ -90,10 +90,13 @@ wall that does not exist for this kind. Scoping user tables is booked, not done
|
|
| 90 |
*"Shared with me"*, who may open a source View, and who may re-share. Saved Views carry no
|
| 91 |
view-level permissions member or secondary access model.
|
| 92 |
"""
|
|
|
|
|
|
|
| 93 |
from fastapi import APIRouter, Body, Depends
|
| 94 |
|
| 95 |
import core.shares as shares
|
| 96 |
import core.users as users
|
|
|
|
| 97 |
from deps import Session, err, require_session
|
| 98 |
# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
|
| 99 |
# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
|
|
@@ -103,6 +106,11 @@ from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
|
|
| 103 |
|
| 104 |
router = APIRouter(prefix="/api/v1")
|
| 105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
|
| 107 |
def _kind_or_400(raw):
|
| 108 |
try:
|
|
@@ -249,11 +257,10 @@ def _field_resolve(session, table_key, field_key, strata=_FIELD_STRATA):
|
|
| 249 |
none, because residency IS authorship in a per-account document.
|
| 250 |
4. **the connector catalogue** — the columns the database itself declares.
|
| 251 |
|
| 252 |
-
⚠ `strata` NARROWS THE WALK, AND IT IS A COST CONTROL AS WELL AS A CORRECTNESS ONE.
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
That arm passes `("shared", "mine")`, which is exactly the pair it could see before W50-T23.
|
| 257 |
|
| 258 |
⛔ EACH STRATUM RESOLVES IN ITS OWN `try`, AND THE BARE `except Exception: pass` THAT WRAPPED
|
| 259 |
ALL OF THEM IS GONE. It conflated *"the store threw"* with *"no such field"*, so one
|
|
@@ -765,54 +772,12 @@ def _unremovable(held_rows, entries, uname):
|
|
| 765 |
stuck.add(user)
|
| 766 |
return stuck
|
| 767 |
|
| 768 |
-
# ──
|
| 769 |
-
#
|
| 770 |
-
#
|
| 771 |
-
#
|
| 772 |
-
#
|
| 773 |
-
#
|
| 774 |
-
# LOUDLY in the three places where a grant would otherwise be a lie.
|
| 775 |
-
#
|
| 776 |
-
# ⛔⛔ TRAP 1 — A FIELD GRANT IS TOPIC-WIDE, NOT VIEW-SCOPED. It is keyed
|
| 777 |
-
# `shares.field_oid(topic, key)`, so it hands the receiver that column on EVERY view of that
|
| 778 |
-
# database, for as long as the grant stands. That is the existing grant primitive and this
|
| 779 |
-
# ticket does not narrow it; what it does is refuse to let the response PRETEND otherwise, which
|
| 780 |
-
# is why `sharedFieldScope` rides back beside the list.
|
| 781 |
-
#
|
| 782 |
-
# ⛔⛔ TRAP 2 — THE SHARER CANNOT GRANT A COLUMN THEY DO NOT OWN. `field` is absent from
|
| 783 |
-
# `shares.RESHARE_KINDS` and `_owns_object`'s field branch demands `createdBy == session.uname`,
|
| 784 |
-
# so a view built over a colleague's column simply cannot carry that column with it. Silently
|
| 785 |
-
# skipping it ships an empty column; silently granting it is a permission hole. It is REFUSED
|
| 786 |
-
# BY NAME, inside the 200 response, with the sentence that says who to ask.
|
| 787 |
-
#
|
| 788 |
-
# ⛔⛔ TRAP 3 — A GRANT CANNOT UN-HIDE AN ADMIN-WALLED COLUMN. `perm_scope.hidden_keys` UNIONS
|
| 789 |
-
# the administrator's `hiddenFields` with the field-grant source, so granting a walled column
|
| 790 |
-
# answers 200, writes the record, and the receiver still sees nothing: granted with no effect,
|
| 791 |
-
# which is the failure this ticket exists to end. Detected BEFORE the grant, refused by name,
|
| 792 |
-
# and the receiver it is walled from is named too.
|
| 793 |
-
#
|
| 794 |
-
# ⚠ THE WALL IS READ FROM THE ADMIN SOURCE DIRECTLY AND NEVER THROUGH `hidden_keys`, AND THAT IS
|
| 795 |
-
# NOT AN OPTIMISATION. `hidden_keys` unions `field_grant_hidden` — "hidden because you were not
|
| 796 |
-
# granted it" — which is true of every column this route is ABOUT to grant. Asking it here would
|
| 797 |
-
# refuse the whole set on the grounds that the grant has not happened yet.
|
| 798 |
-
|
| 799 |
-
|
| 800 |
-
def _view_configs(view):
|
| 801 |
-
"""The dicts a saved view keeps its column state in.
|
| 802 |
-
|
| 803 |
-
`SavedView.config` is the contract (`types.ts::ViewConfig`) and is where every key below
|
| 804 |
-
actually lives. The record ITSELF is read as well because a legacy row that stored the same
|
| 805 |
-
members flat costs one dict lookup to cover and would otherwise hand back an empty key set,
|
| 806 |
-
which is the shape that grants nothing and reports nothing wrong.
|
| 807 |
-
"""
|
| 808 |
-
out = []
|
| 809 |
-
if not isinstance(view, dict):
|
| 810 |
-
return out
|
| 811 |
-
cfg = view.get("config")
|
| 812 |
-
if isinstance(cfg, dict):
|
| 813 |
-
out.append(cfg)
|
| 814 |
-
out.append(view)
|
| 815 |
-
return out
|
| 816 |
|
| 817 |
|
| 818 |
def _view_field_keys(view):
|
|
@@ -826,45 +791,57 @@ def _view_field_keys(view):
|
|
| 826 |
⚠ AND `filters` IS A TREE, NOT A LIST. Condition groups nest under `children`, so the walk
|
| 827 |
recurses rather than iterating once: a column named only inside a group is still named.
|
| 828 |
"""
|
| 829 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 830 |
|
| 831 |
-
|
|
|
|
|
|
|
| 832 |
key = str(raw or "").strip()
|
| 833 |
-
|
| 834 |
-
|
| 835 |
-
|
| 836 |
-
|
| 837 |
-
|
| 838 |
-
|
| 839 |
-
for
|
| 840 |
-
|
| 841 |
-
|
| 842 |
-
|
| 843 |
-
|
| 844 |
-
|
| 845 |
-
|
| 846 |
-
|
| 847 |
-
|
| 848 |
-
|
| 849 |
-
if isinstance(
|
| 850 |
-
|
| 851 |
-
|
| 852 |
-
|
| 853 |
-
for key in (cfg.get("order") or ()):
|
| 854 |
-
add(key)
|
| 855 |
-
for key in (cfg.get("visible") or ()):
|
| 856 |
-
add(key)
|
| 857 |
-
walk(cfg.get("filters"))
|
| 858 |
-
add(cfg.get("groupBy"))
|
| 859 |
-
colour = cfg.get("colorBy")
|
| 860 |
-
add(colour.get("colId") if isinstance(colour, dict) else colour)
|
| 861 |
-
sorts = cfg.get("sorts")
|
| 862 |
-
if isinstance(sorts, dict):
|
| 863 |
-
sorts = [sorts]
|
| 864 |
-
for spec in (sorts or ()):
|
| 865 |
-
if isinstance(spec, dict):
|
| 866 |
-
add(spec.get("colId"))
|
| 867 |
-
return keys
|
| 868 |
|
| 869 |
|
| 870 |
def _find_view_topic(session, oid):
|
|
@@ -891,7 +868,7 @@ def _find_view_topic(session, oid):
|
|
| 891 |
return None, None
|
| 892 |
|
| 893 |
|
| 894 |
-
def _promote_field_for_grant(session, table_key, field_key):
|
| 895 |
"""Promote a private column exactly once, stamp it shared, and answer its GRANT OID.
|
| 896 |
|
| 897 |
⛔⛔ ONE SEQUENCE, TWO ARMS, AND THAT IS THE WHOLE REASON THIS IS A FUNCTION. `put_share`'s
|
|
@@ -921,8 +898,8 @@ def _promote_field_for_grant(session, table_key, field_key):
|
|
| 921 |
person who did not make it. It was a 404 before this ticket and it stays one.
|
| 922 |
"""
|
| 923 |
from core import field_permissions, shared_overlay
|
| 924 |
-
defn, stratum, workspace_key, shared_key, grant_topic =
|
| 925 |
-
session, table_key, field_key)
|
| 926 |
if not isinstance(defn, dict) or stratum == "colleague":
|
| 927 |
return None
|
| 928 |
if stratum == "mine":
|
|
@@ -936,38 +913,6 @@ def _promote_field_for_grant(session, table_key, field_key):
|
|
| 936 |
return shares.field_oid(grant_topic, field_key)
|
| 937 |
|
| 938 |
|
| 939 |
-
def _grant_receivers(session, entries):
|
| 940 |
-
"""`{username: display name}` for everybody a grant list actually reaches.
|
| 941 |
-
|
| 942 |
-
`shares.EVERYONE` is a real audience, not a placeholder, so it expands to the tenant roster
|
| 943 |
-
`_people` already computes for the picker. Reading it from a second place would be a second
|
| 944 |
-
answer to "who is in this workspace".
|
| 945 |
-
"""
|
| 946 |
-
people = None
|
| 947 |
-
out = {}
|
| 948 |
-
for entry in shares._clean_entries(entries):
|
| 949 |
-
who = str(entry.get("user") or "").strip().lower()
|
| 950 |
-
if not who:
|
| 951 |
-
continue
|
| 952 |
-
if who == shares.EVERYONE:
|
| 953 |
-
if people is None:
|
| 954 |
-
people = _people(session.tenant)
|
| 955 |
-
for person in people:
|
| 956 |
-
out.setdefault(str(person["username"]).strip().lower(), person["name"])
|
| 957 |
-
continue
|
| 958 |
-
out.setdefault(who, who)
|
| 959 |
-
if out:
|
| 960 |
-
try:
|
| 961 |
-
reg = users.registry() or {}
|
| 962 |
-
except Exception: # noqa: BLE001
|
| 963 |
-
reg = {}
|
| 964 |
-
for uname in list(out):
|
| 965 |
-
rec = reg.get(uname)
|
| 966 |
-
if isinstance(rec, dict) and rec.get("name"):
|
| 967 |
-
out[uname] = str(rec["name"])
|
| 968 |
-
return out
|
| 969 |
-
|
| 970 |
-
|
| 971 |
def _admin_walled(session, grant_topic, defs, receivers):
|
| 972 |
"""`{field key: [display names]}` for receivers an ADMINISTRATOR has already walled off.
|
| 973 |
|
|
@@ -1020,187 +965,74 @@ def _admin_walled(session, grant_topic, defs, receivers):
|
|
| 1020 |
return walled
|
| 1021 |
|
| 1022 |
|
| 1023 |
-
|
| 1024 |
-
|
| 1025 |
-
#: [[a-declared-gate-is-an-unchecked-claim]] failure `verify_field_doors` section 8 exists to
|
| 1026 |
-
#: prevent: every cause fails by its own name, or it is not gated at all.
|
| 1027 |
-
#: ⛔ STANDING RULE 2 — no em dash and no en dash in any sentence below; these reach a screen.
|
| 1028 |
-
FIELD_REFUSAL_CODES = ("not_a_shared_column", "not_owned", "admin_hidden")
|
| 1029 |
-
|
| 1030 |
-
#: ⭐ TRAP 1, SAID OUT LOUD. A column grant is keyed by DATABASE, so it is not scoped to the view
|
| 1031 |
-
#: that occasioned it. The response carries this sentence rather than letting a caller infer a
|
| 1032 |
-
#: narrower promise than the store actually makes.
|
| 1033 |
-
FIELD_GRANT_SCOPE = ("A column shared this way is shared on the whole database, so the people "
|
| 1034 |
-
"you named can see it on every view of it, not only on this one.")
|
| 1035 |
-
|
| 1036 |
-
|
| 1037 |
-
def _refusal(key, code, names=()):
|
| 1038 |
-
"""One refused column, with the sentence a person reads."""
|
| 1039 |
-
if code == "not_a_shared_column":
|
| 1040 |
-
why = ('"{k}" belongs to the database itself rather than to a person, so there was '
|
| 1041 |
-
'nothing to grant. Anybody who can open the database already sees it.')
|
| 1042 |
-
elif code == "not_owned":
|
| 1043 |
-
why = ('"{k}" is somebody else\'s column, and only its owner (or an administrator) can '
|
| 1044 |
-
'share it. Ask them to share it, then share this view again.')
|
| 1045 |
-
else:
|
| 1046 |
-
why = ('"{k}" is hidden from {who} by an administrator, so sharing it would change '
|
| 1047 |
-
'nothing. Ask an administrator to unhide it first.')
|
| 1048 |
-
return {"field": key, "reason": code,
|
| 1049 |
-
"message": why.format(k=key, who=_named(names) if names else "them")}
|
| 1050 |
-
|
| 1051 |
-
|
| 1052 |
-
def _classify_view_fields(keys, owned, defined, walled):
|
| 1053 |
-
"""`(sharedFieldKeys, refusedFieldKeys)` for the columns a view names. PURE.
|
| 1054 |
-
|
| 1055 |
-
⛔ EVERY KEY IS DECIDED BEFORE ANY OF THEM IS WRITTEN, which is the same property the ceiling
|
| 1056 |
-
block above claims for the three role walls: a refusal must not be able to leave part of the
|
| 1057 |
-
set promoted and part of it untouched.
|
| 1058 |
-
|
| 1059 |
-
⚠ THE ORDER OF THE CAUSES IS FIXED AND IT DECIDES WHICH SENTENCE A PERSON READS. A built-in
|
| 1060 |
-
column is owned by nobody, so "there was nothing to grant" has to be asked before "you do not
|
| 1061 |
-
own it", or every database column in `visible` would be reported as somebody else's.
|
| 1062 |
-
"""
|
| 1063 |
-
granted, refused = [], []
|
| 1064 |
-
for key in keys:
|
| 1065 |
-
if key not in defined:
|
| 1066 |
-
refused.append(_refusal(key, "not_a_shared_column"))
|
| 1067 |
-
elif key not in owned:
|
| 1068 |
-
refused.append(_refusal(key, "not_owned"))
|
| 1069 |
-
elif walled.get(key):
|
| 1070 |
-
refused.append(_refusal(key, "admin_hidden", walled.get(key)))
|
| 1071 |
-
else:
|
| 1072 |
-
granted.append(key)
|
| 1073 |
-
return granted, refused
|
| 1074 |
-
|
| 1075 |
-
|
| 1076 |
-
def _merge_field_entries(existing, incoming):
|
| 1077 |
-
"""A column's grant list with `incoming` ADDED to it. Nobody is removed, nobody is lowered.
|
| 1078 |
-
|
| 1079 |
-
⛔⛔ THE SHARE PUT REPLACES, AND A COLUMN'S AUDIENCE IS NOT THE VIEW'S AUDIENCE. This route's
|
| 1080 |
-
contract is a REPLACING set (`PUT ... entries` at the top of the file), which is right for the
|
| 1081 |
-
object the caller is looking at and catastrophic for the columns underneath it: a column grant
|
| 1082 |
-
is TOPIC-WIDE, so handing the view's list straight to `set_grants` would rewrite the column's
|
| 1083 |
-
audience to whoever this view happens to be shared with. A colleague granted that column
|
| 1084 |
-
through the `field` door would lose it, on every view of the database, silently, inside a 200.
|
| 1085 |
-
That is exactly the harm D-474 closed at the view door, arriving one door over.
|
| 1086 |
-
|
| 1087 |
-
⚠ AND A LENGTH CHECK DOES NOT SEE IT. Replacing a one-person list with a different one-person
|
| 1088 |
-
list keeps the count identical, so the gate asserts the HELD USER IS STILL THERE AT THEIR OWN
|
| 1089 |
-
ROLE rather than counting rows.
|
| 1090 |
-
|
| 1091 |
-
⭐ RAISED, NEVER LOWERED. A receiver already held at `view` who arrives on an `edit` share is
|
| 1092 |
-
raised, because only the column's owner (or an administrator) reaches this path at all and
|
| 1093 |
-
that is the access they just asked to hand over. An `edit` holder meeting a `view` share keeps
|
| 1094 |
-
`edit`: a share is an addition, and nothing here is a revocation.
|
| 1095 |
-
|
| 1096 |
-
⚠ `by` RIDES ON EXISTING ROWS AND IS NEVER TAKEN FROM AN INCOMING ONE. Provenance decides who
|
| 1097 |
-
may revoke a grant (`_unremovable`), so an incoming payload must not be able to claim it;
|
| 1098 |
-
`set_grants` stamps a genuinely new row from `granter`.
|
| 1099 |
-
"""
|
| 1100 |
-
rank = {"view": 0, "edit": 1}
|
| 1101 |
-
order, merged = [], {}
|
| 1102 |
-
for row in (existing or ()):
|
| 1103 |
-
if not isinstance(row, dict):
|
| 1104 |
-
continue
|
| 1105 |
-
who = str(row.get("user") or "").strip().lower()
|
| 1106 |
-
role = str(row.get("role") or "").strip().lower()
|
| 1107 |
-
if not who or role not in rank or who in merged:
|
| 1108 |
-
continue
|
| 1109 |
-
kept = {"user": who, "role": role}
|
| 1110 |
-
stamp = str(row.get("by") or "").strip().lower()
|
| 1111 |
-
if stamp:
|
| 1112 |
-
kept["by"] = stamp
|
| 1113 |
-
merged[who] = kept
|
| 1114 |
-
order.append(who)
|
| 1115 |
-
for row in (incoming or ()):
|
| 1116 |
-
if not isinstance(row, dict):
|
| 1117 |
-
continue
|
| 1118 |
-
who = str(row.get("user") or "").strip().lower()
|
| 1119 |
-
role = str(row.get("role") or "").strip().lower()
|
| 1120 |
-
if not who or role not in rank:
|
| 1121 |
-
continue
|
| 1122 |
-
if who not in merged:
|
| 1123 |
-
merged[who] = {"user": who, "role": role}
|
| 1124 |
-
order.append(who)
|
| 1125 |
-
elif rank[role] > rank[merged[who]["role"]]:
|
| 1126 |
-
merged[who]["role"] = role
|
| 1127 |
-
return [merged[who] for who in order]
|
| 1128 |
|
|
|
|
|
|
|
|
|
|
| 1129 |
|
| 1130 |
-
def _share_view_fields(session, oid, entries):
|
| 1131 |
-
"""The `view` arm's field half: classify every column the view names, then grant the ones
|
| 1132 |
-
that survive. `(sharedFieldKeys, refusedFieldKeys)`.
|
| 1133 |
|
| 1134 |
-
|
| 1135 |
-
|
| 1136 |
-
view's columns promoted.
|
| 1137 |
|
| 1138 |
-
|
| 1139 |
-
|
| 1140 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1141 |
"""
|
| 1142 |
topic, view = _find_view_topic(session, oid)
|
| 1143 |
if not topic or not isinstance(view, dict):
|
| 1144 |
-
|
| 1145 |
keys = _view_field_keys(view)
|
| 1146 |
-
if not keys:
|
| 1147 |
-
return [], []
|
| 1148 |
table_key = f"{topic}_table_workspace"
|
| 1149 |
grant_topic = _field_storage_keys(table_key)[2]
|
| 1150 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1151 |
for key in keys:
|
| 1152 |
-
|
| 1153 |
-
|
| 1154 |
-
|
| 1155 |
-
|
| 1156 |
-
|
| 1157 |
-
|
| 1158 |
-
|
| 1159 |
-
|
| 1160 |
-
# W42-T25 states the intent in as many words: *"Throwing instead would make the
|
| 1161 |
-
# ordinary case (a view over one built-in column) unshareable."*
|
| 1162 |
-
# * `colleague` would put a refusal row per colleague-private column into the dialog's
|
| 1163 |
-
# refused list, for columns the sharer never named.
|
| 1164 |
-
# The `field` arm keeps both, because there the column IS the request.
|
| 1165 |
-
if not isinstance(defn, dict) or stratum in ("catalogue", "colleague"):
|
| 1166 |
continue
|
| 1167 |
-
|
| 1168 |
-
|
| 1169 |
-
|
| 1170 |
-
|
| 1171 |
-
|
| 1172 |
-
owners[key] = _field_owner(session, defn, stratum == "shared", stratum=stratum)
|
| 1173 |
-
# ⛔ ONE OWNERSHIP PREDICATE. `_owns_object` is the wall the `field` arm claims through
|
| 1174 |
-
# and the DELETE door reads; a second "does this person own this column" here is how the
|
| 1175 |
-
# two answers come apart. It short circuits True for an administrator, which is right:
|
| 1176 |
-
# an administrator may already share any column through the `field` door.
|
| 1177 |
-
if _owns_object(session, "field", shares.field_oid(grant_topic, key)):
|
| 1178 |
-
owned.add(key)
|
| 1179 |
-
walled = _admin_walled(session, grant_topic, defs, _grant_receivers(session, entries))
|
| 1180 |
-
granted, refused = _classify_view_fields(keys, owned, defined, walled)
|
| 1181 |
-
written = []
|
| 1182 |
-
for key in granted:
|
| 1183 |
-
field_oid = _promote_field_for_grant(session, table_key, key)
|
| 1184 |
-
if not field_oid:
|
| 1185 |
-
refused.append(_refusal(key, "not_a_shared_column"))
|
| 1186 |
continue
|
| 1187 |
-
|
| 1188 |
-
|
| 1189 |
-
|
| 1190 |
-
|
| 1191 |
-
|
| 1192 |
-
|
| 1193 |
-
|
| 1194 |
-
|
| 1195 |
-
|
| 1196 |
-
|
| 1197 |
-
|
| 1198 |
-
|
| 1199 |
-
|
| 1200 |
-
|
| 1201 |
-
|
| 1202 |
-
|
| 1203 |
-
|
|
|
|
|
|
|
| 1204 |
|
| 1205 |
|
| 1206 |
@router.get("/share/mine")
|
|
@@ -1253,7 +1085,16 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
|
| 1253 |
# administrator-shaped Farhan session must not turn Shantal's absent grant into a record.
|
| 1254 |
# Other share kinds retain their established administrator behaviour.
|
| 1255 |
_admin_share = bool(session.admin) and kind != "cohort"
|
| 1256 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1257 |
may_admin = shares.may_administer(kind, oid, session.uname, is_admin=_admin_share,
|
| 1258 |
st=session.runtime)
|
| 1259 |
# W39-T29 — the dialog opens with GET before its first PUT. Until that PUT exists the grant
|
|
@@ -1561,6 +1402,44 @@ def put_share(kind: str, oid: str, body: dict = Body(default=None),
|
|
| 1561 |
# write rather than at the moment of the read: a concurrent save lands between the two, and
|
| 1562 |
# a wave-40 adversarial probe drove a re-sharer's PUT being ACCEPTED while the grant the
|
| 1563 |
# owner had just added disappeared. See `shares.set_grants`' own note.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1564 |
try:
|
| 1565 |
out = shares.set_grants(kind, oid, entries,
|
| 1566 |
owner=session.uname if claiming else rec["owner"],
|
|
@@ -1570,20 +1449,6 @@ def put_share(kind: str, oid: str, body: dict = Body(default=None),
|
|
| 1570 |
except shares.GrantsChanged as exc:
|
| 1571 |
raise err(409, "grants_changed", str(exc))
|
| 1572 |
_notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
|
| 1573 |
-
# ⭐⭐ W42-T25 — SHARING A VIEW SHARES THE FIELDS IT NAMES. A view is a selection expressed
|
| 1574 |
-
# in columns, so a grant that carries the view and none of its columns puts a rail row on
|
| 1575 |
-
# the receiver's screen over an empty grid. The three refusals are named rather than thrown:
|
| 1576 |
-
# this PUT succeeded, the view IS shared, and what could not follow it is reported by name
|
| 1577 |
-
# with the reason. Throwing instead would make the ordinary case (a view over one built-in
|
| 1578 |
-
# column) unshareable.
|
| 1579 |
-
# ⛔ THE EXISTING KEYS ARE UNTOUCHED. `{owner, entries}` is what every current client reads
|
| 1580 |
-
# off this route; the field report is additive, and a caller that ignores it is unaffected.
|
| 1581 |
-
if kind == "view":
|
| 1582 |
-
shared_fields, refused_fields = _share_view_fields(session, oid, entries)
|
| 1583 |
-
return {**out,
|
| 1584 |
-
"sharedFieldKeys": shared_fields,
|
| 1585 |
-
"refusedFieldKeys": refused_fields,
|
| 1586 |
-
"sharedFieldScope": FIELD_GRANT_SCOPE}
|
| 1587 |
return out
|
| 1588 |
|
| 1589 |
|
|
|
|
| 90 |
*"Shared with me"*, who may open a source View, and who may re-share. Saved Views carry no
|
| 91 |
view-level permissions member or secondary access model.
|
| 92 |
"""
|
| 93 |
+
import secrets
|
| 94 |
+
|
| 95 |
from fastapi import APIRouter, Body, Depends
|
| 96 |
|
| 97 |
import core.shares as shares
|
| 98 |
import core.users as users
|
| 99 |
+
from core import field_permissions as _field_permissions
|
| 100 |
from deps import Session, err, require_session
|
| 101 |
# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
|
| 102 |
# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
|
|
|
|
| 106 |
|
| 107 |
router = APIRouter(prefix="/api/v1")
|
| 108 |
|
| 109 |
+
# C8's derived field leg belongs to the field-permission seam, while every existing read/write
|
| 110 |
+
# wall already calls ``shares.role_for``. Install it explicitly at application-router startup;
|
| 111 |
+
# no caller gets a second permission predicate and non-field kinds are byte-for-byte delegated.
|
| 112 |
+
_field_permissions.install_view_field_resolver()
|
| 113 |
+
|
| 114 |
|
| 115 |
def _kind_or_400(raw):
|
| 116 |
try:
|
|
|
|
| 257 |
none, because residency IS authorship in a per-account document.
|
| 258 |
4. **the connector catalogue** — the columns the database itself declares.
|
| 259 |
|
| 260 |
+
⚠ `strata` NARROWS THE WALK, AND IT IS A COST CONTROL AS WELL AS A CORRECTNESS ONE. A caller
|
| 261 |
+
that already knows which definition strata are relevant can avoid the remaining reads; the
|
| 262 |
+
View share preflight deliberately checks all four because unreadable references must refuse
|
| 263 |
+
the whole transaction rather than be misclassified as database-owned.
|
|
|
|
| 264 |
|
| 265 |
⛔ EACH STRATUM RESOLVES IN ITS OWN `try`, AND THE BARE `except Exception: pass` THAT WRAPPED
|
| 266 |
ALL OF THEM IS GONE. It conflated *"the store threw"* with *"no such field"*, so one
|
|
|
|
| 772 |
stuck.add(user)
|
| 773 |
return stuck
|
| 774 |
|
| 775 |
+
# ── SDLC63 / C8 — ONE VIEW AUDIENCE, ONE DERIVED FIELD BOUNDARY ───────────────────────────────
|
| 776 |
+
# The pre-SDLC63 route copied a View audience into every named field grant and returned every
|
| 777 |
+
# field/refusal to the browser. The current path resolves the deliberate field set once,
|
| 778 |
+
# preflights the editor's own readability, and commits only the View audience plus its compact
|
| 779 |
+
# field boundary. Field detail leaves the server only as a count on success; failures retain it
|
| 780 |
+
# under a support reference rather than turning a large View into a large error message.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 781 |
|
| 782 |
|
| 783 |
def _view_field_keys(view):
|
|
|
|
| 791 |
⚠ AND `filters` IS A TREE, NOT A LIST. Condition groups nest under `children`, so the walk
|
| 792 |
recurses rather than iterating once: a column named only inside a group is still named.
|
| 793 |
"""
|
| 794 |
+
return _field_permissions.view_field_keys(view)
|
| 795 |
+
|
| 796 |
+
|
| 797 |
+
def _view_field_definitions(session, table_key, keys):
|
| 798 |
+
"""Resolve a View's field definitions with a constant number of source reads."""
|
| 799 |
+
workspace_key, shared_key, grant_topic = _field_storage_keys(table_key)
|
| 800 |
+
shared, mine, document, catalogue = {}, {}, {}, {}
|
| 801 |
+
try:
|
| 802 |
+
from core import shared_overlay
|
| 803 |
+
shared = shared_overlay.fields(shared_key, st=session.runtime) or {}
|
| 804 |
+
except Exception: # noqa: BLE001
|
| 805 |
+
pass
|
| 806 |
+
try:
|
| 807 |
+
import core.table_store as table_store
|
| 808 |
+
mine = (table_store.make(workspace_key, st=session.runtime)
|
| 809 |
+
.workspace(session.uname).get("fields") or {})
|
| 810 |
+
except Exception: # noqa: BLE001
|
| 811 |
+
pass
|
| 812 |
+
try:
|
| 813 |
+
document = session.runtime.get(workspace_key) or {}
|
| 814 |
+
except Exception: # noqa: BLE001
|
| 815 |
+
pass
|
| 816 |
+
try:
|
| 817 |
+
catalogue = _catalogue_fields(session, table_key, workspace_key, grant_topic)
|
| 818 |
+
except Exception: # noqa: BLE001
|
| 819 |
+
pass
|
| 820 |
|
| 821 |
+
me = str(session.uname or "").strip().lower()
|
| 822 |
+
out = {}
|
| 823 |
+
for raw in keys or ():
|
| 824 |
key = str(raw or "").strip()
|
| 825 |
+
defn, stratum = None, None
|
| 826 |
+
if isinstance(shared, dict) and isinstance(shared.get(key), dict):
|
| 827 |
+
defn, stratum = shared[key], "shared"
|
| 828 |
+
elif isinstance(mine, dict) and isinstance(mine.get(key), dict):
|
| 829 |
+
defn, stratum = mine[key], "mine"
|
| 830 |
+
elif isinstance(document, dict):
|
| 831 |
+
for member, workspace in document.items():
|
| 832 |
+
if str(member).strip().lower() == me or not isinstance(workspace, dict):
|
| 833 |
+
continue
|
| 834 |
+
theirs = (workspace.get("fields") or {}).get(key)
|
| 835 |
+
if isinstance(theirs, dict):
|
| 836 |
+
defn = dict(theirs)
|
| 837 |
+
if not str(defn.get("createdBy") or "").strip():
|
| 838 |
+
defn["createdBy"] = str(member).strip().lower()
|
| 839 |
+
stratum = "colleague"
|
| 840 |
+
break
|
| 841 |
+
if defn is None and isinstance(catalogue, dict) and isinstance(catalogue.get(key), dict):
|
| 842 |
+
defn, stratum = catalogue[key], "catalogue"
|
| 843 |
+
out[key] = (defn, stratum, workspace_key, shared_key, grant_topic)
|
| 844 |
+
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 845 |
|
| 846 |
|
| 847 |
def _find_view_topic(session, oid):
|
|
|
|
| 868 |
return None, None
|
| 869 |
|
| 870 |
|
| 871 |
+
def _promote_field_for_grant(session, table_key, field_key, resolved=None):
|
| 872 |
"""Promote a private column exactly once, stamp it shared, and answer its GRANT OID.
|
| 873 |
|
| 874 |
⛔⛔ ONE SEQUENCE, TWO ARMS, AND THAT IS THE WHOLE REASON THIS IS A FUNCTION. `put_share`'s
|
|
|
|
| 898 |
person who did not make it. It was a 404 before this ticket and it stays one.
|
| 899 |
"""
|
| 900 |
from core import field_permissions, shared_overlay
|
| 901 |
+
defn, stratum, workspace_key, shared_key, grant_topic = (
|
| 902 |
+
resolved if resolved is not None else _field_resolve(session, table_key, field_key))
|
| 903 |
if not isinstance(defn, dict) or stratum == "colleague":
|
| 904 |
return None
|
| 905 |
if stratum == "mine":
|
|
|
|
| 913 |
return shares.field_oid(grant_topic, field_key)
|
| 914 |
|
| 915 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 916 |
def _admin_walled(session, grant_topic, defs, receivers):
|
| 917 |
"""`{field key: [display names]}` for receivers an ADMINISTRATOR has already walled off.
|
| 918 |
|
|
|
|
| 965 |
return walled
|
| 966 |
|
| 967 |
|
| 968 |
+
class _ViewSharePreparationError(RuntimeError):
|
| 969 |
+
"""Server-only field details for a refused all-or-none View share."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 970 |
|
| 971 |
+
def __init__(self, details):
|
| 972 |
+
self.details = [str(value) for value in (details or ()) if str(value)]
|
| 973 |
+
super().__init__("one or more View fields could not be prepared")
|
| 974 |
|
|
|
|
|
|
|
|
|
|
| 975 |
|
| 976 |
+
def _prepare_view_share(session, oid, *, preview=False):
|
| 977 |
+
"""Resolve one View and make its personal definitions tenant-readable before the CAS.
|
|
|
|
| 978 |
|
| 979 |
+
Database-owned fields need no field write and some connector columns have no overlay
|
| 980 |
+
definition at all. A personal definition is promoted while it
|
| 981 |
+
still has no derived audience; if a later transaction fails, it remains an owner-controlled
|
| 982 |
+
shared definition rather than leaking a value. Resolved unreadable references refuse the
|
| 983 |
+
whole operation and ride only in server diagnostics; keys absent from definition strata are
|
| 984 |
+
database-owned fields already admitted by the validated View-save door.
|
| 985 |
+
Preview performs the same admission checks without promoting personal definitions.
|
| 986 |
"""
|
| 987 |
topic, view = _find_view_topic(session, oid)
|
| 988 |
if not topic or not isinstance(view, dict):
|
| 989 |
+
raise _ViewSharePreparationError(["view:not_found"])
|
| 990 |
keys = _view_field_keys(view)
|
|
|
|
|
|
|
| 991 |
table_key = f"{topic}_table_workspace"
|
| 992 |
grant_topic = _field_storage_keys(table_key)[2]
|
| 993 |
+
details = []
|
| 994 |
+
definitions = {}
|
| 995 |
+
database_owned = 0
|
| 996 |
+
resolved = _view_field_definitions(session, table_key, keys)
|
| 997 |
+
shared_roles = _field_permissions.field_roles_for_keys(
|
| 998 |
+
grant_topic,
|
| 999 |
+
[key for key, row in resolved.items() if row[1] == "shared"],
|
| 1000 |
+
session.uname, is_admin=session.admin, st=session.runtime)
|
| 1001 |
for key in keys:
|
| 1002 |
+
row = resolved.get(key) or (None, None, None, None, grant_topic)
|
| 1003 |
+
defn, stratum = row[:2]
|
| 1004 |
+
if not isinstance(defn, dict):
|
| 1005 |
+
# The validated View-save door already intersected references with the editor's
|
| 1006 |
+
# served schema. Absence from the overlay/catalogue resolver therefore means a
|
| 1007 |
+
# database-owned column, not a private definition to publish. It belongs in the
|
| 1008 |
+
# compact boundary and needs no per-field ownership or grant mutation.
|
| 1009 |
+
database_owned += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1010 |
continue
|
| 1011 |
+
definitions[key] = defn
|
| 1012 |
+
if stratum == "catalogue":
|
| 1013 |
+
database_owned += 1
|
| 1014 |
+
if stratum == "colleague":
|
| 1015 |
+
details.append(f"{key}:not_readable")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1016 |
continue
|
| 1017 |
+
if stratum == "shared":
|
| 1018 |
+
if (shared_roles.get(key) or {}).get("role") is None:
|
| 1019 |
+
details.append(f"{key}:not_readable")
|
| 1020 |
+
continue
|
| 1021 |
+
if stratum == "mine" and not preview:
|
| 1022 |
+
field_oid = _promote_field_for_grant(session, table_key, key, resolved=row)
|
| 1023 |
+
if not field_oid:
|
| 1024 |
+
details.append(f"{key}:promotion_failed")
|
| 1025 |
+
|
| 1026 |
+
# The account's permanent hidden-field wall wins even for catalogue fields. The View save
|
| 1027 |
+
# path already removes these references; repeating that answer here keeps a stale or imported
|
| 1028 |
+
# View from becoming a permission bypass at its first share.
|
| 1029 |
+
walled = _admin_walled(
|
| 1030 |
+
session, grant_topic, definitions,
|
| 1031 |
+
{session.uname: _display_name(session.uname)})
|
| 1032 |
+
details.extend(f"{key}:not_readable" for key in sorted(walled))
|
| 1033 |
+
if details:
|
| 1034 |
+
raise _ViewSharePreparationError(details)
|
| 1035 |
+
return topic, view, grant_topic, keys, database_owned
|
| 1036 |
|
| 1037 |
|
| 1038 |
@router.get("/share/mine")
|
|
|
|
| 1085 |
# administrator-shaped Farhan session must not turn Shantal's absent grant into a record.
|
| 1086 |
# Other share kinds retain their established administrator behaviour.
|
| 1087 |
_admin_share = bool(session.admin) and kind != "cohort"
|
| 1088 |
+
if kind == "field":
|
| 1089 |
+
table_key, field_key = shares.split_field_oid(oid)
|
| 1090 |
+
# C8's inherited path grants the data, not the column's Manage Access roster. Use the
|
| 1091 |
+
# direct leg here so a View audience cannot enumerate a field owner's separate grants.
|
| 1092 |
+
role = _field_permissions.field_role_sources(
|
| 1093 |
+
table_key, field_key, session.uname,
|
| 1094 |
+
is_admin=_admin_share, st=session.runtime)["direct"]
|
| 1095 |
+
else:
|
| 1096 |
+
role = shares.role_for(kind, oid, session.uname,
|
| 1097 |
+
is_admin=_admin_share, st=session.runtime)
|
| 1098 |
may_admin = shares.may_administer(kind, oid, session.uname, is_admin=_admin_share,
|
| 1099 |
st=session.runtime)
|
| 1100 |
# W39-T29 — the dialog opens with GET before its first PUT. Until that PUT exists the grant
|
|
|
|
| 1402 |
# write rather than at the moment of the read: a concurrent save lands between the two, and
|
| 1403 |
# a wave-40 adversarial probe drove a re-sharer's PUT being ACCEPTED while the grant the
|
| 1404 |
# owner had just added disappeared. See `shares.set_grants`' own note.
|
| 1405 |
+
if kind == "view":
|
| 1406 |
+
fields = []
|
| 1407 |
+
stage = "prepare"
|
| 1408 |
+
try:
|
| 1409 |
+
_topic, view, grant_topic, fields, database_owned = _prepare_view_share(session, oid)
|
| 1410 |
+
stage = "transaction"
|
| 1411 |
+
out = _field_permissions.set_view_grants_with_boundary(
|
| 1412 |
+
view, grant_topic, entries,
|
| 1413 |
+
owner=session.uname if claiming else rec["owner"],
|
| 1414 |
+
granter=session.uname, expect=rec["entries"], st=session.runtime)
|
| 1415 |
+
except shares.GrantsChanged as exc:
|
| 1416 |
+
raise err(409, "grants_changed", str(exc))
|
| 1417 |
+
except Exception as exc: # noqa: BLE001
|
| 1418 |
+
# The browser gets one short, actionable sentence. Field names and the internal
|
| 1419 |
+
# cause remain in the tenant-scoped diagnostic ring under the same support reference.
|
| 1420 |
+
reference = f"VS-{secrets.token_hex(4).upper()}"
|
| 1421 |
+
details = getattr(exc, "details", None) or fields
|
| 1422 |
+
try:
|
| 1423 |
+
_field_permissions.record_view_share_diagnostic(
|
| 1424 |
+
reference, oid, session.uname, stage, details, exc,
|
| 1425 |
+
st=session.runtime)
|
| 1426 |
+
except Exception: # noqa: BLE001
|
| 1427 |
+
pass
|
| 1428 |
+
raise err(
|
| 1429 |
+
500, "view_share_failed",
|
| 1430 |
+
"We could not update this View's access. Try again. If it keeps failing, "
|
| 1431 |
+
f"contact support with reference {reference}.")
|
| 1432 |
+
_notify_new_grantees(
|
| 1433 |
+
session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
|
| 1434 |
+
# One field-free compatibility reason keeps older API callers from mistaking connector
|
| 1435 |
+
# columns for promoted custom definitions. It is never rendered and never scales with
|
| 1436 |
+
# the field count; all actual field detail stays server-side under a support reference.
|
| 1437 |
+
compatibility = ([{"reason": "not_a_shared_column"}] if database_owned else [])
|
| 1438 |
+
return {"owner": out.get("owner"), "entries": out.get("entries") or [],
|
| 1439 |
+
"fieldShare": {"includedCount": int(out.get("fieldCount") or 0),
|
| 1440 |
+
"databaseOwnedCount": database_owned},
|
| 1441 |
+
"refusedFieldKeys": compatibility}
|
| 1442 |
+
|
| 1443 |
try:
|
| 1444 |
out = shares.set_grants(kind, oid, entries,
|
| 1445 |
owner=session.uname if claiming else rec["owner"],
|
|
|
|
| 1449 |
except shares.GrantsChanged as exc:
|
| 1450 |
raise err(409, "grants_changed", str(exc))
|
| 1451 |
_notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1452 |
return out
|
| 1453 |
|
| 1454 |
|
api/routes_sheet.py
CHANGED
|
@@ -44,6 +44,7 @@ import threading
|
|
| 44 |
import uuid
|
| 45 |
|
| 46 |
from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
|
|
|
|
| 47 |
|
| 48 |
from deps import Session, err, require_session
|
| 49 |
|
|
@@ -974,7 +975,36 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str:
|
|
| 974 |
import core.perm_scope as perm_scope
|
| 975 |
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 976 |
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 977 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 978 |
|
| 979 |
|
| 980 |
def _has_ending(binding):
|
|
@@ -988,128 +1018,131 @@ def _has_cell_inputs(binding):
|
|
| 988 |
return True
|
| 989 |
scope = binding.get("scope") if isinstance(binding, dict) else None
|
| 990 |
nodes, _ = _fe_parts(scope.get("filter") if isinstance(scope, dict) else None)
|
|
|
|
|
|
|
|
|
|
| 991 |
# Linked values and related-table grants can change independently of the orders' cache key.
|
| 992 |
if sheet_mod.linked_filter_ids(nodes):
|
| 993 |
return True
|
| 994 |
return any(sheet_mod.filter_cell_nodes(nodes))
|
| 995 |
|
| 996 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 997 |
def _reresolved_for_reader(session: Session, book, *, endings_only=False):
|
| 998 |
-
"""
|
| 999 |
-
|
| 1000 |
-
|
| 1001 |
-
|
| 1002 |
-
|
| 1003 |
-
|
| 1004 |
-
|
| 1005 |
-
|
| 1006 |
-
|
| 1007 |
-
|
| 1008 |
-
|
| 1009 |
-
|
| 1010 |
-
|
| 1011 |
-
⚠ `refreshedFor` IS STAMPED ONLY WHEN A BOUND CELL WAS ACTUALLY RECOMPUTED. A workbook of
|
| 1012 |
-
literals has nothing to re-resolve, and a stamp there would be a claim with nothing behind it.
|
| 1013 |
-
|
| 1014 |
-
⭐⭐ W52-T12 / R3: A CACHE HIT SKIPS `_resolve_binding` ENTIRELY, AND A MISS FILLS ONLY THE
|
| 1015 |
-
CALLER'S OWN SLOT. The cache lives in a document `core.sheets` owns and keys by
|
| 1016 |
-
`(database, wall, binding scope)` (`_binding_scope_key`), never by user id or role name, so
|
| 1017 |
-
two readers who share a row restriction share an answer and a reader whose restriction differs
|
| 1018 |
-
never reads the other's slot. Only `refresh_sheet` may ever clear it; a GET here only ever
|
| 1019 |
-
fills the ONE slot the caller themselves is entitled to.
|
| 1020 |
-
"""
|
| 1021 |
-
from core import sheets
|
| 1022 |
-
import copy as _copy
|
| 1023 |
-
fresh = _copy.deepcopy(book)
|
| 1024 |
-
stamp = _now_iso()
|
| 1025 |
-
seen_db = {}
|
| 1026 |
-
# ⭐ The same per-request memo as `refresh_sheet`, for the same reason and with the same
|
| 1027 |
-
# lifetime: this loop resolves cell by cell too, and a reader's GET must not rebuild one
|
| 1028 |
-
# database's workspace assembly once per bound cell.
|
| 1029 |
-
view_memo = {}
|
| 1030 |
-
touched = 0
|
| 1031 |
-
store = _store(session)
|
| 1032 |
-
wid = book.get("id") or ""
|
| 1033 |
-
for tab in fresh.get("sheets") or ():
|
| 1034 |
-
tab_ref = str(tab.get("id") or tab.get("name") or "")
|
| 1035 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 1036 |
-
if
|
| 1037 |
continue
|
| 1038 |
-
if
|
| 1039 |
continue
|
| 1040 |
-
|
| 1041 |
binding = cell.get("binding") or {}
|
| 1042 |
db = str(binding.get("database") or "")
|
| 1043 |
-
if db not in
|
| 1044 |
try:
|
| 1045 |
_require_database(session, db)
|
| 1046 |
-
|
| 1047 |
-
except
|
| 1048 |
-
|
| 1049 |
-
|
| 1050 |
-
|
| 1051 |
-
|
| 1052 |
-
|
| 1053 |
-
|
| 1054 |
-
|
| 1055 |
-
|
| 1056 |
-
|
| 1057 |
-
|
| 1058 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1059 |
try:
|
| 1060 |
-
|
| 1061 |
-
|
| 1062 |
-
|
| 1063 |
-
|
| 1064 |
-
|
| 1065 |
-
|
| 1066 |
-
|
| 1067 |
-
|
| 1068 |
-
|
| 1069 |
-
|
| 1070 |
-
|
| 1071 |
-
|
| 1072 |
-
|
| 1073 |
-
|
| 1074 |
-
|
| 1075 |
-
|
| 1076 |
-
|
| 1077 |
-
except HTTPException as exc:
|
| 1078 |
-
detail = getattr(exc, "detail", None) or {}
|
| 1079 |
-
info = detail.get("error") if isinstance(detail, dict) else None
|
| 1080 |
-
cell.update(_binding_refusal(
|
| 1081 |
-
(info or {}).get("code", "sheet_unresolved"),
|
| 1082 |
-
(info or {}).get("message", "This cell could not be refreshed."),
|
| 1083 |
-
))
|
| 1084 |
-
cell.pop("limits", None)
|
| 1085 |
-
continue
|
| 1086 |
-
cell["state"] = "fresh"
|
| 1087 |
-
cell["value"] = out["value"]
|
| 1088 |
-
cell["display"] = "" if out["value"] is None else str(out["value"])
|
| 1089 |
-
cell["refreshed_at"] = stamp
|
| 1090 |
-
cell.pop("refusal", None)
|
| 1091 |
-
if out["limits"]:
|
| 1092 |
-
cell["limits"] = out["limits"]
|
| 1093 |
else:
|
| 1094 |
-
cell
|
| 1095 |
-
|
| 1096 |
-
|
| 1097 |
-
|
| 1098 |
-
|
| 1099 |
-
|
| 1100 |
-
|
| 1101 |
-
|
| 1102 |
-
|
| 1103 |
-
|
| 1104 |
-
|
| 1105 |
-
|
| 1106 |
-
#
|
| 1107 |
-
|
| 1108 |
-
|
| 1109 |
-
|
| 1110 |
-
|
| 1111 |
-
|
| 1112 |
-
|
|
|
|
|
|
|
|
|
|
| 1113 |
return fresh
|
| 1114 |
|
| 1115 |
|
|
@@ -1121,17 +1154,12 @@ def _response_workbook(book):
|
|
| 1121 |
|
| 1122 |
|
| 1123 |
def _read_for(session: Session, book):
|
| 1124 |
-
"""
|
| 1125 |
|
| 1126 |
W60-T09, 2026-09-14: R28 checks visibility before this projection. Admitted legacy readers
|
| 1127 |
-
and grantees
|
| 1128 |
"""
|
| 1129 |
book = _response_workbook(book)
|
| 1130 |
-
if _role_on_workbook(session, book) == "owner":
|
| 1131 |
-
if any(_has_cell_inputs(cell.get("binding"))
|
| 1132 |
-
for tab in book.get("sheets") or [] for cell in (tab.get("cells") or {}).values()):
|
| 1133 |
-
return _reresolved_for_reader(session, book, endings_only=True)
|
| 1134 |
-
return book
|
| 1135 |
return _reresolved_for_reader(session, book)
|
| 1136 |
|
| 1137 |
|
|
@@ -1484,7 +1512,9 @@ async def start_sheet_import(file: UploadFile = File(...),
|
|
| 1484 |
if not name.lower().endswith(".xlsx"):
|
| 1485 |
raise err(400, "import_not_xlsx",
|
| 1486 |
"Only an .xlsx workbook can be imported. Save your file as .xlsx and try again.")
|
| 1487 |
-
|
|
|
|
|
|
|
| 1488 |
if len(data) > sheet_import.MAX_IMPORT_BYTES:
|
| 1489 |
raise HTTPException(status_code=413, detail={"error": {
|
| 1490 |
"code": "import_too_large",
|
|
@@ -1530,6 +1560,51 @@ def _mb(byte_count):
|
|
| 1530 |
return "%.1f MB" % (float(byte_count) / (1024.0 * 1024.0))
|
| 1531 |
|
| 1532 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1533 |
@router.get("/sheets/{workbook_id}")
|
| 1534 |
def read_sheet(workbook_id: str, tabs: str = "",
|
| 1535 |
session: Session = Depends(require_session)):
|
|
@@ -1695,6 +1770,8 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
|
|
| 1695 |
cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
|
| 1696 |
# Date and filter-cell edits also change database totals. Recompute all readers, then formulas.
|
| 1697 |
ending_cells = set()
|
|
|
|
|
|
|
| 1698 |
for source in book.get("sheets") or []:
|
| 1699 |
for addr, cell in (source.get("cells") or {}).items():
|
| 1700 |
binding = cell.get("binding") or {}
|
|
@@ -1703,6 +1780,11 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
|
|
| 1703 |
try:
|
| 1704 |
result = _resolve_cell_binding(session, binding, book, source.get("id"))
|
| 1705 |
cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1706 |
cell.pop("refusal", None)
|
| 1707 |
except HTTPException as exc:
|
| 1708 |
info = ((exc.detail or {}).get("error") or {})
|
|
@@ -1711,7 +1793,16 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
|
|
| 1711 |
info.get("message", "The Ending date could not be read. Choose a valid date."),
|
| 1712 |
))
|
| 1713 |
ending_cells.add((source.get("id"), addr))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1714 |
if ending_cells:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1715 |
_recompute_formulas(book, fr, sheet_mod)
|
| 1716 |
found = sheets.find_tab(book, tab_ref)
|
| 1717 |
out = {}
|
|
@@ -1887,117 +1978,221 @@ def unshare_sheet(workbook_id: str, who: str, session: Session = Depends(require
|
|
| 1887 |
|
| 1888 |
@router.post("/sheets/{workbook_id}/refresh")
|
| 1889 |
def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)):
|
| 1890 |
-
"""
|
| 1891 |
-
|
| 1892 |
-
|
| 1893 |
-
|
| 1894 |
-
refusal on every one of its cells rather than a stale number quietly repainting.
|
| 1895 |
-
|
| 1896 |
-
⚠ Each distinct `view_id` is resolved ONCE, and as of the wave-53 follow-up to W53-T12 that
|
| 1897 |
-
sentence is finally TRUE OF THE CODE. It was written of `seen_db`, which deduplicates the
|
| 1898 |
-
DATABASE WALL and nothing else: the resolve underneath it ran per cell, so a forty-cell
|
| 1899 |
-
workbook did forty `table_store.find_view` calls and forty legacy migrations with them.
|
| 1900 |
-
`view_memo` below is the dedupe this sentence describes.
|
| 1901 |
-
|
| 1902 |
-
⛔⛔ C5 — THIS IS A WRITE DOOR, BECAUSE IT ENDS IN `put_workbook`, AND A VIEWER IS REFUSED
|
| 1903 |
-
HERE FOR A REASON THAT IS NOT ABOUT PERMISSION TO SEE. A viewer whose refresh were allowed
|
| 1904 |
-
would compute every bound cell under THEIR scope and then SAVE those numbers into the owner's
|
| 1905 |
-
document, so a narrowly walled reader pressing Refresh would silently shrink the owner's
|
| 1906 |
-
workbook. The reader's own re-resolve happens on the READ path, in memory, and never lands.
|
| 1907 |
-
⚠ An EDITOR still writes their own numbers here, which is what an edit grant means and is
|
| 1908 |
-
booked as a residual rather than widened away.
|
| 1909 |
"""
|
| 1910 |
-
from
|
|
|
|
| 1911 |
from modules import sheet as sheet_mod
|
| 1912 |
import formula_runtime as fr
|
|
|
|
| 1913 |
|
|
|
|
|
|
|
| 1914 |
try:
|
| 1915 |
-
book = sheets.get_workbook(
|
| 1916 |
-
|
| 1917 |
except sheets.SheetRefusal as exc:
|
| 1918 |
raise _refuse(exc)
|
| 1919 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1920 |
stamp = _now_iso()
|
| 1921 |
-
seen_db = {}
|
| 1922 |
-
|
| 1923 |
-
|
| 1924 |
-
|
| 1925 |
-
|
| 1926 |
-
|
| 1927 |
-
|
| 1928 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1929 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 1930 |
-
if
|
| 1931 |
continue
|
| 1932 |
-
if _is_view_binding(cell):
|
| 1933 |
-
continue # C2: a placed view refreshes through its own door
|
| 1934 |
binding = cell.get("binding") or {}
|
| 1935 |
db = str(binding.get("database") or "")
|
| 1936 |
-
|
| 1937 |
-
|
| 1938 |
-
|
| 1939 |
-
|
| 1940 |
-
|
| 1941 |
-
seen_db[db] = exc
|
| 1942 |
-
failure = seen_db.get(db)
|
| 1943 |
-
if failure is not None:
|
| 1944 |
-
cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}."))
|
| 1945 |
-
cell.pop("limits", None)
|
| 1946 |
-
continue
|
| 1947 |
try:
|
| 1948 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1949 |
except HTTPException as exc:
|
| 1950 |
-
|
| 1951 |
-
|
| 1952 |
-
|
| 1953 |
-
|
| 1954 |
-
|
| 1955 |
-
|
| 1956 |
-
|
| 1957 |
-
|
| 1958 |
-
|
| 1959 |
-
|
| 1960 |
-
|
| 1961 |
-
|
| 1962 |
-
|
| 1963 |
-
|
| 1964 |
-
|
| 1965 |
-
|
| 1966 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1967 |
|
| 1968 |
-
# ⭐⭐ R4 / A1: THE RECALC IS TIMED AND THE TIME IS SHOWN. The prototype measured 2.9 s as the
|
| 1969 |
-
# FLOOR for the owner's 11,300 formulas (2.9 to 6.2 s observed), so a refresh is a wait a
|
| 1970 |
-
# person notices, and a wait nobody explains reads as a hang. `recalc_ms` is what the client
|
| 1971 |
-
# renders as "Recalculated in N.N s".
|
| 1972 |
-
#
|
| 1973 |
-
# ⚠ IT TIMES THE RECALC, NOT THE ROUTE. The bound-cell loop above is database work whose
|
| 1974 |
-
# duration says nothing about the formula engine, and the save below is store work; folding
|
| 1975 |
-
# either into this number would make the figure on the screen unable to answer the question it
|
| 1976 |
-
# exists for, which is whether the ENGINE is inside A1's 5 s target.
|
| 1977 |
-
from time import perf_counter
|
| 1978 |
started = perf_counter()
|
| 1979 |
_recompute_formulas(book, fr, sheet_mod)
|
| 1980 |
recalc_ms = round((perf_counter() - started) * 1000.0, 1)
|
| 1981 |
-
|
| 1982 |
-
|
| 1983 |
-
|
| 1984 |
-
|
| 1985 |
-
|
| 1986 |
-
|
| 1987 |
-
|
| 1988 |
-
|
| 1989 |
-
|
| 1990 |
-
|
| 1991 |
-
|
| 1992 |
-
|
| 1993 |
-
|
| 1994 |
-
|
| 1995 |
-
return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms
|
|
|
|
| 1996 |
|
| 1997 |
|
| 1998 |
def _now_iso():
|
| 1999 |
-
from
|
| 2000 |
-
return
|
| 2001 |
|
| 2002 |
|
| 2003 |
#: ⛔ THE NAME A TAB GETS WHEN IT CANNOT KEEP ITS OWN, and it is built to be UNTYPEABLE on two
|
|
@@ -2288,8 +2483,11 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
|
|
| 2288 |
refuse("the cell or its sheet is missing or blank.")
|
| 2289 |
if cell.get("kind") == "bound":
|
| 2290 |
try:
|
| 2291 |
-
|
| 2292 |
-
|
|
|
|
|
|
|
|
|
|
| 2293 |
except HTTPException as exc:
|
| 2294 |
reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
|
| 2295 |
refuse(reason)
|
|
@@ -2317,8 +2515,11 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
|
|
| 2317 |
for dependency in needed:
|
| 2318 |
attempted.add(dependency)
|
| 2319 |
try:
|
| 2320 |
-
|
| 2321 |
-
|
|
|
|
|
|
|
|
|
|
| 2322 |
changes[dependency] = {"value": result["value"]}
|
| 2323 |
except HTTPException as exc:
|
| 2324 |
info = ((exc.detail or {}).get("error") or {})
|
|
@@ -2389,6 +2590,116 @@ def _resolve_cell_binding(session, binding, book, tab, **kwargs):
|
|
| 2389 |
return _resolve_binding(session, binding, **kwargs)
|
| 2390 |
|
| 2391 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2392 |
def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
|
| 2393 |
book=None, tab=None, ending_seen=()):
|
| 2394 |
"""Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
|
|
@@ -2450,6 +2761,9 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 2450 |
if binding["aggregate"].get("field"):
|
| 2451 |
needed.add(str(binding["aggregate"]["field"]))
|
| 2452 |
|
|
|
|
|
|
|
|
|
|
| 2453 |
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 2454 |
cols = dict(cols)
|
| 2455 |
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
|
|
@@ -2491,9 +2805,12 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 2491 |
# spec's column expressions; applying it a second time would double-count the exclusion and
|
| 2492 |
# break every joined spec, whose partner column is `sol.order_partner_id` or `aml.partner_id`.
|
| 2493 |
try:
|
|
|
|
|
|
|
|
|
|
| 2494 |
sql, params, limits = sheet_mod.build_aggregate_query(
|
| 2495 |
spec, binding, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
|
| 2496 |
-
wall_nodes=wall_nodes, wall_conj=wall_conj, today=today)
|
| 2497 |
except sheet_mod.SheetRefusal as exc:
|
| 2498 |
raise err(400, exc.code, exc.message)
|
| 2499 |
except ValueError:
|
|
@@ -2534,6 +2851,50 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 2534 |
return {"value": value, "limits": limits, "database": database, "binding": binding}
|
| 2535 |
|
| 2536 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2537 |
def _fe_parts(tree):
|
| 2538 |
"""`(nodes, conj)` for a stored `{conj, nodes}` filter, through the ONE reader of that shape."""
|
| 2539 |
from modules import sheet as sheet_mod
|
|
@@ -2716,6 +3077,11 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 2716 |
saved["refreshed_at"] = stamp
|
| 2717 |
saved.pop("refusal", None)
|
| 2718 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2719 |
try:
|
| 2720 |
landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved},
|
| 2721 |
actor=session.uname,
|
|
@@ -2735,6 +3101,13 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 2735 |
except Exception: # noqa: BLE001
|
| 2736 |
pass
|
| 2737 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2738 |
wire = dict(saved)
|
| 2739 |
#: ⚠ `limits` RIDES THE ANSWER AND IS NOT STORED, because `core.sheets.clean_cell` carries no
|
| 2740 |
#: such key: it describes THIS resolve's truncation, not the cell.
|
|
@@ -2916,24 +3289,35 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
|
|
| 2916 |
window = _ending_window(session, binding, book=book, tab=tab.get("id"))
|
| 2917 |
except sheet_mod.SheetRefusal as exc:
|
| 2918 |
raise err(400, exc.code, exc.message)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2919 |
cur, spec, cols = _mirror(session)(
|
| 2920 |
database, metric_window=window,
|
| 2921 |
-
needed=
|
| 2922 |
-
| sheet_mod.filter_columns_named(wall_nodes)))
|
| 2923 |
cols = dict(cols)
|
| 2924 |
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols)
|
| 2925 |
|
| 2926 |
try:
|
|
|
|
|
|
|
|
|
|
| 2927 |
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
|
| 2928 |
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 2929 |
-
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj)
|
|
|
|
|
|
|
|
|
|
| 2930 |
except sheet_mod.SheetRefusal as exc:
|
| 2931 |
raise err(400, exc.code, exc.message)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2932 |
total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
|
| 2933 |
|
| 2934 |
-
ids_sql, ids_params, _ = sheet_mod.build_ids_query(
|
| 2935 |
-
spec, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
|
| 2936 |
-
wall_nodes=wall_nodes, wall_conj=wall_conj)
|
| 2937 |
record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
|
| 2938 |
|
| 2939 |
out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
|
|
|
|
| 44 |
import uuid
|
| 45 |
|
| 46 |
from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
|
| 47 |
+
from fastapi.responses import Response
|
| 48 |
|
| 49 |
from deps import Session, err, require_session
|
| 50 |
|
|
|
|
| 975 |
import core.perm_scope as perm_scope
|
| 976 |
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 977 |
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 978 |
+
from core.sheet_binding import fingerprint
|
| 979 |
+
# Compare effective access, including field grants. Equivalent callers may share a
|
| 980 |
+
# snapshot unless a personal workspace makes their database values caller-specific.
|
| 981 |
+
memo = getattr(session, "_sheet_scope_context", None)
|
| 982 |
+
if memo is None:
|
| 983 |
+
memo = session._sheet_scope_context = {}
|
| 984 |
+
if database not in memo:
|
| 985 |
+
from core import shared_overlay, shares, user_tables, view_templates
|
| 986 |
+
st = _store(session)
|
| 987 |
+
try:
|
| 988 |
+
if "_definitions" not in memo:
|
| 989 |
+
memo["_definitions"] = st.get_projection(user_tables.STORE_KEY, drop=("rows",)) or {}
|
| 990 |
+
memo["_grants"] = fingerprint((st.get(shares.SHARES_KEY, fresh=True) or {}).get("field"))
|
| 991 |
+
definition = memo["_definitions"].get(database) or {}
|
| 992 |
+
workspace = view_templates.workspace_key(database)
|
| 993 |
+
personal = (st.get_projection(workspace, drop=("overlays",)) or {}).get(session.uname) or {}
|
| 994 |
+
shared_key = database if database.startswith("ut_") else workspace
|
| 995 |
+
fields = {f["key"]: f for f in definition.get("fields", []) if f.get("key")}
|
| 996 |
+
fields.update(shared_overlay.fields(shared_key, st=st))
|
| 997 |
+
fields.update(personal.get("fields") or {})
|
| 998 |
+
memo[database] = [sorted(perm_scope.hidden_keys(session.user, database,
|
| 999 |
+
list(fields.values()), st=st)), session.uname if personal else None]
|
| 1000 |
+
except Exception:
|
| 1001 |
+
# An unprovable wall cannot read a previous slot.
|
| 1002 |
+
memo[database] = uuid.uuid4().hex
|
| 1003 |
+
policy = {key: value for key, value in session.user.items()
|
| 1004 |
+
if key not in ("username", "name", "email", "password", "password_hash", "epoch",
|
| 1005 |
+
"created_at", "updated_at", "last_login")}
|
| 1006 |
+
return fingerprint([sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {}),
|
| 1007 |
+
binding, policy, memo[database], memo.get("_grants")])
|
| 1008 |
|
| 1009 |
|
| 1010 |
def _has_ending(binding):
|
|
|
|
| 1018 |
return True
|
| 1019 |
scope = binding.get("scope") if isinstance(binding, dict) else None
|
| 1020 |
nodes, _ = _fe_parts(scope.get("filter") if isinstance(scope, dict) else None)
|
| 1021 |
+
related = binding.get("related") if isinstance(binding, dict) else None
|
| 1022 |
+
related_nodes, _ = _fe_parts(related.get("filter") if isinstance(related, dict) else None)
|
| 1023 |
+
nodes = [*nodes, *related_nodes]
|
| 1024 |
# Linked values and related-table grants can change independently of the orders' cache key.
|
| 1025 |
if sheet_mod.linked_filter_ids(nodes):
|
| 1026 |
return True
|
| 1027 |
return any(sheet_mod.filter_cell_nodes(nodes))
|
| 1028 |
|
| 1029 |
|
| 1030 |
+
def _refresh_context(book, binding, memo=None):
|
| 1031 |
+
"""Authored inputs and relative dates invalidate refresh, but never an open's snapshot."""
|
| 1032 |
+
from core.sheet_binding import fingerprint
|
| 1033 |
+
from datetime import date
|
| 1034 |
+
payload = {}
|
| 1035 |
+
if _has_cell_inputs(binding):
|
| 1036 |
+
if memo is None:
|
| 1037 |
+
memo = {}
|
| 1038 |
+
if "inputs" not in memo:
|
| 1039 |
+
memo["inputs"] = fingerprint([[tab.get("id"), tab.get("name"), {
|
| 1040 |
+
addr: {key: cell.get(key) for key in ("kind", "text", "formula", "binding", "value")
|
| 1041 |
+
if key != "value" or cell.get("kind") in ("literal", "empty")}
|
| 1042 |
+
for addr, cell in (tab.get("cells") or {}).items()}]
|
| 1043 |
+
for tab in book.get("sheets") or []])
|
| 1044 |
+
payload["inputs"] = memo["inputs"]
|
| 1045 |
+
# Relative filters and measure windows can change their answer at midnight without a write.
|
| 1046 |
+
encoded = json.dumps(binding, sort_keys=True)
|
| 1047 |
+
if any(word in encoded for word in ('"window"', '"within"', '"ending"', '"view_id"')):
|
| 1048 |
+
payload["today"] = date.today().isoformat()
|
| 1049 |
+
return fingerprint(payload)
|
| 1050 |
+
|
| 1051 |
+
|
| 1052 |
+
def _snapshot_cell(cell, snapshot):
|
| 1053 |
+
from copy import deepcopy
|
| 1054 |
+
for key in ("value", "display", "state", "refreshed_at", "refusal", "limits"):
|
| 1055 |
+
cell.pop(key, None)
|
| 1056 |
+
cell.update(deepcopy(snapshot))
|
| 1057 |
+
|
| 1058 |
+
|
| 1059 |
def _reresolved_for_reader(session: Session, book, *, endings_only=False):
|
| 1060 |
+
"""C2/R4: project last resolved values only. This GET path never resolves a binding."""
|
| 1061 |
+
from copy import deepcopy
|
| 1062 |
+
from core import sheet_binding
|
| 1063 |
+
session._sheet_scope_context = {}
|
| 1064 |
+
fresh = deepcopy(book)
|
| 1065 |
+
owner = _role_on_workbook(session, book) == "owner"
|
| 1066 |
+
try:
|
| 1067 |
+
cache = sheet_binding.read_refresh_cache(_store(session), book["id"])
|
| 1068 |
+
except Exception:
|
| 1069 |
+
cache = {}
|
| 1070 |
+
seen = {}
|
| 1071 |
+
changed_values = False
|
| 1072 |
+
for tab in fresh.get("sheets") or []:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1073 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 1074 |
+
if cell.get("kind") != "bound":
|
| 1075 |
continue
|
| 1076 |
+
if _is_view_binding(cell):
|
| 1077 |
continue
|
| 1078 |
+
before = (cell.get("value"), cell.get("state"))
|
| 1079 |
binding = cell.get("binding") or {}
|
| 1080 |
db = str(binding.get("database") or "")
|
| 1081 |
+
if db not in seen:
|
| 1082 |
try:
|
| 1083 |
_require_database(session, db)
|
| 1084 |
+
seen[db] = True
|
| 1085 |
+
except HTTPException:
|
| 1086 |
+
seen[db] = False
|
| 1087 |
+
slot = _binding_scope_key(session, db, binding)
|
| 1088 |
+
hit = (cache.get("refreshCells", {}).get(str(tab.get("id")) + "!" + addr, {})
|
| 1089 |
+
.get(slot)) if seen[db] else None
|
| 1090 |
+
if not hit and seen[db]:
|
| 1091 |
+
# ⛔⛔ THE BRIDGE S65-T03 LEFT OUT (SDLC 65 combine). It moved this reader onto
|
| 1092 |
+
# `sheet_binding.read_refresh_cache` while `core.sheets`' scope cache kept its two
|
| 1093 |
+
# writers: `compute_cell` fills BOTH, but the MCP gateway
|
| 1094 |
+
# (`routes_external.py::cache_binding`) fills only the legacy one. So a binding
|
| 1095 |
+
# written through MCP was invisible to its own next read and answered
|
| 1096 |
+
# `sheet_refresh_required` with NO VALUE. Measured by `verify_perm_scope.py::An
|
| 1097 |
+
# editor binding write replaces the old scope cache before reading its result`:
|
| 1098 |
+
# got None, want 75.
|
| 1099 |
+
# ⭐ A READ, never a second write: the legacy slot is keyed by the SAME
|
| 1100 |
+
# `_binding_scope_key`, so C5's per-scope isolation is the one this already
|
| 1101 |
+
# enforces, and nothing here resolves a binding (C2/R4 is untouched). Writing the
|
| 1102 |
+
# new cache from the gateway instead was tried and REJECTED: it makes
|
| 1103 |
+
# `refreshCells` non-empty, which retires the pre-C2 passthrough below for every
|
| 1104 |
+
# OTHER cell and blanked a bound cell that had a perfectly good stored value.
|
| 1105 |
try:
|
| 1106 |
+
from core import sheets as _sheets
|
| 1107 |
+
legacy = _sheets.cached_scope_value(
|
| 1108 |
+
_store(session), book["id"], str(tab.get("id")) + "!" + addr, slot)
|
| 1109 |
+
except Exception:
|
| 1110 |
+
legacy = None
|
| 1111 |
+
if legacy is not None:
|
| 1112 |
+
value = legacy.get("value")
|
| 1113 |
+
hit = {"cell": {"value": value, "state": "fresh",
|
| 1114 |
+
"display": "" if value is None else str(value),
|
| 1115 |
+
"refreshed_at": legacy.get("refreshedAt")}}
|
| 1116 |
+
if hit:
|
| 1117 |
+
_snapshot_cell(cell, hit["cell"])
|
| 1118 |
+
elif not seen[db]:
|
| 1119 |
+
_snapshot_cell(cell, _binding_refusal("database_unreadable", f"You cannot read {db}."))
|
| 1120 |
+
elif owner and not cache.get("refreshCells") and session.user.get("role") == "admin":
|
| 1121 |
+
# Existing workbooks keep their pre-C2 last result until the first Refresh.
|
| 1122 |
+
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1123 |
else:
|
| 1124 |
+
_snapshot_cell(cell, _binding_refusal("sheet_refresh_required",
|
| 1125 |
+
"Refresh this workbook to resolve this cell under your current access."))
|
| 1126 |
+
changed_values |= before != (cell.get("value"), cell.get("state"))
|
| 1127 |
+
# ⛔⛔ A STALE FORMULA IS A REASON TO RECOMPUTE TOO, AND MISSING IT MADE THE TWO SERVER READS
|
| 1128 |
+
# DISAGREE (SDLC 65 combine). C2/R4 says this GET path never RESOLVES A BINDING, and it still
|
| 1129 |
+
# does not: `_recompute_formulas` evaluates the formula graph over cell values that are already
|
| 1130 |
+
# present and opens no database. Gating it on `changed_values` alone meant that after a
|
| 1131 |
+
# structure edit -- which stores every dependent formula `stale` with no value and moves no
|
| 1132 |
+
# bound cell -- the UI read returned `{'kind': 'formula', 'state': 'stale'}` with no figure
|
| 1133 |
+
# while `sheet_read_cells` (MCP) returned the computed one. `verify_perm_scope.py::The UI read
|
| 1134 |
+
# and MCP read agree on stored cells, figures and formats` measured exactly that pair: D1 and
|
| 1135 |
+
# F1 fresh 30.0 / 60.0 from MCP against stale-and-blank from the UI. A non-browser consumer has
|
| 1136 |
+
# no client engine to cover for it, so the blank is what it would have shown.
|
| 1137 |
+
stale_formula = any(
|
| 1138 |
+
cell.get("kind") == "formula" and (cell.get("state") != "fresh" or "value" not in cell)
|
| 1139 |
+
for tab in (fresh.get("sheets") or []) for cell in (tab.get("cells") or {}).values())
|
| 1140 |
+
if changed_values or stale_formula:
|
| 1141 |
+
from modules import sheet as sheet_mod
|
| 1142 |
+
import formula_runtime as fr
|
| 1143 |
+
_recompute_formulas(fresh, fr, sheet_mod)
|
| 1144 |
+
if not owner:
|
| 1145 |
+
fresh["refreshedFor"] = str(session.uname or "")
|
| 1146 |
return fresh
|
| 1147 |
|
| 1148 |
|
|
|
|
| 1154 |
|
| 1155 |
|
| 1156 |
def _read_for(session: Session, book):
|
| 1157 |
+
"""Project saved results under the current access wall without resolving any binding.
|
| 1158 |
|
| 1159 |
W60-T09, 2026-09-14: R28 checks visibility before this projection. Admitted legacy readers
|
| 1160 |
+
and grantees receive a proven scoped snapshot or a request to Refresh.
|
| 1161 |
"""
|
| 1162 |
book = _response_workbook(book)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1163 |
return _reresolved_for_reader(session, book)
|
| 1164 |
|
| 1165 |
|
|
|
|
| 1512 |
if not name.lower().endswith(".xlsx"):
|
| 1513 |
raise err(400, "import_not_xlsx",
|
| 1514 |
"Only an .xlsx workbook can be imported. Save your file as .xlsx and try again.")
|
| 1515 |
+
# Read at most one byte past the cap. UploadFile may be backed by a large temporary file; an
|
| 1516 |
+
# unbounded read would allocate hostile input merely to discover that it is too large.
|
| 1517 |
+
data = await file.read(sheet_import.MAX_IMPORT_BYTES + 1)
|
| 1518 |
if len(data) > sheet_import.MAX_IMPORT_BYTES:
|
| 1519 |
raise HTTPException(status_code=413, detail={"error": {
|
| 1520 |
"code": "import_too_large",
|
|
|
|
| 1560 |
return "%.1f MB" % (float(byte_count) / (1024.0 * 1024.0))
|
| 1561 |
|
| 1562 |
|
| 1563 |
+
# ── C5: Excel-shaped bounded automation (S63-T22) ──────────────────────────────────────────
|
| 1564 |
+
|
| 1565 |
+
@router.get("/sheets/automation/manifest")
|
| 1566 |
+
def sheet_automation_manifest(session: Session = Depends(require_session)):
|
| 1567 |
+
"""The canonical capability bytes shared with the Python SDK.
|
| 1568 |
+
|
| 1569 |
+
The session wall is deliberate: this is product capability metadata, but keeping it beside
|
| 1570 |
+
the authenticated workbook doors avoids creating a second unauthenticated API convention.
|
| 1571 |
+
"""
|
| 1572 |
+
del session
|
| 1573 |
+
from core import sheet_compat
|
| 1574 |
+
return Response(content=sheet_compat.capability_manifest_bytes(), media_type="application/json")
|
| 1575 |
+
|
| 1576 |
+
|
| 1577 |
+
def _sheet_automation_calculator(book):
|
| 1578 |
+
"""Inject the existing formula runtime without making ``platform/core`` import the API."""
|
| 1579 |
+
from modules import sheet as sheet_mod
|
| 1580 |
+
import formula_runtime as fr
|
| 1581 |
+
_recompute_formulas(book, fr, sheet_mod)
|
| 1582 |
+
|
| 1583 |
+
|
| 1584 |
+
@router.post("/sheets/{workbook_id}/automation")
|
| 1585 |
+
def execute_sheet_automation(workbook_id: str, body: dict = Body(default=None),
|
| 1586 |
+
session: Session = Depends(require_session)):
|
| 1587 |
+
"""Execute one manifest operation through the shared core planner and targeted patcher."""
|
| 1588 |
+
from core import sheet_compat, sheets
|
| 1589 |
+
raw = dict(body or {})
|
| 1590 |
+
operation = str(raw.get("operation") or "")
|
| 1591 |
+
arguments = raw.get("arguments")
|
| 1592 |
+
try:
|
| 1593 |
+
# Unsupported and malformed operation-level calls stop here, before any store update
|
| 1594 |
+
# callback can be entered. Authorization then applies to the declared access class.
|
| 1595 |
+
spec = sheet_compat.require_supported(operation)
|
| 1596 |
+
book = sheets.get_workbook(_store(session), workbook_id)
|
| 1597 |
+
_require_reader(session, book)
|
| 1598 |
+
if spec["access"] == "write":
|
| 1599 |
+
_require_editor(session, book)
|
| 1600 |
+
return sheet_compat.execute(
|
| 1601 |
+
_store(session), workbook_id, operation, arguments,
|
| 1602 |
+
actor=session.uname, actor_name=session.user.get("name") or session.uname,
|
| 1603 |
+
calculator=_sheet_automation_calculator)
|
| 1604 |
+
except sheets.SheetRefusal as exc:
|
| 1605 |
+
raise _refuse(exc)
|
| 1606 |
+
|
| 1607 |
+
|
| 1608 |
@router.get("/sheets/{workbook_id}")
|
| 1609 |
def read_sheet(workbook_id: str, tabs: str = "",
|
| 1610 |
session: Session = Depends(require_session)):
|
|
|
|
| 1770 |
cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
|
| 1771 |
# Date and filter-cell edits also change database totals. Recompute all readers, then formulas.
|
| 1772 |
ending_cells = set()
|
| 1773 |
+
refreshed_entries = []
|
| 1774 |
+
context_memo = {}
|
| 1775 |
for source in book.get("sheets") or []:
|
| 1776 |
for addr, cell in (source.get("cells") or {}).items():
|
| 1777 |
binding = cell.get("binding") or {}
|
|
|
|
| 1780 |
try:
|
| 1781 |
result = _resolve_cell_binding(session, binding, book, source.get("id"))
|
| 1782 |
cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
|
| 1783 |
+
cell["refreshed_at"] = _now_iso()
|
| 1784 |
+
if result.get("limits"):
|
| 1785 |
+
cell["limits"] = result["limits"]
|
| 1786 |
+
else:
|
| 1787 |
+
cell.pop("limits", None)
|
| 1788 |
cell.pop("refusal", None)
|
| 1789 |
except HTTPException as exc:
|
| 1790 |
info = ((exc.detail or {}).get("error") or {})
|
|
|
|
| 1793 |
info.get("message", "The Ending date could not be read. Choose a valid date."),
|
| 1794 |
))
|
| 1795 |
ending_cells.add((source.get("id"), addr))
|
| 1796 |
+
refreshed_entries.append((str(source.get("id")) + "!" + addr,
|
| 1797 |
+
_binding_scope_key(session, str(binding.get("database") or ""), binding),
|
| 1798 |
+
{"tokens": {}, "context": _refresh_context(book, binding, context_memo),
|
| 1799 |
+
"cell": {k: cell[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
|
| 1800 |
+
if k in cell}}))
|
| 1801 |
if ending_cells:
|
| 1802 |
+
from core.sheet_binding import save_refresh_cache
|
| 1803 |
+
save_refresh_cache(_store(session), workbook_id, refreshed_entries,
|
| 1804 |
+
{c.get("binding", {}).get("database") for t in book.get("sheets", [])
|
| 1805 |
+
for c in t.get("cells", {}).values() if c.get("kind") == "bound"})
|
| 1806 |
_recompute_formulas(book, fr, sheet_mod)
|
| 1807 |
found = sheets.find_tab(book, tab_ref)
|
| 1808 |
out = {}
|
|
|
|
| 1978 |
|
| 1979 |
@router.post("/sheets/{workbook_id}/refresh")
|
| 1980 |
def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)):
|
| 1981 |
+
"""C2/R4: Refresh resolves only databases whose token or binding inputs changed.
|
| 1982 |
+
|
| 1983 |
+
Viewers refresh their own server cache. Only editors save the workbook document.
|
| 1984 |
+
One token is read per distinct database on the unchanged path, regardless of cell count.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1985 |
"""
|
| 1986 |
+
from copy import deepcopy
|
| 1987 |
+
from core import sheets, sheet_binding
|
| 1988 |
from modules import sheet as sheet_mod
|
| 1989 |
import formula_runtime as fr
|
| 1990 |
+
from time import perf_counter
|
| 1991 |
|
| 1992 |
+
store = _store(session)
|
| 1993 |
+
session._sheet_scope_context = {}
|
| 1994 |
try:
|
| 1995 |
+
book = sheets.get_workbook(store, workbook_id)
|
| 1996 |
+
_require_reader(session, book)
|
| 1997 |
except sheets.SheetRefusal as exc:
|
| 1998 |
raise _refuse(exc)
|
| 1999 |
+
editable = _role_on_workbook(session, book) in ("owner", "edit")
|
| 2000 |
+
try:
|
| 2001 |
+
cache = sheet_binding.read_refresh_cache(store, workbook_id)
|
| 2002 |
+
except Exception:
|
| 2003 |
+
cache = {}
|
| 2004 |
stamp = _now_iso()
|
| 2005 |
+
seen_db, tokens, entries, status = {}, {}, [], {}
|
| 2006 |
+
view_memo, related_memo = {}, {}
|
| 2007 |
+
context_memo = {}
|
| 2008 |
+
token_reads = 0
|
| 2009 |
+
input_graph = None
|
| 2010 |
+
|
| 2011 |
+
def input_databases(binding, tab_ref):
|
| 2012 |
+
nonlocal input_graph
|
| 2013 |
+
nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
|
| 2014 |
+
related_nodes, _ = _fe_parts((binding.get("related") or {}).get("filter"))
|
| 2015 |
+
refs = [str((node.get("valueRef") or {}).get("ref") or "")
|
| 2016 |
+
for node in sheet_mod.filter_cell_nodes([*nodes, *related_nodes])]
|
| 2017 |
+
period = binding.get("period") or {}
|
| 2018 |
+
if period.get("ref"):
|
| 2019 |
+
refs.append(period["ref"])
|
| 2020 |
+
if not refs:
|
| 2021 |
+
return set()
|
| 2022 |
+
tabs = book.get("sheets") or []
|
| 2023 |
+
names = _tab_scope_names(tabs)
|
| 2024 |
+
if input_graph is None:
|
| 2025 |
+
documents = {fr.a1_key(addr, name): cell for tab, name in zip(tabs, names)
|
| 2026 |
+
for addr, cell in (tab.get("cells") or {}).items()}
|
| 2027 |
+
engine = fr.SheetCalculation({key: fr._document_spec(cell, key.rpartition("!")[0])
|
| 2028 |
+
for key, cell in documents.items()}, sheets=names,
|
| 2029 |
+
named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
|
| 2030 |
+
input_graph = documents, engine
|
| 2031 |
+
documents, engine = input_graph
|
| 2032 |
+
home = _resolve_home_tab(book, tabs, names, tab_ref)
|
| 2033 |
+
pending, visited, deps = [_asked_key(ref, home, fr) for ref in refs], set(), set()
|
| 2034 |
+
while pending:
|
| 2035 |
+
key = pending.pop()
|
| 2036 |
+
if key in visited:
|
| 2037 |
+
continue
|
| 2038 |
+
visited.add(key)
|
| 2039 |
+
cell = documents.get(key) or {}
|
| 2040 |
+
if cell.get("kind") == "bound":
|
| 2041 |
+
deps.add(str((cell.get("binding") or {}).get("database") or ""))
|
| 2042 |
+
pending.extend(engine.consumed.get(key, ()))
|
| 2043 |
+
return deps
|
| 2044 |
+
|
| 2045 |
+
def token(db):
|
| 2046 |
+
nonlocal token_reads
|
| 2047 |
+
if db not in tokens:
|
| 2048 |
+
token_reads += 1
|
| 2049 |
+
try:
|
| 2050 |
+
# ⭐ Arms `core.sheet_binding.mirror_change_token` (SDLC 65 combine). The mirror
|
| 2051 |
+
# reader lives in `harness` because `core` never imports up, and the API is the
|
| 2052 |
+
# layer allowed to reach both ways, so the wiring belongs exactly here.
|
| 2053 |
+
import harness.mirror_tokens # noqa: F401 (registers the provider on import)
|
| 2054 |
+
tokens[db] = sheet_binding.database_change_token(store, db)
|
| 2055 |
+
except Exception:
|
| 2056 |
+
# Failure to prove unchanged means resolve, never silently reuse a value.
|
| 2057 |
+
tokens[db] = None
|
| 2058 |
+
return tokens[db]
|
| 2059 |
+
|
| 2060 |
+
def dependencies(db, binding, tab_ref):
|
| 2061 |
+
deps = {db}
|
| 2062 |
+
if binding.get("related"):
|
| 2063 |
+
key = (db, binding["related"].get("via"))
|
| 2064 |
+
if key not in related_memo:
|
| 2065 |
+
related_memo[key] = _binding_related_entry(session, db, binding)
|
| 2066 |
+
deps.add(related_memo[key]["target"])
|
| 2067 |
+
nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
|
| 2068 |
+
linked = sheet_mod.linked_filter_ids(nodes)
|
| 2069 |
+
if linked:
|
| 2070 |
+
names = {sheet_mod.linked_field_parts(key)[0] for key in linked}
|
| 2071 |
+
deps.update(b["source"] for b in _linked_customer_bridges(db) if b["key"] in names)
|
| 2072 |
+
deps.update(input_databases(binding, tab_ref))
|
| 2073 |
+
return deps
|
| 2074 |
+
|
| 2075 |
+
bound_inputs = {fr.a1_key(addr, name): (tab.get("id"), addr, cell)
|
| 2076 |
+
for tab, name in zip(book.get("sheets") or [], _tab_scope_names(book.get("sheets") or []))
|
| 2077 |
+
for addr, cell in (tab.get("cells") or {}).items() if cell.get("kind") == "bound"}
|
| 2078 |
+
|
| 2079 |
+
def cached_input(key):
|
| 2080 |
+
item = bound_inputs.get(key)
|
| 2081 |
+
if item is None:
|
| 2082 |
+
return None
|
| 2083 |
+
tab_ref, addr, cell = item
|
| 2084 |
+
binding = cell.get("binding") or {}
|
| 2085 |
+
db = str(binding.get("database") or "")
|
| 2086 |
+
_require_database(session, db)
|
| 2087 |
+
scope_key = _binding_scope_key(session, db, binding)
|
| 2088 |
+
hit = cache.get("refreshCells", {}).get(str(tab_ref) + "!" + addr, {}).get(scope_key)
|
| 2089 |
+
sources = {db: token(db) for db in dependencies(db, binding, tab_ref)}
|
| 2090 |
+
if (hit and all(sources.values()) and hit.get("tokens") == sources
|
| 2091 |
+
and hit.get("context") == _refresh_context(book, binding, context_memo)
|
| 2092 |
+
and hit["cell"].get("state") == "fresh"):
|
| 2093 |
+
return hit["cell"]
|
| 2094 |
+
return None
|
| 2095 |
+
|
| 2096 |
+
for tab in book.get("sheets") or []:
|
| 2097 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 2098 |
+
if cell.get("kind") != "bound" or _is_view_binding(cell):
|
| 2099 |
continue
|
|
|
|
|
|
|
| 2100 |
binding = cell.get("binding") or {}
|
| 2101 |
db = str(binding.get("database") or "")
|
| 2102 |
+
cell_key = str(tab.get("id")) + "!" + addr
|
| 2103 |
+
scope_key = _binding_scope_key(session, db, binding)
|
| 2104 |
+
context = _refresh_context(book, binding, context_memo)
|
| 2105 |
+
hit = cache.get("refreshCells", {}).get(cell_key, {}).get(scope_key)
|
| 2106 |
+
source_tokens = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2107 |
try:
|
| 2108 |
+
if db not in seen_db:
|
| 2109 |
+
try:
|
| 2110 |
+
_require_database(session, db)
|
| 2111 |
+
seen_db[db] = None
|
| 2112 |
+
except HTTPException as exc:
|
| 2113 |
+
seen_db[db] = exc
|
| 2114 |
+
if seen_db[db] is not None:
|
| 2115 |
+
raise seen_db[db]
|
| 2116 |
+
source_tokens = {key: token(key) for key in sorted(dependencies(db, binding, tab.get("id")))}
|
| 2117 |
+
if (hit and all(source_tokens.values()) and hit.get("tokens") == source_tokens
|
| 2118 |
+
and hit.get("context") == context
|
| 2119 |
+
and hit["cell"].get("state") == "fresh"):
|
| 2120 |
+
_snapshot_cell(cell, hit["cell"])
|
| 2121 |
+
status.setdefault(db, "skipped")
|
| 2122 |
+
continue
|
| 2123 |
+
status[db] = "resolved"
|
| 2124 |
+
previous = getattr(session, "_sheet_refresh_input", None)
|
| 2125 |
+
session._sheet_refresh_input = cached_input
|
| 2126 |
+
try:
|
| 2127 |
+
out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
|
| 2128 |
+
finally:
|
| 2129 |
+
session._sheet_refresh_input = previous
|
| 2130 |
+
limits = list(out.get("limits") or [])
|
| 2131 |
+
if not all(source_tokens.values()):
|
| 2132 |
+
limits.append({"code": "change_token_unavailable",
|
| 2133 |
+
"cause": "The database change token could not be read. This Refresh resolved the cell.",
|
| 2134 |
+
"recommendation": "Restore the database store and connector sync receipts to enable skipping."})
|
| 2135 |
+
snapshot = {"state": "fresh", "value": out["value"],
|
| 2136 |
+
"display": "" if out["value"] is None else str(out["value"]),
|
| 2137 |
+
"refreshed_at": stamp}
|
| 2138 |
+
if limits:
|
| 2139 |
+
snapshot["limits"] = limits
|
| 2140 |
except HTTPException as exc:
|
| 2141 |
+
info = ((exc.detail or {}).get("error") or {})
|
| 2142 |
+
snapshot = _binding_refusal(info.get("code", "sheet_unresolved"),
|
| 2143 |
+
info.get("message", "This cell could not be refreshed."))
|
| 2144 |
+
status[db] = "refused"
|
| 2145 |
+
_snapshot_cell(cell, snapshot)
|
| 2146 |
+
entries.append((cell_key, scope_key, {"tokens": source_tokens,
|
| 2147 |
+
"context": context, "cell": deepcopy(snapshot)}))
|
| 2148 |
+
|
| 2149 |
+
# Read again only after resolving. A concurrent write must not certify a mixed result
|
| 2150 |
+
# as reusable. The unchanged path still costs exactly one token read per database.
|
| 2151 |
+
resolved_sources = {db for _, _, entry in entries for db in entry["tokens"]}
|
| 2152 |
+
changed = set()
|
| 2153 |
+
for db in resolved_sources:
|
| 2154 |
+
if tokens[db] is None:
|
| 2155 |
+
continue # Already disclosed as unverified and never eligible for a hit.
|
| 2156 |
+
token_reads += 1
|
| 2157 |
+
try:
|
| 2158 |
+
if sheet_binding.database_change_token(store, db) != tokens[db]:
|
| 2159 |
+
changed.add(db)
|
| 2160 |
+
except Exception:
|
| 2161 |
+
changed.add(db)
|
| 2162 |
+
for cell_key, _, entry in entries:
|
| 2163 |
+
if changed.intersection(entry["tokens"]):
|
| 2164 |
+
entry["tokens"] = {} # no next-refresh hit, even if the writer rolls back
|
| 2165 |
+
if entry["cell"].get("state") == "fresh":
|
| 2166 |
+
entry["cell"].setdefault("limits", []).append({"code": "change_during_refresh",
|
| 2167 |
+
"cause": "This database could not be confirmed unchanged while the cell was being resolved.",
|
| 2168 |
+
"recommendation": "Refresh again to resolve the cell from the updated database."})
|
| 2169 |
+
tab_ref, addr = cell_key.split("!", 1)
|
| 2170 |
+
_snapshot_cell(sheets.find_tab(book, tab_ref)["cells"][addr], entry["cell"])
|
| 2171 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2172 |
started = perf_counter()
|
| 2173 |
_recompute_formulas(book, fr, sheet_mod)
|
| 2174 |
recalc_ms = round((perf_counter() - started) * 1000.0, 1)
|
| 2175 |
+
if editable:
|
| 2176 |
+
try:
|
| 2177 |
+
saved = sheets.put_workbook(store, workbook_id,
|
| 2178 |
+
{"name": book.get("name"), "sheets": book.get("sheets"),
|
| 2179 |
+
"refreshed_at": stamp, "rev": book.get("rev")},
|
| 2180 |
+
actor=session.uname, actor_name=session.user.get("name") or session.uname,
|
| 2181 |
+
lossless=True)
|
| 2182 |
+
except sheets.SheetRefusal as exc:
|
| 2183 |
+
raise _refuse(exc)
|
| 2184 |
+
else:
|
| 2185 |
+
saved = book
|
| 2186 |
+
saved["refreshedFor"] = str(session.uname or "")
|
| 2187 |
+
if entries:
|
| 2188 |
+
sheet_binding.save_refresh_cache(store, workbook_id, entries, status)
|
| 2189 |
+
return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms,
|
| 2190 |
+
"binding_refresh": {"databases": status, "token_reads": token_reads}}
|
| 2191 |
|
| 2192 |
|
| 2193 |
def _now_iso():
|
| 2194 |
+
from core import write_execution
|
| 2195 |
+
return write_execution.utcnow().replace(microsecond=0).isoformat()
|
| 2196 |
|
| 2197 |
|
| 2198 |
#: ⛔ THE NAME A TAB GETS WHEN IT CANNOT KEEP ITS OWN, and it is built to be UNTYPEABLE on two
|
|
|
|
| 2483 |
refuse("the cell or its sheet is missing or blank.")
|
| 2484 |
if cell.get("kind") == "bound":
|
| 2485 |
try:
|
| 2486 |
+
lookup = getattr(session, "_sheet_refresh_input", None)
|
| 2487 |
+
cached = lookup(key) if lookup is not None else None
|
| 2488 |
+
value = (cached["value"] if cached is not None else _resolve_binding(
|
| 2489 |
+
session, cell.get("binding"), today=today, book=book,
|
| 2490 |
+
tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"])
|
| 2491 |
except HTTPException as exc:
|
| 2492 |
reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
|
| 2493 |
refuse(reason)
|
|
|
|
| 2515 |
for dependency in needed:
|
| 2516 |
attempted.add(dependency)
|
| 2517 |
try:
|
| 2518 |
+
lookup = getattr(session, "_sheet_refresh_input", None)
|
| 2519 |
+
result = lookup(dependency) if lookup is not None else None
|
| 2520 |
+
if result is None:
|
| 2521 |
+
result = _resolve_binding(session, bound[dependency].get("binding"), today=today,
|
| 2522 |
+
book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency))
|
| 2523 |
changes[dependency] = {"value": result["value"]}
|
| 2524 |
except HTTPException as exc:
|
| 2525 |
info = ((exc.detail or {}).get("error") or {})
|
|
|
|
| 2590 |
return _resolve_binding(session, binding, **kwargs)
|
| 2591 |
|
| 2592 |
|
| 2593 |
+
def _binding_related_entry(session, database, binding):
|
| 2594 |
+
"""Resolve one authored edge key against the permission-scoped declarations."""
|
| 2595 |
+
if "related" not in binding:
|
| 2596 |
+
return None
|
| 2597 |
+
from core import perm_scope, relations, user_tables
|
| 2598 |
+
from routes_grid import _source_fields
|
| 2599 |
+
|
| 2600 |
+
fields = _source_fields(session, database)
|
| 2601 |
+
defs = {}
|
| 2602 |
+
for key, definition in (user_tables.all_defs(st=session.runtime) or {}).items():
|
| 2603 |
+
if perm_scope.nav_may_open(session.user, key, st=session.runtime):
|
| 2604 |
+
declared = definition.get("fields") or []
|
| 2605 |
+
hidden = perm_scope.hidden_keys(session.user, key, declared, st=session.runtime)
|
| 2606 |
+
defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
|
| 2607 |
+
entries = relations.related(database, fields=fields, defs=defs)
|
| 2608 |
+
name = binding["related"]["via"]
|
| 2609 |
+
matches = [entry for entry in entries if entry["key"] == name]
|
| 2610 |
+
via = matches[0]["via"] if len(matches) == 1 else {"key": name}
|
| 2611 |
+
entry = relations.find(database, via, fields=fields, defs=defs)
|
| 2612 |
+
if entry is None:
|
| 2613 |
+
candidates = [item["key"] for item in relations.via_candidates(entries, via)
|
| 2614 |
+
if perm_scope.nav_may_open(session.user, item["target"], st=session.runtime)]
|
| 2615 |
+
offered = (" Available relationships: " + ", ".join(candidates) + ".") if candidates else ""
|
| 2616 |
+
# Write our own sentence: the shared pin refusal does not name via.key.
|
| 2617 |
+
raise err(409, "unknown_relation",
|
| 2618 |
+
f"Relationship '{name}' cannot be resolved on '{database}'."
|
| 2619 |
+
+ offered + " Choose a declared relationship for this database.")
|
| 2620 |
+
target = _require_database(session, entry["target"])
|
| 2621 |
+
if not entry.get("served"):
|
| 2622 |
+
raise err(409, "relation_target_unavailable",
|
| 2623 |
+
f"Relationship '{name}' cannot read '{target}' because it has no read-through "
|
| 2624 |
+
"binding. Choose a relationship to a database with a live connection.")
|
| 2625 |
+
# A through/manual link cannot be represented by this one declared column pair.
|
| 2626 |
+
declaration = next((f for f in fields if f.get("key") == entry["via"].get("field")), {})
|
| 2627 |
+
if entry["via"]["direction"] == "reverse":
|
| 2628 |
+
declaration = next((f for f in defs.get(target, {}).get("fields", [])
|
| 2629 |
+
if f.get("key") == entry["via"].get("field")), {})
|
| 2630 |
+
link = declaration.get("link") or {}
|
| 2631 |
+
if link.get("through") or link.get("selectionMode") == "manual":
|
| 2632 |
+
raise err(409, "relation_source_unresolvable",
|
| 2633 |
+
f"Relationship '{name}' does not declare a direct column join. "
|
| 2634 |
+
"Choose a relationship with one readable column on each database.")
|
| 2635 |
+
return entry
|
| 2636 |
+
|
| 2637 |
+
|
| 2638 |
+
def _binding_related_query(session, binding, entry, cur, spec, columns, *, window=None,
|
| 2639 |
+
today=None, book=None, tab=None, ending_seen=()):
|
| 2640 |
+
"""Materialise the far side on the near cursor, then hand metadata to the compiler."""
|
| 2641 |
+
if entry is None:
|
| 2642 |
+
return None
|
| 2643 |
+
from core import perm_scope, relations
|
| 2644 |
+
from modules import sheet as sheet_mod
|
| 2645 |
+
|
| 2646 |
+
database, target = binding["database"], entry["target"]
|
| 2647 |
+
name = binding["related"]["via"]
|
| 2648 |
+
local_key, remote_key = entry["via"].get("from"), entry["via"].get("on")
|
| 2649 |
+
|
| 2650 |
+
def map_join(key, table, source_spec, cols, side):
|
| 2651 |
+
hidden = perm_scope.hidden_keys(session.user, table, source_spec.get("fields") or [],
|
| 2652 |
+
st=session.runtime)
|
| 2653 |
+
if key and key not in hidden:
|
| 2654 |
+
expr = (cols.get(key) or {}).get("sql")
|
| 2655 |
+
if not expr:
|
| 2656 |
+
# Identity and join columns can be absent from the visible field list.
|
| 2657 |
+
# Only the trusted mirror map may supply their expressions.
|
| 2658 |
+
mapped = (source_spec.get("cols") or {}).get(key)
|
| 2659 |
+
expr = (source_spec.get("id") if key == relations.IDENTITY_COL else
|
| 2660 |
+
mapped[0] if mapped else None)
|
| 2661 |
+
if expr and str(expr).strip().upper() != "NULL":
|
| 2662 |
+
cols[key] = {**cols.get(key, {}), "sql": expr}
|
| 2663 |
+
return
|
| 2664 |
+
code = "relation_source_unresolvable" if side == "source" else "relation_target_unavailable"
|
| 2665 |
+
raise err(409, code,
|
| 2666 |
+
f"Relationship '{name}' cannot read its {side} column '{key}' on '{table}'. "
|
| 2667 |
+
"Choose a relationship whose join columns are available on the live connection.")
|
| 2668 |
+
|
| 2669 |
+
map_join(local_key, database, spec, columns, "source")
|
| 2670 |
+
wall = ((perm_scope.entry(session.user, target) or {}).get("filter")
|
| 2671 |
+
if perm_scope.row_scope_applies(session.user, target) else None)
|
| 2672 |
+
wall_nodes, wall_conj = _fe_parts(wall)
|
| 2673 |
+
scope_nodes, scope_conj = _fe_parts(binding["related"]["filter"])
|
| 2674 |
+
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
|
| 2675 |
+
today=today, seen=ending_seen)
|
| 2676 |
+
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
| 2677 |
+
| sheet_mod.filter_columns_named(wall_nodes) | {remote_key})
|
| 2678 |
+
try:
|
| 2679 |
+
rcur, rspec, rcols = _mirror(session)(target, metric_window=window, needed=needed)
|
| 2680 |
+
except HTTPException as exc:
|
| 2681 |
+
detail = exc.detail if isinstance(exc.detail, dict) else {}
|
| 2682 |
+
if exc.status_code == 404 or (detail.get("error") or {}).get("code") == "not_read_through":
|
| 2683 |
+
raise err(409, "relation_target_unavailable",
|
| 2684 |
+
f"Relationship '{name}' cannot read '{target}' on this live connection. "
|
| 2685 |
+
"Choose a relationship to an available database.") from exc
|
| 2686 |
+
raise
|
| 2687 |
+
if rcur is not cur:
|
| 2688 |
+
raise err(409, "relation_cursor_mismatch",
|
| 2689 |
+
f"Relationship '{name}' cannot be queried on the same live connection as "
|
| 2690 |
+
f"'{database}'. Retry after both databases share a connection.")
|
| 2691 |
+
hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [],
|
| 2692 |
+
st=session.runtime)
|
| 2693 |
+
rcols = {key: value for key, value in rcols.items() if key not in hidden}
|
| 2694 |
+
map_join(remote_key, target, rspec, rcols, "target")
|
| 2695 |
+
depends = sheet_mod.filter_columns_named(scope_nodes)
|
| 2696 |
+
return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key,
|
| 2697 |
+
"spec": rspec, "columns": rcols, "where": entry.get("where"),
|
| 2698 |
+
"scope_nodes": scope_nodes, "scope_conj": scope_conj,
|
| 2699 |
+
"wall_nodes": wall_nodes, "wall_conj": wall_conj,
|
| 2700 |
+
"limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in depends]}
|
| 2701 |
+
|
| 2702 |
+
|
| 2703 |
def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
|
| 2704 |
book=None, tab=None, ending_seen=()):
|
| 2705 |
"""Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
|
|
|
|
| 2761 |
if binding["aggregate"].get("field"):
|
| 2762 |
needed.add(str(binding["aggregate"]["field"]))
|
| 2763 |
|
| 2764 |
+
related_entry = _binding_related_entry(session, database, binding)
|
| 2765 |
+
if related_entry:
|
| 2766 |
+
needed.add(related_entry["via"].get("from"))
|
| 2767 |
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 2768 |
cols = dict(cols)
|
| 2769 |
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
|
|
|
|
| 2805 |
# spec's column expressions; applying it a second time would double-count the exclusion and
|
| 2806 |
# break every joined spec, whose partner column is `sol.order_partner_id` or `aml.partner_id`.
|
| 2807 |
try:
|
| 2808 |
+
related = _binding_related_query(
|
| 2809 |
+
session, binding, related_entry, cur, spec, cols, window=window, today=today,
|
| 2810 |
+
book=book, tab=tab, ending_seen=ending_seen)
|
| 2811 |
sql, params, limits = sheet_mod.build_aggregate_query(
|
| 2812 |
spec, binding, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
|
| 2813 |
+
wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, related=related)
|
| 2814 |
except sheet_mod.SheetRefusal as exc:
|
| 2815 |
raise err(400, exc.code, exc.message)
|
| 2816 |
except ValueError:
|
|
|
|
| 2851 |
return {"value": value, "limits": limits, "database": database, "binding": binding}
|
| 2852 |
|
| 2853 |
|
| 2854 |
+
def execute_daily_sales_repair(session: Session, request, *, backup_store, workbook_id=None):
|
| 2855 |
+
"""Run the explicit T21 repair through the normal scoped binding and formula engines.
|
| 2856 |
+
|
| 2857 |
+
This is intentionally not an HTTP route. A normal workbook request must never acquire an
|
| 2858 |
+
exceptional tenant-data mutation surface. The approved operator calls this helper with the
|
| 2859 |
+
exact marker/revision/source oracle and a backup store, naming the workbook with workbook_id.
|
| 2860 |
+
Omission retains the owner's workbook default. The helper explicitly points the core workflow
|
| 2861 |
+
at that target; the core refuses any workbook it was not pointed at and performs its own marker,
|
| 2862 |
+
snapshot, backup and readback checks.
|
| 2863 |
+
"""
|
| 2864 |
+
from core import daily_sales_repair, sheets
|
| 2865 |
+
import formula_runtime
|
| 2866 |
+
|
| 2867 |
+
try:
|
| 2868 |
+
if str(session.tenant or "") != daily_sales_repair.TARGET_TENANT:
|
| 2869 |
+
raise daily_sales_repair.RepairRefusal(
|
| 2870 |
+
"repair_tenant_mismatch",
|
| 2871 |
+
"This operation may run only in the Royal Imports workspace.")
|
| 2872 |
+
target = str(workbook_id or "").strip() or daily_sales_repair.TARGET_WORKBOOK_ID
|
| 2873 |
+
book = sheets.get_workbook(_store(session), target)
|
| 2874 |
+
_require_editor(session, book)
|
| 2875 |
+
|
| 2876 |
+
view_memo = {}
|
| 2877 |
+
|
| 2878 |
+
def resolve(binding, projected, tab):
|
| 2879 |
+
tab_ref = str(tab.get("id") or tab.get("name") or "")
|
| 2880 |
+
return _resolve_cell_binding(
|
| 2881 |
+
session, binding, projected, tab_ref, view_memo=view_memo)
|
| 2882 |
+
|
| 2883 |
+
def formula_check(projected):
|
| 2884 |
+
formula_runtime.recompute_document(
|
| 2885 |
+
projected, today=(request or {}).get("calculation_date"))
|
| 2886 |
+
return {"book": projected,
|
| 2887 |
+
"errors": daily_sales_repair.formula_error_census(projected)}
|
| 2888 |
+
|
| 2889 |
+
return daily_sales_repair.execute(
|
| 2890 |
+
_store(session), backup_store, request,
|
| 2891 |
+
target_workbook_id=target,
|
| 2892 |
+
resolve_binding=resolve, formula_check=formula_check,
|
| 2893 |
+
actor=session.uname, actor_name=session.user.get("name") or session.uname)
|
| 2894 |
+
except sheets.SheetRefusal as exc:
|
| 2895 |
+
raise _refuse(exc)
|
| 2896 |
+
|
| 2897 |
+
|
| 2898 |
def _fe_parts(tree):
|
| 2899 |
"""`(nodes, conj)` for a stored `{conj, nodes}` filter, through the ONE reader of that shape."""
|
| 2900 |
from modules import sheet as sheet_mod
|
|
|
|
| 3077 |
saved["refreshed_at"] = stamp
|
| 3078 |
saved.pop("refusal", None)
|
| 3079 |
|
| 3080 |
+
if limits:
|
| 3081 |
+
saved["limits"] = limits
|
| 3082 |
+
else:
|
| 3083 |
+
saved.pop("limits", None)
|
| 3084 |
+
|
| 3085 |
try:
|
| 3086 |
landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved},
|
| 3087 |
actor=session.uname,
|
|
|
|
| 3101 |
except Exception: # noqa: BLE001
|
| 3102 |
pass
|
| 3103 |
|
| 3104 |
+
from core.sheet_binding import save_refresh_cache
|
| 3105 |
+
save_refresh_cache(_store(session), workbook_id, [(tab_ref + "!" + key,
|
| 3106 |
+
_binding_scope_key(session, str(binding.get("database") or ""), binding),
|
| 3107 |
+
{"tokens": {}, "context": _refresh_context(book, binding),
|
| 3108 |
+
"cell": {k: saved[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
|
| 3109 |
+
if k in saved}})], [binding.get("database")])
|
| 3110 |
+
|
| 3111 |
wire = dict(saved)
|
| 3112 |
#: ⚠ `limits` RIDES THE ANSWER AND IS NOT STORED, because `core.sheets.clean_cell` carries no
|
| 3113 |
#: such key: it describes THIS resolve's truncation, not the cell.
|
|
|
|
| 3289 |
window = _ending_window(session, binding, book=book, tab=tab.get("id"))
|
| 3290 |
except sheet_mod.SheetRefusal as exc:
|
| 3291 |
raise err(400, exc.code, exc.message)
|
| 3292 |
+
related_entry = _binding_related_entry(session, database, binding)
|
| 3293 |
+
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
| 3294 |
+
| sheet_mod.filter_columns_named(wall_nodes))
|
| 3295 |
+
if related_entry:
|
| 3296 |
+
needed.add(related_entry["via"].get("from"))
|
| 3297 |
cur, spec, cols = _mirror(session)(
|
| 3298 |
database, metric_window=window,
|
| 3299 |
+
needed=needed)
|
|
|
|
| 3300 |
cols = dict(cols)
|
| 3301 |
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols)
|
| 3302 |
|
| 3303 |
try:
|
| 3304 |
+
related = _binding_related_query(
|
| 3305 |
+
session, binding, related_entry, cur, spec, cols, window=window,
|
| 3306 |
+
book=book, tab=tab.get("id"))
|
| 3307 |
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
|
| 3308 |
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 3309 |
+
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
|
| 3310 |
+
ids_sql, ids_params, _ = sheet_mod.build_ids_query(
|
| 3311 |
+
spec, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
|
| 3312 |
+
wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
|
| 3313 |
except sheet_mod.SheetRefusal as exc:
|
| 3314 |
raise err(400, exc.code, exc.message)
|
| 3315 |
+
except ValueError:
|
| 3316 |
+
raise err(409, "scope_not_applicable",
|
| 3317 |
+
"This cell's scope uses a restriction that cannot be applied on its live "
|
| 3318 |
+
"connection. Edit the scope or ask an administrator to update the restriction.")
|
| 3319 |
total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
|
| 3320 |
|
|
|
|
|
|
|
|
|
|
| 3321 |
record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
|
| 3322 |
|
| 3323 |
out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
|
api/routes_slack.py
CHANGED
|
@@ -197,11 +197,11 @@ def agent_may_read(agent, module):
|
|
| 197 |
return bool(perm_scope.may_access(agent_principal(agent), module))
|
| 198 |
|
| 199 |
|
| 200 |
-
def agent_visible_fields(agent, fields, module):
|
| 201 |
"""`fields` minus this channel's hidden closure — the SAME transitive closure a person gets,
|
| 202 |
so a formula over a hidden column cannot leak it back through arithmetic."""
|
| 203 |
import core.perm_scope as perm_scope
|
| 204 |
-
return perm_scope.visible_fields(fields, agent_principal(agent), module)
|
| 205 |
|
| 206 |
|
| 207 |
def agent_rows(agent, rows, module, fields, ctx=None, st=None):
|
|
@@ -218,15 +218,13 @@ def agent_rows(agent, rows, module, fields, ctx=None, st=None):
|
|
| 218 |
default for the same reason `perm_scope.apply_row_scope`'s is: without one this is
|
| 219 |
byte-identical to the wall that shipped before the argument existed.
|
| 220 |
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
rather than what I16 asked for, and widening two walls in one ticket is how a permission
|
| 225 |
-
change stops being reviewable.
|
| 226 |
"""
|
| 227 |
import core.perm_scope as perm_scope
|
| 228 |
p = agent_principal(agent)
|
| 229 |
-
hide = perm_scope.hidden_keys(p, module, fields)
|
| 230 |
kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx, st=st)
|
| 231 |
# Both wires, never one: the field list and the row payload are separate, and stripping only
|
| 232 |
# the first leaves the value sitting in the second where anyone can read it.
|
|
@@ -298,7 +296,8 @@ def channel_answer_context(rt, channel, principal=None, automation_id=None):
|
|
| 298 |
if not isinstance(grants, list) or str(automation_id) not in {
|
| 299 |
str(item) for item in grants if isinstance(item, str)}:
|
| 300 |
return None
|
| 301 |
-
return {"
|
|
|
|
| 302 |
"writeApproval": agent.get("writeApproval", True) is not False,
|
| 303 |
"automationId": str(automation_id or "")}
|
| 304 |
|
|
@@ -336,6 +335,53 @@ def _reply_text(text, provenance):
|
|
| 336 |
return f"{body[:max(0, 4000 - len(footer) - 2)]}\n\n{footer}"
|
| 337 |
|
| 338 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 339 |
def deliver_channel_reply(rt, channel, text, *, principal=None, automation_id=None,
|
| 340 |
provenance=None, post=None):
|
| 341 |
"""Send only after one channel/principal binding and an optional automation grant validate."""
|
|
|
|
| 197 |
return bool(perm_scope.may_access(agent_principal(agent), module))
|
| 198 |
|
| 199 |
|
| 200 |
+
def agent_visible_fields(agent, fields, module, st=None):
|
| 201 |
"""`fields` minus this channel's hidden closure — the SAME transitive closure a person gets,
|
| 202 |
so a formula over a hidden column cannot leak it back through arithmetic."""
|
| 203 |
import core.perm_scope as perm_scope
|
| 204 |
+
return perm_scope.visible_fields(fields, agent_principal(agent), module, st=st)
|
| 205 |
|
| 206 |
|
| 207 |
def agent_rows(agent, rows, module, fields, ctx=None, st=None):
|
|
|
|
| 218 |
default for the same reason `perm_scope.apply_row_scope`'s is: without one this is
|
| 219 |
byte-identical to the wall that shipped before the argument existed.
|
| 220 |
|
| 221 |
+
The field projection takes the same runtime too. Without it a user-generated field can be
|
| 222 |
+
absent from the declared grant while its value survives in the row payload — precisely the
|
| 223 |
+
half-wall a customer-intelligence digest must not cross.
|
|
|
|
|
|
|
| 224 |
"""
|
| 225 |
import core.perm_scope as perm_scope
|
| 226 |
p = agent_principal(agent)
|
| 227 |
+
hide = perm_scope.hidden_keys(p, module, fields, st=st)
|
| 228 |
kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx, st=st)
|
| 229 |
# Both wires, never one: the field list and the row payload are separate, and stripping only
|
| 230 |
# the first leaves the value sitting in the second where anyone can read it.
|
|
|
|
| 296 |
if not isinstance(grants, list) or str(automation_id) not in {
|
| 297 |
str(item) for item in grants if isinstance(item, str)}:
|
| 298 |
return None
|
| 299 |
+
return {"agentId": str(agent.get("id") or ""),
|
| 300 |
+
"channel": str(agent.get("channel") or ""), "principal": expected,
|
| 301 |
"writeApproval": agent.get("writeApproval", True) is not False,
|
| 302 |
"automationId": str(automation_id or "")}
|
| 303 |
|
|
|
|
| 335 |
return f"{body[:max(0, 4000 - len(footer) - 2)]}\n\n{footer}"
|
| 336 |
|
| 337 |
|
| 338 |
+
def permission_safe_automation_digest(rt, context, *, module="", fields=None, rows=None,
|
| 339 |
+
summary="", links=None, limit=12):
|
| 340 |
+
"""Compose a bounded digest only after applying the destination Agent's live read wall.
|
| 341 |
+
|
| 342 |
+
The caller supplies trusted object links, never cell/field values in a URL. For a database
|
| 343 |
+
digest both the schema and rows pass through the exact Slack-agent permission projection;
|
| 344 |
+
for a Sheet digest the body intentionally carries no cell values at all.
|
| 345 |
+
"""
|
| 346 |
+
ctx = context if isinstance(context, dict) else {}
|
| 347 |
+
agent = _agents(rt).get(str(ctx.get("agentId") or ""))
|
| 348 |
+
if not isinstance(agent, dict) or agent.get("active", True) is False:
|
| 349 |
+
return None
|
| 350 |
+
try:
|
| 351 |
+
cap = max(1, min(int(limit), APPROVAL_MAX_ROWS))
|
| 352 |
+
except (TypeError, ValueError):
|
| 353 |
+
cap = APPROVAL_MAX_ROWS
|
| 354 |
+
visible, safe_rows, readable = [], [], True
|
| 355 |
+
if module:
|
| 356 |
+
readable = agent_may_read(agent, module)
|
| 357 |
+
if readable:
|
| 358 |
+
visible = agent_visible_fields(agent, fields or [], module, st=rt)
|
| 359 |
+
safe_rows = agent_rows(agent, rows or [], module, fields or [], st=rt)[:cap]
|
| 360 |
+
labels = {str(field.get("key") or ""): str(field.get("label") or field.get("key") or "")
|
| 361 |
+
for field in visible if isinstance(field, dict) and field.get("key")}
|
| 362 |
+
lines = [" ".join(str(summary or "Customer intelligence found a change.").split())[:500]]
|
| 363 |
+
if readable:
|
| 364 |
+
for row in safe_rows:
|
| 365 |
+
cells = []
|
| 366 |
+
for key in list(labels)[:8]:
|
| 367 |
+
value = (row or {}).get(key)
|
| 368 |
+
if value in (None, ""):
|
| 369 |
+
continue
|
| 370 |
+
cells.append(f"{labels[key]}: {' '.join(str(value).split())[:160]}")
|
| 371 |
+
if cells:
|
| 372 |
+
lines.append(" • " + "; ".join(cells))
|
| 373 |
+
trusted_links = [str(value).strip() for value in (links or [])
|
| 374 |
+
if str(value).strip().startswith("#/")][:cap]
|
| 375 |
+
base = (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/")
|
| 376 |
+
if base:
|
| 377 |
+
trusted_links = [base + "/" + value for value in trusted_links]
|
| 378 |
+
if trusted_links:
|
| 379 |
+
lines.append("Open in Loopable: " + " ".join(trusted_links))
|
| 380 |
+
else:
|
| 381 |
+
return None
|
| 382 |
+
return "\n".join(lines)[:3200]
|
| 383 |
+
|
| 384 |
+
|
| 385 |
def deliver_channel_reply(rt, channel, text, *, principal=None, automation_id=None,
|
| 386 |
provenance=None, post=None):
|
| 387 |
"""Send only after one channel/principal binding and an optional automation grant validate."""
|
api/routes_templates.py
CHANGED
|
@@ -239,13 +239,14 @@ def _requirement(database, definitions, have, database_states, connector=""):
|
|
| 239 |
else "ready" if switched_on else "off"}
|
| 240 |
|
| 241 |
|
| 242 |
-
def _catalog_row(key, name, description, requires):
|
| 243 |
# Prefer the missing connection even when an earlier database is merely off.
|
| 244 |
blocked = next((row for row in requires if row["state"] == "disconnected"), None)
|
| 245 |
blocked = blocked or next((row for row in requires if row["state"] != "ready"), None)
|
| 246 |
return {"key": key, "name": str(name).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
|
| 247 |
"description": str(description).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
|
| 248 |
"requires": requires, "creatable": all(row["state"] == "ready" for row in requires),
|
|
|
|
| 249 |
"action": None if blocked is None else {
|
| 250 |
"kind": "connect" if blocked["state"] == "disconnected" else "switch_on",
|
| 251 |
"connector": blocked["connector"], "database": blocked["database"]}}
|
|
@@ -269,6 +270,8 @@ def _module_templates(session):
|
|
| 269 |
from routes_connectors import _entry_types
|
| 270 |
from routes_keychain import env_odoo_available
|
| 271 |
from routes_nav import application_templates
|
|
|
|
|
|
|
| 272 |
|
| 273 |
definitions = user_tables.all_defs(session.runtime)
|
| 274 |
have, _scopes, _failure = _entry_types(session.runtime, session.uname)
|
|
@@ -299,6 +302,9 @@ def _module_templates(session):
|
|
| 299 |
# These App contracts map fields in the existing wizard and require no fixed connector.
|
| 300 |
app = [_catalog_row(row["kind"], row["label"], "Choose a database and map its fields.", [])
|
| 301 |
for row in application_templates(session=session)["templates"]]
|
|
|
|
|
|
|
|
|
|
| 302 |
return {"sections": [{"module": "sheet", "templates": sheet},
|
| 303 |
{"module": "data", "templates": data},
|
| 304 |
{"module": "app", "templates": app}]}
|
|
@@ -406,6 +412,35 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
|
|
| 406 |
from urllib.parse import quote
|
| 407 |
|
| 408 |
body = body if isinstance(body, dict) else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None)
|
| 410 |
template = next((row for row in section["templates"] if row["key"] == key), None) if section else None
|
| 411 |
if template is None:
|
|
|
|
| 239 |
else "ready" if switched_on else "off"}
|
| 240 |
|
| 241 |
|
| 242 |
+
def _catalog_row(key, name, description, requires, *, creation_mode="wizard"):
|
| 243 |
# Prefer the missing connection even when an earlier database is merely off.
|
| 244 |
blocked = next((row for row in requires if row["state"] == "disconnected"), None)
|
| 245 |
blocked = blocked or next((row for row in requires if row["state"] != "ready"), None)
|
| 246 |
return {"key": key, "name": str(name).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
|
| 247 |
"description": str(description).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
|
| 248 |
"requires": requires, "creatable": all(row["state"] == "ready" for row in requires),
|
| 249 |
+
"creationMode": creation_mode,
|
| 250 |
"action": None if blocked is None else {
|
| 251 |
"kind": "connect" if blocked["state"] == "disconnected" else "switch_on",
|
| 252 |
"connector": blocked["connector"], "database": blocked["database"]}}
|
|
|
|
| 270 |
from routes_connectors import _entry_types
|
| 271 |
from routes_keychain import env_odoo_available
|
| 272 |
from routes_nav import application_templates
|
| 273 |
+
from routes_applications import _may_open
|
| 274 |
+
from modules.finance_apps import finance_templates
|
| 275 |
|
| 276 |
definitions = user_tables.all_defs(session.runtime)
|
| 277 |
have, _scopes, _failure = _entry_types(session.runtime, session.uname)
|
|
|
|
| 302 |
# These App contracts map fields in the existing wizard and require no fixed connector.
|
| 303 |
app = [_catalog_row(row["kind"], row["label"], "Choose a database and map its fields.", [])
|
| 304 |
for row in application_templates(session=session)["templates"]]
|
| 305 |
+
app.extend(_catalog_row("finance:" + row["key"], row["label"], row["description"], [],
|
| 306 |
+
creation_mode="post")
|
| 307 |
+
for row in finance_templates(may_open=lambda permission: _may_open(session, permission)))
|
| 308 |
return {"sections": [{"module": "sheet", "templates": sheet},
|
| 309 |
{"module": "data", "templates": data},
|
| 310 |
{"module": "app", "templates": app}]}
|
|
|
|
| 412 |
from urllib.parse import quote
|
| 413 |
|
| 414 |
body = body if isinstance(body, dict) else {}
|
| 415 |
+
if module == "app" and key.startswith("finance:"):
|
| 416 |
+
import application_registry
|
| 417 |
+
from modules.finance_apps import finance_application
|
| 418 |
+
from routes_applications import _may_open
|
| 419 |
+
|
| 420 |
+
finance_key = key.removeprefix("finance:")
|
| 421 |
+
try:
|
| 422 |
+
descriptor = finance_application(
|
| 423 |
+
finance_key, may_open=lambda permission: _may_open(session, permission))
|
| 424 |
+
except KeyError:
|
| 425 |
+
raise err(404, "unknown_template", "That template does not exist.")
|
| 426 |
+
except PermissionError:
|
| 427 |
+
raise err(403, "finance_forbidden", f"Access to Finance template {finance_key!r} is not granted.")
|
| 428 |
+
if not session.runtime.available():
|
| 429 |
+
raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was saved.")
|
| 430 |
+
try:
|
| 431 |
+
application = application_registry.create(
|
| 432 |
+
session.runtime, session.tenant, session.uname,
|
| 433 |
+
{"templateKind": application_registry.FINANCE_TEMPLATE_KIND,
|
| 434 |
+
"financeKey": finance_key, "name": body.get("name") or descriptor["label"],
|
| 435 |
+
"navPlacement": {"section": "app", "order": 0}},
|
| 436 |
+
allowed_databases=set())
|
| 437 |
+
except application_registry.ApplicationError as exc:
|
| 438 |
+
raise err(400, "bad_application", str(exc))
|
| 439 |
+
except PermissionError as exc:
|
| 440 |
+
raise err(403, "forbidden", str(exc))
|
| 441 |
+
except Exception:
|
| 442 |
+
raise err(503, "store_unavailable", "The Finance application was not saved.")
|
| 443 |
+
return {"address": "#" + application["deepLink"], "application": application}
|
| 444 |
section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None)
|
| 445 |
template = next((row for row in section["templates"] if row["key"] == key), None) if section else None
|
| 446 |
if template is None:
|
api/test_s63_integration.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hermetic regression checks for the SDLC 63 cross-lane repairs."""
|
| 2 |
+
import copy
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import subprocess
|
| 6 |
+
import sys
|
| 7 |
+
import unittest
|
| 8 |
+
from unittest.mock import patch
|
| 9 |
+
|
| 10 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 11 |
+
sys.path[:0] = [str(ROOT / "platform"), str(ROOT / "aios-web/api")]
|
| 12 |
+
|
| 13 |
+
import aios_grid
|
| 14 |
+
import odoo_relational as rel
|
| 15 |
+
from core import user_tables
|
| 16 |
+
from harness import filter_eval
|
| 17 |
+
from verify_odoo_relational import FixtureCursor, MemStore
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class SourceCounts:
|
| 21 |
+
"""Data-free Odoo double whose answers differ by domain, not merely by model."""
|
| 22 |
+
def __init__(self):
|
| 23 |
+
self.calls = []
|
| 24 |
+
|
| 25 |
+
def search_count(self, model, domain):
|
| 26 |
+
self.calls.append(("count", model, domain))
|
| 27 |
+
if model == "account.move":
|
| 28 |
+
move_types = next(value for field, op, value in domain if field == "move_type")
|
| 29 |
+
return 2 if "out_invoice" in move_types else 3
|
| 30 |
+
return {"sale.order": 4, "account.account": 5,
|
| 31 |
+
"sale.order.line": 6, "account.move.line": 7}[model]
|
| 32 |
+
|
| 33 |
+
def search_read(self, model, domain, fields, limit=None):
|
| 34 |
+
self.calls.append(("read", model, domain))
|
| 35 |
+
return [{"id": 1}, {"id": 2}]
|
| 36 |
+
|
| 37 |
+
def read_group(self, model, domain, fields, groupby, lazy=False):
|
| 38 |
+
self.calls.append(("group", model, domain))
|
| 39 |
+
if model == "account.invoice.line.agent":
|
| 40 |
+
return [{"agent_id": [2, "Two"]}, {"agent_id": [3, "Three"]}]
|
| 41 |
+
return [{"partner_id": [8, "Eight"]}, {"partner_id": [9, "Nine"]}]
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class IntegrationTests(unittest.TestCase):
|
| 45 |
+
def test_archived_defaults_and_removable_filter(self):
|
| 46 |
+
fields = json.loads((ROOT / "platform/aios_grid_fields.json").read_text())["fields"]
|
| 47 |
+
config = aios_grid._default_view_config(fields)
|
| 48 |
+
self.assertEqual(config["filters"], [{"colId": "archived", "op": "neq", "value": "true"}])
|
| 49 |
+
tree = {"conj": "and", "nodes": config["filters"]}
|
| 50 |
+
self.assertTrue(filter_eval.evaluate(tree, {"archived": False}, fields))
|
| 51 |
+
self.assertFalse(filter_eval.evaluate(tree, {"archived": True}, fields))
|
| 52 |
+
self.assertTrue(filter_eval.matches({"conj": "and", "nodes": []}, {"archived": True}, fields))
|
| 53 |
+
self.assertEqual(aios_grid._default_view_config([
|
| 54 |
+
{"key": "archived", "type": "checkbox", "source": "overlay"}])["filters"], [])
|
| 55 |
+
script = r'''
|
| 56 |
+
const fs=require('fs'),ts=require('typescript'),vm=require('vm'),assert=require('assert/strict');
|
| 57 |
+
const text=fs.readFileSync('src/customer-grid/useGridColumns.ts','utf8');
|
| 58 |
+
const file=ts.createSourceFile('views.ts',text,ts.ScriptTarget.Latest,true,ts.ScriptKind.TS);
|
| 59 |
+
const wanted=new Set(['isDefaultVisible','defaultViewConfig']);
|
| 60 |
+
const declarations=file.statements.filter(node=>ts.isFunctionDeclaration(node)&&node.name&&wanted.has(node.name.text));
|
| 61 |
+
assert.equal(declarations.length,2);
|
| 62 |
+
const js=ts.transpileModule(declarations.map(node=>node.getText(file)).join('\n'),
|
| 63 |
+
{compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2020}}).outputText;
|
| 64 |
+
const context={exports:{}};vm.runInNewContext(js,context);
|
| 65 |
+
const archived={key:'archived',type:'checkbox',source:'odoo'};
|
| 66 |
+
const ordinary={key:'name',type:'text',source:'odoo'};
|
| 67 |
+
const overlay={key:'archived',type:'checkbox',source:'overlay'};
|
| 68 |
+
console.log(JSON.stringify({odoo:context.exports.defaultViewConfig([ordinary,archived]).filters,
|
| 69 |
+
overlay:context.exports.defaultViewConfig([ordinary,overlay]).filters}));
|
| 70 |
+
'''
|
| 71 |
+
result = subprocess.run(["node", "-e", script], cwd=ROOT / "aios-web/web",
|
| 72 |
+
capture_output=True, text=True, encoding="utf-8", timeout=30)
|
| 73 |
+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
| 74 |
+
client = json.loads(result.stdout)
|
| 75 |
+
self.assertEqual(client["odoo"], config["filters"])
|
| 76 |
+
self.assertEqual(client["overlay"], [])
|
| 77 |
+
|
| 78 |
+
def test_database_read_failure_preserves_rows_and_successful_sibling(self):
|
| 79 |
+
rt = MemStore()
|
| 80 |
+
with patch.object(user_tables, "row_limit", return_value=None), \
|
| 81 |
+
patch.object(rel, "is_read_through_key", return_value=False):
|
| 82 |
+
first = rel.refresh_curated(rt, FixtureCursor(), {"invoices", "bills"}, retire=False)
|
| 83 |
+
self.assertEqual(first[2], {})
|
| 84 |
+
before = copy.deepcopy(rt.data[user_tables.STORE_KEY][rel.INVOICES_KEY])
|
| 85 |
+
with patch.dict(rel._READERS, {"invoices": lambda *_: (_ for _ in ()).throw(
|
| 86 |
+
RuntimeError("private row must never enter receipt"))}):
|
| 87 |
+
built, written, errors, receipts, _ = rel.refresh_curated(
|
| 88 |
+
rt, FixtureCursor(), {"invoices", "bills"}, retire=False)
|
| 89 |
+
self.assertNotIn("invoices", built)
|
| 90 |
+
self.assertNotIn(rel.INVOICES_KEY, written)
|
| 91 |
+
self.assertIn(rel.BILLS_KEY, written)
|
| 92 |
+
self.assertEqual(set(errors), {rel.INVOICES_KEY})
|
| 93 |
+
self.assertEqual(rt.data[user_tables.STORE_KEY][rel.INVOICES_KEY], before)
|
| 94 |
+
self.assertNotIn("private row", json.dumps(receipts))
|
| 95 |
+
self.assertEqual(receipts[rel.BILLS_KEY]["state"], "succeeded")
|
| 96 |
+
self.assertIsNone(receipts[rel.BILLS_KEY]["sourceCount"])
|
| 97 |
+
self.assertEqual(receipts[rel.BILLS_KEY]["mirrorCount"], written[rel.BILLS_KEY]["rows"])
|
| 98 |
+
|
| 99 |
+
def test_duplicate_reader_ids_refuse_before_mutation(self):
|
| 100 |
+
rt = MemStore()
|
| 101 |
+
with patch.object(user_tables, "row_limit", return_value=None), \
|
| 102 |
+
patch.dict(rel._READERS, {"invoices": lambda *_: [{"_id": "1"}, {"_id": "1"}]}):
|
| 103 |
+
_, written, errors, _, _ = rel.refresh_curated(rt, FixtureCursor(), {"invoices"}, retire=False)
|
| 104 |
+
self.assertEqual(written, {})
|
| 105 |
+
self.assertEqual(set(errors), {rel.INVOICES_KEY})
|
| 106 |
+
self.assertEqual(rt.data, {})
|
| 107 |
+
|
| 108 |
+
def test_all_curated_receipts_are_database_and_domain_specific(self):
|
| 109 |
+
rt = MemStore()
|
| 110 |
+
source = SourceCounts()
|
| 111 |
+
expected = {"invoices": 2, "orders": 4, "agents": 3, "accounts": 5,
|
| 112 |
+
"vendors": 2, "bills": 3, "order_lines": 6, "gl_lines": 7}
|
| 113 |
+
by_bucket = {bucket: key for bucket, key, _label, _fields in rel.TABLES}
|
| 114 |
+
|
| 115 |
+
def fake_plan(_cur, rt=None, connection_id=None, buckets=None):
|
| 116 |
+
bucket = next(iter(buckets))
|
| 117 |
+
rows = ([] if bucket in {"order_lines", "gl_lines"} else
|
| 118 |
+
[{"_id": f"{bucket}-{index}"} for index in range(expected[bucket])])
|
| 119 |
+
return {bucket: rows, "problems": [], "limits": {}}
|
| 120 |
+
|
| 121 |
+
def fake_apply(_rt, planned, **_kwargs):
|
| 122 |
+
bucket = next(key for key in expected if key in planned)
|
| 123 |
+
key = by_bucket[bucket]
|
| 124 |
+
return {key: {"rows": len(planned[bucket])}}
|
| 125 |
+
|
| 126 |
+
def fake_population(key, **_kwargs):
|
| 127 |
+
bucket = next(name for name, table_key in by_bucket.items() if table_key == key)
|
| 128 |
+
return expected[bucket]
|
| 129 |
+
|
| 130 |
+
with patch.object(rel, "plan", side_effect=fake_plan), \
|
| 131 |
+
patch.object(rel, "apply_plan", side_effect=fake_apply), \
|
| 132 |
+
patch.object(user_tables, "row_limit", return_value=None), \
|
| 133 |
+
patch.object(rel, "is_read_through_key",
|
| 134 |
+
side_effect=lambda key: key in {rel.ORDER_LINES_KEY, rel.GL_LINES_KEY}), \
|
| 135 |
+
patch("routes_odoo_tables.population", side_effect=fake_population):
|
| 136 |
+
built, written, errors, receipts, _ = rel.refresh_curated(
|
| 137 |
+
rt, FixtureCursor(), set(expected), retire=False, source_client=source)
|
| 138 |
+
|
| 139 |
+
self.assertEqual(errors, {})
|
| 140 |
+
self.assertEqual(set(receipts), set(by_bucket.values()))
|
| 141 |
+
self.assertEqual(set(written), set(by_bucket.values()))
|
| 142 |
+
for bucket, key in by_bucket.items():
|
| 143 |
+
receipt = receipts[key]
|
| 144 |
+
self.assertEqual(receipt["schemaKey"], key)
|
| 145 |
+
self.assertEqual(receipt["domainKey"], rel.TABLE_SUBJECTS[key])
|
| 146 |
+
self.assertEqual(receipt["topicKey"], rel.TABLE_TOPICS[key])
|
| 147 |
+
self.assertEqual(receipt["selection"], {"bucket": bucket, "enabled": True})
|
| 148 |
+
self.assertEqual(receipt["sourceCount"], expected[bucket])
|
| 149 |
+
self.assertEqual(receipt["mirrorCount"], expected[bucket])
|
| 150 |
+
self.assertEqual(receipt["sourceReconciliation"], "matched")
|
| 151 |
+
self.assertEqual(receipt["freshness"]["state"], "current")
|
| 152 |
+
self.assertEqual(receipts[rel.INVOICES_KEY]["model"], "account.move")
|
| 153 |
+
self.assertEqual(receipts[rel.BILLS_KEY]["model"], "account.move")
|
| 154 |
+
self.assertNotEqual(receipts[rel.INVOICES_KEY]["domain"],
|
| 155 |
+
receipts[rel.BILLS_KEY]["domain"])
|
| 156 |
+
self.assertEqual(receipts[rel.VENDORS_KEY]["model"], "res.partner")
|
| 157 |
+
self.assertEqual(receipts[rel.VENDORS_KEY]["countModel"], "account.move")
|
| 158 |
+
self.assertEqual(receipts[rel.ORDER_LINES_KEY]["materializedCount"], 0)
|
| 159 |
+
self.assertEqual(receipts[rel.ORDER_LINES_KEY]["servingCount"], 6)
|
| 160 |
+
route = (ROOT / "aios-web/api/routes_odoo_tables.py").read_text(encoding="utf-8")
|
| 161 |
+
refresh_endpoint = route.split("def refresh_odoo_tables", 1)[1].split(
|
| 162 |
+
'@router.get("/odoo-tables/status")', 1)[0]
|
| 163 |
+
self.assertIn("all_connections=True", refresh_endpoint)
|
| 164 |
+
|
| 165 |
+
def test_browser_grid_header_and_cell_events_with_both_negative_controls(self):
|
| 166 |
+
script = r'''
|
| 167 |
+
const fs=require('fs'),ts=require('typescript'),assert=require('assert/strict');
|
| 168 |
+
const text=fs.readFileSync('src/sheet/SheetGrid.tsx','utf8');
|
| 169 |
+
const file=ts.createSourceFile('grid.tsx',text,ts.ScriptTarget.Latest,true,ts.ScriptKind.TSX);
|
| 170 |
+
const handlers=[];
|
| 171 |
+
function visit(node){
|
| 172 |
+
if(ts.isJsxAttribute(node)&&node.name.getText(file)==='onCellClicked') handlers.push(node.initializer.expression.getText(file));
|
| 173 |
+
ts.forEachChild(node,visit);
|
| 174 |
+
}
|
| 175 |
+
visit(file); assert.equal(handlers.length,2);
|
| 176 |
+
function compiled(handler){
|
| 177 |
+
return ts.transpileModule('globalThis.__sheetClick = '+handler,{compilerOptions:{target:ts.ScriptTarget.ES2020}}).outputText;
|
| 178 |
+
}
|
| 179 |
+
const controls=handlers.map(handler=>{
|
| 180 |
+
const sentinel=handler.indexOf('if (item[0] === -1)');
|
| 181 |
+
const signalAt=handler.indexOf('signal(NAV_MINIMIZE_EVENT);',sentinel);
|
| 182 |
+
assert.ok(sentinel>=0&&signalAt>sentinel,'production signal must follow the sentinel');
|
| 183 |
+
const signal='signal(NAV_MINIMIZE_EVENT);';
|
| 184 |
+
const moved=handler.slice(0,sentinel)+signal+'\n'+handler.slice(sentinel,signalAt)+
|
| 185 |
+
handler.slice(signalAt+signal.length);
|
| 186 |
+
return {positive:compiled(handler),negative:compiled(moved)};
|
| 187 |
+
});
|
| 188 |
+
console.log(JSON.stringify(controls));
|
| 189 |
+
'''
|
| 190 |
+
result = subprocess.run(["node", "-e", script], cwd=ROOT / "aios-web/web",
|
| 191 |
+
capture_output=True, text=True, encoding="utf-8", timeout=30)
|
| 192 |
+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
| 193 |
+
controls = json.loads(result.stdout)
|
| 194 |
+
|
| 195 |
+
# This leg clicks real browser DOM targets wired to the production TypeScript handlers.
|
| 196 |
+
# Each pane also gets the inverse build (signal moved before the row-header sentinel), so
|
| 197 |
+
# the browser proof can demonstrate that it detects the exact regression it guards.
|
| 198 |
+
from playwright.sync_api import sync_playwright
|
| 199 |
+
with sync_playwright() as playwright:
|
| 200 |
+
browser = playwright.chromium.launch(headless=True)
|
| 201 |
+
page = browser.new_page()
|
| 202 |
+
page.set_content('<button id="header">row header</button><button id="cell">cell</button>')
|
| 203 |
+
for pane, pair in zip(("frozen", "unfrozen"), controls, strict=True):
|
| 204 |
+
def run(compiled):
|
| 205 |
+
page.evaluate("""() => {
|
| 206 |
+
globalThis.__signals = 0; globalThis.__rows = 0;
|
| 207 |
+
globalThis.signal = () => globalThis.__signals++;
|
| 208 |
+
globalThis.NAV_MINIMIZE_EVENT = 'aios:nav-minimize';
|
| 209 |
+
globalThis.fromBand = value => value;
|
| 210 |
+
globalThis.selectDocumentAxis = () => globalThis.__rows++;
|
| 211 |
+
globalThis.painter = false; globalThis.addrAt = () => 'A1';
|
| 212 |
+
globalThis.openViewHeader = () => false;
|
| 213 |
+
globalThis.validationOf = () => false;
|
| 214 |
+
globalThis.cellAt = () => null; globalThis.tab = {};
|
| 215 |
+
globalThis.followCellLink = () => false; globalThis.setAnchor = () => {};
|
| 216 |
+
}""")
|
| 217 |
+
page.evaluate("code => { (0, eval)(code); }", compiled)
|
| 218 |
+
page.evaluate("""() => {
|
| 219 |
+
document.querySelector('#header').onclick = () =>
|
| 220 |
+
globalThis.__sheetClick([-1, 0], {shiftKey: false, bounds: {}});
|
| 221 |
+
document.querySelector('#cell').onclick = () =>
|
| 222 |
+
globalThis.__sheetClick([0, 0], {shiftKey: false, bounds: {}});
|
| 223 |
+
}""")
|
| 224 |
+
page.click("#header")
|
| 225 |
+
after_header = page.evaluate("() => ({signals: __signals, rows: __rows})")
|
| 226 |
+
page.click("#cell")
|
| 227 |
+
after_cell = page.evaluate("() => ({signals: __signals, rows: __rows})")
|
| 228 |
+
return after_header, after_cell
|
| 229 |
+
|
| 230 |
+
header, cell = run(pair["positive"])
|
| 231 |
+
self.assertEqual(header, {"signals": 0, "rows": 1}, pane)
|
| 232 |
+
self.assertEqual(cell, {"signals": 1, "rows": 1}, pane)
|
| 233 |
+
broken_header, _ = run(pair["negative"])
|
| 234 |
+
self.assertEqual(broken_header, {"signals": 1, "rows": 1},
|
| 235 |
+
f"{pane} negative control did not expose the regression")
|
| 236 |
+
browser.close()
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
if __name__ == "__main__":
|
| 240 |
+
unittest.main()
|
api/test_sheet_commit_events.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Exercise value-only workbook commit notifications without crossing cell contents."""
|
| 2 |
+
import copy
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
import sys
|
| 5 |
+
import unittest
|
| 6 |
+
from unittest.mock import patch
|
| 7 |
+
|
| 8 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 9 |
+
sys.path[:0] = [str(ROOT / "platform"), str(ROOT / "aios-web/api")]
|
| 10 |
+
from core import sheets, sheet_compat, sheet_events
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class MemoryStore:
|
| 14 |
+
key = "sheet-event-fixture"
|
| 15 |
+
|
| 16 |
+
def __init__(self):
|
| 17 |
+
self.data = {}
|
| 18 |
+
self.locked = False
|
| 19 |
+
self.fail_commit = False
|
| 20 |
+
|
| 21 |
+
def get(self, key, default=None):
|
| 22 |
+
return copy.deepcopy(self.data.get(key, default))
|
| 23 |
+
|
| 24 |
+
def update(self, key, fn, **kwargs):
|
| 25 |
+
self.locked = True
|
| 26 |
+
try:
|
| 27 |
+
value = fn(copy.deepcopy(self.data.get(key, {})))
|
| 28 |
+
if self.fail_commit:
|
| 29 |
+
raise RuntimeError("fixture commit failure")
|
| 30 |
+
self.data[key] = copy.deepcopy(value)
|
| 31 |
+
finally:
|
| 32 |
+
self.locked = False
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class CommitEvents(unittest.TestCase):
|
| 36 |
+
def setUp(self):
|
| 37 |
+
self.st = MemoryStore()
|
| 38 |
+
self.book = sheets.create_workbook(self.st, name="Fixture", created_by="qa")
|
| 39 |
+
self.tab = self.book["sheets"][0]["id"]
|
| 40 |
+
self.events = []
|
| 41 |
+
|
| 42 |
+
def observe(st, events):
|
| 43 |
+
self.assertFalse(st.locked, "no Automation call while the workbook lock is held")
|
| 44 |
+
for event in events:
|
| 45 |
+
self.assertEqual(st.get(sheets.workbook_key(self.book["id"]))["rev"], event["revision"])
|
| 46 |
+
self.events.extend(events)
|
| 47 |
+
|
| 48 |
+
self.observer = patch.object(sheet_events, "COMMIT_OBSERVER", observe)
|
| 49 |
+
self.observer.start()
|
| 50 |
+
self.addCleanup(self.observer.stop)
|
| 51 |
+
|
| 52 |
+
def patch_cells(self, cells):
|
| 53 |
+
return sheets.patch_cells(self.st, self.book["id"], self.tab, cells, actor="qa")
|
| 54 |
+
|
| 55 |
+
def test_bulk_patch_is_one_committed_event_without_cell_values(self):
|
| 56 |
+
self.patch_cells({f"B{i}": {"kind": "literal", "value": i} for i in range(1, 20001)})
|
| 57 |
+
self.assertEqual(len(self.events), 1)
|
| 58 |
+
event = self.events[0]
|
| 59 |
+
self.assertEqual(event["ranges"], ["B1:B20000"])
|
| 60 |
+
self.assertEqual(event["recordCount"], 20000)
|
| 61 |
+
self.assertEqual(event["changeKinds"], ["value"])
|
| 62 |
+
self.assertTrue(event["committed"])
|
| 63 |
+
self.assertNotIn("cells", event)
|
| 64 |
+
self.assertNotIn("values", event)
|
| 65 |
+
|
| 66 |
+
def test_formatting_identical_values_and_refused_cells_do_not_emit(self):
|
| 67 |
+
self.patch_cells({"A1": {"kind": "literal", "value": 7}})
|
| 68 |
+
self.events.clear()
|
| 69 |
+
self.patch_cells({"A1": {"kind": "literal", "value": 7, "style": {"bold": True}}})
|
| 70 |
+
self.patch_cells({"A1": {"kind": "literal", "value": 7}, "B2": {"kind": "unknown"}})
|
| 71 |
+
self.assertEqual(self.events, [])
|
| 72 |
+
|
| 73 |
+
def test_failed_commit_and_stale_put_do_not_emit(self):
|
| 74 |
+
self.st.fail_commit = True
|
| 75 |
+
with self.assertRaises(RuntimeError):
|
| 76 |
+
self.patch_cells({"A1": {"kind": "literal", "value": 3}})
|
| 77 |
+
self.st.fail_commit = False
|
| 78 |
+
self.assertEqual(self.events, [])
|
| 79 |
+
self.patch_cells({"A1": {"kind": "literal", "value": 3}})
|
| 80 |
+
self.events.clear()
|
| 81 |
+
with self.assertRaises(sheets.SheetRefusal):
|
| 82 |
+
sheets.put_workbook(self.st, self.book["id"], self.book)
|
| 83 |
+
self.assertEqual(self.events, [])
|
| 84 |
+
|
| 85 |
+
def test_removal_and_sparse_changes_keep_exact_watched_ranges(self):
|
| 86 |
+
self.patch_cells({"A1": {"kind": "literal", "value": 1},
|
| 87 |
+
"A3": {"kind": "literal", "value": 3}})
|
| 88 |
+
self.assertEqual(self.events[-1]["ranges"], ["A1", "A3"])
|
| 89 |
+
self.patch_cells({"A1": None})
|
| 90 |
+
self.assertEqual(self.events[-1]["ranges"], ["A1"])
|
| 91 |
+
|
| 92 |
+
def test_refresh_and_compatibility_share_the_same_commit_boundary(self):
|
| 93 |
+
book = sheets.get_workbook(self.st, self.book["id"])
|
| 94 |
+
book["sheets"][0]["cells"] = {"A1": {"kind": "literal", "value": 4}}
|
| 95 |
+
saved = sheets.put_workbook(self.st, self.book["id"], book, lossless=True)
|
| 96 |
+
self.assertEqual(len(self.events), 1)
|
| 97 |
+
sheet_compat.execute(self.st, self.book["id"], "range.values.write",
|
| 98 |
+
{"sheet": self.tab, "range": "A1", "values": [[5]],
|
| 99 |
+
"expected_revision": saved["rev"]}, actor="qa")
|
| 100 |
+
self.assertEqual(len(self.events), 2)
|
| 101 |
+
self.assertNotEqual(self.events[0]["batchId"], self.events[1]["batchId"])
|
| 102 |
+
|
| 103 |
+
def test_store_retry_publishes_only_the_successful_mutation(self):
|
| 104 |
+
original = self.st.update
|
| 105 |
+
|
| 106 |
+
def retry(key, fn, **kwargs):
|
| 107 |
+
if key == sheets.workbook_key(self.book["id"]):
|
| 108 |
+
discarded = self.st.get(key)
|
| 109 |
+
discarded["rev"] = 900
|
| 110 |
+
fn(discarded)
|
| 111 |
+
return original(key, fn, **kwargs)
|
| 112 |
+
|
| 113 |
+
with patch.object(self.st, "update", retry):
|
| 114 |
+
self.patch_cells({"A1": {"kind": "literal", "value": 2}})
|
| 115 |
+
self.assertEqual(len(self.events), 1)
|
| 116 |
+
self.assertLess(self.events[0]["revision"], 900)
|
| 117 |
+
|
| 118 |
+
def test_new_tab_backfill_is_not_an_existing_watch_change(self):
|
| 119 |
+
book = sheets.get_workbook(self.st, self.book["id"])
|
| 120 |
+
book["sheets"].append({"id": "imported", "name": "Imported", "cells": {
|
| 121 |
+
"A1": {"kind": "literal", "value": 17}}})
|
| 122 |
+
sheets.put_workbook(self.st, self.book["id"], book)
|
| 123 |
+
self.assertEqual(self.events, [])
|
| 124 |
+
|
| 125 |
+
def test_binding_refresh_publishes_output_change_but_not_refresh_stamp(self):
|
| 126 |
+
cell = {"kind": "bound", "value": 10, "binding": {
|
| 127 |
+
"database": "customers", "aggregate": "balance", "id": "current_ytd"}}
|
| 128 |
+
# Drive the persistence seam with already-canonical stored cells to isolate refresh
|
| 129 |
+
# notification from the independently tested binding-schema cleaner.
|
| 130 |
+
key = sheets.workbook_key(self.book["id"])
|
| 131 |
+
self.st.data[key]["sheets"][0]["cells"]["A1"] = copy.deepcopy(cell)
|
| 132 |
+
|
| 133 |
+
def refresh(state):
|
| 134 |
+
state["rev"] += 1
|
| 135 |
+
state["sheets"][0]["cells"]["A1"].update(value=11, refreshed_at="fixture-1")
|
| 136 |
+
return state, state
|
| 137 |
+
|
| 138 |
+
sheets._persist(self.st, key, refresh)
|
| 139 |
+
self.assertEqual(self.events[-1]["changeKinds"], ["value", "binding"])
|
| 140 |
+
self.assertIn("current_ytd", self.events[-1]["bindings"])
|
| 141 |
+
self.assertIn("customers.balance", self.events[-1]["bindings"])
|
| 142 |
+
self.events.clear()
|
| 143 |
+
sheets._persist(self.st, key, refresh)
|
| 144 |
+
self.assertEqual(self.events, [])
|
| 145 |
+
|
| 146 |
+
def test_notification_failure_does_not_turn_committed_edit_into_a_refusal(self):
|
| 147 |
+
with patch.object(sheet_events, "COMMIT_OBSERVER", side_effect=RuntimeError("private detail")), \
|
| 148 |
+
self.assertLogs("core.sheet_events", level="ERROR") as logs:
|
| 149 |
+
result = self.patch_cells({"A1": {"kind": "literal", "value": 42}})
|
| 150 |
+
self.assertEqual(sheets.get_workbook(self.st, self.book["id"])["rev"], result["rev"])
|
| 151 |
+
self.assertNotIn("private detail", str(logs.output))
|
| 152 |
+
|
| 153 |
+
def test_negative_control_detects_disconnected_commit_publisher(self):
|
| 154 |
+
result = unittest.TestResult()
|
| 155 |
+
with patch.object(sheet_events, "publish", return_value=None):
|
| 156 |
+
CommitEvents("test_bulk_patch_is_one_committed_event_without_cell_values").run(result)
|
| 157 |
+
self.assertEqual(len(result.failures), 1)
|
| 158 |
+
self.assertEqual(result.errors, [])
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
if __name__ == "__main__":
|
| 162 |
+
unittest.main()
|
platform/aios_grid.py
CHANGED
|
@@ -3577,10 +3577,13 @@ def _default_view_config(fields):
|
|
| 3577 |
# MUST move together, which is the whole subject of the note above.
|
| 3578 |
shown = [field["key"] for field in fields if _is_default_visible(field)]
|
| 3579 |
hidden = [field["key"] for field in fields if field["key"] not in shown]
|
|
|
|
|
|
|
|
|
|
| 3580 |
return {
|
| 3581 |
# `filters` is the ROOT of the filter tree: leaf conditions and/or nested
|
| 3582 |
# condition groups ({conj, children}); `filterConj` joins the root level.
|
| 3583 |
-
"filters":
|
| 3584 |
"sorts": [], "groupBy": None, "colorBy": None,
|
| 3585 |
"rowHeightMode": "short", "order": shown + hidden, "visible": shown,
|
| 3586 |
"widths": {}, "memberPids": [],
|
|
@@ -3702,7 +3705,7 @@ def views_from_defs(defs, saved_views, fields, system_name="All customers", lock
|
|
| 3702 |
op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq",
|
| 3703 |
"contains": "contains"}
|
| 3704 |
for name, definition in (defs or {}).items():
|
| 3705 |
-
filters = []
|
| 3706 |
for rule in definition.get("rules") or []:
|
| 3707 |
if rule.get("field") not in {field["key"] for field in fields}:
|
| 3708 |
continue
|
|
|
|
| 3577 |
# MUST move together, which is the whole subject of the note above.
|
| 3578 |
shown = [field["key"] for field in fields if _is_default_visible(field)]
|
| 3579 |
hidden = [field["key"] for field in fields if field["key"] not in shown]
|
| 3580 |
+
archive_filter = [{"colId": "archived", "op": "neq", "value": "true"}] if any(
|
| 3581 |
+
field.get("key") == "archived" and field.get("type") == "checkbox"
|
| 3582 |
+
and field.get("source") == "odoo" for field in fields) else []
|
| 3583 |
return {
|
| 3584 |
# `filters` is the ROOT of the filter tree: leaf conditions and/or nested
|
| 3585 |
# condition groups ({conj, children}); `filterConj` joins the root level.
|
| 3586 |
+
"filters": archive_filter, "filterConj": "and",
|
| 3587 |
"sorts": [], "groupBy": None, "colorBy": None,
|
| 3588 |
"rowHeightMode": "short", "order": shown + hidden, "visible": shown,
|
| 3589 |
"widths": {}, "memberPids": [],
|
|
|
|
| 3705 |
op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq",
|
| 3706 |
"contains": "contains"}
|
| 3707 |
for name, definition in (defs or {}).items():
|
| 3708 |
+
filters = [dict(rule) for rule in base["filters"]]
|
| 3709 |
for rule in definition.get("rules") or []:
|
| 3710 |
if rule.get("field") not in {field["key"] for field in fields}:
|
| 3711 |
continue
|
platform/aios_grid_fields.json
CHANGED
|
@@ -32,6 +32,14 @@
|
|
| 32 |
],
|
| 33 |
"description": "Whether this customer still exists in Odoo. Archived means deleted there."
|
| 34 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
{
|
| 36 |
"key": "agent",
|
| 37 |
"label": "Agent",
|
|
|
|
| 32 |
],
|
| 33 |
"description": "Whether this customer still exists in Odoo. Archived means deleted there."
|
| 34 |
},
|
| 35 |
+
{
|
| 36 |
+
"key": "archived",
|
| 37 |
+
"label": "Archived",
|
| 38 |
+
"type": "checkbox",
|
| 39 |
+
"source": "odoo",
|
| 40 |
+
"default": false,
|
| 41 |
+
"description": "Whether this customer is archived in Odoo."
|
| 42 |
+
},
|
| 43 |
{
|
| 44 |
"key": "agent",
|
| 45 |
"label": "Agent",
|
platform/core/audit.py
CHANGED
|
@@ -12,6 +12,8 @@ AUDIT_KINDS = frozenset({
|
|
| 12 |
"auth.mfa_failed", "auth.logout", "admin.grant", "credential.added",
|
| 13 |
"credential.rotated", "credential.revoked", "amazon.sync", "amazon.throttled",
|
| 14 |
"amazon.authorization_lost", "data.purge", "data.restore", "config.changed",
|
|
|
|
|
|
|
| 15 |
})
|
| 16 |
BUCKET = "security_events"
|
| 17 |
MAX_EVENTS = 2000
|
|
@@ -33,6 +35,7 @@ SECRET_KEYS = frozenset(_key_name(key) for key in (
|
|
| 33 |
))
|
| 34 |
_BEARER = re.compile(r"\bbearer\s+\S+", re.IGNORECASE)
|
| 35 |
_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
|
|
|
|
| 36 |
|
| 37 |
|
| 38 |
class AuditRefused(ValueError):
|
|
@@ -54,6 +57,8 @@ def _string(key, value):
|
|
| 54 |
raise AuditRefused(key, "bearer token shape")
|
| 55 |
if _JWT.search(value):
|
| 56 |
raise AuditRefused(key, "JWT token shape")
|
|
|
|
|
|
|
| 57 |
|
| 58 |
|
| 59 |
def _attribution(key, value):
|
|
@@ -174,3 +179,27 @@ def read(tenant, limit=200, now=None) -> list[dict]:
|
|
| 174 |
doc = runtime.get_runtime(str(tenant)).get(BUCKET)
|
| 175 |
rows = ordered(doc, tenant, limit)
|
| 176 |
return [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
"auth.mfa_failed", "auth.logout", "admin.grant", "credential.added",
|
| 13 |
"credential.rotated", "credential.revoked", "amazon.sync", "amazon.throttled",
|
| 14 |
"amazon.authorization_lost", "data.purge", "data.restore", "config.changed",
|
| 15 |
+
"external.query",
|
| 16 |
+
"external.write",
|
| 17 |
})
|
| 18 |
BUCKET = "security_events"
|
| 19 |
MAX_EVENTS = 2000
|
|
|
|
| 35 |
))
|
| 36 |
_BEARER = re.compile(r"\bbearer\s+\S+", re.IGNORECASE)
|
| 37 |
_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
|
| 38 |
+
_LOOPABLE_CREDENTIAL = re.compile(r"\blp(?:b|o|r|c)1\.[A-Za-z0-9._~-]+")
|
| 39 |
|
| 40 |
|
| 41 |
class AuditRefused(ValueError):
|
|
|
|
| 57 |
raise AuditRefused(key, "bearer token shape")
|
| 58 |
if _JWT.search(value):
|
| 59 |
raise AuditRefused(key, "JWT token shape")
|
| 60 |
+
if _LOOPABLE_CREDENTIAL.search(value):
|
| 61 |
+
raise AuditRefused(key, "Loopable credential shape")
|
| 62 |
|
| 63 |
|
| 64 |
def _attribution(key, value):
|
|
|
|
| 179 |
doc = runtime.get_runtime(str(tenant)).get(BUCKET)
|
| 180 |
rows = ordered(doc, tenant, limit)
|
| 181 |
return [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def external_write_receipt(tenant, principal, *, tool, auth_type, affected_count,
|
| 185 |
+
diff_digest, revision_before, revision_after,
|
| 186 |
+
confirmation_id) -> dict:
|
| 187 |
+
"""Emit and return a value-free C7 write receipt.
|
| 188 |
+
|
| 189 |
+
Target ids, field names, record values and arguments deliberately do not belong here. The
|
| 190 |
+
receipt proves who confirmed which normalized diff and how many objects moved; the concrete
|
| 191 |
+
before/after stays in the short-lived preview response instead of the retained security log.
|
| 192 |
+
"""
|
| 193 |
+
detail = {
|
| 194 |
+
"surface": "ai_api",
|
| 195 |
+
"tool": str(tool or "")[:128],
|
| 196 |
+
"authentication": str(auth_type or "")[:32],
|
| 197 |
+
"affected_count": max(0, int(affected_count or 0)),
|
| 198 |
+
"diff_digest": str(diff_digest or "")[:128],
|
| 199 |
+
"revision_before": str(revision_before or "")[:128],
|
| 200 |
+
"revision_after": str(revision_after or "")[:128],
|
| 201 |
+
"confirmation_id": str(confirmation_id or "")[:128],
|
| 202 |
+
}
|
| 203 |
+
item = event("external.write", tenant, principal, detail)
|
| 204 |
+
stored = record("external.write", tenant, principal, detail)
|
| 205 |
+
return {"event": item, "persisted": stored is not None}
|
platform/core/daily_sales_repair.py
ADDED
|
@@ -0,0 +1,977 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""A refusal-first, exact-snapshot repair for a Daily Sales workbook the caller names.
|
| 2 |
+
|
| 3 |
+
The legacy source-oracle plan and the workbook-derived invoice summary plan share the same
|
| 4 |
+
backup, exact-write and readback guards. Canonical periods use core.windows.normalize; the
|
| 5 |
+
summary uses the workbook's own filter valueRefs instead of freezing its calendar in a period.
|
| 6 |
+
The authorized operator injects the independent invoice validator and the full joined partner set.
|
| 7 |
+
The module validates the complete projected document, writes and verifies a backup, then delegates
|
| 8 |
+
one atomic, preconditioned cell mutation to :mod:`core.sheets`.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import copy
|
| 14 |
+
import json
|
| 15 |
+
import math
|
| 16 |
+
import re
|
| 17 |
+
from datetime import date
|
| 18 |
+
|
| 19 |
+
from core import sheets
|
| 20 |
+
from core import windows as W
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
TARGET_WORKBOOK_ID = "b8edba3d3f47"
|
| 24 |
+
TARGET_TENANT = "royal-imports"
|
| 25 |
+
SUMMARY_TAB = "Summary (All)"
|
| 26 |
+
SUMMARY_TAB_ID = "c2545d26c42c"
|
| 27 |
+
SUMMARY_COLUMNS = tuple(chr(column) for column in range(ord("F"), ord("R") + 1))
|
| 28 |
+
TICKET_FORMULA_ERROR_CODES = frozenset({"#VALUE!", "#DIV/0!", "#REF!", "#N/A", "#NAME?"})
|
| 29 |
+
_EXCEL_ERROR_ALIASES = {
|
| 30 |
+
"bad_value": "#VALUE!", "div_zero": "#DIV/0!", "invalid_reference": "#REF!",
|
| 31 |
+
"external_reference": "#REF!", "no_match": "#N/A", "unknown_name": "#NAME?",
|
| 32 |
+
}
|
| 33 |
+
_SUMMARY_RATIO_ROOTS = {
|
| 34 |
+
"F69": "=(F75/F64) * (365 / (F$7 - DATE(YEAR(F$7),1,1) + 1))",
|
| 35 |
+
"F70": "=(F76/F67) * (365 / (F$7 - DATE(YEAR(F$7),1,1) + 1))",
|
| 36 |
+
"R70": "=(R76/R67) * (365 / (R$7 - DATE(YEAR(R$7),1,1) + 1))",
|
| 37 |
+
}
|
| 38 |
+
_SUMMARY_RATIO_CASCADES = {
|
| 39 |
+
"F71": "=F69-F70", "F72": "=F71/F70", "F73": "=G72-F72",
|
| 40 |
+
"R71": "=R69-R70", "R72": "=R71/R70", "R73": "=R72-Q72",
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def operation_for(workbook_id):
|
| 45 |
+
"""Bind a repair operation to the workbook the caller names."""
|
| 46 |
+
return "repair-daily-sales-" + workbook_id
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
OPERATION = operation_for(TARGET_WORKBOOK_ID)
|
| 50 |
+
YTD_NAMES = {"Current YTD": "ytd", "Previous YTD": "ytd_last_year"}
|
| 51 |
+
if not all(kind in W.WINDOW_KINDS for kind in YTD_NAMES.values()):
|
| 52 |
+
raise RuntimeError("C2 requires every YTD window kind to exist in core.windows.WINDOW_KINDS.")
|
| 53 |
+
|
| 54 |
+
_VISUAL_TAB_KEYS = (
|
| 55 |
+
"colWidths", "rowHeight", "rowHeights", "freeze", "rules", "hiddenRows", "hiddenCols",
|
| 56 |
+
"charts", "visibility", "view", "print", "excel",
|
| 57 |
+
)
|
| 58 |
+
_VISUAL_CELL_KEYS = ("style", "validation", "link", "note", "sparkline", "excel")
|
| 59 |
+
_FORBIDDEN_BINDING_KEYS = {"fallback", "static", "staticValue", "defaultValue"}
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class RepairRefusal(sheets.SheetRefusal):
|
| 63 |
+
"""A named refusal from the exceptional repair workflow."""
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _canonical(value):
|
| 67 |
+
return json.loads(json.dumps(value, sort_keys=True, default=str))
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _tab(book, name):
|
| 71 |
+
try:
|
| 72 |
+
return sheets.find_tab(book, name)
|
| 73 |
+
except sheets.SheetRefusal as exc:
|
| 74 |
+
raise RepairRefusal(exc.code, exc.message) from exc
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _addr(raw):
|
| 78 |
+
value = str(raw or "").strip().upper().replace("$", "")
|
| 79 |
+
match = re.fullmatch(r"([A-Z]{1,3})([1-9][0-9]{0,6})", value)
|
| 80 |
+
if not match:
|
| 81 |
+
raise RepairRefusal("repair_address_invalid", "Every repair source needs one A1 address.")
|
| 82 |
+
column = 0
|
| 83 |
+
for char in match.group(1):
|
| 84 |
+
column = column * 26 + ord(char) - ord("A") + 1
|
| 85 |
+
if column > sheets.DOCUMENT_MAX_COLS or int(match.group(2)) > sheets.DOCUMENT_MAX_ROWS:
|
| 86 |
+
raise RepairRefusal("repair_address_invalid",
|
| 87 |
+
"A repair source address is outside the workbook dimensions.")
|
| 88 |
+
return value
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _validate_marker(marker, expected_revision, workbook_id):
|
| 92 |
+
if not isinstance(marker, dict) or marker.get("direct_tenant_data_live") is not True:
|
| 93 |
+
raise RepairRefusal(
|
| 94 |
+
"repair_marker_required",
|
| 95 |
+
"This repair requires the direct tenant data live marker from the owner instruction.")
|
| 96 |
+
expected = {
|
| 97 |
+
"operation": operation_for(workbook_id),
|
| 98 |
+
"tenant": TARGET_TENANT,
|
| 99 |
+
"workbook_id": workbook_id,
|
| 100 |
+
"expected_revision": expected_revision,
|
| 101 |
+
}
|
| 102 |
+
if any(marker.get(key) != value for key, value in expected.items()):
|
| 103 |
+
raise RepairRefusal(
|
| 104 |
+
"repair_marker_mismatch",
|
| 105 |
+
"The direct tenant data marker does not exactly match this workbook and revision.")
|
| 106 |
+
if not str(marker.get("owner_instruction") or "").strip():
|
| 107 |
+
raise RepairRefusal(
|
| 108 |
+
"repair_owner_instruction_required",
|
| 109 |
+
"The direct tenant data marker must carry the owner's exact repair instruction.")
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def _contains_forbidden_key(value):
|
| 113 |
+
if isinstance(value, dict):
|
| 114 |
+
return bool(_FORBIDDEN_BINDING_KEYS.intersection(value)) or any(
|
| 115 |
+
_contains_forbidden_key(child) for child in value.values())
|
| 116 |
+
if isinstance(value, list):
|
| 117 |
+
return any(_contains_forbidden_key(child) for child in value)
|
| 118 |
+
return False
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _validate_binding(name, binding, *, window_kind=None):
|
| 122 |
+
if not isinstance(binding, dict) or _contains_forbidden_key(binding):
|
| 123 |
+
raise RepairRefusal(
|
| 124 |
+
"repair_binding_static",
|
| 125 |
+
name + " must be a canonical database binding without a static fallback.")
|
| 126 |
+
if not str(binding.get("database") or "").strip():
|
| 127 |
+
raise RepairRefusal("repair_binding_invalid", name + " must name its canonical database.")
|
| 128 |
+
aggregate = binding.get("aggregate")
|
| 129 |
+
if not isinstance(aggregate, dict) or not (
|
| 130 |
+
aggregate.get("metric_key") or aggregate.get("fn")):
|
| 131 |
+
raise RepairRefusal("repair_binding_invalid", name + " must name a real aggregate.")
|
| 132 |
+
if not aggregate.get("metric_key") and (
|
| 133 |
+
aggregate.get("fn") not in {"sum", "count", "avg", "min", "max"}
|
| 134 |
+
or (aggregate["fn"] != "count" and not aggregate.get("field"))):
|
| 135 |
+
raise RepairRefusal("repair_binding_invalid", name + " must name a real column aggregate.")
|
| 136 |
+
scope = binding.get("scope")
|
| 137 |
+
if not isinstance(scope, dict):
|
| 138 |
+
scope = {}
|
| 139 |
+
has_view = bool(str(scope.get("view_id") or "").strip())
|
| 140 |
+
tree = scope.get("filter")
|
| 141 |
+
# Accept the same explicit, nonempty conjunction used by rows 108-113. Requiring
|
| 142 |
+
# both referenced date bounds keeps a missing period from becoming an unbounded sum.
|
| 143 |
+
nodes = tree.get("nodes") if isinstance(tree, dict) else None
|
| 144 |
+
has_filter = (isinstance(nodes, list) and bool(nodes) and tree.get("conj") == "and"
|
| 145 |
+
and all(isinstance(n, dict) and n.get("colId") and n.get("op")
|
| 146 |
+
and ("value" in n or isinstance(n.get("valueRef"), dict))
|
| 147 |
+
for n in nodes))
|
| 148 |
+
if not has_view and not has_filter:
|
| 149 |
+
raise RepairRefusal(
|
| 150 |
+
"repair_binding_scope_required",
|
| 151 |
+
name + " must use a saved View or an explicit nonempty filter scope.")
|
| 152 |
+
if "related" in binding:
|
| 153 |
+
from core.sheet_binding import BindingMetadataRefusal, clean_related
|
| 154 |
+
try:
|
| 155 |
+
clean_related(binding["related"])
|
| 156 |
+
except BindingMetadataRefusal as exc:
|
| 157 |
+
raise RepairRefusal(exc.code, exc.message) from exc
|
| 158 |
+
period = binding.get("period")
|
| 159 |
+
if period is None and has_filter and not has_view:
|
| 160 |
+
bounds = {}
|
| 161 |
+
for node in nodes:
|
| 162 |
+
slot = node.get("valueRef")
|
| 163 |
+
if slot is not None:
|
| 164 |
+
if not isinstance(slot, dict) or set(slot) != {"ref"}:
|
| 165 |
+
raise RepairRefusal("repair_binding_invalid", name + " has an invalid cell reference.")
|
| 166 |
+
_addr(slot["ref"])
|
| 167 |
+
if node["op"] in {"gte", "lte"}:
|
| 168 |
+
bounds.setdefault(node["colId"], set()).add(node["op"])
|
| 169 |
+
if any(ops == {"gte", "lte"} for ops in bounds.values()):
|
| 170 |
+
return
|
| 171 |
+
window = W.normalize(copy.deepcopy(period.get("window") if isinstance(period, dict) else None))
|
| 172 |
+
if window is None or (window_kind is not None and window["kind"] != window_kind):
|
| 173 |
+
expected_window = "the " + window_kind if window_kind is not None else "a valid"
|
| 174 |
+
raise RepairRefusal(
|
| 175 |
+
"repair_binding_period_invalid",
|
| 176 |
+
name + " must use " + expected_window + " canonical date window.")
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def _preserve_visual(before, desired):
|
| 180 |
+
"""Carry every T20 cell display property across a content-only repair."""
|
| 181 |
+
held = {key: copy.deepcopy((before or {}).get(key)) for key in _VISUAL_CELL_KEYS
|
| 182 |
+
if isinstance(before, dict) and key in before}
|
| 183 |
+
if desired is None:
|
| 184 |
+
return {"kind": "empty", "state": "typed", **held} if held else None
|
| 185 |
+
out = copy.deepcopy(desired)
|
| 186 |
+
for key, value in held.items():
|
| 187 |
+
out[key] = value
|
| 188 |
+
return out
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def visual_metadata(book):
|
| 192 |
+
"""The T20 visual strata, including per-cell display properties."""
|
| 193 |
+
tabs = []
|
| 194 |
+
for tab in book.get("sheets") or ():
|
| 195 |
+
if not isinstance(tab, dict):
|
| 196 |
+
continue
|
| 197 |
+
item = {"id": tab.get("id"), "name": tab.get("name")}
|
| 198 |
+
for key in _VISUAL_TAB_KEYS:
|
| 199 |
+
if key in tab:
|
| 200 |
+
item[key] = copy.deepcopy(tab[key])
|
| 201 |
+
cell_visuals = {}
|
| 202 |
+
for address, cell in (tab.get("cells") or {}).items():
|
| 203 |
+
kept = {key: copy.deepcopy(cell[key]) for key in _VISUAL_CELL_KEYS
|
| 204 |
+
if isinstance(cell, dict) and key in cell}
|
| 205 |
+
if kept:
|
| 206 |
+
cell_visuals[address] = kept
|
| 207 |
+
item["cells"] = cell_visuals
|
| 208 |
+
tabs.append(item)
|
| 209 |
+
return _canonical(tabs)
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def formula_error_census(book):
|
| 213 |
+
"""Return every stored formula that still carries a calculation refusal/error."""
|
| 214 |
+
errors = []
|
| 215 |
+
for tab in book.get("sheets") or ():
|
| 216 |
+
for address, cell in (tab.get("cells") or {}).items():
|
| 217 |
+
if not isinstance(cell, dict) or cell.get("kind") != "formula":
|
| 218 |
+
continue
|
| 219 |
+
error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
|
| 220 |
+
if error:
|
| 221 |
+
errors.append({"sheet": tab.get("name"), "address": address,
|
| 222 |
+
"code": error.get("code") if isinstance(error, dict) else str(error)})
|
| 223 |
+
return sorted(errors, key=lambda row: (str(row["sheet"]), row["address"]))
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def excel_formula_error_census(book, *, sheet=SUMMARY_TAB):
|
| 227 |
+
"""The ticket's five Excel codes, separate from the unchanged full error census."""
|
| 228 |
+
errors = []
|
| 229 |
+
for tab in book.get("sheets") or ():
|
| 230 |
+
if sheet is not None and tab.get("name") != sheet:
|
| 231 |
+
continue
|
| 232 |
+
for address, cell in (tab.get("cells") or {}).items():
|
| 233 |
+
if not isinstance(cell, dict) or cell.get("kind") != "formula":
|
| 234 |
+
continue
|
| 235 |
+
error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
|
| 236 |
+
if not error:
|
| 237 |
+
continue
|
| 238 |
+
code = error.get("code") if isinstance(error, dict) else str(error)
|
| 239 |
+
excel_code = error.get("excelCode") if isinstance(error, dict) else None
|
| 240 |
+
excel_code = excel_code or _EXCEL_ERROR_ALIASES.get(code, code)
|
| 241 |
+
if excel_code in TICKET_FORMULA_ERROR_CODES:
|
| 242 |
+
errors.append({"sheet": tab.get("name"), "address": address,
|
| 243 |
+
"code": code, "excelCode": excel_code})
|
| 244 |
+
return sorted(errors, key=lambda row: (str(row["sheet"]), row["address"]))
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def check_summary_formula_errors(book):
|
| 248 |
+
"""Count the real tab only after proving this same census finds planted errors.
|
| 249 |
+
|
| 250 |
+
The negative control is an isolated in-memory document, never a tenant edit. Exercise
|
| 251 |
+
both code spellings, excelCode precedence, and the refused-cell fallback on every call.
|
| 252 |
+
"""
|
| 253 |
+
_tab(book, SUMMARY_TAB)
|
| 254 |
+
planted, expected = {}, {}
|
| 255 |
+
for internal, excel in (("bad_value", "#VALUE!"), ("div_zero", "#DIV/0!"),
|
| 256 |
+
("invalid_reference", "#REF!"), ("no_match", "#N/A"),
|
| 257 |
+
("unknown_name", "#NAME?")):
|
| 258 |
+
for field in ("error", "refusal"):
|
| 259 |
+
for payload in ({"code": internal, "excelCode": excel}, {"code": excel},
|
| 260 |
+
{"code": internal}, {"code": "probe", "excelCode": excel}):
|
| 261 |
+
address = "A" + str(len(planted) + 1)
|
| 262 |
+
planted[address] = {"kind": "formula", "state": "refused",
|
| 263 |
+
"formula": "=1/0", field: payload}
|
| 264 |
+
expected[address] = excel
|
| 265 |
+
for index, code in enumerate(("circular_reference", "blank", "#NUM!"), 1):
|
| 266 |
+
planted["B" + str(index)] = {"kind": "formula", "state": "refused",
|
| 267 |
+
"formula": "=1/0", "error": {"code": code}}
|
| 268 |
+
probe = {"sheets": [{"name": SUMMARY_TAB, "cells": planted},
|
| 269 |
+
{"name": "Summary (SR)", "cells": {"A1": planted["A1"]}}]}
|
| 270 |
+
found = excel_formula_error_census(probe)
|
| 271 |
+
if len(found) != len(expected) or {row["address"]: row["excelCode"] for row in found} != expected:
|
| 272 |
+
raise RepairRefusal("repair_formula_census_blind",
|
| 273 |
+
"The formula error census failed to find its planted errors.")
|
| 274 |
+
errors = excel_formula_error_census(book)
|
| 275 |
+
return {"sheet": SUMMARY_TAB, "codes": sorted(TICKET_FORMULA_ERROR_CODES),
|
| 276 |
+
"count": len(errors), "errors": errors,
|
| 277 |
+
"negative_control": {"planted": len(expected), "found": len(found)}}
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def _formula_error_comparison(before, after, edits, reported_errors=()):
|
| 281 |
+
"""Reject new errors anywhere and every residual error in explicitly planned cells.
|
| 282 |
+
|
| 283 |
+
Compare identities, not totals: fixing one cell cannot pay for breaking another. Code
|
| 284 |
+
aliases denote the same failure; unplanned cell changes still have their separate guard.
|
| 285 |
+
Opaque engine diagnostics cannot be matched to a pre-existing error and fail closed.
|
| 286 |
+
"""
|
| 287 |
+
def identity(row):
|
| 288 |
+
if not isinstance(row, dict) or not row.get("sheet") or not row.get("address"):
|
| 289 |
+
return None
|
| 290 |
+
code = row.get("excelCode") or row.get("code")
|
| 291 |
+
return (str(row["sheet"]).casefold(), str(row["address"]).upper(),
|
| 292 |
+
_EXCEL_ERROR_ALIASES.get(code, code))
|
| 293 |
+
|
| 294 |
+
baseline = {identity(row) for row in formula_error_census(before)}
|
| 295 |
+
repaired = {(row["sheet"].casefold(), row["address"]) for row in edits}
|
| 296 |
+
introduced, remaining, unchanged, seen = [], [], [], set()
|
| 297 |
+
for row in [*formula_error_census(after), *reported_errors]:
|
| 298 |
+
key = identity(row)
|
| 299 |
+
if key is not None and key in seen:
|
| 300 |
+
continue
|
| 301 |
+
seen.add(key)
|
| 302 |
+
if key is None or key not in baseline:
|
| 303 |
+
introduced.append(copy.deepcopy(row))
|
| 304 |
+
if key is not None and key[:2] in repaired:
|
| 305 |
+
remaining.append(copy.deepcopy(row))
|
| 306 |
+
elif key is not None and key in baseline:
|
| 307 |
+
unchanged.append(copy.deepcopy(row))
|
| 308 |
+
return {"introduced": introduced, "remaining_in_repair": remaining,
|
| 309 |
+
"preexisting_elsewhere": unchanged}
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def _project(book, edits):
|
| 313 |
+
projected = copy.deepcopy(book)
|
| 314 |
+
for edit in edits:
|
| 315 |
+
tab = _tab(projected, edit["sheet"])
|
| 316 |
+
cells = tab.setdefault("cells", {})
|
| 317 |
+
if edit["after"] is None:
|
| 318 |
+
cells.pop(edit["address"], None)
|
| 319 |
+
else:
|
| 320 |
+
cells[edit["address"]] = copy.deepcopy(edit["after"])
|
| 321 |
+
return projected
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
def _cell_map(book):
|
| 325 |
+
return {(str(tab.get("name") or "").casefold(), address): copy.deepcopy(cell)
|
| 326 |
+
for tab in book.get("sheets") or () if isinstance(tab, dict)
|
| 327 |
+
for address, cell in (tab.get("cells") or {}).items()}
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def _checked_projection(projected, edits, formula_check):
|
| 331 |
+
"""Make formula output part of the plan, never an unpersisted preflight side effect."""
|
| 332 |
+
if not callable(formula_check):
|
| 333 |
+
errors = formula_error_census(projected)
|
| 334 |
+
return projected, errors
|
| 335 |
+
result = formula_check(copy.deepcopy(projected))
|
| 336 |
+
if isinstance(result, dict) and isinstance(result.get("book"), dict):
|
| 337 |
+
checked = result["book"]
|
| 338 |
+
errors = list(result.get("errors") or ())
|
| 339 |
+
else:
|
| 340 |
+
checked = copy.deepcopy(projected)
|
| 341 |
+
errors = list(result or ())
|
| 342 |
+
before, after = _cell_map(projected), _cell_map(checked)
|
| 343 |
+
planned = {(row["sheet"].casefold(), row["address"]): row for row in edits}
|
| 344 |
+
sheet_names = {str(tab.get("name") or "").casefold(): tab.get("name")
|
| 345 |
+
for tab in projected.get("sheets") or () if isinstance(tab, dict)}
|
| 346 |
+
for slot in sorted(before.keys() | after.keys()):
|
| 347 |
+
if before.get(slot) == after.get(slot):
|
| 348 |
+
continue
|
| 349 |
+
row = planned.get(slot)
|
| 350 |
+
if row is None or row["kind"] == "recalculated":
|
| 351 |
+
previous, recalculated = before.get(slot), after.get(slot)
|
| 352 |
+
# Recalculation may refresh a healthy formula's result, but may not repair
|
| 353 |
+
# an unrelated refused formula or change authored content or presentation.
|
| 354 |
+
if not all(isinstance(cell, dict) and cell.get("kind") == "formula"
|
| 355 |
+
and isinstance(cell.get("formula"), str)
|
| 356 |
+
and not cell.get("error") and not cell.get("refusal")
|
| 357 |
+
and cell.get("state") != "refused"
|
| 358 |
+
for cell in (previous, recalculated)) \
|
| 359 |
+
or previous.get("formula") != recalculated.get("formula") \
|
| 360 |
+
or {key: previous[key] for key in _VISUAL_CELL_KEYS if key in previous} \
|
| 361 |
+
!= {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated}:
|
| 362 |
+
raise RepairRefusal(
|
| 363 |
+
"repair_unplanned_formula_change",
|
| 364 |
+
"Formula validation would change a cell outside the supplied source diff.")
|
| 365 |
+
desired = sheets.clean_cell(_preserve_visual(previous, recalculated), slot[1])
|
| 366 |
+
if row is None:
|
| 367 |
+
sheet = sheet_names[slot[0]]
|
| 368 |
+
row = {"kind": "recalculated", "name": sheet + "!" + slot[1],
|
| 369 |
+
"sheet": sheet, "address": slot[1],
|
| 370 |
+
"before": copy.deepcopy(previous), "after": desired}
|
| 371 |
+
edits.append(row)
|
| 372 |
+
planned[slot] = row
|
| 373 |
+
else:
|
| 374 |
+
row["after"] = desired
|
| 375 |
+
continue
|
| 376 |
+
if row is None or row["kind"] not in {"source", "derived"} \
|
| 377 |
+
or not isinstance(row.get("after"), dict) \
|
| 378 |
+
or row["after"].get("kind") != "formula":
|
| 379 |
+
raise RepairRefusal(
|
| 380 |
+
"repair_unplanned_formula_change",
|
| 381 |
+
"Formula validation would change a cell outside the supplied source diff.")
|
| 382 |
+
recalculated = after.get(slot)
|
| 383 |
+
if not isinstance(recalculated, dict) \
|
| 384 |
+
or (row["kind"] == "source"
|
| 385 |
+
and recalculated.get("value") != row.get("cached_value")):
|
| 386 |
+
raise RepairRefusal(
|
| 387 |
+
"repair_formula_cache_mismatch",
|
| 388 |
+
row["name"] + " does not recalculate to its supplied cached Excel value.")
|
| 389 |
+
row["after"] = sheets.clean_cell(recalculated, row["address"])
|
| 390 |
+
return _project(projected, [
|
| 391 |
+
{**row, "before": before.get((row["sheet"].casefold(), row["address"]))}
|
| 392 |
+
for row in edits
|
| 393 |
+
if before.get((row["sheet"].casefold(), row["address"])) != row["after"]
|
| 394 |
+
]), errors
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
def summary_block_snapshot(book):
|
| 398 |
+
"""Read-only QI3 pin. The lead saves this as proto/before/QI3-summary-block.json.
|
| 399 |
+
|
| 400 |
+
This is the measured F31:R35 extent, not a search for a plausible summary. Labels and
|
| 401 |
+
units are included in the evidence but are never proposed as edits.
|
| 402 |
+
"""
|
| 403 |
+
tab = _tab(book, SUMMARY_TAB)
|
| 404 |
+
cells = tab.get("cells") or {}
|
| 405 |
+
labels = {28: ("C", "Invoice Data Metrics"), 30: ("D", "Total Sales"),
|
| 406 |
+
31: ("D", "Current YTD"), 32: ("D", "Previous YTD"),
|
| 407 |
+
33: ("D", "YTD Growth"), 34: ("D", "YTD Growth"),
|
| 408 |
+
35: ("D", "YTD improvement")}
|
| 409 |
+
if book.get("id") != TARGET_WORKBOOK_ID or tab.get("id") != SUMMARY_TAB_ID or any(
|
| 410 |
+
str((cells.get(col + str(row)) or {}).get("value") or "").strip() != label
|
| 411 |
+
for row, (col, label) in labels.items()):
|
| 412 |
+
raise RepairRefusal("repair_summary_extent_mismatch",
|
| 413 |
+
"The workbook does not match the pinned Invoice Data Metrics block.")
|
| 414 |
+
addresses = {col + str(row) for col in ("D", "E", *SUMMARY_COLUMNS)
|
| 415 |
+
for row in range(31, 36)} | {"C28", "D30", "D10"}
|
| 416 |
+
addresses |= {col + str(row) for col in SUMMARY_COLUMNS for row in (5, 6, 7)}
|
| 417 |
+
addresses |= {addr for addr in cells if re.fullmatch(r"[A-Z]+2", addr)}
|
| 418 |
+
return {"workbook_id": book["id"], "revision": book.get("rev"),
|
| 419 |
+
"workbook_digest": sheets.workbook_digest(book), "sheet": SUMMARY_TAB,
|
| 420 |
+
"sheet_id": tab["id"], "figure_extent": "F31:R35", "figure_count": 65,
|
| 421 |
+
"cells": {addr: copy.deepcopy(cells.get(addr)) for addr in sorted(addresses)}}
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def _summary_value(book, address):
|
| 425 |
+
cell = (_tab(book, SUMMARY_TAB).get("cells") or {}).get(address) or {}
|
| 426 |
+
if cell.get("error") or cell.get("state") == "refused":
|
| 427 |
+
raise RepairRefusal("repair_summary_input_unreadable",
|
| 428 |
+
"Summary input " + address + " has a calculation error.")
|
| 429 |
+
return cell.get("value")
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
def _summary_date(value):
|
| 433 |
+
try:
|
| 434 |
+
return date.fromisoformat(str(value))
|
| 435 |
+
except ValueError as exc:
|
| 436 |
+
raise RepairRefusal("repair_summary_date_invalid",
|
| 437 |
+
"Every summary date must be a real ISO date from the sheet or mirror.") from exc
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
def _resolved(binding, book, resolve_binding):
|
| 441 |
+
if not callable(resolve_binding):
|
| 442 |
+
raise RepairRefusal("repair_binding_resolver_required", "The summary needs its scoped source resolver.")
|
| 443 |
+
result = resolve_binding(copy.deepcopy(binding), book, _tab(book, SUMMARY_TAB))
|
| 444 |
+
if not isinstance(result, dict) or "value" not in result or result.get("value") is None:
|
| 445 |
+
raise RepairRefusal("repair_binding_unreadable", "A summary source did not return a value.")
|
| 446 |
+
if result.get("limits"):
|
| 447 |
+
# The route wraps this exception, retaining its message but not custom attributes.
|
| 448 |
+
# Include each cause and fix there too; upstream clause separators become sentences.
|
| 449 |
+
details = []
|
| 450 |
+
for limit in result["limits"]:
|
| 451 |
+
if isinstance(limit, dict):
|
| 452 |
+
cause = str(limit.get("cause") or "The source reported an incomplete answer.")
|
| 453 |
+
fix = str(limit.get("recommendation") or "Repair the source scope and retry.")
|
| 454 |
+
details.append("Cause: " + cause + " Recommended fix: " + fix)
|
| 455 |
+
detail = " ".join(details).replace(chr(8212), ". ").replace(chr(8211), " to ")
|
| 456 |
+
exc = RepairRefusal("repair_binding_limited",
|
| 457 |
+
"A summary source could not apply its complete scope. " + detail)
|
| 458 |
+
exc.limits = copy.deepcopy(result["limits"])
|
| 459 |
+
raise exc
|
| 460 |
+
return result["value"]
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
def _summary_formula_projection(book, edits, formula_check):
|
| 464 |
+
if not callable(formula_check):
|
| 465 |
+
raise RepairRefusal("repair_formula_engine_required",
|
| 466 |
+
"The summary requires the real document formula engine.")
|
| 467 |
+
result = formula_check(copy.deepcopy(book))
|
| 468 |
+
if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
|
| 469 |
+
raise RepairRefusal("repair_formula_engine_required",
|
| 470 |
+
"Formula validation must return the calculated document.")
|
| 471 |
+
# Only adopt planned formulas here. The final whole-document projection records healthy
|
| 472 |
+
# downstream recalculations as exact edits and refuses every other unplanned change.
|
| 473 |
+
for row in edits:
|
| 474 |
+
if row["kind"] == "derived":
|
| 475 |
+
cell = (_tab(result["book"], row["sheet"]).get("cells") or {}).get(row["address"])
|
| 476 |
+
if not isinstance(cell, dict) or cell.get("kind") != "formula" or cell.get("error") \
|
| 477 |
+
or cell.get("state") != "fresh":
|
| 478 |
+
raise RepairRefusal("repair_formula_errors",
|
| 479 |
+
"Summary formula " + row["address"] + " could not calculate.")
|
| 480 |
+
row["after"] = sheets.clean_cell(cell, row["address"])
|
| 481 |
+
return _project(book, edits)
|
| 482 |
+
|
| 483 |
+
|
| 484 |
+
def _append_summary_ratio_plan(current, edits, formula_check):
|
| 485 |
+
"""Plan the measured roots and let the real engine decide cascade blank handling."""
|
| 486 |
+
if not callable(formula_check):
|
| 487 |
+
raise RepairRefusal("repair_formula_engine_required",
|
| 488 |
+
"The ratio repair requires the real document formula engine.")
|
| 489 |
+
cells = _tab(current, SUMMARY_TAB).get("cells") or {}
|
| 490 |
+
planned = {(row["sheet"].casefold(), row["address"]) for row in edits}
|
| 491 |
+
ratios = {}
|
| 492 |
+
for address, authored in {**_SUMMARY_RATIO_ROOTS, **_SUMMARY_RATIO_CASCADES}.items():
|
| 493 |
+
before = copy.deepcopy(cells.get(address))
|
| 494 |
+
guarded = '=IFERROR(' + authored[1:] + ',"")'
|
| 495 |
+
if not isinstance(before, dict) or before.get("kind") != "formula" \
|
| 496 |
+
or before.get("formula") not in (authored, guarded):
|
| 497 |
+
raise RepairRefusal("repair_ratio_formula_mismatch",
|
| 498 |
+
"The measured ratio formula has changed at " + address + ".")
|
| 499 |
+
if (SUMMARY_TAB.casefold(), address) in planned:
|
| 500 |
+
raise RepairRefusal("repair_edit_duplicate", "A repair cell may be named only once.")
|
| 501 |
+
desired = copy.deepcopy(before)
|
| 502 |
+
for key in ("value", "display", "error", "refusal"):
|
| 503 |
+
desired.pop(key, None)
|
| 504 |
+
desired.update(state="stale", formula=(guarded if address in _SUMMARY_RATIO_ROOTS
|
| 505 |
+
else before["formula"]))
|
| 506 |
+
row = {"kind": "derived", "name": SUMMARY_TAB + "!" + address,
|
| 507 |
+
"sheet": SUMMARY_TAB, "address": address, "before": before,
|
| 508 |
+
"after": sheets.clean_cell(_preserve_visual(before, desired), address)}
|
| 509 |
+
edits.append(row)
|
| 510 |
+
ratios[address] = row
|
| 511 |
+
result = formula_check(_project(current, edits))
|
| 512 |
+
if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
|
| 513 |
+
raise RepairRefusal("repair_formula_engine_required",
|
| 514 |
+
"Formula validation must return the calculated document.")
|
| 515 |
+
calculated = _tab(result["book"], SUMMARY_TAB).get("cells") or {}
|
| 516 |
+
guarded_cascades = []
|
| 517 |
+
for address, authored in _SUMMARY_RATIO_CASCADES.items():
|
| 518 |
+
cell = calculated.get(address) or {}
|
| 519 |
+
error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
|
| 520 |
+
# The current engine refuses arithmetic over IFERROR's empty string as `blank`.
|
| 521 |
+
# Only that observed refusal authorizes wrapping a cascade; other failures remain
|
| 522 |
+
# visible to the existing projection refusal. Never replace a formula with a value.
|
| 523 |
+
if isinstance(error, dict) and error.get("code") == "blank":
|
| 524 |
+
ratios[address]["after"]["formula"] = '=IFERROR(' + authored[1:] + ',"")'
|
| 525 |
+
guarded_cascades.append(address)
|
| 526 |
+
return {"roots": list(_SUMMARY_RATIO_ROOTS), "guarded_cascades": guarded_cascades}
|
| 527 |
+
|
| 528 |
+
|
| 529 |
+
def _build_summary_plan(current, request, resolve_binding, formula_check):
|
| 530 |
+
"""Build all 65 figures, date/scope helpers and guarded ratios, without tenant writes.
|
| 531 |
+
|
| 532 |
+
summary_block carries before=summary_block_snapshot(current), pricelist, the complete
|
| 533 |
+
partner_ids behind customer_link, and validate_invoice=modules.sheet.validate. The last
|
| 534 |
+
is a callable injected by the trusted operator (this workflow has no HTTP route).
|
| 535 |
+
No core-to-modules/API import or second source-query door is needed.
|
| 536 |
+
"""
|
| 537 |
+
summary = request["summary_block"]
|
| 538 |
+
if not isinstance(summary, dict) or summary.get("before") != summary_block_snapshot(current):
|
| 539 |
+
raise RepairRefusal("repair_summary_pin_required",
|
| 540 |
+
"The summary requires the exact before snapshot recorded by the lane lead.")
|
| 541 |
+
if request.get("source_cells") or request.get("binding_cells"):
|
| 542 |
+
raise RepairRefusal("repair_summary_plan_conflict",
|
| 543 |
+
"Run the workbook-derived summary separately from an Excel source-oracle repair.")
|
| 544 |
+
if not str(request.get("refreshed_at") or "").strip():
|
| 545 |
+
raise RepairRefusal("repair_timestamp_required", "The summary needs an explicit refresh timestamp.")
|
| 546 |
+
cells = _tab(current, SUMMARY_TAB).get("cells") or {}
|
| 547 |
+
# Allocate four fully empty rows inside the existing sheet dimensions. Never overwrite
|
| 548 |
+
# authored content, add a tab, resize the sheet, or change hidden/print/visual metadata.
|
| 549 |
+
tab_rows = int(_tab(current, SUMMARY_TAB).get("rows") or 0)
|
| 550 |
+
helper_row = next((row for row in range(tab_rows - 3, 35, -1)
|
| 551 |
+
if all((cells.get(col + str(r)) or {}).get("kind", "empty") == "empty"
|
| 552 |
+
for r in range(row, row + 4) for col in SUMMARY_COLUMNS)), None)
|
| 553 |
+
if helper_row is None:
|
| 554 |
+
raise RepairRefusal("repair_summary_helper_space",
|
| 555 |
+
"The summary needs four empty rows in its existing sheet for date and scope formulas.")
|
| 556 |
+
edits = []
|
| 557 |
+
|
| 558 |
+
def add(address, desired, kind):
|
| 559 |
+
before = copy.deepcopy(cells.get(address))
|
| 560 |
+
edits.append({"kind": kind, "name": SUMMARY_TAB + "!" + address,
|
| 561 |
+
"sheet": SUMMARY_TAB, "address": address, "before": before,
|
| 562 |
+
"after": sheets.clean_cell(_preserve_visual(before, desired), address)})
|
| 563 |
+
|
| 564 |
+
def formula(address, text):
|
| 565 |
+
add(address, {"kind": "formula", "state": "stale", "formula": text}, "derived")
|
| 566 |
+
|
| 567 |
+
scope_ref = "F" + str(helper_row + 3)
|
| 568 |
+
formula(scope_ref, '=IF($D$10="RI","Royal 1 (USD)",IF($D$10="FFS","Fisch (USD)",NA()))')
|
| 569 |
+
windows = {}
|
| 570 |
+
for col in SUMMARY_COLUMNS:
|
| 571 |
+
end = _summary_date(_summary_value(current, col + "7"))
|
| 572 |
+
year_refs = [addr for addr, cell in cells.items() if re.fullmatch(r"[A-Z]+2", addr)
|
| 573 |
+
and str((cell or {}).get("value")) in {str(end.year), str(float(end.year))}]
|
| 574 |
+
# The year can be one merged/shared header. Prefer this column, otherwise require
|
| 575 |
+
# an unambiguous year cell; never guess between two independent calendar headers.
|
| 576 |
+
year_ref = col + "2" if col + "2" in year_refs else year_refs[0] if len(year_refs) == 1 else None
|
| 577 |
+
if not year_ref:
|
| 578 |
+
raise RepairRefusal("repair_summary_year_ambiguous",
|
| 579 |
+
"The summary needs an unambiguous year on row 2 for column " + col + ".")
|
| 580 |
+
first, previous_first, previous_end = (col + str(helper_row + offset) for offset in range(3))
|
| 581 |
+
# DATE/EDATE already return ISO dates. String concatenation preserves that value
|
| 582 |
+
# without introducing an inferred date style on a previously blank helper cell.
|
| 583 |
+
formula(first, '=DATE(' + year_ref + ',1,1)&""')
|
| 584 |
+
formula(previous_first, '=DATE(' + year_ref + '-1,1,1)&""')
|
| 585 |
+
formula(previous_end, '=EDATE(' + col + '7,-12)&""')
|
| 586 |
+
windows[col] = ((first, col + "7"), (previous_first, previous_end))
|
| 587 |
+
projected = _summary_formula_projection(_project(current, edits), edits, formula_check)
|
| 588 |
+
if _summary_value(projected, scope_ref) != summary.get("pricelist"):
|
| 589 |
+
raise RepairRefusal("repair_summary_partner_scope",
|
| 590 |
+
"The supplied partner set must use the pricelist computed from D10.")
|
| 591 |
+
partners = summary.get("partner_ids")
|
| 592 |
+
if not isinstance(partners, list) or any(type(pid) is not int or pid <= 0 for pid in partners) \
|
| 593 |
+
or len(set(partners)) != len(partners):
|
| 594 |
+
raise RepairRefusal("repair_summary_partner_scope",
|
| 595 |
+
"Supply every joined partner ID once, with no source row limit.")
|
| 596 |
+
if not callable(summary.get("validate_invoice")):
|
| 597 |
+
raise RepairRefusal("repair_summary_validator_required",
|
| 598 |
+
"Inject modules.sheet.validate for independent invoice reconciliation.")
|
| 599 |
+
as_of = _summary_date(_resolved({"database": "ut_odoo_invoices",
|
| 600 |
+
"aggregate": {"kind": "column", "fn": "max", "field": "invoice_date"},
|
| 601 |
+
"scope": {"filter": {"conj": "and", "nodes": []}}},
|
| 602 |
+
projected, resolve_binding))
|
| 603 |
+
for col, ranges in windows.items():
|
| 604 |
+
for row_number, (first, last) in zip((31, 32), ranges):
|
| 605 |
+
binding = {"database": "ut_odoo_invoices",
|
| 606 |
+
"aggregate": {"field": "amount_untaxed", "fn": "sum", "kind": "column"},
|
| 607 |
+
"scope": {"filter": {"conj": "and", "nodes": [
|
| 608 |
+
{"colId": "invoice_date", "op": "gte", "valueRef": {"ref": first}},
|
| 609 |
+
{"colId": "invoice_date", "op": "lte", "valueRef": {"ref": last}},
|
| 610 |
+
{"colId": "move_type", "op": "eq", "value": "out_invoice"}]}},
|
| 611 |
+
"related": {"via": "customer_link", "filter": {"conj": "and", "nodes": [
|
| 612 |
+
{"colId": "pricelist", "op": "eq", "valueRef": {"ref": scope_ref}}]}}}
|
| 613 |
+
_validate_binding(col + str(row_number), binding)
|
| 614 |
+
value = _resolved(binding, projected, resolve_binding)
|
| 615 |
+
if type(value) not in (int, float) or not math.isfinite(value):
|
| 616 |
+
raise RepairRefusal("repair_binding_unreadable", "An invoice total must be a finite number.")
|
| 617 |
+
add(col + str(row_number), {"kind": "bound", "state": "fresh", "binding": binding,
|
| 618 |
+
"value": value, "display": str(value),
|
| 619 |
+
"refreshed_at": request["refreshed_at"]}, "binding")
|
| 620 |
+
from openpyxl.formula.translate import Translator
|
| 621 |
+
for row_number in range(33, 36):
|
| 622 |
+
origin = "F" + str(row_number)
|
| 623 |
+
source = cells.get(origin) or {}
|
| 624 |
+
if source.get("kind") != "formula" or not str(source.get("formula") or "").startswith("="):
|
| 625 |
+
raise RepairRefusal("repair_summary_formula_missing",
|
| 626 |
+
"The summary requires its authored formula at " + origin + ".")
|
| 627 |
+
for col in SUMMARY_COLUMNS:
|
| 628 |
+
address = col + str(row_number)
|
| 629 |
+
try:
|
| 630 |
+
text = Translator(source["formula"], origin=origin).translate_formula(address)
|
| 631 |
+
except Exception as exc:
|
| 632 |
+
raise RepairRefusal("repair_summary_formula_translate",
|
| 633 |
+
"The authored summary formula could not be filled across.") from exc
|
| 634 |
+
formula(address, text)
|
| 635 |
+
ratio_evidence = _append_summary_ratio_plan(current, edits, formula_check)
|
| 636 |
+
projected = _summary_formula_projection(_project(current, edits), edits, formula_check)
|
| 637 |
+
checks = validate(projected, windows=windows, mirror_as_of=as_of.isoformat(),
|
| 638 |
+
partner_ids=partners, invoice_validate=summary["validate_invoice"],
|
| 639 |
+
formula_check=formula_check)
|
| 640 |
+
return edits, {"before": summary["before"], "helper_rows": list(range(helper_row, helper_row + 4)),
|
| 641 |
+
"pricelist_cell": scope_ref, "mirror_as_of": as_of.isoformat(),
|
| 642 |
+
"partner_count": len(partners), "checks": checks,
|
| 643 |
+
"ratio_repair": ratio_evidence}
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
def validate(book, *, windows, mirror_as_of, partner_ids, invoice_validate, formula_check):
|
| 647 |
+
"""Reconcile 26 invoice figures and 39 formulas against independent Odoo results.
|
| 648 |
+
|
| 649 |
+
Windows are read from the planned sheet, with each upper bound capped at the mirror's
|
| 650 |
+
max(invoice_date). Re-evaluate the authored formulas over the independent amounts too.
|
| 651 |
+
Invoice-date bounding is not a historical snapshot; preserve validate()'s limitation rows.
|
| 652 |
+
"""
|
| 653 |
+
proof, checks = copy.deepcopy(book), []
|
| 654 |
+
as_of = _summary_date(mirror_as_of)
|
| 655 |
+
for col in SUMMARY_COLUMNS:
|
| 656 |
+
for row_number, (first_ref, last_ref) in zip((31, 32), windows[col]):
|
| 657 |
+
first = _summary_date(_summary_value(book, first_ref))
|
| 658 |
+
last = min(_summary_date(_summary_value(book, last_ref)), as_of)
|
| 659 |
+
if first > last:
|
| 660 |
+
raise RepairRefusal("repair_summary_asof_before_window",
|
| 661 |
+
"The invoice mirror ends before a summary window begins. Refresh it first.")
|
| 662 |
+
address = col + str(row_number)
|
| 663 |
+
observed = _summary_value(book, address)
|
| 664 |
+
rows = invoice_validate(first.isoformat(), last.isoformat(), {"invoiced_sales": observed},
|
| 665 |
+
team_id=None, partner_ids=list(partner_ids),
|
| 666 |
+
basis="invoice", include_refunds=False)
|
| 667 |
+
if not isinstance(rows, list) or len(rows) != 1 or rows[0].get("check") != "invoiced_sales" \
|
| 668 |
+
or rows[0].get("ok") is not True or rows[0].get("a") != observed \
|
| 669 |
+
or type(rows[0].get("b")) not in (int, float) or not math.isfinite(rows[0]["b"]) \
|
| 670 |
+
or abs(observed - rows[0]["b"]) > 0.01:
|
| 671 |
+
exc = RepairRefusal("repair_summary_reconciliation",
|
| 672 |
+
"Independent Odoo reconciliation failed for " + address + ".")
|
| 673 |
+
exc.checks = copy.deepcopy(rows)
|
| 674 |
+
raise exc
|
| 675 |
+
checks.append({**copy.deepcopy(rows[0]), "address": address,
|
| 676 |
+
"date_from": first.isoformat(), "date_to": last.isoformat()})
|
| 677 |
+
_tab(proof, SUMMARY_TAB)["cells"][address]["value"] = rows[0]["b"]
|
| 678 |
+
result = formula_check(proof)
|
| 679 |
+
proof = result["book"]
|
| 680 |
+
for col in SUMMARY_COLUMNS:
|
| 681 |
+
for row_number in range(33, 36):
|
| 682 |
+
address = col + str(row_number)
|
| 683 |
+
actual, expected = _summary_value(book, address), _summary_value(proof, address)
|
| 684 |
+
ok = (type(actual) in (int, float) and type(expected) in (int, float)
|
| 685 |
+
and math.isfinite(actual) and math.isfinite(expected)
|
| 686 |
+
and abs(actual - expected) <= (0.01 if row_number == 33 else 1e-9))
|
| 687 |
+
checks.append({"check": "invoice_derived", "address": address, "a": actual,
|
| 688 |
+
"b": expected, "gap": actual - expected if ok else None, "ok": ok})
|
| 689 |
+
if not ok:
|
| 690 |
+
raise RepairRefusal("repair_summary_reconciliation",
|
| 691 |
+
"The summary formula did not reconcile for " + address + ".")
|
| 692 |
+
return checks
|
| 693 |
+
|
| 694 |
+
|
| 695 |
+
def _build_plan(current, request, resolve_binding):
|
| 696 |
+
if not isinstance(request.get("source_cells"), list) or not request["source_cells"]:
|
| 697 |
+
raise RepairRefusal(
|
| 698 |
+
"repair_source_required",
|
| 699 |
+
"The repair requires a one-to-one source cell oracle with cached Excel values.")
|
| 700 |
+
if not isinstance(request.get("binding_cells"), list):
|
| 701 |
+
raise RepairRefusal("repair_bindings_required", "The repair requires both YTD bindings.")
|
| 702 |
+
|
| 703 |
+
edits, names, slots = [], set(), set()
|
| 704 |
+
for row in request["source_cells"]:
|
| 705 |
+
if not isinstance(row, dict):
|
| 706 |
+
raise RepairRefusal("repair_source_invalid", "Every source cell must be an object.")
|
| 707 |
+
name = str(row.get("name") or "").strip()
|
| 708 |
+
tab_name, address = str(row.get("sheet") or "").strip(), _addr(row.get("address"))
|
| 709 |
+
if not name or name in names:
|
| 710 |
+
raise RepairRefusal("repair_source_name_invalid",
|
| 711 |
+
"Every source cell needs a unique report-cell name.")
|
| 712 |
+
names.add(name)
|
| 713 |
+
slot = (tab_name.casefold(), address)
|
| 714 |
+
if slot in slots:
|
| 715 |
+
raise RepairRefusal("repair_edit_duplicate", "A repair cell may be named only once.")
|
| 716 |
+
slots.add(slot)
|
| 717 |
+
tab = _tab(current, tab_name)
|
| 718 |
+
before = copy.deepcopy((tab.get("cells") or {}).get(address))
|
| 719 |
+
supplied = copy.deepcopy(row.get("cell"))
|
| 720 |
+
if isinstance(supplied, dict) and "imported_value" in supplied:
|
| 721 |
+
raise RepairRefusal("repair_cache_invalid",
|
| 722 |
+
name + " carries a temporary imported value instead of its cache.")
|
| 723 |
+
desired = _preserve_visual(before, supplied)
|
| 724 |
+
if desired is not None:
|
| 725 |
+
desired = sheets.clean_cell(desired, addr=address)
|
| 726 |
+
if desired.get("value") != row.get("cached_value"):
|
| 727 |
+
raise RepairRefusal("repair_cache_mismatch",
|
| 728 |
+
name + " does not match its supplied cached Excel value.")
|
| 729 |
+
elif row.get("cached_value") is not None:
|
| 730 |
+
raise RepairRefusal("repair_cache_mismatch", name + " is blank but its cache is not.")
|
| 731 |
+
edits.append({"kind": "source", "name": name, "sheet": tab_name,
|
| 732 |
+
"address": address, "before": before, "after": desired,
|
| 733 |
+
"cached_value": copy.deepcopy(row.get("cached_value"))})
|
| 734 |
+
|
| 735 |
+
e4 = [row for row in edits if row["address"] == "E4"]
|
| 736 |
+
if len(e4) != 1 or (e4[0]["after"] is not None
|
| 737 |
+
and e4[0]["after"].get("kind") != "empty"):
|
| 738 |
+
raise RepairRefusal(
|
| 739 |
+
"repair_e4_not_blank",
|
| 740 |
+
"The source oracle must name E4 exactly once as blank and non-formula.")
|
| 741 |
+
if "Beg Date" not in names:
|
| 742 |
+
raise RepairRefusal("repair_beg_date_missing", "The source oracle must name Beg Date.")
|
| 743 |
+
beg_date = next(row for row in edits if row["name"] == "Beg Date")
|
| 744 |
+
try:
|
| 745 |
+
parsed_beg_date = date.fromisoformat(str((beg_date.get("after") or {}).get("value")))
|
| 746 |
+
except (TypeError, ValueError):
|
| 747 |
+
parsed_beg_date = None
|
| 748 |
+
if parsed_beg_date is None or parsed_beg_date.isoformat() != beg_date["cached_value"]:
|
| 749 |
+
raise RepairRefusal("repair_beg_date_invalid",
|
| 750 |
+
"Beg Date must carry one real ISO date matching its Excel cache.")
|
| 751 |
+
expected_names = request.get("named_report_cells")
|
| 752 |
+
if not isinstance(expected_names, list) or any(
|
| 753 |
+
not isinstance(item, str) or not item.strip() for item in expected_names):
|
| 754 |
+
raise RepairRefusal(
|
| 755 |
+
"repair_source_manifest_required",
|
| 756 |
+
"The repair requires the complete named report cell manifest from the source.")
|
| 757 |
+
normalized_manifest = [item.strip() for item in expected_names]
|
| 758 |
+
if len(set(normalized_manifest)) != len(normalized_manifest) or set(normalized_manifest) != names:
|
| 759 |
+
raise RepairRefusal(
|
| 760 |
+
"repair_source_manifest_mismatch",
|
| 761 |
+
"The source diff must map one-to-one onto the complete named report cell manifest.")
|
| 762 |
+
|
| 763 |
+
binding_names = []
|
| 764 |
+
refreshed_at = str(request.get("refreshed_at") or "").strip()
|
| 765 |
+
if not refreshed_at:
|
| 766 |
+
raise RepairRefusal("repair_timestamp_required",
|
| 767 |
+
"The deterministic repair plan requires an explicit refresh timestamp.")
|
| 768 |
+
for row in request["binding_cells"]:
|
| 769 |
+
if not isinstance(row, dict):
|
| 770 |
+
raise RepairRefusal("repair_binding_invalid", "Every YTD binding must be an object.")
|
| 771 |
+
name = str(row.get("name") or "").strip()
|
| 772 |
+
# S64-T21 widened this loop to ANY named report cell, so a blank or repeated name is now
|
| 773 |
+
# reachable for an Orders row too. The sentence has to cover both, not just the YTD pair.
|
| 774 |
+
if not name or name in binding_names:
|
| 775 |
+
raise RepairRefusal("repair_bindings_required",
|
| 776 |
+
"Every repair binding needs its own report cell name, and the "
|
| 777 |
+
"repair requires one Current YTD and one Previous YTD among them.")
|
| 778 |
+
binding_names.append(name)
|
| 779 |
+
binding = copy.deepcopy(row.get("binding"))
|
| 780 |
+
_validate_binding(name, binding, window_kind=YTD_NAMES.get(name))
|
| 781 |
+
tab_name, address = str(row.get("sheet") or "").strip(), _addr(row.get("address"))
|
| 782 |
+
slot = (tab_name.casefold(), address)
|
| 783 |
+
if slot in slots:
|
| 784 |
+
raise RepairRefusal("repair_edit_duplicate", "A repair cell may be named only once.")
|
| 785 |
+
slots.add(slot)
|
| 786 |
+
tab = _tab(current, tab_name)
|
| 787 |
+
before = copy.deepcopy((tab.get("cells") or {}).get(address))
|
| 788 |
+
if not callable(resolve_binding):
|
| 789 |
+
raise RepairRefusal(
|
| 790 |
+
"repair_binding_resolver_required",
|
| 791 |
+
"Canonical data must resolve both YTD bindings before this repair can run.")
|
| 792 |
+
resolved = resolve_binding(binding, current, tab)
|
| 793 |
+
if not isinstance(resolved, dict) or "value" not in resolved:
|
| 794 |
+
raise RepairRefusal("repair_binding_unreadable", name + " did not resolve canonical data.")
|
| 795 |
+
canonical_binding = copy.deepcopy(resolved.get("binding") or binding)
|
| 796 |
+
_validate_binding(name, canonical_binding, window_kind=YTD_NAMES.get(name))
|
| 797 |
+
desired = _preserve_visual(before, {
|
| 798 |
+
"kind": "bound", "state": "fresh", "binding": canonical_binding,
|
| 799 |
+
"value": copy.deepcopy(resolved["value"]),
|
| 800 |
+
"display": "" if resolved["value"] is None else str(resolved["value"]),
|
| 801 |
+
"refreshed_at": refreshed_at,
|
| 802 |
+
})
|
| 803 |
+
desired = sheets.clean_cell(desired, addr=address)
|
| 804 |
+
edits.append({"kind": "binding", "name": name, "sheet": tab_name,
|
| 805 |
+
"address": address, "before": before, "after": desired})
|
| 806 |
+
if set(YTD_NAMES) - set(binding_names):
|
| 807 |
+
raise RepairRefusal("repair_bindings_required",
|
| 808 |
+
"The repair requires one Current YTD and one Previous YTD binding.")
|
| 809 |
+
|
| 810 |
+
edits.sort(key=lambda row: (row["sheet"].casefold(), row["address"]))
|
| 811 |
+
return edits
|
| 812 |
+
|
| 813 |
+
|
| 814 |
+
def _write_backup(backup_store, current, digest, diff_digest, workbook_id):
|
| 815 |
+
revision = int(current.get("rev") or 0)
|
| 816 |
+
key = "sheet_repair_backup_" + workbook_id + "_r" + str(revision) + "_" + digest[:16]
|
| 817 |
+
record = {"operation": operation_for(workbook_id), "workbook_id": workbook_id,
|
| 818 |
+
"revision": revision, "digest": digest, "diff_digest": diff_digest,
|
| 819 |
+
"workbook": copy.deepcopy(current)}
|
| 820 |
+
|
| 821 |
+
def apply(existing):
|
| 822 |
+
if existing and existing != record:
|
| 823 |
+
raise RepairRefusal("repair_backup_conflict",
|
| 824 |
+
"A different backup already occupies this repair key.")
|
| 825 |
+
return copy.deepcopy(record)
|
| 826 |
+
|
| 827 |
+
backup_store.update(key, apply, flush="sync")
|
| 828 |
+
readback = backup_store.get(key)
|
| 829 |
+
if readback != record or sheets.workbook_digest(readback.get("workbook") or {}) != digest:
|
| 830 |
+
raise RepairRefusal("repair_backup_unverified",
|
| 831 |
+
"The workbook backup could not be read back byte for byte.")
|
| 832 |
+
return key
|
| 833 |
+
|
| 834 |
+
|
| 835 |
+
def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
|
| 836 |
+
resolve_binding, formula_check=None, actor="", actor_name=""):
|
| 837 |
+
"""Validate, back up, repair and read back only the workbook this call is POINTED AT.
|
| 838 |
+
|
| 839 |
+
⭐ `target_workbook_id` is what makes the repair general, and it is the one argument to read
|
| 840 |
+
twice. It is checked against THREE independent things before a byte moves: the request's own
|
| 841 |
+
`workbook_id`, the marker's (through `operation_for`, so a marker minted for one workbook
|
| 842 |
+
cannot authorise another), and the STORED document's identity. A caller pointed at a fixture
|
| 843 |
+
and handed a request for another workbook is refused, which is the whole point of S64-T21.
|
| 844 |
+
|
| 845 |
+
⚠ THE DEFAULT IS THE OWNER'S WORKBOOK, DELIBERATELY, AND IT IS NOT A HOLE. It is the one
|
| 846 |
+
workbook this module was written for and the only one an unpointed call could ever have
|
| 847 |
+
reached; every other target must be NAMED. An EXPLICIT empty or non-string target still
|
| 848 |
+
refuses `repair_target_required`, so "pointed at nothing" and "pointed at the default" stay
|
| 849 |
+
different answers. Making the argument mandatory instead would have been a breaking change to
|
| 850 |
+
`verify_sheet.py`'s five S63-T21 call sites, a gate no lane on this board owns, and it would
|
| 851 |
+
have bought no safety the route does not already give: `routes_sheet.py` carries the same
|
| 852 |
+
default for the operator.
|
| 853 |
+
"""
|
| 854 |
+
if not isinstance(target_workbook_id, str) or not target_workbook_id.strip():
|
| 855 |
+
raise RepairRefusal("repair_target_required",
|
| 856 |
+
"The repair requires an explicitly named target workbook.")
|
| 857 |
+
target_workbook_id = target_workbook_id.strip()
|
| 858 |
+
if backup_store is None or not callable(getattr(backup_store, "update", None)) \
|
| 859 |
+
or not callable(getattr(backup_store, "get", None)):
|
| 860 |
+
raise RepairRefusal("repair_backup_required",
|
| 861 |
+
"A readable, writable backup store is required before repair.")
|
| 862 |
+
if not isinstance(request, dict) or request.get("workbook_id") != target_workbook_id:
|
| 863 |
+
raise RepairRefusal("repair_workbook_mismatch",
|
| 864 |
+
"This operation may repair only workbook " + target_workbook_id + ".")
|
| 865 |
+
revision = request.get("expected_revision")
|
| 866 |
+
if isinstance(revision, bool) or not isinstance(revision, int) or revision < 0:
|
| 867 |
+
raise RepairRefusal("repair_revision_required",
|
| 868 |
+
"An exact expected workbook revision is required for repair.")
|
| 869 |
+
_validate_marker(request.get("marker"), revision, target_workbook_id)
|
| 870 |
+
try:
|
| 871 |
+
calculation_date = date.fromisoformat(str(request.get("calculation_date") or ""))
|
| 872 |
+
except ValueError as exc:
|
| 873 |
+
raise RepairRefusal("repair_calculation_date_required",
|
| 874 |
+
"The deterministic repair plan requires one ISO calculation date.") from exc
|
| 875 |
+
if calculation_date.isoformat() != request["calculation_date"]:
|
| 876 |
+
raise RepairRefusal("repair_calculation_date_required",
|
| 877 |
+
"The deterministic repair plan requires one ISO calculation date.")
|
| 878 |
+
|
| 879 |
+
current = sheets.workbook_snapshot(st, target_workbook_id)
|
| 880 |
+
if str(current.get("id") or "") != target_workbook_id:
|
| 881 |
+
raise RepairRefusal("repair_workbook_mismatch",
|
| 882 |
+
"The stored workbook identity does not match this repair target.")
|
| 883 |
+
if int(current.get("rev") or 0) != revision:
|
| 884 |
+
raise RepairRefusal("repair_revision_mismatch",
|
| 885 |
+
"The named workbook is not at the expected repair revision.")
|
| 886 |
+
digest = sheets.workbook_digest(current)
|
| 887 |
+
summary_evidence = None
|
| 888 |
+
summary_errors_before = None
|
| 889 |
+
if "summary_block" in request:
|
| 890 |
+
summary_errors_before = check_summary_formula_errors(current)
|
| 891 |
+
edits, summary_evidence = _build_summary_plan(current, request, resolve_binding, formula_check)
|
| 892 |
+
else:
|
| 893 |
+
edits = _build_plan(current, request, resolve_binding)
|
| 894 |
+
projected = _project(current, edits)
|
| 895 |
+
projected, projected_errors = _checked_projection(projected, edits, formula_check)
|
| 896 |
+
comparison = _formula_error_comparison(current, projected, edits, projected_errors)
|
| 897 |
+
summary_errors_after = (check_summary_formula_errors(projected)
|
| 898 |
+
if summary_evidence is not None else None)
|
| 899 |
+
if comparison["introduced"] or comparison["remaining_in_repair"] \
|
| 900 |
+
or (summary_errors_after is not None and summary_errors_after["count"]):
|
| 901 |
+
raise RepairRefusal(
|
| 902 |
+
"repair_formula_errors",
|
| 903 |
+
"The projected repair introduces a formula error or leaves one in its repair scope, "
|
| 904 |
+
"so no repair was applied.")
|
| 905 |
+
visual_before = visual_metadata(current)
|
| 906 |
+
if visual_metadata(projected) != visual_before:
|
| 907 |
+
raise RepairRefusal("repair_visual_metadata_changed",
|
| 908 |
+
"The projected repair changes Daily Sales visual metadata.")
|
| 909 |
+
|
| 910 |
+
exact_edits = [
|
| 911 |
+
{key: copy.deepcopy(row[key]) for key in ("sheet", "address", "before", "after")}
|
| 912 |
+
for row in edits]
|
| 913 |
+
diff_digest = sheets.workbook_digest({"edits": exact_edits})
|
| 914 |
+
backup_key = _write_backup(backup_store, current, digest, diff_digest, target_workbook_id)
|
| 915 |
+
write = sheets.apply_exact_cell_edits(
|
| 916 |
+
st, target_workbook_id, exact_edits,
|
| 917 |
+
revision, digest, actor=actor, actor_name=actor_name)
|
| 918 |
+
readback = sheets.workbook_snapshot(st, target_workbook_id)
|
| 919 |
+
|
| 920 |
+
if int(readback.get("rev") or 0) != revision + 1:
|
| 921 |
+
raise RepairRefusal("repair_readback_revision",
|
| 922 |
+
"The repaired workbook revision did not read back as expected.")
|
| 923 |
+
for row in edits:
|
| 924 |
+
actual = copy.deepcopy((_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"]))
|
| 925 |
+
expected = None if row["after"] is None else sheets.clean_cell(row["after"], row["address"])
|
| 926 |
+
if actual != expected:
|
| 927 |
+
raise RepairRefusal("repair_readback_cell",
|
| 928 |
+
row["sheet"] + "!" + row["address"] + " did not read back exactly.")
|
| 929 |
+
checked_readback, readback_errors = _checked_projection(readback, copy.deepcopy(edits), formula_check)
|
| 930 |
+
readback_comparison = _formula_error_comparison(current, checked_readback, edits, readback_errors)
|
| 931 |
+
summary_errors_readback = (check_summary_formula_errors(checked_readback)
|
| 932 |
+
if summary_evidence is not None else None)
|
| 933 |
+
if readback_comparison["introduced"] or readback_comparison["remaining_in_repair"] \
|
| 934 |
+
or (summary_errors_readback is not None and summary_errors_readback["count"]) \
|
| 935 |
+
or _cell_map(checked_readback) != _cell_map(readback):
|
| 936 |
+
raise RepairRefusal("repair_readback_formula_state",
|
| 937 |
+
"The repaired workbook did not read back calculation-current.")
|
| 938 |
+
if visual_metadata(readback) != visual_before:
|
| 939 |
+
raise RepairRefusal("repair_readback_visual_metadata",
|
| 940 |
+
"Daily Sales visual metadata changed during repair.")
|
| 941 |
+
if summary_evidence is not None:
|
| 942 |
+
tab = _tab(readback, SUMMARY_TAB)
|
| 943 |
+
if any((tab.get("cells", {}).get(col + str(row)) or {}).get("kind")
|
| 944 |
+
!= ("bound" if row in (31, 32) else "formula")
|
| 945 |
+
for col in SUMMARY_COLUMNS for row in range(31, 36)):
|
| 946 |
+
raise RepairRefusal("repair_summary_readback_kind",
|
| 947 |
+
"Every summary figure must read back as its planned binding or formula.")
|
| 948 |
+
summary_evidence["readback"] = summary_block_snapshot(readback)
|
| 949 |
+
summary_evidence["formula_error_check"] = {
|
| 950 |
+
"before": summary_errors_before, "projected": summary_errors_after,
|
| 951 |
+
"readback": summary_errors_readback,
|
| 952 |
+
}
|
| 953 |
+
|
| 954 |
+
source_readback = {row["name"]: copy.deepcopy(
|
| 955 |
+
(_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"]))
|
| 956 |
+
for row in edits if row["kind"] == "source"}
|
| 957 |
+
bindings = {row["name"]: copy.deepcopy(
|
| 958 |
+
((_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"]) or {}).get("binding"))
|
| 959 |
+
for row in edits if row["kind"] == "binding"}
|
| 960 |
+
result = {
|
| 961 |
+
"workbook_id": target_workbook_id,
|
| 962 |
+
"revision_before": revision,
|
| 963 |
+
"revision_after": readback["rev"],
|
| 964 |
+
"backup_key": backup_key,
|
| 965 |
+
"backup_digest": digest,
|
| 966 |
+
"diff_digest": diff_digest,
|
| 967 |
+
"applied": write["edits"],
|
| 968 |
+
"paths": write["paths"],
|
| 969 |
+
"source_cells": source_readback,
|
| 970 |
+
"bindings": bindings,
|
| 971 |
+
"formula_errors": formula_error_census(readback),
|
| 972 |
+
"formula_error_comparison": readback_comparison,
|
| 973 |
+
"visual_metadata_preserved": True,
|
| 974 |
+
}
|
| 975 |
+
if summary_evidence is not None:
|
| 976 |
+
result["summary_block"] = summary_evidence
|
| 977 |
+
return result
|
platform/core/external_access.py
CHANGED
|
@@ -8,6 +8,7 @@ credential, user, bucket, or tenant runtime.
|
|
| 8 |
"""
|
| 9 |
import calendar
|
| 10 |
from dataclasses import dataclass
|
|
|
|
| 11 |
import hashlib
|
| 12 |
import hmac
|
| 13 |
import secrets
|
|
@@ -25,6 +26,7 @@ ALL_DATABASES = "all"
|
|
| 25 |
MAX_SCOPE_DATABASES = 200
|
| 26 |
ALGORITHM = "sha256-salt-v1"
|
| 27 |
AUTH_ERROR = "invalid_external_credential"
|
|
|
|
| 28 |
_DUMMY_SALT = bytes(32)
|
| 29 |
_DUMMY_HASH = hashlib.sha256(_DUMMY_SALT + b"invalid").hexdigest()
|
| 30 |
#: Seconds between two durable `last_used` stamps for ONE credential.
|
|
@@ -114,6 +116,28 @@ def _now() -> str:
|
|
| 114 |
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
| 115 |
|
| 116 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
def _stamp_is_due(last_used, now: float) -> bool:
|
| 118 |
"""Is this credential's `last_used` stale enough to be worth a durable write?
|
| 119 |
|
|
@@ -164,7 +188,7 @@ def _assert_bound_runtime(st, tenant: str) -> None:
|
|
| 164 |
raise ExternalAccessError("external_identity_invalid")
|
| 165 |
|
| 166 |
|
| 167 |
-
def
|
| 168 |
try:
|
| 169 |
registry = users.registry() or {}
|
| 170 |
except Exception as exc:
|
|
@@ -180,6 +204,20 @@ def _active_public_user(username: str, tenant: str) -> dict:
|
|
| 180 |
return public
|
| 181 |
|
| 182 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
def _metadata(record: dict) -> dict:
|
| 184 |
"""Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam."""
|
| 185 |
access, databases = scope_of(record)
|
|
@@ -189,6 +227,8 @@ def _metadata(record: dict) -> dict:
|
|
| 189 |
"created": record.get("created"),
|
| 190 |
"last_used": record.get("last_used"),
|
| 191 |
"revoked_at": record.get("revoked_at"),
|
|
|
|
|
|
|
| 192 |
"access": access,
|
| 193 |
"databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases),
|
| 194 |
}
|
|
@@ -205,16 +245,16 @@ def label_for(st, key_id: str) -> str:
|
|
| 205 |
return label
|
| 206 |
|
| 207 |
|
| 208 |
-
def mint(st, username: str, tenant: str, label: str, access=None, databases=None
|
|
|
|
| 209 |
"""Create one credential and return ``(metadata, token)``; the token is never persisted."""
|
| 210 |
access = normalise_access(access)
|
| 211 |
databases = normalise_databases(databases)
|
| 212 |
owner, tenant_key = _normalise_owner(username, tenant)
|
| 213 |
_assert_bound_runtime(st, tenant_key)
|
| 214 |
_active_public_user(owner, tenant_key)
|
| 215 |
-
clean_label =
|
| 216 |
-
|
| 217 |
-
raise ExternalAccessError("external_label_invalid")
|
| 218 |
|
| 219 |
secret = secrets.token_urlsafe(32)
|
| 220 |
salt = secrets.token_bytes(32)
|
|
@@ -244,6 +284,8 @@ def mint(st, username: str, tenant: str, label: str, access=None, databases=None
|
|
| 244 |
"created": created,
|
| 245 |
"last_used": None,
|
| 246 |
"revoked_at": None,
|
|
|
|
|
|
|
| 247 |
}
|
| 248 |
book[key_id] = record
|
| 249 |
minted["record"] = record
|
|
@@ -262,6 +304,94 @@ def mint(st, username: str, tenant: str, label: str, access=None, databases=None
|
|
| 262 |
return _metadata(record), token
|
| 263 |
|
| 264 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
def list_for_user(st, username: str, tenant: str) -> list[dict]:
|
| 266 |
"""List only this user's active credential metadata, newest first."""
|
| 267 |
owner, tenant_key = _normalise_owner(username, tenant)
|
|
@@ -355,6 +485,8 @@ def verify(token: str) -> VerifiedCredential:
|
|
| 355 |
and record.get("tenant") == tenant_key
|
| 356 |
and bool(record.get("username"))
|
| 357 |
and not record.get("revoked_at")
|
|
|
|
|
|
|
| 358 |
)
|
| 359 |
if not record_ok or not matched:
|
| 360 |
raise ExternalAuthError(AUTH_ERROR)
|
|
|
|
| 8 |
"""
|
| 9 |
import calendar
|
| 10 |
from dataclasses import dataclass
|
| 11 |
+
from datetime import datetime, timezone
|
| 12 |
import hashlib
|
| 13 |
import hmac
|
| 14 |
import secrets
|
|
|
|
| 26 |
MAX_SCOPE_DATABASES = 200
|
| 27 |
ALGORITHM = "sha256-salt-v1"
|
| 28 |
AUTH_ERROR = "invalid_external_credential"
|
| 29 |
+
_UNSET = object()
|
| 30 |
_DUMMY_SALT = bytes(32)
|
| 31 |
_DUMMY_HASH = hashlib.sha256(_DUMMY_SALT + b"invalid").hexdigest()
|
| 32 |
#: Seconds between two durable `last_used` stamps for ONE credential.
|
|
|
|
| 116 |
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
| 117 |
|
| 118 |
|
| 119 |
+
def _epoch(value: str | None) -> float | None:
|
| 120 |
+
if value is None:
|
| 121 |
+
return None
|
| 122 |
+
try:
|
| 123 |
+
return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(
|
| 124 |
+
tzinfo=timezone.utc).timestamp()
|
| 125 |
+
except (TypeError, ValueError, OverflowError):
|
| 126 |
+
return None
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def normalise_expiry(value, *, now: float | None = None) -> str | None:
|
| 130 |
+
"""Return one canonical future UTC expiry, or ``None`` for no expiry."""
|
| 131 |
+
if value is None or value == "":
|
| 132 |
+
return None
|
| 133 |
+
if not isinstance(value, str):
|
| 134 |
+
raise ExternalAccessError("external_expiry_invalid")
|
| 135 |
+
parsed = _epoch(value.strip())
|
| 136 |
+
if parsed is None or parsed <= (time.time() if now is None else float(now)):
|
| 137 |
+
raise ExternalAccessError("external_expiry_invalid")
|
| 138 |
+
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(parsed))
|
| 139 |
+
|
| 140 |
+
|
| 141 |
def _stamp_is_due(last_used, now: float) -> bool:
|
| 142 |
"""Is this credential's `last_used` stale enough to be worth a durable write?
|
| 143 |
|
|
|
|
| 188 |
raise ExternalAccessError("external_identity_invalid")
|
| 189 |
|
| 190 |
|
| 191 |
+
def resolve_active_user(username: str, tenant: str) -> dict:
|
| 192 |
try:
|
| 193 |
registry = users.registry() or {}
|
| 194 |
except Exception as exc:
|
|
|
|
| 204 |
return public
|
| 205 |
|
| 206 |
|
| 207 |
+
def _active_public_user(username: str, tenant: str) -> dict:
|
| 208 |
+
"""Backward-compatible private spelling retained for the SDLC 62 seam."""
|
| 209 |
+
return resolve_active_user(username, tenant)
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def _normalise_label(label) -> str:
|
| 213 |
+
clean = str(label or "").strip()
|
| 214 |
+
if (not clean or len(clean) > 80 or any(ord(char) < 32 for char in clean)
|
| 215 |
+
or "lpb1." in clean or "lpo1." in clean or "lpr1." in clean
|
| 216 |
+
or "lpc1." in clean):
|
| 217 |
+
raise ExternalAccessError("external_label_invalid")
|
| 218 |
+
return clean
|
| 219 |
+
|
| 220 |
+
|
| 221 |
def _metadata(record: dict) -> dict:
|
| 222 |
"""Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam."""
|
| 223 |
access, databases = scope_of(record)
|
|
|
|
| 227 |
"created": record.get("created"),
|
| 228 |
"last_used": record.get("last_used"),
|
| 229 |
"revoked_at": record.get("revoked_at"),
|
| 230 |
+
"rotated_at": record.get("rotated_at"),
|
| 231 |
+
"expires_at": record.get("expires_at"),
|
| 232 |
"access": access,
|
| 233 |
"databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases),
|
| 234 |
}
|
|
|
|
| 245 |
return label
|
| 246 |
|
| 247 |
|
| 248 |
+
def mint(st, username: str, tenant: str, label: str, access=None, databases=None,
|
| 249 |
+
expires_at=None) -> tuple[dict, str]:
|
| 250 |
"""Create one credential and return ``(metadata, token)``; the token is never persisted."""
|
| 251 |
access = normalise_access(access)
|
| 252 |
databases = normalise_databases(databases)
|
| 253 |
owner, tenant_key = _normalise_owner(username, tenant)
|
| 254 |
_assert_bound_runtime(st, tenant_key)
|
| 255 |
_active_public_user(owner, tenant_key)
|
| 256 |
+
clean_label = _normalise_label(label)
|
| 257 |
+
expiry = normalise_expiry(expires_at)
|
|
|
|
| 258 |
|
| 259 |
secret = secrets.token_urlsafe(32)
|
| 260 |
salt = secrets.token_bytes(32)
|
|
|
|
| 284 |
"created": created,
|
| 285 |
"last_used": None,
|
| 286 |
"revoked_at": None,
|
| 287 |
+
"rotated_at": None,
|
| 288 |
+
"expires_at": expiry,
|
| 289 |
}
|
| 290 |
book[key_id] = record
|
| 291 |
minted["record"] = record
|
|
|
|
| 304 |
return _metadata(record), token
|
| 305 |
|
| 306 |
|
| 307 |
+
def edit(st, username: str, tenant: str, key_id: str, *, label=_UNSET, access=_UNSET,
|
| 308 |
+
databases=_UNSET, expires_at=_UNSET) -> dict:
|
| 309 |
+
"""Edit metadata/scope in one store update; credential material is left untouched."""
|
| 310 |
+
owner, tenant_key = _normalise_owner(username, tenant)
|
| 311 |
+
_assert_bound_runtime(st, tenant_key)
|
| 312 |
+
resolve_active_user(owner, tenant_key)
|
| 313 |
+
target = str(key_id or "").strip()
|
| 314 |
+
if label is access is databases is expires_at is _UNSET:
|
| 315 |
+
raise ExternalAccessError("external_edit_empty")
|
| 316 |
+
clean_label = _normalise_label(label) if label is not _UNSET else _UNSET
|
| 317 |
+
clean_access = normalise_access(access) if access is not _UNSET else _UNSET
|
| 318 |
+
clean_databases = normalise_databases(databases) if databases is not _UNSET else _UNSET
|
| 319 |
+
clean_expiry = normalise_expiry(expires_at) if expires_at is not _UNSET else _UNSET
|
| 320 |
+
edited: dict = {}
|
| 321 |
+
|
| 322 |
+
def _set(value):
|
| 323 |
+
if not isinstance(value, dict):
|
| 324 |
+
raise ExternalAccessError("external_key_not_found")
|
| 325 |
+
book = dict(value)
|
| 326 |
+
current = book.get(target)
|
| 327 |
+
if (not isinstance(current, dict) or current.get("username") != owner
|
| 328 |
+
or current.get("tenant") != tenant_key or current.get("revoked_at")):
|
| 329 |
+
raise ExternalAccessError("external_key_not_found")
|
| 330 |
+
record = dict(current)
|
| 331 |
+
if clean_label is not _UNSET:
|
| 332 |
+
record["label"] = clean_label
|
| 333 |
+
if clean_access is not _UNSET:
|
| 334 |
+
record["access"] = clean_access
|
| 335 |
+
if clean_databases is not _UNSET:
|
| 336 |
+
record["databases"] = (ALL_DATABASES if clean_databases == ALL_DATABASES
|
| 337 |
+
else list(clean_databases))
|
| 338 |
+
if clean_expiry is not _UNSET:
|
| 339 |
+
record["expires_at"] = clean_expiry
|
| 340 |
+
book[target] = record
|
| 341 |
+
edited["record"] = record
|
| 342 |
+
return book
|
| 343 |
+
|
| 344 |
+
try:
|
| 345 |
+
st.update(BUCKET, _set, flush="sync")
|
| 346 |
+
except ExternalAccessError:
|
| 347 |
+
raise
|
| 348 |
+
except Exception as exc:
|
| 349 |
+
raise ExternalAccessError("external_access_unavailable") from exc
|
| 350 |
+
record = edited.get("record")
|
| 351 |
+
if not isinstance(record, dict):
|
| 352 |
+
raise ExternalAccessError("external_access_unavailable")
|
| 353 |
+
return _metadata(record)
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
def rotate(st, username: str, tenant: str, key_id: str) -> tuple[dict, str]:
|
| 357 |
+
"""Replace one owned key's secret atomically and reveal the replacement exactly once."""
|
| 358 |
+
owner, tenant_key = _normalise_owner(username, tenant)
|
| 359 |
+
_assert_bound_runtime(st, tenant_key)
|
| 360 |
+
resolve_active_user(owner, tenant_key)
|
| 361 |
+
target = str(key_id or "").strip()
|
| 362 |
+
secret = secrets.token_urlsafe(32)
|
| 363 |
+
salt = secrets.token_bytes(32)
|
| 364 |
+
rotated: dict = {}
|
| 365 |
+
|
| 366 |
+
def _set(value):
|
| 367 |
+
if not isinstance(value, dict):
|
| 368 |
+
raise ExternalAccessError("external_key_not_found")
|
| 369 |
+
book = dict(value)
|
| 370 |
+
current = book.get(target)
|
| 371 |
+
if (not isinstance(current, dict) or current.get("username") != owner
|
| 372 |
+
or current.get("tenant") != tenant_key or current.get("revoked_at")):
|
| 373 |
+
raise ExternalAccessError("external_key_not_found")
|
| 374 |
+
record = dict(current)
|
| 375 |
+
record.update({
|
| 376 |
+
"salt": salt.hex(), "hash": _digest(salt, secret), "algo": ALGORITHM,
|
| 377 |
+
"rotated_at": _now(), "last_used": None,
|
| 378 |
+
})
|
| 379 |
+
book[target] = record
|
| 380 |
+
rotated["record"] = record
|
| 381 |
+
return book
|
| 382 |
+
|
| 383 |
+
try:
|
| 384 |
+
st.update(BUCKET, _set, flush="sync")
|
| 385 |
+
except ExternalAccessError:
|
| 386 |
+
raise
|
| 387 |
+
except Exception as exc:
|
| 388 |
+
raise ExternalAccessError("external_access_unavailable") from exc
|
| 389 |
+
record = rotated.get("record")
|
| 390 |
+
if not isinstance(record, dict):
|
| 391 |
+
raise ExternalAccessError("external_access_unavailable")
|
| 392 |
+
return _metadata(record), f"lpb1.{tenant_key}.{target}.{secret}"
|
| 393 |
+
|
| 394 |
+
|
| 395 |
def list_for_user(st, username: str, tenant: str) -> list[dict]:
|
| 396 |
"""List only this user's active credential metadata, newest first."""
|
| 397 |
owner, tenant_key = _normalise_owner(username, tenant)
|
|
|
|
| 485 |
and record.get("tenant") == tenant_key
|
| 486 |
and bool(record.get("username"))
|
| 487 |
and not record.get("revoked_at")
|
| 488 |
+
and (_epoch(record.get("expires_at")) is None
|
| 489 |
+
if not record.get("expires_at") else _epoch(record.get("expires_at")) > time.time())
|
| 490 |
)
|
| 491 |
if not record_ok or not matched:
|
| 492 |
raise ExternalAuthError(AUTH_ERROR)
|
platform/core/external_oauth.py
ADDED
|
@@ -0,0 +1,723 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hash-only OAuth 2.1-style authorization for the existing remote MCP resource.
|
| 2 |
+
|
| 3 |
+
The authorization endpoint is the only browser-session step. Code exchange, refresh, bearer
|
| 4 |
+
verification, and disconnect resolve from opaque tokens and fresh tenant state without a cookie.
|
| 5 |
+
Public clients use PKCE S256 and have no shared client secret to leak or impersonate a user.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import base64
|
| 10 |
+
from dataclasses import dataclass
|
| 11 |
+
import hashlib
|
| 12 |
+
import hmac
|
| 13 |
+
import secrets
|
| 14 |
+
import time
|
| 15 |
+
from urllib.parse import urlsplit, urlunsplit
|
| 16 |
+
|
| 17 |
+
from core import external_access
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
BUCKET = "external_oauth"
|
| 21 |
+
ALGORITHM = "sha256-salt-v1"
|
| 22 |
+
CODE_TTL_SECONDS = 5 * 60
|
| 23 |
+
ACCESS_TTL_SECONDS = 60 * 60
|
| 24 |
+
REFRESH_TTL_SECONDS = 30 * 24 * 60 * 60
|
| 25 |
+
MAX_REFRESH_ROTATIONS = 256
|
| 26 |
+
MAX_REDIRECT_URIS = 10
|
| 27 |
+
SCOPES = ("mcp:read", "mcp:write", "offline_access")
|
| 28 |
+
AUTH_ERROR = external_access.AUTH_ERROR
|
| 29 |
+
_STAMP_EVERY = 900.0
|
| 30 |
+
_DUMMY_SALT = bytes(32)
|
| 31 |
+
_DUMMY_HASH = hashlib.sha256(_DUMMY_SALT + b"invalid").hexdigest()
|
| 32 |
+
_UNSET = object()
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class ExternalOAuthError(external_access.ExternalAccessError):
|
| 36 |
+
"""Stable OAuth refusal; HTTP vocabulary belongs to the API route."""
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class ExternalOAuthAuthError(ExternalOAuthError):
|
| 40 |
+
"""Uniform bearer refusal with no tenant, client, or grant oracle."""
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@dataclass(frozen=True)
|
| 44 |
+
class VerifiedOAuth:
|
| 45 |
+
tenant: str
|
| 46 |
+
grant_id: str
|
| 47 |
+
client_id: str
|
| 48 |
+
user: dict
|
| 49 |
+
runtime: object
|
| 50 |
+
access: str
|
| 51 |
+
databases: object
|
| 52 |
+
scopes: tuple[str, ...]
|
| 53 |
+
resource: str
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
@dataclass(frozen=True)
|
| 57 |
+
class DisconnectedOAuth:
|
| 58 |
+
"""Safe attribution plus public metadata for one token-authenticated disconnect."""
|
| 59 |
+
|
| 60 |
+
tenant: str
|
| 61 |
+
username: str
|
| 62 |
+
connection: dict
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _now(now=None) -> float:
|
| 66 |
+
return time.time() if now is None else float(now)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _iso(epoch: float | None) -> str | None:
|
| 70 |
+
if epoch is None:
|
| 71 |
+
return None
|
| 72 |
+
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch))
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _digest(salt: bytes, secret: str) -> str:
|
| 76 |
+
return hashlib.sha256(salt + secret.encode("utf-8")).hexdigest()
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _state_digest(state: str) -> str:
|
| 80 |
+
return hashlib.sha256(state.encode("utf-8")).hexdigest()
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def _pkce_challenge(verifier: str) -> str:
|
| 84 |
+
return base64.urlsafe_b64encode(hashlib.sha256(verifier.encode("ascii")).digest()).rstrip(b"=").decode("ascii")
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _book(st, *, fresh=True) -> dict:
|
| 88 |
+
try:
|
| 89 |
+
value = st.get(BUCKET, fresh=fresh)
|
| 90 |
+
except Exception as exc:
|
| 91 |
+
raise ExternalOAuthError("oauth_unavailable") from exc
|
| 92 |
+
if value is None:
|
| 93 |
+
return {"clients": {}, "codes": {}, "states": {}, "grants": {}}
|
| 94 |
+
if not isinstance(value, dict):
|
| 95 |
+
raise ExternalOAuthError("oauth_unavailable")
|
| 96 |
+
out = {}
|
| 97 |
+
for key in ("clients", "codes", "states", "grants"):
|
| 98 |
+
child = value.get(key, {})
|
| 99 |
+
if not isinstance(child, dict):
|
| 100 |
+
raise ExternalOAuthError("oauth_unavailable")
|
| 101 |
+
out[key] = child
|
| 102 |
+
return out
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _normalise_redirect_uris(value) -> tuple[str, ...]:
|
| 106 |
+
if not isinstance(value, (list, tuple)) or not 1 <= len(value) <= MAX_REDIRECT_URIS:
|
| 107 |
+
raise ExternalOAuthError("oauth_redirect_uri_invalid")
|
| 108 |
+
clean = []
|
| 109 |
+
for raw in value:
|
| 110 |
+
if not isinstance(raw, str) or not raw.strip() or len(raw.strip()) > 1000:
|
| 111 |
+
raise ExternalOAuthError("oauth_redirect_uri_invalid")
|
| 112 |
+
uri = raw.strip()
|
| 113 |
+
parsed = urlsplit(uri)
|
| 114 |
+
loopback = parsed.scheme == "http" and (parsed.hostname or "").lower() in {
|
| 115 |
+
"localhost", "127.0.0.1", "::1",
|
| 116 |
+
}
|
| 117 |
+
if (parsed.fragment or parsed.username or parsed.password or not parsed.netloc
|
| 118 |
+
or (parsed.scheme != "https" and not loopback)):
|
| 119 |
+
raise ExternalOAuthError("oauth_redirect_uri_invalid")
|
| 120 |
+
clean.append(uri)
|
| 121 |
+
return tuple(dict.fromkeys(clean))
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _normalise_state(value) -> str:
|
| 125 |
+
if not isinstance(value, str) or not 16 <= len(value) <= 512 or any(ord(c) < 32 for c in value):
|
| 126 |
+
raise ExternalOAuthError("oauth_state_invalid")
|
| 127 |
+
return value
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _normalise_challenge(value, method) -> str:
|
| 131 |
+
if method != "S256" or not isinstance(value, str) or not 43 <= len(value) <= 128:
|
| 132 |
+
raise ExternalOAuthError("oauth_pkce_required")
|
| 133 |
+
allowed = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
|
| 134 |
+
if any(char not in allowed for char in value):
|
| 135 |
+
raise ExternalOAuthError("oauth_pkce_required")
|
| 136 |
+
return value
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _normalise_verifier(value) -> str:
|
| 140 |
+
if not isinstance(value, str) or not 43 <= len(value) <= 128:
|
| 141 |
+
raise ExternalOAuthError("oauth_invalid_grant")
|
| 142 |
+
allowed = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~"
|
| 143 |
+
if any(char not in allowed for char in value):
|
| 144 |
+
raise ExternalOAuthError("oauth_invalid_grant")
|
| 145 |
+
return value
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _normalise_scopes(value, access: str) -> tuple[str, ...]:
|
| 149 |
+
requested = str(value or "mcp:read offline_access").split()
|
| 150 |
+
requested = list(dict.fromkeys(requested))
|
| 151 |
+
if (not requested or any(scope not in SCOPES for scope in requested)
|
| 152 |
+
or "mcp:read" not in requested
|
| 153 |
+
or ("mcp:write" in requested and access != external_access.ACCESS_READWRITE)):
|
| 154 |
+
raise ExternalOAuthError("oauth_scope_invalid")
|
| 155 |
+
return tuple(requested)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def normalise_resource(value) -> str:
|
| 159 |
+
"""Canonicalise one RFC 8707 audience URI while rejecting ambiguous targets."""
|
| 160 |
+
if not isinstance(value, str) or not value.strip() or len(value.strip()) > 1000:
|
| 161 |
+
raise ExternalOAuthError("oauth_resource_invalid")
|
| 162 |
+
parsed = urlsplit(value.strip())
|
| 163 |
+
if (parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc
|
| 164 |
+
or parsed.fragment or parsed.username or parsed.password):
|
| 165 |
+
raise ExternalOAuthError("oauth_resource_invalid")
|
| 166 |
+
try:
|
| 167 |
+
port = parsed.port
|
| 168 |
+
except ValueError as exc:
|
| 169 |
+
raise ExternalOAuthError("oauth_resource_invalid") from exc
|
| 170 |
+
hostname = (parsed.hostname or "").lower()
|
| 171 |
+
if not hostname:
|
| 172 |
+
raise ExternalOAuthError("oauth_resource_invalid")
|
| 173 |
+
host = f"[{hostname}]" if ":" in hostname else hostname
|
| 174 |
+
netloc = host + (f":{port}" if port is not None else "")
|
| 175 |
+
return urlunsplit((parsed.scheme.lower(), netloc, parsed.path, parsed.query, ""))
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _comparison_material(record, prefix: str) -> tuple[bool, bytes, str]:
|
| 179 |
+
usable = isinstance(record, dict) and record.get("algo") == ALGORITHM
|
| 180 |
+
if usable:
|
| 181 |
+
try:
|
| 182 |
+
salt = bytes.fromhex(str(record.get(prefix + "_salt") or ""))
|
| 183 |
+
expected = str(record.get(prefix + "_hash") or "")
|
| 184 |
+
if len(salt) != 32 or len(expected) != 64:
|
| 185 |
+
raise ValueError("bad digest")
|
| 186 |
+
int(expected, 16)
|
| 187 |
+
return True, salt, expected
|
| 188 |
+
except (TypeError, ValueError):
|
| 189 |
+
pass
|
| 190 |
+
return False, _DUMMY_SALT, _DUMMY_HASH
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _used_refresh_history(record, secret: str) -> tuple[bool, list[dict], bool]:
|
| 194 |
+
"""Validate and compare every consumed token retained for this live family."""
|
| 195 |
+
rows = record.get("used_refresh", ()) if isinstance(record, dict) else ()
|
| 196 |
+
if not isinstance(rows, list) or len(rows) > MAX_REFRESH_ROTATIONS:
|
| 197 |
+
return False, [], False
|
| 198 |
+
valid = True
|
| 199 |
+
matched = False
|
| 200 |
+
for item in rows:
|
| 201 |
+
usable, salt, expected = _comparison_material({
|
| 202 |
+
"algo": item.get("algo") if isinstance(item, dict) else None,
|
| 203 |
+
"used_salt": item.get("salt") if isinstance(item, dict) else None,
|
| 204 |
+
"used_hash": item.get("hash") if isinstance(item, dict) else None,
|
| 205 |
+
}, "used")
|
| 206 |
+
valid = valid and usable
|
| 207 |
+
matched = hmac.compare_digest(_digest(salt, secret), expected) and usable or matched
|
| 208 |
+
return valid, list(rows), matched
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _token(prefix: str, tenant: str, identifier: str, secret: str) -> str:
|
| 212 |
+
return f"{prefix}.{tenant}.{identifier}.{secret}"
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _parse_token(token, prefixes) -> tuple[str, str, str, str]:
|
| 216 |
+
parts = str(token or "").strip().split(".")
|
| 217 |
+
if (len(parts) != 4 or parts[0] not in prefixes or not all(parts[1:])
|
| 218 |
+
or len(parts[1]) > 128 or len(parts[2]) > 128 or len(parts[3]) > 256):
|
| 219 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 220 |
+
return parts[0], parts[1].lower(), parts[2], parts[3]
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _metadata(record: dict, client: dict | None = None) -> dict:
|
| 224 |
+
return {
|
| 225 |
+
"id": record.get("id"),
|
| 226 |
+
"client_id": record.get("client_id"),
|
| 227 |
+
"label": (client or {}).get("label"),
|
| 228 |
+
"created": _iso(record.get("created_at")),
|
| 229 |
+
"last_used": _iso(record.get("last_used_at")),
|
| 230 |
+
"expires_at": _iso(record.get("refresh_expires_at")),
|
| 231 |
+
"disconnected_at": _iso(record.get("disconnected_at")),
|
| 232 |
+
"access": record.get("access", external_access.ACCESS_READ),
|
| 233 |
+
"databases": (external_access.ALL_DATABASES if record.get("databases") == external_access.ALL_DATABASES
|
| 234 |
+
else list(record.get("databases") or [])),
|
| 235 |
+
"scopes": list(record.get("scopes") or ()),
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def _client_metadata(record: dict) -> dict:
|
| 240 |
+
access, databases = external_access.scope_of(record)
|
| 241 |
+
return {
|
| 242 |
+
"id": record.get("id"), "label": record.get("label"),
|
| 243 |
+
"redirect_uris": list(record.get("redirect_uris") or []),
|
| 244 |
+
"created": _iso(record.get("created_at")), "revoked_at": _iso(record.get("revoked_at")),
|
| 245 |
+
"access": access,
|
| 246 |
+
"databases": (external_access.ALL_DATABASES if databases == external_access.ALL_DATABASES
|
| 247 |
+
else list(databases)),
|
| 248 |
+
"token_endpoint_auth_method": "none", "pkce": "S256",
|
| 249 |
+
}
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def register_client(st, username: str, tenant: str, label, redirect_uris, *, access=None,
|
| 253 |
+
databases=None) -> dict:
|
| 254 |
+
owner, tenant_key = external_access._normalise_owner(username, tenant)
|
| 255 |
+
external_access._assert_bound_runtime(st, tenant_key)
|
| 256 |
+
external_access.resolve_active_user(owner, tenant_key)
|
| 257 |
+
clean_label = external_access._normalise_label(label)
|
| 258 |
+
redirects = _normalise_redirect_uris(redirect_uris)
|
| 259 |
+
clean_access = external_access.normalise_access(access)
|
| 260 |
+
clean_databases = external_access.normalise_databases(databases)
|
| 261 |
+
created = _now()
|
| 262 |
+
made = {}
|
| 263 |
+
|
| 264 |
+
def _set(value):
|
| 265 |
+
book = _book_value(value)
|
| 266 |
+
clients = dict(book["clients"])
|
| 267 |
+
client_id = "lpc_" + secrets.token_urlsafe(18)
|
| 268 |
+
while client_id in clients:
|
| 269 |
+
client_id = "lpc_" + secrets.token_urlsafe(18)
|
| 270 |
+
record = {
|
| 271 |
+
"id": client_id, "username": owner, "tenant": tenant_key, "label": clean_label,
|
| 272 |
+
"redirect_uris": list(redirects), "access": clean_access,
|
| 273 |
+
"databases": (external_access.ALL_DATABASES if clean_databases == external_access.ALL_DATABASES
|
| 274 |
+
else list(clean_databases)),
|
| 275 |
+
"created_at": created, "revoked_at": None,
|
| 276 |
+
}
|
| 277 |
+
clients[client_id] = record
|
| 278 |
+
book["clients"] = clients
|
| 279 |
+
made["record"] = record
|
| 280 |
+
return book
|
| 281 |
+
|
| 282 |
+
_update(st, _set)
|
| 283 |
+
return _client_metadata(made["record"])
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def list_clients(st, username: str, tenant: str) -> list[dict]:
|
| 287 |
+
owner, tenant_key = external_access._normalise_owner(username, tenant)
|
| 288 |
+
external_access._assert_bound_runtime(st, tenant_key)
|
| 289 |
+
return sorted([
|
| 290 |
+
_client_metadata(row) for row in _book(st)["clients"].values()
|
| 291 |
+
if isinstance(row, dict) and row.get("username") == owner
|
| 292 |
+
and row.get("tenant") == tenant_key and not row.get("revoked_at")
|
| 293 |
+
], key=lambda row: str(row.get("created") or ""), reverse=True)
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def edit_client(st, username: str, tenant: str, client_id: str, *, label=_UNSET,
|
| 297 |
+
redirect_uris=_UNSET, access=_UNSET, databases=_UNSET) -> dict:
|
| 298 |
+
owner, tenant_key = external_access._normalise_owner(username, tenant)
|
| 299 |
+
external_access._assert_bound_runtime(st, tenant_key)
|
| 300 |
+
external_access.resolve_active_user(owner, tenant_key)
|
| 301 |
+
if label is redirect_uris is access is databases is _UNSET:
|
| 302 |
+
raise ExternalOAuthError("oauth_edit_empty")
|
| 303 |
+
clean_label = external_access._normalise_label(label) if label is not _UNSET else _UNSET
|
| 304 |
+
clean_redirects = _normalise_redirect_uris(redirect_uris) if redirect_uris is not _UNSET else _UNSET
|
| 305 |
+
clean_access = external_access.normalise_access(access) if access is not _UNSET else _UNSET
|
| 306 |
+
clean_databases = external_access.normalise_databases(databases) if databases is not _UNSET else _UNSET
|
| 307 |
+
edited = {}
|
| 308 |
+
|
| 309 |
+
def _set(value):
|
| 310 |
+
book = _book_value(value)
|
| 311 |
+
clients = dict(book["clients"])
|
| 312 |
+
current = clients.get(client_id)
|
| 313 |
+
if (not isinstance(current, dict) or current.get("username") != owner
|
| 314 |
+
or current.get("tenant") != tenant_key or current.get("revoked_at")):
|
| 315 |
+
raise ExternalOAuthError("oauth_client_not_found")
|
| 316 |
+
row = dict(current)
|
| 317 |
+
if clean_label is not _UNSET:
|
| 318 |
+
row["label"] = clean_label
|
| 319 |
+
if clean_redirects is not _UNSET:
|
| 320 |
+
row["redirect_uris"] = list(clean_redirects)
|
| 321 |
+
if clean_access is not _UNSET:
|
| 322 |
+
row["access"] = clean_access
|
| 323 |
+
if clean_databases is not _UNSET:
|
| 324 |
+
row["databases"] = (external_access.ALL_DATABASES
|
| 325 |
+
if clean_databases == external_access.ALL_DATABASES
|
| 326 |
+
else list(clean_databases))
|
| 327 |
+
clients[client_id] = row
|
| 328 |
+
book["clients"] = clients
|
| 329 |
+
edited["record"] = row
|
| 330 |
+
return book
|
| 331 |
+
|
| 332 |
+
_update(st, _set)
|
| 333 |
+
return _client_metadata(edited["record"])
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def authorize(st, username: str, tenant: str, *, client_id, redirect_uri, state,
|
| 337 |
+
code_challenge, code_challenge_method, resource, scope=None, response_type="code",
|
| 338 |
+
now=None) -> dict:
|
| 339 |
+
owner, tenant_key = external_access._normalise_owner(username, tenant)
|
| 340 |
+
external_access._assert_bound_runtime(st, tenant_key)
|
| 341 |
+
external_access.resolve_active_user(owner, tenant_key)
|
| 342 |
+
if response_type != "code":
|
| 343 |
+
raise ExternalOAuthError("oauth_response_type_invalid")
|
| 344 |
+
clean_state = _normalise_state(state)
|
| 345 |
+
challenge = _normalise_challenge(code_challenge, code_challenge_method)
|
| 346 |
+
audience = normalise_resource(resource)
|
| 347 |
+
issued_at = _now(now)
|
| 348 |
+
code_id, secret, salt = secrets.token_urlsafe(18), secrets.token_urlsafe(32), secrets.token_bytes(32)
|
| 349 |
+
issued = {}
|
| 350 |
+
|
| 351 |
+
def _set(value):
|
| 352 |
+
book = _book_value(value)
|
| 353 |
+
client = book["clients"].get(client_id)
|
| 354 |
+
if (not isinstance(client, dict) or client.get("username") != owner
|
| 355 |
+
or client.get("tenant") != tenant_key or client.get("revoked_at")):
|
| 356 |
+
raise ExternalOAuthError("oauth_client_invalid")
|
| 357 |
+
if redirect_uri not in (client.get("redirect_uris") or []):
|
| 358 |
+
raise ExternalOAuthError("oauth_redirect_uri_invalid")
|
| 359 |
+
access, databases = external_access.scope_of(client)
|
| 360 |
+
scopes = _normalise_scopes(scope, access)
|
| 361 |
+
states = {key: row for key, row in book["states"].items()
|
| 362 |
+
if isinstance(row, dict) and float(row.get("expires_at") or 0) >= issued_at}
|
| 363 |
+
state_id = hashlib.sha256(f"{tenant_key}\0{owner}\0{client_id}\0{clean_state}".encode()).hexdigest()
|
| 364 |
+
if state_id in states:
|
| 365 |
+
raise ExternalOAuthError("oauth_state_replayed")
|
| 366 |
+
states[state_id] = {"expires_at": issued_at + CODE_TTL_SECONDS}
|
| 367 |
+
codes = {key: row for key, row in book["codes"].items()
|
| 368 |
+
if isinstance(row, dict) and float(row.get("expires_at") or 0) >= issued_at - CODE_TTL_SECONDS}
|
| 369 |
+
row = {
|
| 370 |
+
"id": code_id, "tenant": tenant_key, "username": owner, "client_id": client_id,
|
| 371 |
+
"redirect_uri": redirect_uri, "state_hash": _state_digest(clean_state),
|
| 372 |
+
"code_challenge": challenge, "resource": audience,
|
| 373 |
+
"scopes": list(scopes), "access": access,
|
| 374 |
+
"databases": (external_access.ALL_DATABASES if databases == external_access.ALL_DATABASES
|
| 375 |
+
else list(databases)),
|
| 376 |
+
"code_salt": salt.hex(), "code_hash": _digest(salt, secret), "algo": ALGORITHM,
|
| 377 |
+
"created_at": issued_at, "expires_at": issued_at + CODE_TTL_SECONDS,
|
| 378 |
+
"consumed_at": None,
|
| 379 |
+
}
|
| 380 |
+
codes[code_id] = row
|
| 381 |
+
book.update({"states": states, "codes": codes})
|
| 382 |
+
issued["record"] = row
|
| 383 |
+
return book
|
| 384 |
+
|
| 385 |
+
_update(st, _set)
|
| 386 |
+
return {"redirect_uri": redirect_uri, "state": clean_state,
|
| 387 |
+
"code": _token("lpc1", tenant_key, code_id, secret),
|
| 388 |
+
"expires_in": CODE_TTL_SECONDS}
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
def exchange_code(code, *, client_id, redirect_uri, code_verifier, resource,
|
| 392 |
+
runtime_resolver, now=None) -> dict:
|
| 393 |
+
_, tenant_key, code_id, secret = _parse_token(code, {"lpc1"})
|
| 394 |
+
verifier = _normalise_verifier(code_verifier)
|
| 395 |
+
audience = normalise_resource(resource)
|
| 396 |
+
try:
|
| 397 |
+
st = runtime_resolver(tenant_key)
|
| 398 |
+
external_access._assert_bound_runtime(st, tenant_key)
|
| 399 |
+
except (KeyError, external_access.ExternalAccessError) as exc:
|
| 400 |
+
raise ExternalOAuthError("oauth_invalid_grant") from exc
|
| 401 |
+
snapshot = _book(st)
|
| 402 |
+
candidate = snapshot["codes"].get(code_id)
|
| 403 |
+
if isinstance(candidate, dict):
|
| 404 |
+
try:
|
| 405 |
+
external_access.resolve_active_user(candidate.get("username"), tenant_key)
|
| 406 |
+
except external_access.ExternalAccessError as exc:
|
| 407 |
+
raise ExternalOAuthError("oauth_invalid_grant") from exc
|
| 408 |
+
issued_at = _now(now)
|
| 409 |
+
access_secret, refresh_secret = secrets.token_urlsafe(32), secrets.token_urlsafe(32)
|
| 410 |
+
access_salt, refresh_salt = secrets.token_bytes(32), secrets.token_bytes(32)
|
| 411 |
+
grant_id = secrets.token_urlsafe(18)
|
| 412 |
+
result = {}
|
| 413 |
+
|
| 414 |
+
def _set(value):
|
| 415 |
+
book = _book_value(value)
|
| 416 |
+
row = book["codes"].get(code_id)
|
| 417 |
+
usable, salt, expected = _comparison_material(row, "code")
|
| 418 |
+
matched = hmac.compare_digest(_digest(salt, secret), expected)
|
| 419 |
+
client = book["clients"].get(client_id)
|
| 420 |
+
if (not usable or not matched or not isinstance(row, dict) or row.get("consumed_at")
|
| 421 |
+
or float(row.get("expires_at") or 0) < issued_at
|
| 422 |
+
or row.get("tenant") != tenant_key or row.get("client_id") != client_id
|
| 423 |
+
or row.get("redirect_uri") != redirect_uri
|
| 424 |
+
or row.get("resource") != audience
|
| 425 |
+
or not isinstance(client, dict) or client.get("revoked_at")
|
| 426 |
+
or client.get("username") != row.get("username")
|
| 427 |
+
or not hmac.compare_digest(_pkce_challenge(verifier), str(row.get("code_challenge") or ""))):
|
| 428 |
+
raise ExternalOAuthError("oauth_invalid_grant")
|
| 429 |
+
grants = dict(book["grants"])
|
| 430 |
+
while grant_id in grants:
|
| 431 |
+
raise ExternalOAuthError("oauth_unavailable")
|
| 432 |
+
grant = {
|
| 433 |
+
"id": grant_id, "tenant": tenant_key, "username": row["username"],
|
| 434 |
+
"client_id": client_id, "resource": audience,
|
| 435 |
+
"access": row["access"], "databases": row["databases"],
|
| 436 |
+
"scopes": list(row["scopes"]), "algo": ALGORITHM,
|
| 437 |
+
"access_salt": access_salt.hex(), "access_hash": _digest(access_salt, access_secret),
|
| 438 |
+
"refresh_salt": refresh_salt.hex(), "refresh_hash": _digest(refresh_salt, refresh_secret),
|
| 439 |
+
"used_refresh": [], "refresh_generation": 0,
|
| 440 |
+
"created_at": issued_at, "access_expires_at": issued_at + ACCESS_TTL_SECONDS,
|
| 441 |
+
"refresh_expires_at": issued_at + REFRESH_TTL_SECONDS,
|
| 442 |
+
"last_used_at": None, "disconnected_at": None,
|
| 443 |
+
}
|
| 444 |
+
grants[grant_id] = grant
|
| 445 |
+
codes = dict(book["codes"])
|
| 446 |
+
consumed = dict(row)
|
| 447 |
+
consumed["consumed_at"] = issued_at
|
| 448 |
+
codes[code_id] = consumed
|
| 449 |
+
book.update({"codes": codes, "grants": grants})
|
| 450 |
+
result["grant"] = grant
|
| 451 |
+
return book
|
| 452 |
+
|
| 453 |
+
_update(st, _set)
|
| 454 |
+
grant = result["grant"]
|
| 455 |
+
return {
|
| 456 |
+
"token_type": "Bearer", "access_token": _token("lpo1", tenant_key, grant_id, access_secret),
|
| 457 |
+
"expires_in": ACCESS_TTL_SECONDS,
|
| 458 |
+
"refresh_token": _token("lpr1", tenant_key, grant_id, refresh_secret),
|
| 459 |
+
"scope": " ".join(grant["scopes"]),
|
| 460 |
+
}
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
def refresh(refresh_token, *, client_id, resource, runtime_resolver, now=None) -> dict:
|
| 464 |
+
_, tenant_key, grant_id, secret = _parse_token(refresh_token, {"lpr1"})
|
| 465 |
+
try:
|
| 466 |
+
st = runtime_resolver(tenant_key)
|
| 467 |
+
external_access._assert_bound_runtime(st, tenant_key)
|
| 468 |
+
except (KeyError, external_access.ExternalAccessError) as exc:
|
| 469 |
+
raise ExternalOAuthError("oauth_invalid_grant") from exc
|
| 470 |
+
snapshot = _book(st)
|
| 471 |
+
candidate = snapshot["grants"].get(grant_id)
|
| 472 |
+
if isinstance(candidate, dict):
|
| 473 |
+
try:
|
| 474 |
+
external_access.resolve_active_user(candidate.get("username"), tenant_key)
|
| 475 |
+
except external_access.ExternalAccessError as exc:
|
| 476 |
+
raise ExternalOAuthError("oauth_invalid_grant") from exc
|
| 477 |
+
issued_at = _now(now)
|
| 478 |
+
audience = normalise_resource(resource)
|
| 479 |
+
next_access, next_refresh = secrets.token_urlsafe(32), secrets.token_urlsafe(32)
|
| 480 |
+
next_access_salt, next_refresh_salt = secrets.token_bytes(32), secrets.token_bytes(32)
|
| 481 |
+
result = {}
|
| 482 |
+
|
| 483 |
+
def _set(value):
|
| 484 |
+
book = _book_value(value)
|
| 485 |
+
grants = dict(book["grants"])
|
| 486 |
+
row = grants.get(grant_id)
|
| 487 |
+
current_ok, current_salt, current_hash = _comparison_material(row, "refresh")
|
| 488 |
+
current_match = hmac.compare_digest(_digest(current_salt, secret), current_hash)
|
| 489 |
+
history_ok, used, used_match = _used_refresh_history(row, secret)
|
| 490 |
+
if (isinstance(row, dict) and not history_ok and not row.get("disconnected_at")):
|
| 491 |
+
corrupted = dict(row)
|
| 492 |
+
corrupted["disconnected_at"] = issued_at
|
| 493 |
+
grants[grant_id] = corrupted
|
| 494 |
+
book["grants"] = grants
|
| 495 |
+
result["error"] = "oauth_invalid_grant"
|
| 496 |
+
return book
|
| 497 |
+
if (isinstance(row, dict) and used_match and not row.get("disconnected_at")):
|
| 498 |
+
replayed = dict(row)
|
| 499 |
+
replayed["disconnected_at"] = issued_at
|
| 500 |
+
grants[grant_id] = replayed
|
| 501 |
+
book["grants"] = grants
|
| 502 |
+
result["error"] = "oauth_refresh_replayed"
|
| 503 |
+
return book
|
| 504 |
+
client = book["clients"].get(client_id)
|
| 505 |
+
if (not current_ok or not current_match or not isinstance(row, dict)
|
| 506 |
+
or row.get("disconnected_at") or row.get("tenant") != tenant_key
|
| 507 |
+
or row.get("client_id") != client_id
|
| 508 |
+
or row.get("resource") != audience
|
| 509 |
+
or float(row.get("refresh_expires_at") or 0) < issued_at
|
| 510 |
+
or not isinstance(client, dict) or client.get("revoked_at")
|
| 511 |
+
or client.get("username") != row.get("username")):
|
| 512 |
+
raise ExternalOAuthError("oauth_invalid_grant")
|
| 513 |
+
generation = int(row.get("refresh_generation") or 0)
|
| 514 |
+
if generation >= MAX_REFRESH_ROTATIONS or len(used) >= MAX_REFRESH_ROTATIONS:
|
| 515 |
+
exhausted = dict(row)
|
| 516 |
+
exhausted["disconnected_at"] = issued_at
|
| 517 |
+
grants[grant_id] = exhausted
|
| 518 |
+
book["grants"] = grants
|
| 519 |
+
result["error"] = "oauth_refresh_exhausted"
|
| 520 |
+
return book
|
| 521 |
+
used.append({"salt": row.get("refresh_salt"), "hash": row.get("refresh_hash"),
|
| 522 |
+
"algo": ALGORITHM})
|
| 523 |
+
rotated = dict(row)
|
| 524 |
+
rotated.update({
|
| 525 |
+
"used_refresh": used,
|
| 526 |
+
"refresh_generation": generation + 1,
|
| 527 |
+
"access_salt": next_access_salt.hex(),
|
| 528 |
+
"access_hash": _digest(next_access_salt, next_access),
|
| 529 |
+
"refresh_salt": next_refresh_salt.hex(),
|
| 530 |
+
"refresh_hash": _digest(next_refresh_salt, next_refresh),
|
| 531 |
+
"access_expires_at": issued_at + ACCESS_TTL_SECONDS,
|
| 532 |
+
"last_used_at": None,
|
| 533 |
+
})
|
| 534 |
+
grants[grant_id] = rotated
|
| 535 |
+
book["grants"] = grants
|
| 536 |
+
result["grant"] = rotated
|
| 537 |
+
return book
|
| 538 |
+
|
| 539 |
+
_update(st, _set)
|
| 540 |
+
if result.get("error"):
|
| 541 |
+
raise ExternalOAuthError(result["error"])
|
| 542 |
+
grant = result["grant"]
|
| 543 |
+
return {
|
| 544 |
+
"token_type": "Bearer", "access_token": _token("lpo1", tenant_key, grant_id, next_access),
|
| 545 |
+
"expires_in": ACCESS_TTL_SECONDS,
|
| 546 |
+
"refresh_token": _token("lpr1", tenant_key, grant_id, next_refresh),
|
| 547 |
+
"scope": " ".join(grant["scopes"]),
|
| 548 |
+
}
|
| 549 |
+
|
| 550 |
+
|
| 551 |
+
def _intersect_databases(left, right):
|
| 552 |
+
if left == external_access.ALL_DATABASES:
|
| 553 |
+
return right
|
| 554 |
+
if right == external_access.ALL_DATABASES:
|
| 555 |
+
return left
|
| 556 |
+
allowed = set(right or ())
|
| 557 |
+
return tuple(item for item in (left or ()) if item in allowed)
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
def verify_access(token: str, *, resource, runtime_resolver) -> VerifiedOAuth:
|
| 561 |
+
try:
|
| 562 |
+
_, tenant_key, grant_id, secret = _parse_token(token, {"lpo1"})
|
| 563 |
+
try:
|
| 564 |
+
st = runtime_resolver(tenant_key)
|
| 565 |
+
external_access._assert_bound_runtime(st, tenant_key)
|
| 566 |
+
except KeyError as exc:
|
| 567 |
+
raise ExternalOAuthAuthError(AUTH_ERROR) from exc
|
| 568 |
+
book = _book(st)
|
| 569 |
+
audience = normalise_resource(resource)
|
| 570 |
+
row = book["grants"].get(grant_id)
|
| 571 |
+
usable, salt, expected = _comparison_material(row, "access")
|
| 572 |
+
matched = hmac.compare_digest(_digest(salt, secret), expected)
|
| 573 |
+
client = book["clients"].get(row.get("client_id")) if isinstance(row, dict) else None
|
| 574 |
+
now = _now()
|
| 575 |
+
if (not usable or not matched or not isinstance(row, dict) or row.get("disconnected_at")
|
| 576 |
+
or row.get("tenant") != tenant_key or float(row.get("access_expires_at") or 0) < now
|
| 577 |
+
or row.get("resource") != audience
|
| 578 |
+
or not isinstance(client, dict) or client.get("revoked_at")
|
| 579 |
+
or client.get("username") != row.get("username")
|
| 580 |
+
or client.get("tenant") != tenant_key):
|
| 581 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 582 |
+
try:
|
| 583 |
+
public = external_access.resolve_active_user(row["username"], tenant_key)
|
| 584 |
+
except external_access.ExternalAccessError as exc:
|
| 585 |
+
raise ExternalOAuthAuthError(AUTH_ERROR) from exc
|
| 586 |
+
grant_access, grant_databases = external_access.scope_of(row)
|
| 587 |
+
client_access, client_databases = external_access.scope_of(client)
|
| 588 |
+
access = (external_access.ACCESS_READWRITE
|
| 589 |
+
if grant_access == client_access == external_access.ACCESS_READWRITE
|
| 590 |
+
and "mcp:write" in (row.get("scopes") or ())
|
| 591 |
+
else external_access.ACCESS_READ)
|
| 592 |
+
databases = _intersect_databases(grant_databases, client_databases)
|
| 593 |
+
if now - float(row.get("last_used_at") or 0) >= _STAMP_EVERY:
|
| 594 |
+
def _stamp(value):
|
| 595 |
+
latest = _book_value(value)
|
| 596 |
+
grants = dict(latest["grants"])
|
| 597 |
+
current = grants.get(grant_id)
|
| 598 |
+
if (not isinstance(current, dict) or current.get("disconnected_at")
|
| 599 |
+
or current.get("access_hash") != row.get("access_hash")):
|
| 600 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 601 |
+
current = dict(current)
|
| 602 |
+
current["last_used_at"] = now
|
| 603 |
+
grants[grant_id] = current
|
| 604 |
+
latest["grants"] = grants
|
| 605 |
+
return latest
|
| 606 |
+
_update(st, _stamp, auth=True)
|
| 607 |
+
return VerifiedOAuth(tenant_key, grant_id, row["client_id"], public, st, access,
|
| 608 |
+
databases, tuple(row.get("scopes") or ()), audience)
|
| 609 |
+
except ExternalOAuthAuthError:
|
| 610 |
+
raise
|
| 611 |
+
except Exception as exc:
|
| 612 |
+
raise ExternalOAuthAuthError(AUTH_ERROR) from exc
|
| 613 |
+
|
| 614 |
+
|
| 615 |
+
def disconnect(token: str, *, runtime_resolver, now=None) -> DisconnectedOAuth:
|
| 616 |
+
prefix, tenant_key, grant_id, secret = _parse_token(token, {"lpo1", "lpr1"})
|
| 617 |
+
try:
|
| 618 |
+
st = runtime_resolver(tenant_key)
|
| 619 |
+
external_access._assert_bound_runtime(st, tenant_key)
|
| 620 |
+
except (KeyError, external_access.ExternalAccessError) as exc:
|
| 621 |
+
raise ExternalOAuthAuthError(AUTH_ERROR) from exc
|
| 622 |
+
stopped = {}
|
| 623 |
+
at = _now(now)
|
| 624 |
+
material = "access" if prefix == "lpo1" else "refresh"
|
| 625 |
+
|
| 626 |
+
def _set(value):
|
| 627 |
+
book = _book_value(value)
|
| 628 |
+
grants = dict(book["grants"])
|
| 629 |
+
row = grants.get(grant_id)
|
| 630 |
+
usable, salt, expected = _comparison_material(row, material)
|
| 631 |
+
if (not usable or not hmac.compare_digest(_digest(salt, secret), expected)
|
| 632 |
+
or not isinstance(row, dict) or row.get("tenant") != tenant_key
|
| 633 |
+
or not isinstance(row.get("username"), str) or not row.get("username")):
|
| 634 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 635 |
+
disconnected = dict(row)
|
| 636 |
+
if not disconnected.get("disconnected_at"):
|
| 637 |
+
disconnected["disconnected_at"] = at
|
| 638 |
+
grants[grant_id] = disconnected
|
| 639 |
+
book["grants"] = grants
|
| 640 |
+
stopped["record"] = disconnected
|
| 641 |
+
stopped["client"] = book["clients"].get(disconnected.get("client_id"))
|
| 642 |
+
return book
|
| 643 |
+
|
| 644 |
+
_update(st, _set, auth=True)
|
| 645 |
+
record = stopped["record"]
|
| 646 |
+
return DisconnectedOAuth(tenant_key, record["username"],
|
| 647 |
+
_metadata(record, stopped.get("client")))
|
| 648 |
+
|
| 649 |
+
|
| 650 |
+
def disconnect_for_user(st, username: str, tenant: str, grant_id: str) -> dict:
|
| 651 |
+
owner, tenant_key = external_access._normalise_owner(username, tenant)
|
| 652 |
+
external_access._assert_bound_runtime(st, tenant_key)
|
| 653 |
+
stopped = {}
|
| 654 |
+
at = _now()
|
| 655 |
+
|
| 656 |
+
def _set(value):
|
| 657 |
+
book = _book_value(value)
|
| 658 |
+
grants = dict(book["grants"])
|
| 659 |
+
row = grants.get(grant_id)
|
| 660 |
+
if (not isinstance(row, dict) or row.get("username") != owner
|
| 661 |
+
or row.get("tenant") != tenant_key):
|
| 662 |
+
raise ExternalOAuthError("oauth_connection_not_found")
|
| 663 |
+
disconnected = dict(row)
|
| 664 |
+
if not disconnected.get("disconnected_at"):
|
| 665 |
+
disconnected["disconnected_at"] = at
|
| 666 |
+
grants[grant_id] = disconnected
|
| 667 |
+
book["grants"] = grants
|
| 668 |
+
stopped["record"] = disconnected
|
| 669 |
+
stopped["client"] = book["clients"].get(disconnected.get("client_id"))
|
| 670 |
+
return book
|
| 671 |
+
|
| 672 |
+
_update(st, _set)
|
| 673 |
+
return _metadata(stopped["record"], stopped.get("client"))
|
| 674 |
+
|
| 675 |
+
|
| 676 |
+
def list_connections(st, username: str, tenant: str) -> list[dict]:
|
| 677 |
+
owner, tenant_key = external_access._normalise_owner(username, tenant)
|
| 678 |
+
external_access._assert_bound_runtime(st, tenant_key)
|
| 679 |
+
book = _book(st)
|
| 680 |
+
return sorted([
|
| 681 |
+
_metadata(row, book["clients"].get(row.get("client_id")))
|
| 682 |
+
for row in book["grants"].values()
|
| 683 |
+
if isinstance(row, dict) and row.get("username") == owner
|
| 684 |
+
and row.get("tenant") == tenant_key and not row.get("disconnected_at")
|
| 685 |
+
], key=lambda row: str(row.get("created") or ""), reverse=True)
|
| 686 |
+
|
| 687 |
+
|
| 688 |
+
def label_for(st, grant_id: str) -> str:
|
| 689 |
+
book = _book(st)
|
| 690 |
+
grant = book["grants"].get(grant_id)
|
| 691 |
+
client = book["clients"].get(grant.get("client_id")) if isinstance(grant, dict) else None
|
| 692 |
+
if (not isinstance(grant, dict) or grant.get("disconnected_at")
|
| 693 |
+
or not isinstance(client, dict) or client.get("revoked_at")):
|
| 694 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 695 |
+
label = client.get("label")
|
| 696 |
+
if not isinstance(label, str) or not label:
|
| 697 |
+
raise ExternalOAuthError("oauth_unavailable")
|
| 698 |
+
return label
|
| 699 |
+
|
| 700 |
+
|
| 701 |
+
def _book_value(value) -> dict:
|
| 702 |
+
if value is None:
|
| 703 |
+
return {"clients": {}, "codes": {}, "states": {}, "grants": {}}
|
| 704 |
+
if not isinstance(value, dict):
|
| 705 |
+
raise ExternalOAuthError("oauth_unavailable")
|
| 706 |
+
book = dict(value)
|
| 707 |
+
for key in ("clients", "codes", "states", "grants"):
|
| 708 |
+
child = book.get(key, {})
|
| 709 |
+
if not isinstance(child, dict):
|
| 710 |
+
raise ExternalOAuthError("oauth_unavailable")
|
| 711 |
+
book[key] = dict(child)
|
| 712 |
+
return book
|
| 713 |
+
|
| 714 |
+
|
| 715 |
+
def _update(st, callback, *, auth=False) -> None:
|
| 716 |
+
try:
|
| 717 |
+
st.update(BUCKET, callback, flush="sync")
|
| 718 |
+
except (ExternalOAuthError, ExternalOAuthAuthError):
|
| 719 |
+
raise
|
| 720 |
+
except Exception as exc:
|
| 721 |
+
if auth:
|
| 722 |
+
raise ExternalOAuthAuthError(AUTH_ERROR) from exc
|
| 723 |
+
raise ExternalOAuthError("oauth_unavailable") from exc
|
platform/core/external_tools.py
CHANGED
|
@@ -6,11 +6,10 @@ Adding a tool means adding a registry entry; writes also require the router's li
|
|
| 6 |
"""
|
| 7 |
import copy
|
| 8 |
import json
|
| 9 |
-
import uuid
|
| 10 |
from dataclasses import dataclass
|
| 11 |
-
from typing import Callable
|
| 12 |
|
| 13 |
-
from core import sheets, sheet_structure, sheet_templates
|
| 14 |
|
| 15 |
|
| 16 |
@dataclass(frozen=True)
|
|
@@ -24,9 +23,15 @@ class ToolContext:
|
|
| 24 |
cache_binding: Callable
|
| 25 |
list_databases: Callable
|
| 26 |
describe_database: Callable
|
|
|
|
| 27 |
view_event: Callable
|
|
|
|
| 28 |
app_upsert: Callable
|
| 29 |
app_delete: Callable
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
def _object(properties=None, required=(), **extra):
|
|
@@ -156,7 +161,7 @@ def _add_tab(ctx, args):
|
|
| 156 |
raise sheets.SheetRefusal(
|
| 157 |
"tab_name_taken",
|
| 158 |
"Tab " + wanted + " is already in this workbook. Choose another name.")
|
| 159 |
-
tab = {"id":
|
| 160 |
book["sheets"].append(tab)
|
| 161 |
return _put(ctx, args, book, tab["id"])
|
| 162 |
|
|
@@ -252,7 +257,7 @@ def _chart(ctx, args):
|
|
| 252 |
book = _book(ctx, args, write=True)
|
| 253 |
tab = sheets.find_tab(book, args["tab"])
|
| 254 |
chart = copy.deepcopy(args["chart"])
|
| 255 |
-
chart.setdefault("id",
|
| 256 |
tab["charts"] = [row for row in tab.get("charts", []) if row["id"] != chart["id"]] + [chart]
|
| 257 |
return {**_put(ctx, args, book, tab["id"]), "chart_id": chart["id"]}
|
| 258 |
|
|
@@ -269,6 +274,55 @@ def _tool(description, schema, handler, *, write=False, grants=("analyst", "shee
|
|
| 269 |
"grants": grants, "handler": handler}
|
| 270 |
|
| 271 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 272 |
TOOLS = {
|
| 273 |
"sheet_create_workbook": _tool("Create a private workbook with one initial tab.",
|
| 274 |
_object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
|
|
@@ -317,14 +371,44 @@ TOOLS = {
|
|
| 317 |
"describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
|
| 318 |
_object({"database": _STRING}, ("database",), additionalProperties=False),
|
| 319 |
lambda ctx, args: ctx.describe_database(args["database"]), grants=("analyst",)),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 320 |
"view_upsert": _tool(
|
| 321 |
"Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
|
| 322 |
_object({"database": _STRING, "view": _object({
|
| 323 |
"id": {"type": "string", "minLength": 1, "maxLength": 120},
|
| 324 |
"name": {"type": "string", "minLength": 1, "maxLength": 120},
|
| 325 |
-
"config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config")
|
|
|
|
| 326 |
("database", "view"), additionalProperties=False),
|
| 327 |
lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("analyst",)),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
"view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
|
| 329 |
_object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
|
| 330 |
additionalProperties=False),
|
|
@@ -339,9 +423,83 @@ TOOLS = {
|
|
| 339 |
_object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
|
| 340 |
additionalProperties=False),
|
| 341 |
lambda ctx, args: ctx.app_delete(args), write=True, grants=("analyst",)),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 342 |
}
|
| 343 |
|
| 344 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 345 |
def dispatch(ctx, name, arguments):
|
| 346 |
"""Dispatch one declared tool. The API translates core and grant refusals to MCP content."""
|
| 347 |
tool = TOOLS.get(name) if isinstance(name, str) else None
|
|
@@ -358,15 +516,20 @@ def dispatch(ctx, name, arguments):
|
|
| 358 |
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema.")
|
| 359 |
# Validate only the transport shape here. Document semantics belong to the app writers.
|
| 360 |
for key, value in arguments.items():
|
| 361 |
-
|
|
|
|
| 362 |
types = {"object": dict, "array": list, "string": str, "integer": int}
|
| 363 |
if kind in types and (not isinstance(value, types[kind]) or (kind == "integer" and isinstance(value, bool))):
|
| 364 |
raise sheets.SheetRefusal("tool_arguments_invalid", f"Tool argument {key} must be an {kind}.")
|
|
|
|
|
|
|
|
|
|
| 365 |
# A dimension patch must never become an arbitrary tab-document replacement.
|
| 366 |
if name == "sheet_set_dimensions":
|
| 367 |
allowed = schema["properties"]["dimensions"]["properties"]
|
| 368 |
if arguments["dimensions"].keys() - allowed.keys():
|
| 369 |
raise sheets.SheetRefusal("tool_arguments_invalid", "Dimensions may contain only the published geometry fields.")
|
| 370 |
-
|
|
|
|
| 371 |
raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.")
|
| 372 |
return tool["handler"](ctx, arguments)
|
|
|
|
| 6 |
"""
|
| 7 |
import copy
|
| 8 |
import json
|
|
|
|
| 9 |
from dataclasses import dataclass
|
| 10 |
+
from typing import Callable, Optional
|
| 11 |
|
| 12 |
+
from core import sheets, sheet_structure, sheet_templates, write_execution
|
| 13 |
|
| 14 |
|
| 15 |
@dataclass(frozen=True)
|
|
|
|
| 23 |
cache_binding: Callable
|
| 24 |
list_databases: Callable
|
| 25 |
describe_database: Callable
|
| 26 |
+
query_database: Callable
|
| 27 |
view_event: Callable
|
| 28 |
+
view_share_preview: Callable
|
| 29 |
app_upsert: Callable
|
| 30 |
app_delete: Callable
|
| 31 |
+
database_object: Optional[Callable] = None
|
| 32 |
+
sheet_object: Optional[Callable] = None
|
| 33 |
+
validated_batch: Optional[Callable] = None
|
| 34 |
+
confirm_change: Optional[Callable] = None
|
| 35 |
|
| 36 |
|
| 37 |
def _object(properties=None, required=(), **extra):
|
|
|
|
| 161 |
raise sheets.SheetRefusal(
|
| 162 |
"tab_name_taken",
|
| 163 |
"Tab " + wanted + " is already in this workbook. Choose another name.")
|
| 164 |
+
tab = {"id": write_execution.new_id()[:12], "name": args["name"], "cells": {}}
|
| 165 |
book["sheets"].append(tab)
|
| 166 |
return _put(ctx, args, book, tab["id"])
|
| 167 |
|
|
|
|
| 257 |
book = _book(ctx, args, write=True)
|
| 258 |
tab = sheets.find_tab(book, args["tab"])
|
| 259 |
chart = copy.deepcopy(args["chart"])
|
| 260 |
+
chart.setdefault("id", write_execution.new_id()[:12])
|
| 261 |
tab["charts"] = [row for row in tab.get("charts", []) if row["id"] != chart["id"]] + [chart]
|
| 262 |
return {**_put(ctx, args, book, tab["id"]), "chart_id": chart["id"]}
|
| 263 |
|
|
|
|
| 274 |
"grants": grants, "handler": handler}
|
| 275 |
|
| 276 |
|
| 277 |
+
def _callback(ctx, name, args):
|
| 278 |
+
callback = getattr(ctx, name, None)
|
| 279 |
+
if not callable(callback):
|
| 280 |
+
raise sheets.SheetRefusal(
|
| 281 |
+
"tool_capability_unsupported",
|
| 282 |
+
"This deployment does not publish that object operation in its capability manifest.")
|
| 283 |
+
return callback(args)
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
_DATABASE_ACTIONS = (
|
| 287 |
+
"list", "describe", "query", "record_create", "record_update", "record_delete",
|
| 288 |
+
"field_create", "field_update", "field_delete",
|
| 289 |
+
)
|
| 290 |
+
_VIEW_ACTIONS = ("upsert", "delete", "share_preview")
|
| 291 |
+
_SHEET_OBJECT_ACTIONS = {
|
| 292 |
+
"workbook_object": ("list", "get", "create", "named_ranges", "export"),
|
| 293 |
+
"worksheet_object": (
|
| 294 |
+
"list", "get", "add", "update", "delete", "geometry", "hidden_bands",
|
| 295 |
+
"freeze", "gridlines", "named_ranges",
|
| 296 |
+
),
|
| 297 |
+
"range_object": (
|
| 298 |
+
"read", "write_values", "write_formulas", "write_styles", "number_format",
|
| 299 |
+
"fill_down", "clear",
|
| 300 |
+
),
|
| 301 |
+
"calculation_object": ("calculate", "refresh"),
|
| 302 |
+
}
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def _object_tool(name, actions):
|
| 306 |
+
target = {
|
| 307 |
+
"workbook_id": _STRING,
|
| 308 |
+
"worksheet": _STRING,
|
| 309 |
+
"range": _RANGE,
|
| 310 |
+
"revision": {"type": "integer"},
|
| 311 |
+
"arguments": {"type": "object"},
|
| 312 |
+
}
|
| 313 |
+
return _tool(
|
| 314 |
+
"Use the published Excel-shaped capability manifest for one "
|
| 315 |
+
+ name.removesuffix("_object") + " operation.",
|
| 316 |
+
_object({"action": {"enum": list(actions)}, **target}, ("action",),
|
| 317 |
+
additionalProperties=False),
|
| 318 |
+
lambda ctx, args, object_name=name: _callback(
|
| 319 |
+
ctx, "sheet_object", {"object": object_name.removesuffix("_object"), **args}),
|
| 320 |
+
# The action classifier below distinguishes reads. Declaring the surface write-capable
|
| 321 |
+
# keeps the API's literal allow-list fail closed when a new action is added.
|
| 322 |
+
write=True,
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
|
| 326 |
TOOLS = {
|
| 327 |
"sheet_create_workbook": _tool("Create a private workbook with one initial tab.",
|
| 328 |
_object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
|
|
|
|
| 371 |
"describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
|
| 372 |
_object({"database": _STRING}, ("database",), additionalProperties=False),
|
| 373 |
lambda ctx, args: ctx.describe_database(args["database"]), grants=("analyst",)),
|
| 374 |
+
"query_database": _tool(
|
| 375 |
+
"Query currently permitted records with bounded filters, stable sorts, cursor pagination and aggregates.",
|
| 376 |
+
_object({
|
| 377 |
+
"database": _STRING,
|
| 378 |
+
"fields": {"type": "array", "items": _STRING, "maxItems": 200},
|
| 379 |
+
"filters": {"oneOf": [_FILTER, {"type": "array", "items": {"type": "object"}}]},
|
| 380 |
+
"sorts": {"type": "array", "maxItems": 8, "items": _object({
|
| 381 |
+
"field": _STRING, "direction": {"enum": ["asc", "desc"]},
|
| 382 |
+
}, ("field", "direction"), additionalProperties=False)},
|
| 383 |
+
"limit": {"type": "integer", "minimum": 1, "maximum": 500},
|
| 384 |
+
"cursor": _STRING,
|
| 385 |
+
"aggregates": {"type": "array", "maxItems": 16, "items": _object({
|
| 386 |
+
"fn": {"enum": ["count", "count_distinct", "sum", "avg", "min", "max"]},
|
| 387 |
+
"field": _STRING,
|
| 388 |
+
}, ("fn",), additionalProperties=False)},
|
| 389 |
+
}, ("database",), additionalProperties=False),
|
| 390 |
+
lambda ctx, args: ctx.query_database(args["database"], {
|
| 391 |
+
key: value for key, value in args.items() if key != "database"
|
| 392 |
+
}), grants=("analyst",)),
|
| 393 |
"view_upsert": _tool(
|
| 394 |
"Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
|
| 395 |
_object({"database": _STRING, "view": _object({
|
| 396 |
"id": {"type": "string", "minLength": 1, "maxLength": 120},
|
| 397 |
"name": {"type": "string", "minLength": 1, "maxLength": 120},
|
| 398 |
+
"config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"),
|
| 399 |
+
additionalProperties=False)},
|
| 400 |
("database", "view"), additionalProperties=False),
|
| 401 |
lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("analyst",)),
|
| 402 |
+
"view_share_preview": _tool(
|
| 403 |
+
"Preview a personal draft View's C8 audience and field inheritance without sharing or mutating it.",
|
| 404 |
+
_object({
|
| 405 |
+
"database": _STRING,
|
| 406 |
+
"view_id": _STRING,
|
| 407 |
+
"entries": {"type": "array", "maxItems": 200, "items": _object({
|
| 408 |
+
"user": _STRING, "role": {"enum": ["view", "edit"]},
|
| 409 |
+
}, ("user", "role"), additionalProperties=False)},
|
| 410 |
+
}, ("database", "view_id", "entries"), additionalProperties=False),
|
| 411 |
+
lambda ctx, args: ctx.view_share_preview(args), grants=("analyst",)),
|
| 412 |
"view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
|
| 413 |
_object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
|
| 414 |
additionalProperties=False),
|
|
|
|
| 423 |
_object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
|
| 424 |
additionalProperties=False),
|
| 425 |
lambda ctx, args: ctx.app_delete(args), write=True, grants=("analyst",)),
|
| 426 |
+
"database_object": _tool(
|
| 427 |
+
"Discover/query a database or preview-confirm record and field CRUD through its live UI walls.",
|
| 428 |
+
_object({
|
| 429 |
+
"action": {"enum": list(_DATABASE_ACTIONS)},
|
| 430 |
+
"database": _STRING,
|
| 431 |
+
"record_id": {"oneOf": [_STRING, {"type": "integer"}]},
|
| 432 |
+
"values": {"type": "object"},
|
| 433 |
+
"field": {"type": "object"},
|
| 434 |
+
"field_key": _STRING,
|
| 435 |
+
"query": {"type": "object"},
|
| 436 |
+
}, ("action",), additionalProperties=False),
|
| 437 |
+
lambda ctx, args: _callback(ctx, "database_object", args), write=True,
|
| 438 |
+
grants=("analyst",)),
|
| 439 |
+
"view_object": _tool(
|
| 440 |
+
"Read or preview-confirm a personal View operation through the ordinary grid writer.",
|
| 441 |
+
_object({
|
| 442 |
+
"action": {"enum": list(_VIEW_ACTIONS)}, "database": _STRING,
|
| 443 |
+
"view": {"type": "object"}, "view_id": _STRING,
|
| 444 |
+
"entries": {"type": "array", "items": {"type": "object"}, "maxItems": 200},
|
| 445 |
+
}, ("action", "database"), additionalProperties=False),
|
| 446 |
+
lambda ctx, args: (
|
| 447 |
+
ctx.view_event("view_upsert", {"database": args["database"], "view": args["view"]})
|
| 448 |
+
if args["action"] == "upsert" else
|
| 449 |
+
ctx.view_event("view_delete", {"database": args["database"],
|
| 450 |
+
"view_id": args["view_id"]})
|
| 451 |
+
if args["action"] == "delete" else
|
| 452 |
+
ctx.view_share_preview({"database": args["database"],
|
| 453 |
+
"view_id": args["view_id"],
|
| 454 |
+
"entries": args.get("entries") or []})
|
| 455 |
+
), write=True, grants=("analyst",)),
|
| 456 |
+
**{name: _object_tool(name, actions)
|
| 457 |
+
for name, actions in _SHEET_OBJECT_ACTIONS.items()},
|
| 458 |
+
"validated_batch": _tool(
|
| 459 |
+
"Validate a bounded object-operation batch; writes preview and confirm as one unit.",
|
| 460 |
+
_object({"operations": {"type": "array", "minItems": 1, "maxItems": 50,
|
| 461 |
+
"items": _object({"name": _STRING,
|
| 462 |
+
"arguments": {"type": "object"}},
|
| 463 |
+
("name", "arguments"),
|
| 464 |
+
additionalProperties=False)}},
|
| 465 |
+
("operations",), additionalProperties=False),
|
| 466 |
+
lambda ctx, args: _callback(ctx, "validated_batch", args), write=True,
|
| 467 |
+
grants=("analyst",)),
|
| 468 |
+
"confirm_change": _tool(
|
| 469 |
+
"Confirm one unexpired preview. Destructive/schema changes require its exact typed phrase.",
|
| 470 |
+
_object({"confirmation_token": _STRING, "confirmation_phrase": _STRING},
|
| 471 |
+
("confirmation_token",), additionalProperties=False),
|
| 472 |
+
lambda ctx, args: _callback(ctx, "confirm_change", args), grants=("analyst",)),
|
| 473 |
}
|
| 474 |
|
| 475 |
|
| 476 |
+
_OBJECT_READ_ACTIONS = {
|
| 477 |
+
"database_object": frozenset({"list", "describe", "query"}),
|
| 478 |
+
"view_object": frozenset({"share_preview"}),
|
| 479 |
+
"workbook_object": frozenset({"list", "get", "export"}),
|
| 480 |
+
"worksheet_object": frozenset({"list", "get"}),
|
| 481 |
+
"range_object": frozenset({"read"}),
|
| 482 |
+
"calculation_object": frozenset(),
|
| 483 |
+
}
|
| 484 |
+
|
| 485 |
+
|
| 486 |
+
def is_write(name, arguments) -> bool:
|
| 487 |
+
"""Classify dynamic object tools without trusting an unvalidated action as a read."""
|
| 488 |
+
tool = TOOLS.get(name) if isinstance(name, str) else None
|
| 489 |
+
if tool is None or name == "confirm_change":
|
| 490 |
+
return False
|
| 491 |
+
if name == "validated_batch":
|
| 492 |
+
operations = arguments.get("operations") if isinstance(arguments, dict) else None
|
| 493 |
+
if not isinstance(operations, list):
|
| 494 |
+
return True
|
| 495 |
+
return any(is_write(item.get("name"), item.get("arguments"))
|
| 496 |
+
for item in operations if isinstance(item, dict))
|
| 497 |
+
if name in _OBJECT_READ_ACTIONS:
|
| 498 |
+
action = arguments.get("action") if isinstance(arguments, dict) else None
|
| 499 |
+
return action not in _OBJECT_READ_ACTIONS[name]
|
| 500 |
+
return bool(tool.get("write"))
|
| 501 |
+
|
| 502 |
+
|
| 503 |
def dispatch(ctx, name, arguments):
|
| 504 |
"""Dispatch one declared tool. The API translates core and grant refusals to MCP content."""
|
| 505 |
tool = TOOLS.get(name) if isinstance(name, str) else None
|
|
|
|
| 516 |
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema.")
|
| 517 |
# Validate only the transport shape here. Document semantics belong to the app writers.
|
| 518 |
for key, value in arguments.items():
|
| 519 |
+
property_schema = schema["properties"].get(key, {})
|
| 520 |
+
kind = property_schema.get("type")
|
| 521 |
types = {"object": dict, "array": list, "string": str, "integer": int}
|
| 522 |
if kind in types and (not isinstance(value, types[kind]) or (kind == "integer" and isinstance(value, bool))):
|
| 523 |
raise sheets.SheetRefusal("tool_arguments_invalid", f"Tool argument {key} must be an {kind}.")
|
| 524 |
+
if "enum" in property_schema and value not in property_schema["enum"]:
|
| 525 |
+
raise sheets.SheetRefusal(
|
| 526 |
+
"tool_arguments_invalid", f"Tool argument {key} is not a published operation.")
|
| 527 |
# A dimension patch must never become an arbitrary tab-document replacement.
|
| 528 |
if name == "sheet_set_dimensions":
|
| 529 |
allowed = schema["properties"]["dimensions"]["properties"]
|
| 530 |
if arguments["dimensions"].keys() - allowed.keys():
|
| 531 |
raise sheets.SheetRefusal("tool_arguments_invalid", "Dimensions may contain only the published geometry fields.")
|
| 532 |
+
encoded_arguments = json.dumps(arguments, ensure_ascii=False, allow_nan=False)
|
| 533 |
+
if any(prefix in encoded_arguments for prefix in ("lpb1.", "lpo1.", "lpr1.", "lpc1.")):
|
| 534 |
raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.")
|
| 535 |
return tool["handler"](ctx, arguments)
|
platform/core/field_permissions.py
CHANGED
|
@@ -6,6 +6,7 @@ that could never be true. This module is the single normalization point for the
|
|
| 6 |
personal, collaborative, or specific users.
|
| 7 |
"""
|
| 8 |
|
|
|
|
| 9 |
import time
|
| 10 |
|
| 11 |
import core.shared_overlay as shared_overlay
|
|
@@ -26,6 +27,18 @@ FIELD_ORIGINS = ("preset", "user")
|
|
| 26 |
FIELD_AUDIENCES = ("private", "users", "everyone")
|
| 27 |
FIELD_VALUE_STRATA = ("shared", "personal")
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
#: ⭐⭐ W42-T47 / RULING R15 / CONTRACT C10 — WHICH STRATUM A DEFINITION WAS READ FROM.
|
| 30 |
#:
|
| 31 |
#: C10: *"`field_classes` reports, beside `{origin, audience, sharedBy, values, owner}`, WHICH
|
|
@@ -102,6 +115,377 @@ def grant_entries(permissions):
|
|
| 102 |
return []
|
| 103 |
|
| 104 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
def permissions_from_grants(entries):
|
| 106 |
"""⭐⭐ W40-T03 / CONTRACT C4 — THE CLASSIFICATION IS DERIVED FROM THE REGISTRY, NEVER STAMPED.
|
| 107 |
|
|
|
|
| 6 |
personal, collaborative, or specific users.
|
| 7 |
"""
|
| 8 |
|
| 9 |
+
import copy
|
| 10 |
import time
|
| 11 |
|
| 12 |
import core.shared_overlay as shared_overlay
|
|
|
|
| 27 |
FIELD_AUDIENCES = ("private", "users", "everyone")
|
| 28 |
FIELD_VALUE_STRATA = ("shared", "personal")
|
| 29 |
|
| 30 |
+
#: SDLC63 C8 — the View-to-field edges live beside the View grants, in the SAME tenant-scoped
|
| 31 |
+
#: document, but outside ``shares.KINDS``. A boundary names fields once; it never copies the
|
| 32 |
+
#: View's current people into N field grant records. Consequently a later audience edit takes
|
| 33 |
+
#: effect on the next permission read and removing one View cannot erase a direct or second-View
|
| 34 |
+
#: path. ``core.shares`` deliberately remains the direct-grant registry; callers that need to
|
| 35 |
+
#: explain provenance can therefore distinguish its record from this derived edge.
|
| 36 |
+
VIEW_FIELD_BOUNDARIES_KEY = "view_field_boundaries"
|
| 37 |
+
VIEW_FIELD_AUDIT_CAP = 200
|
| 38 |
+
VIEW_SHARE_DIAGNOSTICS_KEY = "view_share_diagnostics"
|
| 39 |
+
VIEW_SHARE_DIAGNOSTIC_CAP = 30
|
| 40 |
+
VIEW_SHARE_DIAGNOSTIC_FIELDS_CAP = 500
|
| 41 |
+
|
| 42 |
#: ⭐⭐ W42-T47 / RULING R15 / CONTRACT C10 — WHICH STRATUM A DEFINITION WAS READ FROM.
|
| 43 |
#:
|
| 44 |
#: C10: *"`field_classes` reports, beside `{origin, audience, sharedBy, values, owner}`, WHICH
|
|
|
|
| 115 |
return []
|
| 116 |
|
| 117 |
|
| 118 |
+
# ---------------------------------------------------------- SDLC63 / C8: View audience inheritance
|
| 119 |
+
|
| 120 |
+
def view_field_keys(view):
|
| 121 |
+
"""Fields deliberately used by one View, in stable encounter order.
|
| 122 |
+
|
| 123 |
+
``config.order`` is intentionally absent. Both the browser reconciler and the server save
|
| 124 |
+
door append every database field to that list, including columns the author never selected;
|
| 125 |
+
treating it as an inclusion list is the old over-share. Visible columns and fields used by a
|
| 126 |
+
View rule/display are deliberate. Hiding a plain column therefore removes that inherited
|
| 127 |
+
path, while a hidden filter/group/sort input stays readable because the View still depends on
|
| 128 |
+
it.
|
| 129 |
+
"""
|
| 130 |
+
if not isinstance(view, dict):
|
| 131 |
+
return []
|
| 132 |
+
cfg = view.get("config") if isinstance(view.get("config"), dict) else view
|
| 133 |
+
keys, seen = [], set()
|
| 134 |
+
|
| 135 |
+
def add(raw):
|
| 136 |
+
key = str(raw or "").strip()
|
| 137 |
+
if key and key not in seen:
|
| 138 |
+
seen.add(key)
|
| 139 |
+
keys.append(key)
|
| 140 |
+
|
| 141 |
+
def walk_filter(node):
|
| 142 |
+
if isinstance(node, list):
|
| 143 |
+
for child in node:
|
| 144 |
+
walk_filter(child)
|
| 145 |
+
return
|
| 146 |
+
if not isinstance(node, dict):
|
| 147 |
+
return
|
| 148 |
+
walk_filter(node.get("children"))
|
| 149 |
+
add(node.get("colId"))
|
| 150 |
+
rhs = node.get("rhs")
|
| 151 |
+
if isinstance(rhs, dict) and rhs.get("kind") != "stat":
|
| 152 |
+
add(rhs.get("colId"))
|
| 153 |
+
|
| 154 |
+
for key in cfg.get("visible") or ():
|
| 155 |
+
add(key)
|
| 156 |
+
walk_filter(cfg.get("filters"))
|
| 157 |
+
add(cfg.get("groupBy"))
|
| 158 |
+
colour = cfg.get("colorBy")
|
| 159 |
+
add(colour.get("colId") if isinstance(colour, dict) else colour)
|
| 160 |
+
sorts = cfg.get("sorts")
|
| 161 |
+
if isinstance(sorts, dict):
|
| 162 |
+
sorts = [sorts]
|
| 163 |
+
for spec in sorts or ():
|
| 164 |
+
if isinstance(spec, dict):
|
| 165 |
+
add(spec.get("colId"))
|
| 166 |
+
for key in (cfg.get("summary") or {}) if isinstance(cfg.get("summary"), dict) else ():
|
| 167 |
+
add(key)
|
| 168 |
+
display = cfg.get("display")
|
| 169 |
+
if isinstance(display, dict):
|
| 170 |
+
for name, value in display.items():
|
| 171 |
+
if str(name).endswith("Field"):
|
| 172 |
+
add(value)
|
| 173 |
+
for chart in display.get("charts") or ():
|
| 174 |
+
if isinstance(chart, dict):
|
| 175 |
+
add(chart.get("x"))
|
| 176 |
+
add(chart.get("y"))
|
| 177 |
+
for metric in display.get("calendarMetrics") or ():
|
| 178 |
+
if isinstance(metric, dict):
|
| 179 |
+
add(metric.get("field"))
|
| 180 |
+
for key in display.get("fields") or ():
|
| 181 |
+
add(key)
|
| 182 |
+
return keys
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _st_for_boundaries(st=None):
|
| 186 |
+
"""Use the same tenant handle ``core.shares`` uses without reaching into a second store."""
|
| 187 |
+
return st if st is not None else shares._st(None)
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def sync_view_field_boundary(view, grant_topic, actor, st=None, *, audit_additions=True,
|
| 191 |
+
force=False):
|
| 192 |
+
"""Persist one View's current field edge and return its delta.
|
| 193 |
+
|
| 194 |
+
The audience snapshot used by an addition audit is read inside the same ``object_shares``
|
| 195 |
+
update as the boundary change. It is evidence only; permission resolution always reads the
|
| 196 |
+
live View grant, so the snapshot can never become an access list.
|
| 197 |
+
"""
|
| 198 |
+
view_id = str((view or {}).get("id") or "").strip()
|
| 199 |
+
topic = str(grant_topic or "").strip()
|
| 200 |
+
who = _uname(actor)
|
| 201 |
+
if not view_id or not topic:
|
| 202 |
+
return {"added": [], "removed": [], "audience": [], "owner": None}
|
| 203 |
+
wanted = view_field_keys(view)
|
| 204 |
+
result = {"added": [], "removed": [], "audience": [], "owner": None}
|
| 205 |
+
handle = _st_for_boundaries(st)
|
| 206 |
+
|
| 207 |
+
# A private View needs no second write on every filter/resize autosave. Its first share calls
|
| 208 |
+
# this helper with ``force=True`` after the View grant lands; from then on the existing edge
|
| 209 |
+
# makes field additions/removals write-bearing. This keeps C8 off the ordinary private-view
|
| 210 |
+
# hot path without opening a share-before-boundary race.
|
| 211 |
+
try:
|
| 212 |
+
current = handle.get(shares.SHARES_KEY) or {}
|
| 213 |
+
except Exception: # noqa: BLE001
|
| 214 |
+
current = {}
|
| 215 |
+
current_boundaries = current.get(VIEW_FIELD_BOUNDARIES_KEY) or {}
|
| 216 |
+
current_grant = (current.get("view") or {}).get(view_id)
|
| 217 |
+
current_grant = current_grant if isinstance(current_grant, dict) else {}
|
| 218 |
+
current_audience = shares._clean_entries(current_grant.get("entries"))
|
| 219 |
+
current_owner = _uname(current_grant.get("owner")) or None
|
| 220 |
+
prior = (current_boundaries.get(view_id)
|
| 221 |
+
if isinstance(current_boundaries, dict) else None)
|
| 222 |
+
if not force and not isinstance(prior, dict) and not current_audience:
|
| 223 |
+
return {"added": [], "removed": [], "audience": [], "owner": current_owner}
|
| 224 |
+
if isinstance(prior, dict):
|
| 225 |
+
before = [str(key) for key in (prior.get("fields") or ()) if str(key)]
|
| 226 |
+
if prior.get("topic") == topic and before == wanted:
|
| 227 |
+
return {"added": [], "removed": [],
|
| 228 |
+
"audience": current_audience, "owner": current_owner}
|
| 229 |
+
|
| 230 |
+
def _apply(data):
|
| 231 |
+
data = data if isinstance(data, dict) else {}
|
| 232 |
+
boundaries = dict(data.get(VIEW_FIELD_BOUNDARIES_KEY) or {})
|
| 233 |
+
prior = boundaries.get(view_id) if isinstance(boundaries.get(view_id), dict) else {}
|
| 234 |
+
before = [str(key) for key in (prior.get("fields") or ()) if str(key)]
|
| 235 |
+
before_set, wanted_set = set(before), set(wanted)
|
| 236 |
+
added = [key for key in wanted if key not in before_set]
|
| 237 |
+
removed = [key for key in before if key not in wanted_set]
|
| 238 |
+
grant = (data.get("view") or {}).get(view_id)
|
| 239 |
+
grant = grant if isinstance(grant, dict) else {}
|
| 240 |
+
audience = shares._clean_entries(grant.get("entries"))
|
| 241 |
+
owner = _uname(grant.get("owner")) or None
|
| 242 |
+
audit = [row for row in (prior.get("audit") or ()) if isinstance(row, dict)]
|
| 243 |
+
if audit_additions and audience:
|
| 244 |
+
stamp = time.time()
|
| 245 |
+
snapshot = [{"user": row["user"], "role": row["role"]} for row in audience]
|
| 246 |
+
for key in added:
|
| 247 |
+
audit.append({"ts": stamp, "actor": who or None, "field": key,
|
| 248 |
+
"view": view_id, "audience": snapshot})
|
| 249 |
+
boundaries[view_id] = {
|
| 250 |
+
"topic": topic,
|
| 251 |
+
"fields": wanted,
|
| 252 |
+
"updatedBy": who or None,
|
| 253 |
+
"audit": audit[-VIEW_FIELD_AUDIT_CAP:],
|
| 254 |
+
}
|
| 255 |
+
data[VIEW_FIELD_BOUNDARIES_KEY] = boundaries
|
| 256 |
+
result.update({"added": added, "removed": removed,
|
| 257 |
+
"audience": audience, "owner": owner})
|
| 258 |
+
return data
|
| 259 |
+
|
| 260 |
+
handle.update(shares.SHARES_KEY, _apply, flush="async")
|
| 261 |
+
return result
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def set_view_grants_with_boundary(view, grant_topic, entries, *, owner, granter, expect,
|
| 265 |
+
st=None, _fail_after_prepare=False):
|
| 266 |
+
"""Replace one View audience and its field edge in one store transaction.
|
| 267 |
+
|
| 268 |
+
The mutation is built on a detached copy. If comparison, normalization, a test-injected
|
| 269 |
+
mid-operation fault, or persistence fails, neither the audience nor its derived boundary is
|
| 270 |
+
visible. The callback writes one bounded View record plus one bounded list of field keys;
|
| 271 |
+
its store-call count is therefore constant for a View with one field or five hundred.
|
| 272 |
+
|
| 273 |
+
``_fail_after_prepare`` is an internal verifier seam. It raises after both changes have
|
| 274 |
+
been prepared but before the detached document is returned, proving rollback at the only
|
| 275 |
+
point a half-applied mutation could otherwise escape.
|
| 276 |
+
"""
|
| 277 |
+
view_id = str((view or {}).get("id") or "").strip()
|
| 278 |
+
topic = str(grant_topic or "").strip()
|
| 279 |
+
if not view_id or not topic:
|
| 280 |
+
raise ValueError("a View share must name its View and database")
|
| 281 |
+
clean = shares._clean_entries(entries)
|
| 282 |
+
wanted = view_field_keys(view)
|
| 283 |
+
owner_l = _uname(owner) or None
|
| 284 |
+
granter_l = _uname(granter) or None
|
| 285 |
+
expected = ([(row["user"], row["role"])
|
| 286 |
+
for row in shares._clean_entries(expect)]
|
| 287 |
+
if expect is not None else None)
|
| 288 |
+
answer = {}
|
| 289 |
+
|
| 290 |
+
def _apply(raw):
|
| 291 |
+
data = copy.deepcopy(raw) if isinstance(raw, dict) else {}
|
| 292 |
+
views = dict(data.get("view") or {})
|
| 293 |
+
prior = views.get(view_id) if isinstance(views.get(view_id), dict) else {}
|
| 294 |
+
prior_entries = shares._clean_entries(prior.get("entries"))
|
| 295 |
+
if expected is not None:
|
| 296 |
+
current = [(row["user"], row["role"]) for row in prior_entries]
|
| 297 |
+
if current != expected:
|
| 298 |
+
raise shares.GrantsChanged(
|
| 299 |
+
"this item was shared with somebody else while you were editing, so nothing "
|
| 300 |
+
"was saved. Reopen the sharing panel to see who has access now, then make "
|
| 301 |
+
"your change again.")
|
| 302 |
+
|
| 303 |
+
prior_by = {row["user"]: row.get("by") for row in prior_entries}
|
| 304 |
+
stamped = []
|
| 305 |
+
for row in clean:
|
| 306 |
+
saved = {"user": row["user"], "role": row["role"]}
|
| 307 |
+
by = prior_by[row["user"]] if row["user"] in prior_by else granter_l
|
| 308 |
+
if by:
|
| 309 |
+
saved["by"] = by
|
| 310 |
+
stamped.append(saved)
|
| 311 |
+
keep_owner = owner_l or _uname(prior.get("owner")) or None
|
| 312 |
+
if not stamped and not keep_owner:
|
| 313 |
+
views.pop(view_id, None)
|
| 314 |
+
else:
|
| 315 |
+
views[view_id] = {"owner": keep_owner, "entries": stamped}
|
| 316 |
+
|
| 317 |
+
boundaries = dict(data.get(VIEW_FIELD_BOUNDARIES_KEY) or {})
|
| 318 |
+
old_boundary = (boundaries.get(view_id)
|
| 319 |
+
if isinstance(boundaries.get(view_id), dict) else {})
|
| 320 |
+
boundaries[view_id] = {
|
| 321 |
+
"topic": topic,
|
| 322 |
+
"fields": list(wanted),
|
| 323 |
+
"updatedBy": granter_l,
|
| 324 |
+
"audit": [dict(row) for row in (old_boundary.get("audit") or ())
|
| 325 |
+
if isinstance(row, dict)][-VIEW_FIELD_AUDIT_CAP:],
|
| 326 |
+
}
|
| 327 |
+
data["view"] = views
|
| 328 |
+
data[VIEW_FIELD_BOUNDARIES_KEY] = boundaries
|
| 329 |
+
if _fail_after_prepare:
|
| 330 |
+
raise RuntimeError("forced View-share transaction failure")
|
| 331 |
+
answer.update({"owner": keep_owner, "entries": stamped,
|
| 332 |
+
"fieldCount": len(wanted)})
|
| 333 |
+
return data
|
| 334 |
+
|
| 335 |
+
_st_for_boundaries(st).update(shares.SHARES_KEY, _apply, flush="sync")
|
| 336 |
+
return dict(answer)
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def record_view_share_diagnostic(reference, view_id, actor, stage, fields, cause, st=None):
|
| 340 |
+
"""Retain bounded server-only failure detail under a short support reference."""
|
| 341 |
+
ref = str(reference or "").strip().upper()[:40]
|
| 342 |
+
if not ref:
|
| 343 |
+
return False
|
| 344 |
+
clean_fields = []
|
| 345 |
+
seen = set()
|
| 346 |
+
for raw in fields or ():
|
| 347 |
+
key = str(raw or "").strip()[:160]
|
| 348 |
+
if key and key not in seen:
|
| 349 |
+
seen.add(key)
|
| 350 |
+
clean_fields.append(key)
|
| 351 |
+
if len(clean_fields) >= VIEW_SHARE_DIAGNOSTIC_FIELDS_CAP:
|
| 352 |
+
break
|
| 353 |
+
|
| 354 |
+
def _apply(raw):
|
| 355 |
+
data = copy.deepcopy(raw) if isinstance(raw, dict) else {}
|
| 356 |
+
rows = [dict(row) for row in (data.get(VIEW_SHARE_DIAGNOSTICS_KEY) or ())
|
| 357 |
+
if isinstance(row, dict)]
|
| 358 |
+
rows.append({
|
| 359 |
+
"reference": ref,
|
| 360 |
+
"ts": time.time(),
|
| 361 |
+
"view": str(view_id or "").strip()[:160],
|
| 362 |
+
"actor": _uname(actor) or None,
|
| 363 |
+
"stage": str(stage or "transaction").strip()[:80],
|
| 364 |
+
"fields": clean_fields,
|
| 365 |
+
"cause": (str(cause or "view share failed").strip()[:500]
|
| 366 |
+
if isinstance(cause, str)
|
| 367 |
+
else cause.__class__.__name__),
|
| 368 |
+
})
|
| 369 |
+
data[VIEW_SHARE_DIAGNOSTICS_KEY] = rows[-VIEW_SHARE_DIAGNOSTIC_CAP:]
|
| 370 |
+
return data
|
| 371 |
+
|
| 372 |
+
_st_for_boundaries(st).update(shares.SHARES_KEY, _apply, flush="sync")
|
| 373 |
+
return True
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
def drop_view_field_boundary(view_id, st=None):
|
| 377 |
+
"""Remove exactly one inherited source; direct and other-View grants remain untouched."""
|
| 378 |
+
view_id = str(view_id or "").strip()
|
| 379 |
+
if not view_id:
|
| 380 |
+
return False
|
| 381 |
+
removed = {"value": False}
|
| 382 |
+
|
| 383 |
+
def _apply(data):
|
| 384 |
+
boundaries = data.get(VIEW_FIELD_BOUNDARIES_KEY)
|
| 385 |
+
if isinstance(boundaries, dict) and view_id in boundaries:
|
| 386 |
+
boundaries.pop(view_id, None)
|
| 387 |
+
removed["value"] = True
|
| 388 |
+
return data
|
| 389 |
+
|
| 390 |
+
_st_for_boundaries(st).update(shares.SHARES_KEY, _apply, flush="async")
|
| 391 |
+
return removed["value"]
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def _field_role_sources_from_data(topic, key, me, is_admin, data):
|
| 395 |
+
"""Resolve one field from one already-read share document."""
|
| 396 |
+
oid = shares.field_oid(topic, key)
|
| 397 |
+
raw_direct = (data.get("field") or {}).get(oid)
|
| 398 |
+
raw_direct = raw_direct if isinstance(raw_direct, dict) else {}
|
| 399 |
+
direct_rec = {"owner": _uname(raw_direct.get("owner")) or None,
|
| 400 |
+
"entries": shares._clean_entries(raw_direct.get("entries"))}
|
| 401 |
+
direct = shares.role_from(direct_rec, me, is_admin=is_admin)
|
| 402 |
+
if direct == "owner" or is_admin:
|
| 403 |
+
return {"role": "owner", "direct": direct, "views": []}
|
| 404 |
+
view_ids = []
|
| 405 |
+
boundaries = data.get(VIEW_FIELD_BOUNDARIES_KEY) or {}
|
| 406 |
+
view_grants = data.get("view") or {}
|
| 407 |
+
if isinstance(boundaries, dict) and isinstance(view_grants, dict):
|
| 408 |
+
for view_id, boundary in boundaries.items():
|
| 409 |
+
if not isinstance(boundary, dict) or boundary.get("topic") != topic:
|
| 410 |
+
continue
|
| 411 |
+
if key not in {str(value) for value in (boundary.get("fields") or ())}:
|
| 412 |
+
continue
|
| 413 |
+
raw = view_grants.get(str(view_id))
|
| 414 |
+
if not isinstance(raw, dict):
|
| 415 |
+
continue
|
| 416 |
+
rec = {"owner": _uname(raw.get("owner")) or None,
|
| 417 |
+
"entries": shares._clean_entries(raw.get("entries"))}
|
| 418 |
+
if shares.role_from(rec, me, is_admin=False) is not None:
|
| 419 |
+
view_ids.append(str(view_id))
|
| 420 |
+
role = direct if direct in ("edit", "view") else ("view" if view_ids else None)
|
| 421 |
+
return {"role": role, "direct": direct, "views": sorted(view_ids)}
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def field_roles_for_keys(grant_topic, field_keys, viewer, *, is_admin=False, st=None):
|
| 425 |
+
"""Resolve many field roles from one share-document read."""
|
| 426 |
+
topic = str(grant_topic or "").strip()
|
| 427 |
+
me = _uname(viewer)
|
| 428 |
+
keys = []
|
| 429 |
+
seen = set()
|
| 430 |
+
for raw in field_keys or ():
|
| 431 |
+
key = str(raw or "").strip()
|
| 432 |
+
if key and key not in seen:
|
| 433 |
+
seen.add(key)
|
| 434 |
+
keys.append(key)
|
| 435 |
+
if not topic or not me:
|
| 436 |
+
return {key: {"role": None, "direct": None, "views": []} for key in keys}
|
| 437 |
+
try:
|
| 438 |
+
data = _st_for_boundaries(st).get(shares.SHARES_KEY) or {}
|
| 439 |
+
except Exception: # noqa: BLE001
|
| 440 |
+
data = {}
|
| 441 |
+
return {key: _field_role_sources_from_data(topic, key, me, is_admin, data)
|
| 442 |
+
for key in keys}
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def field_role_sources(grant_topic, field_key, viewer, *, is_admin=False, st=None):
|
| 446 |
+
"""Resolve one field role and its direct/View provenance.
|
| 447 |
+
|
| 448 |
+
A View role inherits READ access only. ``Can edit`` on a View means editing the View's
|
| 449 |
+
configuration, not taking ownership of or editing a colleague's field definition/value.
|
| 450 |
+
Direct field roles remain stronger and are returned unchanged.
|
| 451 |
+
"""
|
| 452 |
+
key = str(field_key or "").strip()
|
| 453 |
+
if not key:
|
| 454 |
+
return {"role": None, "direct": None, "views": []}
|
| 455 |
+
return field_roles_for_keys(
|
| 456 |
+
grant_topic, [key], viewer, is_admin=is_admin, st=st).get(
|
| 457 |
+
key, {"role": None, "direct": None, "views": []})
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
# Save the unwrapped direct resolver once, including across module reloads in verifier processes.
|
| 461 |
+
_DIRECT_FIELD_ROLE_FOR = getattr(shares.role_for, "_direct_field_role_for", shares.role_for)
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
def install_view_field_resolver():
|
| 465 |
+
"""Install C8 at the existing ``shares.role_for`` wall, once per process.
|
| 466 |
+
|
| 467 |
+
Lane V does not own ``core.shares`` or ``core.perm_scope``. The application imports its
|
| 468 |
+
share router at startup, and that router explicitly installs this owned resolver. Every
|
| 469 |
+
existing caller therefore keeps asking the same wall; only ``kind='field'`` gains the
|
| 470 |
+
View-boundary leg. The original callable remains attached for direct/provenance reads.
|
| 471 |
+
"""
|
| 472 |
+
if getattr(shares.role_for, "_view_field_resolver", False):
|
| 473 |
+
return shares.role_for
|
| 474 |
+
|
| 475 |
+
def _role_for(kind, oid, user, is_admin=False, st=None):
|
| 476 |
+
if kind != "field":
|
| 477 |
+
return _DIRECT_FIELD_ROLE_FOR(kind, oid, user, is_admin=is_admin, st=st)
|
| 478 |
+
topic, key = shares.split_field_oid(oid)
|
| 479 |
+
if not topic or not key:
|
| 480 |
+
return None
|
| 481 |
+
return field_role_sources(topic, key, user, is_admin=is_admin, st=st)["role"]
|
| 482 |
+
|
| 483 |
+
_role_for._view_field_resolver = True
|
| 484 |
+
_role_for._direct_field_role_for = _DIRECT_FIELD_ROLE_FOR
|
| 485 |
+
shares.role_for = _role_for
|
| 486 |
+
return _role_for
|
| 487 |
+
|
| 488 |
+
|
| 489 |
def permissions_from_grants(entries):
|
| 490 |
"""⭐⭐ W40-T03 / CONTRACT C4 — THE CLASSIFICATION IS DERIVED FROM THE REGISTRY, NEVER STAMPED.
|
| 491 |
|
platform/core/grid_events.py
CHANGED
|
@@ -343,6 +343,93 @@ def _field_share_keys(ctx):
|
|
| 343 |
return raw, raw
|
| 344 |
|
| 345 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 346 |
def _field_share_role(ctx, key, definition):
|
| 347 |
"""Return the specific Share field role, if this definition is governed by it.
|
| 348 |
|
|
@@ -1706,6 +1793,7 @@ def handle_one(event, ctx):
|
|
| 1706 |
)
|
| 1707 |
view['name'] = _unique_name(view.get('name'), _view_names)
|
| 1708 |
ws['views'][view_id] = view
|
|
|
|
| 1709 |
name_changed = view.get('name') != requested_name
|
| 1710 |
# PERSISTED, but deliberately NO rerun — see the return-value contract in
|
| 1711 |
# this function's docstring. This is the hot path: every filter, sort,
|
|
@@ -1770,6 +1858,8 @@ def handle_one(event, ctx):
|
|
| 1770 |
try:
|
| 1771 |
import core.shares as _shares
|
| 1772 |
_shares.drop_objects([('view', view_id)], st=getattr(_tops(ctx), 'st', None))
|
|
|
|
|
|
|
| 1773 |
except Exception:
|
| 1774 |
pass
|
| 1775 |
return True
|
|
@@ -1779,11 +1869,18 @@ def handle_one(event, ctx):
|
|
| 1779 |
try:
|
| 1780 |
import core.shares as _shares
|
| 1781 |
_shares.drop_objects([('view', view_id)], st=getattr(_tops(ctx), 'st', None))
|
|
|
|
|
|
|
| 1782 |
except Exception:
|
| 1783 |
pass
|
| 1784 |
else:
|
| 1785 |
_session_ready()
|
| 1786 |
ws['views'].pop(view_id, None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1787 |
# Wave-6 item 3: the client removed the view optimistically; a deleted view simply
|
| 1788 |
# stops being served, so the next render differs in nothing the client still shows.
|
| 1789 |
return False
|
|
|
|
| 343 |
return raw, raw
|
| 344 |
|
| 345 |
|
| 346 |
+
def _sync_view_field_inheritance(ctx, view):
|
| 347 |
+
"""Apply C8 after a validated View save and notify the field owner/tenant admins.
|
| 348 |
+
|
| 349 |
+
``ctx.fields`` is already the caller's permission-projected schema. A key the editor cannot
|
| 350 |
+
read therefore never reaches the saved config or this boundary. New personal overlay fields
|
| 351 |
+
are promoted without changing ``createdBy`` so the inherited READ role points at a definition
|
| 352 |
+
the audience can actually receive; database-owned ``ut_*`` fields already live tenant-wide.
|
| 353 |
+
"""
|
| 354 |
+
from core import field_permissions as _fp
|
| 355 |
+
|
| 356 |
+
tops = _tops(ctx)
|
| 357 |
+
st = getattr(tops, 'st', None) or ctx.st
|
| 358 |
+
shared_key, grant_topic = _field_share_keys(ctx)
|
| 359 |
+
delta = _fp.sync_view_field_boundary(
|
| 360 |
+
view, grant_topic, ctx.uname, st=st, audit_additions=True)
|
| 361 |
+
added = list(delta.get('added') or ())
|
| 362 |
+
audience = list(delta.get('audience') or ())
|
| 363 |
+
if not audience:
|
| 364 |
+
return delta
|
| 365 |
+
|
| 366 |
+
by_key = {str(field.get('key')): field for field in (ctx.fields or ())
|
| 367 |
+
if isinstance(field, dict) and field.get('key')}
|
| 368 |
+
table_key = str(getattr(tops, 'table_key', '') or '')
|
| 369 |
+
# Check every included key, not only this save's delta. If a transient overlay failure left
|
| 370 |
+
# an edge pointing at a still-personal definition, the audience remains fail-closed and the
|
| 371 |
+
# next View save retries residency instead of treating the boundary as finished forever.
|
| 372 |
+
# Successful promotions are idempotent because the projected schema then reports shared=True.
|
| 373 |
+
included = _fp.view_field_keys(view)
|
| 374 |
+
if not table_key.startswith('ut_'):
|
| 375 |
+
for key in included:
|
| 376 |
+
field = by_key.get(key)
|
| 377 |
+
if not isinstance(field, dict) or field.get('shared') is True:
|
| 378 |
+
continue
|
| 379 |
+
owner = str(field.get('createdBy') or '').strip().lower()
|
| 380 |
+
if not owner or owner != str(ctx.uname or '').strip().lower():
|
| 381 |
+
continue
|
| 382 |
+
try:
|
| 383 |
+
promoted = _fp.promote_field(
|
| 384 |
+
table_key, shared_key, grant_topic, owner, field, st=st)
|
| 385 |
+
by_key[key] = promoted
|
| 386 |
+
except Exception as exc: # noqa: BLE001
|
| 387 |
+
# Fail closed: the boundary alone cannot surface a definition that stayed
|
| 388 |
+
# personal. Report the infrastructure failure without putting field values in
|
| 389 |
+
# logs; every later live-audience save retries the idempotent promotion.
|
| 390 |
+
_tel.error('grid_events:view-field-promotion', exc)
|
| 391 |
+
|
| 392 |
+
if not added:
|
| 393 |
+
return delta
|
| 394 |
+
|
| 395 |
+
# Addition notifications are working-inbox facts, not permission authority. Send one small
|
| 396 |
+
# row per field/recipient and never let an inbox outage roll back an already accepted View.
|
| 397 |
+
try:
|
| 398 |
+
import core.alerts as _alerts
|
| 399 |
+
import core.perms as _perms
|
| 400 |
+
import core.shares as _shares
|
| 401 |
+
import core.users as _users
|
| 402 |
+
registry = _users.registry() or {}
|
| 403 |
+
tenant = str(getattr(st, 'key', '') or '').strip().lower()
|
| 404 |
+
admins = {
|
| 405 |
+
str(username).strip().lower()
|
| 406 |
+
for username, record in registry.items()
|
| 407 |
+
if isinstance(record, dict) and _perms.is_admin(record)
|
| 408 |
+
and (not tenant or str(record.get('tenant') or 'royal-imports').strip().lower() == tenant)
|
| 409 |
+
}
|
| 410 |
+
view_id = str((view or {}).get('id') or '')
|
| 411 |
+
view_name = str((view or {}).get('name') or 'this View')[:160]
|
| 412 |
+
actor = str(ctx.uname or '').strip().lower()
|
| 413 |
+
for key in added:
|
| 414 |
+
field = by_key.get(key) or {}
|
| 415 |
+
direct = _shares.grants(
|
| 416 |
+
'field', _shares.field_oid(grant_topic, key), st=st)
|
| 417 |
+
owner = str(direct.get('owner') or field.get('createdBy') or '').strip().lower()
|
| 418 |
+
recipients = set(admins)
|
| 419 |
+
if owner:
|
| 420 |
+
recipients.add(owner)
|
| 421 |
+
label = str(field.get('label') or key)[:160]
|
| 422 |
+
for recipient in sorted(recipients - {actor, ''}):
|
| 423 |
+
_alerts.notify(
|
| 424 |
+
recipient, 'Field added to a shared View', topic='share',
|
| 425 |
+
key=f'view-field:{view_id}:{key}', row_id=view_id,
|
| 426 |
+
detail=f'{label} was added to {view_name}. Its audience can now read it.',
|
| 427 |
+
actor=actor, st=st)
|
| 428 |
+
except Exception as exc: # noqa: BLE001
|
| 429 |
+
_tel.error('grid_events:view-field-notification', exc)
|
| 430 |
+
return delta
|
| 431 |
+
|
| 432 |
+
|
| 433 |
def _field_share_role(ctx, key, definition):
|
| 434 |
"""Return the specific Share field role, if this definition is governed by it.
|
| 435 |
|
|
|
|
| 1793 |
)
|
| 1794 |
view['name'] = _unique_name(view.get('name'), _view_names)
|
| 1795 |
ws['views'][view_id] = view
|
| 1796 |
+
_sync_view_field_inheritance(ctx, view)
|
| 1797 |
name_changed = view.get('name') != requested_name
|
| 1798 |
# PERSISTED, but deliberately NO rerun — see the return-value contract in
|
| 1799 |
# this function's docstring. This is the hot path: every filter, sort,
|
|
|
|
| 1858 |
try:
|
| 1859 |
import core.shares as _shares
|
| 1860 |
_shares.drop_objects([('view', view_id)], st=getattr(_tops(ctx), 'st', None))
|
| 1861 |
+
from core import field_permissions as _fp
|
| 1862 |
+
_fp.drop_view_field_boundary(view_id, st=getattr(_tops(ctx), 'st', None))
|
| 1863 |
except Exception:
|
| 1864 |
pass
|
| 1865 |
return True
|
|
|
|
| 1869 |
try:
|
| 1870 |
import core.shares as _shares
|
| 1871 |
_shares.drop_objects([('view', view_id)], st=getattr(_tops(ctx), 'st', None))
|
| 1872 |
+
from core import field_permissions as _fp
|
| 1873 |
+
_fp.drop_view_field_boundary(view_id, st=getattr(_tops(ctx), 'st', None))
|
| 1874 |
except Exception:
|
| 1875 |
pass
|
| 1876 |
else:
|
| 1877 |
_session_ready()
|
| 1878 |
ws['views'].pop(view_id, None)
|
| 1879 |
+
try:
|
| 1880 |
+
from core import field_permissions as _fp
|
| 1881 |
+
_fp.drop_view_field_boundary(view_id, st=ctx.st)
|
| 1882 |
+
except Exception:
|
| 1883 |
+
pass
|
| 1884 |
# Wave-6 item 3: the client removed the view optimistically; a deleted view simply
|
| 1885 |
# stops being served, so the next render differs in nothing the client still shows.
|
| 1886 |
return False
|
platform/core/registry.py
CHANGED
|
@@ -229,6 +229,53 @@ REGISTRY = [
|
|
| 229 |
BY_KEY = {m['key']: m for m in REGISTRY}
|
| 230 |
|
| 231 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
def brand_filterable(key) -> bool:
|
| 233 |
return bool(BY_KEY.get(key, {}).get('brand'))
|
| 234 |
|
|
|
|
| 229 |
BY_KEY = {m['key']: m for m in REGISTRY}
|
| 230 |
|
| 231 |
|
| 232 |
+
# Source policies - the single source of truth for connector database set-up.
|
| 233 |
+
#
|
| 234 |
+
# The API resolves eligibility from these rows and each source's registered analysis. Adding a
|
| 235 |
+
# source = add one row here + its provider and analysis in the connector's app layer. This is
|
| 236 |
+
# DATA ONLY: core knows neither the mirror nor the API, and imports neither to answer a policy.
|
| 237 |
+
#
|
| 238 |
+
# Fields:
|
| 239 |
+
# key stable source id (matches its registered analysis)
|
| 240 |
+
# label human name
|
| 241 |
+
# prefixes tuple of owned table-key prefixes; the longest matching prefix wins
|
| 242 |
+
# max_rows materialised-home ceiling, or None to use the store default; never an ingest cap
|
| 243 |
+
# window_max whole-table window ceiling, or None to use the datastore default
|
| 244 |
+
# rank_ops "client" or "server": where set-level rank operators are evaluated
|
| 245 |
+
# candidates "declared": only provider-returned grains; "prefix": every owned database
|
| 246 |
+
#
|
| 247 |
+
# None preserves the existing ceilings. Declared candidates preserve the existing grain set;
|
| 248 |
+
# a prefix identifies the policy owner without making an undeclared database read-through.
|
| 249 |
+
SOURCE_POLICIES = [
|
| 250 |
+
{'key': 'odoo', 'label': 'Odoo', 'prefixes': ('ut_odoo_',),
|
| 251 |
+
'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared'},
|
| 252 |
+
{'key': 'amazon', 'label': 'Amazon', 'prefixes': ('ut_amazon_',),
|
| 253 |
+
'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared'},
|
| 254 |
+
{'key': 'meta_ads', 'label': 'Meta Ads', 'prefixes': ('ut_meta_',),
|
| 255 |
+
'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared'},
|
| 256 |
+
]
|
| 257 |
+
DEFAULT_SOURCE_POLICY = {
|
| 258 |
+
'key': 'default', 'label': 'Default', 'prefixes': (),
|
| 259 |
+
'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared',
|
| 260 |
+
}
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def source_policy(table_key):
|
| 264 |
+
"""The source row owning this key, or DEFAULT_SOURCE_POLICY when no prefix matches."""
|
| 265 |
+
key = str(table_key or '')
|
| 266 |
+
match, longest = DEFAULT_SOURCE_POLICY, -1
|
| 267 |
+
for row in SOURCE_POLICIES:
|
| 268 |
+
for prefix in row['prefixes']:
|
| 269 |
+
if key.startswith(prefix) and len(prefix) > longest:
|
| 270 |
+
match, longest = row, len(prefix)
|
| 271 |
+
return match
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def source_policies():
|
| 275 |
+
"""The declared source rows, in declaration order; the fallback row is separate."""
|
| 276 |
+
return tuple(SOURCE_POLICIES)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
def brand_filterable(key) -> bool:
|
| 280 |
return bool(BY_KEY.get(key, {}).get('brand'))
|
| 281 |
|
platform/core/sheet_binding.py
CHANGED
|
@@ -1,18 +1,134 @@
|
|
| 1 |
-
"""
|
| 2 |
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
without importing the higher ``modules.sheet`` layer.
|
| 6 |
"""
|
| 7 |
|
| 8 |
from __future__ import annotations
|
| 9 |
|
| 10 |
import datetime
|
|
|
|
|
|
|
| 11 |
import re
|
|
|
|
| 12 |
|
| 13 |
from core import windows as W
|
| 14 |
|
| 15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
PERIOD_UNITS = ("day", "week", "month", "quarter", "year")
|
| 17 |
PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
|
| 18 |
PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$")
|
|
@@ -21,7 +137,9 @@ DRAG_ACROSS = ("period", "none")
|
|
| 21 |
DRAG_DOWN_NONE = "none"
|
| 22 |
|
| 23 |
_MESSAGES = {
|
|
|
|
| 24 |
"period_invalid": "{subject} is not a period this cell can use. Choose a valid start for the selected period unit.",
|
|
|
|
| 25 |
"ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.",
|
| 26 |
"ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year to date, this month or this quarter.",
|
| 27 |
"ending_cell_unreadable": "Ending from cell {subject} cannot be read. Use a cell containing a date in YYYY-MM-DD format.",
|
|
@@ -44,6 +162,58 @@ class BindingMetadataRefusal(Exception):
|
|
| 44 |
f"cannot be {verb}.", f"cannot be {verb} because {self.detail}")
|
| 45 |
|
| 46 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
def clean_period(raw):
|
| 48 |
"""Return one canonical Sheet period, or ``None`` when no window was named."""
|
| 49 |
if not isinstance(raw, dict):
|
|
@@ -135,7 +305,10 @@ def clean_period(raw):
|
|
| 135 |
out["ref"] = ref
|
| 136 |
if (raw.get("window") is not None and "unit" not in out and "start" not in out
|
| 137 |
and not out.get("frozen")):
|
| 138 |
-
|
|
|
|
|
|
|
|
|
|
| 139 |
if not ({"start", "window", "ref", "frozen"} & set(out)):
|
| 140 |
return None
|
| 141 |
return out
|
|
|
|
| 1 |
+
"""Binding metadata and server-owned Refresh snapshots shared by Sheet storage and routes.
|
| 2 |
|
| 3 |
+
Cleaners remain independent of query resolution. Change-token readers fingerprint source
|
| 4 |
+
state without assembling a database or evaluating an individual cell binding.
|
|
|
|
| 5 |
"""
|
| 6 |
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
import datetime
|
| 10 |
+
import hashlib
|
| 11 |
+
import json
|
| 12 |
import re
|
| 13 |
+
from copy import deepcopy
|
| 14 |
|
| 15 |
from core import windows as W
|
| 16 |
|
| 17 |
|
| 18 |
+
def fingerprint(value):
|
| 19 |
+
return hashlib.sha256(json.dumps(value, sort_keys=True, default=str,
|
| 20 |
+
separators=(",", ":")).encode()).hexdigest()
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def database_change_token(st, database):
|
| 24 |
+
"""Read a database fingerprint, without assembling rows or resolving a binding.
|
| 25 |
+
|
| 26 |
+
Authored databases share one JSON document. Its bucket revision cannot distinguish
|
| 27 |
+
siblings, so Postgres hashes the selected member IN the database and returns only a
|
| 28 |
+
token and row-free definition. This also sees connector writers which bypass the
|
| 29 |
+
ordinary row/field API. The HF adapter hashes the same documents locally.
|
| 30 |
+
"""
|
| 31 |
+
from core import connector_sync, schema_overlay, store_pg, user_tables
|
| 32 |
+
workspace = {"customer_data": "customer_table_workspace",
|
| 33 |
+
"product_data": "product_table_workspace"}.get(
|
| 34 |
+
database, database + "_table_workspace")
|
| 35 |
+
shared = database if database.startswith("ut_") else workspace
|
| 36 |
+
buckets = [workspace, shared + "__shared"]
|
| 37 |
+
handle = st._store() if callable(getattr(st, "_store", None)) else st
|
| 38 |
+
if isinstance(handle, store_pg.PgStore):
|
| 39 |
+
key = getattr(st, "store_key", str)
|
| 40 |
+
# Values stay on the server. No row JSON crosses the connection on this path.
|
| 41 |
+
with store_pg._pool().connection() as con:
|
| 42 |
+
row = con.execute(store_pg._q(
|
| 43 |
+
"WITH source AS (SELECT value -> %s AS d FROM {tbl} WHERE key = %s) "
|
| 44 |
+
"SELECT md5(COALESCE((SELECT d::text FROM source), 'null') || "
|
| 45 |
+
"COALESCE((SELECT jsonb_object_agg(key, rev ORDER BY key)::text "
|
| 46 |
+
"FROM {tbl} WHERE key = ANY(%s)), 'null') || "
|
| 47 |
+
"COALESCE((SELECT (value -> %s)::text FROM {tbl} WHERE key = %s), 'null') || "
|
| 48 |
+
"COALESCE((SELECT (value -> %s)::text FROM {tbl} WHERE key = %s), 'null') || "
|
| 49 |
+
"COALESCE((SELECT jsonb_object_agg(e.key, e.value ORDER BY e.key)::text "
|
| 50 |
+
"FROM {tbl} kv, jsonb_each(kv.value) e WHERE kv.key = %s "
|
| 51 |
+
"AND e.value ->> 'databaseKey' = %s), 'null') || "
|
| 52 |
+
"COALESCE((SELECT jsonb_path_query_array(value, "
|
| 53 |
+
"'$.*.* ? (@.databases[*] == $db)', jsonb_build_object('db', %s::text))::text "
|
| 54 |
+
"FROM {tbl} WHERE key = %s), 'null')), "
|
| 55 |
+
"(SELECT d - 'rows' FROM source)", "store_kv", handle.tenant_slug),
|
| 56 |
+
(database, key(user_tables.STORE_KEY), [key(b) for b in buckets],
|
| 57 |
+
database, key(user_tables.ZONE_STORE_KEY), database, key(schema_overlay.STORE_KEY),
|
| 58 |
+
key(connector_sync.JOBS_KEY), database,
|
| 59 |
+
database, key(connector_sync.STATE_KEY))).fetchone()
|
| 60 |
+
token, definition = row[0], row[1] or {}
|
| 61 |
+
else:
|
| 62 |
+
definition = (st.get(user_tables.STORE_KEY, fresh=True) or {}).get(database) or {}
|
| 63 |
+
jobs = st.get(connector_sync.JOBS_KEY, fresh=True) or {}
|
| 64 |
+
states = st.get(connector_sync.STATE_KEY, fresh=True) or {}
|
| 65 |
+
token = fingerprint([definition, *(st.get(b, fresh=True) for b in buckets),
|
| 66 |
+
(st.get(user_tables.ZONE_STORE_KEY, fresh=True) or {}).get(database),
|
| 67 |
+
(st.get(schema_overlay.STORE_KEY, fresh=True) or {}).get(database),
|
| 68 |
+
{k: v for k, v in jobs.items() if v.get("databaseKey") == database},
|
| 69 |
+
[v for provider in sorted(states) for _, v in sorted(states[provider].items())
|
| 70 |
+
if database in (v.get("databases") or [])]])
|
| 71 |
+
# Registry topics and read-through connectors live in the observation plane. Sync
|
| 72 |
+
# receipts have second precision, so also fingerprint source content. A file-wide
|
| 73 |
+
# mtime would invalidate every database when just one source changed.
|
| 74 |
+
mirror = not definition or not user_tables.materialises(database, defn=definition)
|
| 75 |
+
if mirror:
|
| 76 |
+
token = fingerprint([token, mirror_change_token(st, database)])
|
| 77 |
+
return token
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
#: ⛔⛔ THE MIRROR TOKEN IS PROVIDED FROM ABOVE, NOT IMPORTED FROM BELOW (SDLC 65 combine).
|
| 81 |
+
#: S65-T03 wrote `mirror_change_token` here and it reached for `harness.datastore` and
|
| 82 |
+
#: `harness.runtime` -- a NEW `core -> harness` edge, which `verify_layering.py` (D-460) caught as
|
| 83 |
+
#: the wave's own violation of ARCHITECTURE.md section 1. ⛔ It is FIXED rather than grandfathered:
|
| 84 |
+
#: `GRANDFATHERED_CORE` is the list of edges that PRE-DATE that gate, and adding this wave's own
|
| 85 |
+
#: edge to it would be granting an exemption to the wave that created it.
|
| 86 |
+
#: The body moved to `harness/mirror_tokens.py`, which registers itself here on import, so the
|
| 87 |
+
#: dependency points DOWN the way every other layer does.
|
| 88 |
+
_MIRROR_TOKEN_PROVIDER = None
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def set_mirror_token_provider(fn):
|
| 92 |
+
"""Register the mirror reader. Called by `harness.mirror_tokens` at import."""
|
| 93 |
+
global _MIRROR_TOKEN_PROVIDER
|
| 94 |
+
_MIRROR_TOKEN_PROVIDER = fn
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def mirror_change_token(st, database):
|
| 98 |
+
"""One SQL token read over source tables, delegated to the layer that owns the mirror.
|
| 99 |
+
|
| 100 |
+
⚠ NO PROVIDER IS A RAISE, DELIBERATELY, and it is the safe direction: `database_change_token`'s
|
| 101 |
+
contract is that a token which cannot be READ resolves rather than reusing a value, so an
|
| 102 |
+
unregistered provider costs a recompute and can never serve a stale number.
|
| 103 |
+
"""
|
| 104 |
+
if _MIRROR_TOKEN_PROVIDER is None:
|
| 105 |
+
raise RuntimeError("No mirror change-token provider is registered; import "
|
| 106 |
+
"harness.mirror_tokens before asking for one.")
|
| 107 |
+
return _MIRROR_TOKEN_PROVIDER(st, database)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def read_refresh_cache(st, workbook_id):
|
| 111 |
+
from core import sheets
|
| 112 |
+
return st.get(sheets._scope_cache_key(workbook_id), fresh=True) or {}
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def save_refresh_cache(st, workbook_id, entries, databases):
|
| 116 |
+
"""Server-only snapshots; ordinary workbook PUT/PATCH cannot manufacture these slots."""
|
| 117 |
+
from core import sheets
|
| 118 |
+
|
| 119 |
+
def apply(doc):
|
| 120 |
+
cells = doc.setdefault("refreshCells", {})
|
| 121 |
+
for cell_key, scope_key, entry in entries:
|
| 122 |
+
slots = cells.setdefault(cell_key, {})
|
| 123 |
+
if scope_key not in slots and len(slots) >= sheets._MAX_SCOPE_ENTRIES_PER_CELL:
|
| 124 |
+
slots.pop(next(iter(slots)))
|
| 125 |
+
slots[scope_key] = deepcopy(entry)
|
| 126 |
+
sheets._merge_data_origins(doc, databases=databases)
|
| 127 |
+
return doc, None
|
| 128 |
+
|
| 129 |
+
sheets._persist(st, sheets._scope_cache_key(workbook_id), apply)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
PERIOD_UNITS = ("day", "week", "month", "quarter", "year")
|
| 133 |
PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
|
| 134 |
PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$")
|
|
|
|
| 137 |
DRAG_DOWN_NONE = "none"
|
| 138 |
|
| 139 |
_MESSAGES = {
|
| 140 |
+
"related_invalid": "Choose one declared relationship and a valid filter tree for this cell.",
|
| 141 |
"period_invalid": "{subject} is not a period this cell can use. Choose a valid start for the selected period unit.",
|
| 142 |
+
"period_window_invalid": "This cell's lookback window is invalid. Choose a valid window in the period control.",
|
| 143 |
"ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.",
|
| 144 |
"ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year to date, this month or this quarter.",
|
| 145 |
"ending_cell_unreadable": "Ending from cell {subject} cannot be read. Use a cell containing a date in YYYY-MM-DD format.",
|
|
|
|
| 162 |
f"cannot be {verb}.", f"cannot be {verb} because {self.detail}")
|
| 163 |
|
| 164 |
|
| 165 |
+
def clean_related(raw):
|
| 166 |
+
"""One declared edge key and a structurally valid, losslessly copied filter tree.
|
| 167 |
+
|
| 168 |
+
Absence is handled by the caller. An explicitly supplied null is malformed, as are
|
| 169 |
+
extra relationship bags and nested relationships. Operator support belongs to the
|
| 170 |
+
query compiler, which reports unpushable conditions rather than dropping metadata.
|
| 171 |
+
"""
|
| 172 |
+
def refuse():
|
| 173 |
+
raise BindingMetadataRefusal("related_invalid")
|
| 174 |
+
|
| 175 |
+
if not isinstance(raw, dict) or set(raw) != {"via", "filter"}:
|
| 176 |
+
refuse()
|
| 177 |
+
via = raw["via"]
|
| 178 |
+
if not isinstance(via, str) or not via.strip():
|
| 179 |
+
refuse()
|
| 180 |
+
|
| 181 |
+
def reject_nested(value):
|
| 182 |
+
if isinstance(value, dict):
|
| 183 |
+
if "related" in value:
|
| 184 |
+
refuse()
|
| 185 |
+
for child in value.values():
|
| 186 |
+
reject_nested(child)
|
| 187 |
+
elif isinstance(value, list):
|
| 188 |
+
for child in value:
|
| 189 |
+
reject_nested(child)
|
| 190 |
+
|
| 191 |
+
reject_nested(raw["filter"])
|
| 192 |
+
|
| 193 |
+
def nodes_ok(nodes):
|
| 194 |
+
if not isinstance(nodes, list):
|
| 195 |
+
refuse()
|
| 196 |
+
for node in nodes:
|
| 197 |
+
if not isinstance(node, dict) or "related" in node:
|
| 198 |
+
refuse()
|
| 199 |
+
if "children" in node:
|
| 200 |
+
if ("colId" in node or node.get("conj", "and") not in ("and", "or")):
|
| 201 |
+
refuse()
|
| 202 |
+
nodes_ok(node["children"])
|
| 203 |
+
elif (not isinstance(node.get("colId"), str) or not node["colId"].strip()
|
| 204 |
+
or not isinstance(node.get("op"), str) or not node["op"].strip()):
|
| 205 |
+
refuse()
|
| 206 |
+
|
| 207 |
+
tree = raw["filter"]
|
| 208 |
+
if isinstance(tree, dict):
|
| 209 |
+
if set(tree) - {"nodes", "conj"} or tree.get("conj", "and") not in ("and", "or"):
|
| 210 |
+
refuse()
|
| 211 |
+
nodes_ok(tree.get("nodes"))
|
| 212 |
+
else:
|
| 213 |
+
nodes_ok(tree)
|
| 214 |
+
return {"via": via.strip(), "filter": deepcopy(tree)}
|
| 215 |
+
|
| 216 |
+
|
| 217 |
def clean_period(raw):
|
| 218 |
"""Return one canonical Sheet period, or ``None`` when no window was named."""
|
| 219 |
if not isinstance(raw, dict):
|
|
|
|
| 305 |
out["ref"] = ref
|
| 306 |
if (raw.get("window") is not None and "unit" not in out and "start" not in out
|
| 307 |
and not out.get("frozen")):
|
| 308 |
+
window = W.normalize(raw["window"])
|
| 309 |
+
if window is None:
|
| 310 |
+
raise BindingMetadataRefusal("period_window_invalid")
|
| 311 |
+
out["window"] = window
|
| 312 |
if not ({"start", "window", "ref", "frozen"} & set(out)):
|
| 313 |
return None
|
| 314 |
return out
|
platform/core/sheet_compat.py
ADDED
|
@@ -0,0 +1,875 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Excel-shaped automation compatibility for Loopable Sheet.
|
| 2 |
+
|
| 3 |
+
This module is the one capability declaration and the one operation planner used by both the
|
| 4 |
+
HTTP door and the Python SDK. It deliberately models the bounded workbook operations exercised
|
| 5 |
+
by the refresh-sales script and the refresh-fpa operating procedure. Excel process, window, VBA
|
| 6 |
+
and modal-UI controls are inventoried too, but always refuse before the tenant store callback.
|
| 7 |
+
|
| 8 |
+
Writes are operation-level compare-and-swap patches: every write needs the revision the caller
|
| 9 |
+
read, and only the addressed cells or worksheet/workbook property is changed. This is not a
|
| 10 |
+
second workbook persistence implementation; the final callback uses ``core.sheets``' store seam.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import base64
|
| 16 |
+
import copy
|
| 17 |
+
import json
|
| 18 |
+
import math
|
| 19 |
+
import os
|
| 20 |
+
import re
|
| 21 |
+
import tempfile
|
| 22 |
+
from core import write_execution
|
| 23 |
+
|
| 24 |
+
from core import sheets
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
MANIFEST_VERSION = "2026-09-19"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _cap(name, supported, signature, semantics, evidence, *, access="read", reason=""):
|
| 31 |
+
row = {
|
| 32 |
+
"name": name,
|
| 33 |
+
"supported": bool(supported),
|
| 34 |
+
"access": access if supported else "unsupported",
|
| 35 |
+
"signature": signature,
|
| 36 |
+
"semantics": semantics if supported else "Refuses before any workbook store update.",
|
| 37 |
+
"evidence": list(evidence),
|
| 38 |
+
}
|
| 39 |
+
if not supported:
|
| 40 |
+
row["unsupportedReason"] = reason
|
| 41 |
+
return row
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
_SALES = ".agents/skills/refresh-sales/scripts/refresh_daily_sales_performance.py"
|
| 45 |
+
_FPA = ".agents/skills/refresh-fpa/SKILL.md"
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# The evidence references are intentionally literal and machine-readable. Section 26 of the
|
| 49 |
+
# Sheet gate re-censuses the source operations and requires every token to land on one row here.
|
| 50 |
+
_OPERATIONS = (
|
| 51 |
+
_cap("workbooks.open", True, "Open(id_or_name, UpdateLinks=0, ReadOnly=False)",
|
| 52 |
+
"Looks up one existing workbook by stable id, then exact case-insensitive name.",
|
| 53 |
+
[f"{_SALES}:421", f"{_FPA}:153"]),
|
| 54 |
+
_cap("worksheets.item", True, "Worksheets(name_or_id)",
|
| 55 |
+
"Looks up one worksheet by id, exact name, then case-insensitive name.",
|
| 56 |
+
[f"{_SALES}:425-426"]),
|
| 57 |
+
_cap("worksheet.range", True, "Range(A1 | A1:B9 | Cell, Cell)",
|
| 58 |
+
"Builds one inclusive rectangular A1 range inside Excel worksheet limits.",
|
| 59 |
+
[f"{_SALES}:392", f"{_SALES}:621-624"]),
|
| 60 |
+
_cap("worksheet.cells", True, "Cells(row, column)",
|
| 61 |
+
"Addresses one one-based cell and composes with Range.",
|
| 62 |
+
[f"{_SALES}:438", f"{_SALES}:581"]),
|
| 63 |
+
_cap("range.values.read", True, "Range.Value | Range.Value2",
|
| 64 |
+
"Returns a scalar for one cell or a row-major matrix for a rectangle; formulas return cached values.",
|
| 65 |
+
[f"{_SALES}:466", f"{_SALES}:584"]),
|
| 66 |
+
_cap("range.values.write", True, "Range.Value = scalar | matrix",
|
| 67 |
+
"Writes only addressed cell contents, retaining each cell's style and unrelated metadata.",
|
| 68 |
+
[f"{_SALES}:438", f"{_SALES}:624"], access="write"),
|
| 69 |
+
_cap("range.formulas.read", True, "Range.Formula | Range.FormulaR1C1",
|
| 70 |
+
"Returns authored formulas; FormulaR1C1 returns the preserved R1C1 source when present.",
|
| 71 |
+
[f"{_SALES}:393"]),
|
| 72 |
+
_cap("range.formulas.write", True, "Range.Formula[ R1C1] = scalar | matrix",
|
| 73 |
+
"Writes formulas only in the rectangle; R1C1 references are translated at each destination.",
|
| 74 |
+
[f"{_SALES}:403", f"{_FPA}:177-179"], access="write"),
|
| 75 |
+
_cap("range.styles.read", True, "Range.Style",
|
| 76 |
+
"Returns each cell's canonical style bag without interpreting unrelated cell content.",
|
| 77 |
+
[f"{_SALES}:433-436", f"{_FPA}:112" ]),
|
| 78 |
+
_cap("range.styles.write", True, "Range.Style = style | matrix",
|
| 79 |
+
"Replaces only the addressed cells' canonical style bags.",
|
| 80 |
+
[f"{_SALES}:433-436", f"{_FPA}:112"], access="write"),
|
| 81 |
+
_cap("range.number_formats.read", True, "Range.NumberFormat",
|
| 82 |
+
"Returns Excel format-code strings from canonical cell styles.",
|
| 83 |
+
[f"{_SALES}:10", f"{_FPA}:177" ]),
|
| 84 |
+
_cap("range.number_formats.write", True, "Range.NumberFormat = code | matrix",
|
| 85 |
+
"Changes only style.format on addressed cells; General removes the explicit format.",
|
| 86 |
+
[f"{_SALES}:10", f"{_FPA}:177"], access="write"),
|
| 87 |
+
_cap("range.fill_down", True, "Range.FillDown()",
|
| 88 |
+
"Copies the top row down inside the rectangle and translates A1/R1C1 formulas per row.",
|
| 89 |
+
[f"{_SALES}:404", f"{_FPA}:177-179"], access="write"),
|
| 90 |
+
_cap("range.clear_contents", True, "Range.ClearContents()",
|
| 91 |
+
"Clears values and formulas in the rectangle while retaining styles and worksheet properties.",
|
| 92 |
+
[f"{_SALES}:433-436", f"{_FPA}:139-143"], access="write"),
|
| 93 |
+
_cap("range.replace", True, "Range.Replace(What, Replacement, LookAt=xlWhole, MatchCase=False)",
|
| 94 |
+
"Replaces exact stored literal values only; formulas and partial strings are never rewritten.",
|
| 95 |
+
[f"{_SALES}:672-680"], access="write"),
|
| 96 |
+
_cap("range.copy", True, "target.CopyFrom(source, mode='all'|'values'|'formulas')",
|
| 97 |
+
"Copies equal-sized rectangles and translates formulas; values mode hardcodes cached values.",
|
| 98 |
+
[f"{_FPA}:121-130", f"{_FPA}:177-179"], access="write"),
|
| 99 |
+
_cap("range.paste_values", True, "target.PasteSpecial(xlPasteValues, source=source)",
|
| 100 |
+
"Hardcodes cached source values into an equal-sized target without an ambient process clipboard.",
|
| 101 |
+
[f"{_FPA}:177-179", f"{_FPA}:194-199"], access="write"),
|
| 102 |
+
_cap("worksheet.used_range", True, "Worksheet.UsedRange",
|
| 103 |
+
"Returns the inclusive true used rectangle, or A1 for an empty sheet.",
|
| 104 |
+
[f"{_SALES}:466", f"{_SALES}:656"]),
|
| 105 |
+
_cap("range.end_up", True, "Cells(Rows.Count, column).End(xlUp)",
|
| 106 |
+
"Returns the last nonblank row in the selected column, or row 1 when empty.",
|
| 107 |
+
[f"{_SALES}:581", f"{_SALES}:617"]),
|
| 108 |
+
_cap("worksheet.count_a", True, "WorksheetFunction.CountA(range)",
|
| 109 |
+
"Counts nonblank stored values or formulas inside the rectangle.",
|
| 110 |
+
[f"{_SALES}:457-463", f"{_SALES}:630"]),
|
| 111 |
+
_cap("worksheet.count_if", True, "WorksheetFunction.CountIf(range, criterion)",
|
| 112 |
+
"Supports the workflow's exact-cell, case-insensitive criterion.",
|
| 113 |
+
[f"{_SALES}:656"]),
|
| 114 |
+
_cap("worksheet.geometry.read", True, "Range.ColumnWidth | Range.RowHeight",
|
| 115 |
+
"Reads canonical per-column widths and per-row heights with worksheet defaults.",
|
| 116 |
+
[f"{_SALES}:10", f"{_FPA}:121-130"]),
|
| 117 |
+
_cap("worksheet.geometry.write", True, "Range.ColumnWidth|RowHeight = number",
|
| 118 |
+
"Changes only columns or rows intersecting the addressed range.",
|
| 119 |
+
[f"{_SALES}:10", f"{_FPA}:121-130"], access="write"),
|
| 120 |
+
_cap("worksheet.hidden.read", True, "Range.EntireRow.Hidden | EntireColumn.Hidden",
|
| 121 |
+
"Reads hidden state for rows or columns intersecting the addressed range.",
|
| 122 |
+
[f"{_SALES}:10"]),
|
| 123 |
+
_cap("worksheet.hidden.write", True, "Range.EntireRow|EntireColumn.Hidden = bool",
|
| 124 |
+
"Changes hidden state only for rows or columns intersecting the addressed range.",
|
| 125 |
+
[f"{_SALES}:10"], access="write"),
|
| 126 |
+
_cap("worksheet.freeze.read", True, "Worksheet.FreezePanes",
|
| 127 |
+
"Returns canonical frozen row and column counts.",
|
| 128 |
+
[f"{_SALES}:932-935"]),
|
| 129 |
+
_cap("worksheet.freeze.write", True, "Worksheet.FreezePanes = {rows, cols}",
|
| 130 |
+
"Changes only the worksheet freeze counts.",
|
| 131 |
+
[f"{_SALES}:932-935"], access="write"),
|
| 132 |
+
_cap("worksheet.gridlines.read", True, "Worksheet.DisplayGridlines",
|
| 133 |
+
"Reads the worksheet view's gridline flag, defaulting to shown.",
|
| 134 |
+
[f"{_SALES}:932-935"]),
|
| 135 |
+
_cap("worksheet.gridlines.write", True, "Worksheet.DisplayGridlines = bool",
|
| 136 |
+
"Changes only the worksheet view's gridline flag.",
|
| 137 |
+
[f"{_SALES}:932-935"], access="write"),
|
| 138 |
+
_cap("application.calculation.read", True, "Application.Calculation",
|
| 139 |
+
"Reads manual or automatic mode from workbook calculation metadata.",
|
| 140 |
+
[f"{_SALES}:423", f"{_FPA}:146-153"]),
|
| 141 |
+
_cap("application.calculation.write", True, "Application.Calculation = manual|automatic",
|
| 142 |
+
"Changes only workbook calculation mode and requires expected revision.",
|
| 143 |
+
[f"{_SALES}:423", f"{_SALES}:444", f"{_FPA}:151-157"], access="write"),
|
| 144 |
+
_cap("workbook.calculate_full", True, "Application.CalculateFull()",
|
| 145 |
+
"Runs the host formula evaluator over the workbook and persists refreshed formula caches.",
|
| 146 |
+
[f"{_SALES}:447", f"{_FPA}:187", f"{_FPA}:216"], access="write"),
|
| 147 |
+
_cap("workbook.refresh_all", True, "Workbook.RefreshAll()",
|
| 148 |
+
"Runs the same bounded calculation callback and stamps refreshed_at.",
|
| 149 |
+
[f"{_SALES}:447", f"{_FPA}:233"], access="write"),
|
| 150 |
+
_cap("workbook.names.item", True, "Workbook.Names(name)",
|
| 151 |
+
"Reads one workbook named range without resolving external names.",
|
| 152 |
+
[f"{_SALES}:63-66", f"{_FPA}:177"]),
|
| 153 |
+
_cap("workbook.names.write", True, "Workbook.Names.Add(name, refers_to)",
|
| 154 |
+
"Upserts one workbook-scoped A1 named range and preserves every other name.",
|
| 155 |
+
[f"{_SALES}:63-66", f"{_FPA}:177"], access="write"),
|
| 156 |
+
_cap("workbook.export_xlsx", True, "Workbook.Export(path|bytes)",
|
| 157 |
+
"Exports a data-free xlsx using the existing lossless Sheet export seam; it never runs VBA.",
|
| 158 |
+
[f"{_SALES}:506-538", f"{_FPA}:233"]),
|
| 159 |
+
_cap("workbook.save", True, "Workbook.Save()",
|
| 160 |
+
"Returns the current revision because Loopable operation writes are already atomic and durable.",
|
| 161 |
+
[f"{_SALES}:469", f"{_FPA}:157", f"{_FPA}:233"]),
|
| 162 |
+
_cap("workbook.read_only", True, "Workbook.ReadOnly",
|
| 163 |
+
"Returns false after the route's editor wall; the facade never bypasses authorization.",
|
| 164 |
+
[f"{_FPA}:160"]),
|
| 165 |
+
|
| 166 |
+
_cap("application.dispatch_ex", False, "DispatchEx('Excel.Application')", "", [f"{_SALES}:412"],
|
| 167 |
+
reason="Loopable has no local Excel process; choose the backend at the client boundary."),
|
| 168 |
+
_cap("workbooks.add", False, "Workbooks.Add()", "", [f"{_FPA}:147-153"],
|
| 169 |
+
reason="An Excel scratch workbook exists only to bind process-global calculation mode."),
|
| 170 |
+
_cap("application.visible", False, "Application.Visible = bool", "", [f"{_SALES}:413", f"{_SALES}:935"],
|
| 171 |
+
reason="Browser visibility is not a workbook data operation."),
|
| 172 |
+
_cap("application.window_state", False, "Application.WindowState = value", "", [f"{_SALES}:936"],
|
| 173 |
+
reason="Window maximization is local UI orchestration."),
|
| 174 |
+
_cap("application.display_alerts", False, "Application.DisplayAlerts = bool", "", [f"{_SALES}:414", f"{_SALES}:937"],
|
| 175 |
+
reason="Loopable has no Excel modal-alert process state."),
|
| 176 |
+
_cap("application.screen_updating", False, "Application.ScreenUpdating = bool", "", [f"{_SALES}:415"],
|
| 177 |
+
reason="Loopable writes documents without an Excel repaint loop."),
|
| 178 |
+
_cap("application.enable_events", False, "Application.EnableEvents = bool", "", [f"{_SALES}:416", f"{_SALES}:938"],
|
| 179 |
+
reason="Excel process events are not Sheet document data."),
|
| 180 |
+
_cap("application.automation_security", False, "Application.AutomationSecurity = value", "", [f"{_SALES}:417", f"{_SALES}:940"],
|
| 181 |
+
reason="Loopable never executes workbook macros."),
|
| 182 |
+
_cap("workbook.autosave_on", False, "Workbook.AutoSaveOn = bool", "", [f"{_SALES}:341-351"],
|
| 183 |
+
reason="Loopable writes are atomic store operations, not OneDrive AutoSave sessions."),
|
| 184 |
+
_cap("workbook.vbproject", False, "Workbook.VBProject", "", [f"{_SALES}:898"],
|
| 185 |
+
reason="VBA projects are preserved by import metadata but never exposed for execution or editing."),
|
| 186 |
+
_cap("vba.code_module", False, "CodeModule.Lines|DeleteLines|AddFromString", "", [f"{_SALES}:904-918"],
|
| 187 |
+
reason="Editing executable VBA is outside the Sheet document capability boundary."),
|
| 188 |
+
_cap("application.run", False, "Application.Run(macro, *args)", "", [f"{_SALES}:963-965"],
|
| 189 |
+
reason="Loopable never runs VBA; use workbook.export_xlsx for the supported export."),
|
| 190 |
+
_cap("application.get_save_as_filename", False, "Application.GetSaveAsFilename(...) ", "", [f"{_SALES}:806-863"],
|
| 191 |
+
reason="A modal file picker is browser/UI orchestration."),
|
| 192 |
+
_cap("application.msg_box", False, "MsgBox(message)", "", [f"{_SALES}:865-885"],
|
| 193 |
+
reason="A modal VBA message box is not a workbook data operation."),
|
| 194 |
+
_cap("workbook.close", False, "Workbook.Close(SaveChanges=False)", "", [f"{_SALES}:477", f"{_FPA}:160"],
|
| 195 |
+
reason="Loopable workbooks are request-scoped resources, not local process handles."),
|
| 196 |
+
_cap("application.quit", False, "Application.Quit()", "", [f"{_SALES}:478"],
|
| 197 |
+
reason="Loopable has no Excel process to terminate."),
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
CAPABILITY_MANIFEST = {
|
| 202 |
+
"schema": 1,
|
| 203 |
+
"version": MANIFEST_VERSION,
|
| 204 |
+
"hierarchy": ["Application", "Workbooks", "Workbook", "Worksheets", "Worksheet", "Range", "Cells"],
|
| 205 |
+
"inventory": {
|
| 206 |
+
"method": "mechanical Python attribute-call census plus the refresh-fpa operating workflow evidence",
|
| 207 |
+
"sources": [_SALES, _FPA],
|
| 208 |
+
"supported": sum(1 for row in _OPERATIONS if row["supported"]),
|
| 209 |
+
"unsupported": sum(1 for row in _OPERATIONS if not row["supported"]),
|
| 210 |
+
},
|
| 211 |
+
"operations": list(_OPERATIONS),
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
CAPABILITY_MANIFEST_BYTES = json.dumps(
|
| 215 |
+
CAPABILITY_MANIFEST, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
| 216 |
+
).encode("utf-8")
|
| 217 |
+
|
| 218 |
+
_BY_NAME = {row["name"]: row for row in _OPERATIONS}
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
class CompatibilityRefusal(sheets.SheetRefusal):
|
| 222 |
+
"""A named compatibility refusal suitable for the existing Sheet HTTP envelope."""
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def capability_manifest():
|
| 226 |
+
return copy.deepcopy(CAPABILITY_MANIFEST)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def capability_manifest_bytes():
|
| 230 |
+
return CAPABILITY_MANIFEST_BYTES
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def operation_spec(name):
|
| 234 |
+
return copy.deepcopy(_BY_NAME.get(str(name or "")))
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def require_supported(name):
|
| 238 |
+
key = str(name or "").strip()
|
| 239 |
+
spec = _BY_NAME.get(key)
|
| 240 |
+
if spec is None:
|
| 241 |
+
raise CompatibilityRefusal(
|
| 242 |
+
"sheet_compat_operation_unknown", "That Sheet automation operation is not in the capability manifest.")
|
| 243 |
+
if not spec["supported"]:
|
| 244 |
+
raise CompatibilityRefusal(
|
| 245 |
+
"sheet_compat_operation_unsupported",
|
| 246 |
+
key + " is unsupported: " + spec["unsupportedReason"])
|
| 247 |
+
return spec
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
_CELL = re.compile(r"^\$?([A-Z]{1,3})\$?([1-9][0-9]{0,6})$")
|
| 251 |
+
_RECT = re.compile(r"^\$?([A-Z]{1,3})\$?([1-9][0-9]{0,6})(?::\$?([A-Z]{1,3})\$?([1-9][0-9]{0,6}))?$")
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def _col_number(letters):
|
| 255 |
+
value = 0
|
| 256 |
+
for letter in str(letters).upper():
|
| 257 |
+
value = value * 26 + ord(letter) - 64
|
| 258 |
+
return value
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def _letters(number):
|
| 262 |
+
out = ""
|
| 263 |
+
value = int(number)
|
| 264 |
+
while value:
|
| 265 |
+
value, rem = divmod(value - 1, 26)
|
| 266 |
+
out = chr(65 + rem) + out
|
| 267 |
+
return out
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
def _addr(row, col):
|
| 271 |
+
return _letters(col) + str(row)
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def _range(raw):
|
| 275 |
+
text = str(raw or "").strip().upper().replace(" ", "")
|
| 276 |
+
found = _RECT.fullmatch(text)
|
| 277 |
+
if not found:
|
| 278 |
+
raise CompatibilityRefusal("sheet_compat_range_invalid", "A Sheet automation range needs one rectangular A1 address.")
|
| 279 |
+
c0, r0 = _col_number(found.group(1)), int(found.group(2))
|
| 280 |
+
c1, r1 = _col_number(found.group(3) or found.group(1)), int(found.group(4) or found.group(2))
|
| 281 |
+
if c0 > sheets.DOCUMENT_MAX_COLS or c1 > sheets.DOCUMENT_MAX_COLS \
|
| 282 |
+
or r0 > sheets.DOCUMENT_MAX_ROWS or r1 > sheets.DOCUMENT_MAX_ROWS:
|
| 283 |
+
raise CompatibilityRefusal("sheet_compat_range_invalid", "That Sheet automation range is outside Excel's dimensions.")
|
| 284 |
+
c0, c1 = sorted((c0, c1))
|
| 285 |
+
r0, r1 = sorted((r0, r1))
|
| 286 |
+
if (c1 - c0 + 1) * (r1 - r0 + 1) > sheets._MAX_CELLS_PER_SHEET:
|
| 287 |
+
raise CompatibilityRefusal("sheet_too_many_cells", "That Sheet automation range is larger than one operation accepts.")
|
| 288 |
+
return r0, c0, r1, c1
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def _range_text(rect):
|
| 292 |
+
r0, c0, r1, c1 = rect
|
| 293 |
+
first, last = _addr(r0, c0), _addr(r1, c1)
|
| 294 |
+
return first if first == last else first + ":" + last
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def _addresses(rect):
|
| 298 |
+
r0, c0, r1, c1 = rect
|
| 299 |
+
return [[_addr(row, col) for col in range(c0, c1 + 1)] for row in range(r0, r1 + 1)]
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def _matrix(value, height, width, what, *, scalar=True):
|
| 303 |
+
if scalar and not isinstance(value, (list, tuple)):
|
| 304 |
+
return [[copy.deepcopy(value) for _ in range(width)] for _ in range(height)]
|
| 305 |
+
if not isinstance(value, (list, tuple)):
|
| 306 |
+
raise CompatibilityRefusal("sheet_compat_matrix_invalid", what + " needs a row-major matrix.")
|
| 307 |
+
rows = list(value)
|
| 308 |
+
if height == 1 and len(rows) == width and (not rows or not isinstance(rows[0], (list, tuple))):
|
| 309 |
+
rows = [rows]
|
| 310 |
+
if len(rows) != height or any(not isinstance(row, (list, tuple)) or len(row) != width for row in rows):
|
| 311 |
+
raise CompatibilityRefusal(
|
| 312 |
+
"sheet_compat_matrix_shape", what + " must have exactly " + str(height) + " rows and " + str(width) + " columns.")
|
| 313 |
+
return [[copy.deepcopy(item) for item in row] for row in rows]
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
def _tab(book, ref):
|
| 317 |
+
return sheets.find_tab(book, ref)
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def _cell_value(cell):
|
| 321 |
+
if not isinstance(cell, dict) or cell.get("kind") == "empty":
|
| 322 |
+
return None
|
| 323 |
+
return cell.get("value", cell.get("text"))
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def _display_matrix(tab, rect, getter):
|
| 327 |
+
grid = []
|
| 328 |
+
cells = tab.get("cells") or {}
|
| 329 |
+
for row in _addresses(rect):
|
| 330 |
+
grid.append([copy.deepcopy(getter(cells.get(addr))) for addr in row])
|
| 331 |
+
return grid[0][0] if len(grid) == len(grid[0]) == 1 else grid
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
def _keep(cell, keys=("style", "validation", "note", "excel", "link", "sparkline")):
|
| 335 |
+
return {key: copy.deepcopy(cell[key]) for key in keys if isinstance(cell, dict) and key in cell}
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
def _literal(value, prior):
|
| 339 |
+
if value is None:
|
| 340 |
+
held = _keep(prior)
|
| 341 |
+
return {"kind": "empty", "state": "typed", **held} if held else None
|
| 342 |
+
cell = {"kind": "literal", "state": "typed", "value": copy.deepcopy(value), **_keep(prior)}
|
| 343 |
+
if isinstance(value, str):
|
| 344 |
+
cell["text"] = value
|
| 345 |
+
return sheets.clean_cell(cell)
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
_R1C1 = re.compile(r"(?<![A-Z0-9_])R(?:(\[[-+]?\d+\])|(\d+))?C(?:(\[[-+]?\d+\])|(\d+))?", re.I)
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def _r1c1_to_a1(formula, row, col):
|
| 352 |
+
def replace(found):
|
| 353 |
+
rr = row + int(found.group(1)[1:-1]) if found.group(1) else int(found.group(2) or row)
|
| 354 |
+
cc = col + int(found.group(3)[1:-1]) if found.group(3) else int(found.group(4) or col)
|
| 355 |
+
if rr < 1 or cc < 1 or rr > sheets.DOCUMENT_MAX_ROWS or cc > sheets.DOCUMENT_MAX_COLS:
|
| 356 |
+
raise CompatibilityRefusal("sheet_compat_formula_reference", "That R1C1 formula points outside the worksheet.")
|
| 357 |
+
return _addr(rr, cc)
|
| 358 |
+
return _R1C1.sub(replace, str(formula or ""))
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def _formula(value, prior, addr, *, r1c1=False):
|
| 362 |
+
if value in (None, ""):
|
| 363 |
+
return _literal(None, prior)
|
| 364 |
+
text = str(value)
|
| 365 |
+
if not text.startswith("="):
|
| 366 |
+
text = "=" + text
|
| 367 |
+
found = _CELL.fullmatch(addr)
|
| 368 |
+
authored = _r1c1_to_a1(text, int(found.group(2)), _col_number(found.group(1))) if r1c1 else text
|
| 369 |
+
cell = {"kind": "formula", "state": "stale", "formula": authored, "value": None, **_keep(prior)}
|
| 370 |
+
if r1c1:
|
| 371 |
+
excel = dict(cell.get("excel") or {})
|
| 372 |
+
excel["formulaR1C1"] = text
|
| 373 |
+
cell["excel"] = excel
|
| 374 |
+
return sheets.clean_cell(cell, addr=addr)
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def _translate_formula(cell, origin, destination):
|
| 378 |
+
if not isinstance(cell, dict) or cell.get("kind") != "formula":
|
| 379 |
+
return copy.deepcopy(cell)
|
| 380 |
+
r1c1 = (cell.get("excel") or {}).get("formulaR1C1")
|
| 381 |
+
if isinstance(r1c1, str):
|
| 382 |
+
return _formula(r1c1, cell, destination, r1c1=True)
|
| 383 |
+
try:
|
| 384 |
+
from openpyxl.formula.translate import Translator
|
| 385 |
+
translated = Translator(str(cell.get("formula") or ""), origin=origin).translate_formula(destination)
|
| 386 |
+
except Exception:
|
| 387 |
+
translated = str(cell.get("formula") or "")
|
| 388 |
+
return _formula(translated, cell, destination)
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
def _used(tab):
|
| 392 |
+
rect = tab.get("usedExtent") or sheets._used_extent(
|
| 393 |
+
tab.get("cells") or {}, (tab.get("excel") or {}).get("blankStyles"))
|
| 394 |
+
if not rect:
|
| 395 |
+
return (1, 1, 1, 1)
|
| 396 |
+
return (int(rect["row0"]) + 1, int(rect["col0"]) + 1,
|
| 397 |
+
int(rect["row1"]) + 1, int(rect["col1"]) + 1)
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def _write_required(args):
|
| 401 |
+
revision = args.get("expected_revision")
|
| 402 |
+
if isinstance(revision, bool) or not isinstance(revision, int) or revision < 0:
|
| 403 |
+
raise CompatibilityRefusal(
|
| 404 |
+
"sheet_compat_revision_required", "A Sheet automation write requires the exact workbook revision it read.")
|
| 405 |
+
return revision
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
def _preflight(name, args):
|
| 409 |
+
spec = require_supported(name)
|
| 410 |
+
if not isinstance(args, dict):
|
| 411 |
+
raise CompatibilityRefusal("sheet_compat_arguments_invalid", "Sheet automation arguments must be an object.")
|
| 412 |
+
if spec["access"] == "write":
|
| 413 |
+
_write_required(args)
|
| 414 |
+
range_ops = {key for key in _BY_NAME if key.startswith("range.")} | {
|
| 415 |
+
"worksheet.count_a", "worksheet.count_if", "worksheet.geometry.read",
|
| 416 |
+
"worksheet.geometry.write", "worksheet.hidden.read", "worksheet.hidden.write",
|
| 417 |
+
}
|
| 418 |
+
if name.startswith("worksheet.") or name.startswith("range."):
|
| 419 |
+
if not str(args.get("sheet") or "").strip():
|
| 420 |
+
raise CompatibilityRefusal("sheet_compat_sheet_required", "That Sheet automation operation needs one worksheet.")
|
| 421 |
+
if name in range_ops:
|
| 422 |
+
_range(args.get("range"))
|
| 423 |
+
return spec
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
def _read(book, name, args):
|
| 427 |
+
if name == "workbooks.open":
|
| 428 |
+
return {"id": book.get("id"), "name": book.get("name"), "rev": book.get("rev")}
|
| 429 |
+
if name == "worksheets.item":
|
| 430 |
+
tab = _tab(book, args.get("sheet"))
|
| 431 |
+
return {key: copy.deepcopy(tab.get(key)) for key in ("id", "name", "cols", "rows")}
|
| 432 |
+
if name in ("worksheet.range", "worksheet.cells"):
|
| 433 |
+
return {"sheet": args.get("sheet"), "range": _range_text(_range(args.get("range")))}
|
| 434 |
+
if name.startswith("range.") or name.startswith("worksheet."):
|
| 435 |
+
tab = _tab(book, args.get("sheet"))
|
| 436 |
+
rect = _range(args.get("range")) if args.get("range") else None
|
| 437 |
+
if name == "range.values.read":
|
| 438 |
+
return _display_matrix(tab, rect, _cell_value)
|
| 439 |
+
if name == "range.formulas.read":
|
| 440 |
+
mode = str(args.get("mode") or "a1").lower()
|
| 441 |
+
return _display_matrix(tab, rect, lambda cell: (
|
| 442 |
+
((cell or {}).get("excel") or {}).get("formulaR1C1") if mode == "r1c1"
|
| 443 |
+
else (cell or {}).get("formula")))
|
| 444 |
+
if name == "range.styles.read":
|
| 445 |
+
return _display_matrix(tab, rect, lambda cell: (cell or {}).get("style"))
|
| 446 |
+
if name == "range.number_formats.read":
|
| 447 |
+
def number_format(cell):
|
| 448 |
+
fmt = ((cell or {}).get("style") or {}).get("format")
|
| 449 |
+
return _xlsx_number_format(fmt)
|
| 450 |
+
return _display_matrix(tab, rect, number_format)
|
| 451 |
+
if name == "worksheet.used_range":
|
| 452 |
+
return {"address": _range_text(_used(tab))}
|
| 453 |
+
if name == "range.end_up":
|
| 454 |
+
_r0, c0, _r1, _c1 = rect
|
| 455 |
+
occupied = [int(re.search(r"\d+$", addr).group()) for addr, cell in (tab.get("cells") or {}).items()
|
| 456 |
+
if _col_number(re.match(r"[A-Z]+", addr).group()) == c0 and _cell_value(cell) not in (None, "")]
|
| 457 |
+
row = max(occupied, default=1)
|
| 458 |
+
return {"address": _addr(row, c0), "row": row, "column": c0}
|
| 459 |
+
if name == "worksheet.count_a":
|
| 460 |
+
return sum(1 for row in _addresses(rect) for addr in row
|
| 461 |
+
if ((tab.get("cells") or {}).get(addr) or {}).get("kind") == "formula"
|
| 462 |
+
or _cell_value((tab.get("cells") or {}).get(addr)) not in (None, ""))
|
| 463 |
+
if name == "worksheet.count_if":
|
| 464 |
+
wanted = str(args.get("criterion") if args.get("criterion") is not None else "").casefold()
|
| 465 |
+
return sum(1 for row in _addresses(rect) for addr in row
|
| 466 |
+
if str(_cell_value((tab.get("cells") or {}).get(addr)) or "").casefold() == wanted)
|
| 467 |
+
if name == "worksheet.geometry.read":
|
| 468 |
+
r0, c0, r1, c1 = rect
|
| 469 |
+
return {"columns": {_letters(col): (tab.get("colWidths") or {}).get(_letters(col), sheets.DEFAULT_COL_WIDTH)
|
| 470 |
+
for col in range(c0, c1 + 1)},
|
| 471 |
+
"rows": {str(row): (tab.get("rowHeights") or {}).get(str(row), tab.get("rowHeight", 24))
|
| 472 |
+
for row in range(r0, r1 + 1)}}
|
| 473 |
+
if name == "worksheet.hidden.read":
|
| 474 |
+
r0, c0, r1, c1 = rect
|
| 475 |
+
hidden_rows, hidden_cols = set(tab.get("hiddenRows") or ()), set(tab.get("hiddenCols") or ())
|
| 476 |
+
return {"rows": {str(row): row in hidden_rows for row in range(r0, r1 + 1)},
|
| 477 |
+
"columns": {_letters(col): col in hidden_cols for col in range(c0, c1 + 1)}}
|
| 478 |
+
if name == "worksheet.freeze.read":
|
| 479 |
+
return copy.deepcopy(tab.get("freeze") or {"rows": 0, "cols": 0})
|
| 480 |
+
if name == "worksheet.gridlines.read":
|
| 481 |
+
return (tab.get("view") or {}).get("gridlines", True)
|
| 482 |
+
if name == "application.calculation.read":
|
| 483 |
+
return (book.get("calculation") or {}).get("mode", "automatic")
|
| 484 |
+
if name == "workbook.names.item":
|
| 485 |
+
wanted = str(args.get("name") or "").casefold()
|
| 486 |
+
rows = book.get("namedRanges") or []
|
| 487 |
+
if isinstance(rows, dict):
|
| 488 |
+
rows = [{"name": key, "ref": value} for key, value in rows.items()]
|
| 489 |
+
for row in rows:
|
| 490 |
+
if isinstance(row, dict) and str(row.get("name") or "").casefold() == wanted:
|
| 491 |
+
return copy.deepcopy(row)
|
| 492 |
+
raise CompatibilityRefusal("sheet_compat_name_missing", "That workbook named range does not exist.")
|
| 493 |
+
if name == "workbook.save":
|
| 494 |
+
return {"rev": int(book.get("rev") or 0), "saved": True}
|
| 495 |
+
if name == "workbook.read_only":
|
| 496 |
+
return False
|
| 497 |
+
if name == "workbook.export_xlsx":
|
| 498 |
+
from core import sheet_import
|
| 499 |
+
handle, path = tempfile.mkstemp(prefix="loopable-sheet-export-", suffix=".xlsx")
|
| 500 |
+
os.close(handle)
|
| 501 |
+
try:
|
| 502 |
+
sheet_import.export_imported_workbook(book, path)
|
| 503 |
+
_apply_canonical_export_styles(book, path, sheet_import)
|
| 504 |
+
with open(path, "rb") as source:
|
| 505 |
+
payload = base64.b64encode(source.read()).decode("ascii")
|
| 506 |
+
return {"contentType": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
| 507 |
+
"filename": str(args.get("filename") or (book.get("name") or "workbook") + ".xlsx"),
|
| 508 |
+
"base64": payload}
|
| 509 |
+
finally:
|
| 510 |
+
try:
|
| 511 |
+
os.remove(path)
|
| 512 |
+
except OSError:
|
| 513 |
+
pass
|
| 514 |
+
raise CompatibilityRefusal("sheet_compat_operation_unimplemented", name + " is declared but has no planner.")
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
def _xlsx_colour(value, palette):
|
| 518 |
+
"""One canonical colour token as openpyxl's opaque ARGB value."""
|
| 519 |
+
raw = str(palette.get(str(value or ""), value or "") or "").strip().lstrip("#")
|
| 520 |
+
if re.fullmatch(r"[0-9a-fA-F]{6}", raw):
|
| 521 |
+
return "FF" + raw.upper()
|
| 522 |
+
if re.fullmatch(r"[0-9a-fA-F]{8}", raw):
|
| 523 |
+
return raw.upper()
|
| 524 |
+
return None
|
| 525 |
+
|
| 526 |
+
|
| 527 |
+
def _xlsx_number_format(spec):
|
| 528 |
+
"""Render the closed canonical number-format vocabulary without guessing custom codes."""
|
| 529 |
+
spec = spec if isinstance(spec, dict) else {}
|
| 530 |
+
kind = str(spec.get("kind") or "general")
|
| 531 |
+
if kind == "code":
|
| 532 |
+
return str(spec.get("code") or "General")
|
| 533 |
+
if kind == "date":
|
| 534 |
+
return "yyyy-mm-dd"
|
| 535 |
+
decimals = max(0, int(spec.get("decimals") or 0))
|
| 536 |
+
places = ("." + ("0" * decimals)) if decimals else ""
|
| 537 |
+
grouping = "#,##0" if spec.get("thousands") else "0"
|
| 538 |
+
if kind == "number":
|
| 539 |
+
return grouping + places
|
| 540 |
+
if kind == "percent":
|
| 541 |
+
return "0" + places + "%"
|
| 542 |
+
if kind == "currency":
|
| 543 |
+
symbols = {"USD": "$", "EUR": "€", "GBP": "£", "JPY": "¥"}
|
| 544 |
+
symbol = symbols.get(str(spec.get("currency") or "USD").upper(),
|
| 545 |
+
str(spec.get("currency") or "USD").upper() + " ")
|
| 546 |
+
return symbol + grouping + places
|
| 547 |
+
return "General"
|
| 548 |
+
|
| 549 |
+
|
| 550 |
+
def _apply_canonical_export_styles(book, path, sheet_import):
|
| 551 |
+
"""Add canonical Sheet styles where the import exporter has no source style table.
|
| 552 |
+
|
| 553 |
+
``sheet_import.export_imported_workbook`` owns exact imported OOXML and must stay untouched.
|
| 554 |
+
A workbook authored through the compatibility facade has no ``stylesXml``, however, so that
|
| 555 |
+
seam intentionally emits defaults. In that one bounded case this overlay projects the
|
| 556 |
+
canonical style bags into the generated xlsx. Imported workbooks keep the lossless source
|
| 557 |
+
style path byte-for-byte and never pass through openpyxl a second time.
|
| 558 |
+
"""
|
| 559 |
+
document = sheets.migrate_workbook(book, lossless=True)
|
| 560 |
+
if ((document.get("excel") or {}).get("stylesXml") or ""):
|
| 561 |
+
return
|
| 562 |
+
|
| 563 |
+
import openpyxl
|
| 564 |
+
from openpyxl.styles import Side
|
| 565 |
+
|
| 566 |
+
output = openpyxl.load_workbook(path)
|
| 567 |
+
size_steps = {"s": 9, "m": sheets.DEFAULT_FONT_PT, "l": 12, "xl": 14}
|
| 568 |
+
for tab in document.get("sheets") or []:
|
| 569 |
+
ws = output[str(tab.get("name") or "Sheet")[:31]]
|
| 570 |
+
for address, source in (tab.get("cells") or {}).items():
|
| 571 |
+
style = (source or {}).get("style") if isinstance(source, dict) else None
|
| 572 |
+
if not isinstance(style, dict) or not style:
|
| 573 |
+
continue
|
| 574 |
+
target = ws[address]
|
| 575 |
+
|
| 576 |
+
font = copy.copy(target.font)
|
| 577 |
+
font.bold = bool(style.get("bold"))
|
| 578 |
+
font.italic = bool(style.get("italic"))
|
| 579 |
+
font.underline = "single" if style.get("underline") else None
|
| 580 |
+
if style.get("font"):
|
| 581 |
+
font.name = str(style["font"])
|
| 582 |
+
size = style.get("size")
|
| 583 |
+
if isinstance(size, str):
|
| 584 |
+
font.sz = size_steps.get(size, sheets.DEFAULT_FONT_PT)
|
| 585 |
+
elif not isinstance(size, bool) and isinstance(size, (int, float)):
|
| 586 |
+
font.sz = float(size)
|
| 587 |
+
colour = _xlsx_colour(style.get("color"), sheet_import.PALETTE)
|
| 588 |
+
if colour:
|
| 589 |
+
font.color = colour
|
| 590 |
+
target.font = font
|
| 591 |
+
|
| 592 |
+
fill = _xlsx_colour(style.get("fill"), sheet_import.PALETTE)
|
| 593 |
+
if fill:
|
| 594 |
+
painted = copy.copy(target.fill)
|
| 595 |
+
painted.fill_type = "solid"
|
| 596 |
+
painted.fgColor = fill
|
| 597 |
+
painted.patternType = "solid"
|
| 598 |
+
target.fill = painted
|
| 599 |
+
|
| 600 |
+
alignment = copy.copy(target.alignment)
|
| 601 |
+
if style.get("align"):
|
| 602 |
+
alignment.horizontal = style["align"]
|
| 603 |
+
if style.get("valign"):
|
| 604 |
+
alignment.vertical = {"middle": "center"}.get(style["valign"], style["valign"])
|
| 605 |
+
alignment.wrap_text = bool(style.get("wrap"))
|
| 606 |
+
alignment.indent = int(style.get("indent") or 0)
|
| 607 |
+
alignment.text_rotation = int(style.get("textRotation") or 0)
|
| 608 |
+
target.alignment = alignment
|
| 609 |
+
|
| 610 |
+
borders = style.get("border") or {}
|
| 611 |
+
if borders:
|
| 612 |
+
border = copy.copy(target.border)
|
| 613 |
+
line = Side(style="thin", color="FFCBD1DC")
|
| 614 |
+
for side in ("top", "right", "bottom", "left"):
|
| 615 |
+
if borders.get(side):
|
| 616 |
+
setattr(border, side, line)
|
| 617 |
+
target.border = border
|
| 618 |
+
if style.get("format"):
|
| 619 |
+
target.number_format = _xlsx_number_format(style["format"])
|
| 620 |
+
output.save(path)
|
| 621 |
+
output.close()
|
| 622 |
+
|
| 623 |
+
|
| 624 |
+
def _tab_property(tab, key, value):
|
| 625 |
+
candidate = copy.deepcopy(tab)
|
| 626 |
+
if value in (None, {}, []):
|
| 627 |
+
candidate.pop(key, None)
|
| 628 |
+
else:
|
| 629 |
+
candidate[key] = copy.deepcopy(value)
|
| 630 |
+
cleaned = sheets._clean_sheet(candidate, lossless=True)
|
| 631 |
+
if key in cleaned:
|
| 632 |
+
tab[key] = copy.deepcopy(cleaned[key])
|
| 633 |
+
else:
|
| 634 |
+
tab.pop(key, None)
|
| 635 |
+
|
| 636 |
+
|
| 637 |
+
def _mutate(book, name, args, calculator=None):
|
| 638 |
+
touched = {}
|
| 639 |
+
result = None
|
| 640 |
+
tab = _tab(book, args.get("sheet")) if args.get("sheet") else None
|
| 641 |
+
rect = _range(args.get("range")) if args.get("range") else None
|
| 642 |
+
cells = tab.setdefault("cells", {}) if tab is not None else None
|
| 643 |
+
|
| 644 |
+
def mark(address):
|
| 645 |
+
touched.setdefault(str(tab.get("id") or tab.get("name")), []).append(address)
|
| 646 |
+
|
| 647 |
+
if name in ("range.values.write", "range.formulas.write", "range.styles.write", "range.number_formats.write"):
|
| 648 |
+
addresses = _addresses(rect)
|
| 649 |
+
height, width = len(addresses), len(addresses[0])
|
| 650 |
+
values = _matrix(args.get("values"), height, width, name)
|
| 651 |
+
for row_at, row in enumerate(addresses):
|
| 652 |
+
for col_at, address in enumerate(row):
|
| 653 |
+
prior, value = cells.get(address), values[row_at][col_at]
|
| 654 |
+
if name == "range.values.write":
|
| 655 |
+
after = _literal(value, prior)
|
| 656 |
+
elif name == "range.formulas.write":
|
| 657 |
+
after = _formula(value, prior, address, r1c1=str(args.get("mode") or "").lower() == "r1c1")
|
| 658 |
+
elif name == "range.styles.write":
|
| 659 |
+
style = sheets._clean_style(value)
|
| 660 |
+
after = copy.deepcopy(prior) if isinstance(prior, dict) else {"kind": "empty", "state": "typed"}
|
| 661 |
+
if style:
|
| 662 |
+
after["style"] = style
|
| 663 |
+
else:
|
| 664 |
+
after.pop("style", None)
|
| 665 |
+
after = sheets.clean_cell(after, addr=address)
|
| 666 |
+
else:
|
| 667 |
+
fmt = None if value in (None, "", "General", "general") else {"kind": "code", "code": str(value)}
|
| 668 |
+
after = copy.deepcopy(prior) if isinstance(prior, dict) else {"kind": "empty", "state": "typed"}
|
| 669 |
+
style = dict(after.get("style") or {})
|
| 670 |
+
if fmt:
|
| 671 |
+
style["format"] = fmt
|
| 672 |
+
else:
|
| 673 |
+
style.pop("format", None)
|
| 674 |
+
if style:
|
| 675 |
+
after["style"] = style
|
| 676 |
+
else:
|
| 677 |
+
after.pop("style", None)
|
| 678 |
+
after = sheets.clean_cell(after, addr=address)
|
| 679 |
+
if after is None:
|
| 680 |
+
cells.pop(address, None)
|
| 681 |
+
else:
|
| 682 |
+
cells[address] = after
|
| 683 |
+
mark(address)
|
| 684 |
+
result = {"cells": height * width}
|
| 685 |
+
elif name == "range.clear_contents":
|
| 686 |
+
for row in _addresses(rect):
|
| 687 |
+
for address in row:
|
| 688 |
+
after = _literal(None, cells.get(address))
|
| 689 |
+
if after is None:
|
| 690 |
+
cells.pop(address, None)
|
| 691 |
+
else:
|
| 692 |
+
cells[address] = after
|
| 693 |
+
mark(address)
|
| 694 |
+
result = {"cleared": sum(len(row) for row in _addresses(rect))}
|
| 695 |
+
elif name == "range.fill_down":
|
| 696 |
+
addresses = _addresses(rect)
|
| 697 |
+
sources = {address: copy.deepcopy(cells.get(address)) for address in addresses[0]}
|
| 698 |
+
for row in addresses[1:]:
|
| 699 |
+
for column_at, address in enumerate(row):
|
| 700 |
+
origin = addresses[0][column_at]
|
| 701 |
+
after = _translate_formula(sources[origin], origin, address)
|
| 702 |
+
if after is None:
|
| 703 |
+
cells.pop(address, None)
|
| 704 |
+
else:
|
| 705 |
+
cells[address] = after
|
| 706 |
+
mark(address)
|
| 707 |
+
result = {"filled": max(0, len(addresses) - 1) * len(addresses[0])}
|
| 708 |
+
elif name == "range.replace":
|
| 709 |
+
if str(args.get("look_at") or "whole").lower() != "whole":
|
| 710 |
+
raise CompatibilityRefusal("sheet_compat_replace_mode", "Only exact whole-cell replacement is supported.")
|
| 711 |
+
wanted, replacement = args.get("what"), args.get("replacement")
|
| 712 |
+
match_case = bool(args.get("match_case"))
|
| 713 |
+
matched = 0
|
| 714 |
+
for row in _addresses(rect):
|
| 715 |
+
for address in row:
|
| 716 |
+
prior = cells.get(address)
|
| 717 |
+
if not isinstance(prior, dict) or prior.get("kind") == "formula":
|
| 718 |
+
continue
|
| 719 |
+
value = _cell_value(prior)
|
| 720 |
+
equal = value == wanted if match_case else str(value or "").casefold() == str(wanted or "").casefold()
|
| 721 |
+
if equal:
|
| 722 |
+
cells[address] = _literal(replacement, prior)
|
| 723 |
+
mark(address)
|
| 724 |
+
matched += 1
|
| 725 |
+
result = {"replaced": matched}
|
| 726 |
+
elif name in ("range.copy", "range.paste_values"):
|
| 727 |
+
source = _range(args.get("source_range"))
|
| 728 |
+
source_tab = _tab(book, args.get("source_sheet") or args.get("sheet"))
|
| 729 |
+
source_addresses, target_addresses = _addresses(source), _addresses(rect)
|
| 730 |
+
if (len(source_addresses), len(source_addresses[0])) != (len(target_addresses), len(target_addresses[0])):
|
| 731 |
+
raise CompatibilityRefusal("sheet_compat_copy_shape", "Source and target copy ranges must have the same shape.")
|
| 732 |
+
mode = "values" if name == "range.paste_values" else str(args.get("mode") or "all").lower()
|
| 733 |
+
if mode not in ("all", "values", "formulas"):
|
| 734 |
+
raise CompatibilityRefusal("sheet_compat_copy_mode", "Copy mode must be all, values or formulas.")
|
| 735 |
+
source_cells = copy.deepcopy(source_tab.get("cells") or {})
|
| 736 |
+
for rr, row in enumerate(target_addresses):
|
| 737 |
+
for cc, address in enumerate(row):
|
| 738 |
+
origin = source_addresses[rr][cc]
|
| 739 |
+
source_cell, prior = source_cells.get(origin), cells.get(address)
|
| 740 |
+
if mode == "values":
|
| 741 |
+
after = _literal(_cell_value(source_cell), prior)
|
| 742 |
+
elif mode == "formulas":
|
| 743 |
+
after = _translate_formula(source_cell, origin, address) if (source_cell or {}).get("kind") == "formula" else prior
|
| 744 |
+
else:
|
| 745 |
+
after = _translate_formula(source_cell, origin, address)
|
| 746 |
+
if after is None:
|
| 747 |
+
cells.pop(address, None)
|
| 748 |
+
else:
|
| 749 |
+
cells[address] = after
|
| 750 |
+
mark(address)
|
| 751 |
+
result = {"copied": sum(len(row) for row in target_addresses), "mode": mode}
|
| 752 |
+
elif name == "worksheet.geometry.write":
|
| 753 |
+
r0, c0, r1, c1 = rect
|
| 754 |
+
if "column_width" in args:
|
| 755 |
+
widths = dict(tab.get("colWidths") or {})
|
| 756 |
+
for col in range(c0, c1 + 1):
|
| 757 |
+
widths[_letters(col)] = args["column_width"]
|
| 758 |
+
_tab_property(tab, "colWidths", widths)
|
| 759 |
+
if "row_height" in args:
|
| 760 |
+
heights = dict(tab.get("rowHeights") or {})
|
| 761 |
+
for row in range(r0, r1 + 1):
|
| 762 |
+
heights[str(row)] = args["row_height"]
|
| 763 |
+
_tab_property(tab, "rowHeights", heights)
|
| 764 |
+
result = {"range": _range_text(rect)}
|
| 765 |
+
elif name == "worksheet.hidden.write":
|
| 766 |
+
r0, c0, r1, c1 = rect
|
| 767 |
+
axis = str(args.get("axis") or "rows").lower()
|
| 768 |
+
hidden = bool(args.get("hidden"))
|
| 769 |
+
if axis not in ("rows", "columns"):
|
| 770 |
+
raise CompatibilityRefusal("sheet_compat_hidden_axis", "Hidden state needs rows or columns.")
|
| 771 |
+
key, members = ("hiddenRows", range(r0, r1 + 1)) if axis == "rows" else ("hiddenCols", range(c0, c1 + 1))
|
| 772 |
+
values = set(tab.get(key) or ())
|
| 773 |
+
values = values.union(members) if hidden else values.difference(members)
|
| 774 |
+
_tab_property(tab, key, sorted(values))
|
| 775 |
+
result = {"axis": axis, "hidden": hidden}
|
| 776 |
+
elif name == "worksheet.freeze.write":
|
| 777 |
+
_tab_property(tab, "freeze", {"rows": args.get("rows", 0), "cols": args.get("cols", 0)})
|
| 778 |
+
result = copy.deepcopy(tab.get("freeze") or {"rows": 0, "cols": 0})
|
| 779 |
+
elif name == "worksheet.gridlines.write":
|
| 780 |
+
view = dict(tab.get("view") or {})
|
| 781 |
+
view["gridlines"] = bool(args.get("shown"))
|
| 782 |
+
_tab_property(tab, "view", view)
|
| 783 |
+
result = view["gridlines"]
|
| 784 |
+
elif name == "application.calculation.write":
|
| 785 |
+
mode = str(args.get("mode") or "").lower()
|
| 786 |
+
if mode not in ("manual", "automatic"):
|
| 787 |
+
raise CompatibilityRefusal("sheet_compat_calculation_mode", "Calculation mode must be manual or automatic.")
|
| 788 |
+
calculation = dict(book.get("calculation") or {})
|
| 789 |
+
calculation["mode"] = mode
|
| 790 |
+
book["calculation"] = calculation
|
| 791 |
+
result = mode
|
| 792 |
+
elif name in ("workbook.calculate_full", "workbook.refresh_all"):
|
| 793 |
+
if not callable(calculator):
|
| 794 |
+
raise CompatibilityRefusal("sheet_compat_calculator_required", "This Sheet host has no formula evaluator configured.")
|
| 795 |
+
calculator(book)
|
| 796 |
+
stamp = write_execution.utcnow().replace(microsecond=0).isoformat()
|
| 797 |
+
calculation = dict(book.get("calculation") or {})
|
| 798 |
+
calculation["lastCalculatedAt"] = stamp
|
| 799 |
+
book["calculation"] = calculation
|
| 800 |
+
if name == "workbook.refresh_all":
|
| 801 |
+
book["refreshed_at"] = stamp
|
| 802 |
+
result = {"calculatedAt": stamp}
|
| 803 |
+
elif name == "workbook.names.write":
|
| 804 |
+
named, ref = str(args.get("name") or "").strip(), str(args.get("refers_to") or "").strip()
|
| 805 |
+
if not named or not ref:
|
| 806 |
+
raise CompatibilityRefusal("sheet_compat_name_invalid", "A named range needs a name and refers_to address.")
|
| 807 |
+
rows = book.get("namedRanges") or []
|
| 808 |
+
if isinstance(rows, dict):
|
| 809 |
+
rows = [{"name": key, "ref": value} for key, value in rows.items()]
|
| 810 |
+
kept = [copy.deepcopy(row) for row in rows if isinstance(row, dict)
|
| 811 |
+
and str(row.get("name") or "").casefold() != named.casefold()]
|
| 812 |
+
kept.append({"name": named, "ref": ref})
|
| 813 |
+
book["namedRanges"] = kept
|
| 814 |
+
result = copy.deepcopy(kept[-1])
|
| 815 |
+
else:
|
| 816 |
+
raise CompatibilityRefusal("sheet_compat_operation_unimplemented", name + " is declared but has no patcher.")
|
| 817 |
+
|
| 818 |
+
if tab is not None:
|
| 819 |
+
tab["usedExtent"] = sheets._used_extent(cells or {}, (tab.get("excel") or {}).get("blankStyles"))
|
| 820 |
+
if tab["usedExtent"]:
|
| 821 |
+
tab["cols"] = max(int(tab.get("cols") or 1), int(tab["usedExtent"]["col1"]) + 1)
|
| 822 |
+
tab["rows"] = max(int(tab.get("rows") or 1), int(tab["usedExtent"]["row1"]) + 1)
|
| 823 |
+
return result, touched
|
| 824 |
+
|
| 825 |
+
|
| 826 |
+
def execute(st, workbook_id, operation, arguments=None, *, actor="", actor_name="", calculator=None):
|
| 827 |
+
"""Execute one declared operation through the shared planner/patcher.
|
| 828 |
+
|
| 829 |
+
Unsupported and malformed operation-level requests are checked before ``st.update`` is ever
|
| 830 |
+
reachable. A supported write validates its expected revision again under the store lock.
|
| 831 |
+
"""
|
| 832 |
+
if arguments is None:
|
| 833 |
+
args = {}
|
| 834 |
+
elif not isinstance(arguments, dict):
|
| 835 |
+
raise CompatibilityRefusal(
|
| 836 |
+
"sheet_compat_arguments_invalid", "Sheet automation arguments must be an object.")
|
| 837 |
+
else:
|
| 838 |
+
args = dict(arguments)
|
| 839 |
+
spec = _preflight(operation, args)
|
| 840 |
+
if spec["access"] != "write":
|
| 841 |
+
book = sheets.get_workbook(st, workbook_id)
|
| 842 |
+
return {"operation": operation, "rev": int(book.get("rev") or 0),
|
| 843 |
+
"result": _read(book, operation, args)}
|
| 844 |
+
|
| 845 |
+
expected = _write_required(args)
|
| 846 |
+
wid = sheets._slug_id(workbook_id)
|
| 847 |
+
saved, envelope = {}, {}
|
| 848 |
+
|
| 849 |
+
def apply(state):
|
| 850 |
+
if not isinstance(state, dict) or not state.get("id"):
|
| 851 |
+
raise sheets.SheetRefusal("workbook_missing", "That workbook no longer exists.")
|
| 852 |
+
current = int(state.get("rev") or 0)
|
| 853 |
+
if current != expected:
|
| 854 |
+
raise sheets.SheetStaleRefusal(
|
| 855 |
+
"workbook_revision_stale",
|
| 856 |
+
"That workbook changed somewhere else, so this automation operation was not applied.",
|
| 857 |
+
rev_current=current)
|
| 858 |
+
book = copy.deepcopy(state)
|
| 859 |
+
result, touched = _mutate(book, operation, args, calculator=calculator)
|
| 860 |
+
new_rev = current + 1
|
| 861 |
+
by_id = {str(tab.get("id")): tab for tab in book.get("sheets") or []}
|
| 862 |
+
for tab_id, addresses in touched.items():
|
| 863 |
+
if tab_id in by_id:
|
| 864 |
+
sheets._append_write_log(by_id[tab_id], new_rev, sorted(set(addresses)),
|
| 865 |
+
actor=actor, actor_name=actor_name)
|
| 866 |
+
book["rev"] = new_rev
|
| 867 |
+
book["updated_at"] = write_execution.utcnow().replace(microsecond=0).isoformat()
|
| 868 |
+
sheets._json_safe(book, "workbook")
|
| 869 |
+
saved.update(copy.deepcopy(book))
|
| 870 |
+
envelope.update({"operation": operation, "rev": new_rev, "result": copy.deepcopy(result)})
|
| 871 |
+
return book, envelope
|
| 872 |
+
|
| 873 |
+
out = sheets._persist(st, sheets.workbook_key(wid), apply)
|
| 874 |
+
sheets._index_upsert(st, saved)
|
| 875 |
+
return out
|
platform/core/sheet_events.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Value-only workbook commit notifications; the API supplies the Automation consumer.
|
| 2 |
+
|
| 3 |
+
No row or cell values cross this boundary. Publication happens after the synchronous store
|
| 4 |
+
commit, outside its lock. A new workbook/tab is initial backfill, not a watched change.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import logging
|
| 9 |
+
import re
|
| 10 |
+
|
| 11 |
+
COMMIT_OBSERVER = None
|
| 12 |
+
_ADDRESS = re.compile(r"^([A-Z]+)([1-9][0-9]*)$")
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _value(cell):
|
| 16 |
+
cell = cell if isinstance(cell, dict) else {}
|
| 17 |
+
# Paint, formula text, refresh stamps and provenance do not change the computed insight.
|
| 18 |
+
return cell.get("value"), cell.get("error"), cell.get("refusal")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _binding_keys(cell, address):
|
| 22 |
+
binding = (cell or {}).get("binding") or {}
|
| 23 |
+
if not binding:
|
| 24 |
+
return set()
|
| 25 |
+
keys = {address}
|
| 26 |
+
for key in ("id", "key", "name"):
|
| 27 |
+
if binding.get(key):
|
| 28 |
+
keys.add(str(binding[key]))
|
| 29 |
+
field = binding.get("field") or binding.get("aggregate")
|
| 30 |
+
if field:
|
| 31 |
+
keys.add(str(field))
|
| 32 |
+
if binding.get("database"):
|
| 33 |
+
keys.add(f'{binding["database"]}.{field}')
|
| 34 |
+
return keys
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _ranges(addresses):
|
| 38 |
+
"""Compress adjacent rows only; sparse gaps must never trigger an unrelated watch."""
|
| 39 |
+
columns = {}
|
| 40 |
+
for address in addresses:
|
| 41 |
+
match = _ADDRESS.fullmatch(address)
|
| 42 |
+
if match:
|
| 43 |
+
columns.setdefault(match[1], []).append(int(match[2]))
|
| 44 |
+
ranges = []
|
| 45 |
+
for column, rows in sorted(columns.items()):
|
| 46 |
+
start = end = None
|
| 47 |
+
for row in sorted(rows):
|
| 48 |
+
if start is not None and row == end + 1:
|
| 49 |
+
end = row
|
| 50 |
+
continue
|
| 51 |
+
if start is not None:
|
| 52 |
+
ranges.append(f"{column}{start}" if start == end else f"{column}{start}:{column}{end}")
|
| 53 |
+
start = end = row
|
| 54 |
+
if start is not None:
|
| 55 |
+
ranges.append(f"{column}{start}" if start == end else f"{column}{start}:{column}{end}")
|
| 56 |
+
return ranges
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def committed_changes(before, after):
|
| 60 |
+
"""Derive at most one event per existing tab from the transaction's actual before/after."""
|
| 61 |
+
if not isinstance(before, dict) or not isinstance(after, dict):
|
| 62 |
+
return []
|
| 63 |
+
workbook = str(after.get("id") or "")
|
| 64 |
+
if not workbook or before.get("id") != workbook:
|
| 65 |
+
return []
|
| 66 |
+
prior = {str(tab.get("id")): tab for tab in before.get("sheets") or []}
|
| 67 |
+
revision = int(after.get("rev") or 0)
|
| 68 |
+
events = []
|
| 69 |
+
for tab in after.get("sheets") or []:
|
| 70 |
+
sheet = str(tab.get("id") or "")
|
| 71 |
+
if sheet not in prior:
|
| 72 |
+
continue
|
| 73 |
+
old, new = prior[sheet].get("cells") or {}, tab.get("cells") or {}
|
| 74 |
+
changed, bindings = set(), set()
|
| 75 |
+
for address in old.keys() | new.keys():
|
| 76 |
+
left, right = old.get(address) or {}, new.get(address) or {}
|
| 77 |
+
value_changed = _value(left) != _value(right)
|
| 78 |
+
binding_changed = (left.get("binding") or {}) != (right.get("binding") or {})
|
| 79 |
+
if value_changed:
|
| 80 |
+
changed.add(address)
|
| 81 |
+
if value_changed or binding_changed:
|
| 82 |
+
bindings.update(_binding_keys(left, address) | _binding_keys(right, address))
|
| 83 |
+
if not changed and not bindings:
|
| 84 |
+
continue
|
| 85 |
+
events.append({
|
| 86 |
+
"kind": "sheet_insight_changed", "workbookId": workbook, "sheetId": sheet,
|
| 87 |
+
"batchId": f"{workbook}:{revision}:{sheet}", "revision": revision,
|
| 88 |
+
"committed": True, "changeKinds": (["value"] if changed else []) + (["binding"] if bindings else []),
|
| 89 |
+
"ranges": _ranges(changed), "bindings": sorted(bindings), "recordCount": len(changed),
|
| 90 |
+
})
|
| 91 |
+
return events
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def publish(st, events):
|
| 95 |
+
if events and callable(COMMIT_OBSERVER):
|
| 96 |
+
try:
|
| 97 |
+
COMMIT_OBSERVER(st, events)
|
| 98 |
+
except Exception:
|
| 99 |
+
# A consumer failure cannot turn a committed edit into a purported failed save.
|
| 100 |
+
# Never log cell contents, credentials, or the consumer's raw exception message.
|
| 101 |
+
logging.getLogger(__name__).error("Sheet commit notification failed after persistence")
|
platform/core/sheet_excel.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Windows Excel adapter and shared spreadsheet-backend protocols.
|
| 2 |
+
|
| 3 |
+
Importing this module is safe on Linux. The optional ``win32com`` dependency is loaded only when
|
| 4 |
+
``excel_application`` is selected at the client boundary. Excel itself already implements the
|
| 5 |
+
canonical object hierarchy, so the adapter returns the genuine COM Application rather than
|
| 6 |
+
maintaining a second imitation of it.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from typing import Any, Protocol, runtime_checkable
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@runtime_checkable
|
| 15 |
+
class OperationBackend(Protocol):
|
| 16 |
+
"""Transport used by the Loopable facade."""
|
| 17 |
+
|
| 18 |
+
def manifest_bytes(self) -> bytes: ...
|
| 19 |
+
|
| 20 |
+
def open_workbook(self, identifier: str) -> dict[str, Any]: ...
|
| 21 |
+
|
| 22 |
+
def execute(self, workbook_id: str, operation: str, arguments: dict[str, Any]) -> dict[str, Any]: ...
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@runtime_checkable
|
| 26 |
+
class SpreadsheetApplication(Protocol):
|
| 27 |
+
"""The common boundary accepted by data-only workbook workflow functions."""
|
| 28 |
+
|
| 29 |
+
Workbooks: Any
|
| 30 |
+
WorksheetFunction: Any
|
| 31 |
+
|
| 32 |
+
def CalculateFull(self) -> Any: ...
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def excel_application(*, visible: bool | None = None, dispatch_ex: bool = True):
|
| 36 |
+
"""Return a genuine Excel COM Application, loading pywin32 only on Windows.
|
| 37 |
+
|
| 38 |
+
Process/UI settings remain the Excel caller's responsibility. They are not projected onto
|
| 39 |
+
Loopable, where the public capability manifest names them as refusal-only operations.
|
| 40 |
+
"""
|
| 41 |
+
try:
|
| 42 |
+
import win32com.client as win32
|
| 43 |
+
except (ImportError, ModuleNotFoundError) as exc: # pragma: no cover - Windows-only path
|
| 44 |
+
raise RuntimeError(
|
| 45 |
+
"The Excel backend needs Windows, desktop Excel and pywin32. Choose backend='loopable' on this host."
|
| 46 |
+
) from exc
|
| 47 |
+
application = (win32.DispatchEx if dispatch_ex else win32.Dispatch)("Excel.Application")
|
| 48 |
+
if visible is not None:
|
| 49 |
+
application.Visible = bool(visible)
|
| 50 |
+
return application
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def select_application(backend: str, **options):
|
| 54 |
+
"""Select Excel or Loopable once, outside business workflow logic."""
|
| 55 |
+
kind = str(backend or "").strip().lower()
|
| 56 |
+
if kind == "excel":
|
| 57 |
+
return excel_application(
|
| 58 |
+
visible=options.get("visible"), dispatch_ex=options.get("dispatch_ex", True))
|
| 59 |
+
if kind == "loopable":
|
| 60 |
+
from sdk.loopable_sheet import Application
|
| 61 |
+
transport = options.get("transport")
|
| 62 |
+
if transport is None:
|
| 63 |
+
raise ValueError("The Loopable backend needs an operation transport.")
|
| 64 |
+
return Application(transport)
|
| 65 |
+
raise ValueError("Spreadsheet backend must be 'excel' or 'loopable'.")
|
platform/core/sheet_import.py
CHANGED
|
@@ -44,6 +44,9 @@ import re
|
|
| 44 |
import zipfile
|
| 45 |
import hashlib
|
| 46 |
import json
|
|
|
|
|
|
|
|
|
|
| 47 |
import xml.etree.ElementTree as ET
|
| 48 |
from datetime import date, datetime, time, timezone
|
| 49 |
|
|
@@ -55,6 +58,15 @@ from core import sheets
|
|
| 55 |
#: parse in reasonable time.
|
| 56 |
MAX_IMPORT_BYTES = 25 * 1024 * 1024
|
| 57 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
#: C4's job states, in the order a job walks them. `refused` is the exit that replaces `done`.
|
| 59 |
IMPORT_STATES = ("reading", "cells", "formats", "recalc", "done", "refused")
|
| 60 |
|
|
@@ -126,10 +138,6 @@ PALETTE = {
|
|
| 126 |
DEFAULT_INK = "--lp-ink"
|
| 127 |
DEFAULT_FILL = "--lp-surface"
|
| 128 |
|
| 129 |
-
#: The four font steps A1 froze, in points. A size is a STEP and never a pixel count, for
|
| 130 |
-
#: `_clean_style`'s reason: a stored pixel size is a layout decision that outlives the layout.
|
| 131 |
-
_SIZE_STEPS = ((9.5, "s"), (12.5, "m"), (16.5, "l"))
|
| 132 |
-
|
| 133 |
#: Excel's horizontal alignments that have a home in `CELL_ALIGNS`. Everything else is named in
|
| 134 |
#: `unsupported` rather than guessed at: `justify` and `distributed` are not `left`.
|
| 135 |
_ALIGNS = {"left": "left", "center": "center", "centre": "center", "right": "right"}
|
|
@@ -314,22 +322,13 @@ def classify_format(code):
|
|
| 314 |
|
| 315 |
# ── one cell's style ─────────────────────────────────────────────────────────────────────────
|
| 316 |
|
| 317 |
-
def _size_step(points):
|
| 318 |
-
if not isinstance(points, (int, float)):
|
| 319 |
-
return ""
|
| 320 |
-
for limit, step in _SIZE_STEPS:
|
| 321 |
-
if points < limit:
|
| 322 |
-
return step
|
| 323 |
-
return "xl"
|
| 324 |
-
|
| 325 |
-
|
| 326 |
def style_of(cell, notes):
|
| 327 |
"""The C2 style for one openpyxl cell, or `{}` when nothing about it is worth storing.
|
| 328 |
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
"""
|
| 334 |
out = {}
|
| 335 |
font = getattr(cell, "font", None)
|
|
@@ -340,9 +339,17 @@ def style_of(cell, notes):
|
|
| 340 |
out["italic"] = True
|
| 341 |
if font.u:
|
| 342 |
out["underline"] = True
|
| 343 |
-
|
| 344 |
-
if
|
| 345 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 346 |
argb = argb_of(font.color)
|
| 347 |
if argb:
|
| 348 |
token = quantize(argb)
|
|
@@ -902,6 +909,134 @@ def _geometry_of(ws, notes):
|
|
| 902 |
return cols, rows, widths, freeze, sorted(set(hidden_rows)), sorted(set(hidden_cols)), heights
|
| 903 |
|
| 904 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 905 |
def _print_of(ws):
|
| 906 |
"""Store print setup and inclusive print areas; export executes these settings later."""
|
| 907 |
setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
|
|
@@ -977,8 +1112,6 @@ def import_workbook(path, *, source="", progress=None):
|
|
| 977 |
⚠ `progress(state, fraction)` IS CALLED FROM WHATEVER THREAD RUNS THIS. It must not raise and
|
| 978 |
must not block; the job's registry entry is the only thing it touches.
|
| 979 |
"""
|
| 980 |
-
import openpyxl
|
| 981 |
-
|
| 982 |
def _say(state, fraction):
|
| 983 |
if progress:
|
| 984 |
try:
|
|
@@ -987,6 +1120,11 @@ def import_workbook(path, *, source="", progress=None):
|
|
| 987 |
pass
|
| 988 |
|
| 989 |
_say("reading", 0.02)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 990 |
try:
|
| 991 |
formulas = openpyxl.load_workbook(path, data_only=False, read_only=False,
|
| 992 |
keep_links=False)
|
|
@@ -1088,6 +1226,9 @@ def import_workbook(path, *, source="", progress=None):
|
|
| 1088 |
tab["hiddenRows"] = hidden_rows
|
| 1089 |
if hidden_cols:
|
| 1090 |
tab["hiddenCols"] = hidden_cols
|
|
|
|
|
|
|
|
|
|
| 1091 |
charts = _charts_of(ws, name, notes)
|
| 1092 |
if name in chart_tabs:
|
| 1093 |
tab["id"] = chart_tabs[name]["id"]
|
|
@@ -1103,11 +1244,6 @@ def import_workbook(path, *, source="", progress=None):
|
|
| 1103 |
values.close()
|
| 1104 |
except Exception: # noqa: BLE001
|
| 1105 |
pass
|
| 1106 |
-
#: ⭐ THE FONT NOTE IS WORKBOOK-WIDE AND UNCONDITIONAL, because the answer is the same on
|
| 1107 |
-
#: every file: the app is one typeface, so no imported font survives and the reader is told
|
| 1108 |
-
#: once rather than per tab.
|
| 1109 |
-
book_notes.add("every font was replaced for display; original fonts and formats are kept for export")
|
| 1110 |
-
|
| 1111 |
# Original styles are shared once, with integer references on cells, so exact formats,
|
| 1112 |
# theme colours, border weights and blank-cell styles survive without repeating XML per cell.
|
| 1113 |
original = {}
|
|
@@ -1129,6 +1265,132 @@ def import_workbook(path, *, source="", progress=None):
|
|
| 1129 |
}
|
| 1130 |
|
| 1131 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1132 |
def take_imported_values(book):
|
| 1133 |
"""Pull every `imported_value` off the document and return `{(tab, addr): cached}`.
|
| 1134 |
|
|
|
|
| 44 |
import zipfile
|
| 45 |
import hashlib
|
| 46 |
import json
|
| 47 |
+
import math
|
| 48 |
+
import os
|
| 49 |
+
import tempfile
|
| 50 |
import xml.etree.ElementTree as ET
|
| 51 |
from datetime import date, datetime, time, timezone
|
| 52 |
|
|
|
|
| 58 |
#: parse in reasonable time.
|
| 59 |
MAX_IMPORT_BYTES = 25 * 1024 * 1024
|
| 60 |
|
| 61 |
+
#: An xlsx is a ZIP and the compressed upload limit is therefore not a parse budget. These
|
| 62 |
+
#: ceilings are checked from the central directory before openpyxl is imported or called. They
|
| 63 |
+
#: deliberately leave more room than the 64 MiB stored document: worksheet XML repeats tags and
|
| 64 |
+
#: shared strings before the importer turns them into the compact JSON document.
|
| 65 |
+
MAX_ARCHIVE_UNCOMPRESSED_BYTES = 256 * 1024 * 1024
|
| 66 |
+
MAX_ARCHIVE_MEMBER_BYTES = 128 * 1024 * 1024
|
| 67 |
+
MAX_ARCHIVE_MEMBERS = 10_000
|
| 68 |
+
MAX_ARCHIVE_COMPRESSION_RATIO = 250
|
| 69 |
+
|
| 70 |
#: C4's job states, in the order a job walks them. `refused` is the exit that replaces `done`.
|
| 71 |
IMPORT_STATES = ("reading", "cells", "formats", "recalc", "done", "refused")
|
| 72 |
|
|
|
|
| 138 |
DEFAULT_INK = "--lp-ink"
|
| 139 |
DEFAULT_FILL = "--lp-surface"
|
| 140 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
#: Excel's horizontal alignments that have a home in `CELL_ALIGNS`. Everything else is named in
|
| 142 |
#: `unsupported` rather than guessed at: `justify` and `distributed` are not `left`.
|
| 143 |
_ALIGNS = {"left": "left", "center": "center", "centre": "center", "right": "right"}
|
|
|
|
| 322 |
|
| 323 |
# ── one cell's style ─────────────────────────────────────────────────────────────────────────
|
| 324 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
def style_of(cell, notes):
|
| 326 |
"""The C2 style for one openpyxl cell, or `{}` when nothing about it is worth storing.
|
| 327 |
|
| 328 |
+
Imported workbook styling is cell data, not product chrome. A supported font family and the
|
| 329 |
+
authored point size therefore enter the normal display model. Anything that model cannot
|
| 330 |
+
express still survives exactly through the cell's style id plus the workbook's original
|
| 331 |
+
styles XML; the import report names the display fallback instead of pretending it was exact.
|
| 332 |
"""
|
| 333 |
out = {}
|
| 334 |
font = getattr(cell, "font", None)
|
|
|
|
| 339 |
out["italic"] = True
|
| 340 |
if font.u:
|
| 341 |
out["underline"] = True
|
| 342 |
+
size = getattr(font, "sz", None)
|
| 343 |
+
if not isinstance(size, bool) and isinstance(size, (int, float)) and math.isfinite(size):
|
| 344 |
+
points = max(sheets.MIN_FONT_PT, min(sheets.MAX_FONT_PT, float(size)))
|
| 345 |
+
if points != sheets.DEFAULT_FONT_PT:
|
| 346 |
+
out["size"] = points
|
| 347 |
+
name = str(getattr(font, "name", None) or "").strip()
|
| 348 |
+
if name in sheets.CELL_FONTS:
|
| 349 |
+
if name != sheets.DEFAULT_CELL_FONT:
|
| 350 |
+
out["font"] = name
|
| 351 |
+
elif name:
|
| 352 |
+
notes.add("the " + name[:60] + " font is kept for export but uses the default face on screen")
|
| 353 |
argb = argb_of(font.color)
|
| 354 |
if argb:
|
| 355 |
token = quantize(argb)
|
|
|
|
| 909 |
return cols, rows, widths, freeze, sorted(set(hidden_rows)), sorted(set(hidden_cols)), heights
|
| 910 |
|
| 911 |
|
| 912 |
+
def _view_of(ws):
|
| 913 |
+
"""Explicit worksheet display state in the existing tab ``view`` vocabulary.
|
| 914 |
+
|
| 915 |
+
``None`` means the source omitted the property and is intentionally different from an
|
| 916 |
+
explicit ``True`` or 100 percent. That distinction is what lets compact product defaults
|
| 917 |
+
apply only when the file said nothing while an authored value always wins.
|
| 918 |
+
"""
|
| 919 |
+
source = getattr(ws, "sheet_view", None)
|
| 920 |
+
if source is None:
|
| 921 |
+
return {}
|
| 922 |
+
out = {}
|
| 923 |
+
if isinstance(getattr(source, "showGridLines", None), bool):
|
| 924 |
+
out["gridlines"] = source.showGridLines
|
| 925 |
+
if isinstance(getattr(source, "showRowColHeaders", None), bool):
|
| 926 |
+
out["headings"] = source.showRowColHeaders
|
| 927 |
+
zoom = getattr(source, "zoomScale", None)
|
| 928 |
+
if not isinstance(zoom, bool) and isinstance(zoom, (int, float)) and math.isfinite(zoom):
|
| 929 |
+
out["zoom"] = zoom
|
| 930 |
+
page_view = {"normal": "normal", "pageBreakPreview": "pageBreak",
|
| 931 |
+
"pageLayout": "pageLayout"}.get(getattr(source, "view", None))
|
| 932 |
+
if page_view:
|
| 933 |
+
out["pageView"] = page_view
|
| 934 |
+
return out
|
| 935 |
+
|
| 936 |
+
|
| 937 |
+
def _archive_refusal(code, message):
|
| 938 |
+
raise ImportRefusal(code, message)
|
| 939 |
+
|
| 940 |
+
|
| 941 |
+
def _address_in_document(ref):
|
| 942 |
+
point = _corner(ref)
|
| 943 |
+
return (point is not None and 0 <= point[0] < sheets.DOCUMENT_MAX_COLS
|
| 944 |
+
and 0 <= point[1] < sheets.DOCUMENT_MAX_ROWS)
|
| 945 |
+
|
| 946 |
+
|
| 947 |
+
def _guard_worksheet_xml(stream):
|
| 948 |
+
"""Stream one worksheet and reject impossible dimensions/cell counts before openpyxl.
|
| 949 |
+
|
| 950 |
+
ElementTree sees only bounded, central-directory-admitted bytes. It is used here as an
|
| 951 |
+
admission parser, not as the workbook parser: no cell value, formula or style is interpreted.
|
| 952 |
+
"""
|
| 953 |
+
dimension_seen = False
|
| 954 |
+
cells = 0
|
| 955 |
+
try:
|
| 956 |
+
for _event, element in ET.iterparse(stream, events=("end",)):
|
| 957 |
+
tag = str(element.tag).rsplit("}", 1)[-1]
|
| 958 |
+
if tag == "dimension":
|
| 959 |
+
ref = str(element.get("ref") or "").replace("$", "").upper()
|
| 960 |
+
parts = ref.split(":")
|
| 961 |
+
if (len(parts) not in (1, 2) or not all(_address_in_document(part) for part in parts)
|
| 962 |
+
or (len(parts) == 2 and rect_of(ref) is None)):
|
| 963 |
+
_archive_refusal(
|
| 964 |
+
"import_declared_dimension_invalid",
|
| 965 |
+
"A worksheet declares cells outside Excel's supported dimensions, so it cannot be imported.")
|
| 966 |
+
dimension_seen = True
|
| 967 |
+
elif tag == "c":
|
| 968 |
+
cells += 1
|
| 969 |
+
ref = str(element.get("r") or "").replace("$", "").upper()
|
| 970 |
+
if not _address_in_document(ref):
|
| 971 |
+
_archive_refusal(
|
| 972 |
+
"import_declared_dimension_invalid",
|
| 973 |
+
"A worksheet contains a cell outside Excel's supported dimensions, so it cannot be imported.")
|
| 974 |
+
if cells > sheets._MAX_CELLS_PER_SHEET:
|
| 975 |
+
_archive_refusal(
|
| 976 |
+
"sheet_too_many_cells",
|
| 977 |
+
"A worksheet declares more than " + str(sheets._MAX_CELLS_PER_SHEET)
|
| 978 |
+
+ " populated cells, so it cannot be imported.")
|
| 979 |
+
element.clear()
|
| 980 |
+
except ImportRefusal:
|
| 981 |
+
raise
|
| 982 |
+
except ET.ParseError as exc:
|
| 983 |
+
raise ImportRefusal(
|
| 984 |
+
"import_unreadable",
|
| 985 |
+
"That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
|
| 986 |
+
# A dimension element is optional in OOXML. When it is absent, the independently bounded
|
| 987 |
+
# cell-address walk above is the authority; absence must not invent a default declaration.
|
| 988 |
+
return dimension_seen
|
| 989 |
+
|
| 990 |
+
|
| 991 |
+
def guard_import_archive(path):
|
| 992 |
+
"""Admit a bounded OOXML archive before any openpyxl workbook parse occurs."""
|
| 993 |
+
try:
|
| 994 |
+
byte_count = os.path.getsize(path)
|
| 995 |
+
except OSError as exc:
|
| 996 |
+
raise ImportRefusal("import_unreadable", "That workbook file could not be read.") from exc
|
| 997 |
+
if byte_count > MAX_IMPORT_BYTES:
|
| 998 |
+
raise ImportRefusal(
|
| 999 |
+
"import_too_large",
|
| 1000 |
+
"That file is larger than the " + str(MAX_IMPORT_BYTES // (1024 * 1024))
|
| 1001 |
+
+ " MB import limit, so it cannot be read.")
|
| 1002 |
+
try:
|
| 1003 |
+
with zipfile.ZipFile(path) as archive:
|
| 1004 |
+
members = archive.infolist()
|
| 1005 |
+
if len(members) > MAX_ARCHIVE_MEMBERS:
|
| 1006 |
+
_archive_refusal("import_zip_bloat",
|
| 1007 |
+
"That workbook archive holds too many parts to import safely.")
|
| 1008 |
+
names = [entry.filename for entry in members]
|
| 1009 |
+
if len(set(names)) != len(names):
|
| 1010 |
+
_archive_refusal("import_zip_bloat",
|
| 1011 |
+
"That workbook archive repeats a part name, so it cannot be imported safely.")
|
| 1012 |
+
total = sum(max(0, int(entry.file_size)) for entry in members)
|
| 1013 |
+
if total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
|
| 1014 |
+
_archive_refusal("import_zip_bloat",
|
| 1015 |
+
"That workbook expands beyond the safe import limit.")
|
| 1016 |
+
for entry in members:
|
| 1017 |
+
if entry.flag_bits & 0x1:
|
| 1018 |
+
_archive_refusal("import_encrypted",
|
| 1019 |
+
"Password protected workbooks cannot be imported.")
|
| 1020 |
+
if entry.file_size > MAX_ARCHIVE_MEMBER_BYTES:
|
| 1021 |
+
_archive_refusal("import_zip_bloat",
|
| 1022 |
+
"One workbook part expands beyond the safe import limit.")
|
| 1023 |
+
ratio = entry.file_size / max(1, entry.compress_size)
|
| 1024 |
+
if entry.file_size and ratio > MAX_ARCHIVE_COMPRESSION_RATIO:
|
| 1025 |
+
_archive_refusal("import_zip_bloat",
|
| 1026 |
+
"That workbook archive is compressed too densely to import safely.")
|
| 1027 |
+
worksheets = [entry for entry in members
|
| 1028 |
+
if re.fullmatch(r"xl/worksheets/sheet[^/]*\.xml", entry.filename)]
|
| 1029 |
+
for entry in worksheets:
|
| 1030 |
+
with archive.open(entry) as stream:
|
| 1031 |
+
_guard_worksheet_xml(stream)
|
| 1032 |
+
except ImportRefusal:
|
| 1033 |
+
raise
|
| 1034 |
+
except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile) as exc:
|
| 1035 |
+
raise ImportRefusal(
|
| 1036 |
+
"import_unreadable",
|
| 1037 |
+
"That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
|
| 1038 |
+
|
| 1039 |
+
|
| 1040 |
def _print_of(ws):
|
| 1041 |
"""Store print setup and inclusive print areas; export executes these settings later."""
|
| 1042 |
setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
|
|
|
|
| 1112 |
⚠ `progress(state, fraction)` IS CALLED FROM WHATEVER THREAD RUNS THIS. It must not raise and
|
| 1113 |
must not block; the job's registry entry is the only thing it touches.
|
| 1114 |
"""
|
|
|
|
|
|
|
| 1115 |
def _say(state, fraction):
|
| 1116 |
if progress:
|
| 1117 |
try:
|
|
|
|
| 1120 |
pass
|
| 1121 |
|
| 1122 |
_say("reading", 0.02)
|
| 1123 |
+
# The guard is deliberately before both the import and the first load_workbook call. A
|
| 1124 |
+
# hostile declared dimension or ZIP bomb must not reach openpyxl's object allocator.
|
| 1125 |
+
guard_import_archive(path)
|
| 1126 |
+
import openpyxl
|
| 1127 |
+
|
| 1128 |
try:
|
| 1129 |
formulas = openpyxl.load_workbook(path, data_only=False, read_only=False,
|
| 1130 |
keep_links=False)
|
|
|
|
| 1226 |
tab["hiddenRows"] = hidden_rows
|
| 1227 |
if hidden_cols:
|
| 1228 |
tab["hiddenCols"] = hidden_cols
|
| 1229 |
+
view = _view_of(ws)
|
| 1230 |
+
if view:
|
| 1231 |
+
tab["view"] = view
|
| 1232 |
charts = _charts_of(ws, name, notes)
|
| 1233 |
if name in chart_tabs:
|
| 1234 |
tab["id"] = chart_tabs[name]["id"]
|
|
|
|
| 1244 |
values.close()
|
| 1245 |
except Exception: # noqa: BLE001
|
| 1246 |
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1247 |
# Original styles are shared once, with integer references on cells, so exact formats,
|
| 1248 |
# theme colours, border weights and blank-cell styles survive without repeating XML per cell.
|
| 1249 |
original = {}
|
|
|
|
| 1265 |
}
|
| 1266 |
|
| 1267 |
|
| 1268 |
+
def export_imported_workbook(book, path):
|
| 1269 |
+
"""Export current cell values with an imported workbook's exact source styles.
|
| 1270 |
+
|
| 1271 |
+
This is the lossless import round-trip seam, not the general Sheet exporter: cells without an
|
| 1272 |
+
imported ``excel.styleId`` use the generated default style. The original style table is
|
| 1273 |
+
restored only after openpyxl has written a structurally valid workbook, so current values,
|
| 1274 |
+
formulas, dimensions, visibility, freeze and view state come from the cleaned document while
|
| 1275 |
+
source fills, fonts, alignments, borders and number formats keep their exact OOXML records.
|
| 1276 |
+
"""
|
| 1277 |
+
import openpyxl
|
| 1278 |
+
|
| 1279 |
+
document = sheets.migrate_workbook(book, lossless=True)
|
| 1280 |
+
target = str(path)
|
| 1281 |
+
output = openpyxl.Workbook()
|
| 1282 |
+
output.remove(output.active)
|
| 1283 |
+
source_styles = ((document.get("excel") or {}).get("stylesXml") or "")
|
| 1284 |
+
source_theme = ((document.get("excel") or {}).get("themeXml") or "")
|
| 1285 |
+
style_ids = []
|
| 1286 |
+
for tab in document.get("sheets") or []:
|
| 1287 |
+
ws = output.create_sheet(str(tab.get("name") or "Sheet")[:31])
|
| 1288 |
+
ws.sheet_state = tab.get("visibility", "visible")
|
| 1289 |
+
by_addr = {}
|
| 1290 |
+
for addr, cell in (tab.get("cells") or {}).items():
|
| 1291 |
+
if not isinstance(cell, dict):
|
| 1292 |
+
continue
|
| 1293 |
+
target_cell = ws[addr]
|
| 1294 |
+
if cell.get("kind") == "formula" and isinstance(cell.get("formula"), str):
|
| 1295 |
+
target_cell.value = cell["formula"]
|
| 1296 |
+
elif cell.get("kind") != "empty":
|
| 1297 |
+
target_cell.value = cell.get("value", cell.get("text"))
|
| 1298 |
+
style_id = (cell.get("excel") or {}).get("styleId")
|
| 1299 |
+
if isinstance(style_id, int) and not isinstance(style_id, bool) and style_id >= 0:
|
| 1300 |
+
by_addr[addr] = style_id
|
| 1301 |
+
for addr, style_id in ((tab.get("excel") or {}).get("blankStyles") or {}).items():
|
| 1302 |
+
if isinstance(style_id, int) and not isinstance(style_id, bool) and style_id >= 0:
|
| 1303 |
+
ws[addr] # Ensure the style-only cell exists in sheetData before the OOXML patch.
|
| 1304 |
+
by_addr.setdefault(addr, style_id)
|
| 1305 |
+
|
| 1306 |
+
# Exact Excel dimension records win. The canonical pixel maps remain the fallback for a
|
| 1307 |
+
# document whose source metadata predates this import seam.
|
| 1308 |
+
column_records = (tab.get("excel") or {}).get("columnDimensions") or {}
|
| 1309 |
+
for key, record in column_records.items():
|
| 1310 |
+
if not isinstance(record, dict):
|
| 1311 |
+
continue
|
| 1312 |
+
dim = ws.column_dimensions[str(key)]
|
| 1313 |
+
if record.get("width") is not None:
|
| 1314 |
+
dim.width = float(record["width"])
|
| 1315 |
+
if record.get("hidden") is not None:
|
| 1316 |
+
dim.hidden = str(record["hidden"]).lower() in ("1", "true")
|
| 1317 |
+
for key, width in (tab.get("colWidths") or {}).items():
|
| 1318 |
+
if key not in column_records:
|
| 1319 |
+
ws.column_dimensions[key].width = 0 if width == 0 else max(0, (float(width) - 5) / 7)
|
| 1320 |
+
row_records = (tab.get("excel") or {}).get("rowDimensions") or {}
|
| 1321 |
+
for key, record in row_records.items():
|
| 1322 |
+
if not isinstance(record, dict) or not str(key).isdigit():
|
| 1323 |
+
continue
|
| 1324 |
+
dim = ws.row_dimensions[int(key)]
|
| 1325 |
+
if record.get("ht") is not None:
|
| 1326 |
+
dim.height = float(record["ht"])
|
| 1327 |
+
if record.get("hidden") is not None:
|
| 1328 |
+
dim.hidden = str(record["hidden"]).lower() in ("1", "true")
|
| 1329 |
+
for key, height in (tab.get("rowHeights") or {}).items():
|
| 1330 |
+
if key not in row_records:
|
| 1331 |
+
ws.row_dimensions[int(key)].height = float(height) * 3 / 4
|
| 1332 |
+
for number in tab.get("hiddenRows") or []:
|
| 1333 |
+
ws.row_dimensions[int(number)].hidden = True
|
| 1334 |
+
for number in tab.get("hiddenCols") or []:
|
| 1335 |
+
ws.column_dimensions[_addr(int(number) - 1, 0).rstrip("0123456789")].hidden = True
|
| 1336 |
+
|
| 1337 |
+
freeze = tab.get("freeze") or {}
|
| 1338 |
+
if freeze.get("rows") or freeze.get("cols"):
|
| 1339 |
+
ws.freeze_panes = _addr(int(freeze.get("cols") or 0), int(freeze.get("rows") or 0))
|
| 1340 |
+
view = tab.get("view") or {}
|
| 1341 |
+
if "gridlines" in view:
|
| 1342 |
+
ws.sheet_view.showGridLines = bool(view["gridlines"])
|
| 1343 |
+
if "headings" in view:
|
| 1344 |
+
ws.sheet_view.showRowColHeaders = bool(view["headings"])
|
| 1345 |
+
if "zoom" in view:
|
| 1346 |
+
ws.sheet_view.zoomScale = int(view["zoom"])
|
| 1347 |
+
if view.get("pageView") in ("normal", "pageBreak", "pageLayout"):
|
| 1348 |
+
ws.sheet_view.view = {"normal": "normal", "pageBreak": "pageBreakPreview",
|
| 1349 |
+
"pageLayout": "pageLayout"}[view["pageView"]]
|
| 1350 |
+
style_ids.append(by_addr)
|
| 1351 |
+
|
| 1352 |
+
output.save(target)
|
| 1353 |
+
output.close()
|
| 1354 |
+
if not source_styles:
|
| 1355 |
+
return target
|
| 1356 |
+
|
| 1357 |
+
namespace = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
|
| 1358 |
+
ET.register_namespace("", namespace)
|
| 1359 |
+
with zipfile.ZipFile(target) as source:
|
| 1360 |
+
parts = [(entry, source.read(entry.filename)) for entry in source.infolist()]
|
| 1361 |
+
patched = []
|
| 1362 |
+
for entry, blob in parts:
|
| 1363 |
+
match = re.fullmatch(r"xl/worksheets/sheet([0-9]+)\.xml", entry.filename)
|
| 1364 |
+
if match and int(match.group(1)) <= len(style_ids):
|
| 1365 |
+
wanted = style_ids[int(match.group(1)) - 1]
|
| 1366 |
+
if wanted:
|
| 1367 |
+
root = ET.fromstring(blob)
|
| 1368 |
+
for cell in root.iter("{" + namespace + "}c"):
|
| 1369 |
+
style_id = wanted.get(cell.get("r"))
|
| 1370 |
+
if style_id is not None:
|
| 1371 |
+
cell.set("s", str(style_id))
|
| 1372 |
+
blob = ET.tostring(root, encoding="utf-8", xml_declaration=True)
|
| 1373 |
+
elif entry.filename == "xl/styles.xml":
|
| 1374 |
+
blob = source_styles.encode("utf-8")
|
| 1375 |
+
elif entry.filename == "xl/theme/theme1.xml" and source_theme:
|
| 1376 |
+
blob = source_theme.encode("utf-8")
|
| 1377 |
+
patched.append((entry, blob))
|
| 1378 |
+
handle, replacement = tempfile.mkstemp(prefix="sheet-export-", suffix=".xlsx",
|
| 1379 |
+
dir=os.path.dirname(os.path.abspath(target)))
|
| 1380 |
+
os.close(handle)
|
| 1381 |
+
try:
|
| 1382 |
+
with zipfile.ZipFile(replacement, "w") as archive:
|
| 1383 |
+
for entry, blob in patched:
|
| 1384 |
+
archive.writestr(entry, blob)
|
| 1385 |
+
os.replace(replacement, target)
|
| 1386 |
+
finally:
|
| 1387 |
+
try:
|
| 1388 |
+
os.remove(replacement)
|
| 1389 |
+
except OSError:
|
| 1390 |
+
pass
|
| 1391 |
+
return target
|
| 1392 |
+
|
| 1393 |
+
|
| 1394 |
def take_imported_values(book):
|
| 1395 |
"""Pull every `imported_value` off the document and return `{(tab, addr): cached}`.
|
| 1396 |
|
platform/core/sheet_templates.py
CHANGED
|
@@ -9,7 +9,7 @@ from copy import deepcopy
|
|
| 9 |
from datetime import date, datetime, timedelta, timezone
|
| 10 |
import re
|
| 11 |
|
| 12 |
-
from core import sheets, user_tables
|
| 13 |
|
| 14 |
|
| 15 |
# Do not use connector_spec.connector_of: it only covers SPEC'd tables and calls
|
|
@@ -205,7 +205,7 @@ def save_sheet_template(st, workbook, key, *, name="", description="", created_b
|
|
| 205 |
databases.add(database)
|
| 206 |
record = {"key": key, "name": name.strip() or book.get("name") or key,
|
| 207 |
"description": description, "databases": sorted(databases), "workbook": book,
|
| 208 |
-
"created_by": created_by, "created_at":
|
| 209 |
"byName": actor_name or created_by}
|
| 210 |
|
| 211 |
def save(state):
|
|
|
|
| 9 |
from datetime import date, datetime, timedelta, timezone
|
| 10 |
import re
|
| 11 |
|
| 12 |
+
from core import sheets, user_tables, write_execution
|
| 13 |
|
| 14 |
|
| 15 |
# Do not use connector_spec.connector_of: it only covers SPEC'd tables and calls
|
|
|
|
| 205 |
databases.add(database)
|
| 206 |
record = {"key": key, "name": name.strip() or book.get("name") or key,
|
| 207 |
"description": description, "databases": sorted(databases), "workbook": book,
|
| 208 |
+
"created_by": created_by, "created_at": write_execution.utcnow().isoformat(),
|
| 209 |
"byName": actor_name or created_by}
|
| 210 |
|
| 211 |
def save(state):
|
platform/core/sheets.py
CHANGED
|
@@ -60,8 +60,7 @@ import hashlib
|
|
| 60 |
import json
|
| 61 |
import math
|
| 62 |
import re
|
| 63 |
-
import
|
| 64 |
-
from datetime import datetime, timezone
|
| 65 |
|
| 66 |
from core.sheet_binding import (
|
| 67 |
BindingMetadataRefusal,
|
|
@@ -247,7 +246,7 @@ SHEET_CHART_STACKING = ("clustered", "stacked", "percent")
|
|
| 247 |
#:
|
| 248 |
#: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
|
| 249 |
SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
|
| 250 |
-
"rules", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
|
| 251 |
"visibility", "view", "print", "excel")
|
| 252 |
|
| 253 |
#: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
|
|
@@ -261,8 +260,17 @@ _MAX_HIDDEN = 5000
|
|
| 261 |
_MAX_CHARTS = 50
|
| 262 |
_MAX_UNSUPPORTED = 200
|
| 263 |
|
| 264 |
-
#:
|
| 265 |
-
#:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
_COLOUR = re.compile(r"^[A-Za-z0-9#_-]{1,40}$")
|
| 267 |
|
| 268 |
_MAX_DECIMALS = 10
|
|
@@ -280,7 +288,7 @@ _MAX_DECIMALS = 10
|
|
| 280 |
#: sheet at the ceiling is still far above `_MAX_CELLS_PER_SHEET`, so content runs out first.
|
| 281 |
DEFAULT_SHEET_COLS = 200
|
| 282 |
DEFAULT_SHEET_ROWS = 10000
|
| 283 |
-
DEFAULT_COL_WIDTH =
|
| 284 |
MIN_SHEET_COLS = 1
|
| 285 |
MIN_SHEET_ROWS = 1
|
| 286 |
MAX_SHEET_COLS = 702
|
|
@@ -367,7 +375,7 @@ class SheetStaleRefusal(SheetRefusal):
|
|
| 367 |
|
| 368 |
|
| 369 |
def _now():
|
| 370 |
-
return
|
| 371 |
|
| 372 |
|
| 373 |
def workbook_key(workbook_id):
|
|
@@ -399,6 +407,7 @@ def _read(st, key):
|
|
| 399 |
|
| 400 |
def _persist(st, key, mutate, seed=None):
|
| 401 |
"""Read-modify-write under the store's own lock, mirroring `chart_system._persist`."""
|
|
|
|
| 402 |
result = {}
|
| 403 |
|
| 404 |
def apply(current):
|
|
@@ -408,9 +417,12 @@ def _persist(st, key, mutate, seed=None):
|
|
| 408 |
state.setdefault(k, copy.deepcopy(v))
|
| 409 |
next_state, value = mutate(state)
|
| 410 |
result["value"] = copy.deepcopy(value)
|
|
|
|
|
|
|
| 411 |
return next_state
|
| 412 |
|
| 413 |
st.update(key, apply, flush="sync")
|
|
|
|
| 414 |
return result.get("value")
|
| 415 |
|
| 416 |
|
|
@@ -530,6 +542,29 @@ def _clean_format(raw):
|
|
| 530 |
return out
|
| 531 |
|
| 532 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 533 |
def _clean_style(raw):
|
| 534 |
"""Normalize a cell's A1 style, or None when nothing survives.
|
| 535 |
|
|
@@ -552,8 +587,8 @@ def _clean_style(raw):
|
|
| 552 |
out[mark] = True
|
| 553 |
|
| 554 |
for key in ("color", "fill"):
|
| 555 |
-
token = _text(raw.get(key), 40)
|
| 556 |
-
if token
|
| 557 |
out[key] = token
|
| 558 |
|
| 559 |
align = _text(raw.get("align"), 12)
|
|
@@ -583,13 +618,16 @@ def _clean_style(raw):
|
|
| 583 |
#: picked, gone on the next read with nothing to say so. The type is decided FIRST, and a step
|
| 584 |
#: name still round trips exactly as it always did. A point size is CLAMPED, not refused, for
|
| 585 |
#: the rotation's reason below, and the default is stored as an absence just as `m` is.
|
|
|
|
|
|
|
|
|
|
| 586 |
size = raw.get("size")
|
| 587 |
if isinstance(size, str):
|
| 588 |
step = _text(size, 4)
|
| 589 |
if step in CELL_SIZES and step != "m":
|
| 590 |
out["size"] = step
|
| 591 |
-
elif not isinstance(size, bool) and isinstance(size, (int, float)):
|
| 592 |
-
points = max(MIN_FONT_PT, min(MAX_FONT_PT,
|
| 593 |
if points != DEFAULT_FONT_PT:
|
| 594 |
out["size"] = points
|
| 595 |
|
|
@@ -1365,6 +1403,32 @@ def _clean_imported(raw):
|
|
| 1365 |
return out if (out["at"] or out["source"] or notes) else None
|
| 1366 |
|
| 1367 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1368 |
def _clean_sheet(raw, index=0, *, lossless=False):
|
| 1369 |
"""Normalize a tab's known strata and retain additive JSON metadata without interpretation.
|
| 1370 |
|
|
@@ -1404,6 +1468,7 @@ def _clean_sheet(raw, index=0, *, lossless=False):
|
|
| 1404 |
"colWidths": _clean_col_widths(raw.get("colWidths"), cols, lossless=lossless),
|
| 1405 |
"freeze": _clean_freeze(raw.get("freeze"), cols, rows, lossless=lossless),
|
| 1406 |
"rules": _clean_rules(raw.get("rules"), lossless=lossless),
|
|
|
|
| 1407 |
"hiddenRows": _clean_hidden(raw.get("hiddenRows"), "rows"),
|
| 1408 |
"hiddenCols": _clean_hidden(raw.get("hiddenCols"), "columns"),
|
| 1409 |
"charts": _clean_charts(raw.get("charts")),
|
|
@@ -1481,13 +1546,13 @@ def migrate_workbook(raw, *, lossless=True):
|
|
| 1481 |
|
| 1482 |
def new_workbook(name="", created_by="", workbook_id=""):
|
| 1483 |
"""The in-memory shape of a fresh workbook with ONE tab (R7: a workbook has tabs)."""
|
| 1484 |
-
wid = _slug_id(workbook_id) if workbook_id else
|
| 1485 |
stamp = _now()
|
| 1486 |
return {
|
| 1487 |
"id": wid,
|
| 1488 |
"schemaVersion": SHEET_SCHEMA_VERSION,
|
| 1489 |
"name": _text(name, _MAX_NAME) or "Untitled workbook",
|
| 1490 |
-
"sheets": [{"id":
|
| 1491 |
"cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {},
|
| 1492 |
"usedExtent": None, "writesFrom": 0}],
|
| 1493 |
"rev": 1,
|
|
@@ -1764,6 +1829,132 @@ def find_tab(book, tab_ref):
|
|
| 1764 |
return tabs[at]
|
| 1765 |
|
| 1766 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1767 |
def workbook_meta(book):
|
| 1768 |
"""The workbook WITHOUT any cells, for C3's `?tabs=meta`.
|
| 1769 |
|
|
|
|
| 60 |
import json
|
| 61 |
import math
|
| 62 |
import re
|
| 63 |
+
from core import write_execution
|
|
|
|
| 64 |
|
| 65 |
from core.sheet_binding import (
|
| 66 |
BindingMetadataRefusal,
|
|
|
|
| 246 |
#:
|
| 247 |
#: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
|
| 248 |
SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
|
| 249 |
+
"rules", "rowGroups", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
|
| 250 |
"visibility", "view", "print", "excel")
|
| 251 |
|
| 252 |
#: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
|
|
|
|
| 260 |
_MAX_CHARTS = 50
|
| 261 |
_MAX_UNSUPPORTED = 200
|
| 262 |
|
| 263 |
+
#: ⭐⭐ SDLC 64 R2 / CONTRACT C1 — THE ACCEPTED FORM OF A CELL COLOUR, AS ONE NAMED PATTERN.
|
| 264 |
+
#: Mirrored BYTE FOR BYTE in `aios-web/web/src/sheet/types.ts::CELL_COLOUR_PATTERN`, and
|
| 265 |
+
#: `verify_sheet.py`'s C1 leg compares the two source strings, so the client cannot start
|
| 266 |
+
#: writing a form this cleaner would drop.
|
| 267 |
+
CELL_COLOUR_PATTERN = r"^(--lp-[a-z0-9-]{1,40}|#[0-9a-f]{6})$"
|
| 268 |
+
_CELL_COLOUR = re.compile(CELL_COLOUR_PATTERN)
|
| 269 |
+
|
| 270 |
+
#: R2 / C1: CELL `color` / `fill` use CELL_COLOUR_PATTERN above. This older guard stays on
|
| 271 |
+
#: `_clean_rule_style` and `_clean_scale`: their RULE_FILLS / RULE_INKS galleries remain token
|
| 272 |
+
#: only. Its existing storage allowance is unchanged; a cell fill's custom door does not widen
|
| 273 |
+
#: the conditional-format palettes or make either guard dead.
|
| 274 |
_COLOUR = re.compile(r"^[A-Za-z0-9#_-]{1,40}$")
|
| 275 |
|
| 276 |
_MAX_DECIMALS = 10
|
|
|
|
| 288 |
#: sheet at the ceiling is still far above `_MAX_CELLS_PER_SHEET`, so content runs out first.
|
| 289 |
DEFAULT_SHEET_COLS = 200
|
| 290 |
DEFAULT_SHEET_ROWS = 10000
|
| 291 |
+
DEFAULT_COL_WIDTH = 100
|
| 292 |
MIN_SHEET_COLS = 1
|
| 293 |
MIN_SHEET_ROWS = 1
|
| 294 |
MAX_SHEET_COLS = 702
|
|
|
|
| 375 |
|
| 376 |
|
| 377 |
def _now():
|
| 378 |
+
return write_execution.utcnow().replace(microsecond=0).isoformat()
|
| 379 |
|
| 380 |
|
| 381 |
def workbook_key(workbook_id):
|
|
|
|
| 407 |
|
| 408 |
def _persist(st, key, mutate, seed=None):
|
| 409 |
"""Read-modify-write under the store's own lock, mirroring `chart_system._persist`."""
|
| 410 |
+
from core import sheet_events
|
| 411 |
result = {}
|
| 412 |
|
| 413 |
def apply(current):
|
|
|
|
| 417 |
state.setdefault(k, copy.deepcopy(v))
|
| 418 |
next_state, value = mutate(state)
|
| 419 |
result["value"] = copy.deepcopy(value)
|
| 420 |
+
# A store retry replaces this delta; no observer runs for a discarded transaction.
|
| 421 |
+
result["events"] = sheet_events.committed_changes(current, next_state)
|
| 422 |
return next_state
|
| 423 |
|
| 424 |
st.update(key, apply, flush="sync")
|
| 425 |
+
sheet_events.publish(st, result.get("events", []))
|
| 426 |
return result.get("value")
|
| 427 |
|
| 428 |
|
|
|
|
| 542 |
return out
|
| 543 |
|
| 544 |
|
| 545 |
+
def _normalise_colour(token):
|
| 546 |
+
"""One cell colour in C1's accepted form, or "" when nothing survives.
|
| 547 |
+
|
| 548 |
+
⭐⭐ C1's FORM IS LOWERCASE `#rrggbb`, AND THAT IS WHAT IS STORED, SO A RELOAD COMPARES
|
| 549 |
+
BYTE FOR BYTE EQUAL TO THE SAVE. A colour already in that form is returned unchanged:
|
| 550 |
+
normalisation is the identity on everything the client writes, which is the whole point.
|
| 551 |
+
|
| 552 |
+
⚠ THE TWO COERCIONS ARE A RESCUE, NOT A CONVENIENCE. `#AABBCC` and `#abc` are forms an
|
| 553 |
+
IMPORTED workbook can carry, and this cleaner's house rule is to DROP what it does not
|
| 554 |
+
recognise, in silence. Dropping a colour somebody can see is the exact failure R2 named,
|
| 555 |
+
so a case difference and the three digit shorthand are widened onto the stored form
|
| 556 |
+
instead of being thrown away. Anything else is still dropped.
|
| 557 |
+
"""
|
| 558 |
+
if not token:
|
| 559 |
+
return ""
|
| 560 |
+
if token.startswith("#"):
|
| 561 |
+
body = token[1:].lower()
|
| 562 |
+
if len(body) == 3 and all(c in "0123456789abcdef" for c in body):
|
| 563 |
+
body = body[0] * 2 + body[1] * 2 + body[2] * 2
|
| 564 |
+
token = "#" + body
|
| 565 |
+
return token if _CELL_COLOUR.match(token) else ""
|
| 566 |
+
|
| 567 |
+
|
| 568 |
def _clean_style(raw):
|
| 569 |
"""Normalize a cell's A1 style, or None when nothing survives.
|
| 570 |
|
|
|
|
| 587 |
out[mark] = True
|
| 588 |
|
| 589 |
for key in ("color", "fill"):
|
| 590 |
+
token = _normalise_colour(_text(raw.get(key), 40))
|
| 591 |
+
if token:
|
| 592 |
out[key] = token
|
| 593 |
|
| 594 |
align = _text(raw.get("align"), 12)
|
|
|
|
| 618 |
#: picked, gone on the next read with nothing to say so. The type is decided FIRST, and a step
|
| 619 |
#: name still round trips exactly as it always did. A point size is CLAMPED, not refused, for
|
| 620 |
#: the rotation's reason below, and the default is stored as an absence just as `m` is.
|
| 621 |
+
#: Imported fractional points remain fractional: 11.5pt is authored workbook data, while
|
| 622 |
+
#: ribbon steps happen to be integers. Rounding here made an import unequal to its source on
|
| 623 |
+
#: the first clean/save cycle.
|
| 624 |
size = raw.get("size")
|
| 625 |
if isinstance(size, str):
|
| 626 |
step = _text(size, 4)
|
| 627 |
if step in CELL_SIZES and step != "m":
|
| 628 |
out["size"] = step
|
| 629 |
+
elif not isinstance(size, bool) and isinstance(size, (int, float)) and math.isfinite(size):
|
| 630 |
+
points = max(MIN_FONT_PT, min(MAX_FONT_PT, size))
|
| 631 |
if points != DEFAULT_FONT_PT:
|
| 632 |
out["size"] = points
|
| 633 |
|
|
|
|
| 1403 |
return out if (out["at"] or out["source"] or notes) else None
|
| 1404 |
|
| 1405 |
|
| 1406 |
+
def _clean_row_groups(raw, rows):
|
| 1407 |
+
"""Inclusive sheet ranges, with independent nested collapse state and no empty stratum."""
|
| 1408 |
+
if raw is None:
|
| 1409 |
+
return None
|
| 1410 |
+
if not isinstance(raw, list):
|
| 1411 |
+
raise SheetRefusal("sheet_row_groups_invalid", "Row groups must be a list.")
|
| 1412 |
+
out, ids = [], set()
|
| 1413 |
+
for item in raw:
|
| 1414 |
+
if not isinstance(item, dict):
|
| 1415 |
+
raise SheetRefusal("sheet_row_groups_invalid", "A row group must be an object.")
|
| 1416 |
+
gid, start, end, collapsed = (item.get(key) for key in ("id", "start", "end", "collapsed"))
|
| 1417 |
+
if not isinstance(gid, str) or not gid.strip() or len(gid) > _MAX_ID or gid in ids \
|
| 1418 |
+
or any(isinstance(n, bool) or not isinstance(n, int) for n in (start, end)) \
|
| 1419 |
+
or not 1 <= start <= end <= rows or not isinstance(collapsed, bool):
|
| 1420 |
+
raise SheetRefusal("sheet_row_groups_invalid", "A row group needs an id, a valid row range and a collapse state.")
|
| 1421 |
+
for prior in out:
|
| 1422 |
+
disjoint = end < prior["start"] or start > prior["end"]
|
| 1423 |
+
nested = (start >= prior["start"] and end <= prior["end"]) \
|
| 1424 |
+
or (start <= prior["start"] and end >= prior["end"])
|
| 1425 |
+
if (start, end) == (prior["start"], prior["end"]) or not (disjoint or nested):
|
| 1426 |
+
raise SheetRefusal("sheet_row_groups_overlap", "Groups can be nested, but cannot duplicate or partially overlap.")
|
| 1427 |
+
ids.add(gid)
|
| 1428 |
+
out.append({"id": gid, "start": start, "end": end, "collapsed": collapsed})
|
| 1429 |
+
return out or None
|
| 1430 |
+
|
| 1431 |
+
|
| 1432 |
def _clean_sheet(raw, index=0, *, lossless=False):
|
| 1433 |
"""Normalize a tab's known strata and retain additive JSON metadata without interpretation.
|
| 1434 |
|
|
|
|
| 1468 |
"colWidths": _clean_col_widths(raw.get("colWidths"), cols, lossless=lossless),
|
| 1469 |
"freeze": _clean_freeze(raw.get("freeze"), cols, rows, lossless=lossless),
|
| 1470 |
"rules": _clean_rules(raw.get("rules"), lossless=lossless),
|
| 1471 |
+
"rowGroups": _clean_row_groups(raw.get("rowGroups"), rows),
|
| 1472 |
"hiddenRows": _clean_hidden(raw.get("hiddenRows"), "rows"),
|
| 1473 |
"hiddenCols": _clean_hidden(raw.get("hiddenCols"), "columns"),
|
| 1474 |
"charts": _clean_charts(raw.get("charts")),
|
|
|
|
| 1546 |
|
| 1547 |
def new_workbook(name="", created_by="", workbook_id=""):
|
| 1548 |
"""The in-memory shape of a fresh workbook with ONE tab (R7: a workbook has tabs)."""
|
| 1549 |
+
wid = _slug_id(workbook_id) if workbook_id else write_execution.new_id()[:12]
|
| 1550 |
stamp = _now()
|
| 1551 |
return {
|
| 1552 |
"id": wid,
|
| 1553 |
"schemaVersion": SHEET_SCHEMA_VERSION,
|
| 1554 |
"name": _text(name, _MAX_NAME) or "Untitled workbook",
|
| 1555 |
+
"sheets": [{"id": write_execution.new_id()[:12], "name": "Sheet1",
|
| 1556 |
"cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {},
|
| 1557 |
"usedExtent": None, "writesFrom": 0}],
|
| 1558 |
"rev": 1,
|
|
|
|
| 1829 |
return tabs[at]
|
| 1830 |
|
| 1831 |
|
| 1832 |
+
def workbook_snapshot(st, workbook_id):
|
| 1833 |
+
"""Return the stored workbook bytes as JSON data, without migration or cleaning.
|
| 1834 |
+
|
| 1835 |
+
The ordinary read door intentionally migrates old workbooks. A narrowly scoped repair needs
|
| 1836 |
+
a different primitive: its backup and compare-and-swap digest must describe the document that
|
| 1837 |
+
is actually stored, or a migration performed only in memory could make the precondition look
|
| 1838 |
+
stronger than it is. The returned copy is detached from the store.
|
| 1839 |
+
"""
|
| 1840 |
+
wid = _slug_id(workbook_id)
|
| 1841 |
+
doc = _read(st, workbook_key(wid))
|
| 1842 |
+
if not doc or not doc.get("id"):
|
| 1843 |
+
raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
|
| 1844 |
+
return copy.deepcopy(doc)
|
| 1845 |
+
|
| 1846 |
+
|
| 1847 |
+
def workbook_digest(book):
|
| 1848 |
+
"""A stable digest for an exact workbook snapshot."""
|
| 1849 |
+
if not isinstance(book, dict):
|
| 1850 |
+
raise SheetRefusal("workbook_not_an_object", "A workbook must be an object.")
|
| 1851 |
+
encoded = json.dumps(book, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
| 1852 |
+
default=str).encode("utf-8")
|
| 1853 |
+
return hashlib.sha256(encoded).hexdigest()
|
| 1854 |
+
|
| 1855 |
+
|
| 1856 |
+
def apply_exact_cell_edits(st, workbook_id, edits, expect_rev, expect_digest, *,
|
| 1857 |
+
actor="", actor_name=""):
|
| 1858 |
+
"""Apply a preconditioned list of cell edits under the store lock.
|
| 1859 |
+
|
| 1860 |
+
This is deliberately narrower than PUT and stricter than the collaborative range PATCH. Each
|
| 1861 |
+
edit names one tab/address and carries the exact value observed while the repair was planned.
|
| 1862 |
+
The workbook revision, whole-document digest and every cell precondition must still match.
|
| 1863 |
+
Nothing outside those cells, normal revision/write-audit fields and derived used extents is
|
| 1864 |
+
rewritten. It is the storage primitive for exceptional, owner-authorized workbook repair;
|
| 1865 |
+
normal Sheet editing continues to use ``put_workbook`` and ``patch_cells``.
|
| 1866 |
+
"""
|
| 1867 |
+
wid = _slug_id(workbook_id)
|
| 1868 |
+
if isinstance(expect_rev, bool) or not isinstance(expect_rev, int) or expect_rev < 0:
|
| 1869 |
+
raise SheetRefusal("repair_revision_required",
|
| 1870 |
+
"An exact workbook revision is required for this repair.")
|
| 1871 |
+
if not isinstance(expect_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", expect_digest):
|
| 1872 |
+
raise SheetRefusal("repair_digest_required",
|
| 1873 |
+
"An exact workbook snapshot digest is required for this repair.")
|
| 1874 |
+
if not isinstance(edits, list) or not edits:
|
| 1875 |
+
raise SheetRefusal("repair_edits_required", "This repair does not name any cell edits.")
|
| 1876 |
+
|
| 1877 |
+
prepared, seen = [], set()
|
| 1878 |
+
for raw in edits:
|
| 1879 |
+
if not isinstance(raw, dict):
|
| 1880 |
+
raise SheetRefusal("repair_edit_invalid", "Every repair edit must be an object.")
|
| 1881 |
+
tab_ref = str(raw.get("sheet") or "").strip()
|
| 1882 |
+
addr = str(raw.get("address") or "").strip().upper().replace("$", "")
|
| 1883 |
+
matched = _ADDR.fullmatch(addr)
|
| 1884 |
+
if (not tab_ref or not matched
|
| 1885 |
+
or _col_index(re.match(r"[A-Z]+", addr).group(0)) >= DOCUMENT_MAX_COLS
|
| 1886 |
+
or int(re.search(r"[0-9]+$", addr).group(0)) > DOCUMENT_MAX_ROWS):
|
| 1887 |
+
raise SheetRefusal("repair_edit_invalid",
|
| 1888 |
+
"Every repair edit must name one sheet and one A1 address.")
|
| 1889 |
+
slot = (tab_ref.casefold(), addr)
|
| 1890 |
+
if slot in seen:
|
| 1891 |
+
raise SheetRefusal("repair_edit_duplicate",
|
| 1892 |
+
"A workbook repair may name each cell only once.")
|
| 1893 |
+
seen.add(slot)
|
| 1894 |
+
after = raw.get("after")
|
| 1895 |
+
prepared.append({"sheet": tab_ref, "address": addr,
|
| 1896 |
+
"before": copy.deepcopy(raw.get("before")),
|
| 1897 |
+
"after": None if after is None else clean_cell(after, addr=addr)})
|
| 1898 |
+
prepared.sort(key=lambda row: (row["sheet"].casefold(), row["address"]))
|
| 1899 |
+
|
| 1900 |
+
saved = {}
|
| 1901 |
+
|
| 1902 |
+
def _apply(state):
|
| 1903 |
+
prior = state if state.get("id") else None
|
| 1904 |
+
if prior is None:
|
| 1905 |
+
raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
|
| 1906 |
+
if _slug_id(prior.get("id")) != wid:
|
| 1907 |
+
raise SheetRefusal("repair_workbook_mismatch",
|
| 1908 |
+
"The stored workbook identity does not match this repair target.")
|
| 1909 |
+
current = int(prior.get("rev") or 0)
|
| 1910 |
+
if current != expect_rev:
|
| 1911 |
+
raise SheetStaleRefusal(
|
| 1912 |
+
"workbook_revision_stale",
|
| 1913 |
+
"That workbook changed after the repair was planned, so no repair was applied.",
|
| 1914 |
+
rev_current=current)
|
| 1915 |
+
if workbook_digest(prior) != expect_digest:
|
| 1916 |
+
raise SheetRefusal(
|
| 1917 |
+
"repair_snapshot_changed",
|
| 1918 |
+
"That workbook changed without a revision change, so no repair was applied.")
|
| 1919 |
+
|
| 1920 |
+
book = copy.deepcopy(prior)
|
| 1921 |
+
touched = {}
|
| 1922 |
+
for edit in prepared:
|
| 1923 |
+
at = _tab_index(book.get("sheets") or [], edit["sheet"])
|
| 1924 |
+
if at is None:
|
| 1925 |
+
raise SheetRefusal("tab_missing", "Tab " + edit["sheet"]
|
| 1926 |
+
+ " is not in this workbook.")
|
| 1927 |
+
tab = book["sheets"][at]
|
| 1928 |
+
cells = tab.setdefault("cells", {})
|
| 1929 |
+
actual = copy.deepcopy(cells.get(edit["address"]))
|
| 1930 |
+
if actual != edit["before"]:
|
| 1931 |
+
raise SheetRefusal(
|
| 1932 |
+
"repair_cell_changed",
|
| 1933 |
+
"Cell " + edit["sheet"] + "!" + edit["address"]
|
| 1934 |
+
+ " changed after the repair was planned, so no repair was applied.")
|
| 1935 |
+
if edit["after"] is None:
|
| 1936 |
+
cells.pop(edit["address"], None)
|
| 1937 |
+
else:
|
| 1938 |
+
cells[edit["address"]] = copy.deepcopy(edit["after"])
|
| 1939 |
+
touched.setdefault(at, []).append(edit["address"])
|
| 1940 |
+
|
| 1941 |
+
new_rev = current + 1
|
| 1942 |
+
for at, addrs in touched.items():
|
| 1943 |
+
tab = book["sheets"][at]
|
| 1944 |
+
tab["usedExtent"] = _used_extent(
|
| 1945 |
+
tab.get("cells") or {}, (tab.get("excel") or {}).get("blankStyles"))
|
| 1946 |
+
_append_write_log(tab, new_rev, addrs, actor=actor, actor_name=actor_name)
|
| 1947 |
+
book["rev"] = new_rev
|
| 1948 |
+
book["updated_at"] = _now()
|
| 1949 |
+
_json_safe(book, "workbook")
|
| 1950 |
+
saved.update(book)
|
| 1951 |
+
return book, {"rev": new_rev, "edits": len(prepared),
|
| 1952 |
+
"paths": [row["sheet"] + "!" + row["address"] for row in prepared]}
|
| 1953 |
+
|
| 1954 |
+
out = _persist(st, workbook_key(wid), _apply)
|
| 1955 |
+
return {**out, "digest": workbook_digest(saved)}
|
| 1956 |
+
|
| 1957 |
+
|
| 1958 |
def workbook_meta(book):
|
| 1959 |
"""The workbook WITHOUT any cells, for C3's `?tabs=meta`.
|
| 1960 |
|
platform/core/table_store.py
CHANGED
|
@@ -19,6 +19,7 @@ import time
|
|
| 19 |
|
| 20 |
import core.store as store
|
| 21 |
import core.view_audit as view_audit
|
|
|
|
| 22 |
|
| 23 |
#: ``__shared__`` remains the tenant-wide FIELD-annotation stratum. Before Wave 48 it also
|
| 24 |
#: carried a second, legacy home for shared Views. That view map is migrated away on the first
|
|
@@ -1727,7 +1728,7 @@ class TableStore:
|
|
| 1727 |
data.pop(ASSIGNED_KEY, None)
|
| 1728 |
if isinstance(before, dict):
|
| 1729 |
bucket = data.setdefault(owner, {}).setdefault(TRASH_KEY, {})
|
| 1730 |
-
stamp =
|
| 1731 |
bucket[vid] = {'view': before, 'deleted_at': float(stamp),
|
| 1732 |
'deleted_by': who, 'assignments': carried}
|
| 1733 |
purge_trash(bucket, now=stamp)
|
|
@@ -1818,7 +1819,7 @@ class TableStore:
|
|
| 1818 |
current.get(owner), dict) else None
|
| 1819 |
if isinstance(live, dict):
|
| 1820 |
live.pop(vid, None)
|
| 1821 |
-
purge_trash(live, now=
|
| 1822 |
if not live:
|
| 1823 |
current[owner].pop(TRASH_KEY, None)
|
| 1824 |
audit(action=None, owner=owner, view_id=vid)
|
|
@@ -1854,7 +1855,7 @@ class TableStore:
|
|
| 1854 |
bucket = ((data.get(owner) or {}).get(TRASH_KEY) or {})
|
| 1855 |
if not isinstance(bucket, dict) or not bucket:
|
| 1856 |
return []
|
| 1857 |
-
stamp = float(now if now is not None else
|
| 1858 |
expired = [vid for vid, entry in bucket.items() if trash_expired(entry, now=stamp)]
|
| 1859 |
# ⛔⛔ R11's ADOPTION RUNS BEFORE THE UPDATE AND OUTSIDE ITS CALLBACK (W51-T26).
|
| 1860 |
# `sheets.py::save_folders` states the rule for this codebase in as many words: validation
|
|
@@ -1985,7 +1986,7 @@ class TableStore:
|
|
| 1985 |
if isinstance(assignment, dict):
|
| 1986 |
target[str(aid)] = dict(assignment)
|
| 1987 |
bucket.pop(vid, None)
|
| 1988 |
-
purge_trash(bucket, now=
|
| 1989 |
if not bucket:
|
| 1990 |
data[owner].pop(TRASH_KEY, None)
|
| 1991 |
restored.clear()
|
|
|
|
| 19 |
|
| 20 |
import core.store as store
|
| 21 |
import core.view_audit as view_audit
|
| 22 |
+
from core import write_execution
|
| 23 |
|
| 24 |
#: ``__shared__`` remains the tenant-wide FIELD-annotation stratum. Before Wave 48 it also
|
| 25 |
#: carried a second, legacy home for shared Views. That view map is migrated away on the first
|
|
|
|
| 1728 |
data.pop(ASSIGNED_KEY, None)
|
| 1729 |
if isinstance(before, dict):
|
| 1730 |
bucket = data.setdefault(owner, {}).setdefault(TRASH_KEY, {})
|
| 1731 |
+
stamp = write_execution.timestamp()
|
| 1732 |
bucket[vid] = {'view': before, 'deleted_at': float(stamp),
|
| 1733 |
'deleted_by': who, 'assignments': carried}
|
| 1734 |
purge_trash(bucket, now=stamp)
|
|
|
|
| 1819 |
current.get(owner), dict) else None
|
| 1820 |
if isinstance(live, dict):
|
| 1821 |
live.pop(vid, None)
|
| 1822 |
+
purge_trash(live, now=write_execution.timestamp())
|
| 1823 |
if not live:
|
| 1824 |
current[owner].pop(TRASH_KEY, None)
|
| 1825 |
audit(action=None, owner=owner, view_id=vid)
|
|
|
|
| 1855 |
bucket = ((data.get(owner) or {}).get(TRASH_KEY) or {})
|
| 1856 |
if not isinstance(bucket, dict) or not bucket:
|
| 1857 |
return []
|
| 1858 |
+
stamp = float(now if now is not None else write_execution.timestamp())
|
| 1859 |
expired = [vid for vid, entry in bucket.items() if trash_expired(entry, now=stamp)]
|
| 1860 |
# ⛔⛔ R11's ADOPTION RUNS BEFORE THE UPDATE AND OUTSIDE ITS CALLBACK (W51-T26).
|
| 1861 |
# `sheets.py::save_folders` states the rule for this codebase in as many words: validation
|
|
|
|
| 1986 |
if isinstance(assignment, dict):
|
| 1987 |
target[str(aid)] = dict(assignment)
|
| 1988 |
bucket.pop(vid, None)
|
| 1989 |
+
purge_trash(bucket, now=write_execution.timestamp())
|
| 1990 |
if not bucket:
|
| 1991 |
data[owner].pop(TRASH_KEY, None)
|
| 1992 |
restored.clear()
|
platform/core/user_tables.py
CHANGED
|
@@ -69,6 +69,22 @@ SHARED_NOTE_READS_KEY = 'shared_note_reads'
|
|
| 69 |
ODOO_SCHEMA_SNAPSHOT_KEY = 'odoo_schema_snapshot'
|
| 70 |
ODOO_SCHEMA_ENABLEMENTS_KEY = 'odoo_schema_enablements'
|
| 71 |
ODOO_SYNC_STATUS_KEY = 'odoo_sync_status'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
# A reserved, derived multi-select field. It is table data backed by durable Zone membership,
|
| 73 |
# not a user-editable column and therefore cannot collide with a field a user creates.
|
| 74 |
ZONE_FIELD_KEY = '__zone__'
|
|
@@ -467,6 +483,74 @@ def connector_database_identity(table_key, rt=None, st=None, definition=None):
|
|
| 467 |
declared.get('model') or (getattr(spec, 'native', '') if spec else ''))
|
| 468 |
|
| 469 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 470 |
def credential_footprint(entry_type, connection_id, rt=None, st=None):
|
| 471 |
"""Disclose what revoking this ONE credential purges and what its policy keeps.
|
| 472 |
|
|
|
|
| 69 |
ODOO_SCHEMA_SNAPSHOT_KEY = 'odoo_schema_snapshot'
|
| 70 |
ODOO_SCHEMA_ENABLEMENTS_KEY = 'odoo_schema_enablements'
|
| 71 |
ODOO_SYNC_STATUS_KEY = 'odoo_sync_status'
|
| 72 |
+
CONNECTOR_HEALTH_FIELD = 'connectorHealth'
|
| 73 |
+
CONNECTOR_HEALTH_STATES = (
|
| 74 |
+
'never_run', 'queued', 'running', 'succeeded', 'partial', 'failed', 'disabled',
|
| 75 |
+
)
|
| 76 |
+
_CONNECTOR_SAFE_ERRORS = {
|
| 77 |
+
'maps_credential_missing': 'Add the scraper credential in Keychain before choosing Run now.',
|
| 78 |
+
'scraper_credential_missing': 'Add the scraper credential in Keychain before choosing Run now.',
|
| 79 |
+
'maps_unbounded_request': 'Google Maps discovery requires an explicit bounded result cap.',
|
| 80 |
+
'maps_invalid_search': 'Google Maps discovery needs a valid category or search.',
|
| 81 |
+
'maps_vendor_error': 'The Maps provider could not complete this run.',
|
| 82 |
+
'maps_schema_drift': 'The Maps provider returned an unreadable result shape.',
|
| 83 |
+
'scraper_run_failed': 'The provider could not complete this run. No result was written.',
|
| 84 |
+
'provider_result_failed': 'The provider returned no usable result for this database.',
|
| 85 |
+
'scraper_result_not_saved': 'The provider answered, but the result could not be saved.',
|
| 86 |
+
'sync_status_not_saved': 'The run did not start because its durable status could not be saved.',
|
| 87 |
+
}
|
| 88 |
# A reserved, derived multi-select field. It is table data backed by durable Zone membership,
|
| 89 |
# not a user-editable column and therefore cannot collide with a field a user creates.
|
| 90 |
ZONE_FIELD_KEY = '__zone__'
|
|
|
|
| 483 |
declared.get('model') or (getattr(spec, 'native', '') if spec else ''))
|
| 484 |
|
| 485 |
|
| 486 |
+
def connector_health_receipt(state, *, last_success=None, source_timestamp=None,
|
| 487 |
+
materialized_timestamp=None, row_count=None, error=None):
|
| 488 |
+
"""Validate the durable, data-free connector database health document."""
|
| 489 |
+
def stamp(value):
|
| 490 |
+
text = str(value or '').strip()
|
| 491 |
+
if not text:
|
| 492 |
+
return None
|
| 493 |
+
try:
|
| 494 |
+
parsed = _dt.datetime.fromisoformat(text.replace('Z', '+00:00'))
|
| 495 |
+
except (TypeError, ValueError, OverflowError):
|
| 496 |
+
return None
|
| 497 |
+
return parsed.isoformat()
|
| 498 |
+
|
| 499 |
+
clean_state = str(state or '').strip()
|
| 500 |
+
if clean_state not in CONNECTOR_HEALTH_STATES:
|
| 501 |
+
raise ValueError('unknown connector database health state')
|
| 502 |
+
count = None
|
| 503 |
+
if row_count is not None:
|
| 504 |
+
if type(row_count) is not int or row_count < 0:
|
| 505 |
+
raise ValueError('row_count must be a non-negative integer or None')
|
| 506 |
+
count = row_count
|
| 507 |
+
clean_error = None
|
| 508 |
+
if error is not None:
|
| 509 |
+
raw = error if isinstance(error, dict) else {}
|
| 510 |
+
requested_code = str(raw.get('code') or '').strip()
|
| 511 |
+
code = requested_code if requested_code in _CONNECTOR_SAFE_ERRORS else 'connector_sync_failed'
|
| 512 |
+
# Never persist provider/exception prose. Only named classifications get specific copy.
|
| 513 |
+
message = _CONNECTOR_SAFE_ERRORS.get(code, 'This database could not be synced.')
|
| 514 |
+
clean_error = {'code': code, 'message': message}
|
| 515 |
+
return {
|
| 516 |
+
'state': clean_state,
|
| 517 |
+
'lastSuccess': stamp(last_success),
|
| 518 |
+
'sourceTimestamp': stamp(source_timestamp),
|
| 519 |
+
'materializedTimestamp': stamp(materialized_timestamp),
|
| 520 |
+
'rowCount': count,
|
| 521 |
+
'error': clean_error,
|
| 522 |
+
}
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
def record_connector_health(table_key, state, *, st=None, last_success=None,
|
| 526 |
+
source_timestamp=None, materialized_timestamp=None,
|
| 527 |
+
row_count=None, error=None):
|
| 528 |
+
"""Persist one data-free database sync receipt beside its governed definition.
|
| 529 |
+
|
| 530 |
+
Provider runners supply already-sanitized error ``code``/``message`` values. This seam
|
| 531 |
+
deliberately accepts neither exceptions nor arbitrary provider payloads, so a failed run can
|
| 532 |
+
survive a process restart without turning the tenant store into a credential or row log.
|
| 533 |
+
"""
|
| 534 |
+
key = str(table_key or '').strip()
|
| 535 |
+
if not key.startswith(KEY_PREFIX):
|
| 536 |
+
raise ValueError('a governed database key is required')
|
| 537 |
+
receipt = connector_health_receipt(
|
| 538 |
+
state, last_success=last_success, source_timestamp=source_timestamp,
|
| 539 |
+
materialized_timestamp=materialized_timestamp, row_count=row_count, error=error)
|
| 540 |
+
|
| 541 |
+
def _set(current):
|
| 542 |
+
document = dict(current or {})
|
| 543 |
+
definition = dict(document.get(key) or {})
|
| 544 |
+
if not definition:
|
| 545 |
+
raise KeyError(key)
|
| 546 |
+
definition[CONNECTOR_HEALTH_FIELD] = receipt
|
| 547 |
+
document[key] = definition
|
| 548 |
+
return document
|
| 549 |
+
|
| 550 |
+
landed = _st(st).update(STORE_KEY, _set, flush='sync')
|
| 551 |
+
return dict((landed.get(key) or {}).get(CONNECTOR_HEALTH_FIELD) or receipt)
|
| 552 |
+
|
| 553 |
+
|
| 554 |
def credential_footprint(entry_type, connection_id, rt=None, st=None):
|
| 555 |
"""Disclose what revoking this ONE credential purges and what its policy keeps.
|
| 556 |
|
platform/core/write_execution.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Request-local server metadata for replaying an explicitly confirmed write.
|
| 2 |
+
|
| 3 |
+
The gateway signs the seed and timestamp with the preview. Revalidation and the
|
| 4 |
+
ordinary writer reuse them so generated ids and timestamps cannot change the diff.
|
| 5 |
+
This does not replace document revision or permission checks. Unscoped UI writes
|
| 6 |
+
continue to use fresh ids and the wall clock.
|
| 7 |
+
"""
|
| 8 |
+
from contextlib import contextmanager
|
| 9 |
+
from contextvars import ContextVar
|
| 10 |
+
from datetime import datetime, timezone
|
| 11 |
+
import time
|
| 12 |
+
import uuid
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
_execution = ContextVar('confirmed_write_execution', default=None)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@contextmanager
|
| 19 |
+
def replay(metadata):
|
| 20 |
+
state = {'seed': str(metadata['seed']), 'timestamp': float(metadata['timestamp']), 'index': 0}
|
| 21 |
+
token = _execution.set(state)
|
| 22 |
+
try:
|
| 23 |
+
yield
|
| 24 |
+
finally:
|
| 25 |
+
_execution.reset(token)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def new_id():
|
| 29 |
+
state = _execution.get()
|
| 30 |
+
if state is None:
|
| 31 |
+
return uuid.uuid4().hex
|
| 32 |
+
# Replace, never mutate, the context value: independently copied async contexts
|
| 33 |
+
# must not share a mutable sequence counter.
|
| 34 |
+
index = state['index']
|
| 35 |
+
_execution.set({**state, 'index': index + 1})
|
| 36 |
+
return uuid.uuid5(uuid.NAMESPACE_URL, f"loopable-write:{state['seed']}:{index}").hex
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def timestamp():
|
| 40 |
+
state = _execution.get()
|
| 41 |
+
return time.time() if state is None else state['timestamp']
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def utcnow():
|
| 45 |
+
return datetime.fromtimestamp(timestamp(), timezone.utc)
|
platform/harness/filter_sql.py
CHANGED
|
@@ -157,6 +157,9 @@ _DATE_CMP = dict(_NUM_CMP)
|
|
| 157 |
#: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
|
| 158 |
#: always-true `1=1` branch references nothing, and must not force HAVING routing).
|
| 159 |
Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
|
|
|
|
|
|
|
|
|
|
| 160 |
|
| 161 |
|
| 162 |
# --------------------------------------------------------------------------- value coercion
|
|
@@ -612,6 +615,10 @@ def _compile_cohort(rule, state):
|
|
| 612 |
|
| 613 |
parts, params = [], []
|
| 614 |
for ids in members:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 615 |
ids = [int(x) for x in ids]
|
| 616 |
if not ids:
|
| 617 |
parts.append('1=0')
|
|
|
|
| 157 |
#: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
|
| 158 |
#: always-true `1=1` branch references nothing, and must not force HAVING routing).
|
| 159 |
Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
|
| 160 |
+
#: HOTFIX 2026-09-23: a cohort set given as a SELECT of ids rather than a list. A server-side
|
| 161 |
+
#: rank answer (Top N, above average ...) over 256k rows is a subquery, never a 128k-id IN list.
|
| 162 |
+
SqlMembership = namedtuple('SqlMembership', 'sql params')
|
| 163 |
|
| 164 |
|
| 165 |
# --------------------------------------------------------------------------- value coercion
|
|
|
|
| 615 |
|
| 616 |
parts, params = [], []
|
| 617 |
for ids in members:
|
| 618 |
+
if isinstance(ids, SqlMembership):
|
| 619 |
+
parts.append(f"COALESCE({id_sql} IN ({ids.sql}), FALSE)")
|
| 620 |
+
params.extend(ids.params)
|
| 621 |
+
continue
|
| 622 |
ids = [int(x) for x in ids]
|
| 623 |
if not ids:
|
| 624 |
parts.append('1=0')
|
platform/harness/mirror_tokens.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The mirror's own change token, in the layer that owns the mirror (SDLC 65 combine).
|
| 2 |
+
|
| 3 |
+
⛔⛔ WHY IT LIVES HERE AND NOT IN `core/sheet_binding.py`, WHERE S65-T03 WROTE IT.
|
| 4 |
+
It reads the analytical mirror: `harness.datastore.ENTITIES` and `harness.runtime.mirror_cursor`.
|
| 5 |
+
`core` never imports up (ARCHITECTURE.md section 1), and `verify_layering.py`'s D-460 leg caught
|
| 6 |
+
the new `core -> harness` edge as this wave's own violation. Moving the body up and registering it
|
| 7 |
+
back down is the fix the gate's first remedy names -- "move it down" read from the other end. The
|
| 8 |
+
alternative, a `GRANDFATHERED_CORE` entry, was REFUSED: that list is edges which PRE-DATE the gate,
|
| 9 |
+
and a wave adding its own edge to it exempts itself from the rule it just broke.
|
| 10 |
+
|
| 11 |
+
⭐ Importing this module is what arms `core.sheet_binding.mirror_change_token`; `routes_sheet`
|
| 12 |
+
does it. Until then that function RAISES, which `database_change_token` already treats as "cannot
|
| 13 |
+
prove unchanged, therefore resolve" -- the safe direction.
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
import re
|
| 17 |
+
|
| 18 |
+
from core import sheet_binding
|
| 19 |
+
from core.sheet_binding import fingerprint
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def mirror_change_token(st, database):
|
| 23 |
+
"""One SQL token read over source tables, not N binding queries or Python row copies.
|
| 24 |
+
|
| 25 |
+
The current mirror has no per-table monotonic revision. Content fingerprints close
|
| 26 |
+
that gap without changing its writer or adding a row cap. Count, xor and sum include
|
| 27 |
+
entire records; schema and sync receipts cover field changes and no-op syncs too.
|
| 28 |
+
Registry metrics depend on the Odoo observation set, so their token includes it.
|
| 29 |
+
"""
|
| 30 |
+
from core import connector_spec
|
| 31 |
+
from harness import datastore
|
| 32 |
+
from harness.runtime import mirror_cursor
|
| 33 |
+
spec = connector_spec.spec_for(database)
|
| 34 |
+
if database in ("customer_data", "product_data"):
|
| 35 |
+
sources = set(datastore.ENTITIES)
|
| 36 |
+
elif spec:
|
| 37 |
+
sources = {spec.source}
|
| 38 |
+
# A connector grid also reads joined label/identity columns declared by its spec.
|
| 39 |
+
sources.update(target.source for _, key, _, _ in spec.links
|
| 40 |
+
if (target := connector_spec.spec_for(key)) is not None)
|
| 41 |
+
else:
|
| 42 |
+
raise RuntimeError("The connector has no declared source for its change token.")
|
| 43 |
+
if not sources or any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", s) for s in sources):
|
| 44 |
+
raise RuntimeError("The connector source identity is invalid.")
|
| 45 |
+
sources = sorted(sources)
|
| 46 |
+
receipt_sources = sorted(set(sources) | ({"meta"} if any(s.startswith("meta_") for s in sources) else set()))
|
| 47 |
+
clauses = [f"SELECT '{source}' AS source, CAST(count(*) AS VARCHAR) AS n, "
|
| 48 |
+
f"CAST(bit_xor(hash(t)) AS VARCHAR) AS x, "
|
| 49 |
+
f"CAST(sum(CAST(hash(t) AS HUGEINT)) AS VARCHAR) AS s FROM \"{source}\" t"
|
| 50 |
+
for source in sources]
|
| 51 |
+
placeholders = ",".join("?" for _ in sources)
|
| 52 |
+
clauses.append("SELECT 'schema', table_name, column_name, data_type FROM duckdb_columns() "
|
| 53 |
+
f"WHERE table_name IN ({placeholders})")
|
| 54 |
+
# _sync_state is the resident mirror writer's receipt, not the app's status cache.
|
| 55 |
+
clauses.append("SELECT 'sync', entity, phase, concat_ws('|', last_id, cursor_wd, rows, updated_at) "
|
| 56 |
+
"FROM _sync_state WHERE split_part(entity, '.', 1) IN ("
|
| 57 |
+
+ ",".join("?" for _ in receipt_sources) + ")")
|
| 58 |
+
rows = mirror_cursor(st).execute(" UNION ALL ".join(clauses) + " ORDER BY 1,2,3,4",
|
| 59 |
+
[*sources, *receipt_sources]).fetchall()
|
| 60 |
+
if not any(row[0] == "sync" for row in rows):
|
| 61 |
+
raise RuntimeError("The connector has no sync receipt.")
|
| 62 |
+
return fingerprint(rows)
|
| 63 |
+
|
| 64 |
+
sheet_binding.set_mirror_token_provider(mirror_change_token)
|