Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- RELEASES.json +7 -1
- VERSION +1 -1
- api/ai_review.py +3 -2
- api/automation_engine.py +68 -0
- api/connectors_amazon.py +132 -1
- api/main.py +175 -3
- api/odoo_relational.py +74 -5
- api/providers.py +2 -1
- api/routes_admin.py +6 -1
- api/routes_automation.py +121 -10
- api/routes_connectors.py +192 -25
- api/routes_customers.py +22 -4
- api/routes_dependents.py +19 -0
- api/routes_external.py +489 -26
- api/routes_grid.py +197 -22
- api/routes_keychain.py +8 -0
- api/routes_module_index.py +132 -0
- api/routes_nav.py +94 -20
- api/routes_odoo_tables.py +310 -306
- api/routes_products.py +23 -3
- api/routes_query.py +326 -29
- api/routes_sheet.py +717 -115
- api/routes_starred.py +196 -19
- api/routes_tables.py +99 -56
- api/routes_templates.py +179 -9
- api/semantic_registry.py +152 -14
- api/test_odoo_rank_refresh.py +65 -0
- platform/aios_grid.py +48 -0
- platform/core/connector_sync.py +133 -2
- platform/core/dependents.py +233 -40
- platform/core/external_access.py +80 -7
- platform/core/external_tools.py +372 -0
- platform/core/measure_resolve.py +4 -2
- platform/core/odoo.py +3 -1
- platform/core/perm_scope.py +17 -4
- platform/core/relations.py +33 -0
- platform/core/shares.py +51 -3
- platform/core/sheet_binding.py +180 -0
- platform/core/sheet_import.py +24 -0
- platform/core/sheet_structure.py +316 -0
- platform/core/sheet_templates.py +111 -5
- platform/core/sheets.py +543 -60
- platform/core/store_pg.py +69 -0
- platform/core/table_store.py +28 -0
- platform/core/user_tables.py +52 -26
- platform/core/windows.py +492 -0
- platform/harness/amazon_client.py +73 -1
- platform/harness/amazon_store.py +430 -21
- platform/harness/measure_filter.py +7 -2
- platform/harness/runtime.py +25 -0
RELEASES.json
CHANGED
|
@@ -1,6 +1,12 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
{
|
| 5 |
"version": "v74",
|
| 6 |
"sha": "40dceba",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "v75 (2c51469)",
|
| 3 |
"releases": [
|
| 4 |
+
{
|
| 5 |
+
"version": "v75",
|
| 6 |
+
"sha": "2c51469",
|
| 7 |
+
"date": "2026-09-19",
|
| 8 |
+
"subject": "release v75"
|
| 9 |
+
},
|
| 10 |
{
|
| 11 |
"version": "v74",
|
| 12 |
"sha": "40dceba",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
v75 (2c51469)
|
api/ai_review.py
CHANGED
|
@@ -174,8 +174,9 @@ def _call_openai(p, model, system, user, timeout):
|
|
| 174 |
|
| 175 |
|
| 176 |
#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any.
|
| 177 |
-
#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so
|
| 178 |
-
#:
|
|
|
|
| 179 |
LAST_ANTHROPIC_TRANSPORT = ""
|
| 180 |
|
| 181 |
|
|
|
|
| 174 |
|
| 175 |
|
| 176 |
#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any.
|
| 177 |
+
#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so an administrator can ask WHICH path ran
|
| 178 |
+
#: rather than infer it from a requirements file: `GET /api/v1/automations/ai/transport`
|
| 179 |
+
#: (`routes_automation.ai_transport`, D-384) serves it beside the installed SDK version.
|
| 180 |
LAST_ANTHROPIC_TRANSPORT = ""
|
| 181 |
|
| 182 |
|
api/automation_engine.py
CHANGED
|
@@ -14553,6 +14553,74 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14553 |
return changes, limits
|
| 14554 |
|
| 14555 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14556 |
def related_product_metric_values(rt, customer_id, products, fields, offer, *,
|
| 14557 |
team_id=None, agent=None, today=None):
|
| 14558 |
"""Read-only customer × real product metrics; the caller supplies walled server rows.
|
|
|
|
| 14553 |
return changes, limits
|
| 14554 |
|
| 14555 |
|
| 14556 |
+
def related_metric_values(rt, table_key, rows, fields, offer, related, *, team_id=None,
|
| 14557 |
+
today=None):
|
| 14558 |
+
"""Read-only linked cells at customer, product or declared entity grain.
|
| 14559 |
+
|
| 14560 |
+
The host supplies the same resolved membership as its rows door and its walled catalogue.
|
| 14561 |
+
A request-local memo never survives a parent, account, permission or tenant change.
|
| 14562 |
+
"""
|
| 14563 |
+
from core import measure_resolve, relations
|
| 14564 |
+
from harness import semantic as sem, windows
|
| 14565 |
+
import time
|
| 14566 |
+
|
| 14567 |
+
rt.assert_datastore_matches()
|
| 14568 |
+
today = today or time.strftime('%Y-%m-%d')
|
| 14569 |
+
by_key = {m['key']: m for m in offer}
|
| 14570 |
+
out = relations.MeasureAnswer({r['pid']: {f['key']: None for f in fields} for r in rows})
|
| 14571 |
+
memo = {}
|
| 14572 |
+
for field in fields:
|
| 14573 |
+
spec = field.get('measure') or {}
|
| 14574 |
+
key = spec.get('key')
|
| 14575 |
+
if key not in by_key:
|
| 14576 |
+
continue # The host removes unavailable catalogue entries before serialization.
|
| 14577 |
+
try:
|
| 14578 |
+
bounds = windows.resolve(spec.get('window'), today)
|
| 14579 |
+
if bounds is None:
|
| 14580 |
+
raise ValueError('The metric date window cannot be resolved. Choose a valid period.')
|
| 14581 |
+
if table_key == 'customer_data':
|
| 14582 |
+
errors = []
|
| 14583 |
+
values = measure_resolve.column_values(
|
| 14584 |
+
[field], team_id, [r['pid'] for r in rows], today, today, memo,
|
| 14585 |
+
on_error=lambda _where, exc: errors.append(exc), related=related)
|
| 14586 |
+
if errors:
|
| 14587 |
+
raise ValueError('This metric could not be read for the linked record. Try again after the database is ready.')
|
| 14588 |
+
out.refusals.extend(getattr(values, 'refusals', ()))
|
| 14589 |
+
for pid, cells in values.items():
|
| 14590 |
+
out[pid].update(cells)
|
| 14591 |
+
continue
|
| 14592 |
+
topic = sem.topic_for_grid(table_key)
|
| 14593 |
+
cache_key = (topic, key, bounds, team_id)
|
| 14594 |
+
if cache_key not in memo:
|
| 14595 |
+
memo[cache_key] = sem.entity_measure_values(
|
| 14596 |
+
topic, [key], date_from=bounds[0], date_to=bounds[1], team_id=team_id,
|
| 14597 |
+
offer=offer, related=related)
|
| 14598 |
+
values = memo[cache_key]
|
| 14599 |
+
markers = getattr(values, 'refusals', ())
|
| 14600 |
+
if markers:
|
| 14601 |
+
out.refusals.extend(dict(marker, field=field['key']) for marker in markers)
|
| 14602 |
+
continue # Never zero-fill a refused answer or expose its unscoped values.
|
| 14603 |
+
for row in rows:
|
| 14604 |
+
identity = row.get('code') if table_key == 'product_data' else row['pid']
|
| 14605 |
+
if identity in (None, ''):
|
| 14606 |
+
raise ValueError('The metric identity cannot be read for this record.')
|
| 14607 |
+
cell = sem.entity_zero_fill(values.get(identity), [key], offer)
|
| 14608 |
+
out[row['pid']][field['key']] = cell.get(key)
|
| 14609 |
+
except Exception as exc:
|
| 14610 |
+
message = (str(exc) if isinstance(exc, (relations.PinRefusal, sem._EntityMeasureError)) else
|
| 14611 |
+
'This metric cannot be read for the linked record. Check its period and source binding, then try again.')
|
| 14612 |
+
out.refusals.append(dict(code='metric_scope_unavailable', message=message,
|
| 14613 |
+
path=related.path, field=field['key'], scoped=False))
|
| 14614 |
+
# Customer resolvers deliberately retain diagnostics even when their low-level pin refuses.
|
| 14615 |
+
for marker in out.refusals:
|
| 14616 |
+
marker['message'] = marker['message'].replace('Values are unscoped.', 'Values are unavailable.').replace(
|
| 14617 |
+
'Metric values are unscoped.', 'Metric values are unavailable.')
|
| 14618 |
+
for cells in out.values():
|
| 14619 |
+
if marker.get('field') in cells:
|
| 14620 |
+
cells[marker['field']] = None
|
| 14621 |
+
return out
|
| 14622 |
+
|
| 14623 |
+
|
| 14624 |
def related_product_metric_values(rt, customer_id, products, fields, offer, *,
|
| 14625 |
team_id=None, agent=None, today=None):
|
| 14626 |
"""Read-only customer × real product metrics; the caller supplies walled server rows.
|
api/connectors_amazon.py
CHANGED
|
@@ -94,6 +94,137 @@ def assert_safe_status(payload):
|
|
| 94 |
return payload
|
| 95 |
|
| 96 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
#: W55-T19 / R7 / C10 — AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
|
| 98 |
#: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and
|
| 99 |
#: its seven databases are enabled at the provider boundary rather than masquerading as Odoo
|
|
@@ -340,7 +471,7 @@ def main(argv):
|
|
| 340 |
return 1
|
| 341 |
print("\nreport types this connector will pull (owner ruling R4):")
|
| 342 |
for table, spec in REPORT_TYPES.items():
|
| 343 |
-
rep = spec.get("report") or "NO CONFIRMED REPORT FLAVOR (see O5)"
|
| 344 |
print(f" {table:<16} {rep}")
|
| 345 |
|
| 346 |
if "--probe-report" in argv:
|
|
|
|
| 94 |
return payload
|
| 95 |
|
| 96 |
|
| 97 |
+
def first_load_status(tenant_key, rt=None):
|
| 98 |
+
"""Writer-side progress helper; request status uses sync_state's read-only cursor."""
|
| 99 |
+
from harness import amazon_store
|
| 100 |
+
return assert_safe_status(amazon_store.first_load_status(tenant_key, rt=rt))
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
| 104 |
+
"""Each Amazon table's sync state, or None when this mirror cannot be read.
|
| 105 |
+
|
| 106 |
+
Read only: never _connect_tenant_mirror or first_load_status, which issue DDL.
|
| 107 |
+
Missing bookkeeping is unknown, not a measured zero. First-load numbers describe the
|
| 108 |
+
whole connection, because that is the scope of the producer's durable progress marker.
|
| 109 |
+
"""
|
| 110 |
+
import json
|
| 111 |
+
from datetime import datetime, timezone
|
| 112 |
+
import duckdb
|
| 113 |
+
from harness import amazon_store as az
|
| 114 |
+
from routes_odoo_tables import _sync_entities, stuck_verdict
|
| 115 |
+
|
| 116 |
+
def stamp(value):
|
| 117 |
+
if not value:
|
| 118 |
+
return None
|
| 119 |
+
try:
|
| 120 |
+
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
| 121 |
+
return (parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)).astimezone(timezone.utc)
|
| 122 |
+
except (ValueError, TypeError, OverflowError):
|
| 123 |
+
return None
|
| 124 |
+
|
| 125 |
+
def iso(value):
|
| 126 |
+
return value.isoformat().replace("+00:00", "Z") if value else None
|
| 127 |
+
|
| 128 |
+
cadence = cadence_seconds if cadence_seconds is not None else 1800
|
| 129 |
+
try:
|
| 130 |
+
path = az._tenant_mirror_path(tenant_key, rt=rt)
|
| 131 |
+
if not path.is_file():
|
| 132 |
+
return assert_safe_status(None)
|
| 133 |
+
# ⛔⛔ A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES
|
| 134 |
+
# `ConnectionException: Can't open a connection to same database file with a different
|
| 135 |
+
# configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global
|
| 136 |
+
# analytical mirror (`datastore.DB_PATH`), so the swallowed refusal below turned this whole
|
| 137 |
+
# screen into a permanent "sync status is unknown" on exactly the deployment the ticket is
|
| 138 |
+
# for. Measured on staging 2026-09-14, W60-T35. `datastore.ro_con()` is the shipped
|
| 139 |
+
# per-thread READ cursor on that same instance, and it takes no lock against the writer.
|
| 140 |
+
from harness import datastore as _ds
|
| 141 |
+
if path.resolve() == _ds.DB_PATH.resolve():
|
| 142 |
+
# `ro_cursor()` is a cursor on the instance this process ALREADY holds: no second
|
| 143 |
+
# connection, no lock contention with the writer, and no DDL. Deliberately NOT
|
| 144 |
+
# `ro_con()`, which refuses until `ready()` and would report the whole screen as
|
| 145 |
+
# unknown whenever the Odoo backfill is mid-flight, and NOT `datastore.connect()`,
|
| 146 |
+
# which issues `CREATE TABLE IF NOT EXISTS`.
|
| 147 |
+
entities = _sync_entities(_ds.ro_cursor(), cadence)
|
| 148 |
+
else:
|
| 149 |
+
with duckdb.connect(str(path), read_only=True) as cur:
|
| 150 |
+
entities = _sync_entities(cur, cadence)
|
| 151 |
+
except Exception: # noqa: BLE001
|
| 152 |
+
# No exception text: paths, provider payloads and credentials never become status copy.
|
| 153 |
+
return assert_safe_status(None)
|
| 154 |
+
if entities is None:
|
| 155 |
+
return assert_safe_status(None)
|
| 156 |
+
|
| 157 |
+
clock = stamp(now) if now is not None else datetime.now(timezone.utc)
|
| 158 |
+
states, bad_ticks = {}, set()
|
| 159 |
+
for table in az.TABLES:
|
| 160 |
+
tick = entities.get(az.entity_key("sync_tick_" + table))
|
| 161 |
+
if tick:
|
| 162 |
+
updated = stamp(tick.get("updatedAt"))
|
| 163 |
+
if updated is None or tick.get("phase") not in ("attempted", "done", "running"):
|
| 164 |
+
bad_ticks.add(table)
|
| 165 |
+
else:
|
| 166 |
+
states[table] = {"state": tick["phase"], "updated_at": iso(updated)}
|
| 167 |
+
# An unreadable input cannot support a cadence promise for any table in the rotation.
|
| 168 |
+
plan = az.sync_plan(states, now=clock) if not bad_ticks and clock is not None else None
|
| 169 |
+
first_load, load_unknown = None, ""
|
| 170 |
+
load = entities.get(az.entity_key(az.FIRST_LOAD_STATUS_KEY))
|
| 171 |
+
if load:
|
| 172 |
+
try:
|
| 173 |
+
progress = json.loads(load.get("cursor") or "")
|
| 174 |
+
remaining, eta = progress["remaining"], progress["etaSeconds"]
|
| 175 |
+
if (type(remaining) is not int or not 0 <= remaining <= 2**53 - 1
|
| 176 |
+
or type(eta) is not int or not 0 <= eta <= 2**53 - 1
|
| 177 |
+
or (load.get("phase") == "done" and remaining > 0)):
|
| 178 |
+
raise ValueError("Invalid first-load progress")
|
| 179 |
+
if remaining > 0:
|
| 180 |
+
first_load = {"remaining": remaining, "etaSeconds": eta,
|
| 181 |
+
"cause": az.first_load_eta_cause(remaining),
|
| 182 |
+
"needsAttention": load.get("phase") == "failed"}
|
| 183 |
+
except (ValueError, TypeError, KeyError, OverflowError, RecursionError):
|
| 184 |
+
load_unknown = "First-load progress is unknown because its saved marker could not be read. Retry sync to refresh it."
|
| 185 |
+
elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"):
|
| 186 |
+
load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass."
|
| 187 |
+
|
| 188 |
+
answer = {}
|
| 189 |
+
for table in az.TABLES:
|
| 190 |
+
landed = entities.get(az.entity_key(table)) or {}
|
| 191 |
+
tick = entities.get(az.entity_key("sync_tick_" + table)) or {}
|
| 192 |
+
watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {}
|
| 193 |
+
rows = landed.get("rows")
|
| 194 |
+
if type(rows) is not int or rows < 0:
|
| 195 |
+
rows = None
|
| 196 |
+
successes = [when for marker in (landed, tick, watermark)
|
| 197 |
+
if marker.get("phase") == "done"
|
| 198 |
+
and (when := stamp(marker.get("updatedAt"))) is not None]
|
| 199 |
+
stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or ""
|
| 200 |
+
if tick.get("phase") == "attempted":
|
| 201 |
+
# The shared stall helper calls an active phase 'running'; Amazon calls it 'attempted'.
|
| 202 |
+
updated = stamp(tick.get("updatedAt"))
|
| 203 |
+
stuck, stuck_cause = stuck_verdict(
|
| 204 |
+
"running", iso(updated), cadence,
|
| 205 |
+
now=clock.replace(tzinfo=None) if clock else None)
|
| 206 |
+
reason = plan["deferred"].get(table) if plan else None
|
| 207 |
+
# Only the planner's explicit due stamp is a due date. Minimum stagger spacing is not
|
| 208 |
+
# a promise about when a later resident tick will select this table.
|
| 209 |
+
next_due = stamp(reason.partition("next due ")[2]) if reason else None
|
| 210 |
+
unknown = [load_unknown] if load_unknown else []
|
| 211 |
+
if rows is None:
|
| 212 |
+
unknown.append("This container has no readable row count for this table.")
|
| 213 |
+
if plan is None:
|
| 214 |
+
unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.")
|
| 215 |
+
answer[table] = {
|
| 216 |
+
"rowsLoaded": rows,
|
| 217 |
+
"lastSuccess": iso(max(successes)) if successes else None,
|
| 218 |
+
"inProgress": tick.get("phase") == "attempted" and stuck is False,
|
| 219 |
+
"deferred": {"reason": reason} if reason else None,
|
| 220 |
+
"nextDue": iso(next_due),
|
| 221 |
+
"firstLoad": first_load,
|
| 222 |
+
"stuck": stuck, "stuckCause": stuck_cause,
|
| 223 |
+
"unknownCause": " ".join(unknown),
|
| 224 |
+
}
|
| 225 |
+
return assert_safe_status(answer)
|
| 226 |
+
|
| 227 |
+
|
| 228 |
#: W55-T19 / R7 / C10 — AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
|
| 229 |
#: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and
|
| 230 |
#: its seven databases are enabled at the provider boundary rather than masquerading as Odoo
|
|
|
|
| 471 |
return 1
|
| 472 |
print("\nreport types this connector will pull (owner ruling R4):")
|
| 473 |
for table, spec in REPORT_TYPES.items():
|
| 474 |
+
rep = spec.get("report") or spec.get("report_discover") or "NO CONFIRMED REPORT FLAVOR (see O5)"
|
| 475 |
print(f" {table:<16} {rep}")
|
| 476 |
|
| 477 |
if "--probe-report" in argv:
|
api/main.py
CHANGED
|
@@ -20,6 +20,7 @@ session's tenant claim resolves through `harness.runtime.get_runtime`, which is
|
|
| 20 |
is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB
|
| 21 |
commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
|
| 22 |
"""
|
|
|
|
| 23 |
import os
|
| 24 |
import sys
|
| 25 |
from pathlib import Path
|
|
@@ -273,7 +274,7 @@ def _view_revision_unverifiable(request: Request, exc: view_audit.RevisionUnveri
|
|
| 273 |
|
| 274 |
|
| 275 |
@app.get("/api/health")
|
| 276 |
-
def health():
|
| 277 |
"""Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
|
| 278 |
reveal anything about the deployment beyond "the process is up". No version, no tenant list,
|
| 279 |
no config: a health endpoint is the one URL every scanner finds first.
|
|
@@ -282,7 +283,17 @@ def health():
|
|
| 282 |
release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated
|
| 283 |
caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind
|
| 284 |
a session — and the authoritative copy is the `VERSION` file in the Space repo, which
|
| 285 |
-
`deploy_web.space_version()` reads without needing the app to be up at all.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
return {"ok": True}
|
| 287 |
|
| 288 |
|
|
@@ -398,6 +409,7 @@ app.include_router(routes_charts.router)
|
|
| 398 |
# and that router, so this is the one mount in this block that crosses no fence. It is still
|
| 399 |
# written HERE rather than at the end, for the reason the block below spells out at length.
|
| 400 |
app.include_router(routes_sheet.router)
|
|
|
|
| 401 |
# ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the
|
| 402 |
# FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses.
|
| 403 |
# ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own
|
|
@@ -766,6 +778,158 @@ def _pull_amazon(why):
|
|
| 766 |
return False
|
| 767 |
|
| 768 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 769 |
def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations=False):
|
| 770 |
"""Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used.
|
| 771 |
|
|
@@ -1320,6 +1484,7 @@ def _store_resync_loop():
|
|
| 1320 |
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
| 1321 |
refreshed pool re-reads the freshly synced store."""
|
| 1322 |
import time as _t
|
|
|
|
| 1323 |
passes = 0
|
| 1324 |
last_relational_success_at = None
|
| 1325 |
last_meta_fingerprint = None
|
|
@@ -1388,10 +1553,13 @@ def _store_resync_loop():
|
|
| 1388 |
# stored fields such as customer_rank can change without partner write_date advancing,
|
| 1389 |
# so staging served a provably stale customer population for its first hour. Subsequent
|
| 1390 |
# passes still sleep at the freshly resolved cadence, and manual still skips only Odoo.
|
|
|
|
| 1391 |
if passes:
|
| 1392 |
-
|
|
|
|
| 1393 |
passes += 1
|
| 1394 |
RESYNC_STATE["passes"] = passes
|
|
|
|
| 1395 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
|
| 1396 |
RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
|
| 1397 |
RESYNC_STATE["lastPassOk"] = _err is None
|
|
@@ -1444,6 +1612,10 @@ def _store_resync_loop():
|
|
| 1444 |
_amzn_previous = last_amzn_fingerprint if _meta_reason is None else None
|
| 1445 |
_reason = _resync_rebuild_reason(sync_report, delete_report,
|
| 1446 |
last_relational_success_at, _meta_now)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1447 |
_meta_state = {}
|
| 1448 |
if _reason is not None:
|
| 1449 |
print(f"[aios-api] odoo relational rebuild required ({_reason})")
|
|
|
|
| 20 |
is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB
|
| 21 |
commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
|
| 22 |
"""
|
| 23 |
+
import hmac
|
| 24 |
import os
|
| 25 |
import sys
|
| 26 |
from pathlib import Path
|
|
|
|
| 274 |
|
| 275 |
|
| 276 |
@app.get("/api/health")
|
| 277 |
+
def health(challenge: str = ""):
|
| 278 |
"""Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
|
| 279 |
reveal anything about the deployment beyond "the process is up". No version, no tenant list,
|
| 280 |
no config: a health endpoint is the one URL every scanner finds first.
|
|
|
|
| 283 |
release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated
|
| 284 |
caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind
|
| 285 |
a session — and the authoritative copy is the `VERSION` file in the Space repo, which
|
| 286 |
+
`deploy_web.space_version()` reads without needing the app to be up at all.
|
| 287 |
+
|
| 288 |
+
W-265 adds a possession challenge for the deployer. It reveals only whether this process was
|
| 289 |
+
born after one high-entropy, single-deploy nonce was stamped; it never returns the nonce,
|
| 290 |
+
version, Space id, or configuration. A normal liveness request keeps the exact
|
| 291 |
+
``{"ok": true}`` response shape.
|
| 292 |
+
"""
|
| 293 |
+
if challenge:
|
| 294 |
+
expected = os.environ.get("AIOS_DEPLOY_NONCE") or ""
|
| 295 |
+
return {"ok": True,
|
| 296 |
+
"deployReady": bool(expected and hmac.compare_digest(challenge, expected))}
|
| 297 |
return {"ok": True}
|
| 298 |
|
| 299 |
|
|
|
|
| 409 |
# and that router, so this is the one mount in this block that crosses no fence. It is still
|
| 410 |
# written HERE rather than at the end, for the reason the block below spells out at length.
|
| 411 |
app.include_router(routes_sheet.router)
|
| 412 |
+
import routes_module_index; app.include_router(routes_module_index.router) # W61-T20 / C6: App and Agents folder index
|
| 413 |
# ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the
|
| 414 |
# FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses.
|
| 415 |
# ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own
|
|
|
|
| 778 |
return False
|
| 779 |
|
| 780 |
|
| 781 |
+
class _SyncNowRefused(Exception):
|
| 782 |
+
"""An application-authored, credential-free sentence for a Sync now receipt."""
|
| 783 |
+
|
| 784 |
+
|
| 785 |
+
def _sync_now_failure(cause):
|
| 786 |
+
"""Classify actual provider failures without returning vendor text or exception reprs."""
|
| 787 |
+
import re
|
| 788 |
+
text = str(cause).lower()
|
| 789 |
+
causes = (
|
| 790 |
+
(("reduce the amount of data", "response too large"),
|
| 791 |
+
"The provider refused the pull because the response contains too much data."),
|
| 792 |
+
(("no ad accounts",), "The Meta token cannot access any ad accounts."),
|
| 793 |
+
(("max_pages", "page cap"), "The pull stopped at the provider page limit."),
|
| 794 |
+
(("sandbox",), "Amazon sync requires a production credential; this credential is for the sandbox."),
|
| 795 |
+
(("deletion", "retention"), "Amazon sync is blocked by an outstanding data deletion requirement."),
|
| 796 |
+
(("no stored", "no odoo credentials", "access_token is not set"),
|
| 797 |
+
"This tenant has no usable credential for the connector."),
|
| 798 |
+
(("enablement document", "capability unavailable"),
|
| 799 |
+
"The connector database settings could not be read."),
|
| 800 |
+
(("throttl", "rate limit", "too many requests"),
|
| 801 |
+
"The provider rate limit stopped the sync. Retry after the provider delay."),
|
| 802 |
+
(("timed out", "timeout"), "The sync failed because the provider request timed out."),
|
| 803 |
+
(("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
|
| 804 |
+
"expired", "authentication"), "The provider refused the connector credential or its permissions."),
|
| 805 |
+
(("duplicate key", "primary key", "constraint"),
|
| 806 |
+
"The mirror rejected rows because they violate a database constraint."),
|
| 807 |
+
(("connection", "network", "name resolution"),
|
| 808 |
+
"The sync could not connect to the provider or tenant database."),
|
| 809 |
+
# A-META / D-201: a Meta pull on a container whose mirror never held `meta_*` raises
|
| 810 |
+
# DuckDB's CatalogException. Standing rule 1 wants the CAUSE, not "unexpected error":
|
| 811 |
+
# "the loader is broken" and "this server has never held these tables" are different
|
| 812 |
+
# operator actions, and only the second one is true here.
|
| 813 |
+
(("does not exist", "no such table", "catalog error"),
|
| 814 |
+
"This server's mirror does not hold this connector's tables yet, so there was nothing "
|
| 815 |
+
"to update. Run the connector's first load on the server that owns this tenant."),
|
| 816 |
+
)
|
| 817 |
+
for needles, sentence in causes:
|
| 818 |
+
if any(needle in text for needle in needles):
|
| 819 |
+
return sentence
|
| 820 |
+
status = re.search(r"\bhttp\s+(\d{3})\b", text)
|
| 821 |
+
if status:
|
| 822 |
+
return f"The provider refused the sync request with HTTP status {status.group(1)}."
|
| 823 |
+
if isinstance(cause, TypeError):
|
| 824 |
+
return "The connector failed because its provider call used incompatible arguments."
|
| 825 |
+
if isinstance(cause, (KeyError, ValueError)):
|
| 826 |
+
return "The connector could not read an expected value in the sync response."
|
| 827 |
+
return "The connector encountered an unexpected error while processing the sync response."
|
| 828 |
+
|
| 829 |
+
|
| 830 |
+
def _run_sync_requests():
|
| 831 |
+
"""Drain Sync now intents on the resident resync thread, the mirror's single writer.
|
| 832 |
+
|
| 833 |
+
D-29 / intake F6: HTTP handlers only queue intents. Provider calls stay here, before the
|
| 834 |
+
resident projection pass. A failed request cannot discard another tenant's queued work.
|
| 835 |
+
"""
|
| 836 |
+
from core import connector_sync
|
| 837 |
+
odoo_requested = False
|
| 838 |
+
for receipt in connector_sync.take_sync_requests():
|
| 839 |
+
provider, tenant = receipt["provider"], receipt["tenant"]
|
| 840 |
+
ok, detail = False, ""
|
| 841 |
+
first_load_progress, continuing = None, False
|
| 842 |
+
try:
|
| 843 |
+
from harness import runtime, datastore
|
| 844 |
+
rt = runtime.get_runtime(tenant)
|
| 845 |
+
if rt.key != tenant:
|
| 846 |
+
raise _SyncNowRefused("The sync runtime does not match the requesting tenant.")
|
| 847 |
+
# These two loaders use a process-global mirror. Never repoint it under readers.
|
| 848 |
+
# Amazon's writer opens an explicit tenant file and does not need this restriction.
|
| 849 |
+
if provider in ("odoo", "meta_ads") and (
|
| 850 |
+
rt.datastore_path.resolve() != datastore.DB_PATH.resolve() or
|
| 851 |
+
datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()):
|
| 852 |
+
raise _SyncNowRefused(
|
| 853 |
+
"This tenant's mirror is not resident on this server. Sync requires its tenant worker.")
|
| 854 |
+
if provider == "odoo":
|
| 855 |
+
import core.odoo as odoo
|
| 856 |
+
import odoo_relational as rel
|
| 857 |
+
if rt.odoo_paused() or rel.frozen(rt):
|
| 858 |
+
raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.")
|
| 859 |
+
source = rt.odoo_source()
|
| 860 |
+
if source is None:
|
| 861 |
+
raise _SyncNowRefused("This tenant has no Odoo credential. Connect Odoo before syncing.")
|
| 862 |
+
odoo_requested = True
|
| 863 |
+
previous_client = getattr(odoo._tlocal, "client", None)
|
| 864 |
+
previous_context = getattr(odoo._tlocal, "tenant_ctx", None)
|
| 865 |
+
try:
|
| 866 |
+
# Raw mirror pulls use the resolved tenant credential and a dedicated
|
| 867 |
+
# transport. Do not borrow a request's cached XML-RPC client.
|
| 868 |
+
odoo._tlocal.client = odoo.OdooClient(creds=source._creds)
|
| 869 |
+
odoo._tlocal.tenant_ctx = None
|
| 870 |
+
report = datastore.sync_all(log=lambda *a, **k: None)
|
| 871 |
+
finally:
|
| 872 |
+
odoo._tlocal.client = previous_client
|
| 873 |
+
odoo._tlocal.tenant_ctx = previous_context
|
| 874 |
+
if not report:
|
| 875 |
+
raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
|
| 876 |
+
if any(row.get("phase") != "live" for row in report.values()):
|
| 877 |
+
raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
|
| 878 |
+
problems = []
|
| 879 |
+
elif provider == "amazon_sp":
|
| 880 |
+
from harness import amazon_store as amazon
|
| 881 |
+
if not amazon.credential(tenant, rt=rt):
|
| 882 |
+
raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
|
| 883 |
+
first_load = amazon.needs_first_load(tenant, rt=rt)
|
| 884 |
+
options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS} if first_load else {}
|
| 885 |
+
report = amazon.sync(tenant, rt=rt, scheduled=False, **options, log=lambda *a, **k: None)
|
| 886 |
+
from connectors_amazon import assert_safe_status
|
| 887 |
+
first_load_progress = assert_safe_status(report.get("firstLoad"))
|
| 888 |
+
continuing = report.get("state") == "partial"
|
| 889 |
+
problems = list(report.get("problems") or []) + list(report.get("audit_problems") or [])
|
| 890 |
+
tables = report.get("tables") or {}
|
| 891 |
+
if not problems and not tables and not continuing:
|
| 892 |
+
raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.")
|
| 893 |
+
if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()):
|
| 894 |
+
raise _SyncNowRefused("All Amazon databases are disabled. Enable a database before syncing.")
|
| 895 |
+
elif provider == "meta_ads":
|
| 896 |
+
from harness import meta_store as meta
|
| 897 |
+
if not meta.token(rt):
|
| 898 |
+
raise _SyncNowRefused("Meta Ads has no access token. Connect Meta Ads before syncing.")
|
| 899 |
+
report = meta.sync(tenant, log=lambda *a, **k: None, insights_days=7)
|
| 900 |
+
problems = list(report.get("problems") or [])
|
| 901 |
+
if any(row.get("capped") for row in (report.get("tables") or {}).values()):
|
| 902 |
+
problems.append("The pull hit the page cap.")
|
| 903 |
+
if not problems and not report.get("tables"):
|
| 904 |
+
raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.")
|
| 905 |
+
else:
|
| 906 |
+
raise _SyncNowRefused("This connector has no Sync now runner.")
|
| 907 |
+
if continuing:
|
| 908 |
+
ok = True
|
| 909 |
+
detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; "
|
| 910 |
+
f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.")
|
| 911 |
+
if problems:
|
| 912 |
+
detail += " " + _sync_now_failure(problems[0])
|
| 913 |
+
elif problems:
|
| 914 |
+
detail = _sync_now_failure(problems[0])
|
| 915 |
+
else:
|
| 916 |
+
ok, detail = True, "Sync finished."
|
| 917 |
+
except _SyncNowRefused as exc:
|
| 918 |
+
detail = str(exc)
|
| 919 |
+
except Exception as exc: # noqa: BLE001
|
| 920 |
+
detail = _sync_now_failure(exc)
|
| 921 |
+
if first_load_progress is not None:
|
| 922 |
+
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail,
|
| 923 |
+
first_load=first_load_progress)
|
| 924 |
+
else:
|
| 925 |
+
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail)
|
| 926 |
+
if continuing and ok:
|
| 927 |
+
# Finish first: the desk refuses a second in-flight intent. Return to the resident
|
| 928 |
+
# loop so its Odoo mirror pass runs before this continuation is claimed.
|
| 929 |
+
connector_sync.request_sync("amazon_sp", tenant, requested_by="first load")
|
| 930 |
+
return odoo_requested
|
| 931 |
+
|
| 932 |
+
|
| 933 |
def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations=False):
|
| 934 |
"""Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used.
|
| 935 |
|
|
|
|
| 1484 |
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
| 1485 |
refreshed pool re-reads the freshly synced store."""
|
| 1486 |
import time as _t
|
| 1487 |
+
from core import connector_sync
|
| 1488 |
passes = 0
|
| 1489 |
last_relational_success_at = None
|
| 1490 |
last_meta_fingerprint = None
|
|
|
|
| 1553 |
# stored fields such as customer_rank can change without partner write_date advancing,
|
| 1554 |
# so staging served a provably stale customer population for its first hour. Subsequent
|
| 1555 |
# passes still sleep at the freshly resolved cadence, and manual still skips only Odoo.
|
| 1556 |
+
# The wait is now interruptible so an administrator's Sync now wakes this writer.
|
| 1557 |
if passes:
|
| 1558 |
+
connector_sync.sync_wake_event().wait(_every)
|
| 1559 |
+
connector_sync.sync_wake_event().clear()
|
| 1560 |
passes += 1
|
| 1561 |
RESYNC_STATE["passes"] = passes
|
| 1562 |
+
_odoo_requested = _run_sync_requests()
|
| 1563 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
|
| 1564 |
RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
|
| 1565 |
RESYNC_STATE["lastPassOk"] = _err is None
|
|
|
|
| 1612 |
_amzn_previous = last_amzn_fingerprint if _meta_reason is None else None
|
| 1613 |
_reason = _resync_rebuild_reason(sync_report, delete_report,
|
| 1614 |
last_relational_success_at, _meta_now)
|
| 1615 |
+
# The scheduled pass may now be quiet because Sync now just advanced this mirror.
|
| 1616 |
+
# Its zero-change report cannot suppress projection of the explicit Odoo pull.
|
| 1617 |
+
if _odoo_requested:
|
| 1618 |
+
_reason = "sync now"
|
| 1619 |
_meta_state = {}
|
| 1620 |
if _reason is not None:
|
| 1621 |
print(f"[aios-api] odoo relational rebuild required ({_reason})")
|
api/odoo_relational.py
CHANGED
|
@@ -1876,6 +1876,9 @@ def authoritative_schema_snapshot():
|
|
| 1876 |
MODEL_CATALOG_KEY = "odoo_readable_model_catalog"
|
| 1877 |
MODEL_CATALOG_VERSION = 1
|
| 1878 |
MODEL_READ_PAGE = 2000
|
|
|
|
|
|
|
|
|
|
| 1879 |
MODEL_CATALOG_TTL = _dt.timedelta(hours=6)
|
| 1880 |
_MODEL_FIELD_TYPES = {
|
| 1881 |
"boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency",
|
|
@@ -3776,7 +3779,7 @@ def _model_cell(value):
|
|
| 3776 |
return value
|
| 3777 |
|
| 3778 |
|
| 3779 |
-
def _read_model_rows(client, catalogue_row):
|
| 3780 |
"""Read one explicitly enabled model in stable pages; every call is a read method."""
|
| 3781 |
model = str(catalogue_row.get("model") or "")
|
| 3782 |
metadata = catalogue_row.get("fields") or {}
|
|
@@ -3792,19 +3795,76 @@ def _read_model_rows(client, catalogue_row):
|
|
| 3792 |
continue
|
| 3793 |
rows.append({"_id": str(rid),
|
| 3794 |
**{name: _model_cell((item or {}).get(name)) for name in fields}})
|
| 3795 |
-
|
|
|
|
|
|
|
|
|
|
| 3796 |
break
|
| 3797 |
offset += len(page)
|
| 3798 |
return rows
|
| 3799 |
|
| 3800 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3801 |
def sync_discovered_models(rt, source_connection_id=None, namespace=None,
|
| 3802 |
-
username="automation", today=None, table_keys=None):
|
| 3803 |
-
"""Materialise
|
| 3804 |
|
| 3805 |
Disabled models are absent from this pass rather than represented by an empty row set. That
|
| 3806 |
distinction is the preservation guarantee: no updater touches their definition, rows, fields,
|
| 3807 |
grants, ownership, or tenant rename, and re-enabling resumes against the same stable key.
|
|
|
|
|
|
|
| 3808 |
"""
|
| 3809 |
requested = set(table_keys) if table_keys is not None else None
|
| 3810 |
if requested is not None and not requested:
|
|
@@ -3834,6 +3894,10 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None,
|
|
| 3834 |
from core.odoo import get_odoo
|
| 3835 |
with source._bound(): # tenant credential + scope
|
| 3836 |
client = get_odoo()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3837 |
for choice, key in zip(selected, keys):
|
| 3838 |
model = str(choice["model"])
|
| 3839 |
row = by_model[model]
|
|
@@ -3936,7 +4000,7 @@ def available(rt=None, tenant_key=None, connection_id=None):
|
|
| 3936 |
|
| 3937 |
|
| 3938 |
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
|
| 3939 |
-
all_connections=False, table_keys=None):
|
| 3940 |
"""THE entry point — the store-resync path and the route both call this.
|
| 3941 |
|
| 3942 |
⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row
|
|
@@ -4046,6 +4110,11 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
|
|
| 4046 |
# Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no
|
| 4047 |
# longer built) SAYS SO, with the keys.
|
| 4048 |
plan_report = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4049 |
written = apply_plan(rt, built, username=username, today=today, report=plan_report,
|
| 4050 |
connection_id=ns, retire=requested is None)
|
| 4051 |
# The fixed mirror roster above and the discovered long tail deliberately use separate
|
|
|
|
| 1876 |
MODEL_CATALOG_KEY = "odoo_readable_model_catalog"
|
| 1877 |
MODEL_CATALOG_VERSION = 1
|
| 1878 |
MODEL_READ_PAGE = 2000
|
| 1879 |
+
# Inline rows rewrite the whole store document, so batch publications after the first page.
|
| 1880 |
+
MODEL_PUBLISH_PAGES = 5
|
| 1881 |
+
MODEL_PUBLISH_SECONDS = 10
|
| 1882 |
MODEL_CATALOG_TTL = _dt.timedelta(hours=6)
|
| 1883 |
_MODEL_FIELD_TYPES = {
|
| 1884 |
"boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency",
|
|
|
|
| 3779 |
return value
|
| 3780 |
|
| 3781 |
|
| 3782 |
+
def _read_model_rows(client, catalogue_row, *, on_page=None):
|
| 3783 |
"""Read one explicitly enabled model in stable pages; every call is a read method."""
|
| 3784 |
model = str(catalogue_row.get("model") or "")
|
| 3785 |
metadata = catalogue_row.get("fields") or {}
|
|
|
|
| 3795 |
continue
|
| 3796 |
rows.append({"_id": str(rid),
|
| 3797 |
**{name: _model_cell((item or {}).get(name)) for name in fields}})
|
| 3798 |
+
final = len(page or ()) < MODEL_READ_PAGE
|
| 3799 |
+
if on_page is not None:
|
| 3800 |
+
on_page(rows, final)
|
| 3801 |
+
if final:
|
| 3802 |
break
|
| 3803 |
offset += len(page)
|
| 3804 |
return rows
|
| 3805 |
|
| 3806 |
|
| 3807 |
+
def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, today,
|
| 3808 |
+
on_progress):
|
| 3809 |
+
"""Publish a single job's growing prefix; source reads and progress have no row ceiling."""
|
| 3810 |
+
ut = _ut()
|
| 3811 |
+
key, model = choice["key"], choice["model"]
|
| 3812 |
+
fields, stamp = _model_field_specs(catalogue_row), today or _iso_today()
|
| 3813 |
+
# A bounded census is never a denominator. Failure to count does not prevent row reads.
|
| 3814 |
+
try:
|
| 3815 |
+
total = client.search_count(model, [])
|
| 3816 |
+
except Exception: # noqa: BLE001
|
| 3817 |
+
total = None
|
| 3818 |
+
if isinstance(total, bool) or not isinstance(total, int) or total < 0:
|
| 3819 |
+
total = None
|
| 3820 |
+
|
| 3821 |
+
stored_n = len(((rt.get(ut.STORE_KEY) or {}).get(key) or {}).get("rows") or {})
|
| 3822 |
+
if total is not None and stored_n and total < stored_n * MAX_SHRINK:
|
| 3823 |
+
raise Refused(f"{model}: source count is {total} rows against {stored_n} stored. "
|
| 3824 |
+
"The prior database was preserved. Check Odoo read access and retry sync.")
|
| 3825 |
+
|
| 3826 |
+
written, published_rows, pages_since_publish = {}, 0, 0
|
| 3827 |
+
last_publish = _time.monotonic()
|
| 3828 |
+
|
| 3829 |
+
def on_page(rows, final):
|
| 3830 |
+
nonlocal published_rows, pages_since_publish, last_publish
|
| 3831 |
+
pages_since_publish += 1
|
| 3832 |
+
if (not written or final or pages_since_publish >= MODEL_PUBLISH_PAGES
|
| 3833 |
+
or _time.monotonic() - last_publish >= MODEL_PUBLISH_SECONDS):
|
| 3834 |
+
def apply(current):
|
| 3835 |
+
current = current if isinstance(current, dict) else {}
|
| 3836 |
+
written[key] = _ensure_table_inplace(
|
| 3837 |
+
current, key, choice["label"], fields, rows, username, stamp)
|
| 3838 |
+
from core import connector_sync
|
| 3839 |
+
current[key]["connector"] = {
|
| 3840 |
+
**connector_sync.database_identity(
|
| 3841 |
+
"odoo", connection_for_table_key(key, rt), key, model),
|
| 3842 |
+
"key": "odoo", "readOnly": True}
|
| 3843 |
+
return current
|
| 3844 |
+
|
| 3845 |
+
# The caller supplies its fenced runtime. A superseded write must propagate.
|
| 3846 |
+
rt.update(ut.STORE_KEY, apply, flush="sync")
|
| 3847 |
+
ut.register_connected(key)
|
| 3848 |
+
published_rows = written[key]["rows"]
|
| 3849 |
+
pages_since_publish, last_publish = 0, _time.monotonic()
|
| 3850 |
+
on_progress(published_rows, total)
|
| 3851 |
+
|
| 3852 |
+
values = _read_model_rows(client, catalogue_row, on_page=on_page)
|
| 3853 |
+
if total is not None and len(values) < total:
|
| 3854 |
+
raise Refused("Odoo returned fewer rows than its count. "
|
| 3855 |
+
"Retry sync to load the remaining rows.")
|
| 3856 |
+
return {"tables": written, "errors": []}
|
| 3857 |
+
|
| 3858 |
+
|
| 3859 |
def sync_discovered_models(rt, source_connection_id=None, namespace=None,
|
| 3860 |
+
username="automation", today=None, table_keys=None, on_progress=None):
|
| 3861 |
+
"""Materialise explicitly enabled long-tail models through the tenant's store.
|
| 3862 |
|
| 3863 |
Disabled models are absent from this pass rather than represented by an empty row set. That
|
| 3864 |
distinction is the preservation guarantee: no updater touches their definition, rows, fields,
|
| 3865 |
grants, ownership, or tenant rename, and re-enabling resumes against the same stable key.
|
| 3866 |
+
A single-database job supplies on_progress and publishes growing prefixes. Other callers
|
| 3867 |
+
retain the all-model atomic publication and its shrink guards.
|
| 3868 |
"""
|
| 3869 |
requested = set(table_keys) if table_keys is not None else None
|
| 3870 |
if requested is not None and not requested:
|
|
|
|
| 3894 |
from core.odoo import get_odoo
|
| 3895 |
with source._bound(): # tenant credential + scope
|
| 3896 |
client = get_odoo()
|
| 3897 |
+
if requested is not None and len(requested) == 1 and on_progress is not None:
|
| 3898 |
+
return _publish_discovered_model(
|
| 3899 |
+
rt, client, selected[0], by_model[str(selected[0]["model"])],
|
| 3900 |
+
username=username, today=today, on_progress=on_progress)
|
| 3901 |
for choice, key in zip(selected, keys):
|
| 3902 |
model = str(choice["model"])
|
| 3903 |
row = by_model[model]
|
|
|
|
| 4000 |
|
| 4001 |
|
| 4002 |
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
|
| 4003 |
+
all_connections=False, table_keys=None, on_apply=None):
|
| 4004 |
"""THE entry point — the store-resync path and the route both call this.
|
| 4005 |
|
| 4006 |
⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row
|
|
|
|
| 4110 |
# Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no
|
| 4111 |
# longer built) SAYS SO, with the keys.
|
| 4112 |
plan_report = {}
|
| 4113 |
+
if on_apply is not None:
|
| 4114 |
+
try:
|
| 4115 |
+
on_apply()
|
| 4116 |
+
except Exception: # noqa: BLE001
|
| 4117 |
+
pass # Optional status reporting cannot fail a valid plan's publication.
|
| 4118 |
written = apply_plan(rt, built, username=username, today=today, report=plan_report,
|
| 4119 |
connection_id=ns, retire=requested is None)
|
| 4120 |
# The fixed mirror roster above and the discovered long tail deliberately use separate
|
api/providers.py
CHANGED
|
@@ -641,7 +641,8 @@ def anthropic_send(req, timeout=None, use_sdk=None):
|
|
| 641 |
⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader
|
| 642 |
who cannot tell which path ran cannot tell whether the requirements pin reached the container
|
| 643 |
[[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on
|
| 644 |
-
`ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for
|
|
|
|
| 645 |
check keyed to it passes on a stale value written by an earlier check in the same process.
|
| 646 |
|
| 647 |
`status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error —
|
|
|
|
| 641 |
⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader
|
| 642 |
who cannot tell which path ran cannot tell whether the requirements pin reached the container
|
| 643 |
[[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on
|
| 644 |
+
`ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for the admin view
|
| 645 |
+
`GET /api/v1/automations/ai/transport` (D-384), and a
|
| 646 |
check keyed to it passes on a stale value written by an earlier check in the same process.
|
| 647 |
|
| 648 |
`status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error —
|
api/routes_admin.py
CHANGED
|
@@ -1956,10 +1956,15 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non
|
|
| 1956 |
# client, or a copy of a record written before this ticket, must not read as a
|
| 1957 |
# revocation. `perm_scope.may_metrics` applies the identical rule on the read side, so
|
| 1958 |
# the validator and the wall cannot disagree about what an absent key means.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1959 |
out[key] = {"access": bool(raw.get("access", True)),
|
| 1960 |
"filter": clean_tree,
|
| 1961 |
"hiddenFields": sorted(submitted),
|
| 1962 |
-
"metrics":
|
| 1963 |
_refuse_unshapeable_bu(out)
|
| 1964 |
return out
|
| 1965 |
|
|
|
|
| 1956 |
# client, or a copy of a record written before this ticket, must not read as a
|
| 1957 |
# revocation. `perm_scope.may_metrics` applies the identical rule on the read side, so
|
| 1958 |
# the validator and the wall cannot disagree about what an absent key means.
|
| 1959 |
+
# D-465: type checked like `hiddenFields`. `bool("false")` is True, so a coerced string
|
| 1960 |
+
# stored a GRANT the administrator meant as a denial. Absence still grants.
|
| 1961 |
+
metrics = raw.get("metrics", True)
|
| 1962 |
+
if not isinstance(metrics, bool):
|
| 1963 |
+
raise err(400, "bad_perms", f"{key}: metrics must be true or false")
|
| 1964 |
out[key] = {"access": bool(raw.get("access", True)),
|
| 1965 |
"filter": clean_tree,
|
| 1966 |
"hiddenFields": sorted(submitted),
|
| 1967 |
+
"metrics": metrics}
|
| 1968 |
_refuse_unshapeable_bu(out)
|
| 1969 |
return out
|
| 1970 |
|
api/routes_automation.py
CHANGED
|
@@ -126,10 +126,27 @@ def _voice_options(session):
|
|
| 126 |
tables.append(option)
|
| 127 |
except Exception: # noqa: BLE001 - permission/absence.
|
| 128 |
pass
|
| 129 |
-
# User tables are permissioned by their own object wall inside
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
if option:
|
| 134 |
tables.append(option)
|
| 135 |
return {"tables": tables}
|
|
@@ -210,8 +227,23 @@ def _voice_view_input(session, audience):
|
|
| 210 |
|
| 211 |
@router.get("/voice-agents")
|
| 212 |
def list_voice_agents(session: Session = Depends(_GATE)):
|
| 213 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 214 |
rows = voice.list_browser_dtos(session.runtime)
|
|
|
|
| 215 |
for row in rows:
|
| 216 |
row["audienceOptions"] = options
|
| 217 |
return {"voiceAgents": rows}
|
|
@@ -1081,6 +1113,49 @@ def _discover_default(session, tables):
|
|
| 1081 |
lambda: engine.discover_default_table(session.runtime))
|
| 1082 |
|
| 1083 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1084 |
def warm_default(rt):
|
| 1085 |
"""⭐⭐ WAVE 30 · T12, THE COLD HALF. Elect the discovery picker's default at BOOT.
|
| 1086 |
|
|
@@ -1396,7 +1471,22 @@ def _table_views(session, key):
|
|
| 1396 |
|
| 1397 |
@router.get("/automations/tables")
|
| 1398 |
def automation_tables(session: Session = Depends(_GATE)):
|
| 1399 |
-
"""The blank databases an automation can target, with their fields.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1400 |
|
| 1401 |
⚠ A READ-ONLY MIRROR of C3-UT's `GET /api/v1/tables`, under this router's own prefix so the
|
| 1402 |
two can never collide. It exists because the automation editor needs the table+field list to
|
|
@@ -1442,11 +1532,12 @@ def automation_tables(session: Session = Depends(_GATE)):
|
|
| 1442 |
"type": f.get("type") or "text",
|
| 1443 |
"automation": f.get("automation") or None}
|
| 1444 |
for f in (t.get("fields") or [])]}
|
| 1445 |
-
|
| 1446 |
-
|
| 1447 |
-
|
|
|
|
| 1448 |
out.append(row)
|
| 1449 |
-
return
|
| 1450 |
|
| 1451 |
|
| 1452 |
@router.get("/automations/presets")
|
|
@@ -1953,6 +2044,26 @@ def agent_author_action(auto_id: str, body: dict = Body(default=None),
|
|
| 1953 |
"actionId": action_id, "agent": agent_id, "notes": notes}
|
| 1954 |
|
| 1955 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1956 |
@router.get("/automations/{auto_id}")
|
| 1957 |
def get_automation(auto_id: str, session: Session = Depends(_GATE)):
|
| 1958 |
"""One automation, by id.
|
|
|
|
| 126 |
tables.append(option)
|
| 127 |
except Exception: # noqa: BLE001 - permission/absence.
|
| 128 |
pass
|
| 129 |
+
# User tables are permissioned by their own object wall inside _automation_table_rows().
|
| 130 |
+
#
|
| 131 |
+
# ⭐⭐ W62-T17 — THE VIEW LIST IS BOUGHT PER DATABASE AND `_voice_option` THROWS MOST OF IT
|
| 132 |
+
# AWAY. That helper returns None unless a database has BOTH a phone-type field AND a saved
|
| 133 |
+
# View, so a database with no phone field is dropped whatever its Views say — and resolving
|
| 134 |
+
# them cost a `shares.grants` read plus a whole workspace-document read EACH. MEASURED on
|
| 135 |
+
# staging: `_table_views` x24 = 6,810 ms of this pass
|
| 136 |
+
# (`AutoWiki/waves/wave62/proto/T17-profile.md`). The phone test is free (the field list is
|
| 137 |
+
# already in hand), so it is asked FIRST and the view read is spent only where it can matter.
|
| 138 |
+
#
|
| 139 |
+
# ⚠ THE ANSWER IS IDENTICAL, not merely similar: the skipped databases are exactly the ones
|
| 140 |
+
# `_voice_option` returned None for, and `_table_views` is still the single source of the
|
| 141 |
+
# view list for every database that survives — ABSENT (None) still reads as "nobody looked"
|
| 142 |
+
# and still drops the option, exactly as it did when the list rode on the row.
|
| 143 |
+
for table in _automation_table_rows(session, with_views=False):
|
| 144 |
+
if not any(isinstance(field, dict) and field.get("key") and field.get("type") == "phone"
|
| 145 |
+
for field in (table.get("fields") or [])):
|
| 146 |
+
continue
|
| 147 |
+
key = table.get("key")
|
| 148 |
+
option = _voice_option(key, table.get("label") or key,
|
| 149 |
+
table.get("fields"), _table_views(session, key))
|
| 150 |
if option:
|
| 151 |
tables.append(option)
|
| 152 |
return {"tables": tables}
|
|
|
|
| 227 |
|
| 228 |
@router.get("/voice-agents")
|
| 229 |
def list_voice_agents(session: Session = Depends(_GATE)):
|
| 230 |
+
"""Every voice agent, each carrying the audience choices its editor offers.
|
| 231 |
+
|
| 232 |
+
⭐⭐ W62-T17 (R12, D-739) — **THE OPTIONS ARE RESOLVED ONLY IF A ROW WILL CARRY THEM**, and on
|
| 233 |
+
the store that produced this ticket that is the whole fix. `_voice_options` is the most
|
| 234 |
+
expensive read pass in this file (MEASURED at 14,705 ms of this route's 14,748 ms on staging's
|
| 235 |
+
Royal Imports, `AutoWiki/waves/wave62/proto/T17-profile.md`), and Royal Imports has NO voice
|
| 236 |
+
agents, so every one of those seconds was assembled and then thrown away with the empty list.
|
| 237 |
+
The payload is unchanged by construction: with no rows the loop below never ran, so the
|
| 238 |
+
options never reached a caller.
|
| 239 |
+
|
| 240 |
+
⛔ THE ORDER IS THE FIX, not a tidy-up — read the rows first, then decide. Reinstating the
|
| 241 |
+
eager call "because it is simpler" restores a 14 second answer to an 18 byte payload. This is
|
| 242 |
+
the same shape this file already carries twice (W29-T01 and W30-T12, both on `list_automations`
|
| 243 |
+
below): work whose result is discarded is still paid for.
|
| 244 |
+
"""
|
| 245 |
rows = voice.list_browser_dtos(session.runtime)
|
| 246 |
+
options = _voice_options(session) if rows else None
|
| 247 |
for row in rows:
|
| 248 |
row["audienceOptions"] = options
|
| 249 |
return {"voiceAgents": rows}
|
|
|
|
| 1113 |
lambda: engine.discover_default_table(session.runtime))
|
| 1114 |
|
| 1115 |
|
| 1116 |
+
def automation_ids(session):
|
| 1117 |
+
"""Every automation id `GET /automations` would list, WITHOUT wiring a single DTO.
|
| 1118 |
+
|
| 1119 |
+
⭐⭐ W62-T17 — THE ID-ONLY ACCESSOR `routes_module_index._readable_ids` ASKED FOR IN WRITING:
|
| 1120 |
+
*"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until it
|
| 1121 |
+
has an id-only accessor."* That route needs the id SET and nothing else — it uses it to drop a
|
| 1122 |
+
folder placement whose item the caller cannot read — and it was building the whole Agents rail
|
| 1123 |
+
payload to get it. MEASURED on staging: `GET /api/v1/module-index/agents` spent 1,101 ms of its
|
| 1124 |
+
1,180 ms inside `list_automations`, whose `_wire` reads `agent_contracts` and
|
| 1125 |
+
`automation_agent_actions` once PER AUTOMATION (`AutoWiki/waves/wave62/proto/T17-profile.md`).
|
| 1126 |
+
|
| 1127 |
+
⛔⛔ THE SET MUST BE `list_automations`' SET EXACTLY, BECAUSE IT IS A WALL. An id this omits is
|
| 1128 |
+
a tile that silently stops rendering — the worst failure shape on a folder home, because the
|
| 1129 |
+
placement is still stored and the page simply looks emptier. So all three strata stay: the
|
| 1130 |
+
stored definitions, the synthetic field agents (C3), and the derived Odoo sync row (C4). Add a
|
| 1131 |
+
stratum to `list_automations` and it belongs here in the same commit.
|
| 1132 |
+
|
| 1133 |
+
⛔ AND THE STATEMENTS SEED STAYS, for a reason that is not obvious: `seed_statements_agent`
|
| 1134 |
+
MINTS a row that `all_definitions` then lists (W35-T37 / R10), once per tenant per process. If
|
| 1135 |
+
only `list_automations` seeded it, this set would depend on WHICH ROUTE the process happened to
|
| 1136 |
+
serve first — the Agents home would hide the statements agent's placement until somebody
|
| 1137 |
+
opened the rail. Same guard, same idempotency key, same never-take-the-rail-down posture.
|
| 1138 |
+
|
| 1139 |
+
⚠ WHAT IT DELIBERATELY DOES NOT DO is the board-retirement scan. That is hygiene over stored
|
| 1140 |
+
state (`all_definitions` strips retired board state on every read anyway), it changes no id,
|
| 1141 |
+
and `list_automations` still runs it on its own first call per tenant.
|
| 1142 |
+
"""
|
| 1143 |
+
if session.tenant not in _STATEMENTS_SEEDED:
|
| 1144 |
+
try:
|
| 1145 |
+
engine.seed_statements_agent(session.runtime, username=session.uname or "system")
|
| 1146 |
+
except Exception: # noqa: BLE001
|
| 1147 |
+
pass
|
| 1148 |
+
_STATEMENTS_SEEDED.add(session.tenant)
|
| 1149 |
+
ids = {str(defn.get("id")) for defn in engine.all_definitions(session.runtime).values()
|
| 1150 |
+
if isinstance(defn, dict) and defn.get("id")}
|
| 1151 |
+
ids |= {str(row.get("id")) for row in _field_agent_rows(session)
|
| 1152 |
+
if isinstance(row, dict) and row.get("id")}
|
| 1153 |
+
odoo = _odoo_sync_row(session)
|
| 1154 |
+
if isinstance(odoo, dict) and odoo.get("id"):
|
| 1155 |
+
ids.add(str(odoo["id"]))
|
| 1156 |
+
return ids
|
| 1157 |
+
|
| 1158 |
+
|
| 1159 |
def warm_default(rt):
|
| 1160 |
"""⭐⭐ WAVE 30 · T12, THE COLD HALF. Elect the discovery picker's default at BOOT.
|
| 1161 |
|
|
|
|
| 1471 |
|
| 1472 |
@router.get("/automations/tables")
|
| 1473 |
def automation_tables(session: Session = Depends(_GATE)):
|
| 1474 |
+
"""The blank databases an automation can target, with their fields. See the helper below."""
|
| 1475 |
+
return {"tables": _automation_table_rows(session)}
|
| 1476 |
+
|
| 1477 |
+
|
| 1478 |
+
def _automation_table_rows(session, with_views=True):
|
| 1479 |
+
"""The blank databases an automation can target, with their fields — the rows behind
|
| 1480 |
+
`GET /automations/tables`, as a plain list.
|
| 1481 |
+
|
| 1482 |
+
⭐⭐ W62-T17 — SPLIT OUT OF THE ROUTE SO ONE IN-PROCESS CALLER CAN DECLINE THE EXPENSIVE HALF.
|
| 1483 |
+
The route's payload is unchanged (`with_views=True` is the default and the route passes
|
| 1484 |
+
nothing else); `_voice_options` above is the caller that asks for `with_views=False`, because
|
| 1485 |
+
it filters on phone fields first and most databases never reach the view list at all. See that
|
| 1486 |
+
call site for the measurement.
|
| 1487 |
+
|
| 1488 |
+
⛔ A PLAIN FUNCTION, NOT A SECOND ROUTE. The wall, the C8 migration and the lent document all
|
| 1489 |
+
stay exactly where they were; only the `views` read became optional.
|
| 1490 |
|
| 1491 |
⚠ A READ-ONLY MIRROR of C3-UT's `GET /api/v1/tables`, under this router's own prefix so the
|
| 1492 |
two can never collide. It exists because the automation editor needs the table+field list to
|
|
|
|
| 1532 |
"type": f.get("type") or "text",
|
| 1533 |
"automation": f.get("automation") or None}
|
| 1534 |
for f in (t.get("fields") or [])]}
|
| 1535 |
+
if with_views:
|
| 1536 |
+
views = _table_views(session, key)
|
| 1537 |
+
if views is not None:
|
| 1538 |
+
row["views"] = views # absent when the bucket did not answer — never []
|
| 1539 |
out.append(row)
|
| 1540 |
+
return out
|
| 1541 |
|
| 1542 |
|
| 1543 |
@router.get("/automations/presets")
|
|
|
|
| 2044 |
"actionId": action_id, "agent": agent_id, "notes": notes}
|
| 2045 |
|
| 2046 |
|
| 2047 |
+
@router.get("/automations/ai/transport")
|
| 2048 |
+
def ai_transport(session: Session = Depends(_GATE)):
|
| 2049 |
+
"""D-384: which Anthropic path THIS process ran, read-only, administrators only.
|
| 2050 |
+
|
| 2051 |
+
A verifier could prove the SDK path on its own box and never in the container, because the
|
| 2052 |
+
installed version and `ai_review.LAST_ANTHROPIC_TRANSPORT` had no reader. Two segments under
|
| 2053 |
+
`/automations/`, so the one-segment `/automations/{auto_id}` below cannot shadow it.
|
| 2054 |
+
⚠ `anthropic` is imported HERE, lazily: `verify_no_streamlit.py` boots the API with optional
|
| 2055 |
+
packages blocked, and it drives this handler with the SDK blocked too."""
|
| 2056 |
+
if not session.admin:
|
| 2057 |
+
raise err(403, "admin_only", "AI transport diagnostics are an admin view")
|
| 2058 |
+
try:
|
| 2059 |
+
import anthropic # noqa: PLC0415
|
| 2060 |
+
version = str(getattr(anthropic, "__version__", "") or "")
|
| 2061 |
+
except ImportError:
|
| 2062 |
+
version = ""
|
| 2063 |
+
return {"anthropicVersion": version or None, "sdkInstalled": bool(version),
|
| 2064 |
+
"lastTransport": ai_review.LAST_ANTHROPIC_TRANSPORT or None}
|
| 2065 |
+
|
| 2066 |
+
|
| 2067 |
@router.get("/automations/{auto_id}")
|
| 2068 |
def get_automation(auto_id: str, session: Session = Depends(_GATE)):
|
| 2069 |
"""One automation, by id.
|
api/routes_connectors.py
CHANGED
|
@@ -23,6 +23,8 @@ from __future__ import annotations
|
|
| 23 |
import os
|
| 24 |
import time
|
| 25 |
import threading
|
|
|
|
|
|
|
| 26 |
from datetime import datetime, timezone
|
| 27 |
|
| 28 |
from fastapi import APIRouter, Body, Depends
|
|
@@ -1201,13 +1203,12 @@ def _sync_payload(connector_key, connection_id, row=None):
|
|
| 1201 |
def _odoo_sync_runner(session, connection_id):
|
| 1202 |
"""Run Odoo through C6 without moving provider imports into the data layer."""
|
| 1203 |
connector_sync.record(session.runtime, "odoo", connection_id, "syncing")
|
| 1204 |
-
# The current Odoo entry point plans and applies in one call, so the closest truthful seam is
|
| 1205 |
-
# immediately before it starts applying the tenant mirror into governed database definitions.
|
| 1206 |
-
connector_sync.record(session.runtime, "odoo", connection_id, "materializing")
|
| 1207 |
import odoo_relational
|
| 1208 |
result = odoo_relational.refresh(session.runtime, session.tenant,
|
| 1209 |
username=session.uname or "automation",
|
| 1210 |
-
connection_id=connection_id
|
|
|
|
|
|
|
| 1211 |
tables = result.get("tables") if isinstance(result, dict) else {}
|
| 1212 |
databases = sorted(str(key) for key in (tables or {}).keys())
|
| 1213 |
sync_errors = []
|
|
@@ -1235,6 +1236,11 @@ def _odoo_sync_runner(session, connection_id):
|
|
| 1235 |
_SYNC_RUNNERS = {"odoo": _odoo_sync_runner}
|
| 1236 |
|
| 1237 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1238 |
class _SupersededMaterialization(Exception):
|
| 1239 |
pass
|
| 1240 |
|
|
@@ -1296,15 +1302,29 @@ class _MaterializationRuntime:
|
|
| 1296 |
marker = [self.job["id"], self.job["leaseToken"]]
|
| 1297 |
def apply(current):
|
| 1298 |
self.valid()
|
| 1299 |
-
|
|
|
|
|
|
|
| 1300 |
raise _SupersededMaterialization()
|
| 1301 |
answer = fn(current)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1302 |
# Lease expiry during a long merge cannot acknowledge a stale write either.
|
| 1303 |
self.valid()
|
| 1304 |
return answer
|
| 1305 |
return self.rt.update(name, apply, flush=flush, **kwargs)
|
| 1306 |
|
| 1307 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1308 |
def _run_database_job(rt, candidate):
|
| 1309 |
"""Run one persisted receipt. Repeating a completed publication reconciles the same rows."""
|
| 1310 |
import core.user_tables as ut
|
|
@@ -1312,13 +1332,14 @@ def _run_database_job(rt, candidate):
|
|
| 1312 |
# A disconnected source is paused work, not a failed attempt. Preserve the receipt so
|
| 1313 |
# reconnect can resume it; the publication fence still rechecks a mid-run disconnect.
|
| 1314 |
if (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"):
|
|
|
|
| 1315 |
return
|
| 1316 |
job = connector_sync.claim(rt, candidate)
|
| 1317 |
if job is None:
|
| 1318 |
return
|
| 1319 |
fenced = _MaterializationRuntime(rt, job)
|
| 1320 |
key = job["databaseKey"]
|
| 1321 |
-
def finish(success=False):
|
| 1322 |
# The pre-claim check cannot cover a disconnect during source I/O or publication.
|
| 1323 |
# Only the current lease may requeue; disable and a newer Save still win.
|
| 1324 |
interrupted = bool((rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"))
|
|
@@ -1326,7 +1347,13 @@ def _run_database_job(rt, candidate):
|
|
| 1326 |
# A newer field Save can persist after rows commit but before this acknowledgement.
|
| 1327 |
# The old rows remain valid history; they cannot establish the new settings ready.
|
| 1328 |
fenced.valid()
|
| 1329 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1330 |
|
| 1331 |
stopped = threading.Event()
|
| 1332 |
def heartbeat():
|
|
@@ -1339,9 +1366,8 @@ def _run_database_job(rt, candidate):
|
|
| 1339 |
try:
|
| 1340 |
def install(current):
|
| 1341 |
fenced.valid()
|
| 1342 |
-
if key
|
| 1343 |
-
|
| 1344 |
-
current[key]["_connectorJob"] = [job["id"], job["leaseToken"]]
|
| 1345 |
return current
|
| 1346 |
rt.update(ut.STORE_KEY, install, flush="sync")
|
| 1347 |
heartbeat_thread = threading.Thread(target=heartbeat, daemon=True,
|
|
@@ -1349,6 +1375,15 @@ def _run_database_job(rt, candidate):
|
|
| 1349 |
heartbeat_thread.start()
|
| 1350 |
cid = job["connectionId"]
|
| 1351 |
source_id = None if cid == "default" else cid
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1352 |
if rel.base_table_key(key) in rel.ODOO_TABLE_MODELS:
|
| 1353 |
result = rel.refresh(
|
| 1354 |
fenced, getattr(rt, "tenant", None), username="automation",
|
|
@@ -1358,26 +1393,58 @@ def _run_database_job(rt, candidate):
|
|
| 1358 |
_cid, namespace = rel.connection_namespace(rt, source_id)
|
| 1359 |
result = {"discovered": rel.sync_discovered_models(
|
| 1360 |
fenced, source_connection_id=source_id, namespace=namespace,
|
| 1361 |
-
username="automation", table_keys={key}
|
|
|
|
|
|
|
| 1362 |
discovered = result.get("discovered") or {}
|
| 1363 |
published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
|
| 1364 |
success = (key in published and not result.get("failed") and not result.get("frozen")
|
| 1365 |
and not result.get("limits") and not discovered.get("errors"))
|
| 1366 |
-
finish(success=success)
|
| 1367 |
except _SupersededMaterialization:
|
| 1368 |
# Disable/new Save already invalidated the table marker; never restore it here.
|
| 1369 |
finish()
|
| 1370 |
-
except
|
|
|
|
|
|
|
| 1371 |
# Raw provider exceptions may carry credentials or source values. Store only safe copy.
|
| 1372 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1373 |
finally:
|
| 1374 |
stopped.set()
|
| 1375 |
if heartbeat_thread:
|
| 1376 |
heartbeat_thread.join(timeout=1)
|
| 1377 |
|
| 1378 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1379 |
def _resume_database_jobs(rt):
|
| 1380 |
"""One serial pass, also usable by isolated synthetic fixtures."""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1381 |
for candidate in connector_sync.jobs(rt):
|
| 1382 |
if candidate.get("provider") == "odoo" and (
|
| 1383 |
candidate.get("state") == "queued" or
|
|
@@ -1394,6 +1461,52 @@ _materialization_wake = threading.Event()
|
|
| 1394 |
_materialization_stop = threading.Event()
|
| 1395 |
_materialization_thread = None
|
| 1396 |
_materialization_lifecycle = threading.Lock()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1397 |
|
| 1398 |
|
| 1399 |
@router.on_event("startup")
|
|
@@ -1417,7 +1530,7 @@ def _start_database_jobs():
|
|
| 1417 |
if _materialization_stop.is_set():
|
| 1418 |
break
|
| 1419 |
try:
|
| 1420 |
-
|
| 1421 |
except Exception:
|
| 1422 |
continue # Durable queued/leased work is retried by the next scan.
|
| 1423 |
except Exception:
|
|
@@ -1435,9 +1548,14 @@ def _start_database_jobs():
|
|
| 1435 |
|
| 1436 |
@router.on_event("shutdown")
|
| 1437 |
def _stop_database_jobs():
|
|
|
|
| 1438 |
with _materialization_lifecycle:
|
| 1439 |
_materialization_stop.set()
|
| 1440 |
_materialization_wake.set()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1441 |
|
| 1442 |
|
| 1443 |
def _sync_refusals():
|
|
@@ -1575,6 +1693,30 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
|
|
| 1575 |
return _sync_payload(connector_key, connection_id, row)
|
| 1576 |
|
| 1577 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1578 |
@router.post("/connectors/{connector_key}/sync/retry")
|
| 1579 |
def connector_sync_retry(connector_key: str, connection_id: str = None,
|
| 1580 |
session: Session = Depends(require_session), database_key: str = None):
|
|
@@ -1809,6 +1951,7 @@ def _enable_odoo_database(table_key, session, connection_id=None):
|
|
| 1809 |
# materialisation remains separate work.
|
| 1810 |
if failure is not None:
|
| 1811 |
raise err(*failure)
|
|
|
|
| 1812 |
if not exists:
|
| 1813 |
reason = _materialisation_failure_reason(result)
|
| 1814 |
message = "The setting was enabled, but the database was not created."
|
|
@@ -1816,11 +1959,14 @@ def _enable_odoo_database(table_key, session, connection_id=None):
|
|
| 1816 |
message = f"{message} {reason}"
|
| 1817 |
else:
|
| 1818 |
message = f"{message} The field catalogue could not be published."
|
| 1819 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1820 |
|
| 1821 |
# Every permission-visible field is available from schema alone. Existing per-database
|
| 1822 |
# choices survive a retry; population counts are optional background observations.
|
| 1823 |
-
model = str(choice.get("model") or "").strip()
|
| 1824 |
try:
|
| 1825 |
schema_keys = {_odoo_field_key(session.runtime, key, model, name): True
|
| 1826 |
for name in result["sourceFields"]}
|
|
@@ -1832,9 +1978,12 @@ def _enable_odoo_database(table_key, session, connection_id=None):
|
|
| 1832 |
},
|
| 1833 |
flush="sync")
|
| 1834 |
except Exception: # noqa: BLE001
|
| 1835 |
-
|
| 1836 |
-
|
| 1837 |
-
|
|
|
|
|
|
|
|
|
|
| 1838 |
_evict_odoo_product_pools(session.runtime)
|
| 1839 |
|
| 1840 |
# W56-T36: there is deliberately no navigation projection eviction here. Data's database
|
|
@@ -2578,11 +2727,25 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
|
|
| 2578 |
return _named_databases(rt, is_admin, uname, out)
|
| 2579 |
|
| 2580 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2581 |
def _amazon_databases(rt, is_admin, spec=None):
|
| 2582 |
"""⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM.
|
| 2583 |
|
| 2584 |
-
Fills the
|
| 2585 |
-
|
| 2586 |
|
| 2587 |
⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five
|
| 2588 |
of the seven tables have NO published schema, because Amazon publishes artifacts only for its
|
|
@@ -2611,12 +2774,13 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 2611 |
enabled = enablement.get("databases") or {}
|
| 2612 |
cat = amazon_schema.catalogue()
|
| 2613 |
tables = cat.get("tables") or {}
|
|
|
|
| 2614 |
|
| 2615 |
# Which OTHER declared tables are filled from the SAME report document. Derived from the
|
| 2616 |
# report map so a table added to `REPORT_TYPES` gets this disclosure for free.
|
| 2617 |
by_report = {}
|
| 2618 |
for name in amazon_schema.PROVENANCE:
|
| 2619 |
-
rep =
|
| 2620 |
if rep:
|
| 2621 |
by_report.setdefault(rep, []).append("ut_amzn_" + name)
|
| 2622 |
|
|
@@ -2625,7 +2789,7 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 2625 |
key = "ut_amzn_" + name
|
| 2626 |
spec = tables.get(name) or {}
|
| 2627 |
cols = spec.get("columns") or []
|
| 2628 |
-
rep =
|
| 2629 |
fields = [{"key": str(c.get("name") or ""),
|
| 2630 |
"label": str(c.get("name") or "").replace("_", " ").capitalize(),
|
| 2631 |
"type": "text" if str(c.get("type")) == "VARCHAR" else "number",
|
|
@@ -2656,6 +2820,7 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 2656 |
# read from Odoo's grid config, and disabling it leaves every stored row and
|
| 2657 |
# definition untouched: only this provider config document changes.
|
| 2658 |
"enabled": enabled.get(key) is True,
|
|
|
|
| 2659 |
"fields": fields,
|
| 2660 |
"excluded": excluded,
|
| 2661 |
"sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
|
|
@@ -2876,9 +3041,11 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
|
|
| 2876 |
return _safe_connector_status(key, {"overview": overview,
|
| 2877 |
"keychain": keychain,
|
| 2878 |
"databases": databases,
|
|
|
|
|
|
|
| 2879 |
"materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
|
| 2880 |
"retryPath": f"/connectors/{key}/sync/retry"}
|
| 2881 |
-
if is_admin and key
|
| 2882 |
# W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it
|
| 2883 |
# from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints.
|
| 2884 |
"enablement": enablement,
|
|
|
|
| 23 |
import os
|
| 24 |
import time
|
| 25 |
import threading
|
| 26 |
+
import xmlrpc.client
|
| 27 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 28 |
from datetime import datetime, timezone
|
| 29 |
|
| 30 |
from fastapi import APIRouter, Body, Depends
|
|
|
|
| 1203 |
def _odoo_sync_runner(session, connection_id):
|
| 1204 |
"""Run Odoo through C6 without moving provider imports into the data layer."""
|
| 1205 |
connector_sync.record(session.runtime, "odoo", connection_id, "syncing")
|
|
|
|
|
|
|
|
|
|
| 1206 |
import odoo_relational
|
| 1207 |
result = odoo_relational.refresh(session.runtime, session.tenant,
|
| 1208 |
username=session.uname or "automation",
|
| 1209 |
+
connection_id=connection_id,
|
| 1210 |
+
on_apply=lambda: connector_sync.record(
|
| 1211 |
+
session.runtime, "odoo", connection_id, "materializing"))
|
| 1212 |
tables = result.get("tables") if isinstance(result, dict) else {}
|
| 1213 |
databases = sorted(str(key) for key in (tables or {}).keys())
|
| 1214 |
sync_errors = []
|
|
|
|
| 1236 |
_SYNC_RUNNERS = {"odoo": _odoo_sync_runner}
|
| 1237 |
|
| 1238 |
|
| 1239 |
+
ODOO_DISCONNECTED_CAUSE = (
|
| 1240 |
+
"Odoo is disconnected, so rows cannot load. "
|
| 1241 |
+
"Reconnect Odoo in the Keychain and this resumes on its own.")
|
| 1242 |
+
|
| 1243 |
+
|
| 1244 |
class _SupersededMaterialization(Exception):
|
| 1245 |
pass
|
| 1246 |
|
|
|
|
| 1302 |
marker = [self.job["id"], self.job["leaseToken"]]
|
| 1303 |
def apply(current):
|
| 1304 |
self.valid()
|
| 1305 |
+
missing = name == ut.STORE_KEY and key not in current
|
| 1306 |
+
if (name == ut.STORE_KEY and not missing
|
| 1307 |
+
and (current.get(key) or {}).get("_connectorJob") != marker):
|
| 1308 |
raise _SupersededMaterialization()
|
| 1309 |
answer = fn(current)
|
| 1310 |
+
if missing and key in answer:
|
| 1311 |
+
# A reconciled receipt may precede its definition. Its first schema write
|
| 1312 |
+
# installs the marker under the same lease and configuration fence.
|
| 1313 |
+
answer[key]["_connectorJob"] = marker
|
| 1314 |
# Lease expiry during a long merge cannot acknowledge a stale write either.
|
| 1315 |
self.valid()
|
| 1316 |
return answer
|
| 1317 |
return self.rt.update(name, apply, flush=flush, **kwargs)
|
| 1318 |
|
| 1319 |
|
| 1320 |
+
def _odoo_read_access_refused(exc):
|
| 1321 |
+
"""Match only XML-RPC faults with explicit access/read refusal markers."""
|
| 1322 |
+
if not isinstance(exc, xmlrpc.client.Fault):
|
| 1323 |
+
return False
|
| 1324 |
+
message = str(exc).casefold()
|
| 1325 |
+
return "security restrictions" in message or "operation: read" in message
|
| 1326 |
+
|
| 1327 |
+
|
| 1328 |
def _run_database_job(rt, candidate):
|
| 1329 |
"""Run one persisted receipt. Repeating a completed publication reconciles the same rows."""
|
| 1330 |
import core.user_tables as ut
|
|
|
|
| 1332 |
# A disconnected source is paused work, not a failed attempt. Preserve the receipt so
|
| 1333 |
# reconnect can resume it; the publication fence still rechecks a mid-run disconnect.
|
| 1334 |
if (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"):
|
| 1335 |
+
connector_sync.stall(rt, candidate, ODOO_DISCONNECTED_CAUSE)
|
| 1336 |
return
|
| 1337 |
job = connector_sync.claim(rt, candidate)
|
| 1338 |
if job is None:
|
| 1339 |
return
|
| 1340 |
fenced = _MaterializationRuntime(rt, job)
|
| 1341 |
key = job["databaseKey"]
|
| 1342 |
+
def finish(success=False, cause=""):
|
| 1343 |
# The pre-claim check cannot cover a disconnect during source I/O or publication.
|
| 1344 |
# Only the current lease may requeue; disable and a newer Save still win.
|
| 1345 |
interrupted = bool((rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"))
|
|
|
|
| 1347 |
# A newer field Save can persist after rows commit but before this acknowledgement.
|
| 1348 |
# The old rows remain valid history; they cannot establish the new settings ready.
|
| 1349 |
fenced.valid()
|
| 1350 |
+
# Only provider-authored sentences reach this seam, never raw exception messages.
|
| 1351 |
+
cause = str(cause or "").replace("\u2014", ",").replace("\u2013", "-")
|
| 1352 |
+
result = connector_sync.finish(
|
| 1353 |
+
rt, job, success=success, interrupted=interrupted, cause=cause)
|
| 1354 |
+
if interrupted:
|
| 1355 |
+
connector_sync.stall(rt, job, ODOO_DISCONNECTED_CAUSE)
|
| 1356 |
+
return result
|
| 1357 |
|
| 1358 |
stopped = threading.Event()
|
| 1359 |
def heartbeat():
|
|
|
|
| 1366 |
try:
|
| 1367 |
def install(current):
|
| 1368 |
fenced.valid()
|
| 1369 |
+
if key in (current or {}):
|
| 1370 |
+
current[key]["_connectorJob"] = [job["id"], job["leaseToken"]]
|
|
|
|
| 1371 |
return current
|
| 1372 |
rt.update(ut.STORE_KEY, install, flush="sync")
|
| 1373 |
heartbeat_thread = threading.Thread(target=heartbeat, daemon=True,
|
|
|
|
| 1375 |
heartbeat_thread.start()
|
| 1376 |
cid = job["connectionId"]
|
| 1377 |
source_id = None if cid == "default" else cid
|
| 1378 |
+
if key not in (rt.get(ut.STORE_KEY, fresh=True) or {}):
|
| 1379 |
+
schema = rel.publish_database_schema(fenced, key, connection_id=source_id,
|
| 1380 |
+
username="automation")
|
| 1381 |
+
if not job["model"]:
|
| 1382 |
+
# Reconciliation cannot infer a missing definition's model. Now the provider
|
| 1383 |
+
# has resolved it, queue its real field fingerprint before reading any rows.
|
| 1384 |
+
fenced.valid()
|
| 1385 |
+
_queue_odoo_database(rt, key, model=schema["identity"]["model"], changed=True)
|
| 1386 |
+
return
|
| 1387 |
if rel.base_table_key(key) in rel.ODOO_TABLE_MODELS:
|
| 1388 |
result = rel.refresh(
|
| 1389 |
fenced, getattr(rt, "tenant", None), username="automation",
|
|
|
|
| 1393 |
_cid, namespace = rel.connection_namespace(rt, source_id)
|
| 1394 |
result = {"discovered": rel.sync_discovered_models(
|
| 1395 |
fenced, source_connection_id=source_id, namespace=namespace,
|
| 1396 |
+
username="automation", table_keys={key},
|
| 1397 |
+
on_progress=lambda rows, total: connector_sync.progress(
|
| 1398 |
+
fenced, job, rows=rows, total=total))}
|
| 1399 |
discovered = result.get("discovered") or {}
|
| 1400 |
published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
|
| 1401 |
success = (key in published and not result.get("failed") and not result.get("frozen")
|
| 1402 |
and not result.get("limits") and not discovered.get("errors"))
|
| 1403 |
+
finish(success=success, cause=result.get("note") or discovered.get("note") or "")
|
| 1404 |
except _SupersededMaterialization:
|
| 1405 |
# Disable/new Save already invalidated the table marker; never restore it here.
|
| 1406 |
finish()
|
| 1407 |
+
except rel.Refused as exc:
|
| 1408 |
+
finish(cause=str(exc))
|
| 1409 |
+
except Exception as exc:
|
| 1410 |
# Raw provider exceptions may carry credentials or source values. Store only safe copy.
|
| 1411 |
+
cause = ""
|
| 1412 |
+
if _odoo_read_access_refused(exc):
|
| 1413 |
+
cause = (f"Odoo refused to read {job['model']} because the connected Odoo user has no "
|
| 1414 |
+
"read access, so retrying cannot load rows. "
|
| 1415 |
+
"Give that user read access in Odoo, or switch this database off.")
|
| 1416 |
+
finish(cause=cause)
|
| 1417 |
finally:
|
| 1418 |
stopped.set()
|
| 1419 |
if heartbeat_thread:
|
| 1420 |
heartbeat_thread.join(timeout=1)
|
| 1421 |
|
| 1422 |
|
| 1423 |
+
def _reconcile_database_jobs(rt):
|
| 1424 |
+
"""Repair enabled Odoo grids with no receipt using only tenant-local definitions."""
|
| 1425 |
+
import core.user_tables as ut
|
| 1426 |
+
import odoo_relational as rel
|
| 1427 |
+
grids = (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("grids") or {}
|
| 1428 |
+
receipts = {job["databaseKey"] for job in connector_sync.jobs(rt, "odoo")}
|
| 1429 |
+
missing = [key for key, enabled in grids.items() if enabled is True and key not in receipts]
|
| 1430 |
+
if not missing:
|
| 1431 |
+
return
|
| 1432 |
+
definitions = rt.get_projection(ut.STORE_KEY, drop=("rows",)) or {}
|
| 1433 |
+
for key in missing:
|
| 1434 |
+
try:
|
| 1435 |
+
model = str(((definitions.get(key) or {}).get("connector") or {}).get("model") or "")
|
| 1436 |
+
# The queue's identity validator also decides whether an unknown model is legal.
|
| 1437 |
+
_queue_odoo_database(rt, key, model=model)
|
| 1438 |
+
except Exception:
|
| 1439 |
+
continue # One unresolvable definition cannot starve other queued work.
|
| 1440 |
+
|
| 1441 |
+
|
| 1442 |
def _resume_database_jobs(rt):
|
| 1443 |
"""One serial pass, also usable by isolated synthetic fixtures."""
|
| 1444 |
+
try:
|
| 1445 |
+
_reconcile_database_jobs(rt)
|
| 1446 |
+
except Exception:
|
| 1447 |
+
pass # A reconciliation failure must still fall through to the receipt scan.
|
| 1448 |
for candidate in connector_sync.jobs(rt):
|
| 1449 |
if candidate.get("provider") == "odoo" and (
|
| 1450 |
candidate.get("state") == "queued" or
|
|
|
|
| 1461 |
_materialization_stop = threading.Event()
|
| 1462 |
_materialization_thread = None
|
| 1463 |
_materialization_lifecycle = threading.Lock()
|
| 1464 |
+
# 3 is a starting point, not tuned: each worker holds an Odoo read and a growing store document in a small Space container.
|
| 1465 |
+
MATERIALIZATION_WORKERS = 3
|
| 1466 |
+
_materialization_pool = None
|
| 1467 |
+
_materialization_inflight = set()
|
| 1468 |
+
_materialization_inflight_lock = threading.Lock()
|
| 1469 |
+
|
| 1470 |
+
|
| 1471 |
+
def _release_database_job(key):
|
| 1472 |
+
# Completion and cancellation both release the reservation, including failed claims.
|
| 1473 |
+
with _materialization_inflight_lock:
|
| 1474 |
+
_materialization_inflight.discard(key)
|
| 1475 |
+
|
| 1476 |
+
|
| 1477 |
+
def _dispatch_database_jobs(rt):
|
| 1478 |
+
"""Submit a resident scan without waiting for source reads or row publication."""
|
| 1479 |
+
global _materialization_pool
|
| 1480 |
+
try:
|
| 1481 |
+
_reconcile_database_jobs(rt)
|
| 1482 |
+
except Exception:
|
| 1483 |
+
pass # Match the serial fixture pass even when reconciliation is unavailable.
|
| 1484 |
+
for candidate in connector_sync.jobs(rt):
|
| 1485 |
+
if candidate.get("provider") == "odoo" and (
|
| 1486 |
+
candidate.get("state") == "queued" or
|
| 1487 |
+
candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()):
|
| 1488 |
+
key = (rt.key, candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
|
| 1489 |
+
with _materialization_inflight_lock:
|
| 1490 |
+
if key in _materialization_inflight:
|
| 1491 |
+
continue
|
| 1492 |
+
_materialization_inflight.add(key)
|
| 1493 |
+
try:
|
| 1494 |
+
with _materialization_lifecycle:
|
| 1495 |
+
if _materialization_stop.is_set():
|
| 1496 |
+
_release_database_job(key)
|
| 1497 |
+
return
|
| 1498 |
+
if _materialization_pool is None:
|
| 1499 |
+
_materialization_pool = ThreadPoolExecutor(
|
| 1500 |
+
max_workers=MATERIALIZATION_WORKERS,
|
| 1501 |
+
thread_name_prefix="connector-materialization-job")
|
| 1502 |
+
pool = _materialization_pool
|
| 1503 |
+
future = pool.submit(_run_database_job, rt, candidate)
|
| 1504 |
+
# Register outside both locks: an already finished future calls back inline.
|
| 1505 |
+
future.add_done_callback(lambda _future, key=key: _release_database_job(key))
|
| 1506 |
+
except Exception:
|
| 1507 |
+
_release_database_job(key)
|
| 1508 |
+
# A concurrent shutdown or failed submit leaves the receipt for the next pass.
|
| 1509 |
+
continue
|
| 1510 |
|
| 1511 |
|
| 1512 |
@router.on_event("startup")
|
|
|
|
| 1530 |
if _materialization_stop.is_set():
|
| 1531 |
break
|
| 1532 |
try:
|
| 1533 |
+
_dispatch_database_jobs(runtime.get_runtime(tenant))
|
| 1534 |
except Exception:
|
| 1535 |
continue # Durable queued/leased work is retried by the next scan.
|
| 1536 |
except Exception:
|
|
|
|
| 1548 |
|
| 1549 |
@router.on_event("shutdown")
|
| 1550 |
def _stop_database_jobs():
|
| 1551 |
+
global _materialization_pool
|
| 1552 |
with _materialization_lifecycle:
|
| 1553 |
_materialization_stop.set()
|
| 1554 |
_materialization_wake.set()
|
| 1555 |
+
pool, _materialization_pool = _materialization_pool, None
|
| 1556 |
+
if pool is not None:
|
| 1557 |
+
# Running reads retain their reservation until done; lease expiry permits recovery.
|
| 1558 |
+
pool.shutdown(wait=False, cancel_futures=True)
|
| 1559 |
|
| 1560 |
|
| 1561 |
def _sync_refusals():
|
|
|
|
| 1693 |
return _sync_payload(connector_key, connection_id, row)
|
| 1694 |
|
| 1695 |
|
| 1696 |
+
@router.post("/connectors/{connector_key}/sync/now")
|
| 1697 |
+
def connector_sync_now(connector_key: str, session: Session = Depends(require_session)):
|
| 1698 |
+
_sync_admin(session)
|
| 1699 |
+
if connector_key not in connector_sync.providers():
|
| 1700 |
+
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
|
| 1701 |
+
try:
|
| 1702 |
+
return connector_sync.request_sync(connector_key, session.runtime.key,
|
| 1703 |
+
requested_by=session.uname)
|
| 1704 |
+
except ValueError:
|
| 1705 |
+
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None
|
| 1706 |
+
|
| 1707 |
+
|
| 1708 |
+
@router.get("/connectors/{connector_key}/sync/now")
|
| 1709 |
+
def connector_sync_now_status(connector_key: str, session: Session = Depends(require_session)):
|
| 1710 |
+
_sync_admin(session)
|
| 1711 |
+
if connector_key not in connector_sync.providers():
|
| 1712 |
+
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
|
| 1713 |
+
return connector_sync.sync_request(connector_key, session.runtime.key) or {
|
| 1714 |
+
"provider": connector_key, "tenant": session.runtime.key, "state": "idle",
|
| 1715 |
+
"requestedAt": None, "startedAt": None, "finishedAt": None,
|
| 1716 |
+
"requestedBy": "", "detail": "", "accepted": False,
|
| 1717 |
+
}
|
| 1718 |
+
|
| 1719 |
+
|
| 1720 |
@router.post("/connectors/{connector_key}/sync/retry")
|
| 1721 |
def connector_sync_retry(connector_key: str, connection_id: str = None,
|
| 1722 |
session: Session = Depends(require_session), database_key: str = None):
|
|
|
|
| 1951 |
# materialisation remains separate work.
|
| 1952 |
if failure is not None:
|
| 1953 |
raise err(*failure)
|
| 1954 |
+
model = str(choice.get("model") or "").strip()
|
| 1955 |
if not exists:
|
| 1956 |
reason = _materialisation_failure_reason(result)
|
| 1957 |
message = "The setting was enabled, but the database was not created."
|
|
|
|
| 1959 |
message = f"{message} {reason}"
|
| 1960 |
else:
|
| 1961 |
message = f"{message} The field catalogue could not be published."
|
| 1962 |
+
try:
|
| 1963 |
+
_queue_odoo_database(session.runtime, key, model=model)
|
| 1964 |
+
finally:
|
| 1965 |
+
# Preserve the original refusal envelope even if the queue store is unavailable.
|
| 1966 |
+
raise err(409, "database_not_created", message)
|
| 1967 |
|
| 1968 |
# Every permission-visible field is available from schema alone. Existing per-database
|
| 1969 |
# choices survive a retry; population counts are optional background observations.
|
|
|
|
| 1970 |
try:
|
| 1971 |
schema_keys = {_odoo_field_key(session.runtime, key, model, name): True
|
| 1972 |
for name in result["sourceFields"]}
|
|
|
|
| 1978 |
},
|
| 1979 |
flush="sync")
|
| 1980 |
except Exception: # noqa: BLE001
|
| 1981 |
+
try:
|
| 1982 |
+
_queue_odoo_database(session.runtime, key, model=model)
|
| 1983 |
+
finally:
|
| 1984 |
+
raise err(503, "field_auto_enable_failed",
|
| 1985 |
+
"The database was created, but its field choices could not be saved. "
|
| 1986 |
+
"Try again.")
|
| 1987 |
_evict_odoo_product_pools(session.runtime)
|
| 1988 |
|
| 1989 |
# W56-T36: there is deliberately no navigation projection eviction here. Data's database
|
|
|
|
| 2727 |
return _named_databases(rt, is_admin, uname, out)
|
| 2728 |
|
| 2729 |
|
| 2730 |
+
def _amazon_report_name(table):
|
| 2731 |
+
"""The report type this Amazon table reads, whichever key declares it.
|
| 2732 |
+
|
| 2733 |
+
⛔ SETTLEMENTS DECLARE `report_discover`, NOT `report` (W60-T36), because Amazon issues them on
|
| 2734 |
+
its own schedule and refuses a createReport for that type: it is a getReports DISCOVERY filter
|
| 2735 |
+
and is deliberately absent from `_APPROVED_REPORT_TYPES`. Reading only `report` here printed an
|
| 2736 |
+
empty Source name for settlements on the connector screen, which reads as "this table has no
|
| 2737 |
+
report" when the truth is "this table is not requested, it is listed".
|
| 2738 |
+
"""
|
| 2739 |
+
import connectors_amazon as cx # noqa: PLC0415 - provider owns state
|
| 2740 |
+
spec = cx.REPORT_TYPES.get(table) or {}
|
| 2741 |
+
return str(spec.get("report") or spec.get("report_discover") or "")
|
| 2742 |
+
|
| 2743 |
+
|
| 2744 |
def _amazon_databases(rt, is_admin, spec=None):
|
| 2745 |
"""⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM.
|
| 2746 |
|
| 2747 |
+
Fills the shared `ConnectorDatabase` shape and adds an administrator-only Amazon `sync`
|
| 2748 |
+
block. Members retain the shared shape without the field catalogue or sync bookkeeping.
|
| 2749 |
|
| 2750 |
⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five
|
| 2751 |
of the seven tables have NO published schema, because Amazon publishes artifacts only for its
|
|
|
|
| 2774 |
enabled = enablement.get("databases") or {}
|
| 2775 |
cat = amazon_schema.catalogue()
|
| 2776 |
tables = cat.get("tables") or {}
|
| 2777 |
+
states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
|
| 2778 |
|
| 2779 |
# Which OTHER declared tables are filled from the SAME report document. Derived from the
|
| 2780 |
# report map so a table added to `REPORT_TYPES` gets this disclosure for free.
|
| 2781 |
by_report = {}
|
| 2782 |
for name in amazon_schema.PROVENANCE:
|
| 2783 |
+
rep = _amazon_report_name(name)
|
| 2784 |
if rep:
|
| 2785 |
by_report.setdefault(rep, []).append("ut_amzn_" + name)
|
| 2786 |
|
|
|
|
| 2789 |
key = "ut_amzn_" + name
|
| 2790 |
spec = tables.get(name) or {}
|
| 2791 |
cols = spec.get("columns") or []
|
| 2792 |
+
rep = _amazon_report_name(name)
|
| 2793 |
fields = [{"key": str(c.get("name") or ""),
|
| 2794 |
"label": str(c.get("name") or "").replace("_", " ").capitalize(),
|
| 2795 |
"type": "text" if str(c.get("type")) == "VARCHAR" else "number",
|
|
|
|
| 2820 |
# read from Odoo's grid config, and disabling it leaves every stored row and
|
| 2821 |
# definition untouched: only this provider config document changes.
|
| 2822 |
"enabled": enabled.get(key) is True,
|
| 2823 |
+
**({"sync": (states or {}).get(name)} if is_admin else {}),
|
| 2824 |
"fields": fields,
|
| 2825 |
"excluded": excluded,
|
| 2826 |
"sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
|
|
|
|
| 3041 |
return _safe_connector_status(key, {"overview": overview,
|
| 3042 |
"keychain": keychain,
|
| 3043 |
"databases": databases,
|
| 3044 |
+
"syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"}
|
| 3045 |
+
if is_admin and key in connector_sync.providers() else None),
|
| 3046 |
"materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
|
| 3047 |
"retryPath": f"/connectors/{key}/sync/retry"}
|
| 3048 |
+
if is_admin and key == "odoo" else None),
|
| 3049 |
# W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it
|
| 3050 |
# from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints.
|
| 3051 |
"enablement": enablement,
|
api/routes_customers.py
CHANGED
|
@@ -513,6 +513,8 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 513 |
if _ov:
|
| 514 |
ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden}
|
| 515 |
for pid, cells in _ov.items()}
|
|
|
|
|
|
|
| 516 |
|
| 517 |
today = time.strftime("%Y-%m-%d")
|
| 518 |
fields = aios_grid.project_date_recurrences(fields, today=today)
|
|
@@ -603,7 +605,7 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 603 |
"team_id": team_id, "routes": normalized_routes, "zones": normalized_zones}
|
| 604 |
|
| 605 |
|
| 606 |
-
def _payload(session: Session):
|
| 607 |
"""`{fields, rows, today, docs, pulled_at}` — X2's shape, which `verify_fields_contract.py`
|
| 608 |
referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by
|
| 609 |
construction); see `grid_assembly` for what that fixed.
|
|
@@ -617,11 +619,18 @@ def _payload(session: Session):
|
|
| 617 |
"""
|
| 618 |
import aios_grid
|
| 619 |
|
| 620 |
-
g = grid_assembly(session)
|
| 621 |
rows = aios_grid.rows_from_pool(
|
| 622 |
g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"])
|
| 623 |
from core import route_system
|
| 624 |
rows = route_system.apply_customer_projections(rows, g["routes"], g["zones"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 625 |
# ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door
|
| 626 |
# (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is
|
| 627 |
# complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key.
|
|
@@ -636,6 +645,8 @@ def _payload(session: Session):
|
|
| 636 |
# "every document in the tenant" mode to reach for, deliberately.
|
| 637 |
from core import grid_events as _ge
|
| 638 |
return {"fields": g["fields"], "rows": rows,
|
|
|
|
|
|
|
| 639 |
# `today` rides the payload because every relative date condition must resolve against
|
| 640 |
# the TENANT's day, never the browser's — a client that falls back to its own clock
|
| 641 |
# disagrees with the server for everyone west of it.
|
|
@@ -718,8 +729,15 @@ def allowed_pids(session: Session):
|
|
| 718 |
|
| 719 |
|
| 720 |
@router.get("/customers")
|
| 721 |
-
def customers(session: Session = Depends(module_gate(MODULE))):
|
| 722 |
-
return _payload(session)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 723 |
|
| 724 |
|
| 725 |
@router.get("/customers/assigned/{view_id}/rows")
|
|
|
|
| 513 |
if _ov:
|
| 514 |
ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden}
|
| 515 |
for pid, cells in _ov.items()}
|
| 516 |
+
# D-464: and the saved views are a FOURTH wire, served by `/workspace` from `workspace`.
|
| 517 |
+
workspace["views"] = aios_grid.narrow_view_keys(workspace.get("views"), hidden)
|
| 518 |
|
| 519 |
today = time.strftime("%Y-%m-%d")
|
| 520 |
fields = aios_grid.project_date_recurrences(fields, today=today)
|
|
|
|
| 605 |
"team_id": team_id, "routes": normalized_routes, "zones": normalized_zones}
|
| 606 |
|
| 607 |
|
| 608 |
+
def _payload(session: Session, related=None):
|
| 609 |
"""`{fields, rows, today, docs, pulled_at}` — X2's shape, which `verify_fields_contract.py`
|
| 610 |
referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by
|
| 611 |
construction); see `grid_assembly` for what that fixed.
|
|
|
|
| 619 |
"""
|
| 620 |
import aios_grid
|
| 621 |
|
| 622 |
+
g = grid_assembly(session, consume_corrections=not bool(related))
|
| 623 |
rows = aios_grid.rows_from_pool(
|
| 624 |
g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"])
|
| 625 |
from core import route_system
|
| 626 |
rows = route_system.apply_customer_projections(rows, g["routes"], g["zones"])
|
| 627 |
+
from routes_odoo_tables import narrow_related_rows
|
| 628 |
+
rows = narrow_related_rows(session, MODULE, rows, related)
|
| 629 |
+
from routes_tables import scope_linked_metrics, linked_metric_limits
|
| 630 |
+
rows, g['fields'], metric_refusals = scope_linked_metrics(
|
| 631 |
+
session, MODULE, rows, g['fields'], g['measures'], related)
|
| 632 |
+
if related:
|
| 633 |
+
g["pids"] = frozenset(row["pid"] for row in rows)
|
| 634 |
# ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door
|
| 635 |
# (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is
|
| 636 |
# complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key.
|
|
|
|
| 645 |
# "every document in the tenant" mode to reach for, deliberately.
|
| 646 |
from core import grid_events as _ge
|
| 647 |
return {"fields": g["fields"], "rows": rows,
|
| 648 |
+
"refusals": metric_refusals,
|
| 649 |
+
"limits": linked_metric_limits(metric_refusals),
|
| 650 |
# `today` rides the payload because every relative date condition must resolve against
|
| 651 |
# the TENANT's day, never the browser's — a client that falls back to its own clock
|
| 652 |
# disagrees with the server for everyone west of it.
|
|
|
|
| 729 |
|
| 730 |
|
| 731 |
@router.get("/customers")
|
| 732 |
+
def customers(session: Session = Depends(module_gate(MODULE)), related: str = None):
|
| 733 |
+
return _payload(session, related=related)
|
| 734 |
+
|
| 735 |
+
|
| 736 |
+
@router.post("/customers/aggregates")
|
| 737 |
+
def customer_aggregates(body: dict = Body(default=None),
|
| 738 |
+
session: Session = Depends(module_gate(MODULE))):
|
| 739 |
+
from routes_odoo_tables import odoo_table_aggregates
|
| 740 |
+
return odoo_table_aggregates(MODULE, body=body, session=session)
|
| 741 |
|
| 742 |
|
| 743 |
@router.get("/customers/assigned/{view_id}/rows")
|
api/routes_dependents.py
CHANGED
|
@@ -42,6 +42,8 @@ docstring names the cost of publishing it. This door returns names and a count,
|
|
| 42 |
"Shared with Anna and 2 others" consumes.
|
| 43 |
"""
|
| 44 |
|
|
|
|
|
|
|
| 45 |
from fastapi import APIRouter, Depends, Query
|
| 46 |
|
| 47 |
import core.dependents as dependents_registry
|
|
@@ -148,6 +150,23 @@ def read_dependents(kind: str = Query(...), key: str = Query(...),
|
|
| 148 |
|
| 149 |
target = {"kind": target_kind, "key": target_key, "database": resolved}
|
| 150 |
report = dependents_registry.dependents_report(session.runtime, target)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
label = _label_for(session, resolved, target_kind, target_key)
|
| 152 |
body = {
|
| 153 |
"target": target,
|
|
|
|
| 42 |
"Shared with Anna and 2 others" consumes.
|
| 43 |
"""
|
| 44 |
|
| 45 |
+
from urllib.parse import parse_qs
|
| 46 |
+
|
| 47 |
from fastapi import APIRouter, Depends, Query
|
| 48 |
|
| 49 |
import core.dependents as dependents_registry
|
|
|
|
| 150 |
|
| 151 |
target = {"kind": target_kind, "key": target_key, "database": resolved}
|
| 152 |
report = dependents_registry.dependents_report(session.runtime, target)
|
| 153 |
+
seen = {resolved: True}
|
| 154 |
+
for dependent in report['dependents']:
|
| 155 |
+
# An address is not an authorisation. Inbound fields can live on a different
|
| 156 |
+
# database, so reuse the same wall before offering either C3 address shape.
|
| 157 |
+
for item in [dependent, *dependent.get('items', [])]:
|
| 158 |
+
href = item.get('href')
|
| 159 |
+
if not isinstance(href, str):
|
| 160 |
+
continue
|
| 161 |
+
if not href.startswith(('#/data?', '#/sheet?')):
|
| 162 |
+
item['href'] = None
|
| 163 |
+
item['reason'] = 'This dependent has no supported address available.'
|
| 164 |
+
continue
|
| 165 |
+
destinations = parse_qs(href.partition('?')[2]).get('database', [])
|
| 166 |
+
if (href.startswith('#/data?') and not destinations) or any(
|
| 167 |
+
not _may_open(session, destination, seen) for destination in destinations):
|
| 168 |
+
item['href'] = None
|
| 169 |
+
item['reason'] = 'You do not have access to open this dependent\'s database.'
|
| 170 |
label = _label_for(session, resolved, target_kind, target_key)
|
| 171 |
body = {
|
| 172 |
"target": target,
|
api/routes_external.py
CHANGED
|
@@ -1,13 +1,16 @@
|
|
| 1 |
-
"""Per-user
|
| 2 |
|
|
|
|
| 3 |
import json
|
|
|
|
| 4 |
from urllib.parse import quote
|
| 5 |
|
| 6 |
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response
|
|
|
|
| 7 |
|
| 8 |
import deps
|
| 9 |
from deps import Session, err, require_session
|
| 10 |
-
from core import external_access
|
| 11 |
|
| 12 |
|
| 13 |
router = APIRouter(prefix="/api/v1/external", tags=["external"])
|
|
@@ -15,6 +18,53 @@ router = APIRouter(prefix="/api/v1/external", tags=["external"])
|
|
| 15 |
_CREDENTIAL_PREFIX = "lpb1."
|
| 16 |
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
def _auth_refusal():
|
| 19 |
raise err(401, external_access.AUTH_ERROR, "The credential is invalid or revoked.")
|
| 20 |
|
|
@@ -40,6 +90,10 @@ def require_external_session(
|
|
| 40 |
"u": verified.user["username"],
|
| 41 |
"external": True,
|
| 42 |
"external_key_id": verified.key_id,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
},
|
| 44 |
runtime=verified.runtime,
|
| 45 |
)
|
|
@@ -64,13 +118,17 @@ def _can_use_analyst(session: Session) -> bool:
|
|
| 64 |
|
| 65 |
|
| 66 |
def _management_refusal(exc: external_access.ExternalAccessError):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
if exc.code == "external_label_invalid":
|
| 68 |
raise err(400, exc.code, "Enter a label between 1 and 80 characters.")
|
| 69 |
if exc.code == "external_key_not_found":
|
| 70 |
raise err(404, exc.code, "That credential was not found.")
|
| 71 |
if exc.code in {"external_identity_invalid", "external_account_unavailable"}:
|
| 72 |
raise err(409, exc.code, "This account cannot create an external credential.")
|
| 73 |
-
raise err(503, "external_access_unavailable", "
|
| 74 |
|
| 75 |
|
| 76 |
def _resolve_external_scope(
|
|
@@ -128,6 +186,8 @@ def _resolve_external_scope(
|
|
| 128 |
else:
|
| 129 |
row["label"] = fallback_label(database)
|
| 130 |
row["fields"] = []
|
|
|
|
|
|
|
| 131 |
resolved.append(row)
|
| 132 |
return resolved
|
| 133 |
|
|
@@ -150,13 +210,14 @@ def _portable_skill_document(session: Session, base_url: str) -> str:
|
|
| 150 |
lines = [
|
| 151 |
"---",
|
| 152 |
"name: loopable-external-data",
|
| 153 |
-
"description: Read
|
| 154 |
"---",
|
| 155 |
"",
|
| 156 |
"# Loopable external data",
|
| 157 |
"",
|
| 158 |
-
("This surface
|
| 159 |
-
"
|
|
|
|
| 160 |
"",
|
| 161 |
"## Authentication",
|
| 162 |
"",
|
|
@@ -268,12 +329,67 @@ def _portable_skill_document(session: Session, base_url: str) -> str:
|
|
| 268 |
])
|
| 269 |
lines.append("")
|
| 270 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
lines.extend([
|
| 272 |
"## Limits",
|
| 273 |
"",
|
| 274 |
-
("This surface
|
| 275 |
-
"
|
| 276 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
"",
|
| 278 |
])
|
| 279 |
document = "\n".join(lines)
|
|
@@ -287,8 +403,11 @@ def create_key(body=Body(default=None), session: Session = Depends(require_sessi
|
|
| 287 |
"""Mint a credential for the signed-in human. Bearer auth is not accepted here."""
|
| 288 |
session.require("analyst")
|
| 289 |
label = body.get("label") if isinstance(body, dict) else None
|
|
|
|
|
|
|
| 290 |
try:
|
| 291 |
-
key, token = external_access.mint(session.runtime, session.uname, session.tenant, label
|
|
|
|
| 292 |
except external_access.ExternalAccessError as exc:
|
| 293 |
_management_refusal(exc)
|
| 294 |
return {"key": key, "secret": token}
|
|
@@ -316,7 +435,7 @@ def revoke_key(key_id: str, session: Session = Depends(require_session)):
|
|
| 316 |
|
| 317 |
@router.get("/skill.md")
|
| 318 |
def portable_skill(request: Request, session: Session = Depends(require_session)):
|
| 319 |
-
"""Download
|
| 320 |
session.require("analyst")
|
| 321 |
document = _portable_skill_document(session, str(request.base_url))
|
| 322 |
return Response(
|
|
@@ -328,21 +447,39 @@ def portable_skill(request: Request, session: Session = Depends(require_session)
|
|
| 328 |
|
| 329 |
@router.get("/whoami")
|
| 330 |
def whoami(session: Session = Depends(require_external_analyst)):
|
| 331 |
-
"""
|
|
|
|
| 332 |
return {
|
| 333 |
"username": session.uname,
|
| 334 |
"name": session.user.get("name") or session.uname,
|
| 335 |
"tenant": session.tenant,
|
| 336 |
"key_id": session.claims["external_key_id"],
|
| 337 |
-
"
|
|
|
|
|
|
|
|
|
|
| 338 |
}
|
| 339 |
|
| 340 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 341 |
@router.get("/databases")
|
| 342 |
def list_databases(session: Session = Depends(require_external_analyst)):
|
| 343 |
"""List only sources this user may read or already sees elsewhere in the product."""
|
| 344 |
databases = []
|
|
|
|
| 345 |
for row in _resolve_external_scope(session):
|
|
|
|
|
|
|
| 346 |
if not (row.get("permitted") or row.get("visible")):
|
| 347 |
continue
|
| 348 |
public = {
|
|
@@ -353,12 +490,14 @@ def list_databases(session: Session = Depends(require_external_analyst)):
|
|
| 353 |
if not public["answerable"]:
|
| 354 |
public["reason"] = row.get("reason") or ""
|
| 355 |
databases.append(public)
|
| 356 |
-
return {"databases": databases,
|
|
|
|
| 357 |
|
| 358 |
|
| 359 |
@router.get("/databases/{database}/schema")
|
| 360 |
def database_schema(database: str, session: Session = Depends(require_external_analyst)):
|
| 361 |
"""Return the field contract after resolving the ordinary strict Assistant read wall."""
|
|
|
|
| 362 |
resolved = _resolve_external_scope(
|
| 363 |
session, databases=[database], include_schema=True, strict_schema=True,
|
| 364 |
)
|
|
@@ -372,6 +511,7 @@ def query_database(
|
|
| 372 |
session: Session = Depends(require_external_analyst),
|
| 373 |
):
|
| 374 |
"""Read one detached Assistant snapshot; FastAPI rejects non-object request bodies."""
|
|
|
|
| 375 |
payload = body or {}
|
| 376 |
return deps.assistant_read_scope(
|
| 377 |
session,
|
|
@@ -381,19 +521,342 @@ def query_database(
|
|
| 381 |
)
|
| 382 |
|
| 383 |
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
# above. No bearer dependency appears on those lifecycle routes.
|
| 387 |
-
_READ_ONLY_METHODS = frozenset({"GET", "POST", "HEAD", "OPTIONS"})
|
| 388 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 389 |
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 397 |
|
| 398 |
|
| 399 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Per-user credentials, governed reads and explicitly allowed bearer build tools."""
|
| 2 |
|
| 3 |
+
import copy
|
| 4 |
import json
|
| 5 |
+
import uuid
|
| 6 |
from urllib.parse import quote
|
| 7 |
|
| 8 |
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response
|
| 9 |
+
from starlette.concurrency import run_in_threadpool
|
| 10 |
|
| 11 |
import deps
|
| 12 |
from deps import Session, err, require_session
|
| 13 |
+
from core import external_access, external_tools, sheets
|
| 14 |
|
| 15 |
|
| 16 |
router = APIRouter(prefix="/api/v1/external", tags=["external"])
|
|
|
|
| 18 |
_CREDENTIAL_PREFIX = "lpb1."
|
| 19 |
|
| 20 |
|
| 21 |
+
def _key_scope(session):
|
| 22 |
+
"""(access, databases) for THIS request. A session without the claims is an in-app human."""
|
| 23 |
+
claims = session.claims or {}
|
| 24 |
+
return external_access.scope_of({
|
| 25 |
+
"access": claims.get("external_access"),
|
| 26 |
+
"databases": claims.get("external_databases"),
|
| 27 |
+
})
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _assert_database_scope(session, database):
|
| 31 |
+
if external_access.database_allowed(_key_scope(session)[1], database):
|
| 32 |
+
return
|
| 33 |
+
# A binding can carry no database at all. Name what was asked for when there is a name, and
|
| 34 |
+
# never interpolate a bare None into a sentence a caller reads.
|
| 35 |
+
named = str(database or "").strip()
|
| 36 |
+
raise err(403, "key_database_out_of_scope",
|
| 37 |
+
f"Database {named} is outside this credential's scope." if named
|
| 38 |
+
else "This credential is limited to named databases, and none was named.")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _assert_write_scope(session, tool_name):
|
| 42 |
+
if not external_access.may_write(_key_scope(session)[0]):
|
| 43 |
+
raise err(403, "key_read_only", f"{tool_name} writes, and this credential is read only.")
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _databases_in_arguments(value):
|
| 47 |
+
"""Find database references, including bound cell documents, without unbounded recursion."""
|
| 48 |
+
found = []
|
| 49 |
+
|
| 50 |
+
def scan(node, depth):
|
| 51 |
+
if depth > 12:
|
| 52 |
+
return
|
| 53 |
+
if isinstance(node, dict):
|
| 54 |
+
for key, item in node.items():
|
| 55 |
+
if key in ("database", "sourceDatabase") and isinstance(item, str):
|
| 56 |
+
found.append(item)
|
| 57 |
+
elif key == "compatibleDatabases" and isinstance(item, list):
|
| 58 |
+
found.extend(database for database in item if isinstance(database, str))
|
| 59 |
+
scan(item, depth + 1)
|
| 60 |
+
elif isinstance(node, (list, tuple)):
|
| 61 |
+
for item in node:
|
| 62 |
+
scan(item, depth + 1)
|
| 63 |
+
|
| 64 |
+
scan(value, 0)
|
| 65 |
+
return found
|
| 66 |
+
|
| 67 |
+
|
| 68 |
def _auth_refusal():
|
| 69 |
raise err(401, external_access.AUTH_ERROR, "The credential is invalid or revoked.")
|
| 70 |
|
|
|
|
| 90 |
"u": verified.user["username"],
|
| 91 |
"external": True,
|
| 92 |
"external_key_id": verified.key_id,
|
| 93 |
+
"external_access": verified.access,
|
| 94 |
+
"external_databases": (external_access.ALL_DATABASES
|
| 95 |
+
if verified.databases == external_access.ALL_DATABASES
|
| 96 |
+
else list(verified.databases)),
|
| 97 |
},
|
| 98 |
runtime=verified.runtime,
|
| 99 |
)
|
|
|
|
| 118 |
|
| 119 |
|
| 120 |
def _management_refusal(exc: external_access.ExternalAccessError):
|
| 121 |
+
if exc.code == "external_access_scope_invalid":
|
| 122 |
+
raise err(400, exc.code, "Choose Read only or Read and write.")
|
| 123 |
+
if exc.code == "external_databases_invalid":
|
| 124 |
+
raise err(400, exc.code, "Choose all databases, or name at least one database.")
|
| 125 |
if exc.code == "external_label_invalid":
|
| 126 |
raise err(400, exc.code, "Enter a label between 1 and 80 characters.")
|
| 127 |
if exc.code == "external_key_not_found":
|
| 128 |
raise err(404, exc.code, "That credential was not found.")
|
| 129 |
if exc.code in {"external_identity_invalid", "external_account_unavailable"}:
|
| 130 |
raise err(409, exc.code, "This account cannot create an external credential.")
|
| 131 |
+
raise err(503, "external_access_unavailable", "The AI API is unavailable right now.")
|
| 132 |
|
| 133 |
|
| 134 |
def _resolve_external_scope(
|
|
|
|
| 186 |
else:
|
| 187 |
row["label"] = fallback_label(database)
|
| 188 |
row["fields"] = []
|
| 189 |
+
else:
|
| 190 |
+
row["label"] = fallback_label(database)
|
| 191 |
resolved.append(row)
|
| 192 |
return resolved
|
| 193 |
|
|
|
|
| 210 |
lines = [
|
| 211 |
"---",
|
| 212 |
"name: loopable-external-data",
|
| 213 |
+
"description: Read governed data and build Sheet workbooks, saved Views and Apps as the key's maker.",
|
| 214 |
"---",
|
| 215 |
"",
|
| 216 |
"# Loopable external data",
|
| 217 |
"",
|
| 218 |
+
("This surface reads governed data and writes only through the named build tools, "
|
| 219 |
+
"as the key's maker with the maker's own grants. It applies the same database, "
|
| 220 |
+
"field, row and workbook permissions as the Loopable app."),
|
| 221 |
"",
|
| 222 |
"## Authentication",
|
| 223 |
"",
|
|
|
|
| 329 |
])
|
| 330 |
lines.append("")
|
| 331 |
|
| 332 |
+
example_call = {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {
|
| 333 |
+
"name": "sheet_create_workbook", "arguments": {"name": "Gateway workbook"}}}
|
| 334 |
+
lines.extend([
|
| 335 |
+
"## Remote MCP tools",
|
| 336 |
+
"",
|
| 337 |
+
"Endpoint: `POST /api/v1/external/mcp`",
|
| 338 |
+
"",
|
| 339 |
+
("Send one JSON-RPC 2.0 object per request. Methods: initialize, tools/list, tools/call "
|
| 340 |
+
"and ping. Initialize echoes your protocolVersion and advertises tools. Requests "
|
| 341 |
+
"without an id, or with a null id, receive HTTP 204."),
|
| 342 |
+
"",
|
| 343 |
+
"```json",
|
| 344 |
+
json.dumps(example_call, indent=2),
|
| 345 |
+
"```",
|
| 346 |
+
"",
|
| 347 |
+
("A tools/call response has jsonrpc, id and result. The result contains "
|
| 348 |
+
"content: [{type: 'text', text: '<JSON {ok, data or error}>'}] and isError. "
|
| 349 |
+
"A tool refusal uses HTTP 200 with ok: false and isError: true; its error names "
|
| 350 |
+
"the tool, code, status and reason. Invalid or revoked credentials receive HTTP 401."),
|
| 351 |
+
"",
|
| 352 |
+
"```sh",
|
| 353 |
+
"curl --request POST \\",
|
| 354 |
+
" --header 'Authorization: Bearer <your Loopable credential>' \\",
|
| 355 |
+
" --header 'Content-Type: application/json' \\",
|
| 356 |
+
f" --data '{json.dumps(example_call, separators=(',', ':'))}' \\",
|
| 357 |
+
f" '{surface_url}/mcp'",
|
| 358 |
+
"```",
|
| 359 |
+
"",
|
| 360 |
+
("Sheet tools require analyst and sheets grants. Geometry patches carry the revision "
|
| 361 |
+
"read by the server; pass rev to also protect the revision you read. Cell writes "
|
| 362 |
+
"return accepted and refused addresses, retaining the app's partial-write behavior."),
|
| 363 |
+
"",
|
| 364 |
+
("View tools use the grid's native config: filters, sorts, groupBy, visible and display. "
|
| 365 |
+
"Fields omitted from visible are hidden. The grid validates the config. App tools use "
|
| 366 |
+
"the Application registry's typed config. app_delete requires the exact current "
|
| 367 |
+
"App name in confirmName, including after a rename."),
|
| 368 |
+
"",
|
| 369 |
+
])
|
| 370 |
+
for name, tool in external_tools.TOOLS.items():
|
| 371 |
+
lines.extend([
|
| 372 |
+
f"### `{name}`",
|
| 373 |
+
"",
|
| 374 |
+
_markdown_inline(tool["description"]),
|
| 375 |
+
"",
|
| 376 |
+
"Writes: " + ("yes" if tool["write"] else "no") + ".",
|
| 377 |
+
"",
|
| 378 |
+
"```json",
|
| 379 |
+
json.dumps(tool["inputSchema"], indent=2, ensure_ascii=True),
|
| 380 |
+
"```",
|
| 381 |
+
"",
|
| 382 |
+
])
|
| 383 |
lines.extend([
|
| 384 |
"## Limits",
|
| 385 |
"",
|
| 386 |
+
("This surface writes only through the named tools, as the key's maker with the maker's "
|
| 387 |
+
"own grants. Workbook roles and database restrictions still apply. Each credential also "
|
| 388 |
+
"carries its own access level and database scope, set when it is created and shown in "
|
| 389 |
+
"Loopable Settings. It returns no data "
|
| 390 |
+
"the maker cannot already see in Loopable. Sheet cell history records the maker and "
|
| 391 |
+
"key label. Saved Views and Apps keep the UI's username attribution; their writers "
|
| 392 |
+
"have no author display-name field. No tool writes to Odoo."),
|
| 393 |
"",
|
| 394 |
])
|
| 395 |
document = "\n".join(lines)
|
|
|
|
| 403 |
"""Mint a credential for the signed-in human. Bearer auth is not accepted here."""
|
| 404 |
session.require("analyst")
|
| 405 |
label = body.get("label") if isinstance(body, dict) else None
|
| 406 |
+
access = body.get("access") if isinstance(body, dict) else None
|
| 407 |
+
databases = body.get("databases") if isinstance(body, dict) else None
|
| 408 |
try:
|
| 409 |
+
key, token = external_access.mint(session.runtime, session.uname, session.tenant, label,
|
| 410 |
+
access=access, databases=databases)
|
| 411 |
except external_access.ExternalAccessError as exc:
|
| 412 |
_management_refusal(exc)
|
| 413 |
return {"key": key, "secret": token}
|
|
|
|
| 435 |
|
| 436 |
@router.get("/skill.md")
|
| 437 |
def portable_skill(request: Request, session: Session = Depends(require_session)):
|
| 438 |
+
"""Download current governed external read and build instructions."""
|
| 439 |
session.require("analyst")
|
| 440 |
document = _portable_skill_document(session, str(request.base_url))
|
| 441 |
return Response(
|
|
|
|
| 447 |
|
| 448 |
@router.get("/whoami")
|
| 449 |
def whoami(session: Session = Depends(require_external_analyst)):
|
| 450 |
+
"""Name the authenticated maker and the explicit limits on bearer writes."""
|
| 451 |
+
access, databases = _key_scope(session)
|
| 452 |
return {
|
| 453 |
"username": session.uname,
|
| 454 |
"name": session.user.get("name") or session.uname,
|
| 455 |
"tenant": session.tenant,
|
| 456 |
"key_id": session.claims["external_key_id"],
|
| 457 |
+
"access": access,
|
| 458 |
+
"databases": (external_access.ALL_DATABASES
|
| 459 |
+
if databases == external_access.ALL_DATABASES else list(databases)),
|
| 460 |
+
"write_scope": "Named tools only, as the key's maker with the maker's own grants.",
|
| 461 |
}
|
| 462 |
|
| 463 |
|
| 464 |
+
@router.get("/scope/databases")
|
| 465 |
+
def scope_databases(session: Session = Depends(require_session)):
|
| 466 |
+
"""List the databases this signed-in maker could put inside a new credential's scope."""
|
| 467 |
+
session.require("analyst")
|
| 468 |
+
return {"databases": [
|
| 469 |
+
{key: row[key] for key in ("database", "label", "permitted", "answerable")}
|
| 470 |
+
for row in _resolve_external_scope(session)
|
| 471 |
+
if row.get("permitted") or row.get("visible")
|
| 472 |
+
]}
|
| 473 |
+
|
| 474 |
+
|
| 475 |
@router.get("/databases")
|
| 476 |
def list_databases(session: Session = Depends(require_external_analyst)):
|
| 477 |
"""List only sources this user may read or already sees elsewhere in the product."""
|
| 478 |
databases = []
|
| 479 |
+
key_databases = _key_scope(session)[1]
|
| 480 |
for row in _resolve_external_scope(session):
|
| 481 |
+
if not external_access.database_allowed(key_databases, row["database"]):
|
| 482 |
+
continue
|
| 483 |
if not (row.get("permitted") or row.get("visible")):
|
| 484 |
continue
|
| 485 |
public = {
|
|
|
|
| 490 |
if not public["answerable"]:
|
| 491 |
public["reason"] = row.get("reason") or ""
|
| 492 |
databases.append(public)
|
| 493 |
+
return {"databases": databases,
|
| 494 |
+
"write_scope": "Named tools only, as the key's maker with the maker's own grants."}
|
| 495 |
|
| 496 |
|
| 497 |
@router.get("/databases/{database}/schema")
|
| 498 |
def database_schema(database: str, session: Session = Depends(require_external_analyst)):
|
| 499 |
"""Return the field contract after resolving the ordinary strict Assistant read wall."""
|
| 500 |
+
_assert_database_scope(session, database)
|
| 501 |
resolved = _resolve_external_scope(
|
| 502 |
session, databases=[database], include_schema=True, strict_schema=True,
|
| 503 |
)
|
|
|
|
| 511 |
session: Session = Depends(require_external_analyst),
|
| 512 |
):
|
| 513 |
"""Read one detached Assistant snapshot; FastAPI rejects non-object request bodies."""
|
| 514 |
+
_assert_database_scope(session, database)
|
| 515 |
payload = body or {}
|
| 516 |
return deps.assistant_read_scope(
|
| 517 |
session,
|
|
|
|
| 521 |
)
|
| 522 |
|
| 523 |
|
| 524 |
+
def _view_context(session, database, event):
|
| 525 |
+
import routes_grid
|
|
|
|
|
|
|
| 526 |
|
| 527 |
+
scope = routes_grid._scope_or_400(
|
| 528 |
+
{"customer_data": "customer", "product_data": "product"}.get(database, database))
|
| 529 |
+
try:
|
| 530 |
+
if scope.startswith("ut_"):
|
| 531 |
+
from routes_tables import ut_write_ctx
|
| 532 |
+
|
| 533 |
+
g = ut_write_ctx(session, scope)
|
| 534 |
+
if g.get("limits") and (event.get("pid") is not None or event.get("pids")):
|
| 535 |
+
limit = g["limits"][0]
|
| 536 |
+
raise err(409, "pid_scope_unresolved",
|
| 537 |
+
"This connector mirror's record scope is unresolved, so a change "
|
| 538 |
+
"addressed to particular records cannot be admitted. "
|
| 539 |
+
f"{limit.get('cause') or 'The row set is unresolved'}. "
|
| 540 |
+
f"{limit.get('recommendation') or ''}".strip())
|
| 541 |
+
elif scope == "product":
|
| 542 |
+
from routes_products import MODULE, product_assembly
|
| 543 |
+
|
| 544 |
+
session.require(MODULE)
|
| 545 |
+
g = product_assembly(session, consume_corrections=False)
|
| 546 |
+
else:
|
| 547 |
+
from routes_customers import grid_assembly
|
| 548 |
|
| 549 |
+
session.require(routes_grid.MODULE)
|
| 550 |
+
g = grid_assembly(session, scope=scope, consume_corrections=False)
|
| 551 |
+
except HTTPException as exc:
|
| 552 |
+
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 553 |
+
raise err(exc.status_code, info.get("code", "view_refused"),
|
| 554 |
+
f"Database {database}: {info.get('message') or 'The app refused access.'}") from exc
|
| 555 |
+
ctx = routes_grid._ctx(
|
| 556 |
+
session, g["fields"], g["pids"], scope_key=scope,
|
| 557 |
+
measure_keys=frozenset(m["key"] for m in g["measures"]),
|
| 558 |
+
resolved_ids=frozenset(g["measure_sets"]),
|
| 559 |
+
cohort_ids=frozenset(c["id"] for c in g["lists"]),
|
| 560 |
+
measure_offer=tuple(g["measures"]), visible_views=tuple(g["views"]),
|
| 561 |
+
recurrence_validator=lambda raw: routes_grid.govern_date_recurrence(
|
| 562 |
+
session, scope, g["fields"], raw))
|
| 563 |
+
if scope.startswith("ut_"):
|
| 564 |
+
ctx.hidden_keys = frozenset(g["hidden"])
|
| 565 |
+
return ctx
|
| 566 |
+
|
| 567 |
+
|
| 568 |
+
def _application_access(session, databases, *, strict=False):
|
| 569 |
+
import routes_assets
|
| 570 |
+
|
| 571 |
+
if not strict:
|
| 572 |
+
return routes_assets.application_access_context(session, databases)
|
| 573 |
+
allowed, records = set(), {}
|
| 574 |
+
for database in databases or ():
|
| 575 |
+
try:
|
| 576 |
+
permitted, semantic = routes_assets.application_access_context(
|
| 577 |
+
session, [database], strict=True)
|
| 578 |
+
except PermissionError as exc:
|
| 579 |
+
# Name only the submitted database, never a hidden registry object's source.
|
| 580 |
+
raise err(403, "forbidden", f"Database {database}: {exc}") from exc
|
| 581 |
+
allowed |= permitted
|
| 582 |
+
records.update(semantic)
|
| 583 |
+
return allowed, records
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def _view_event(session, kind, args):
|
| 587 |
+
from core import grid_events
|
| 588 |
+
|
| 589 |
+
database = args["database"]
|
| 590 |
+
_assert_database_scope(session, database)
|
| 591 |
+
event = {"id": uuid.uuid4().hex, "type": kind}
|
| 592 |
+
if kind == "view_upsert":
|
| 593 |
+
raw = args["view"]
|
| 594 |
+
if (not isinstance(raw.get("config"), dict)
|
| 595 |
+
or not isinstance(raw.get("id"), str) or not raw["id"].strip()
|
| 596 |
+
or not isinstance(raw.get("name"), str) or not raw["name"].strip()):
|
| 597 |
+
raise err(400, "tool_arguments_invalid", "A View needs an id, a name and an object config.")
|
| 598 |
+
view_id = raw["id"]
|
| 599 |
+
event["view"] = raw
|
| 600 |
+
# A membership config also addresses records, even though the payload nests its ids.
|
| 601 |
+
if raw["config"].get("memberPids"):
|
| 602 |
+
event["pids"] = raw["config"]["memberPids"]
|
| 603 |
+
else:
|
| 604 |
+
view_id = args["view_id"]
|
| 605 |
+
event["viewId"] = view_id
|
| 606 |
+
if not view_id.strip() or len(view_id) > 120 or view_id != view_id.strip():
|
| 607 |
+
raise err(400, "tool_arguments_invalid", "A View id must have 1 to 120 characters without surrounding spaces.")
|
| 608 |
+
ctx = _view_context(session, database, event)
|
| 609 |
+
try:
|
| 610 |
+
before = ctx.table.find_view(view_id, prefer_owner=session.uname)
|
| 611 |
+
rerender = grid_events.handle_one(event, ctx)
|
| 612 |
+
except grid_events.StoreUnavailable as exc:
|
| 613 |
+
raise err(503, "store_unavailable", f"View {view_id}: the tenant store is unavailable.") from exc
|
| 614 |
+
if ctx.out.refusals:
|
| 615 |
+
refusal = ctx.out.refusals[0]
|
| 616 |
+
raise err(400, refusal.get("code", "view_refused"),
|
| 617 |
+
f"View {view_id}: {refusal.get('message') or 'The app refused this change.'}")
|
| 618 |
+
if ctx.out.toast:
|
| 619 |
+
raise err(403, "view_refused", f"View {view_id}: {ctx.out.toast}")
|
| 620 |
+
after = ctx.table.find_view(view_id, prefer_owner=session.uname)
|
| 621 |
+
changed = before != after
|
| 622 |
+
if not changed:
|
| 623 |
+
raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.")
|
| 624 |
+
# ⛔⛔ `handle_one`'s BOOL IS A RERENDER FLAG, NOT A SUCCESS FLAG, and reading it as success is
|
| 625 |
+
# the mistake this comment exists to stop the next reader making. Its own contract block
|
| 626 |
+
# (`grid_events.py:1182`) says so in as many words: "view_delete - False (the deleted view
|
| 627 |
+
# simply stops being served). view_upsert - unchanged: False, except an unresolved measure
|
| 628 |
+
# condition." So an ordinary successful view write returns False, and refusing on it would
|
| 629 |
+
# refuse every save that worked.
|
| 630 |
+
# ⭐ THE SILENT NO-OP IS CAUGHT BY `changed` ABOVE INSTEAD, which is the scar this tool was
|
| 631 |
+
# written against: `view_upsert` answers 200 and writes nothing when the view id belongs to
|
| 632 |
+
# another user ([[a-view-id-another-user-holds-refuses-silently]]), and a store read before and
|
| 633 |
+
# after is the only thing that can tell that apart from a save that landed
|
| 634 |
+
# ([[lost-write-looks-like-failed-read]]). `rerender` rides out as information, never as the
|
| 635 |
+
# verdict.
|
| 636 |
+
return {"database": database, "view_id": view_id, "rerender": bool(rerender)}
|
| 637 |
+
|
| 638 |
+
|
| 639 |
+
def _application_upsert(session, args):
|
| 640 |
+
import application_registry as applications
|
| 641 |
+
import routes_assets
|
| 642 |
+
|
| 643 |
+
for database in args.get("compatibleDatabases") or []:
|
| 644 |
+
_assert_database_scope(session, database)
|
| 645 |
+
if args.get("sourceDatabase") is not None:
|
| 646 |
+
_assert_database_scope(session, args["sourceDatabase"])
|
| 647 |
+
payload = {key: value for key, value in args.items() if key != "app_id"}
|
| 648 |
+
app_id = args.get("app_id")
|
| 649 |
+
if "app_id" in args and not str(app_id or "").strip():
|
| 650 |
+
raise err(400, "bad_application", "app_id must name an App to update.")
|
| 651 |
+
if not session.runtime.available():
|
| 652 |
+
raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was saved.")
|
| 653 |
+
try:
|
| 654 |
+
if not app_id:
|
| 655 |
+
databases = payload.get("compatibleDatabases") or []
|
| 656 |
+
if payload.get("sourceDatabase"):
|
| 657 |
+
databases = [*databases, payload["sourceDatabase"]]
|
| 658 |
+
allowed, records = _application_access(session, databases, strict=True)
|
| 659 |
+
return {"application": applications.create(
|
| 660 |
+
session.runtime, session.tenant, session.uname, payload,
|
| 661 |
+
semantic_records=records, allowed_databases=allowed)}
|
| 662 |
+
current = routes_assets._application_databases(session.runtime, app_id)
|
| 663 |
+
allowed, _ = _application_access(session, current)
|
| 664 |
+
if not current or not allowed:
|
| 665 |
+
raise applications.ApplicationError("application does not exist")
|
| 666 |
+
proposed = payload.get("compatibleDatabases") if "compatibleDatabases" in payload else current
|
| 667 |
+
proposed_allowed, records = _application_access(session, proposed, strict=True)
|
| 668 |
+
return {"application": applications.mutate(
|
| 669 |
+
session.runtime, app_id, session.uname, session.admin, payload,
|
| 670 |
+
semantic_records=records, allowed_databases=allowed | proposed_allowed,
|
| 671 |
+
target_database=next((database for database in proposed
|
| 672 |
+
if database in proposed_allowed), None))}
|
| 673 |
+
except (applications.ApplicationError, PermissionError) as exc:
|
| 674 |
+
routes_assets._application_error(exc)
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
def _tool_context(session: Session) -> external_tools.ToolContext:
|
| 678 |
+
# Import within the API layer. Core receives callbacks, never an upward dependency.
|
| 679 |
+
import routes_sheet
|
| 680 |
+
import formula_runtime
|
| 681 |
+
import routes_assets
|
| 682 |
+
from modules import sheet as sheet_mod
|
| 683 |
+
|
| 684 |
+
def project(book):
|
| 685 |
+
served = routes_sheet._read_for(session, book)
|
| 686 |
+
databases = _key_scope(session)[1]
|
| 687 |
+
if databases != external_access.ALL_DATABASES:
|
| 688 |
+
# The owner projection can share cells with its input; key scope is response-only.
|
| 689 |
+
served = copy.deepcopy(served)
|
| 690 |
+
for tab in served.get("sheets", []):
|
| 691 |
+
for cell in tab.get("cells", {}).values():
|
| 692 |
+
if not isinstance(cell, dict) or cell.get("kind") != "bound":
|
| 693 |
+
continue
|
| 694 |
+
binding = cell.get("binding")
|
| 695 |
+
database = binding.get("database") if isinstance(binding, dict) else None
|
| 696 |
+
if not external_access.database_allowed(databases, database):
|
| 697 |
+
cell["value"] = None
|
| 698 |
+
cell["display"] = ""
|
| 699 |
+
routes_sheet._recompute_formulas(served, formula_runtime, sheet_mod)
|
| 700 |
+
return served
|
| 701 |
+
|
| 702 |
+
def resolve_binding(binding, book, tab):
|
| 703 |
+
if isinstance(binding, dict):
|
| 704 |
+
_assert_database_scope(session, binding.get("database"))
|
| 705 |
+
return routes_sheet._resolve_cell_binding(session, binding, book, tab)
|
| 706 |
+
|
| 707 |
+
def cache_binding(workbook_id, tab, addr, binding, resolved):
|
| 708 |
+
# Same best-effort cache fill as the UI's compute_cell. A fresh value must replace
|
| 709 |
+
# this scope's old slot before an editor's read projection can use that slot.
|
| 710 |
+
try:
|
| 711 |
+
sheets.fill_scope_cache(session.runtime, workbook_id, tab + "!" + addr,
|
| 712 |
+
routes_sheet._binding_scope_key(session, resolved["database"], binding),
|
| 713 |
+
resolved["value"], routes_sheet._now_iso(), resolved["database"])
|
| 714 |
+
except Exception:
|
| 715 |
+
pass
|
| 716 |
+
|
| 717 |
+
label = external_access.label_for(session.runtime, session.claims["external_key_id"])
|
| 718 |
+
maker = session.user.get("name") or session.uname
|
| 719 |
+
actor_name = f"{maker} via key {label}"
|
| 720 |
+
if _CREDENTIAL_PREFIX in actor_name:
|
| 721 |
+
raise sheets.SheetRefusal("credential_in_attribution", "Maker names and key labels must not contain credential values.")
|
| 722 |
+
return external_tools.ToolContext(
|
| 723 |
+
session=session, actor_name=actor_name,
|
| 724 |
+
require_reader=lambda book: routes_sheet._require_reader(session, book),
|
| 725 |
+
require_editor=lambda book: routes_sheet._require_editor(session, book),
|
| 726 |
+
project_book=project,
|
| 727 |
+
resolve_binding=resolve_binding,
|
| 728 |
+
cache_binding=cache_binding,
|
| 729 |
+
list_databases=lambda: list_databases(session),
|
| 730 |
+
describe_database=lambda database: database_schema(database, session),
|
| 731 |
+
view_event=lambda kind, args: _view_event(session, kind, args),
|
| 732 |
+
app_upsert=lambda args: _application_upsert(session, args),
|
| 733 |
+
# Reuse the UI's atomic NameConfirmedDelete wrapper as well as the registry delete.
|
| 734 |
+
app_delete=lambda args: routes_assets.delete_application(
|
| 735 |
+
args["app_id"], {"confirmName": args["confirmName"]}, session),
|
| 736 |
+
)
|
| 737 |
|
| 738 |
|
| 739 |
+
def _tool_refusal(name, code, message, status=400, **extra):
|
| 740 |
+
# Never echo arbitrary method/tool names or raw exception text across the bearer seam.
|
| 741 |
+
tool_name = name if isinstance(name, str) and name in external_tools.TOOLS else "tools/call"
|
| 742 |
+
sentence = f"{tool_name}: {_markdown_inline(message)}"
|
| 743 |
+
if _CREDENTIAL_PREFIX in sentence:
|
| 744 |
+
sentence = f"{tool_name}: The result contained a credential value and was withheld."
|
| 745 |
+
return {"ok": False, "error": {"tool": tool_name, "code": code,
|
| 746 |
+
"message": sentence, "status": status, **extra}}
|
| 747 |
+
|
| 748 |
+
|
| 749 |
+
def _mcp_content(rid, out):
|
| 750 |
+
encoded = json.dumps(out, default=str, ensure_ascii=True, allow_nan=False)
|
| 751 |
+
if _CREDENTIAL_PREFIX in encoded:
|
| 752 |
+
out = _tool_refusal(None, "credential_in_output", "The result contained a credential value and was withheld.")
|
| 753 |
+
encoded = json.dumps(out)
|
| 754 |
+
return {"jsonrpc": "2.0", "id": rid, "result": {
|
| 755 |
+
"content": [{"type": "text", "text": encoded}], "isError": not out.get("ok", False)}}
|
| 756 |
+
|
| 757 |
+
|
| 758 |
+
def _handle_mcp(req, session: Session):
|
| 759 |
+
"""One HTTP request object; the stdio precedent's JSON-RPC and content envelopes."""
|
| 760 |
+
def protocol_error(rid, code, message):
|
| 761 |
+
return {"jsonrpc": "2.0", "id": rid, "error": {"code": code, "message": message}}
|
| 762 |
+
|
| 763 |
+
if not isinstance(req, dict):
|
| 764 |
+
return protocol_error(None, -32600, "An MCP request must be a JSON-RPC object.")
|
| 765 |
+
rid, method = req.get("id"), req.get("method")
|
| 766 |
+
if (rid is not None and (type(rid) not in (str, int) or _CREDENTIAL_PREFIX in str(rid))):
|
| 767 |
+
return protocol_error(None, -32600, "The request id must be a number or a safe string.")
|
| 768 |
+
if rid is None:
|
| 769 |
+
# Notifications never dispatch writes and never return a JSON-RPC body.
|
| 770 |
+
return None
|
| 771 |
+
if req.get("jsonrpc") != "2.0" or not isinstance(method, str):
|
| 772 |
+
return protocol_error(rid, -32600, "An MCP request needs jsonrpc 2.0 and a method.")
|
| 773 |
+
params = req.get("params", {})
|
| 774 |
+
if not isinstance(params, dict):
|
| 775 |
+
return protocol_error(rid, -32602, "MCP params must be an object.")
|
| 776 |
+
name = params.get("name")
|
| 777 |
+
try:
|
| 778 |
+
# Same Analyst wall as the read doors, inside the dispatcher so a grant refusal
|
| 779 |
+
# remains MCP content. Only the unchanged bearer dependency produces an HTTP 401.
|
| 780 |
+
require_external_analyst(session)
|
| 781 |
+
if method == "initialize":
|
| 782 |
+
protocol = params.get("protocolVersion") or "2025-06-18"
|
| 783 |
+
if not isinstance(protocol, str) or _CREDENTIAL_PREFIX in protocol:
|
| 784 |
+
return protocol_error(rid, -32602, "The protocol version must be a safe string.")
|
| 785 |
+
return {"jsonrpc": "2.0", "id": rid, "result": {
|
| 786 |
+
"protocolVersion": protocol, "capabilities": {"tools": {}},
|
| 787 |
+
"serverInfo": {"name": "loopable-external", "version": "1.0.0"}}}
|
| 788 |
+
if method == "ping":
|
| 789 |
+
return {"jsonrpc": "2.0", "id": rid, "result": {}}
|
| 790 |
+
if method == "tools/list":
|
| 791 |
+
return {"jsonrpc": "2.0", "id": rid, "result": {"tools": [
|
| 792 |
+
{"name": key, "description": tool["description"], "inputSchema": tool["inputSchema"]}
|
| 793 |
+
for key, tool in external_tools.TOOLS.items()]}}
|
| 794 |
+
if method != "tools/call":
|
| 795 |
+
return protocol_error(rid, -32601, "The requested MCP method was not found.")
|
| 796 |
+
if isinstance(name, str) and name in _WRITE_TOOLS:
|
| 797 |
+
_assert_write_scope(session, name)
|
| 798 |
+
for database in _databases_in_arguments(params.get("arguments", {})):
|
| 799 |
+
_assert_database_scope(session, database)
|
| 800 |
+
data = external_tools.dispatch(_tool_context(session), name, params.get("arguments", {}))
|
| 801 |
+
if isinstance(data, dict) and data.get("refused"):
|
| 802 |
+
first = data["refused"][0]
|
| 803 |
+
out = _tool_refusal(name, first["code"], first["sentence"])
|
| 804 |
+
out["error"]["data"] = data # Accepted edits remain visible even on partial refusal.
|
| 805 |
+
else:
|
| 806 |
+
out = {"ok": True, "data": data}
|
| 807 |
+
except external_access.ExternalAuthError:
|
| 808 |
+
_auth_refusal()
|
| 809 |
+
except external_access.ExternalAccessError:
|
| 810 |
+
out = _tool_refusal(name, "external_access_unavailable", "The AI API is unavailable right now.", 503)
|
| 811 |
+
except sheets.SheetRefusal as exc:
|
| 812 |
+
import routes_sheet
|
| 813 |
+
translated = routes_sheet._refuse(exc)
|
| 814 |
+
info = translated.detail["error"]
|
| 815 |
+
out = _tool_refusal(name, info["code"], info["message"], translated.status_code,
|
| 816 |
+
**{k: v for k, v in info.items() if k not in ("code", "message")})
|
| 817 |
+
except HTTPException as exc:
|
| 818 |
+
if exc.status_code == 401:
|
| 819 |
+
_auth_refusal()
|
| 820 |
+
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 821 |
+
out = _tool_refusal(name, info.get("code", "tool_refused"),
|
| 822 |
+
info.get("message", "The app refused this operation."), exc.status_code)
|
| 823 |
+
except (TypeError, ValueError, KeyError, AttributeError):
|
| 824 |
+
out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
|
| 825 |
+
except Exception:
|
| 826 |
+
out = _tool_refusal(name, "tool_unavailable", "The tool could not complete this operation.", 503)
|
| 827 |
+
return _mcp_content(rid, out)
|
| 828 |
+
|
| 829 |
+
|
| 830 |
+
@router.post("/mcp")
|
| 831 |
+
async def external_mcp(request: Request, session: Session = Depends(require_external_session)):
|
| 832 |
+
"""Remote MCP without a browser session, with the read doors' maker and Analyst wall."""
|
| 833 |
+
try:
|
| 834 |
+
req = await request.json()
|
| 835 |
+
except (ValueError, UnicodeError):
|
| 836 |
+
return {"jsonrpc": "2.0", "id": None,
|
| 837 |
+
"error": {"code": -32700, "message": "The request body must be valid JSON."}}
|
| 838 |
+
reply = await run_in_threadpool(_handle_mcp, req, session)
|
| 839 |
+
return Response(status_code=204) if reply is None else reply
|
| 840 |
+
|
| 841 |
+
|
| 842 |
+
# R8 (SDLC 61) revokes the read-only surface. Every write must be named here deliberately.
|
| 843 |
+
# A new write in the registry without a matching literal raises at import, before serving.
|
| 844 |
+
_WRITE_TOOLS = frozenset({
|
| 845 |
+
"sheet_create_workbook", "sheet_add_tab", "sheet_write_cells", "sheet_set_binding",
|
| 846 |
+
"sheet_set_styles", "sheet_set_dimensions", "sheet_set_validation", "sheet_set_freeze",
|
| 847 |
+
"sheet_add_rule", "sheet_set_chart", "sheet_structure",
|
| 848 |
+
"sheet_save_template",
|
| 849 |
+
"view_upsert", "view_delete", "app_upsert", "app_delete",
|
| 850 |
+
})
|
| 851 |
+
|
| 852 |
+
|
| 853 |
+
def _assert_write_allow_list() -> None:
|
| 854 |
+
declared = {name for name, tool in external_tools.TOOLS.items() if tool.get("write")}
|
| 855 |
+
undeclared = declared - _WRITE_TOOLS
|
| 856 |
+
stale = _WRITE_TOOLS - declared
|
| 857 |
+
if undeclared or stale:
|
| 858 |
+
raise RuntimeError("External write allow-list mismatch: undeclared="
|
| 859 |
+
+ repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
|
| 860 |
+
|
| 861 |
+
|
| 862 |
+
_assert_write_allow_list()
|
api/routes_grid.py
CHANGED
|
@@ -2491,9 +2491,11 @@ def _topic_rt(st, module):
|
|
| 2491 |
return st
|
| 2492 |
|
| 2493 |
|
| 2494 |
-
def _customer_rows(table_key, user, st):
|
| 2495 |
"""`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses."""
|
| 2496 |
import aios_grid
|
|
|
|
|
|
|
| 2497 |
import core.perm_scope as perm_scope
|
| 2498 |
from routes_customers import _pool_for
|
| 2499 |
|
|
@@ -2502,15 +2504,16 @@ def _customer_rows(table_key, user, st):
|
|
| 2502 |
return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent)
|
| 2503 |
|
| 2504 |
|
| 2505 |
-
def _product_rows(table_key, user, st):
|
| 2506 |
"""`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a
|
| 2507 |
BU-pinned reader gets that BU's field contract rather than the consolidated one."""
|
| 2508 |
import core.perm_scope as perm_scope
|
| 2509 |
from routes_products import _pool_for, pd_fields
|
| 2510 |
|
| 2511 |
-
rt = _topic_rt(st, table_key)
|
| 2512 |
team_id, _agent = perm_scope.derive_pool_scope(user, table_key)
|
| 2513 |
-
return pd_fields(consolidated=team_id is None, st=st),
|
|
|
|
| 2514 |
|
| 2515 |
|
| 2516 |
def _register_topic_rows():
|
|
@@ -2747,7 +2750,8 @@ def _may_open(session: Session, key: str, st=None, memo: dict = None):
|
|
| 2747 |
return _walled_fields(session, key, st=st, memo=memo) is not None
|
| 2748 |
|
| 2749 |
|
| 2750 |
-
def _source_fields(session: Session, table: str, st=None, memo: dict = None
|
|
|
|
| 2751 |
"""The link-bearing field list for `table`, under this session's field wall.
|
| 2752 |
|
| 2753 |
⛔⛔ THE PRE-SET COLUMNS ARE NOT IN `scoped_fields` AND THAT IS THE TRAP THIS FUNCTION EXISTS
|
|
@@ -2769,7 +2773,8 @@ def _source_fields(session: Session, table: str, st=None, memo: dict = None):
|
|
| 2769 |
import odoo_relational
|
| 2770 |
fields = fields + list(odoo_relational.module_link_fields(table) or ())
|
| 2771 |
except Exception: # noqa: BLE001
|
| 2772 |
-
|
|
|
|
| 2773 |
return fields
|
| 2774 |
|
| 2775 |
|
|
@@ -2826,7 +2831,7 @@ def _ensure_odoo_spec():
|
|
| 2826 |
return False
|
| 2827 |
|
| 2828 |
|
| 2829 |
-
def _unwalled_fields(defs: dict, key: str):
|
| 2830 |
"""`key`'s field list WITHOUT this session's FIELD wall, for the W53-T18 diff pass only.
|
| 2831 |
|
| 2832 |
⛔⛔ IT IS NEVER SERVED AND NEVER WIDENS A TAB. Its ONE consumer is the unfiltered half of
|
|
@@ -2849,7 +2854,8 @@ def _unwalled_fields(defs: dict, key: str):
|
|
| 2849 |
import odoo_relational
|
| 2850 |
fields = fields + list(odoo_relational.module_link_fields(key) or ())
|
| 2851 |
except Exception: # noqa: BLE001
|
| 2852 |
-
|
|
|
|
| 2853 |
return fields
|
| 2854 |
|
| 2855 |
|
|
@@ -2884,7 +2890,7 @@ def _relation_context(session: Session, table: str, memo: dict = None):
|
|
| 2884 |
than a silently empty grid — and `scoped_fields` is a `want_rows=False` door by construction.
|
| 2885 |
Nothing here is handed to a mutation; `grid_related` is a GET.
|
| 2886 |
⛔ IT CANNOT WIDEN A WALL, and that was checked rather than assumed: the lent handle serves
|
| 2887 |
-
|
| 2888 |
the real runtime, and `session.require` is not given it at all. MEASURED 2026-09-06 across all
|
| 2889 |
ten of tenant #0's databases: `scoped_fields` under the lent handle returned a field list
|
| 2890 |
EQUAL, key for key and value for value, to the one it returned under the bare runtime.
|
|
@@ -2895,16 +2901,19 @@ def _relation_context(session: Session, table: str, memo: dict = None):
|
|
| 2895 |
"""
|
| 2896 |
import core.user_tables as user_tables
|
| 2897 |
|
|
|
|
| 2898 |
try:
|
| 2899 |
defs = dict(user_tables.all_defs(st=session.runtime) or {})
|
| 2900 |
except Exception: # noqa: BLE001
|
| 2901 |
defs = {}
|
|
|
|
| 2902 |
try:
|
| 2903 |
lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs})
|
| 2904 |
except Exception: # noqa: BLE001
|
| 2905 |
# A store handle too old to lend is CORRECT-but-slow, exactly as `all_defs`' own fallback
|
| 2906 |
# is: the pass simply goes back to paying a round trip per wall resolution.
|
| 2907 |
lent = session.runtime
|
|
|
|
| 2908 |
|
| 2909 |
# ⭐ ONE WALL RESOLUTION PER TABLE, SHARED BY BOTH CALLBACKS. `open_memo` and `field_memo`
|
| 2910 |
# below still memoise their own composition (the grant gate, the pre-set link append); this is
|
|
@@ -2919,24 +2928,30 @@ def _relation_context(session: Session, table: str, memo: dict = None):
|
|
| 2919 |
def may_open(key):
|
| 2920 |
if key not in open_memo:
|
| 2921 |
open_memo[key] = _may_open(session, key, st=lent, memo=wall_memo)
|
|
|
|
|
|
|
| 2922 |
return open_memo[key]
|
| 2923 |
|
| 2924 |
def fields_for(key):
|
| 2925 |
if key not in field_memo:
|
| 2926 |
-
field_memo[key] = _source_fields(session, key, st=lent, memo=wall_memo
|
|
|
|
|
|
|
|
|
|
| 2927 |
return field_memo[key]
|
| 2928 |
|
| 2929 |
def open_fields_for(key):
|
| 2930 |
if key not in wide_memo:
|
| 2931 |
walled = list(fields_for(key) or ())
|
| 2932 |
seen = {str(f.get("key")) for f in walled if isinstance(f, dict)}
|
| 2933 |
-
extra = [f for f in _unwalled_fields(defs, key)
|
| 2934 |
if isinstance(f, dict) and str(f.get("key")) not in seen]
|
| 2935 |
wide_memo[key] = walled + extra
|
| 2936 |
return wide_memo[key]
|
| 2937 |
|
| 2938 |
-
|
| 2939 |
-
|
|
|
|
| 2940 |
|
| 2941 |
|
| 2942 |
def related_path_entries(session: Session, table: str, depth: int, ctx: dict = None):
|
|
@@ -2952,14 +2967,120 @@ def related_path_entries(session: Session, table: str, depth: int, ctx: dict = N
|
|
| 2952 |
"""
|
| 2953 |
import core.relations as relations
|
| 2954 |
|
| 2955 |
-
_ensure_odoo_spec()
|
| 2956 |
table = str(table or "")
|
| 2957 |
ctx = ctx or _relation_context(session, table)
|
|
|
|
|
|
|
| 2958 |
return relations.related_paths(table, fields=ctx["fields_for"](table), defs=ctx["defs"],
|
| 2959 |
depth=depth, may_open=ctx["may_open"],
|
| 2960 |
fields_for=ctx["fields_for"])
|
| 2961 |
|
| 2962 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2963 |
def related_refusals(session: Session, table: str, depth: int, entries: list, ctx: dict = None):
|
| 2964 |
"""W53-T18 / R13 / C3 — WHAT THE WALL REMOVED from `entries`, by name. The SIBLING of `related`.
|
| 2965 |
|
|
@@ -3031,6 +3152,33 @@ def _refusal_error(diag: dict):
|
|
| 3031 |
return exc
|
| 3032 |
|
| 3033 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3034 |
@router.get("/grid/{table}/related")
|
| 3035 |
def grid_related(table: str,
|
| 3036 |
depth: int = Query(default=2),
|
|
@@ -3102,12 +3250,27 @@ def grid_related(table: str,
|
|
| 3102 |
f"{relations.MAX_DEPTH} (those plus everything one further named hop away), and "
|
| 3103 |
f"this request asked for {depth}. Ask for 1 or {relations.MAX_DEPTH}")
|
| 3104 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3105 |
ctx = _relation_context(session, table, memo=wall_memo)
|
| 3106 |
entries = related_path_entries(session, table, depth, ctx)
|
| 3107 |
-
if
|
| 3108 |
-
wanted =
|
| 3109 |
-
if len(wanted) > relations.MAX_DEPTH:
|
| 3110 |
-
raise err(400, "over_depth", relations.over_depth_refusal(table, wanted))
|
| 3111 |
hit = relations.pick_path(entries, wanted)
|
| 3112 |
if hit is None:
|
| 3113 |
# ⭐⭐ W53-T18 / R13 / C3 — THE ONE GENERIC `unknown_via` IS NOW ONE OF FOUR NAMED
|
|
@@ -3127,7 +3290,19 @@ def grid_related(table: str,
|
|
| 3127 |
# ⚠ A RESOLVED PIN REPORTS NO REFUSALS. The caller named one path and got it; the other
|
| 3128 |
# tabs the wall removed are the LISTING's answer, not this one, and repeating them here
|
| 3129 |
# would attach an unasked-for permission report to every successful read.
|
| 3130 |
-
|
| 3131 |
-
|
| 3132 |
-
|
| 3133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2491 |
return st
|
| 2492 |
|
| 2493 |
|
| 2494 |
+
def _customer_rows(table_key, user, st, want_rows=True):
|
| 2495 |
"""`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses."""
|
| 2496 |
import aios_grid
|
| 2497 |
+
if not want_rows:
|
| 2498 |
+
return list(aios_grid.FIELDS), []
|
| 2499 |
import core.perm_scope as perm_scope
|
| 2500 |
from routes_customers import _pool_for
|
| 2501 |
|
|
|
|
| 2504 |
return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent)
|
| 2505 |
|
| 2506 |
|
| 2507 |
+
def _product_rows(table_key, user, st, want_rows=True):
|
| 2508 |
"""`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a
|
| 2509 |
BU-pinned reader gets that BU's field contract rather than the consolidated one."""
|
| 2510 |
import core.perm_scope as perm_scope
|
| 2511 |
from routes_products import _pool_for, pd_fields
|
| 2512 |
|
| 2513 |
+
rt = _topic_rt(st, table_key) if want_rows else None
|
| 2514 |
team_id, _agent = perm_scope.derive_pool_scope(user, table_key)
|
| 2515 |
+
return pd_fields(consolidated=team_id is None, st=st), (
|
| 2516 |
+
_pool_for(rt, team_id) if want_rows else [])
|
| 2517 |
|
| 2518 |
|
| 2519 |
def _register_topic_rows():
|
|
|
|
| 2750 |
return _walled_fields(session, key, st=st, memo=memo) is not None
|
| 2751 |
|
| 2752 |
|
| 2753 |
+
def _source_fields(session: Session, table: str, st=None, memo: dict = None,
|
| 2754 |
+
read_status: dict = None):
|
| 2755 |
"""The link-bearing field list for `table`, under this session's field wall.
|
| 2756 |
|
| 2757 |
⛔⛔ THE PRE-SET COLUMNS ARE NOT IN `scoped_fields` AND THAT IS THE TRAP THIS FUNCTION EXISTS
|
|
|
|
| 2773 |
import odoo_relational
|
| 2774 |
fields = fields + list(odoo_relational.module_link_fields(table) or ())
|
| 2775 |
except Exception: # noqa: BLE001
|
| 2776 |
+
if read_status is not None:
|
| 2777 |
+
read_status["cacheable"] = False
|
| 2778 |
return fields
|
| 2779 |
|
| 2780 |
|
|
|
|
| 2831 |
return False
|
| 2832 |
|
| 2833 |
|
| 2834 |
+
def _unwalled_fields(defs: dict, key: str, read_status: dict = None):
|
| 2835 |
"""`key`'s field list WITHOUT this session's FIELD wall, for the W53-T18 diff pass only.
|
| 2836 |
|
| 2837 |
⛔⛔ IT IS NEVER SERVED AND NEVER WIDENS A TAB. Its ONE consumer is the unfiltered half of
|
|
|
|
| 2854 |
import odoo_relational
|
| 2855 |
fields = fields + list(odoo_relational.module_link_fields(key) or ())
|
| 2856 |
except Exception: # noqa: BLE001
|
| 2857 |
+
if read_status is not None:
|
| 2858 |
+
read_status["cacheable"] = False
|
| 2859 |
return fields
|
| 2860 |
|
| 2861 |
|
|
|
|
| 2890 |
than a silently empty grid — and `scoped_fields` is a `want_rows=False` door by construction.
|
| 2891 |
Nothing here is handed to a mutation; `grid_related` is a GET.
|
| 2892 |
⛔ IT CANNOT WIDEN A WALL, and that was checked rather than assumed: the lent handle serves
|
| 2893 |
+
`user_tables`, (lazily, once) `object_shares` and read-only `__shared` overlays; other keys fall through to
|
| 2894 |
the real runtime, and `session.require` is not given it at all. MEASURED 2026-09-06 across all
|
| 2895 |
ten of tenant #0's databases: `scoped_fields` under the lent handle returned a field list
|
| 2896 |
EQUAL, key for key and value for value, to the one it returned under the bare runtime.
|
|
|
|
| 2901 |
"""
|
| 2902 |
import core.user_tables as user_tables
|
| 2903 |
|
| 2904 |
+
read_status = {"cacheable": True}
|
| 2905 |
try:
|
| 2906 |
defs = dict(user_tables.all_defs(st=session.runtime) or {})
|
| 2907 |
except Exception: # noqa: BLE001
|
| 2908 |
defs = {}
|
| 2909 |
+
read_status["cacheable"] = False
|
| 2910 |
try:
|
| 2911 |
lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs})
|
| 2912 |
except Exception: # noqa: BLE001
|
| 2913 |
# A store handle too old to lend is CORRECT-but-slow, exactly as `all_defs`' own fallback
|
| 2914 |
# is: the pass simply goes back to paying a round trip per wall resolution.
|
| 2915 |
lent = session.runtime
|
| 2916 |
+
read_status["cacheable"] = False
|
| 2917 |
|
| 2918 |
# ⭐ ONE WALL RESOLUTION PER TABLE, SHARED BY BOTH CALLBACKS. `open_memo` and `field_memo`
|
| 2919 |
# below still memoise their own composition (the grant gate, the pre-set link append); this is
|
|
|
|
| 2928 |
def may_open(key):
|
| 2929 |
if key not in open_memo:
|
| 2930 |
open_memo[key] = _may_open(session, key, st=lent, memo=wall_memo)
|
| 2931 |
+
if not open_memo[key]:
|
| 2932 |
+
read_status["cacheable"] = False
|
| 2933 |
return open_memo[key]
|
| 2934 |
|
| 2935 |
def fields_for(key):
|
| 2936 |
if key not in field_memo:
|
| 2937 |
+
field_memo[key] = _source_fields(session, key, st=lent, memo=wall_memo,
|
| 2938 |
+
read_status=read_status)
|
| 2939 |
+
if wall_memo.get(key) is None:
|
| 2940 |
+
read_status["cacheable"] = False
|
| 2941 |
return field_memo[key]
|
| 2942 |
|
| 2943 |
def open_fields_for(key):
|
| 2944 |
if key not in wide_memo:
|
| 2945 |
walled = list(fields_for(key) or ())
|
| 2946 |
seen = {str(f.get("key")) for f in walled if isinstance(f, dict)}
|
| 2947 |
+
extra = [f for f in _unwalled_fields(defs, key, read_status=read_status)
|
| 2948 |
if isinstance(f, dict) and str(f.get("key")) not in seen]
|
| 2949 |
wide_memo[key] = walled + extra
|
| 2950 |
return wide_memo[key]
|
| 2951 |
|
| 2952 |
+
read_status.update(defs=defs, may_open=may_open, fields_for=fields_for,
|
| 2953 |
+
open_fields_for=open_fields_for)
|
| 2954 |
+
return read_status
|
| 2955 |
|
| 2956 |
|
| 2957 |
def related_path_entries(session: Session, table: str, depth: int, ctx: dict = None):
|
|
|
|
| 2967 |
"""
|
| 2968 |
import core.relations as relations
|
| 2969 |
|
| 2970 |
+
spec_ready = _ensure_odoo_spec()
|
| 2971 |
table = str(table or "")
|
| 2972 |
ctx = ctx or _relation_context(session, table)
|
| 2973 |
+
if not spec_ready:
|
| 2974 |
+
ctx["cacheable"] = False
|
| 2975 |
return relations.related_paths(table, fields=ctx["fields_for"](table), defs=ctx["defs"],
|
| 2976 |
depth=depth, may_open=ctx["may_open"],
|
| 2977 |
fields_for=ctx["fields_for"])
|
| 2978 |
|
| 2979 |
|
| 2980 |
+
def related_measure_pin(session, target, related, requested):
|
| 2981 |
+
"""Compile metric membership with the rows door's exact path and leg resolvers.
|
| 2982 |
+
|
| 2983 |
+
SQL bindings are host declarations. No SQL, callbacks or row identities are trusted from
|
| 2984 |
+
the client beyond the origin id, named edge and requested subset of resolved target ids.
|
| 2985 |
+
Retain through rows as well as explicit hops so a metric cannot expand behind a row wall.
|
| 2986 |
+
"""
|
| 2987 |
+
from core import perm_scope, relations
|
| 2988 |
+
import routes_odoo_tables as door
|
| 2989 |
+
|
| 2990 |
+
if isinstance(related, str):
|
| 2991 |
+
related = door._json_arg(related, 'related')
|
| 2992 |
+
allowed = door._relation_target_ids(session, target, related)
|
| 2993 |
+
if set(requested) - allowed:
|
| 2994 |
+
raise err(403, 'metric_record_refused', 'A requested record is outside this linked grid.')
|
| 2995 |
+
source, via = str(related['table']), related.get('via')
|
| 2996 |
+
path = door._via_path(via)
|
| 2997 |
+
entries = related_path_entries(session, source, len(path) if path else 1)
|
| 2998 |
+
entry = relations.pick_path(entries, path) if path else relations.pick(entries, via)
|
| 2999 |
+
legs = [(source, entry)]
|
| 3000 |
+
if len(entry.get('path') or ()) > 1:
|
| 3001 |
+
first, middle, last = door._related_path_legs(session, source, entry, entries)
|
| 3002 |
+
legs = [(source, first), (middle, last)]
|
| 3003 |
+
|
| 3004 |
+
bindings = {
|
| 3005 |
+
'sale_order_line': [
|
| 3006 |
+
dict(table='customer_data', column='pid', sql='{alias}.order_partner_id'),
|
| 3007 |
+
dict(table='product_data', column='product_id', sql='{alias}.product_id'),
|
| 3008 |
+
dict(table='ut_odoo_orders', column='pid', sql='{alias}.order_id'),
|
| 3009 |
+
dict(table='ut_odoo_order_lines', column='pid', sql='{alias}.id'),
|
| 3010 |
+
dict(table='ut_odoo_agents', column='pid', sql='rp.agent_id'),
|
| 3011 |
+
],
|
| 3012 |
+
'sale_order': [
|
| 3013 |
+
dict(table='customer_data', column='pid', sql='{alias}.partner_id'),
|
| 3014 |
+
dict(table='ut_odoo_orders', column='pid', sql='{alias}.id'),
|
| 3015 |
+
dict(table='ut_odoo_agents', column='pid', sql='rp.agent_id'),
|
| 3016 |
+
dict(table='ut_odoo_order_lines', column='order_id', sql='{alias}.id'),
|
| 3017 |
+
],
|
| 3018 |
+
}
|
| 3019 |
+
members = {}
|
| 3020 |
+
def read(table, column, values, where):
|
| 3021 |
+
columns = {'pid', column}
|
| 3022 |
+
if table == 'product_data':
|
| 3023 |
+
columns.add('product_id')
|
| 3024 |
+
if table == 'ut_odoo_order_lines':
|
| 3025 |
+
columns.add('order_id')
|
| 3026 |
+
return door._relation_rows(session, table, columns, match=(column, values), where=where)
|
| 3027 |
+
|
| 3028 |
+
def retain(table, ids):
|
| 3029 |
+
rows = read(table, 'pid', ids, None)
|
| 3030 |
+
if table in members:
|
| 3031 |
+
existing = {r['pid'] for r in members[table]}
|
| 3032 |
+
if {r['pid'] for r in rows} != existing:
|
| 3033 |
+
raise err(409, 'metric_scope_unsupported',
|
| 3034 |
+
'This metric cannot distinguish repeated databases in the record path. '
|
| 3035 |
+
'Use a direct relation or add a metric binding for these separate roles.')
|
| 3036 |
+
rows = [r for r in rows if r['pid'] in existing]
|
| 3037 |
+
members[table] = rows
|
| 3038 |
+
|
| 3039 |
+
with relations.measure_pin_context(read=read, bindings=bindings,
|
| 3040 |
+
may_open=lambda key: _may_open(session, key)):
|
| 3041 |
+
ids = {int(related['record'])}
|
| 3042 |
+
retain(source, ids)
|
| 3043 |
+
if not members[source]:
|
| 3044 |
+
raise err(403, 'metric_record_refused', 'The origin record is unavailable to this account.')
|
| 3045 |
+
for table, leg in legs:
|
| 3046 |
+
link, via = leg.get('link') or {}, leg.get('via') or {}
|
| 3047 |
+
through = link.get('through')
|
| 3048 |
+
if through and link.get('selectionMode') != 'manual':
|
| 3049 |
+
reverse = via.get('direction') == 'reverse'
|
| 3050 |
+
left = str(link['throughTo'] if reverse else link['throughOn'])
|
| 3051 |
+
frm = str(via.get('from') or 'pid')
|
| 3052 |
+
origins = door._relation_rows(session, table, [frm], ids=ids)
|
| 3053 |
+
bridge = read(str(through), left,
|
| 3054 |
+
[r[frm] for r in origins if r.get(frm) is not None], None)
|
| 3055 |
+
retain(str(through), {r['pid'] for r in bridge})
|
| 3056 |
+
ids = door._relation_leg_ids(session, table, ids, leg)
|
| 3057 |
+
retain(str(leg['target']), ids)
|
| 3058 |
+
required = tuple(members)
|
| 3059 |
+
# A manual link need not name the sales bridge. Its metrics still obey that
|
| 3060 |
+
# database's row wall, intersected with the already resolved record membership.
|
| 3061 |
+
if 'ut_odoo_order_lines' not in members:
|
| 3062 |
+
for table, column, join in [('customer_data', 'pid', 'partner_id'),
|
| 3063 |
+
('product_data', 'product_id', 'product_id'),
|
| 3064 |
+
('ut_odoo_orders', 'pid', 'order_id')]:
|
| 3065 |
+
if table in members:
|
| 3066 |
+
bridge = read('ut_odoo_order_lines', join,
|
| 3067 |
+
[r[column] for r in members[table]], None)
|
| 3068 |
+
# This extra wall belongs to line metrics. Header metrics use their own
|
| 3069 |
+
# traversed order membership, including legitimate orders with no lines.
|
| 3070 |
+
retain('ut_odoo_order_lines', {r['pid'] for r in bridge})
|
| 3071 |
+
bindings['sale_order'] = [binding for binding in bindings['sale_order']
|
| 3072 |
+
if binding['table'] != 'ut_odoo_order_lines']
|
| 3073 |
+
break
|
| 3074 |
+
# Required path tables must all bind. Binding only the destination would return globals.
|
| 3075 |
+
pin = relations.MeasurePin(entry['path'], members, bindings,
|
| 3076 |
+
required=required)
|
| 3077 |
+
scopes = [perm_scope.derive_pool_scope(session.user, table) for table in required]
|
| 3078 |
+
teams = {team for team, _agent in scopes if team is not None}
|
| 3079 |
+
if len(teams) > 1:
|
| 3080 |
+
raise err(403, 'metric_scope_conflict', 'The linked databases have incompatible business unit scopes.')
|
| 3081 |
+
return pin, next(iter(teams), None)
|
| 3082 |
+
|
| 3083 |
+
|
| 3084 |
def related_refusals(session: Session, table: str, depth: int, entries: list, ctx: dict = None):
|
| 3085 |
"""W53-T18 / R13 / C3 — WHAT THE WALL REMOVED from `entries`, by name. The SIBLING of `related`.
|
| 3086 |
|
|
|
|
| 3152 |
return exc
|
| 3153 |
|
| 3154 |
|
| 3155 |
+
_RELATED_CACHE_LIMIT = 128
|
| 3156 |
+
|
| 3157 |
+
|
| 3158 |
+
def _related_cache_key(session: Session, table: str, depth: int, via):
|
| 3159 |
+
"""Names-only answers follow this user's schema, grant, sync and registrar revisions.
|
| 3160 |
+
|
| 3161 |
+
Tokens are read-through dict lookups, not document downloads. A runtime without revision
|
| 3162 |
+
support can still serve an uncached answer. Never reuse an answer with unknown revisions.
|
| 3163 |
+
"""
|
| 3164 |
+
import core.store as store
|
| 3165 |
+
import core.shares as shares
|
| 3166 |
+
import core.user_tables as user_tables
|
| 3167 |
+
import odoo_relational
|
| 3168 |
+
|
| 3169 |
+
try:
|
| 3170 |
+
if not session.uname:
|
| 3171 |
+
return None
|
| 3172 |
+
return (session.uname, table, depth, via,
|
| 3173 |
+
session.runtime.revision(user_tables.STORE_KEY)["token"],
|
| 3174 |
+
session.runtime.revision(shares.SHARES_KEY)["token"],
|
| 3175 |
+
store.revision("users")["token"],
|
| 3176 |
+
session.runtime.revision(odoo_relational.CONFIG_KEY)["token"],
|
| 3177 |
+
user_tables.register_read_through(), user_tables.register_connected_prefix())
|
| 3178 |
+
except Exception: # noqa: BLE001
|
| 3179 |
+
return None
|
| 3180 |
+
|
| 3181 |
+
|
| 3182 |
@router.get("/grid/{table}/related")
|
| 3183 |
def grid_related(table: str,
|
| 3184 |
depth: int = Query(default=2),
|
|
|
|
| 3250 |
f"{relations.MAX_DEPTH} (those plus everything one further named hop away), and "
|
| 3251 |
f"this request asked for {depth}. Ask for 1 or {relations.MAX_DEPTH}")
|
| 3252 |
|
| 3253 |
+
wanted = tuple(str(v).strip() for v in via if str(v or "").strip()) if via else None
|
| 3254 |
+
if wanted is not None and len(wanted) > relations.MAX_DEPTH:
|
| 3255 |
+
raise err(400, "over_depth", relations.over_depth_refusal(table, list(wanted)))
|
| 3256 |
+
|
| 3257 |
+
# This registration must run even on a hit; an import order repair cannot be skipped by
|
| 3258 |
+
# memoisation. Failed registration or reads are served as before, but never remembered.
|
| 3259 |
+
spec_ready = _ensure_odoo_spec()
|
| 3260 |
+
cache = getattr(session.runtime, "related_cache", None)
|
| 3261 |
+
cache_key = (_related_cache_key(session, table, depth, wanted)
|
| 3262 |
+
if spec_ready and cache is not None else None)
|
| 3263 |
+
if spec_ready and cache_key is not None:
|
| 3264 |
+
from copy import deepcopy
|
| 3265 |
+
with session.runtime._lock:
|
| 3266 |
+
cached = cache.get(cache_key)
|
| 3267 |
+
if cached is not None:
|
| 3268 |
+
return deepcopy(cached)
|
| 3269 |
+
|
| 3270 |
ctx = _relation_context(session, table, memo=wall_memo)
|
| 3271 |
entries = related_path_entries(session, table, depth, ctx)
|
| 3272 |
+
if wanted is not None:
|
| 3273 |
+
wanted = list(wanted)
|
|
|
|
|
|
|
| 3274 |
hit = relations.pick_path(entries, wanted)
|
| 3275 |
if hit is None:
|
| 3276 |
# ⭐⭐ W53-T18 / R13 / C3 — THE ONE GENERIC `unknown_via` IS NOW ONE OF FOUR NAMED
|
|
|
|
| 3290 |
# ⚠ A RESOLVED PIN REPORTS NO REFUSALS. The caller named one path and got it; the other
|
| 3291 |
# tabs the wall removed are the LISTING's answer, not this one, and repeating them here
|
| 3292 |
# would attach an unasked-for permission report to every successful read.
|
| 3293 |
+
answer = {"ok": True, "table": table, "depth": depth, "related": [hit], "refusals": []}
|
| 3294 |
+
else:
|
| 3295 |
+
answer = {"ok": True, "table": table, "depth": depth, "related": entries,
|
| 3296 |
+
"refusals": related_refusals(session, table, depth, entries, ctx)}
|
| 3297 |
+
if (spec_ready and cache_key is not None and ctx.get("cacheable", False) and ctx["defs"]
|
| 3298 |
+
and not answer["refusals"]
|
| 3299 |
+
and cache_key == _related_cache_key(session, table, depth, cache_key[3])):
|
| 3300 |
+
from copy import deepcopy
|
| 3301 |
+
# The Amazon purge's pool-cache tuple does not include this memo. That is safe because
|
| 3302 |
+
# every lookup checks the current tokens; old entries cannot match changed metadata.
|
| 3303 |
+
# Limit memory, not the number of related databases: eviction just rebuilds next open.
|
| 3304 |
+
with session.runtime._lock:
|
| 3305 |
+
if len(cache) >= _RELATED_CACHE_LIMIT:
|
| 3306 |
+
cache.clear()
|
| 3307 |
+
cache[cache_key] = deepcopy(answer)
|
| 3308 |
+
return answer
|
api/routes_keychain.py
CHANGED
|
@@ -920,6 +920,14 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 920 |
_rel_reconnect(session.runtime)
|
| 921 |
except Exception: # noqa: BLE001
|
| 922 |
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 923 |
if scope != DEFAULT_SCOPE:
|
| 924 |
try:
|
| 925 |
_write_scope(session.runtime, row["id"], scope, session.uname)
|
|
|
|
| 920 |
_rel_reconnect(session.runtime)
|
| 921 |
except Exception: # noqa: BLE001
|
| 922 |
pass
|
| 923 |
+
elif etype == "amazon_sp":
|
| 924 |
+
# D-29 / intake F6: the HTTP thread only records intent. The resident resync thread
|
| 925 |
+
# remains the mirror's single writer. A queue failure must not undo a stored credential.
|
| 926 |
+
try:
|
| 927 |
+
from core import connector_sync
|
| 928 |
+
connector_sync.request_sync("amazon_sp", session.runtime.key, requested_by=session.uname)
|
| 929 |
+
except Exception: # noqa: BLE001
|
| 930 |
+
pass
|
| 931 |
if scope != DEFAULT_SCOPE:
|
| 932 |
try:
|
| 933 |
_write_scope(session.runtime, row["id"], scope, session.uname)
|
api/routes_module_index.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""W61-T20 / C6: personal App and Agents folder trees in the tenant store."""
|
| 2 |
+
import copy
|
| 3 |
+
|
| 4 |
+
from fastapi import APIRouter, Body, Depends, HTTPException
|
| 5 |
+
|
| 6 |
+
from core.sheets import SheetFolderRefusal, validate_folder_tree
|
| 7 |
+
from deps import Session, err, module_gate, require_session
|
| 8 |
+
import routes_assets
|
| 9 |
+
import routes_automation
|
| 10 |
+
import voice_control_plane as voice
|
| 11 |
+
|
| 12 |
+
router = APIRouter(prefix="/api/v1/module-index")
|
| 13 |
+
_AGENTS_GATE = module_gate("automation")
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _sentence(value):
|
| 17 |
+
return " ".join(str(value).replace("\u2014", "-").replace("\u2013", "-").split())
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _refuse(exc):
|
| 21 |
+
status = {"module_index_unknown": 404, "module_index_user_forbidden": 403,
|
| 22 |
+
"module_index_item_forbidden": 403, "index_revision_stale": 409}.get(exc.code, 400)
|
| 23 |
+
return err(status, exc.code, _sentence(exc.message))
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _access(module: str, session: Session = Depends(require_session)):
|
| 27 |
+
if module not in {"app", "agents"}:
|
| 28 |
+
raise _refuse(SheetFolderRefusal(
|
| 29 |
+
"module_index_unknown", f'Folder index "{module}" is not supported; use app or agents.'))
|
| 30 |
+
# App has the same session-only door as GET /applications; its rows have their own wall.
|
| 31 |
+
if module == "agents":
|
| 32 |
+
try:
|
| 33 |
+
_AGENTS_GATE(session)
|
| 34 |
+
except HTTPException as exc:
|
| 35 |
+
if exc.status_code != 403:
|
| 36 |
+
raise
|
| 37 |
+
raise err(403, "forbidden", "Your account or workspace does not have access to Agents (automation).")
|
| 38 |
+
return session
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _owner(session, user):
|
| 42 |
+
owner = user or session.uname
|
| 43 |
+
if owner != session.uname and not session.admin:
|
| 44 |
+
raise SheetFolderRefusal(
|
| 45 |
+
"module_index_user_forbidden", f'Only an administrator may read the folder tree for "{owner}".')
|
| 46 |
+
return owner
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _readable_ids(module, session):
|
| 50 |
+
# The projection is hygiene; the per-item list door is the wall. The client renders
|
| 51 |
+
# only listed items, so an unverifiable placement renders no tile. Use local voice
|
| 52 |
+
# records, never audience-option assembly, just to verify placement ids.
|
| 53 |
+
if module == "app":
|
| 54 |
+
return {row["id"] for row in routes_assets.application_index(session=session)["applications"]}
|
| 55 |
+
# ⭐⭐ W62-T17 — THE ID-ONLY ACCESSOR THIS COMMENT USED TO ASK FOR NOW EXISTS. It read
|
| 56 |
+
# *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until
|
| 57 |
+
# it has an id-only accessor"*, and the whole Agents rail payload was being built to answer a
|
| 58 |
+
# membership question. `routes_automation.automation_ids` keeps all three strata (stored
|
| 59 |
+
# definitions, field agents, the derived Odoo sync row) and the statements seed, so the SET is
|
| 60 |
+
# unchanged; what it drops is `_wire` per automation, which read `agent_contracts` and
|
| 61 |
+
# `automation_agent_actions` once EACH TIME (MEASURED: 1,101 ms of this route's 1,180 ms on
|
| 62 |
+
# staging, `AutoWiki/waves/wave62/proto/T17-profile.md`).
|
| 63 |
+
return ({"agent:" + aid for aid in routes_automation.automation_ids(session)}
|
| 64 |
+
| {"voice:" + row["id"] for row in
|
| 65 |
+
voice._store(session.runtime)["voice_agent"].values() if isinstance(row, dict)})
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _project(bucket, ids):
|
| 69 |
+
folders, placed, _ = validate_folder_tree(bucket.get("folders"), bucket.get("itemFolders"))
|
| 70 |
+
return {"folders": folders, "itemFolders": {k: v for k, v in placed.items() if k in ids},
|
| 71 |
+
"rev": bucket.get("rev", 0)}
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _available(session):
|
| 75 |
+
if not session.runtime.available():
|
| 76 |
+
raise err(503, "store_unavailable", "The tenant store is unavailable. Reload folders to retry.")
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
@router.get("/{module}")
|
| 80 |
+
def read_module_index(module: str, user: str = "", session: Session = Depends(_access)):
|
| 81 |
+
try:
|
| 82 |
+
owner = _owner(session, user)
|
| 83 |
+
_available(session)
|
| 84 |
+
doc = session.runtime.get("module_index_" + module) or {}
|
| 85 |
+
bucket = (doc.get("users") or {}).get(owner) or {}
|
| 86 |
+
ids = _readable_ids(module, session) if bucket.get("itemFolders") else set()
|
| 87 |
+
return {"index": _project(bucket, ids)}
|
| 88 |
+
except SheetFolderRefusal as exc:
|
| 89 |
+
raise _refuse(exc)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
@router.put("/{module}")
|
| 93 |
+
def write_module_index(module: str, body: dict = Body(default=None), user: str = "",
|
| 94 |
+
session: Session = Depends(_access)):
|
| 95 |
+
try:
|
| 96 |
+
owner = _owner(session, user)
|
| 97 |
+
# The admin override is for reading another user's tree, never for replacing it.
|
| 98 |
+
if owner != session.uname:
|
| 99 |
+
raise SheetFolderRefusal("module_index_user_forbidden",
|
| 100 |
+
f'The folder tree for "{owner}" can only be changed by its owner.')
|
| 101 |
+
_available(session)
|
| 102 |
+
raw = body or {}
|
| 103 |
+
expect_rev = raw.get("rev")
|
| 104 |
+
if type(expect_rev) is not int or expect_rev < 0:
|
| 105 |
+
raise SheetFolderRefusal("index_revision_required", "Reload folders before saving a folder change.")
|
| 106 |
+
placements = raw.get("itemFolders")
|
| 107 |
+
if not isinstance(raw.get("folders"), list) or not isinstance(placements, dict):
|
| 108 |
+
raise SheetFolderRefusal("module_index_invalid", "Send folders and itemFolders together to save the folder tree.")
|
| 109 |
+
ids = _readable_ids(module, session) if placements else set()
|
| 110 |
+
if any(item_id not in ids for item_id in placements):
|
| 111 |
+
raise SheetFolderRefusal("module_index_item_forbidden",
|
| 112 |
+
"A folder placement names an item you cannot read in this module.")
|
| 113 |
+
folders, placed, _ = validate_folder_tree(raw["folders"], placements)
|
| 114 |
+
saved = {}
|
| 115 |
+
|
| 116 |
+
def apply(current):
|
| 117 |
+
state = copy.deepcopy(current) if isinstance(current, dict) else {}
|
| 118 |
+
users = state.setdefault("users", {})
|
| 119 |
+
rev = (users.get(owner) or {}).get("rev", 0)
|
| 120 |
+
# Compare inside the store transaction, before touching any user's bucket.
|
| 121 |
+
if expect_rev != rev:
|
| 122 |
+
raise SheetFolderRefusal("index_revision_stale",
|
| 123 |
+
"Folders changed somewhere else. Reload folders and try again.")
|
| 124 |
+
saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
|
| 125 |
+
users[owner] = dict(saved)
|
| 126 |
+
state["schema"] = 1
|
| 127 |
+
return state
|
| 128 |
+
|
| 129 |
+
session.runtime.update("module_index_" + module, apply, flush="sync")
|
| 130 |
+
return {"index": _project(saved, ids)}
|
| 131 |
+
except SheetFolderRefusal as exc:
|
| 132 |
+
raise _refuse(exc)
|
api/routes_nav.py
CHANGED
|
@@ -81,6 +81,36 @@ def _now() -> int:
|
|
| 81 |
return int(time.time())
|
| 82 |
|
| 83 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False):
|
| 85 |
"""Validated wholesale replacement, the `clean_folders` posture: prune, never invent.
|
| 86 |
|
|
@@ -118,7 +148,35 @@ def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False):
|
|
| 118 |
placement[k] = v
|
| 119 |
elif keep_unknown_ut and k.startswith("ut_"):
|
| 120 |
placement[k] = v
|
| 121 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
|
| 123 |
|
| 124 |
def _placeable_top_keys(session):
|
|
@@ -397,6 +455,28 @@ def _visible_database_keys(session):
|
|
| 397 |
return keys, enumerated
|
| 398 |
|
| 399 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 400 |
def _granted_view_ids(session):
|
| 401 |
"""Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`.
|
| 402 |
|
|
@@ -406,21 +486,13 @@ def _granted_view_ids(session):
|
|
| 406 |
answer "which views has this person starred", so the derivation moves here beside the reader
|
| 407 |
it feeds rather than being copied into a second file ([[one-evaluator-per-question]]).
|
| 408 |
|
| 409 |
-
⚠ THE READ
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
⚠ Fail-closed: no grants is a NARROWER answer, never a wider one.
|
| 414 |
"""
|
| 415 |
-
|
| 416 |
-
try:
|
| 417 |
-
return {str(v) for v in
|
| 418 |
-
((shares.shared_with(session.uname, kind="view", st=session.runtime) or {})
|
| 419 |
-
.get("view") or [])
|
| 420 |
-
if shares.may_see("view", str(v), session.uname,
|
| 421 |
-
st=session.runtime)}
|
| 422 |
-
except Exception:
|
| 423 |
-
return set()
|
| 424 |
|
| 425 |
|
| 426 |
#: ⭐⭐ W54-T48 / OWNER RULING R4 / CONTRACT C4 — ONE CONNECTION, ONE FOLDER IN THE RAIL.
|
|
@@ -959,7 +1031,7 @@ def nav_prefs(session: Session = Depends(require_session)):
|
|
| 959 |
try:
|
| 960 |
stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {}
|
| 961 |
except Exception:
|
| 962 |
-
|
| 963 |
keys, enumerated = _placeable_top_keys(session)
|
| 964 |
return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)}
|
| 965 |
|
|
@@ -967,17 +1039,19 @@ def nav_prefs(session: Session = Depends(require_session)):
|
|
| 967 |
@router.post("/nav/prefs")
|
| 968 |
def save_nav_prefs(body: dict = Body(default=None),
|
| 969 |
session: Session = Depends(require_session)):
|
| 970 |
-
"""
|
| 971 |
-
picture, validated here; a partial merge would resurrect deleted folders forever."""
|
| 972 |
keys, enumerated = _placeable_top_keys(session)
|
| 973 |
-
clean =
|
| 974 |
if not session.runtime.available():
|
| 975 |
raise err(503, "store_unavailable",
|
| 976 |
"the tenant store is unavailable. Nothing was saved.")
|
| 977 |
|
| 978 |
def _up(data):
|
|
|
|
| 979 |
data = data if isinstance(data, dict) else {}
|
| 980 |
-
|
|
|
|
|
|
|
| 981 |
data[session.uname] = clean
|
| 982 |
else:
|
| 983 |
data.pop(session.uname, None)
|
|
@@ -987,7 +1061,7 @@ def save_nav_prefs(body: dict = Body(default=None),
|
|
| 987 |
session.runtime.update(_NAV_PREFS_KEY, _up)
|
| 988 |
except Exception:
|
| 989 |
raise err(503, "store_unavailable",
|
| 990 |
-
"the
|
| 991 |
return {"prefs": clean}
|
| 992 |
|
| 993 |
|
|
|
|
| 81 |
return int(time.time())
|
| 82 |
|
| 83 |
|
| 84 |
+
def _clean_linked_layouts(raw):
|
| 85 |
+
"""R8 personal presentation only. Never accept a SavedView or a field definition."""
|
| 86 |
+
if not isinstance(raw, dict):
|
| 87 |
+
return {}
|
| 88 |
+
layouts = {}
|
| 89 |
+
for key, value in raw.items():
|
| 90 |
+
if not isinstance(key, str) or not key or not isinstance(value, dict):
|
| 91 |
+
continue
|
| 92 |
+
layout = {}
|
| 93 |
+
for name in ("order", "visible"):
|
| 94 |
+
if isinstance(value.get(name), list):
|
| 95 |
+
layout[name] = list(dict.fromkeys(k for k in value[name] if isinstance(k, str)))
|
| 96 |
+
if isinstance(value.get("widths"), dict):
|
| 97 |
+
layout["widths"] = {k: w for k, w in value["widths"].items()
|
| 98 |
+
if isinstance(k, str) and type(w) in (int, float) and w > 0}
|
| 99 |
+
if isinstance(value.get("sorts"), list):
|
| 100 |
+
layout["sorts"] = [{"colId": s["colId"], "dir": s["dir"]}
|
| 101 |
+
for s in value["sorts"] if isinstance(s, dict)
|
| 102 |
+
and isinstance(s.get("colId"), str) and s.get("dir") in ("asc", "desc")]
|
| 103 |
+
if isinstance(value.get("filters"), list):
|
| 104 |
+
layout["filters"] = [n for n in value["filters"] if isinstance(n, dict)]
|
| 105 |
+
if value.get("filterConj") in ("and", "or"):
|
| 106 |
+
layout["filterConj"] = value["filterConj"]
|
| 107 |
+
if "groupBy" in value and (value["groupBy"] is None or isinstance(value["groupBy"], str)):
|
| 108 |
+
layout["groupBy"] = value["groupBy"]
|
| 109 |
+
if layout:
|
| 110 |
+
layouts[key] = layout
|
| 111 |
+
return layouts
|
| 112 |
+
|
| 113 |
+
|
| 114 |
def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False):
|
| 115 |
"""Validated wholesale replacement, the `clean_folders` posture: prune, never invent.
|
| 116 |
|
|
|
|
| 148 |
placement[k] = v
|
| 149 |
elif keep_unknown_ut and k.startswith("ut_"):
|
| 150 |
placement[k] = v
|
| 151 |
+
clean = {"folders": folders, "placement": placement}
|
| 152 |
+
layouts = _clean_linked_layouts(raw.get("linkedLayouts"))
|
| 153 |
+
if layouts:
|
| 154 |
+
clean["linkedLayouts"] = layouts
|
| 155 |
+
return clean
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def _merge_nav_prefs(stored, body, page_keys, *, keep_unknown_ut=False):
|
| 159 |
+
"""Replace navigation only when supplied; patch each personal link independently.
|
| 160 |
+
|
| 161 |
+
Called inside runtime.update so concurrent link and folder writes see the latest entry.
|
| 162 |
+
Folder clients may echo a stale linkedLayouts from GET; only a layout-only request owns it.
|
| 163 |
+
"""
|
| 164 |
+
stored = stored if isinstance(stored, dict) else {}
|
| 165 |
+
body = body if isinstance(body, dict) else {}
|
| 166 |
+
navigation = "folders" in body or "placement" in body
|
| 167 |
+
merged = {**stored}
|
| 168 |
+
if navigation:
|
| 169 |
+
merged.update(folders=body.get("folders", []), placement=body.get("placement", {}))
|
| 170 |
+
else:
|
| 171 |
+
layouts = _clean_linked_layouts(stored.get("linkedLayouts"))
|
| 172 |
+
layouts.update(_clean_linked_layouts(body.get("linkedLayouts")))
|
| 173 |
+
merged["linkedLayouts"] = layouts
|
| 174 |
+
# A layout write does not own navigation pruning either.
|
| 175 |
+
clean = _clean_nav_prefs(merged, page_keys, keep_unknown_ut=keep_unknown_ut)
|
| 176 |
+
if not navigation:
|
| 177 |
+
for name, default in (("folders", []), ("placement", {})):
|
| 178 |
+
clean[name] = stored.get(name, default)
|
| 179 |
+
return clean
|
| 180 |
|
| 181 |
|
| 182 |
def _placeable_top_keys(session):
|
|
|
|
| 455 |
return keys, enumerated
|
| 456 |
|
| 457 |
|
| 458 |
+
def _granted_view_sources(session):
|
| 459 |
+
"""`{view_id: owner}` for grants this caller may read, from one registry read.
|
| 460 |
+
|
| 461 |
+
`owner` may be ``None`` for a legacy ownerless grant. Callers preserve the old fail-closed
|
| 462 |
+
source lookup for that case; modern grants point directly at the canonical personal stratum.
|
| 463 |
+
"""
|
| 464 |
+
import core.shares as shares
|
| 465 |
+
try:
|
| 466 |
+
out = {}
|
| 467 |
+
for raw_id, record in shares.grants_bulk("view", st=session.runtime).items():
|
| 468 |
+
owner = str(record.get("owner") or "").strip().lower() or None
|
| 469 |
+
if owner == session.uname:
|
| 470 |
+
continue
|
| 471 |
+
# Match the historical `shared_with` + `may_see` path: being an admin alone did not
|
| 472 |
+
# add a View to "shared with me"; a valid named or everyone grant did.
|
| 473 |
+
if shares.role_from(record, session.uname) is not None:
|
| 474 |
+
out[str(raw_id)] = owner
|
| 475 |
+
return out
|
| 476 |
+
except Exception:
|
| 477 |
+
return {}
|
| 478 |
+
|
| 479 |
+
|
| 480 |
def _granted_view_ids(session):
|
| 481 |
"""Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`.
|
| 482 |
|
|
|
|
| 486 |
answer "which views has this person starred", so the derivation moves here beside the reader
|
| 487 |
it feeds rather than being copied into a second file ([[one-evaluator-per-question]]).
|
| 488 |
|
| 489 |
+
⚠ THE READ PREDICATE STILL HAS ONE OWNER. `_granted_view_sources` obtains the registry once
|
| 490 |
+
through `grants_bulk` and delegates each held record to `shares.role_from`, the same pure
|
| 491 |
+
evaluator `may_see` uses. Named and dynamic-everyone grants therefore keep identical semantics
|
| 492 |
+
without one registry round trip per id.
|
| 493 |
⚠ Fail-closed: no grants is a NARROWER answer, never a wider one.
|
| 494 |
"""
|
| 495 |
+
return set(_granted_view_sources(session))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 496 |
|
| 497 |
|
| 498 |
#: ⭐⭐ W54-T48 / OWNER RULING R4 / CONTRACT C4 — ONE CONNECTION, ONE FOLDER IN THE RAIL.
|
|
|
|
| 1031 |
try:
|
| 1032 |
stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {}
|
| 1033 |
except Exception:
|
| 1034 |
+
raise err(503, "store_unavailable", "Your saved preferences could not be loaded.")
|
| 1035 |
keys, enumerated = _placeable_top_keys(session)
|
| 1036 |
return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)}
|
| 1037 |
|
|
|
|
| 1039 |
@router.post("/nav/prefs")
|
| 1040 |
def save_nav_prefs(body: dict = Body(default=None),
|
| 1041 |
session: Session = Depends(require_session)):
|
| 1042 |
+
"""Replace the supplied navigation picture or patch personal linked-grid layouts."""
|
|
|
|
| 1043 |
keys, enumerated = _placeable_top_keys(session)
|
| 1044 |
+
clean = {}
|
| 1045 |
if not session.runtime.available():
|
| 1046 |
raise err(503, "store_unavailable",
|
| 1047 |
"the tenant store is unavailable. Nothing was saved.")
|
| 1048 |
|
| 1049 |
def _up(data):
|
| 1050 |
+
nonlocal clean
|
| 1051 |
data = data if isinstance(data, dict) else {}
|
| 1052 |
+
clean = _merge_nav_prefs(data.get(session.uname), body, keys,
|
| 1053 |
+
keep_unknown_ut=not enumerated)
|
| 1054 |
+
if clean["folders"] or clean["placement"] or clean.get("linkedLayouts"):
|
| 1055 |
data[session.uname] = clean
|
| 1056 |
else:
|
| 1057 |
data.pop(session.uname, None)
|
|
|
|
| 1061 |
session.runtime.update(_NAV_PREFS_KEY, _up)
|
| 1062 |
except Exception:
|
| 1063 |
raise err(503, "store_unavailable",
|
| 1064 |
+
"the preference change was not saved: the store refused the write.")
|
| 1065 |
return {"prefs": clean}
|
| 1066 |
|
| 1067 |
|
api/routes_odoo_tables.py
CHANGED
|
@@ -23,6 +23,7 @@ import hashlib
|
|
| 23 |
import hmac
|
| 24 |
import json
|
| 25 |
import time
|
|
|
|
| 26 |
|
| 27 |
from fastapi import Body, Depends, HTTPException
|
| 28 |
from fastapi import APIRouter, Query
|
|
@@ -1516,6 +1517,189 @@ def _related_source_value(session, entry, record):
|
|
| 1516 |
return None
|
| 1517 |
|
| 1518 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1519 |
def _cache_relation_plan(session, source, via, target):
|
| 1520 |
"""The offered module link and its explicit bridge, or a cause-specific refusal.
|
| 1521 |
|
|
@@ -1538,9 +1722,8 @@ def _cache_relation_plan(session, source, via, target):
|
|
| 1538 |
raise err(404, "relation_target_missing",
|
| 1539 |
f"'{target}' is not a current module/cache relation target in this workspace")
|
| 1540 |
if not _may_open(session, source):
|
| 1541 |
-
raise err(403, "
|
| 1542 |
-
f"'{source}'
|
| 1543 |
-
"be followed")
|
| 1544 |
|
| 1545 |
offered = next((entry for entry in related_entries(session, source)
|
| 1546 |
if str(entry.get("key") or "") == via
|
|
@@ -1557,7 +1740,7 @@ def _cache_relation_plan(session, source, via, target):
|
|
| 1557 |
if not isinstance(link, dict) or str(link.get("table") or "") != target:
|
| 1558 |
raise err(404, "relation_target_missing",
|
| 1559 |
f"the relation '{via}' no longer has a current target in this workspace")
|
| 1560 |
-
through =
|
| 1561 |
through_on = str(link.get("throughOn") or "")
|
| 1562 |
through_to = str(link.get("throughTo") or "")
|
| 1563 |
on = str(link.get("on") or relations.IDENTITY_COL)
|
|
@@ -1567,13 +1750,11 @@ def _cache_relation_plan(session, source, via, target):
|
|
| 1567 |
if relations.is_retired_table(through):
|
| 1568 |
raise err(404, "relation_target_retired",
|
| 1569 |
f"'{through}' is retired, so the relation to '{target}' is no longer available")
|
| 1570 |
-
|
| 1571 |
-
|
| 1572 |
-
f"the relation to '{target}' crosses '{through}', which your account cannot "
|
| 1573 |
-
"open")
|
| 1574 |
if not _may_open(session, target):
|
| 1575 |
raise err(403, "forbidden_table",
|
| 1576 |
-
f"the relation ends at '{target}', which your account cannot open")
|
| 1577 |
return dict(offered, __source__=source), link, through, through_on, through_to, on
|
| 1578 |
|
| 1579 |
|
|
@@ -1601,15 +1782,23 @@ def _cache_relation_source_value(session, entry, record):
|
|
| 1601 |
|
| 1602 |
|
| 1603 |
def _cache_relation_target_ids(session, source, record, via, target):
|
| 1604 |
-
"""Resolve
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1605 |
entry, _link, through, through_on, through_to, on = _cache_relation_plan(
|
| 1606 |
-
session, source,
|
| 1607 |
source_value = _cache_relation_source_value(session, entry, record)
|
| 1608 |
if source_value in (None, ""):
|
| 1609 |
raise err(400, "unknown_relation",
|
| 1610 |
-
f"record {record} has no readable source value for relation '{
|
| 1611 |
|
| 1612 |
-
|
|
|
|
|
|
|
|
|
|
| 1613 |
spec = compiled["spec"]
|
| 1614 |
left = (spec.get("cols") or {}).get(through_on)
|
| 1615 |
right = (spec.get("cols") or {}).get(through_to)
|
|
@@ -1658,7 +1847,11 @@ def odoo_relation_targets(source: str = Query(...), record: int = Query(..., ge=
|
|
| 1658 |
via: str = Query(...), target: str = Query(...),
|
| 1659 |
session: Session = Depends(require_session)):
|
| 1660 |
"""Target pids for a declared relation whose destination is a runtime-cache database."""
|
| 1661 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1662 |
|
| 1663 |
|
| 1664 |
def _via_path(via):
|
|
@@ -1721,308 +1914,24 @@ def _related_path_legs(session, source, entry, entries):
|
|
| 1721 |
return dict(first, __source__=source), mid_key, dict(last, __source__=mid_key)
|
| 1722 |
|
| 1723 |
|
| 1724 |
-
def _related_hop_values(session, source, record, first, mid_key, last, notes):
|
| 1725 |
-
"""The values hop 2 joins on, resolved from the INTERMEDIATE database's own rows.
|
| 1726 |
-
|
| 1727 |
-
⭐⭐ W54-T40 — THIS IS THE HALF THAT MAKES A TWO-HOP TAB ANSWER ROWS INSTEAD OF A SENTENCE.
|
| 1728 |
-
Hop 1 turns the source record into a set of intermediate rows; this returns the value each of
|
| 1729 |
-
those rows carries in the column hop 2 is keyed on, and the caller pins THIS database by them.
|
| 1730 |
-
|
| 1731 |
-
⛔⛔ THE INTERMEDIATE IS READ THROUGH `_compiled_read`, WHICH IS THE SAME COMPILER THIS DOOR
|
| 1732 |
-
USES FOR AN ORDINARY WINDOW, AND THAT IS THE WHOLE PERMISSION ARGUMENT. Its grant gate, its
|
| 1733 |
-
administrator row wall and its hidden-field closure all run before a single id is produced, so
|
| 1734 |
-
a traversal cannot be used to learn about rows of the middle database a reader may not see. A
|
| 1735 |
-
second, lighter read here would be a second permission wall, and this file already carries the
|
| 1736 |
-
rule that a second copy of that block does not drift towards refusing.
|
| 1737 |
-
⛔ AND AN INTERMEDIATE THIS DOOR CANNOT READ REFUSES BY NAME. `product_data` is a compiled
|
| 1738 |
-
registry module with no read-through binding, so the one enumerated path that passes through it
|
| 1739 |
-
answers a sentence carrying the cause and a way round it, rather than an empty grid.
|
| 1740 |
-
|
| 1741 |
-
⚠ NO CAP, AND THE COST IS REPORTED INSTEAD (standing rule 1). Every distinct value comes back,
|
| 1742 |
-
however many there are, and the reader is told through `limits` that this tab resolves a second
|
| 1743 |
-
database on every page.
|
| 1744 |
-
"""
|
| 1745 |
-
import core.relations as relations
|
| 1746 |
-
from fastapi import HTTPException # noqa: PLC0415
|
| 1747 |
-
|
| 1748 |
-
value = _related_source_value(session, first, record)
|
| 1749 |
-
if value in (None, ""):
|
| 1750 |
-
raise err(400, "unknown_relation",
|
| 1751 |
-
relations.unresolvable_source_refusal(source, first))
|
| 1752 |
-
try:
|
| 1753 |
-
mid = _compiled_read(mid_key, session)
|
| 1754 |
-
except HTTPException as exc:
|
| 1755 |
-
why = ""
|
| 1756 |
-
if isinstance(getattr(exc, "detail", None), dict):
|
| 1757 |
-
why = str((exc.detail.get("error") or {}).get("message") or "")
|
| 1758 |
-
raise err(409, "relation_not_served",
|
| 1759 |
-
f"this relation reaches '{last.get('target')}' through '{mid_key}', and that "
|
| 1760 |
-
f"database could not be read here to follow it"
|
| 1761 |
-
+ (f" ({why})" if why else "")
|
| 1762 |
-
+ f". Open '{mid_key}' on its own, find the records this one is linked to, and "
|
| 1763 |
-
f"follow the next relation from there")
|
| 1764 |
-
mid_spec = mid["spec"]
|
| 1765 |
-
on1 = str((first.get("via") or {}).get("on") or "")
|
| 1766 |
-
step = (mid_spec.get("cols") or {}).get(on1)
|
| 1767 |
-
if not step:
|
| 1768 |
-
raise err(409, "relation_not_served",
|
| 1769 |
-
f"'{mid_key}' has no live expression for '{on1}', which is the column the first "
|
| 1770 |
-
f"hop of this relation joins on, so the traversal cannot be resolved. Open "
|
| 1771 |
-
f"'{mid_key}' on its own, or ask for '{on1}' to be added to its live connection")
|
| 1772 |
-
# ⭐ THE COLUMN HOP 2 IS KEYED ON, READ OFF THE INTERMEDIATE. `pid` is the row IDENTITY and is
|
| 1773 |
-
# not a column of the mirror spec, so it resolves to the spec's own identity expression, which
|
| 1774 |
-
# is the same one this route trusts everywhere else and is per table rather than a bare `id`.
|
| 1775 |
-
frm2 = str((last.get("via") or {}).get("from") or relations.IDENTITY_COL)
|
| 1776 |
-
take = (mid_spec["id"] if frm2 == relations.IDENTITY_COL
|
| 1777 |
-
else ((mid_spec.get("cols") or {}).get(frm2) or (None,))[0])
|
| 1778 |
-
if not take:
|
| 1779 |
-
raise err(409, "relation_not_served",
|
| 1780 |
-
f"'{mid_key}' has no live expression for '{frm2}', which is the value the second "
|
| 1781 |
-
f"hop of this relation is keyed on, so the traversal cannot be resolved. Open "
|
| 1782 |
-
f"'{mid_key}' on its own, or ask for '{frm2}' to be added to its live connection")
|
| 1783 |
-
try:
|
| 1784 |
-
value = step[1](value)
|
| 1785 |
-
except Exception: # noqa: BLE001
|
| 1786 |
-
value = str(value)
|
| 1787 |
-
mid_from = mid_spec["from_sql"] if mid_spec.get("from_sql") else mid_spec["table"]
|
| 1788 |
-
parts = [p for p in (mid["base_where"], first.get("where")) if p]
|
| 1789 |
-
parts.append(f"{step[0]} = ?")
|
| 1790 |
-
pred = " AND ".join(f"({p})" for p in parts)
|
| 1791 |
-
rows = mid["cur"].execute(
|
| 1792 |
-
f"SELECT DISTINCT {take} FROM {mid_from} WHERE {pred}",
|
| 1793 |
-
list(mid["wall_params"]) + [value]).fetchall()
|
| 1794 |
-
values = [r[0] for r in rows if r and r[0] is not None]
|
| 1795 |
-
# ⭐⭐ STANDING RULE 1'S SECOND SENTENCE, AND `effect` IS CHOSEN SO THE READER GETS **THIS**
|
| 1796 |
-
# SENTENCE RATHER THAN A NEIGHBOUR'S. `counts.ts::plainLimit` maps a known `effect` to one
|
| 1797 |
-
# fixed client phrase and falls through to the SERVER's own first sentence for an effect it has
|
| 1798 |
-
# not learned; `slow` maps to *"Rows this far down load slowly, each page is reached by walking
|
| 1799 |
-
# every row above it"*, which is a true statement about a deep page and a FALSE one about a
|
| 1800 |
-
# seven-row traversal. `traversal` is deliberately not in that table, so the strip prints the
|
| 1801 |
-
# cause below, which is what actually happened.
|
| 1802 |
-
# ⚠ THE FIRST CLAUSE IS THE ONE THAT REACHES THE STRIP: `firstSentence` cuts at the first full
|
| 1803 |
-
# stop and truncates past 120 characters, so the mechanism is stated before the count.
|
| 1804 |
-
notes.append({
|
| 1805 |
-
"subject": "related", "effect": "traversal",
|
| 1806 |
-
"cause": f"these rows are reached through '{mid_key}', so every page resolves that "
|
| 1807 |
-
f"database first. This record matched {len(values):,} of its records, and this "
|
| 1808 |
-
f"database is then narrowed to the rows those reach",
|
| 1809 |
-
"recommendation": f"open '{mid_key}' on its own if you are paging through a large set, or "
|
| 1810 |
-
f"filter this database beside the relation to ask a smaller question"})
|
| 1811 |
-
return values
|
| 1812 |
|
| 1813 |
|
| 1814 |
def _related_pin(table_key, session, spec, base_where, wall_params, cur, related):
|
| 1815 |
-
"""
|
| 1816 |
-
|
| 1817 |
-
⛔⛔ WHY THIS DOES NOT CALL `compile_filter_tree(member_ids=)`, WHICH IS WHAT C6'S SENTENCE
|
| 1818 |
-
SAYS. Measured against `platform/harness/filter_sql.py` on 2026-09-03, that call cannot express
|
| 1819 |
-
a pinned read and the failure is silent in the widening direction, twice over:
|
| 1820 |
-
|
| 1821 |
-
1. `:583` returns None the moment the tree is INACTIVE, and the `member_ids` branch is BELOW
|
| 1822 |
-
that return. A pin with no client filter is exactly the inactive case, so the ids would be
|
| 1823 |
-
dropped and the whole table served.
|
| 1824 |
-
2. `:588` emits `(tree OR id_sql IN (...))`. Its own docstring: *"`member_ids` reproduces
|
| 1825 |
-
`members.has(r.pid) || matchFilterTree(...)`"*. That is a UNION. ORing a pin onto a
|
| 1826 |
-
filter WIDENS; a pinned read needs an INTERSECTION.
|
| 1827 |
-
|
| 1828 |
-
So the tail is compiled by `core.relations.pin_sql`, in `filter_sql`'s OWN spelling (proven
|
| 1829 |
-
byte for byte by `verify_pivot.py`, not by this comment), and ANDed in. `filter_sql.py` is lane
|
| 1830 |
-
B's file this wave; a narrowing member mode there is an AMENDMENT and is booked in the handoff,
|
| 1831 |
-
not edited in from here.
|
| 1832 |
-
|
| 1833 |
-
⭐ `id_sql` IS PASSED EXPLICITLY, AS `spec["id"]`, AND IS NEVER DEFAULTED. `harness/semantic.py`
|
| 1834 |
-
defaults it to `f"{alias}.id"` inside its OWN compile calls; a call from the route layer never
|
| 1835 |
-
reaches that line, and `filter_sql:586` RAISES without it. It matters that it is per table
|
| 1836 |
-
rather than a literal `id`: `ut_odoo_order_lines` is join-shaped and its identity is `sol.id`,
|
| 1837 |
-
so a bare `id` would be ambiguous across `sale_order_line` and `sale_order`. It is the same
|
| 1838 |
-
expression this route already trusts as `tiebreak_sql` and as the fallback ORDER BY.
|
| 1839 |
-
|
| 1840 |
-
⭐⭐ W54-T40 / INSTRUCTION 22 / R12 — **A `via` MAY NOW BE A PATH, AND THE PATH IS SERVED.**
|
| 1841 |
-
The owner's second quoted error was this function's own `bad_relation` sentence, and it fired
|
| 1842 |
-
on 18 of the 25 rail tabs of `customer_data`: R12 says the client pins a traversal by naming
|
| 1843 |
-
each edge, so a two-hop tab sends `via` as the ARRAY `["invoices","order_link"]`, and the
|
| 1844 |
-
guard below accepted only the one-hop bag. MEASURED 2026-09-06, 6 of 25 tabs answered 200 and
|
| 1845 |
-
18 answered 400 on the first click.
|
| 1846 |
-
|
| 1847 |
-
⛔⛔ ACCEPTING THE ARRAY IS HALF A FIX AND WOULD HAVE MOVED THE ERROR RATHER THAN CLOSED IT.
|
| 1848 |
-
`relations.pick` matches `via.direction` against `DIRECTIONS`, and a two-hop entry's direction
|
| 1849 |
-
is the literal `path`, deliberately outside that tuple so nothing serves half a traversal. So
|
| 1850 |
-
a path resolves through `relations.pick_path` against the DEPTH-AWARE listing, and the read is
|
| 1851 |
-
then answered in TWO steps: hop 1 resolves the intermediate database's own rows to the values
|
| 1852 |
-
hop 2 joins on, and hop 2 pins this database by them. Sending the object form alone answered
|
| 1853 |
-
`400 unknown_relation` and changed nothing the reader could see.
|
| 1854 |
-
|
| 1855 |
-
⭐ EVERY HOP IS STILL PERMISSION CHECKED, AND AT PIN TIME, NOT ONLY AT LISTING TIME (R13). The
|
| 1856 |
-
intermediate is resolved through `_compiled_read`, i.e. THE SAME compiler this door uses for
|
| 1857 |
-
an ordinary read, so its grant, its row wall and its hidden-field closure all apply before a
|
| 1858 |
-
single id crosses into this table's predicate. A second, laxer read of the intermediate would
|
| 1859 |
-
be exactly the permission wall with a hole in it that `related_entries` exists to prevent.
|
| 1860 |
-
⛔ AND AN INTERMEDIATE THIS DOOR CANNOT READ IS A **REPORTED LIMIT**, never a silent empty:
|
| 1861 |
-
`product_data` is a registry module with no read-through binding, so the one enumerated path
|
| 1862 |
-
that passes through it refuses by name with a cause and a recommendation.
|
| 1863 |
-
"""
|
| 1864 |
import core.relations as relations
|
| 1865 |
|
| 1866 |
if isinstance(related, str):
|
| 1867 |
related = _json_arg(related, "related")
|
| 1868 |
if related in (None, "", {}):
|
| 1869 |
return None, [], []
|
| 1870 |
-
|
| 1871 |
-
|
| 1872 |
-
"`related` must be an object naming the record to pin these rows to: "
|
| 1873 |
-
"{table, record, via}")
|
| 1874 |
-
|
| 1875 |
-
source = str(related.get("table") or "")
|
| 1876 |
-
record = related.get("record")
|
| 1877 |
-
via = related.get("via")
|
| 1878 |
-
# ⭐⭐ W54-T40 — `via` IS EITHER ONE EDGE'S BAG OR A PATH OF EDGE KEYS, and `_via_path` is the
|
| 1879 |
-
# one place that decides which. It answers None for the bag, which is what keeps the guard
|
| 1880 |
-
# below spelling the SAME refusal for a genuinely incomplete pin.
|
| 1881 |
-
path = _via_path(via)
|
| 1882 |
-
if not source or record in (None, "") or (path is None and not isinstance(via, dict)):
|
| 1883 |
-
raise err(400, "bad_relation",
|
| 1884 |
-
"a pinned read needs the record's database, the record id and the relation to "
|
| 1885 |
-
"follow, as {table, record, via}, where `via` is one relation or the array of "
|
| 1886 |
-
"edge keys naming each hop. Ask GET /api/v1/grid/" + (source or "{table}")
|
| 1887 |
-
+ "/related for the relations this account may follow")
|
| 1888 |
-
|
| 1889 |
-
from routes_grid import _may_open, related_entries, related_path_entries
|
| 1890 |
-
|
| 1891 |
-
if not _may_open(session, source):
|
| 1892 |
-
# ⚠ THE CAUSE IS NAMED AS A PAIR, ON PURPOSE. `_may_open` is fail-closed and answers False
|
| 1893 |
-
# both for a genuine permission refusal and for a database it could not read at all, so a
|
| 1894 |
-
# sentence claiming only the first would tell a reader their access was revoked when the
|
| 1895 |
-
# store had merely hiccupped. Naming both is honest and still refuses.
|
| 1896 |
-
raise err(403, "not_permitted",
|
| 1897 |
-
f"'{source}' could not be opened for your account, either because it is not "
|
| 1898 |
-
f"granted to you or because it could not be read just now, so rows here cannot "
|
| 1899 |
-
f"be pinned to one of its records. Ask an administrator for access to that "
|
| 1900 |
-
f"database, or try again")
|
| 1901 |
-
|
| 1902 |
-
# ⛔ RESOLVED AGAINST THE **SCOPED** LISTING, which is what makes "an edge you cannot be
|
| 1903 |
-
# offered is an edge you cannot pin" structural rather than remembered. `related_entries` is
|
| 1904 |
-
# the same function `GET /grid/{table}/related` answers with; there is no second resolution.
|
| 1905 |
-
# ⭐ W54-T40 — AND A PATH RESOLVES AGAINST THE DEPTH-AWARE LISTING, WHICH IS THE SAME RULE ONE
|
| 1906 |
-
# LEVEL DEEPER: `related_path_entries` is what `GET /grid/{table}/related?depth=2` answers
|
| 1907 |
-
# with, so a traversal this account is not offered is a traversal it cannot pin either.
|
| 1908 |
-
# ⚠ THE DEPTH ASKED FOR IS THE LENGTH OF THE PATH, never a flat 2. A one-element path costs
|
| 1909 |
-
# the depth-1 enumeration it needs and not the depth-2 one, which on `customer_data` is the
|
| 1910 |
-
# difference between roughly half a second and roughly two.
|
| 1911 |
-
if path is None:
|
| 1912 |
-
entries = related_entries(session, source)
|
| 1913 |
-
entry = relations.pick(entries, via)
|
| 1914 |
-
else:
|
| 1915 |
-
if len(path) > relations.MAX_DEPTH:
|
| 1916 |
-
raise err(400, "over_depth", relations.over_depth_refusal(source, path))
|
| 1917 |
-
entries = related_path_entries(session, source, len(path))
|
| 1918 |
-
entry = relations.pick_path(entries, path)
|
| 1919 |
-
if entry is None:
|
| 1920 |
-
raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via))
|
| 1921 |
-
entry = dict(entry, __source__=source)
|
| 1922 |
-
|
| 1923 |
-
if str(entry.get("target") or "") != str(table_key):
|
| 1924 |
-
raise err(400, "unknown_relation",
|
| 1925 |
-
f"the relation '{entry.get('key')}' on '{source}' points at "
|
| 1926 |
-
f"'{entry.get('target')}', not at '{table_key}', so it cannot narrow this "
|
| 1927 |
-
f"database. Read the database the relation names, or ask for the relation that "
|
| 1928 |
-
f"reaches this one")
|
| 1929 |
-
if not entry.get("served", True):
|
| 1930 |
-
raise err(409, "relation_not_served", relations.unserved_target_refusal(entry))
|
| 1931 |
-
|
| 1932 |
-
# ⭐⭐ W54-T40 — THE **LAST** HOP IS WHAT JOINS THIS DATABASE, and on a two-hop entry that is
|
| 1933 |
-
# not `entry`. A composite carries `via = {direction: 'path', key, path}` and NO `on`, no
|
| 1934 |
-
# `from` and no `where`: `relations._two_hop_entry` refuses to flatten the two hops' predicates
|
| 1935 |
-
# onto the pair, because one of them may carry a predicate the other does not (the open-invoice
|
| 1936 |
-
# bridge is exactly that edge). So the legs are resolved, each from its OWN table's scoped
|
| 1937 |
-
# listing, and the last one is what this block below reads.
|
| 1938 |
notes = []
|
| 1939 |
-
|
| 1940 |
-
if len(entry.get("path") or ()) > 1:
|
| 1941 |
-
first, mid_key, last = _related_path_legs(session, source, entry, entries)
|
| 1942 |
-
|
| 1943 |
-
on = str((last.get("via") or {}).get("on") or "")
|
| 1944 |
-
# ⚠ `spec["cols"]`, NOT the caller's narrowed `cols`. The join column is a JOIN, not a value
|
| 1945 |
-
# this response returns: a relation must keep working when an administrator has hidden the id
|
| 1946 |
-
# column it runs over, and no cell of it is ever projected. The hidden-field closure still
|
| 1947 |
-
# governs everything that reaches the wire, one layer up.
|
| 1948 |
-
# ⭐⭐ W54-T40 — AND THE ROW IDENTITY IS NOT ONE OF THOSE COLUMNS. `relations` defaults an edge
|
| 1949 |
-
# with no declared `on` to `pid`, which is the wire's name for the row identity and is never a
|
| 1950 |
-
# key of a mirror spec's `cols` (measured across all eight bound grains on 2026-09-06: not one
|
| 1951 |
-
# carries it). Every such edge therefore refused with "has no live expression for 'pid'" while
|
| 1952 |
-
# the expression was in hand the whole time, as `spec["id"]` - the same per-table identity this
|
| 1953 |
-
# route already uses for the pin itself, for the tiebreak and for the fallback ORDER BY, and
|
| 1954 |
-
# deliberately not a bare `id`, which is ambiguous on a join-shaped grain.
|
| 1955 |
-
if on == relations.IDENTITY_COL:
|
| 1956 |
-
binding = (spec["id"], int)
|
| 1957 |
-
else:
|
| 1958 |
-
binding = (spec.get("cols") or {}).get(on)
|
| 1959 |
-
if not binding:
|
| 1960 |
-
raise err(409, "relation_not_served",
|
| 1961 |
-
f"'{table_key}' has no live expression for '{on}', which is the column this "
|
| 1962 |
-
f"relation joins on, so the rows cannot be narrowed to one record here. Open "
|
| 1963 |
-
f"this database directly, or ask for '{on}' to be added to its live connection")
|
| 1964 |
-
on_expr, on_cast = binding[0], binding[1]
|
| 1965 |
-
|
| 1966 |
-
if mid_key:
|
| 1967 |
-
# ⭐ HOP 1, ANSWERED BY THE INTERMEDIATE DATABASE ITSELF, UNDER ITS OWN WALL. See
|
| 1968 |
-
# `_related_hop_values`. An empty answer is an ANSWER (this record reaches nothing through
|
| 1969 |
-
# that database) and compiles to `1=0` two blocks down, never to "no pin".
|
| 1970 |
-
values = _related_hop_values(session, source, record, first, mid_key, last, notes)
|
| 1971 |
-
else:
|
| 1972 |
-
value = _related_source_value(session, entry, record)
|
| 1973 |
-
if value in (None, ""):
|
| 1974 |
-
raise err(400, "unknown_relation",
|
| 1975 |
-
relations.unresolvable_source_refusal(source, entry))
|
| 1976 |
-
values = [value]
|
| 1977 |
-
cast = []
|
| 1978 |
-
for one in values:
|
| 1979 |
-
try:
|
| 1980 |
-
cast.append(on_cast(one))
|
| 1981 |
-
except Exception: # noqa: BLE001
|
| 1982 |
-
cast.append(str(one))
|
| 1983 |
-
|
| 1984 |
-
# ── the member ids, in ONE SQL over the bridge column (the ticket's `how:`) ────────────────
|
| 1985 |
-
# ⛔ NOT THROUGH `datastore.window`: that clamps at `WINDOW_MAX` (5,000) rows, so a customer
|
| 1986 |
-
# with more order lines than that would be pinned to the first 5,000 of them and the tab would
|
| 1987 |
-
# show a smaller, entirely plausible number with nothing red anywhere. Standing rule 1: no cap
|
| 1988 |
-
# on data from a connected source.
|
| 1989 |
-
# ⚠ THE WALL RIDES ALONG. Resolving ids under `base_where` means the id SET a scoped reader can
|
| 1990 |
-
# produce is already narrowed to rows they may see, so the pin cannot be used to learn how many
|
| 1991 |
-
# rows exist beyond their boundary.
|
| 1992 |
-
# ⭐ W54-T40 — `= ?` FOR ONE VALUE AND `IN (...)` FOR MANY, and the one-value spelling is kept
|
| 1993 |
-
# BYTE FOR BYTE so a one-hop pin compiles exactly the SQL it compiled before this ticket. The
|
| 1994 |
-
# many-value arm is only ever reached by the second hop of a traversal.
|
| 1995 |
-
frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
|
| 1996 |
-
ids = []
|
| 1997 |
-
if cast:
|
| 1998 |
-
parts = [p for p in (base_where, last.get("where")) if p]
|
| 1999 |
-
parts.append(f"{on_expr} = ?" if len(cast) == 1
|
| 2000 |
-
else f"{on_expr} IN ({','.join('?' for _ in cast)})")
|
| 2001 |
-
pred = " AND ".join(f"({p})" for p in parts)
|
| 2002 |
-
ids = [int(r[0]) for r in cur.execute(
|
| 2003 |
-
f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {pred}',
|
| 2004 |
-
list(wall_params) + list(cast)).fetchall() if r and r[0] is not None]
|
| 2005 |
-
|
| 2006 |
-
# ⭐ STANDING RULE 1'S SECOND SENTENCE, ON A COST RATHER THAN ON A CAP. Nothing is limited
|
| 2007 |
-
# here; the reader is told that this particular relation is answered by reading the source
|
| 2008 |
-
# database once per page, because it is keyed on a column rather than on the record id.
|
| 2009 |
-
if str((first.get("via") or {}).get("from") or "") != relations.IDENTITY_COL:
|
| 2010 |
notes.append({
|
| 2011 |
"subject": "related", "effect": "slow",
|
| 2012 |
-
"cause": f"
|
| 2013 |
-
|
| 2014 |
-
f"resolved by reading that database first",
|
| 2015 |
-
"recommendation": "read this database on its own and filter it on "
|
| 2016 |
-
f"'{on}' if you are paging through a large set"})
|
| 2017 |
-
if len(ids) >= _PIN_REPORT_AT:
|
| 2018 |
-
notes.append({
|
| 2019 |
-
"subject": "related", "effect": "slow",
|
| 2020 |
-
"cause": f"this record is linked to {len(ids):,} rows, and each page names every one "
|
| 2021 |
-
f"of them in the query that narrows this database to them",
|
| 2022 |
-
"recommendation": "add a filter beside the relation to ask a smaller question, or "
|
| 2023 |
-
"read this database on its own"})
|
| 2024 |
-
|
| 2025 |
-
sql, params = relations.pin_sql(spec["id"], ids)
|
| 2026 |
return sql, params, notes
|
| 2027 |
|
| 2028 |
|
|
@@ -2300,7 +2209,7 @@ def _may_see_view(session, vid, store):
|
|
| 2300 |
|
| 2301 |
|
| 2302 |
def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
|
| 2303 |
-
search=None, tree=_TREE_UNSET, related=None):
|
| 2304 |
"""EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
|
| 2305 |
|
| 2306 |
Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits,
|
|
@@ -2360,7 +2269,10 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 2360 |
# tenant #0's eight databases is read-through. Left un-applied, an administrator could set a
|
| 2361 |
# filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows
|
| 2362 |
# used to be. So the wall is APPLIED here, in SQL, three paragraphs down.
|
| 2363 |
-
|
|
|
|
|
|
|
|
|
|
| 2364 |
fields = None
|
| 2365 |
for _bucket, key, _label, mk_fields in rel.TABLES:
|
| 2366 |
if key == table_key:
|
|
@@ -2615,6 +2527,87 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 2615 |
"pred": pred}
|
| 2616 |
|
| 2617 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2618 |
@router.get("/odoo-tables/{table_key}/rows")
|
| 2619 |
def odoo_table_rows(table_key: str,
|
| 2620 |
offset: int = Query(default=0, ge=0),
|
|
@@ -2652,6 +2645,10 @@ def odoo_table_rows(table_key: str,
|
|
| 2652 |
import core.perm_scope as perm_scope
|
| 2653 |
from harness import datastore
|
| 2654 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2655 |
# ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
|
| 2656 |
# hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
|
| 2657 |
# because C1 promises that `/aggregates` answers over *the same compiled predicate this window
|
|
@@ -3041,6 +3038,13 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 3041 |
if group_by is not None and not isinstance(group_by, str):
|
| 3042 |
raise err(400, "bad_selection", "groupBy must be a column id")
|
| 3043 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3044 |
# ⛔⛔ THE PIN RIDES THE **SAME** COMPILER, AND THIS ONE LINE IS THE WHOLE OF CORRECTION (3).
|
| 3045 |
# `/rows` takes `related` as a JSON query STRING and this door takes it as a BODY KEY; a worker
|
| 3046 |
# who assumed both were bodies would have wired the rows and left the summaries answering over
|
|
|
|
| 23 |
import hmac
|
| 24 |
import json
|
| 25 |
import time
|
| 26 |
+
from contextlib import contextmanager
|
| 27 |
|
| 28 |
from fastapi import Body, Depends, HTTPException
|
| 29 |
from fastapi import APIRouter, Query
|
|
|
|
| 1517 |
return None
|
| 1518 |
|
| 1519 |
|
| 1520 |
+
def _materialised_table(table_key):
|
| 1521 |
+
"""Storage dispatch, independent of the browser's fallible status roster."""
|
| 1522 |
+
return module_cache_source(table_key) is not None or table_key not in _sources()
|
| 1523 |
+
|
| 1524 |
+
|
| 1525 |
+
def _traversal_defn_or_refuse(session, table_key):
|
| 1526 |
+
"""R50 / C4: admit an intermediate after the caller has established both end grants.
|
| 1527 |
+
|
| 1528 |
+
The session may already open both ends of a declared two-hop link. Its intermediate
|
| 1529 |
+
is read only to find target ids, and only join values leave this traversal. The compiled
|
| 1530 |
+
read still applies the row wall, hidden-field closure, BU scope and mirror guard.
|
| 1531 |
+
"""
|
| 1532 |
+
ut = _ut()
|
| 1533 |
+
st = ut.lend_defs(session.runtime)
|
| 1534 |
+
defn = ut.get(table_key, st=st)
|
| 1535 |
+
if not defn:
|
| 1536 |
+
raise err(404, "unknown_table", "that database does not exist")
|
| 1537 |
+
return defn
|
| 1538 |
+
|
| 1539 |
+
|
| 1540 |
+
def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
|
| 1541 |
+
traversal_of=None):
|
| 1542 |
+
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
|
| 1543 |
+
|
| 1544 |
+
Join columns may be hidden on the wire. Only identities leave the resolver; the native
|
| 1545 |
+
payload builders still apply their complete hidden-field closure before returning cells.
|
| 1546 |
+
With traversal_of, the caller must already have established both ends of the declared
|
| 1547 |
+
edge named by that key. Only join values leave this intermediate read, never a cell.
|
| 1548 |
+
"""
|
| 1549 |
+
import core.perm_scope as perm_scope
|
| 1550 |
+
import core.relations as relations
|
| 1551 |
+
|
| 1552 |
+
columns = set(columns) | {relations.IDENTITY_COL}
|
| 1553 |
+
if not _materialised_table(table):
|
| 1554 |
+
read = (_compiled_read(table, session, traversal_intermediate=True)
|
| 1555 |
+
if traversal_of is not None else _compiled_read(table, session))
|
| 1556 |
+
spec = read["spec"]
|
| 1557 |
+
bindings = dict(spec["cols"], pid=(spec["id"], int))
|
| 1558 |
+
if columns - set(bindings):
|
| 1559 |
+
raise err(409, "relation_target_unavailable", "A required relation column is unavailable.")
|
| 1560 |
+
parts = [p for p in (read["where"], where) if p]
|
| 1561 |
+
params = list(read["params"])
|
| 1562 |
+
for key, values in (("pid", ids), match or ("pid", None)):
|
| 1563 |
+
if values is None:
|
| 1564 |
+
continue
|
| 1565 |
+
expr, cast = bindings[key]
|
| 1566 |
+
values = list(values)
|
| 1567 |
+
parts.append(f"{expr} IN ({','.join('?' for _ in values)})" if values else "1=0")
|
| 1568 |
+
params.extend(cast(value) for value in values)
|
| 1569 |
+
names = sorted(columns)
|
| 1570 |
+
projection = ", ".join(bindings[key][0] for key in names)
|
| 1571 |
+
source = spec.get("from_sql") or spec["table"]
|
| 1572 |
+
predicate = " AND ".join(f"({part})" for part in parts) or "1=1"
|
| 1573 |
+
return [dict(zip(names, row)) for row in read["cur"].execute(
|
| 1574 |
+
f"SELECT {projection} FROM {source} WHERE {predicate}", params).fetchall()]
|
| 1575 |
+
|
| 1576 |
+
# This is an internal join, never a cell response. Apply the same permanent wall on the
|
| 1577 |
+
# complete contract before taking any join value, including a join on a hidden column.
|
| 1578 |
+
if traversal_of is None and not perm_scope.may_read(session.user, table, st=session.runtime):
|
| 1579 |
+
raise err(403, "forbidden_table", "This account cannot open the linked database.")
|
| 1580 |
+
if module_cache_source(table) is not None:
|
| 1581 |
+
session.require(table)
|
| 1582 |
+
fields, rows = perm_scope._read(table, session.user, session.runtime)
|
| 1583 |
+
rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime)
|
| 1584 |
+
if where:
|
| 1585 |
+
raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.")
|
| 1586 |
+
if ids is not None:
|
| 1587 |
+
wanted = {str(value) for value in ids}
|
| 1588 |
+
rows = [row for row in rows if str(row.get("pid")) in wanted]
|
| 1589 |
+
if match is not None:
|
| 1590 |
+
key, values = match
|
| 1591 |
+
wanted = {str(value) for value in values}
|
| 1592 |
+
rows = [row for row in rows if str(row.get(key)) in wanted]
|
| 1593 |
+
return [{key: row.get(key) for key in columns} for row in rows]
|
| 1594 |
+
|
| 1595 |
+
|
| 1596 |
+
def _relation_cell_ids(value):
|
| 1597 |
+
"""Stored link cells contain comma-separated identities, not a scalar join value."""
|
| 1598 |
+
parts = value if isinstance(value, (list, tuple, set)) else str(value or "").split(",")
|
| 1599 |
+
return {str(int(part)) for part in parts if str(part).strip().isdigit() and int(part) > 0}
|
| 1600 |
+
|
| 1601 |
+
|
| 1602 |
+
def _relation_leg_ids(session, source, source_ids, entry):
|
| 1603 |
+
"""Follow one declared leg, including manual, inverse and through links."""
|
| 1604 |
+
import core.relations as relations
|
| 1605 |
+
|
| 1606 |
+
target = str(entry["target"])
|
| 1607 |
+
via = entry.get("via") or {}
|
| 1608 |
+
link = entry.get("link") or {}
|
| 1609 |
+
reverse = via.get("direction") == "reverse"
|
| 1610 |
+
frm, on = str(via.get("from") or "pid"), str(via.get("on") or "pid")
|
| 1611 |
+
field = str(via.get("field") or "")
|
| 1612 |
+
manual = bool(field and link and not link.get("inverse")
|
| 1613 |
+
and (link.get("selectionMode") == "manual" or not link.get("on")))
|
| 1614 |
+
inverse = str(link.get("inverse") or "")
|
| 1615 |
+
source_col, target_col = frm, on
|
| 1616 |
+
if manual:
|
| 1617 |
+
source_col, target_col = ("pid", field) if reverse else (field, "pid")
|
| 1618 |
+
elif inverse:
|
| 1619 |
+
source_col, target_col = (inverse, "pid") if reverse else ("pid", inverse)
|
| 1620 |
+
source_rows = _relation_rows(session, source, [source_col], ids=source_ids)
|
| 1621 |
+
values = {str(row[source_col]) for row in source_rows if row.get(source_col) not in (None, "")}
|
| 1622 |
+
# A column-keyed edge cannot fall back to the wire record id. In particular,
|
| 1623 |
+
# product_data.pid is a stable hash while sale_order_line.product_id is an Odoo
|
| 1624 |
+
# identity. Treating a missing/hidden/blank source value as an empty match would
|
| 1625 |
+
# turn "the join input is unavailable" into the business claim "no related rows".
|
| 1626 |
+
if source_col != relations.IDENTITY_COL and not values:
|
| 1627 |
+
raise err(400, "unknown_relation", relations.unresolvable_source_refusal(source, entry))
|
| 1628 |
+
if (manual and not reverse) or (inverse and reverse):
|
| 1629 |
+
values = set().union(*(_relation_cell_ids(row.get(source_col)) for row in source_rows))
|
| 1630 |
+
target_col = "pid"
|
| 1631 |
+
through = relations.declared_through(link) if not manual else None
|
| 1632 |
+
if through:
|
| 1633 |
+
left = str(link["throughTo"] if reverse else link["throughOn"])
|
| 1634 |
+
right = str(link["throughOn"] if reverse else link["throughTo"])
|
| 1635 |
+
middle = _relation_rows(session, str(through), [left, right], match=(left, values),
|
| 1636 |
+
traversal_of=str(entry.get("key") or via.get("key") or ""))
|
| 1637 |
+
values = {str(row[right]) for row in middle if row.get(right) not in (None, "")}
|
| 1638 |
+
if (manual and reverse) or (inverse and not reverse):
|
| 1639 |
+
rows = _relation_rows(session, target, [target_col])
|
| 1640 |
+
rows = [row for row in rows if _relation_cell_ids(row.get(target_col)) & values]
|
| 1641 |
+
else:
|
| 1642 |
+
rows = _relation_rows(session, target, [target_col], match=(target_col, values),
|
| 1643 |
+
where=entry.get("where"))
|
| 1644 |
+
return {int(row["pid"]) for row in rows}
|
| 1645 |
+
|
| 1646 |
+
|
| 1647 |
+
def _relation_label(key):
|
| 1648 |
+
"""Prefer the database's display label in relation refusals."""
|
| 1649 |
+
source = module_cache_source(key)
|
| 1650 |
+
if source and source.get("label"):
|
| 1651 |
+
return source["label"]
|
| 1652 |
+
return next((label for _bucket, table, label, _fields in _rel().TABLES
|
| 1653 |
+
if table == key and label), str(key))
|
| 1654 |
+
|
| 1655 |
+
|
| 1656 |
+
def _relation_target_ids(session, target, related):
|
| 1657 |
+
"""One pin resolver for SQL, pool and tenant rows, selected from the scoped path list."""
|
| 1658 |
+
import core.relations as relations
|
| 1659 |
+
from routes_grid import _may_open, related_path_entries
|
| 1660 |
+
|
| 1661 |
+
if isinstance(related, str):
|
| 1662 |
+
related = _json_arg(related, "related")
|
| 1663 |
+
if not isinstance(related, dict) or not related.get("table") or related.get("record") in (None, ""):
|
| 1664 |
+
raise err(400, "bad_relation", "A pinned read needs {table, record, via}.")
|
| 1665 |
+
source = str(related["table"])
|
| 1666 |
+
via = related.get("via")
|
| 1667 |
+
path = _via_path(via)
|
| 1668 |
+
if path and len(path) > relations.MAX_DEPTH:
|
| 1669 |
+
raise err(400, "over_depth", relations.over_depth_refusal(source, path))
|
| 1670 |
+
if not _may_open(session, source):
|
| 1671 |
+
raise err(403, "forbidden_table",
|
| 1672 |
+
f"the relation starts at '{_relation_label(source)}', which your account cannot open")
|
| 1673 |
+
entries = related_path_entries(session, source, len(path) if path else 1)
|
| 1674 |
+
entry = relations.pick_path(entries, path) if path else relations.pick(entries, via)
|
| 1675 |
+
if entry is None or str(entry.get("target")) != str(target):
|
| 1676 |
+
raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via))
|
| 1677 |
+
try:
|
| 1678 |
+
ids = {int(related["record"])}
|
| 1679 |
+
except (TypeError, ValueError):
|
| 1680 |
+
raise err(400, "bad_relation", "The source record must have an integer identity.")
|
| 1681 |
+
legs = [(source, entry)]
|
| 1682 |
+
if len(entry.get("path") or ()) > 1:
|
| 1683 |
+
first, middle, last = _related_path_legs(session, source, entry, entries)
|
| 1684 |
+
legs = [(source, first), (middle, last)]
|
| 1685 |
+
for table, leg in legs:
|
| 1686 |
+
if not leg.get("served"):
|
| 1687 |
+
raise err(409, "relation_not_served", relations.unserved_target_refusal(leg))
|
| 1688 |
+
if not _may_open(session, leg["target"]):
|
| 1689 |
+
raise err(403, "forbidden_table",
|
| 1690 |
+
f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
|
| 1691 |
+
ids = _relation_leg_ids(session, table, ids, leg)
|
| 1692 |
+
return ids
|
| 1693 |
+
|
| 1694 |
+
|
| 1695 |
+
def narrow_related_rows(session, table_key, rows, related):
|
| 1696 |
+
"""Intersect an already walled and stripped payload with one resolved identity set."""
|
| 1697 |
+
if related in (None, "", {}):
|
| 1698 |
+
return rows
|
| 1699 |
+
ids = _relation_target_ids(session, table_key, related)
|
| 1700 |
+
return [row for row in rows if int(row["pid"]) in ids]
|
| 1701 |
+
|
| 1702 |
+
|
| 1703 |
def _cache_relation_plan(session, source, via, target):
|
| 1704 |
"""The offered module link and its explicit bridge, or a cause-specific refusal.
|
| 1705 |
|
|
|
|
| 1722 |
raise err(404, "relation_target_missing",
|
| 1723 |
f"'{target}' is not a current module/cache relation target in this workspace")
|
| 1724 |
if not _may_open(session, source):
|
| 1725 |
+
raise err(403, "forbidden_table",
|
| 1726 |
+
f"the relation starts at '{_relation_label(source)}', which your account cannot open")
|
|
|
|
| 1727 |
|
| 1728 |
offered = next((entry for entry in related_entries(session, source)
|
| 1729 |
if str(entry.get("key") or "") == via
|
|
|
|
| 1740 |
if not isinstance(link, dict) or str(link.get("table") or "") != target:
|
| 1741 |
raise err(404, "relation_target_missing",
|
| 1742 |
f"the relation '{via}' no longer has a current target in this workspace")
|
| 1743 |
+
through = relations.declared_through(link)
|
| 1744 |
through_on = str(link.get("throughOn") or "")
|
| 1745 |
through_to = str(link.get("throughTo") or "")
|
| 1746 |
on = str(link.get("on") or relations.IDENTITY_COL)
|
|
|
|
| 1750 |
if relations.is_retired_table(through):
|
| 1751 |
raise err(404, "relation_target_retired",
|
| 1752 |
f"'{through}' is retired, so the relation to '{target}' is no longer available")
|
| 1753 |
+
# R50: both end grants suffice. Traversal admission in _compiled_read replaces the
|
| 1754 |
+
# intermediate grant check and still applies the intermediate's row wall.
|
|
|
|
|
|
|
| 1755 |
if not _may_open(session, target):
|
| 1756 |
raise err(403, "forbidden_table",
|
| 1757 |
+
f"the relation ends at '{_relation_label(target)}', which your account cannot open")
|
| 1758 |
return dict(offered, __source__=source), link, through, through_on, through_to, on
|
| 1759 |
|
| 1760 |
|
|
|
|
| 1782 |
|
| 1783 |
|
| 1784 |
def _cache_relation_target_ids(session, source, record, via, target):
|
| 1785 |
+
"""Resolve any offered path; retain the legacy Products ordered string transport."""
|
| 1786 |
+
via_key = str(via.get("key") or "") if isinstance(via, dict) else str(via or "")
|
| 1787 |
+
if not (source == "customer_data" and target == "product_data"
|
| 1788 |
+
and via_key == "products_ordered"):
|
| 1789 |
+
ids = _relation_target_ids(session, target, {"table": source, "record": record, "via": via})
|
| 1790 |
+
return {"ok": True, "target": str(target), "ids": sorted(ids)}
|
| 1791 |
entry, _link, through, through_on, through_to, on = _cache_relation_plan(
|
| 1792 |
+
session, source, via_key, target)
|
| 1793 |
source_value = _cache_relation_source_value(session, entry, record)
|
| 1794 |
if source_value in (None, ""):
|
| 1795 |
raise err(400, "unknown_relation",
|
| 1796 |
+
f"record {record} has no readable source value for relation '{via_key}'")
|
| 1797 |
|
| 1798 |
+
# R50 / C4: the intermediate of a DECLARED two-hop link is read to find target ids
|
| 1799 |
+
# only, so it is admitted on both ends' grants. The compiler still applies its row
|
| 1800 |
+
# wall, hidden-field closure and mirror guard to it.
|
| 1801 |
+
compiled = _compiled_read(through, session, traversal_intermediate=True)
|
| 1802 |
spec = compiled["spec"]
|
| 1803 |
left = (spec.get("cols") or {}).get(through_on)
|
| 1804 |
right = (spec.get("cols") or {}).get(through_to)
|
|
|
|
| 1847 |
via: str = Query(...), target: str = Query(...),
|
| 1848 |
session: Session = Depends(require_session)):
|
| 1849 |
"""Target pids for a declared relation whose destination is a runtime-cache database."""
|
| 1850 |
+
parsed_via = _json_arg(via, "via") if via.startswith(("[", "{")) else via
|
| 1851 |
+
answer = _cache_relation_target_ids(session, source, record, parsed_via, target)
|
| 1852 |
+
# Internal cache diagnostics (sourceKind/path) stay internal; the public door has one stable
|
| 1853 |
+
# response shape for generic, legacy-string and structured-bag transports.
|
| 1854 |
+
return {"ok": True, "target": str(target), "ids": list(answer.get("ids") or ())}
|
| 1855 |
|
| 1856 |
|
| 1857 |
def _via_path(via):
|
|
|
|
| 1914 |
return dict(first, __source__=source), mid_key, dict(last, __source__=mid_key)
|
| 1915 |
|
| 1916 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1917 |
|
| 1918 |
|
| 1919 |
def _related_pin(table_key, session, spec, base_where, wall_params, cur, related):
|
| 1920 |
+
"""Compile the same resolved identity set on both mirror transports, as an intersection."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1921 |
import core.relations as relations
|
| 1922 |
|
| 1923 |
if isinstance(related, str):
|
| 1924 |
related = _json_arg(related, "related")
|
| 1925 |
if related in (None, "", {}):
|
| 1926 |
return None, [], []
|
| 1927 |
+
ids = _relation_target_ids(session, table_key, related)
|
| 1928 |
+
sql, params = relations.pin_sql(spec["id"], sorted(ids))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1929 |
notes = []
|
| 1930 |
+
if len(ids) > _PIN_REPORT_AT:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1931 |
notes.append({
|
| 1932 |
"subject": "related", "effect": "slow",
|
| 1933 |
+
"cause": f"This relation matched {len(ids):,} records, all of which are included.",
|
| 1934 |
+
"recommendation": "Add a filter to reduce the result while continuing to page through every match."})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1935 |
return sql, params, notes
|
| 1936 |
|
| 1937 |
|
|
|
|
| 2209 |
|
| 2210 |
|
| 2211 |
def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
|
| 2212 |
+
search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False):
|
| 2213 |
"""EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
|
| 2214 |
|
| 2215 |
Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits,
|
|
|
|
| 2269 |
# tenant #0's eight databases is read-through. Left un-applied, an administrator could set a
|
| 2270 |
# filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows
|
| 2271 |
# used to be. So the wall is APPLIED here, in SQL, three paragraphs down.
|
| 2272 |
+
if traversal_intermediate:
|
| 2273 |
+
defn = _traversal_defn_or_refuse(session, table_key)
|
| 2274 |
+
else:
|
| 2275 |
+
defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
|
| 2276 |
fields = None
|
| 2277 |
for _bucket, key, _label, mk_fields in rel.TABLES:
|
| 2278 |
if key == table_key:
|
|
|
|
| 2527 |
"pred": pred}
|
| 2528 |
|
| 2529 |
|
| 2530 |
+
@contextmanager
|
| 2531 |
+
def _materialised_read(table_key, session, query, related):
|
| 2532 |
+
"""One compiler for native rows and their aggregate sibling, over the walled payload.
|
| 2533 |
+
|
| 2534 |
+
The temporary connection contains only this request's already stripped, pinned rows and is
|
| 2535 |
+
closed before returning. It never registers data on the tenant's shared mirror connection.
|
| 2536 |
+
"""
|
| 2537 |
+
import duckdb
|
| 2538 |
+
|
| 2539 |
+
if table_key == "customer_data":
|
| 2540 |
+
from routes_customers import _payload
|
| 2541 |
+
session.require(table_key)
|
| 2542 |
+
payload = _payload(session, related=related)
|
| 2543 |
+
elif table_key == "product_data":
|
| 2544 |
+
from routes_products import products
|
| 2545 |
+
session.require(table_key)
|
| 2546 |
+
payload = products(session=session, related=related)
|
| 2547 |
+
else:
|
| 2548 |
+
from routes_tables import table_rows
|
| 2549 |
+
payload = table_rows(table_key, session=session, related=related)
|
| 2550 |
+
fields = payload["fields"]
|
| 2551 |
+
with duckdb.connect(":memory:") as cur:
|
| 2552 |
+
cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)",
|
| 2553 |
+
[json.dumps(payload["rows"], default=str)])
|
| 2554 |
+
bindings = {}
|
| 2555 |
+
for field in fields:
|
| 2556 |
+
key = str(field["key"])
|
| 2557 |
+
pointer = ("/" + key.replace("~", "~0").replace("/", "~1")).replace("'", "''")
|
| 2558 |
+
expr = f"json_extract_string(payload, '{pointer}')"
|
| 2559 |
+
numeric = field.get("type") in ("int", "currency", "pct", "float", "number")
|
| 2560 |
+
bindings[key] = (f"TRY_CAST({expr} AS DOUBLE)" if numeric else expr,
|
| 2561 |
+
float if numeric else str)
|
| 2562 |
+
spec = {"table": "pinned_rows", "id": "CAST(json_extract_string(payload, '/pid') AS BIGINT)",
|
| 2563 |
+
"cols": bindings, "where": None}
|
| 2564 |
+
cols = _filter_columns(spec, fields)
|
| 2565 |
+
limits = list(payload.get("limits") or [])
|
| 2566 |
+
tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
|
| 2567 |
+
None, [], cur, limits)
|
| 2568 |
+
missing = _leaf_cols(tree) - _PSEUDO_COLS - set(cols)
|
| 2569 |
+
if missing:
|
| 2570 |
+
raise err(400, "filter_unsupported", "A filter names a column unavailable to this account.")
|
| 2571 |
+
try:
|
| 2572 |
+
pred = _fs().compile_filter_tree(tree, conj=query["filterConj"], columns=cols,
|
| 2573 |
+
id_sql=spec["id"], today=payload.get("today") or _today(),
|
| 2574 |
+
cohort_sets=cohorts)
|
| 2575 |
+
except ValueError as exc:
|
| 2576 |
+
raise err(400, "filter_unsupported", str(exc))
|
| 2577 |
+
if pred is not None and pred.uses_aggregate:
|
| 2578 |
+
raise err(400, "filter_unsupported", "This condition cannot be evaluated for individual rows.")
|
| 2579 |
+
where, params = (pred.sql, list(pred.params)) if pred is not None else (None, [])
|
| 2580 |
+
search = _fs().compile_search(query["search"], cols) if query["search"] else None
|
| 2581 |
+
if search is not None:
|
| 2582 |
+
where = f"({where}) AND {search.sql}" if where else search.sql
|
| 2583 |
+
params.extend(search.params)
|
| 2584 |
+
yield {"cur": cur, "spec": spec, "cols": cols, "where": where, "params": params,
|
| 2585 |
+
"frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload}
|
| 2586 |
+
|
| 2587 |
+
|
| 2588 |
+
def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related):
|
| 2589 |
+
from harness import datastore
|
| 2590 |
+
|
| 2591 |
+
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
|
| 2592 |
+
"sorts": _json_arg(sorts, "sorts"), "search": search})
|
| 2593 |
+
with _materialised_read(table_key, session, query, related) as read:
|
| 2594 |
+
spec = read["spec"]
|
| 2595 |
+
order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
|
| 2596 |
+
win = datastore.window(select=f'{spec["id"]} AS pid', where=read["where"],
|
| 2597 |
+
params=tuple(read["params"]), order_by=order or spec["id"],
|
| 2598 |
+
offset=offset, limit=limit or None, cur=read["cur"], **read["frm"])
|
| 2599 |
+
payload = read["payload"]
|
| 2600 |
+
by_id = {int(row["pid"]): row for row in payload["rows"]}
|
| 2601 |
+
rows = [by_id[int(row[0])] for row in win["rows"]]
|
| 2602 |
+
if win["clamped"]:
|
| 2603 |
+
read["limits"].append({"subject": "limit", "effect": "window_clamped",
|
| 2604 |
+
"cause": f"One response carries at most {datastore.WINDOW_MAX} rows to bound response memory.",
|
| 2605 |
+
"recommendation": "Continue paging with offset. Every matching row remains available."})
|
| 2606 |
+
return {**payload, "ok": True, "rows": rows, "total": win["total"],
|
| 2607 |
+
"totalUnfiltered": len(by_id), "offset": win["offset"], "limit": win["limit"],
|
| 2608 |
+
"limits": read["limits"], "scope": {"table": table_key, "readThrough": True}}
|
| 2609 |
+
|
| 2610 |
+
|
| 2611 |
@router.get("/odoo-tables/{table_key}/rows")
|
| 2612 |
def odoo_table_rows(table_key: str,
|
| 2613 |
offset: int = Query(default=0, ge=0),
|
|
|
|
| 2645 |
import core.perm_scope as perm_scope
|
| 2646 |
from harness import datastore
|
| 2647 |
|
| 2648 |
+
if _materialised_table(table_key):
|
| 2649 |
+
return _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
| 2650 |
+
sorts, search, related)
|
| 2651 |
+
|
| 2652 |
# ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
|
| 2653 |
# hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
|
| 2654 |
# because C1 promises that `/aggregates` answers over *the same compiled predicate this window
|
|
|
|
| 3038 |
if group_by is not None and not isinstance(group_by, str):
|
| 3039 |
raise err(400, "bad_selection", "groupBy must be a column id")
|
| 3040 |
|
| 3041 |
+
if _materialised_table(table_key):
|
| 3042 |
+
with _materialised_read(table_key, session, query, payload.get("related")) as read:
|
| 3043 |
+
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
|
| 3044 |
+
read["params"], read["frm"], aggs, group_by=group_by,
|
| 3045 |
+
limits=read["limits"])
|
| 3046 |
+
return {"ok": True, **out, "scope": {"table": table_key, "readThrough": True}}
|
| 3047 |
+
|
| 3048 |
# ⛔⛔ THE PIN RIDES THE **SAME** COMPILER, AND THIS ONE LINE IS THE WHOLE OF CORRECTION (3).
|
| 3049 |
# `/rows` takes `related` as a JSON query STRING and this door takes it as a BODY KEY; a worker
|
| 3050 |
# who assumed both were bodies would have wired the rows and left the summaries answering over
|
api/routes_products.py
CHANGED
|
@@ -498,6 +498,8 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str
|
|
| 498 |
str(pid): {key: value for key, value in (row or {}).items() if key not in hidden}
|
| 499 |
for pid, row in (ws.get("overlays") or {}).items()
|
| 500 |
}
|
|
|
|
|
|
|
| 501 |
|
| 502 |
today = time.strftime("%Y-%m-%d")
|
| 503 |
fields = aios_grid.project_date_recurrences(fields, today=today)
|
|
@@ -561,7 +563,7 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str
|
|
| 561 |
|
| 562 |
|
| 563 |
@router.get("/products")
|
| 564 |
-
def products(session: Session = Depends(module_gate(MODULE))):
|
| 565 |
"""The product table for this session's scope — the /customers envelope byte-for-byte
|
| 566 |
(`{fields, rows, today, pulled_at}`) plus two additive keys (`identity`, `scope`).
|
| 567 |
|
|
@@ -573,13 +575,21 @@ def products(session: Session = Depends(module_gate(MODULE))):
|
|
| 573 |
import aios_grid
|
| 574 |
import modules.product_data as pd
|
| 575 |
|
| 576 |
-
g = product_assembly(session)
|
| 577 |
rows = aios_grid.rows_from_pool(
|
| 578 |
g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"])
|
| 579 |
rows = _seed_image(_seed_shared(rows, g["rows_src"], g["fields"]), g["fields"])
|
| 580 |
from core import user_tables as _zones
|
| 581 |
rows = _zones.zone_cells("product_data", rows, allowed_row_ids=g["pids"],
|
| 582 |
st=session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 583 |
# ⭐⭐ W41-T17 (owner instruction 25) — THE IDENTITY REPORT RIDES THE ENVELOPE, because a
|
| 584 |
# report nothing reads is not a report. `product_data.identity_report` is standing rule 1's
|
| 585 |
# second sentence as data for this grid's two identity columns: `product_id` (the Odoo
|
|
@@ -599,12 +609,21 @@ def products(session: Session = Depends(module_gate(MODULE))):
|
|
| 599 |
# The envelope check beside it filters to the four `/customers` keys, so an additive key is
|
| 600 |
# the shape this route already established with `identity` and `scope`.
|
| 601 |
return {"fields": g["fields"], "rows": rows,
|
|
|
|
|
|
|
| 602 |
"today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
|
| 603 |
"identity": {"pid": "pid", "businessKey": "code"},
|
| 604 |
"identity_report": pd.identity_report(g["rows_src"]),
|
| 605 |
"scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}}
|
| 606 |
|
| 607 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 608 |
@router.patch("/products/{pid}")
|
| 609 |
def patch_product(pid: int, body: dict = Body(default=None),
|
| 610 |
session: Session = Depends(module_gate(MODULE))):
|
|
@@ -769,7 +788,8 @@ def pd_fields(consolidated=True, st=None):
|
|
| 769 |
# `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a
|
| 770 |
# column `modules.product_data.pool()` reads off a SKU — and this one is neither read from
|
| 771 |
# Odoo nor written to it. Injected after the JSON so the canonical file stays the answer to
|
| 772 |
-
# "what does Odoo give us"
|
|
|
|
| 773 |
if st is not None:
|
| 774 |
image = pd.image_field_definition(st=st)
|
| 775 |
else:
|
|
|
|
| 498 |
str(pid): {key: value for key, value in (row or {}).items() if key not in hidden}
|
| 499 |
for pid, row in (ws.get("overlays") or {}).items()
|
| 500 |
}
|
| 501 |
+
# D-464: the saved views' column keys are the next wire `/workspace` serves.
|
| 502 |
+
workspace["views"] = aios_grid.narrow_view_keys(workspace.get("views"), hidden)
|
| 503 |
|
| 504 |
today = time.strftime("%Y-%m-%d")
|
| 505 |
fields = aios_grid.project_date_recurrences(fields, today=today)
|
|
|
|
| 563 |
|
| 564 |
|
| 565 |
@router.get("/products")
|
| 566 |
+
def products(session: Session = Depends(module_gate(MODULE)), related: str = None):
|
| 567 |
"""The product table for this session's scope — the /customers envelope byte-for-byte
|
| 568 |
(`{fields, rows, today, pulled_at}`) plus two additive keys (`identity`, `scope`).
|
| 569 |
|
|
|
|
| 575 |
import aios_grid
|
| 576 |
import modules.product_data as pd
|
| 577 |
|
| 578 |
+
g = product_assembly(session, consume_corrections=not bool(related))
|
| 579 |
rows = aios_grid.rows_from_pool(
|
| 580 |
g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"])
|
| 581 |
rows = _seed_image(_seed_shared(rows, g["rows_src"], g["fields"]), g["fields"])
|
| 582 |
from core import user_tables as _zones
|
| 583 |
rows = _zones.zone_cells("product_data", rows, allowed_row_ids=g["pids"],
|
| 584 |
st=session.runtime)
|
| 585 |
+
from routes_odoo_tables import narrow_related_rows
|
| 586 |
+
rows = narrow_related_rows(session, MODULE, rows, related)
|
| 587 |
+
from routes_tables import scope_linked_metrics, linked_metric_limits
|
| 588 |
+
rows, g['fields'], metric_refusals = scope_linked_metrics(
|
| 589 |
+
session, MODULE, rows, g['fields'], g['measures'], related)
|
| 590 |
+
if related:
|
| 591 |
+
pinned = {row["pid"] for row in rows}
|
| 592 |
+
g["rows_src"] = [row for row in g["rows_src"] if row["pid"] in pinned]
|
| 593 |
# ⭐⭐ W41-T17 (owner instruction 25) — THE IDENTITY REPORT RIDES THE ENVELOPE, because a
|
| 594 |
# report nothing reads is not a report. `product_data.identity_report` is standing rule 1's
|
| 595 |
# second sentence as data for this grid's two identity columns: `product_id` (the Odoo
|
|
|
|
| 609 |
# The envelope check beside it filters to the four `/customers` keys, so an additive key is
|
| 610 |
# the shape this route already established with `identity` and `scope`.
|
| 611 |
return {"fields": g["fields"], "rows": rows,
|
| 612 |
+
"refusals": metric_refusals,
|
| 613 |
+
"limits": linked_metric_limits(metric_refusals),
|
| 614 |
"today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
|
| 615 |
"identity": {"pid": "pid", "businessKey": "code"},
|
| 616 |
"identity_report": pd.identity_report(g["rows_src"]),
|
| 617 |
"scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}}
|
| 618 |
|
| 619 |
|
| 620 |
+
@router.post("/products/aggregates")
|
| 621 |
+
def product_aggregates(body: dict = Body(default=None),
|
| 622 |
+
session: Session = Depends(module_gate(MODULE))):
|
| 623 |
+
from routes_odoo_tables import odoo_table_aggregates
|
| 624 |
+
return odoo_table_aggregates(MODULE, body=body, session=session)
|
| 625 |
+
|
| 626 |
+
|
| 627 |
@router.patch("/products/{pid}")
|
| 628 |
def patch_product(pid: int, body: dict = Body(default=None),
|
| 629 |
session: Session = Depends(module_gate(MODULE))):
|
|
|
|
| 788 |
# `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a
|
| 789 |
# column `modules.product_data.pool()` reads off a SKU — and this one is neither read from
|
| 790 |
# Odoo nor written to it. Injected after the JSON so the canonical file stays the answer to
|
| 791 |
+
# "what does Odoo give us". `verify_fields_contract` does NOT referee the product contract
|
| 792 |
+
# (D-400); `platform/verify_product_pool.py::section_product_field_contract` does.
|
| 793 |
if st is not None:
|
| 794 |
image = pd.image_field_definition(st=st)
|
| 795 |
else:
|
api/routes_query.py
CHANGED
|
@@ -74,6 +74,19 @@ MAX_HISTORY_CHARS = 6000
|
|
| 74 |
MAX_RATING_REASON = 200
|
| 75 |
QUERY_RATINGS = ("up", "down")
|
| 76 |
MODEL_AUTO = "auto"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
NO_CITED_SOURCES_REASON = (
|
| 78 |
"No authorized application data or retrieved knowledge was cited for this answer."
|
| 79 |
)
|
|
@@ -166,6 +179,11 @@ def _answer_provenance(artifact=None):
|
|
| 166 |
def _public_message(row, allowed_citations):
|
| 167 |
"""Project a stored answer through its current citation wall before it reaches the browser."""
|
| 168 |
out = _safe(row) if isinstance(row, dict) else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
raw_citations = out.get("citationIds")
|
| 170 |
citation_ids = [str(citation_id) for citation_id in (raw_citations if isinstance(raw_citations, list) else [])
|
| 171 |
if str(citation_id) in allowed_citations]
|
|
@@ -1114,6 +1132,8 @@ def _public_state(state, session):
|
|
| 1114 |
messages = sorted(state["messages"].values(), key=lambda row: row.get("createdAt", ""))
|
| 1115 |
views = sorted((_public_view(row) for row in allowed_views), key=lambda row: row["createdAt"], reverse=True)
|
| 1116 |
messages = [row for row in messages if not row.get("viewId") or row.get("viewId") in allowed_ids]
|
|
|
|
|
|
|
| 1117 |
messages = [_public_message(row, allowed_citations) for row in messages]
|
| 1118 |
return {"threads": _safe(threads), "messages": messages, "views": views,
|
| 1119 |
"citations": _safe([row for key, row in state["citations"].items()
|
|
@@ -1344,14 +1364,21 @@ def _submit(body, session, chat=None):
|
|
| 1344 |
raise err(400, "bad_request", "no question was asked")
|
| 1345 |
if len(question) > MAX_QUESTION:
|
| 1346 |
raise err(400, "question_too_long", f"a question must be under {MAX_QUESTION} characters")
|
| 1347 |
-
if not target:
|
| 1348 |
-
raise err(400, "bad_request", "one target database must be selected")
|
| 1349 |
if selected_model not in model_choices():
|
| 1350 |
raise err(400, "unknown_model", "that model is not available in Query")
|
| 1351 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1352 |
# This call happens before model selection and is the only permitted source-data read.
|
| 1353 |
-
snapshot = assistant_read_scope(session, target, fields=None, filters=body.get("filters"))
|
| 1354 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1355 |
state = _state(session)
|
| 1356 |
thread_id = str(body.get("threadId") or "").strip()
|
| 1357 |
if thread_id and thread_id not in state["threads"]:
|
|
@@ -1365,9 +1392,25 @@ def _submit(body, session, chat=None):
|
|
| 1365 |
# turn's own messages exist, so the replay is strictly the prior conversation.
|
| 1366 |
# ⭐ C7/T31: `session` rides along so the meter can attribute this call. See `_call_model`'s
|
| 1367 |
# docstring for why it is an argument and not ambient state (PRD amendment A1).
|
| 1368 |
-
|
| 1369 |
-
|
| 1370 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1371 |
|
| 1372 |
artifact = None
|
| 1373 |
citation = None
|
|
@@ -1404,12 +1447,17 @@ def _submit(body, session, chat=None):
|
|
| 1404 |
assistant_message = {
|
| 1405 |
"id": assistant_message_id, "threadId": thread_id, "role": "assistant", "createdAt": now,
|
| 1406 |
"content": (said or _said_fallback(artifact)) if artifact else (refusal or "the assistant could not answer"),
|
|
|
|
| 1407 |
"targetDatabase": target, "requestedModel": selected_model, "model": provider,
|
|
|
|
| 1408 |
**_model_identity(provider),
|
| 1409 |
"reason": refusal_code, "viewId": artifact["id"] if artifact else None,
|
| 1410 |
"provenance": _answer_provenance(artifact),
|
| 1411 |
"citationIds": artifact["citationIds"] if artifact else [], "numeric": artifact["numeric"] if artifact else None,
|
| 1412 |
}
|
|
|
|
|
|
|
|
|
|
| 1413 |
user_message = {"id": user_message_id, "threadId": thread_id, "role": "user", "createdAt": now,
|
| 1414 |
"content": question, "sources": sources, "targetDatabase": target,
|
| 1415 |
"requestedModel": selected_model}
|
|
@@ -1451,18 +1499,147 @@ def submit_chat(body: dict = Body(default=None), session: Session = Depends(requ
|
|
| 1451 |
# Governed Query contract (Wave 45 C2). The existing chat workspace remains the presentation
|
| 1452 |
# surface; this provider-neutral door is the shared execution seam for Assistant, Agents, voice,
|
| 1453 |
# Slack and future adapters.
|
| 1454 |
-
def _semantic_baseline(database):
|
| 1455 |
from harness import semantic as sem
|
| 1456 |
topic = next((row for row in sem.topics().values()
|
| 1457 |
if isinstance(row, dict) and row.get("grid") == database), None)
|
| 1458 |
-
if
|
| 1459 |
-
|
| 1460 |
-
|
| 1461 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1462 |
|
| 1463 |
|
| 1464 |
def _semantic_records(session, database):
|
| 1465 |
-
return semantic_rt.load(session.runtime, _semantic_baseline(database))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1466 |
|
| 1467 |
|
| 1468 |
def _semantic_authorize(session, database, *, require_answerable=False):
|
|
@@ -1536,7 +1713,7 @@ def update_semantic_field(database: str, field: str, body: dict = Body(default=N
|
|
| 1536 |
session: Session = Depends(require_session)):
|
| 1537 |
_semantic_authorize(session, database)
|
| 1538 |
try:
|
| 1539 |
-
row = semantic_rt.update(session.runtime, _semantic_baseline(database), database, field,
|
| 1540 |
body, actor=session.uname, is_admin=session.admin)
|
| 1541 |
except semantic_rt.SemanticError as exc:
|
| 1542 |
code = "forbidden" if "only a tenant admin" in str(exc) else "bad_semantic_contract"
|
|
@@ -1550,7 +1727,7 @@ def profile_semantic_field(database: str, field: str,
|
|
| 1550 |
session: Session = Depends(require_session)):
|
| 1551 |
"""Profile values through Query's permissioned snapshot and persist only a bounded summary."""
|
| 1552 |
_semantic_authorize(session, database, require_answerable=True)
|
| 1553 |
-
baseline = _semantic_baseline(database)
|
| 1554 |
records = semantic_rt.load(session.runtime, baseline)
|
| 1555 |
try:
|
| 1556 |
current = records[semantic_rt.field_key(database, field)]
|
|
@@ -1574,6 +1751,115 @@ def profile_semantic_field(database: str, field: str,
|
|
| 1574 |
return {"field": row}
|
| 1575 |
|
| 1576 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1577 |
def _resume_b64(raw):
|
| 1578 |
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
| 1579 |
|
|
@@ -1713,12 +1999,10 @@ def query_metrics(database: str = "", session: Session = Depends(require_session
|
|
| 1713 |
_semantic_authorize_schema_only(session, database)
|
| 1714 |
metrics, reason = _metric_catalogue(session, database)
|
| 1715 |
# ⛔⛔ THE GOVERNED SOURCE WALL STAYS, BUT IT CANNOT BE THE FIRST THING ASKED.
|
| 1716 |
-
#
|
| 1717 |
-
#
|
| 1718 |
-
#
|
| 1719 |
-
#
|
| 1720 |
-
# asked only once a topic is known to exist, which is the only case where it has a contract to
|
| 1721 |
-
# enforce; a topic-less database answers 200 with `reason: no_topic`, and that IS its answer.
|
| 1722 |
if reason != "no_topic":
|
| 1723 |
_semantic_records(session, database)
|
| 1724 |
return {"metrics": metrics, "reason": reason}
|
|
@@ -1838,22 +2122,34 @@ def governed_query(body: dict = Body(default=None),
|
|
| 1838 |
body = body if isinstance(body, dict) else {}
|
| 1839 |
database = str(body.get("database") or "").strip()
|
| 1840 |
question = " ".join(str(body.get("prompt") or body.get("question") or "").split())
|
| 1841 |
-
if not
|
| 1842 |
-
raise err(400, "bad_request", "a
|
| 1843 |
if len(question) > MAX_QUESTION:
|
| 1844 |
raise err(400, "question_too_long", f"a prompt must be under {MAX_QUESTION} characters")
|
| 1845 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1846 |
records = _semantic_records(session, database)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1847 |
approved = semantic_rt.approved_fields(records, database)
|
| 1848 |
if not approved:
|
| 1849 |
-
|
| 1850 |
-
"this database has no approved semantic fields yet")
|
| 1851 |
raw_requested = body.get("fieldKeys")
|
| 1852 |
if raw_requested is not None and not isinstance(raw_requested, list):
|
| 1853 |
raise err(400, "bad_request", "fieldKeys must be a list")
|
| 1854 |
requested = list(dict.fromkeys(str(x) for x in (raw_requested or ()) if str(x)))
|
| 1855 |
if not requested:
|
| 1856 |
-
return _clarification(session, database, question, approved)
|
| 1857 |
_verify_resume_token(body.get("resumeToken"), session, database, question,
|
| 1858 |
approved, requested)
|
| 1859 |
try:
|
|
@@ -1861,12 +2157,13 @@ def governed_query(body: dict = Body(default=None),
|
|
| 1861 |
except semantic_rt.SemanticError as exc:
|
| 1862 |
raise err(403, "semantic_field_unapproved", str(exc))
|
| 1863 |
snapshot = assistant_read_scope(session, database, fields=requested, filters=None)
|
|
|
|
| 1864 |
snapshot, formulas = _tool_formula_snapshot(body, snapshot, requested)
|
| 1865 |
spec, planned_wording, provider = _tool_plan(question, body, snapshot, session)
|
| 1866 |
if spec is None:
|
| 1867 |
return {"status": "refused", "wording": planned_wording,
|
| 1868 |
"speakable": planned_wording, "artifacts": None,
|
| 1869 |
-
"model": _model_identity(provider)}
|
| 1870 |
view, refusal, refusal_code = _validate(spec, snapshot.get("fields") or selected)
|
| 1871 |
if view is None:
|
| 1872 |
raise err(400, refusal_code or "bad_query_artifact", refusal or "that Query is invalid")
|
|
@@ -1891,7 +2188,7 @@ def governed_query(body: dict = Body(default=None),
|
|
| 1891 |
"rows": [{key: row.get(key) for key in view["visible"]} for row in rows],
|
| 1892 |
"groups": groups, "numeric": numeric,
|
| 1893 |
"artifacts": artifacts, "semanticRouter": semantic_rt.router_projection(records, database),
|
| 1894 |
-
"model": _model_identity(provider)}
|
| 1895 |
|
| 1896 |
|
| 1897 |
@router.post("/query/formulas/parse")
|
|
|
|
| 74 |
MAX_RATING_REASON = 200
|
| 75 |
QUERY_RATINGS = ("up", "down")
|
| 76 |
MODEL_AUTO = "auto"
|
| 77 |
+
QUERY_NO_SOURCE_MATCHED = (
|
| 78 |
+
"None of the databases you can read cover that question. Pick one from the list, "
|
| 79 |
+
"or ask about a database you can open."
|
| 80 |
+
)
|
| 81 |
+
QUERY_DRAFT_ADMIN = "This database has no approved fields yet. Pick the ones this answer should use and they will be approved for everyone."
|
| 82 |
+
QUERY_DRAFT_MEMBER = "This database has no approved fields yet. Pick the ones you need and your administrators will be asked to approve them."
|
| 83 |
+
QUERY_DRAFT_SENT = "Sent to your administrators. Ask again once one of them approves."
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _draft_approved_line(name, date):
|
| 87 |
+
return f"Approved by {_no_dashes(name)} on {str(date)[:10]}."
|
| 88 |
+
|
| 89 |
+
|
| 90 |
NO_CITED_SOURCES_REASON = (
|
| 91 |
"No authorized application data or retrieved knowledge was cited for this answer."
|
| 92 |
)
|
|
|
|
| 179 |
def _public_message(row, allowed_citations):
|
| 180 |
"""Project a stored answer through its current citation wall before it reaches the browser."""
|
| 181 |
out = _safe(row) if isinstance(row, dict) else {}
|
| 182 |
+
if isinstance(out.get("draft"), dict):
|
| 183 |
+
# Historical cards cannot replay samples admitted under an older sensitivity or row
|
| 184 |
+
# grant. GET /semantics/{database}/draft computes their fresh permissioned profiles.
|
| 185 |
+
for field in out["draft"].get("fields") or ():
|
| 186 |
+
field["sample"] = semantic_rt.redacted_profile()
|
| 187 |
raw_citations = out.get("citationIds")
|
| 188 |
citation_ids = [str(citation_id) for citation_id in (raw_citations if isinstance(raw_citations, list) else [])
|
| 189 |
if str(citation_id) in allowed_citations]
|
|
|
|
| 1132 |
messages = sorted(state["messages"].values(), key=lambda row: row.get("createdAt", ""))
|
| 1133 |
views = sorted((_public_view(row) for row in allowed_views), key=lambda row: row["createdAt"], reverse=True)
|
| 1134 |
messages = [row for row in messages if not row.get("viewId") or row.get("viewId") in allowed_ids]
|
| 1135 |
+
messages = [row for row in messages if not isinstance(row.get("draft"), dict)
|
| 1136 |
+
or row["draft"].get("database") in permitted]
|
| 1137 |
messages = [_public_message(row, allowed_citations) for row in messages]
|
| 1138 |
return {"threads": _safe(threads), "messages": messages, "views": views,
|
| 1139 |
"citations": _safe([row for key, row in state["citations"].items()
|
|
|
|
| 1364 |
raise err(400, "bad_request", "no question was asked")
|
| 1365 |
if len(question) > MAX_QUESTION:
|
| 1366 |
raise err(400, "question_too_long", f"a question must be under {MAX_QUESTION} characters")
|
|
|
|
|
|
|
| 1367 |
if selected_model not in model_choices():
|
| 1368 |
raise err(400, "unknown_model", "that model is not available in Query")
|
| 1369 |
|
| 1370 |
+
routed = _route(session, question) if not target else {
|
| 1371 |
+
"database": target, "auto": False, "alternatives": []}
|
| 1372 |
+
target = routed["database"]
|
| 1373 |
+
route_refusal = (QUERY_NO_SOURCE_MATCHED if not target else routed.get("refusal"))
|
| 1374 |
+
|
| 1375 |
# This call happens before model selection and is the only permitted source-data read.
|
| 1376 |
+
snapshot = (assistant_read_scope(session, target, fields=None, filters=body.get("filters"))
|
| 1377 |
+
if not route_refusal else None)
|
| 1378 |
+
sources = _sources(body, target, session) if not route_refusal else ([target] if target else [])
|
| 1379 |
+
if snapshot is not None and not routed["auto"]:
|
| 1380 |
+
routed.update({"label": _no_dashes(snapshot["label"]),
|
| 1381 |
+
"reason": f"You selected {_no_dashes(snapshot['label'])}."})
|
| 1382 |
state = _state(session)
|
| 1383 |
thread_id = str(body.get("threadId") or "").strip()
|
| 1384 |
if thread_id and thread_id not in state["threads"]:
|
|
|
|
| 1392 |
# turn's own messages exist, so the replay is strictly the prior conversation.
|
| 1393 |
# ⭐ C7/T31: `session` rides along so the meter can attribute this call. See `_call_model`'s
|
| 1394 |
# docstring for why it is an argument and not ambient state (PRD amendment A1).
|
| 1395 |
+
draft_card = None
|
| 1396 |
+
if not route_refusal:
|
| 1397 |
+
_semantic_authorize(session, target)
|
| 1398 |
+
records = _semantic_records(session, target)
|
| 1399 |
+
if not semantic_rt.approved_fields(records, target):
|
| 1400 |
+
draft_card = _draft_card(session, target, question, records)
|
| 1401 |
+
else:
|
| 1402 |
+
snapshot = _semantic_answer_snapshot(snapshot, records)
|
| 1403 |
+
if route_refusal:
|
| 1404 |
+
said, provider = route_refusal, None
|
| 1405 |
+
view, refusal, refusal_code = (None, route_refusal,
|
| 1406 |
+
"query_source_unavailable" if target else "query_no_source_matched")
|
| 1407 |
+
elif draft_card:
|
| 1408 |
+
said, provider = draft_card["alignmentQuestion"]["question"], None
|
| 1409 |
+
view, refusal, refusal_code = None, said, "query_semantics_drafting"
|
| 1410 |
+
else:
|
| 1411 |
+
spec, said, provider, reason = _call_model(question, snapshot, model=selected_model, chat=chat,
|
| 1412 |
+
history=_history(state, thread_id), session=session)
|
| 1413 |
+
view, refusal, refusal_code = _validate(spec, snapshot["fields"]) if spec is not None else (None, said, reason)
|
| 1414 |
|
| 1415 |
artifact = None
|
| 1416 |
citation = None
|
|
|
|
| 1447 |
assistant_message = {
|
| 1448 |
"id": assistant_message_id, "threadId": thread_id, "role": "assistant", "createdAt": now,
|
| 1449 |
"content": (said or _said_fallback(artifact)) if artifact else (refusal or "the assistant could not answer"),
|
| 1450 |
+
# D-745: the Assistant restores routing from targetDatabase; keep it equal to routed["database"].
|
| 1451 |
"targetDatabase": target, "requestedModel": selected_model, "model": provider,
|
| 1452 |
+
"routedTo": routed,
|
| 1453 |
**_model_identity(provider),
|
| 1454 |
"reason": refusal_code, "viewId": artifact["id"] if artifact else None,
|
| 1455 |
"provenance": _answer_provenance(artifact),
|
| 1456 |
"citationIds": artifact["citationIds"] if artifact else [], "numeric": artifact["numeric"] if artifact else None,
|
| 1457 |
}
|
| 1458 |
+
if draft_card:
|
| 1459 |
+
assistant_message.update({key: draft_card[key] for key in
|
| 1460 |
+
("status", "draft", "alignmentQuestion", "resumeToken")})
|
| 1461 |
user_message = {"id": user_message_id, "threadId": thread_id, "role": "user", "createdAt": now,
|
| 1462 |
"content": question, "sources": sources, "targetDatabase": target,
|
| 1463 |
"requestedModel": selected_model}
|
|
|
|
| 1499 |
# Governed Query contract (Wave 45 C2). The existing chat workspace remains the presentation
|
| 1500 |
# surface; this provider-neutral door is the shared execution seam for Assistant, Agents, voice,
|
| 1501 |
# Slack and future adapters.
|
| 1502 |
+
def _semantic_baseline(database, session=None):
|
| 1503 |
from harness import semantic as sem
|
| 1504 |
topic = next((row for row in sem.topics().values()
|
| 1505 |
if isinstance(row, dict) and row.get("grid") == database), None)
|
| 1506 |
+
if topic:
|
| 1507 |
+
return semantic_rt.topic_records(database, topic)
|
| 1508 |
+
if session is not None:
|
| 1509 |
+
try:
|
| 1510 |
+
import odoo_relational
|
| 1511 |
+
base_key, connection_id = odoo_relational.split_table_key(database)
|
| 1512 |
+
if base_key is not None:
|
| 1513 |
+
namespace = (odoo_relational.connection_namespace(session.runtime, connection_id)[1]
|
| 1514 |
+
if connection_id else None)
|
| 1515 |
+
choice = next((row for row in odoo_relational.grid_choices(
|
| 1516 |
+
session.runtime, connection_id) if row.get("key") == database), None)
|
| 1517 |
+
model = (choice or {}).get("model")
|
| 1518 |
+
catalog = odoo_relational.readable_model_catalog(session.runtime, namespace)
|
| 1519 |
+
row = next((row for row in catalog.get("models") or ()
|
| 1520 |
+
if model and row.get("model") == model), None)
|
| 1521 |
+
if row is not None:
|
| 1522 |
+
return semantic_rt.generated_records(database, row, model=model)
|
| 1523 |
+
except Exception:
|
| 1524 |
+
# An unavailable cached catalogue is an unknown schema, not a server failure.
|
| 1525 |
+
pass
|
| 1526 |
+
# A custom database's own declared schema supplies draft rows. This is metadata only;
|
| 1527 |
+
# routing must not read records from every candidate just to recognize its label.
|
| 1528 |
+
projection = getattr(session.runtime, "get_projection", None)
|
| 1529 |
+
definitions = (projection("user_tables", drop=("rows",)) or {}) if projection else {}
|
| 1530 |
+
definition = definitions.get(database)
|
| 1531 |
+
if (database.startswith("ut_") and not database.startswith("ut_odoo_")
|
| 1532 |
+
and isinstance(definition, dict)):
|
| 1533 |
+
baseline = semantic_rt.topic_records(database, {"fields": definition.get("fields") or []})
|
| 1534 |
+
for field in definition.get("fields") or ():
|
| 1535 |
+
row = baseline.get(semantic_rt.field_key(database, field.get("key")))
|
| 1536 |
+
if row is None:
|
| 1537 |
+
continue
|
| 1538 |
+
row["relation"] = str((field.get("link") or {}).get("table") or "")
|
| 1539 |
+
if row["relation"] or row["dataType"] == "link":
|
| 1540 |
+
row["role"] = "link"
|
| 1541 |
+
elif row["dataType"] in ("date", "datetime"):
|
| 1542 |
+
row["role"] = "date"
|
| 1543 |
+
# Creating a source column does not itself grant tenant-wide approval.
|
| 1544 |
+
row.update({"state": "draft", "approvedBy": None, "approvedAt": None})
|
| 1545 |
+
return baseline
|
| 1546 |
+
raise err(404, "semantic_source_unknown",
|
| 1547 |
+
"that database does not have a semantic contract")
|
| 1548 |
|
| 1549 |
|
| 1550 |
def _semantic_records(session, database):
|
| 1551 |
+
return semantic_rt.load(session.runtime, _semantic_baseline(database, session))
|
| 1552 |
+
|
| 1553 |
+
|
| 1554 |
+
def _route_candidates(session):
|
| 1555 |
+
"""The status wall is the sole source of candidate keys; read semantic metadata only."""
|
| 1556 |
+
from harness import semantic as sem
|
| 1557 |
+
statuses = assistant_source_status(session, None)
|
| 1558 |
+
candidates = []
|
| 1559 |
+
definitions = None
|
| 1560 |
+
for database, status in statuses.items():
|
| 1561 |
+
if not status.get("permitted"):
|
| 1562 |
+
continue
|
| 1563 |
+
try:
|
| 1564 |
+
topic = next((row for row in sem.topics().values()
|
| 1565 |
+
if row.get("grid") == database), {})
|
| 1566 |
+
records = _semantic_records(session, database)
|
| 1567 |
+
approved = semantic_rt.approved_fields(records, database)
|
| 1568 |
+
if definitions is None:
|
| 1569 |
+
projection = getattr(session.runtime, "get_projection", None)
|
| 1570 |
+
definitions = (projection("user_tables", drop=("rows",)) or {}) if projection else {}
|
| 1571 |
+
definition = definitions.get(database) or {}
|
| 1572 |
+
label = _said(status.get("label") or definition.get("label") or
|
| 1573 |
+
topic.get("label") or database.removeprefix("ut_").replace("_", " ")) or database
|
| 1574 |
+
terms = []
|
| 1575 |
+
|
| 1576 |
+
def words(value):
|
| 1577 |
+
if isinstance(value, str) and value.strip():
|
| 1578 |
+
terms.append(value)
|
| 1579 |
+
terms.extend(re.findall(r"[^\W_]+", value))
|
| 1580 |
+
|
| 1581 |
+
words(label)
|
| 1582 |
+
words(topic.get("label"))
|
| 1583 |
+
words(topic.get("grain"))
|
| 1584 |
+
scope = topic.get("scope")
|
| 1585 |
+
for value in (scope.values() if isinstance(scope, dict) else [scope]):
|
| 1586 |
+
words(value)
|
| 1587 |
+
words((topic.get("semantic") or {}).get("scope"))
|
| 1588 |
+
# ⭐⭐ ONLY A CURATED TOPIC LENDS ITS FIELDS TO THE VOCABULARY, AND THIS WAS MEASURED
|
| 1589 |
+
# ON A RUNNING TENANT RATHER THAN REASONED ABOUT. W61-T28 gives every topic-less Odoo
|
| 1590 |
+
# database a pre-approved map generated from Odoo's own `fields_get`, so
|
| 1591 |
+
# `ut_odoo_order_lines` arrived carrying every raw field name `sale.order.line` has.
|
| 1592 |
+
# Asked "invoiced sales last week", the router picked ODOO ORDER LINES: raw names
|
| 1593 |
+
# matched both "invoiced" and "sale", and a generated map has dozens of fields where a
|
| 1594 |
+
# curated topic has a dozen, so the long tail outvoted all eight curated topics. A raw
|
| 1595 |
+
# column name is not a description of what a person is asking about. A generated
|
| 1596 |
+
# database therefore routes on WHAT IT IS (its label, below) and never on the field
|
| 1597 |
+
# names nobody wrote for this purpose; it stays fully readable once chosen, which is
|
| 1598 |
+
# what R27 pre-approved it for.
|
| 1599 |
+
if topic:
|
| 1600 |
+
for field in approved.values():
|
| 1601 |
+
words(field.get("label"))
|
| 1602 |
+
terms.extend(field.get("synonyms") or ())
|
| 1603 |
+
# Head words describe the field; explanatory tails often name OTHER databases.
|
| 1604 |
+
words(" ".join(str(field.get("purpose") or "").split()[:4]))
|
| 1605 |
+
candidates.append({"database": database, "label": label,
|
| 1606 |
+
"topic": topic.get("label") or "", "terms": terms,
|
| 1607 |
+
# What this database IS, as opposed to what it HOLDS. See
|
| 1608 |
+
# `route_question`: a subject word outweighs a field word, which is
|
| 1609 |
+
# what stops an orders question landing in the customer book.
|
| 1610 |
+
"subject": [label, topic.get("label") or ""],
|
| 1611 |
+
"curated": bool(topic),
|
| 1612 |
+
"answerable": bool(status.get("answerable")),
|
| 1613 |
+
"reason": _said(status.get("reason")) or ""})
|
| 1614 |
+
except Exception:
|
| 1615 |
+
# One unavailable semantic contract must not hide the other permitted databases.
|
| 1616 |
+
continue
|
| 1617 |
+
return candidates
|
| 1618 |
+
|
| 1619 |
+
|
| 1620 |
+
def _route(session, question):
|
| 1621 |
+
from harness import semantic as sem
|
| 1622 |
+
candidates = _route_candidates(session)
|
| 1623 |
+
result = sem.route_question(question, candidates)
|
| 1624 |
+
if not result["database"]:
|
| 1625 |
+
return {"database": "", "label": "", "reason": QUERY_NO_SOURCE_MATCHED,
|
| 1626 |
+
"auto": True, "alternatives": []}
|
| 1627 |
+
readable = sem.route_question(question, [row for row in candidates if row["answerable"]])
|
| 1628 |
+
# Availability breaks equally strong matches, never substitutes a worse semantic match.
|
| 1629 |
+
strength = lambda row: (row["score"], max(
|
| 1630 |
+
(len(" ".join(sem._route_tokens(term))) for term in row["matched"]), default=0))
|
| 1631 |
+
chosen = readable if readable["database"] and strength(readable) == strength(result) else result
|
| 1632 |
+
candidate = next(row for row in candidates if row["database"] == chosen["database"])
|
| 1633 |
+
matched = " and ".join(" ".join(re.findall(r"[^\W_]+", term))
|
| 1634 |
+
for term in chosen["matched"][:2])
|
| 1635 |
+
routed = {"database": candidate["database"], "label": candidate["label"],
|
| 1636 |
+
"reason": f"{candidate['label']} matched {matched}.", "auto": True,
|
| 1637 |
+
"alternatives": [{"database": row["database"], "label": _no_dashes(row["label"])}
|
| 1638 |
+
for row in result["ranked"] if row["database"] != candidate["database"]][:2]}
|
| 1639 |
+
if not candidate["answerable"]:
|
| 1640 |
+
cause = candidate["reason"] or "this database cannot answer Query yet"
|
| 1641 |
+
routed["refusal"] = f"{candidate['label']} cannot answer because {cause.rstrip('.')}."
|
| 1642 |
+
return routed
|
| 1643 |
|
| 1644 |
|
| 1645 |
def _semantic_authorize(session, database, *, require_answerable=False):
|
|
|
|
| 1713 |
session: Session = Depends(require_session)):
|
| 1714 |
_semantic_authorize(session, database)
|
| 1715 |
try:
|
| 1716 |
+
row = semantic_rt.update(session.runtime, _semantic_baseline(database, session), database, field,
|
| 1717 |
body, actor=session.uname, is_admin=session.admin)
|
| 1718 |
except semantic_rt.SemanticError as exc:
|
| 1719 |
code = "forbidden" if "only a tenant admin" in str(exc) else "bad_semantic_contract"
|
|
|
|
| 1727 |
session: Session = Depends(require_session)):
|
| 1728 |
"""Profile values through Query's permissioned snapshot and persist only a bounded summary."""
|
| 1729 |
_semantic_authorize(session, database, require_answerable=True)
|
| 1730 |
+
baseline = _semantic_baseline(database, session)
|
| 1731 |
records = semantic_rt.load(session.runtime, baseline)
|
| 1732 |
try:
|
| 1733 |
current = records[semantic_rt.field_key(database, field)]
|
|
|
|
| 1751 |
return {"field": row}
|
| 1752 |
|
| 1753 |
|
| 1754 |
+
def _semantic_answer_snapshot(snapshot, records):
|
| 1755 |
+
"""Custom approvals govern model input too; sensitive values never become answer rows."""
|
| 1756 |
+
database = snapshot.get("database") or ""
|
| 1757 |
+
custom = (database.startswith("ut_") and not database.startswith("ut_odoo_")
|
| 1758 |
+
and not any(row.get("topic") for row in records.values()))
|
| 1759 |
+
hidden = {row["field"] for row in records.values()
|
| 1760 |
+
if row.get("sensitivity") in ("sensitive", "restricted")}
|
| 1761 |
+
if not hidden and not custom:
|
| 1762 |
+
return snapshot
|
| 1763 |
+
out = copy.deepcopy(snapshot)
|
| 1764 |
+
if custom:
|
| 1765 |
+
approved = semantic_rt.approved_fields(records, database)
|
| 1766 |
+
out["fields"] = [field for field in out.get("fields") or () if field.get("key") in approved]
|
| 1767 |
+
out["records"] = [{key: value for key, value in row.items() if key == "pid" or key in approved}
|
| 1768 |
+
for row in out.get("records") or ()]
|
| 1769 |
+
for row in out.get("records") or ():
|
| 1770 |
+
for field in hidden.intersection(row):
|
| 1771 |
+
row[field] = None
|
| 1772 |
+
return out
|
| 1773 |
+
|
| 1774 |
+
|
| 1775 |
+
def _draft_card(session, database, question, records):
|
| 1776 |
+
snapshot = assistant_read_scope(session, database, fields=None, filters=None)
|
| 1777 |
+
fields = semantic_rt.draft_fields(records, snapshot)
|
| 1778 |
+
statuses = assistant_source_status(session, None)
|
| 1779 |
+
projection = getattr(session.runtime, "get_projection", None)
|
| 1780 |
+
definitions = (projection("user_tables", drop=("rows",)) or {}) if projection else {}
|
| 1781 |
+
from harness import semantic as sem
|
| 1782 |
+
topic_labels = {row.get("grid"): row.get("label") for row in sem.topics().values()}
|
| 1783 |
+
for field in fields:
|
| 1784 |
+
record = records[semantic_rt.field_key(database, field["field"])]
|
| 1785 |
+
relation = record.get("relation") or ""
|
| 1786 |
+
relation = {"customer": "customer_data", "product": "product_data"}.get(relation, relation)
|
| 1787 |
+
status = statuses.get(relation) or {}
|
| 1788 |
+
if status.get("permitted"):
|
| 1789 |
+
label = (status.get("label") or (definitions.get(relation) or {}).get("label")
|
| 1790 |
+
or topic_labels.get(relation))
|
| 1791 |
+
if label:
|
| 1792 |
+
field["why"] = field["why"].rstrip(".") + f", linked to {_no_dashes(label)}."
|
| 1793 |
+
field["label"] = _no_dashes(field["label"])
|
| 1794 |
+
# Profiles are projections for display, never an authority for field keys or approval.
|
| 1795 |
+
for key in ("values", "sample"):
|
| 1796 |
+
field["sample"][key] = [_no_dashes(value) for value in field["sample"].get(key) or ()]
|
| 1797 |
+
request = semantic_rt.draft_request(session.runtime, database) or {}
|
| 1798 |
+
approved = semantic_rt.approved_fields(records, database)
|
| 1799 |
+
approval = request if request.get("state") == "approved" else next(iter(approved.values()), {})
|
| 1800 |
+
state = "approved" if approved else "pending"
|
| 1801 |
+
draft = {"database": database, "label": _no_dashes(snapshot.get("label") or database),
|
| 1802 |
+
"canApprove": bool(session.admin), "fields": fields,
|
| 1803 |
+
"sentToAdmins": not session.admin, "state": state,
|
| 1804 |
+
"requestedBy": request.get("requestedBy") or session.uname,
|
| 1805 |
+
"approvedBy": approval.get("approvedBy"), "approvedAt": approval.get("approvedAt"),
|
| 1806 |
+
"sentMessage": QUERY_DRAFT_SENT}
|
| 1807 |
+
if draft["approvedBy"] and draft["approvedAt"]:
|
| 1808 |
+
draft["approvalLine"] = _draft_approved_line(draft["approvedBy"], draft["approvedAt"])
|
| 1809 |
+
return {"status": "clarification", "alignmentQuestion": {
|
| 1810 |
+
"id": "draft_fields", "question": QUERY_DRAFT_ADMIN if session.admin else QUERY_DRAFT_MEMBER,
|
| 1811 |
+
"choices": [{"field": field["field"], "label": field["label"]} for field in fields],
|
| 1812 |
+
"maxSelections": 8}, "draft": draft, "resumeToken": ""}
|
| 1813 |
+
|
| 1814 |
+
|
| 1815 |
+
@router.post("/query/semantics/{database}/approve")
|
| 1816 |
+
def approve_semantic_draft(database: str, body: dict = Body(default=None),
|
| 1817 |
+
session: Session = Depends(require_session)):
|
| 1818 |
+
_semantic_authorize(session, database)
|
| 1819 |
+
baseline = _semantic_baseline(database, session)
|
| 1820 |
+
body = body if isinstance(body, dict) else {}
|
| 1821 |
+
fields = body.get("fields")
|
| 1822 |
+
if (not isinstance(fields, list) or not fields or len(fields) > 8
|
| 1823 |
+
or any(not isinstance(field, str) for field in fields)):
|
| 1824 |
+
raise err(400, "bad_semantic_contract", "pick between one and eight database fields")
|
| 1825 |
+
fields = list(dict.fromkeys(fields))
|
| 1826 |
+
question = " ".join(str(body.get("question") or "").split())
|
| 1827 |
+
if not question or len(question) > MAX_QUESTION:
|
| 1828 |
+
raise err(400, "bad_request", "a question of up to 500 characters is required")
|
| 1829 |
+
try:
|
| 1830 |
+
# Validate the whole selection before the first write, including member requests.
|
| 1831 |
+
if any(semantic_rt.field_key(database, field) not in baseline for field in fields):
|
| 1832 |
+
raise semantic_rt.SemanticError("that field is not in the database contract")
|
| 1833 |
+
now = _dt.datetime.now(_dt.timezone.utc)
|
| 1834 |
+
if not session.admin:
|
| 1835 |
+
request = semantic_rt.record_draft_request(
|
| 1836 |
+
session.runtime, database, question, fields, actor=session.uname, now=now)
|
| 1837 |
+
return {"approved": [], "state": "pending", "sentToAdmins": True,
|
| 1838 |
+
"requestedBy": request["requestedBy"], "sentMessage": QUERY_DRAFT_SENT}
|
| 1839 |
+
for field in fields:
|
| 1840 |
+
semantic_rt.update(session.runtime, baseline, database, field, {"state": "approved"},
|
| 1841 |
+
actor=session.uname, is_admin=session.admin, now=now)
|
| 1842 |
+
if not semantic_rt.draft_request(session.runtime, database):
|
| 1843 |
+
semantic_rt.record_draft_request(
|
| 1844 |
+
session.runtime, database, question, fields, actor=session.uname, now=now)
|
| 1845 |
+
request = semantic_rt.approve_draft_request(
|
| 1846 |
+
session.runtime, database, fields, actor=session.uname, is_admin=session.admin, now=now)
|
| 1847 |
+
except semantic_rt.SemanticError as exc:
|
| 1848 |
+
raise err(400, "bad_semantic_contract", str(exc))
|
| 1849 |
+
return {"approved": fields, "approvedBy": request["approvedBy"],
|
| 1850 |
+
"approvedAt": request["approvedAt"], "state": "approved",
|
| 1851 |
+
"approvalLine": _draft_approved_line(request["approvedBy"], request["approvedAt"])}
|
| 1852 |
+
|
| 1853 |
+
|
| 1854 |
+
@router.get("/query/semantics/{database}/draft")
|
| 1855 |
+
def semantic_draft(database: str, session: Session = Depends(require_session)):
|
| 1856 |
+
_semantic_authorize(session, database)
|
| 1857 |
+
records = _semantic_records(session, database)
|
| 1858 |
+
request = semantic_rt.draft_request(session.runtime, database)
|
| 1859 |
+
return {**_draft_card(session, database, (request or {}).get("question") or "", records),
|
| 1860 |
+
"request": request}
|
| 1861 |
+
|
| 1862 |
+
|
| 1863 |
def _resume_b64(raw):
|
| 1864 |
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
| 1865 |
|
|
|
|
| 1999 |
_semantic_authorize_schema_only(session, database)
|
| 2000 |
metrics, reason = _metric_catalogue(session, database)
|
| 2001 |
# ⛔⛔ THE GOVERNED SOURCE WALL STAYS, BUT IT CANNOT BE THE FIRST THING ASKED.
|
| 2002 |
+
# Cached Odoo fields can now supply a baseline without an entity topic, including order and
|
| 2003 |
+
# GL lines. That schema does not create a validated entity-measure binding: keep returning
|
| 2004 |
+
# `reason: no_topic` for this picker until a topic owns the measure offer. Other callers of
|
| 2005 |
+
# `_semantic_records` still resolve the generated schema through their existing record wall.
|
|
|
|
|
|
|
| 2006 |
if reason != "no_topic":
|
| 2007 |
_semantic_records(session, database)
|
| 2008 |
return {"metrics": metrics, "reason": reason}
|
|
|
|
| 2122 |
body = body if isinstance(body, dict) else {}
|
| 2123 |
database = str(body.get("database") or "").strip()
|
| 2124 |
question = " ".join(str(body.get("prompt") or body.get("question") or "").split())
|
| 2125 |
+
if not question:
|
| 2126 |
+
raise err(400, "bad_request", "a prompt is required")
|
| 2127 |
if len(question) > MAX_QUESTION:
|
| 2128 |
raise err(400, "question_too_long", f"a prompt must be under {MAX_QUESTION} characters")
|
| 2129 |
+
routed = _route(session, question) if not database else {
|
| 2130 |
+
"database": database, "label": "", "reason": "", "auto": False, "alternatives": []}
|
| 2131 |
+
database = routed["database"]
|
| 2132 |
+
if database:
|
| 2133 |
+
_semantic_authorize(session, database, require_answerable=not routed["auto"])
|
| 2134 |
+
route_refusal = QUERY_NO_SOURCE_MATCHED if not database else routed.get("refusal")
|
| 2135 |
+
if route_refusal:
|
| 2136 |
+
return {"status": "refused", "wording": route_refusal, "speakable": route_refusal,
|
| 2137 |
+
"artifacts": None, "routedTo": routed, "model": _model_identity(None)}
|
| 2138 |
records = _semantic_records(session, database)
|
| 2139 |
+
if not routed["auto"]:
|
| 2140 |
+
from harness import semantic as sem
|
| 2141 |
+
label = next((row.get("label") for row in sem.topics().values()
|
| 2142 |
+
if row.get("grid") == database), database)
|
| 2143 |
+
routed.update({"label": _no_dashes(label), "reason": f"You selected {_no_dashes(label)}."})
|
| 2144 |
approved = semantic_rt.approved_fields(records, database)
|
| 2145 |
if not approved:
|
| 2146 |
+
return {**_draft_card(session, database, question, records), "routedTo": routed}
|
|
|
|
| 2147 |
raw_requested = body.get("fieldKeys")
|
| 2148 |
if raw_requested is not None and not isinstance(raw_requested, list):
|
| 2149 |
raise err(400, "bad_request", "fieldKeys must be a list")
|
| 2150 |
requested = list(dict.fromkeys(str(x) for x in (raw_requested or ()) if str(x)))
|
| 2151 |
if not requested:
|
| 2152 |
+
return {**_clarification(session, database, question, approved), "routedTo": routed}
|
| 2153 |
_verify_resume_token(body.get("resumeToken"), session, database, question,
|
| 2154 |
approved, requested)
|
| 2155 |
try:
|
|
|
|
| 2157 |
except semantic_rt.SemanticError as exc:
|
| 2158 |
raise err(403, "semantic_field_unapproved", str(exc))
|
| 2159 |
snapshot = assistant_read_scope(session, database, fields=requested, filters=None)
|
| 2160 |
+
snapshot = _semantic_answer_snapshot(snapshot, records)
|
| 2161 |
snapshot, formulas = _tool_formula_snapshot(body, snapshot, requested)
|
| 2162 |
spec, planned_wording, provider = _tool_plan(question, body, snapshot, session)
|
| 2163 |
if spec is None:
|
| 2164 |
return {"status": "refused", "wording": planned_wording,
|
| 2165 |
"speakable": planned_wording, "artifacts": None,
|
| 2166 |
+
"model": _model_identity(provider), "routedTo": routed}
|
| 2167 |
view, refusal, refusal_code = _validate(spec, snapshot.get("fields") or selected)
|
| 2168 |
if view is None:
|
| 2169 |
raise err(400, refusal_code or "bad_query_artifact", refusal or "that Query is invalid")
|
|
|
|
| 2188 |
"rows": [{key: row.get(key) for key in view["visible"]} for row in rows],
|
| 2189 |
"groups": groups, "numeric": numeric,
|
| 2190 |
"artifacts": artifacts, "semanticRouter": semantic_rt.router_projection(records, database),
|
| 2191 |
+
"model": _model_identity(provider), "routedTo": routed}
|
| 2192 |
|
| 2193 |
|
| 2194 |
@router.post("/query/formulas/parse")
|
api/routes_sheet.py
CHANGED
|
@@ -221,6 +221,11 @@ def _registry_assembly(session: Session, table_key: str):
|
|
| 221 |
if table_key in memo:
|
| 222 |
return memo[table_key]
|
| 223 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
if table_key == "product_data":
|
| 225 |
from routes_products import product_assembly
|
| 226 |
memo[table_key] = product_assembly(session, scope="product",
|
|
@@ -248,6 +253,159 @@ def _registry_assembly(session: Session, table_key: str):
|
|
| 248 |
"shown rather than shown from an older set of columns. Try again in a moment.")
|
| 249 |
|
| 250 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
|
| 252 |
"""`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period.
|
| 253 |
|
|
@@ -616,7 +774,7 @@ _CAP_CODES = ("sheet_too_many_cells", "workbook_too_large")
|
|
| 616 |
|
| 617 |
#: The refusals that mean "that thing is not here", answered 404 so a client can tell a missing
|
| 618 |
#: workbook from a malformed one without reading the sentence.
|
| 619 |
-
_MISSING_CODES = ("workbook_missing", "tab_missing")
|
| 620 |
|
| 621 |
#: ⛔⛔ C5's TWO SHARING REFUSALS ANSWER 403, AND THE TUPLE EXISTS BECAUSE THE FALL-THROUGH BELOW
|
| 622 |
#: IS 400 (wave 51, W51-T25). Without this branch `workbook_read_only` would arrive as a NAMED 400
|
|
@@ -642,10 +800,8 @@ def _refuse(exc):
|
|
| 642 |
nothing else, so the 409 and the 413 are raised here directly rather than by bending `err`,
|
| 643 |
which would change the error shape of every route in the app.
|
| 644 |
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
working. ⭐ D-639's `index_revision_stale` lands here too and DOES carry one, so the rail's
|
| 648 |
-
new guard needed no branch of its own: matching the family's spelling was the whole cost.
|
| 649 |
"""
|
| 650 |
if exc.code in _CAP_CODES:
|
| 651 |
return HTTPException(status_code=413,
|
|
@@ -739,13 +895,23 @@ def _role_on_workbook(session: Session, book):
|
|
| 739 |
round because a workbook that has never been shared has no registry record at all. Asking
|
| 740 |
`shares.role_for` alone would answer `None` for the person who created it.
|
| 741 |
"""
|
| 742 |
-
from core import
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
| 747 |
-
|
| 748 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 749 |
|
| 750 |
|
| 751 |
def _require_editor(session: Session, book):
|
|
@@ -778,6 +944,8 @@ def _wire_shares(rows):
|
|
| 778 |
out = []
|
| 779 |
for row in rows or ():
|
| 780 |
row = dict(row)
|
|
|
|
|
|
|
| 781 |
row["shares"] = [{"who": s.get("who"), "role": wire_role(s.get("role"))}
|
| 782 |
for s in (row.get("shares") or [])
|
| 783 |
if wire_role(s.get("role"))]
|
|
@@ -809,7 +977,24 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str:
|
|
| 809 |
return sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {})
|
| 810 |
|
| 811 |
|
| 812 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 813 |
"""A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5).
|
| 814 |
|
| 815 |
⛔⛔ IN MEMORY, AND IT MUST NEVER PERSIST. `refresh_sheet` runs the same loop and ends with
|
|
@@ -850,6 +1035,8 @@ def _reresolved_for_reader(session: Session, book):
|
|
| 850 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 851 |
if not isinstance(cell, dict) or cell.get("kind") != "bound":
|
| 852 |
continue
|
|
|
|
|
|
|
| 853 |
touched += 1
|
| 854 |
binding = cell.get("binding") or {}
|
| 855 |
db = str(binding.get("database") or "")
|
|
@@ -861,18 +1048,14 @@ def _reresolved_for_reader(session: Session, book):
|
|
| 861 |
seen_db[db] = exc
|
| 862 |
failure = seen_db.get(db)
|
| 863 |
if failure is not None:
|
| 864 |
-
cell
|
| 865 |
-
cell["value"] = None
|
| 866 |
-
cell["display"] = ""
|
| 867 |
-
cell["refusal"] = {"code": "database_unreadable",
|
| 868 |
-
"message": f"You cannot read {db}."}
|
| 869 |
cell.pop("limits", None)
|
| 870 |
continue
|
| 871 |
|
| 872 |
cell_key = tab_ref + "!" + addr
|
| 873 |
scope_key = _binding_scope_key(session, db, binding)
|
| 874 |
hit = None
|
| 875 |
-
if wid and scope_key:
|
| 876 |
try:
|
| 877 |
hit = sheets.cached_scope_value(store, wid, cell_key, scope_key)
|
| 878 |
except Exception: # noqa: BLE001
|
|
@@ -890,17 +1073,15 @@ def _reresolved_for_reader(session: Session, book):
|
|
| 890 |
continue
|
| 891 |
|
| 892 |
try:
|
| 893 |
-
out =
|
| 894 |
except HTTPException as exc:
|
| 895 |
detail = getattr(exc, "detail", None) or {}
|
| 896 |
info = detail.get("error") if isinstance(detail, dict) else None
|
| 897 |
-
cell
|
| 898 |
-
|
| 899 |
-
|
| 900 |
-
|
| 901 |
-
|
| 902 |
-
"message": (info or {}).get("message", "This cell could not be refreshed."),
|
| 903 |
-
}
|
| 904 |
continue
|
| 905 |
cell["state"] = "fresh"
|
| 906 |
cell["value"] = out["value"]
|
|
@@ -932,15 +1113,24 @@ def _reresolved_for_reader(session: Session, book):
|
|
| 932 |
return fresh
|
| 933 |
|
| 934 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 935 |
def _read_for(session: Session, book):
|
| 936 |
"""The workbook this session is entitled to see: stored for the owner, re-resolved otherwise.
|
| 937 |
|
| 938 |
-
|
| 939 |
-
|
| 940 |
-
fail-closed answer to "what should they be shown" is: whatever THEIR OWN scope computes,
|
| 941 |
-
never the owner's cached numbers.
|
| 942 |
"""
|
|
|
|
| 943 |
if _role_on_workbook(session, book) == "owner":
|
|
|
|
|
|
|
|
|
|
| 944 |
return book
|
| 945 |
return _reresolved_for_reader(session, book)
|
| 946 |
|
|
@@ -951,8 +1141,7 @@ def list_sheets(session: Session = Depends(require_session)):
|
|
| 951 |
|
| 952 |
⚠ C5's `shares` rides here as well as on `/sheets/index`, because the client reads both.
|
| 953 |
"""
|
| 954 |
-
|
| 955 |
-
rows = sheets.project_shares(_store(session), sheets.list_workbooks(_store(session)))
|
| 956 |
return {"workbooks": _wire_shares(rows)}
|
| 957 |
|
| 958 |
|
|
@@ -964,10 +1153,33 @@ def create_sheet(body: dict = Body(default=None), session: Session = Depends(req
|
|
| 964 |
try:
|
| 965 |
book = sheets.create_workbook(_store(session),
|
| 966 |
name=raw.get("name") or "",
|
| 967 |
-
created_by=session.uname
|
|
|
|
| 968 |
except sheets.SheetRefusal as exc:
|
| 969 |
raise _refuse(exc)
|
| 970 |
-
return {"workbook": book}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 971 |
|
| 972 |
|
| 973 |
# ── the rail's own document (wave 50, W50-T17) ─────────────────────���─────────────────────────
|
|
@@ -982,17 +1194,10 @@ def create_sheet(body: dict = Body(default=None), session: Session = Depends(req
|
|
| 982 |
# ⚠ `Depends(require_session)` ONLY, like every other door in this file, and that is correct: the
|
| 983 |
# Sheet is granted to everyone, and the rail's folders name no database.
|
| 984 |
#
|
| 985 |
-
#
|
| 986 |
-
#
|
| 987 |
-
#
|
| 988 |
-
#
|
| 989 |
-
# ⇒ `sharedWithMe` is therefore a PARTITION of a list everyone can already see, and C5's
|
| 990 |
-
# viewer/editor wall guards the WRITE doors rather than closing a read that was never shut.
|
| 991 |
-
# Refusing a stranger's read is not in W51-T25's `done-when` and widening it here would change the
|
| 992 |
-
# behaviour of a door six other tickets are built on, so it is an amendment finding for the root.
|
| 993 |
-
# ⚠ WHAT THIS TICKET DOES GUARANTEE MEANWHILE is the half that leaks data rather than existence:
|
| 994 |
-
# a non-owner never receives the owner's stored numbers, because every cell-bearing read
|
| 995 |
-
# re-resolves under the caller's own scope first.
|
| 996 |
|
| 997 |
|
| 998 |
@router.get("/sheets/index")
|
|
@@ -1007,14 +1212,13 @@ def read_sheet_index(session: Session = Depends(require_session)):
|
|
| 1007 |
can reach: a client cannot send a revision it was never given, and a server that never checks
|
| 1008 |
one is a client stamping a number nobody reads.
|
| 1009 |
"""
|
| 1010 |
-
|
| 1011 |
-
return {"index": _wire_index(sheets.read_index(_store(session), viewer=session.uname))}
|
| 1012 |
|
| 1013 |
|
| 1014 |
@router.put("/sheets/index")
|
| 1015 |
def write_sheet_index(body: dict = Body(default=None),
|
| 1016 |
session: Session = Depends(require_session)):
|
| 1017 |
-
"""
|
| 1018 |
|
| 1019 |
⭐ `repairs` RIDES BACK OUT. `core.sheets.save_folders` records every change it made to the
|
| 1020 |
payload, and the rail says so in a sentence rather than letting a workbook reappear at the
|
|
@@ -1029,13 +1233,102 @@ def write_sheet_index(body: dict = Body(default=None),
|
|
| 1029 |
"""
|
| 1030 |
from core import sheets
|
| 1031 |
raw = dict(body or {})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1032 |
try:
|
| 1033 |
-
|
| 1034 |
-
|
| 1035 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1036 |
except sheets.SheetRefusal as exc:
|
| 1037 |
raise _refuse(exc)
|
| 1038 |
-
return {"index": _wire_index(out)}
|
| 1039 |
|
| 1040 |
|
| 1041 |
# ── C4: the .xlsx IMPORT, which is a JOB and not a request (wave 51, W51-T24) ────────────────
|
|
@@ -1131,9 +1424,11 @@ def _run_import(job_id, path, filename, runtime, uname):
|
|
| 1131 |
_job_set(job_id, report=report)
|
| 1132 |
|
| 1133 |
stamp = _now_iso()
|
| 1134 |
-
made = sheets.create_workbook(runtime, name=book.get("name") or "", created_by=uname
|
|
|
|
| 1135 |
saved = sheets.put_workbook(runtime, made["id"],
|
| 1136 |
-
dict(book, refreshed_at=stamp, rev=made["rev"])
|
|
|
|
| 1137 |
_job_set(job_id, state="done", progress=1.0, workbook_id=saved["id"],
|
| 1138 |
sentence=_import_sentence(report))
|
| 1139 |
except sheet_import.ImportRefusal as exc:
|
|
@@ -1258,6 +1553,7 @@ def read_sheet(workbook_id: str, tabs: str = "",
|
|
| 1258 |
from core import sheets
|
| 1259 |
try:
|
| 1260 |
book = sheets.get_workbook(_store(session), workbook_id)
|
|
|
|
| 1261 |
except sheets.SheetRefusal as exc:
|
| 1262 |
raise _refuse(exc)
|
| 1263 |
if str(tabs or "").strip().lower() == "meta":
|
|
@@ -1294,6 +1590,7 @@ def read_sheet_tab(workbook_id: str, tab: str,
|
|
| 1294 |
from core import sheets
|
| 1295 |
try:
|
| 1296 |
book = sheets.get_workbook(_store(session), workbook_id)
|
|
|
|
| 1297 |
served = _read_for(session, book)
|
| 1298 |
found = sheets.find_tab(served, tab)
|
| 1299 |
except sheets.SheetRefusal as exc:
|
|
@@ -1319,9 +1616,8 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None
|
|
| 1319 |
same patch has landed. A CAP breach is the opposite: it is a whole request 413, because the
|
| 1320 |
document as a whole cannot be stored and there is no partial version of that.
|
| 1321 |
|
| 1322 |
-
|
| 1323 |
-
|
| 1324 |
-
`sheet_revision_stale` carrying `rev_current`, which is the revision to refetch.
|
| 1325 |
|
| 1326 |
⛔⛔ C5's READ-ONLY CHECK RUNS BEFORE THAT COMPARISON AND THE ORDER IS THE CONTRACT. A viewer
|
| 1327 |
is refused 403 `workbook_read_only`; a viewer holding a stale rev is ALSO refused
|
|
@@ -1347,7 +1643,8 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None
|
|
| 1347 |
expect = raw.get("rev")
|
| 1348 |
try:
|
| 1349 |
out = sheets.patch_cells(_store(session), workbook_id, tab, raw.get("cells"),
|
| 1350 |
-
expect_rev=expect
|
|
|
|
| 1351 |
except sheets.SheetRefusal as exc:
|
| 1352 |
raise _refuse(exc)
|
| 1353 |
accepted = out.pop("accepted", [])
|
|
@@ -1396,11 +1693,32 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
|
|
| 1396 |
(found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes}
|
| 1397 |
_recompute_formulas(book, fr, sheet_mod,
|
| 1398 |
cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1399 |
found = sheets.find_tab(book, tab_ref)
|
| 1400 |
out = {}
|
| 1401 |
for addr, cell in (found.get("cells") or {}).items():
|
| 1402 |
if not isinstance(cell, dict) or (cell.get("kind") != "formula"
|
| 1403 |
-
and "spill" not in cell and "spillParent" not in cell
|
|
|
|
| 1404 |
continue
|
| 1405 |
row = {"value": cell.get("value"),
|
| 1406 |
"display": "" if cell.get("value") is None else str(cell.get("display") or ""),
|
|
@@ -1435,10 +1753,12 @@ def write_sheet(workbook_id: str, body: dict = Body(default=None),
|
|
| 1435 |
# `rev` inside the body IS the expectation; `core.sheets.put_workbook` reads it. This door
|
| 1436 |
# does not translate it into a second argument, because two ways to say the same thing is
|
| 1437 |
# how one of them ends up ignored.
|
| 1438 |
-
book = sheets.put_workbook(_store(session), workbook_id, raw
|
|
|
|
|
|
|
| 1439 |
except sheets.SheetRefusal as exc:
|
| 1440 |
raise _refuse(exc)
|
| 1441 |
-
return {"workbook": book}
|
| 1442 |
|
| 1443 |
|
| 1444 |
@router.delete("/sheets/{workbook_id}")
|
|
@@ -1509,6 +1829,14 @@ def share_sheet(workbook_id: str, body: dict = Body(default=None),
|
|
| 1509 |
rec = shares.set_grants("workbook", book["id"], entries,
|
| 1510 |
owner=(None if held.get("owner") else _workbook_owner(book)),
|
| 1511 |
granter=session.uname, st=_store(session), expect=prior)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1512 |
except shares.GrantsChanged as exc:
|
| 1513 |
# ⚠ `grants_changed`, THE SAME CODE `routes_shares.py` ALREADY ANSWERS, and a plain 409
|
| 1514 |
# rather than the `_stale` shape: there is no revision to hand back here, and a
|
|
@@ -1601,6 +1929,8 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 1601 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 1602 |
if not isinstance(cell, dict) or cell.get("kind") != "bound":
|
| 1603 |
continue
|
|
|
|
|
|
|
| 1604 |
binding = cell.get("binding") or {}
|
| 1605 |
db = str(binding.get("database") or "")
|
| 1606 |
if db not in seen_db:
|
|
@@ -1611,22 +1941,19 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 1611 |
seen_db[db] = exc
|
| 1612 |
failure = seen_db.get(db)
|
| 1613 |
if failure is not None:
|
| 1614 |
-
cell
|
| 1615 |
-
cell
|
| 1616 |
-
cell["refusal"] = {"code": "database_unreadable",
|
| 1617 |
-
"message": f"You cannot read {db}."}
|
| 1618 |
continue
|
| 1619 |
try:
|
| 1620 |
-
out =
|
| 1621 |
except HTTPException as exc:
|
| 1622 |
detail = getattr(exc, "detail", None) or {}
|
| 1623 |
info = detail.get("error") if isinstance(detail, dict) else None
|
| 1624 |
-
cell
|
| 1625 |
-
|
| 1626 |
-
|
| 1627 |
-
|
| 1628 |
-
|
| 1629 |
-
}
|
| 1630 |
continue
|
| 1631 |
cell["state"] = "fresh"
|
| 1632 |
cell["value"] = out["value"]
|
|
@@ -1652,16 +1979,20 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 1652 |
_recompute_formulas(book, fr, sheet_mod)
|
| 1653 |
recalc_ms = round((perf_counter() - started) * 1000.0, 1)
|
| 1654 |
try:
|
|
|
|
| 1655 |
saved = sheets.put_workbook(_store(session), workbook_id,
|
| 1656 |
{"name": book.get("name"), "sheets": book.get("sheets"),
|
| 1657 |
-
"refreshed_at": stamp, "rev": book.get("rev")}
|
|
|
|
|
|
|
|
|
|
| 1658 |
except sheets.SheetRefusal as exc:
|
| 1659 |
raise _refuse(exc)
|
| 1660 |
# ⭐⭐ R3 / W52-T12: REFRESH IS THE ONE EVENT ALLOWED TO INVALIDATE A READER'S CACHED SCOPE
|
| 1661 |
# VALUE, so it is the one door that clears the whole per-scope cache. Every reader's next GET
|
| 1662 |
# recomputes once, under their own scope, and fills its own slot again from there.
|
| 1663 |
sheets.clear_scope_cache(_store(session), workbook_id)
|
| 1664 |
-
return {"workbook": saved, "recalc_ms": recalc_ms}
|
| 1665 |
|
| 1666 |
|
| 1667 |
def _now_iso():
|
|
@@ -1916,7 +2247,150 @@ def _metric_column_aggregate(session, database, aggregate, window):
|
|
| 1916 |
return column(raw)
|
| 1917 |
|
| 1918 |
|
| 1919 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1920 |
"""Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
|
| 1921 |
|
| 1922 |
The order below is the contract, not a preference:
|
|
@@ -1944,7 +2418,10 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 1944 |
raise err(400, exc.code, exc.message)
|
| 1945 |
|
| 1946 |
database = _require_database(session, binding["database"])
|
| 1947 |
-
|
|
|
|
|
|
|
|
|
|
| 1948 |
# W59-T14 / R10: the per-database wall is contract step 1. A metric key must not be looked
|
| 1949 |
# up or acted on before the caller passes it; the ordinary column guards below still apply.
|
| 1950 |
if binding["aggregate"].get("metric_key"):
|
|
@@ -1966,12 +2443,16 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 1966 |
scope.get("view_name", ""), memo=view_memo)
|
| 1967 |
else:
|
| 1968 |
scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
|
|
|
|
|
|
|
| 1969 |
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
| 1970 |
| sheet_mod.filter_columns_named(wall_nodes))
|
| 1971 |
if binding["aggregate"].get("field"):
|
| 1972 |
needed.add(str(binding["aggregate"]["field"]))
|
| 1973 |
|
| 1974 |
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
|
|
|
|
|
|
| 1975 |
|
| 1976 |
# ⛔ THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
|
| 1977 |
# real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
|
|
@@ -2069,7 +2550,15 @@ def preview_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 2069 |
field through the database's ordinary field writer; reusing a column needs no field write.
|
| 2070 |
"""
|
| 2071 |
raw = dict(body or {})
|
| 2072 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2073 |
return {"value": out["value"], "display": "" if out["value"] is None else str(out["value"]),
|
| 2074 |
"limits": out["limits"]}
|
| 2075 |
|
|
@@ -2196,6 +2685,11 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 2196 |
raise err(404, "cell_missing", f"Cell {key} is not filled in.")
|
| 2197 |
if cell.get("kind") != "bound":
|
| 2198 |
raise err(409, "cell_not_bound", f"Cell {key} is not bound to a database.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2199 |
|
| 2200 |
tab_ref = str((tab or {}).get("id") or (tab or {}).get("name") or "")
|
| 2201 |
binding = cell.get("binding") or {}
|
|
@@ -2203,20 +2697,17 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 2203 |
saved = dict(cell)
|
| 2204 |
limits, resolved = [], None
|
| 2205 |
try:
|
| 2206 |
-
resolved =
|
| 2207 |
except HTTPException as exc:
|
| 2208 |
#: The same per-cell refusal shape `refresh_sheet` writes, so a cell refused by this door
|
| 2209 |
#: and a cell refused by that one read identically. It is PERSISTED rather than raised:
|
| 2210 |
#: a cell left `stale` would shimmer forever, and the reason belongs on the cell.
|
| 2211 |
detail = getattr(exc, "detail", None) or {}
|
| 2212 |
info = detail.get("error") if isinstance(detail, dict) else None
|
| 2213 |
-
saved
|
| 2214 |
-
|
| 2215 |
-
|
| 2216 |
-
|
| 2217 |
-
"code": (info or {}).get("code", "sheet_unresolved"),
|
| 2218 |
-
"message": (info or {}).get("message", "This cell could not be refreshed."),
|
| 2219 |
-
}
|
| 2220 |
else:
|
| 2221 |
limits = resolved["limits"]
|
| 2222 |
saved["state"] = "fresh"
|
|
@@ -2226,7 +2717,9 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 2226 |
saved.pop("refusal", None)
|
| 2227 |
|
| 2228 |
try:
|
| 2229 |
-
landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved}
|
|
|
|
|
|
|
| 2230 |
except sheets.SheetRefusal as exc:
|
| 2231 |
raise _refuse(exc)
|
| 2232 |
|
|
@@ -2250,6 +2743,104 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 2250 |
return {"cell": wire, "rev": landed.get("rev"), "tab": tab_ref, "addr": key}
|
| 2251 |
|
| 2252 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2253 |
@router.get("/sheets/{workbook_id}/cells/{addr}/drill")
|
| 2254 |
def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_session)):
|
| 2255 |
"""`{database, view_id?, filter?, record_ids, count}` for `LinkGridModal` (R6, R13, R17).
|
|
@@ -2285,6 +2876,9 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
|
|
| 2285 |
if cell.get("kind") != "bound":
|
| 2286 |
raise err(409, "literal_no_drill",
|
| 2287 |
"A hand typed value has no rows behind it.")
|
|
|
|
|
|
|
|
|
|
| 2288 |
|
| 2289 |
try:
|
| 2290 |
binding = sheet_mod.clean_binding(cell.get("binding"))
|
|
@@ -2317,14 +2911,24 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
|
|
| 2317 |
# `_metric_derived` at a month window is 27 ms of an 8,437 ms drill on `customer_data`. So the
|
| 2318 |
# window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from
|
| 2319 |
# the column set instead, where the time actually is.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2320 |
cur, spec, cols = _mirror(session)(
|
| 2321 |
-
database, metric_window=
|
| 2322 |
needed=(sheet_mod.filter_columns_named(scope_nodes)
|
| 2323 |
| sheet_mod.filter_columns_named(wall_nodes)))
|
|
|
|
|
|
|
| 2324 |
|
| 2325 |
-
|
| 2326 |
-
|
| 2327 |
-
|
|
|
|
|
|
|
|
|
|
| 2328 |
total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
|
| 2329 |
|
| 2330 |
ids_sql, ids_params, _ = sheet_mod.build_ids_query(
|
|
@@ -2388,7 +2992,7 @@ def _asked_key(asked, home, fr):
|
|
| 2388 |
@router.post("/sheets/{workbook_id}/resolve-bindings")
|
| 2389 |
def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
|
| 2390 |
session: Session = Depends(require_session)):
|
| 2391 |
-
"""
|
| 2392 |
|
| 2393 |
⭐⭐ THIS IS THE DOOR THE COMMIT PATH USES, AND ITS WHOLE POINT IS THAT IT IS SMALL.
|
| 2394 |
Under R2 a formula evaluates in the browser the instant it is committed, so the only thing the
|
|
@@ -2404,20 +3008,13 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
|
|
| 2404 |
by THE EXACT STRING THE CALLER ASKED WITH, so a client looks its answer up under the
|
| 2405 |
reference it already holds and never has to re-derive this file's key spelling.
|
| 2406 |
|
| 2407 |
-
|
| 2408 |
-
|
| 2409 |
-
|
| 2410 |
-
|
| 2411 |
-
|
| 2412 |
-
|
| 2413 |
-
|
| 2414 |
-
cannot compile) comes back as `null`, which is the same thing an aggregate over zero matching
|
| 2415 |
-
rows returns. The cause is real and this door knows it; `{addr: value}` has nowhere to put it.
|
| 2416 |
-
`POST /sheets/{id}/refresh` and `GET /sheets/{id}` both still carry the full
|
| 2417 |
-
`{code, message}` refusal on the cell, so nothing is lost from the document - what is lost is
|
| 2418 |
-
the client's ability to paint THIS cell's refusal at commit time. Raised as a dated PRD
|
| 2419 |
-
amendment rather than fixed by widening the shape on one lane's authority, because lane A is
|
| 2420 |
-
the consumer and a second shape agreed by nobody is worse than a known gap.
|
| 2421 |
|
| 2422 |
⛔ ONE BAD CELL NEVER FAILS THE BATCH, exactly as in `refresh_sheet`'s bound-cell loop: a
|
| 2423 |
refusal is caught per cell and the siblings still answer.
|
|
@@ -2479,16 +3076,21 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
|
|
| 2479 |
if cell is None:
|
| 2480 |
continue
|
| 2481 |
binding = cell.get("binding") or {}
|
| 2482 |
-
|
|
|
|
| 2483 |
if slot not in by_binding:
|
| 2484 |
try:
|
| 2485 |
-
by_binding[slot] =
|
| 2486 |
-
|
| 2487 |
-
except HTTPException:
|
| 2488 |
-
|
| 2489 |
-
|
| 2490 |
-
|
| 2491 |
-
|
| 2492 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2493 |
answer[label] = by_binding[slot]
|
| 2494 |
return answer
|
|
|
|
| 221 |
if table_key in memo:
|
| 222 |
return memo[table_key]
|
| 223 |
try:
|
| 224 |
+
# W62 C2: a placed ut_* view needs its own field vocabulary, without fetching its rows.
|
| 225 |
+
if table_key.startswith("ut_"):
|
| 226 |
+
from routes_tables import ut_assembly
|
| 227 |
+
memo[table_key] = ut_assembly(session, table_key, consume_corrections=False, with_rows=False)
|
| 228 |
+
return memo[table_key]
|
| 229 |
if table_key == "product_data":
|
| 230 |
from routes_products import product_assembly
|
| 231 |
memo[table_key] = product_assembly(session, scope="product",
|
|
|
|
| 253 |
"shown rather than shown from an older set of columns. Try again in a moment.")
|
| 254 |
|
| 255 |
|
| 256 |
+
def _linked_customer_bridges(database):
|
| 257 |
+
"""Only declared reverse customer edges with an unconditional, served relation."""
|
| 258 |
+
from core.relations import BRIDGES
|
| 259 |
+
return [bridge for bridge in BRIDGES if bridge.get("target") == database
|
| 260 |
+
and bridge.get("source") == "customer_data" and bridge.get("served")
|
| 261 |
+
and bridge.get("cardinality") == "one_to_many" and not bridge.get("where")]
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def _linked_customer_fields(session, bridge):
|
| 265 |
+
from core import perm_scope
|
| 266 |
+
_require_database(session, bridge["source"])
|
| 267 |
+
g = _registry_assembly(session, bridge["source"])
|
| 268 |
+
fields = list(g.get("fields") or [])
|
| 269 |
+
hidden = perm_scope.hidden_keys(session.user, bridge["source"], fields, st=session.runtime)
|
| 270 |
+
# Only values the server can read per record. A browser formula or linked rollup needs
|
| 271 |
+
# another evaluator; offering it here would make a missing value look like a blank value.
|
| 272 |
+
fields = [f for f in fields if f.get("key") not in hidden
|
| 273 |
+
and f.get("type") not in {"formula", "link", "lookup", "rollup", "metric"}
|
| 274 |
+
and not f.get("measure")]
|
| 275 |
+
return g, fields
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _linked_relation_column(session, bridge, cur, spec, columns):
|
| 279 |
+
from core import perm_scope
|
| 280 |
+
from modules import sheet as sheet_mod
|
| 281 |
+
key = str(bridge.get("target_col") or "")
|
| 282 |
+
subject = bridge["key"] + ":" + (key or "(missing column)")
|
| 283 |
+
hidden = perm_scope.hidden_keys(session.user, bridge["target"], spec.get("fields") or [],
|
| 284 |
+
st=session.runtime)
|
| 285 |
+
if key in hidden:
|
| 286 |
+
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 287 |
+
"the relation column is hidden from this account.")
|
| 288 |
+
# The catalogue owns expressions such as sol.order_partner_id and aml.partner_id.
|
| 289 |
+
# The declaration supplies the key, and is a fallback only when the catalogue has none.
|
| 290 |
+
if (columns.get(key) or {}).get("sql"):
|
| 291 |
+
return key
|
| 292 |
+
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key):
|
| 293 |
+
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 294 |
+
"the link does not declare a readable relation column.")
|
| 295 |
+
source = spec.get("from_sql") or spec.get("table")
|
| 296 |
+
try:
|
| 297 |
+
cur.execute(f'SELECT {key} FROM {source} LIMIT 0')
|
| 298 |
+
except Exception:
|
| 299 |
+
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 300 |
+
"the declared relation column is unavailable on this database.")
|
| 301 |
+
columns[key] = {"sql": key, "type": "int", "aggregate": False}
|
| 302 |
+
return key
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
@router.get("/sheets/filter-fields/{database}")
|
| 306 |
+
def linked_filter_fields(database: str, session: Session = Depends(require_session)):
|
| 307 |
+
"""The linked Filter offer, through the same field wall as the related grid."""
|
| 308 |
+
from modules import sheet as sheet_mod
|
| 309 |
+
database = _require_database(session, database)
|
| 310 |
+
bridges = _linked_customer_bridges(database)
|
| 311 |
+
if not bridges:
|
| 312 |
+
return {"fields": []}
|
| 313 |
+
cur, spec, columns = _mirror(session)(database)
|
| 314 |
+
columns = dict(columns)
|
| 315 |
+
offered = []
|
| 316 |
+
for bridge in bridges:
|
| 317 |
+
try:
|
| 318 |
+
_linked_relation_column(session, bridge, cur, spec, columns)
|
| 319 |
+
_g, fields = _linked_customer_fields(session, bridge)
|
| 320 |
+
except HTTPException as exc:
|
| 321 |
+
if exc.status_code == 403:
|
| 322 |
+
continue
|
| 323 |
+
raise
|
| 324 |
+
except sheet_mod.SheetRefusal:
|
| 325 |
+
continue
|
| 326 |
+
link_label = f'{bridge.get("source_label") or bridge["source"]} via {bridge.get("label") or bridge["key"]}'
|
| 327 |
+
for field in fields:
|
| 328 |
+
# Copy the existing field vocabulary, including options. No customer values or
|
| 329 |
+
# hidden definitions travel on the picker wire.
|
| 330 |
+
offered.append({**field,
|
| 331 |
+
"key": f'{sheet_mod.LINKED_FILTER_PREFIX}{bridge["key"]}:{field["key"]}',
|
| 332 |
+
"label": f'{link_label}: {field.get("label") or field["key"]}'})
|
| 333 |
+
return {"fields": offered}
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def _linked_filter_scope(session, database, nodes, cur, spec, columns, *, today=None):
|
| 337 |
+
"""Read linked values in Python, then give the relation-blind compiler flat predicates."""
|
| 338 |
+
from modules import sheet as sheet_mod
|
| 339 |
+
if not sheet_mod.linked_filter_ids(nodes):
|
| 340 |
+
return nodes
|
| 341 |
+
import aios_grid
|
| 342 |
+
from core import perm_scope, shared_overlay, view_templates
|
| 343 |
+
from harness import filter_eval, filter_sql
|
| 344 |
+
|
| 345 |
+
bridges = {b["key"]: b for b in _linked_customer_bridges(database)}
|
| 346 |
+
related = {}
|
| 347 |
+
|
| 348 |
+
def resolve(leaf):
|
| 349 |
+
subject = leaf["colId"]
|
| 350 |
+
bridge_key, field_key = sheet_mod.linked_field_parts(subject)
|
| 351 |
+
bridge = bridges.get(bridge_key)
|
| 352 |
+
if bridge is None:
|
| 353 |
+
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 354 |
+
"this database no longer has that available customer link.")
|
| 355 |
+
try:
|
| 356 |
+
relation_column = _linked_relation_column(session, bridge, cur, spec, columns)
|
| 357 |
+
if bridge_key not in related:
|
| 358 |
+
g, fields = _linked_customer_fields(session, bridge)
|
| 359 |
+
# Use the rows door's wall, including its shared-field enrichment. The pool's
|
| 360 |
+
# identity is pid; it must never be confused with the orders' partner_id.
|
| 361 |
+
rows = perm_scope.apply_row_scope(g["rows_src"], session.user, bridge["source"],
|
| 362 |
+
aios_grid.FIELDS, st=session.runtime)
|
| 363 |
+
related[bridge_key] = (g, fields, rows)
|
| 364 |
+
g, fields, source_rows = related[bridge_key]
|
| 365 |
+
except HTTPException as exc:
|
| 366 |
+
if exc.status_code != 403:
|
| 367 |
+
raise
|
| 368 |
+
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 369 |
+
"this account cannot read the linked customer database.")
|
| 370 |
+
field = next((f for f in fields if f.get("key") == field_key), None)
|
| 371 |
+
if field is None:
|
| 372 |
+
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 373 |
+
"the field is hidden, removed, or has no readable per-record value.")
|
| 374 |
+
local = {**leaf, "colId": field_key}
|
| 375 |
+
# Validate before evaluating even an empty pool, so malformed/stale leaves refuse
|
| 376 |
+
# rather than looking like a genuine zero. Cell slots were resolved by W60-T06 above.
|
| 377 |
+
probe_columns = {field_key: {"sql": "linked_value", "type": sheet_mod.mirror_field_type(field)}}
|
| 378 |
+
try:
|
| 379 |
+
compiled = sheet_mod.compile_scope([local], "and", probe_columns, today=today)
|
| 380 |
+
if (local.get("op") not in filter_sql.FILTER_OPS or compiled is None
|
| 381 |
+
or field_key not in compiled.columns_used):
|
| 382 |
+
raise ValueError("unusable linked condition")
|
| 383 |
+
except (ValueError, sheet_mod.SheetRefusal):
|
| 384 |
+
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 385 |
+
"its value or operator cannot be applied to this field.")
|
| 386 |
+
storage_key = view_templates.workspace_key(bridge["source"])
|
| 387 |
+
try:
|
| 388 |
+
definitions = shared_overlay.fields(storage_key, st=session.runtime)
|
| 389 |
+
rows = aios_grid.rows_from_pool(source_rows, fields, (g.get("ws") or {}).get("overlays"),
|
| 390 |
+
derived=g.get("derived"))
|
| 391 |
+
if field_key in definitions:
|
| 392 |
+
cells = shared_overlay.cells(storage_key, [r["pid"] for r in source_rows], st=session.runtime)
|
| 393 |
+
rows = [{**row, field_key: (cells.get(str(int(row["pid"]))) or {}).get(field_key, "")}
|
| 394 |
+
for row in rows]
|
| 395 |
+
ctx = filter_eval.EvalCtx(today=today)
|
| 396 |
+
ids = [row[bridge["source_col"]] for row in rows
|
| 397 |
+
if filter_eval.permits({"conj": "and", "nodes": [local]}, row, fields, ctx)]
|
| 398 |
+
return sheet_mod.linked_id_predicate(relation_column, ids)
|
| 399 |
+
except Exception:
|
| 400 |
+
raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
|
| 401 |
+
"the linked customer values could not be read. Retry after the data is available.")
|
| 402 |
+
|
| 403 |
+
try:
|
| 404 |
+
return sheet_mod.rewrite_linked_filters(nodes, resolve)
|
| 405 |
+
except sheet_mod.SheetRefusal as exc:
|
| 406 |
+
raise err(409, exc.code, exc.message)
|
| 407 |
+
|
| 408 |
+
|
| 409 |
def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
|
| 410 |
"""`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period.
|
| 411 |
|
|
|
|
| 774 |
|
| 775 |
#: The refusals that mean "that thing is not here", answered 404 so a client can tell a missing
|
| 776 |
#: workbook from a malformed one without reading the sentence.
|
| 777 |
+
_MISSING_CODES = ("workbook_missing", "folder_missing", "tab_missing", "unknown_sheet_template")
|
| 778 |
|
| 779 |
#: ⛔⛔ C5's TWO SHARING REFUSALS ANSWER 403, AND THE TUPLE EXISTS BECAUSE THE FALL-THROUGH BELOW
|
| 780 |
#: IS 400 (wave 51, W51-T25). Without this branch `workbook_read_only` would arrive as a NAMED 400
|
|
|
|
| 800 |
nothing else, so the 409 and the 413 are raised here directly rather than by bending `err`,
|
| 801 |
which would change the error shape of every route in the app.
|
| 802 |
|
| 803 |
+
Workbook, retained cell history and folder index refusals all carry the current revision
|
| 804 |
+
through the same existing envelope.
|
|
|
|
|
|
|
| 805 |
"""
|
| 806 |
if exc.code in _CAP_CODES:
|
| 807 |
return HTTPException(status_code=413,
|
|
|
|
| 895 |
round because a workbook that has never been shared has no registry record at all. Asking
|
| 896 |
`shares.role_for` alone would answer `None` for the person who created it.
|
| 897 |
"""
|
| 898 |
+
from core import sheets
|
| 899 |
+
return sheets.workbook_role(_store(session), book, session.uname, is_admin=session.admin)
|
| 900 |
+
|
| 901 |
+
|
| 902 |
+
def _require_reader(session: Session, book):
|
| 903 |
+
"""R28: use the same predicate as the index; hidden and missing ids answer identically."""
|
| 904 |
+
from core import sheets
|
| 905 |
+
if not sheets.may_read_workbook(book, _role_on_workbook(session, book)):
|
| 906 |
+
raise _refuse(sheets.SheetRefusal("workbook_missing", "That workbook no longer exists."))
|
| 907 |
+
|
| 908 |
+
|
| 909 |
+
def _index_for(session: Session):
|
| 910 |
+
"""C2: every list response uses the index's sole visibility filter and role helper."""
|
| 911 |
+
from core import sheets
|
| 912 |
+
# The index already owns the bulk grant join. Passing the per-object route helper here
|
| 913 |
+
# converts that one read back into N full registry reads and made 50-row lists miss budget.
|
| 914 |
+
return sheets.read_index(_store(session), viewer=session.uname, is_admin=session.admin)
|
| 915 |
|
| 916 |
|
| 917 |
def _require_editor(session: Session, book):
|
|
|
|
| 944 |
out = []
|
| 945 |
for row in rows or ():
|
| 946 |
row = dict(row)
|
| 947 |
+
if "role" in row:
|
| 948 |
+
row["role"] = "owner" if row["role"] == "owner" else wire_role(row["role"])
|
| 949 |
row["shares"] = [{"who": s.get("who"), "role": wire_role(s.get("role"))}
|
| 950 |
for s in (row.get("shares") or [])
|
| 951 |
if wire_role(s.get("role"))]
|
|
|
|
| 977 |
return sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {})
|
| 978 |
|
| 979 |
|
| 980 |
+
def _has_ending(binding):
|
| 981 |
+
return (isinstance(binding, dict) and isinstance(binding.get("period"), dict)
|
| 982 |
+
and "ending" in binding["period"])
|
| 983 |
+
|
| 984 |
+
|
| 985 |
+
def _has_cell_inputs(binding):
|
| 986 |
+
from modules import sheet as sheet_mod
|
| 987 |
+
if _has_ending(binding):
|
| 988 |
+
return True
|
| 989 |
+
scope = binding.get("scope") if isinstance(binding, dict) else None
|
| 990 |
+
nodes, _ = _fe_parts(scope.get("filter") if isinstance(scope, dict) else None)
|
| 991 |
+
# Linked values and related-table grants can change independently of the orders' cache key.
|
| 992 |
+
if sheet_mod.linked_filter_ids(nodes):
|
| 993 |
+
return True
|
| 994 |
+
return any(sheet_mod.filter_cell_nodes(nodes))
|
| 995 |
+
|
| 996 |
+
|
| 997 |
+
def _reresolved_for_reader(session: Session, book, *, endings_only=False):
|
| 998 |
"""A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5).
|
| 999 |
|
| 1000 |
⛔⛔ IN MEMORY, AND IT MUST NEVER PERSIST. `refresh_sheet` runs the same loop and ends with
|
|
|
|
| 1035 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 1036 |
if not isinstance(cell, dict) or cell.get("kind") != "bound":
|
| 1037 |
continue
|
| 1038 |
+
if endings_only and not _has_cell_inputs(cell.get("binding")):
|
| 1039 |
+
continue
|
| 1040 |
touched += 1
|
| 1041 |
binding = cell.get("binding") or {}
|
| 1042 |
db = str(binding.get("database") or "")
|
|
|
|
| 1048 |
seen_db[db] = exc
|
| 1049 |
failure = seen_db.get(db)
|
| 1050 |
if failure is not None:
|
| 1051 |
+
cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}."))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1052 |
cell.pop("limits", None)
|
| 1053 |
continue
|
| 1054 |
|
| 1055 |
cell_key = tab_ref + "!" + addr
|
| 1056 |
scope_key = _binding_scope_key(session, db, binding)
|
| 1057 |
hit = None
|
| 1058 |
+
if wid and scope_key and not _has_cell_inputs(binding):
|
| 1059 |
try:
|
| 1060 |
hit = sheets.cached_scope_value(store, wid, cell_key, scope_key)
|
| 1061 |
except Exception: # noqa: BLE001
|
|
|
|
| 1073 |
continue
|
| 1074 |
|
| 1075 |
try:
|
| 1076 |
+
out = _resolve_cell_binding(session, binding, fresh, tab_ref, view_memo=view_memo)
|
| 1077 |
except HTTPException as exc:
|
| 1078 |
detail = getattr(exc, "detail", None) or {}
|
| 1079 |
info = detail.get("error") if isinstance(detail, dict) else None
|
| 1080 |
+
cell.update(_binding_refusal(
|
| 1081 |
+
(info or {}).get("code", "sheet_unresolved"),
|
| 1082 |
+
(info or {}).get("message", "This cell could not be refreshed."),
|
| 1083 |
+
))
|
| 1084 |
+
cell.pop("limits", None)
|
|
|
|
|
|
|
| 1085 |
continue
|
| 1086 |
cell["state"] = "fresh"
|
| 1087 |
cell["value"] = out["value"]
|
|
|
|
| 1113 |
return fresh
|
| 1114 |
|
| 1115 |
|
| 1116 |
+
def _response_workbook(book):
|
| 1117 |
+
"""Omit private write history from response tabs without changing the stored document."""
|
| 1118 |
+
return {**book, "sheets": [{key: value for key, value in tab.items()
|
| 1119 |
+
if key not in ("writes", "writesFrom")}
|
| 1120 |
+
for tab in book.get("sheets") or []]}
|
| 1121 |
+
|
| 1122 |
+
|
| 1123 |
def _read_for(session: Session, book):
|
| 1124 |
"""The workbook this session is entitled to see: stored for the owner, re-resolved otherwise.
|
| 1125 |
|
| 1126 |
+
W60-T09, 2026-09-14: R28 checks visibility before this projection. Admitted legacy readers
|
| 1127 |
+
and grantees still receive their own scoped numbers, never the owner's cached numbers.
|
|
|
|
|
|
|
| 1128 |
"""
|
| 1129 |
+
book = _response_workbook(book)
|
| 1130 |
if _role_on_workbook(session, book) == "owner":
|
| 1131 |
+
if any(_has_cell_inputs(cell.get("binding"))
|
| 1132 |
+
for tab in book.get("sheets") or [] for cell in (tab.get("cells") or {}).values()):
|
| 1133 |
+
return _reresolved_for_reader(session, book, endings_only=True)
|
| 1134 |
return book
|
| 1135 |
return _reresolved_for_reader(session, book)
|
| 1136 |
|
|
|
|
| 1141 |
|
| 1142 |
⚠ C5's `shares` rides here as well as on `/sheets/index`, because the client reads both.
|
| 1143 |
"""
|
| 1144 |
+
rows = _index_for(session)["workbooks"]
|
|
|
|
| 1145 |
return {"workbooks": _wire_shares(rows)}
|
| 1146 |
|
| 1147 |
|
|
|
|
| 1153 |
try:
|
| 1154 |
book = sheets.create_workbook(_store(session),
|
| 1155 |
name=raw.get("name") or "",
|
| 1156 |
+
created_by=session.uname,
|
| 1157 |
+
on_duplicate="refuse")
|
| 1158 |
except sheets.SheetRefusal as exc:
|
| 1159 |
raise _refuse(exc)
|
| 1160 |
+
return {"workbook": _response_workbook(book)}
|
| 1161 |
+
|
| 1162 |
+
|
| 1163 |
+
@router.post("/sheets/from-template/{key}", status_code=201)
|
| 1164 |
+
def create_sheet_from_template(key: str, body: dict = Body(default=None),
|
| 1165 |
+
session: Session = Depends(require_session)):
|
| 1166 |
+
"""Create a private workbook from a tenant-saved Sheet template."""
|
| 1167 |
+
from core import sheets, sheet_templates
|
| 1168 |
+
book = None
|
| 1169 |
+
try:
|
| 1170 |
+
record = sheet_templates.saved_template(_store(session), key)
|
| 1171 |
+
for database in (record or {}).get("databases", []):
|
| 1172 |
+
_require_database(session, database)
|
| 1173 |
+
book = sheet_templates.apply_saved_template(
|
| 1174 |
+
_store(session), key, name=(body or {}).get("name") or "", created_by=session.uname)
|
| 1175 |
+
# Owners read persisted results, so refresh under this maker before their first open.
|
| 1176 |
+
return {"workbook": _response_workbook(refresh_sheet(book["id"], session=session)["workbook"])}
|
| 1177 |
+
except Exception as exc:
|
| 1178 |
+
if book is not None:
|
| 1179 |
+
sheets.delete_workbook(_store(session), book["id"])
|
| 1180 |
+
if isinstance(exc, sheets.SheetRefusal):
|
| 1181 |
+
raise _refuse(exc)
|
| 1182 |
+
raise
|
| 1183 |
|
| 1184 |
|
| 1185 |
# ── the rail's own document (wave 50, W50-T17) ─────────────────────���─────────────────────────
|
|
|
|
| 1194 |
# ⚠ `Depends(require_session)` ONLY, like every other door in this file, and that is correct: the
|
| 1195 |
# Sheet is granted to everyone, and the rail's folders name no database.
|
| 1196 |
#
|
| 1197 |
+
# W60-T09, 2026-09-14: R28 supersedes the W51-T25 amendment finding. Both lists and all three
|
| 1198 |
+
# workbook read shapes use core.sheets.may_read_workbook with _role_on_workbook. New workbooks
|
| 1199 |
+
# are private to their creator and grantees; administrators see all. Unmarked or creatorless
|
| 1200 |
+
# documents retain legacy visibility. Admitted non-owners still re-resolve under their own scope.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1201 |
|
| 1202 |
|
| 1203 |
@router.get("/sheets/index")
|
|
|
|
| 1212 |
can reach: a client cannot send a revision it was never given, and a server that never checks
|
| 1213 |
one is a client stamping a number nobody reads.
|
| 1214 |
"""
|
| 1215 |
+
return {"index": _wire_index(_index_for(session))}
|
|
|
|
| 1216 |
|
| 1217 |
|
| 1218 |
@router.put("/sheets/index")
|
| 1219 |
def write_sheet_index(body: dict = Body(default=None),
|
| 1220 |
session: Session = Depends(require_session)):
|
| 1221 |
+
"""D-639 / W62-T25: replace the rail wholesale or merge a placements-only delta.
|
| 1222 |
|
| 1223 |
⭐ `repairs` RIDES BACK OUT. `core.sheets.save_folders` records every change it made to the
|
| 1224 |
payload, and the rail says so in a sentence rather than letting a workbook reappear at the
|
|
|
|
| 1233 |
"""
|
| 1234 |
from core import sheets
|
| 1235 |
raw = dict(body or {})
|
| 1236 |
+
# This document is tenant-shared while its workbook rows are object-scoped. A response-only
|
| 1237 |
+
# projection is not a write wall: without carrying these ids into the mutation, a stranger can
|
| 1238 |
+
# move a hidden workbook by guessing its id, or save their empty projection wholesale and wipe
|
| 1239 |
+
# the owner's folders/stars. Read the same projected index the GET returns, then make the core
|
| 1240 |
+
# transaction preserve every stratum outside that projection.
|
| 1241 |
+
visible = _index_for(session)
|
| 1242 |
+
visible_ids = {str(row.get("id")) for row in (visible.get("workbooks") or [])
|
| 1243 |
+
if isinstance(row, dict) and row.get("id")}
|
| 1244 |
+
visible_folder_ids = {str(row.get("id")) for row in (visible.get("folders") or [])
|
| 1245 |
+
if isinstance(row, dict) and row.get("id")}
|
| 1246 |
try:
|
| 1247 |
+
if "placements" in raw:
|
| 1248 |
+
if set(raw) != {"placements"}:
|
| 1249 |
+
raise sheets.SheetRefusal(
|
| 1250 |
+
"index_request_mixed",
|
| 1251 |
+
"Send placements alone, or send the full Sheet rail with its revision.")
|
| 1252 |
+
if (not session.admin and isinstance(raw.get("placements"), dict)
|
| 1253 |
+
and any(str(wid) not in visible_ids for wid in raw["placements"])):
|
| 1254 |
+
# Hidden and nonexistent ids are deliberately identical: this write door is not
|
| 1255 |
+
# an oracle for private workbook existence.
|
| 1256 |
+
raise sheets.SheetRefusal("workbook_missing",
|
| 1257 |
+
"That workbook no longer exists.")
|
| 1258 |
+
out = sheets.merge_placements(_store(session), raw["placements"])
|
| 1259 |
+
else:
|
| 1260 |
+
folders = raw.get("folders")
|
| 1261 |
+
item_folders = raw.get("itemFolders")
|
| 1262 |
+
if not session.admin:
|
| 1263 |
+
submitted_workbooks = (set(str(wid) for wid in item_folders)
|
| 1264 |
+
if isinstance(item_folders, dict) else set())
|
| 1265 |
+
submitted_stars = (set(str(wid) for wid in raw.get("starred"))
|
| 1266 |
+
if isinstance(raw.get("starred"), list) else set())
|
| 1267 |
+
if (submitted_workbooks | submitted_stars) - visible_ids:
|
| 1268 |
+
raise sheets.SheetRefusal("workbook_missing",
|
| 1269 |
+
"That workbook no longer exists.")
|
| 1270 |
+
# A wholesale snapshot cannot otherwise distinguish "create this client id" from
|
| 1271 |
+
# "rename the private folder whose id I guessed". Current clients declare their
|
| 1272 |
+
# newly-created temporary ids. Every created id is replaced server-side, whether
|
| 1273 |
+
# it collides with a hidden folder or with nothing, so the response is not an
|
| 1274 |
+
# existence oracle.
|
| 1275 |
+
#
|
| 1276 |
+
# Older clients predate `createdFolderIds`. Preserve their one safe, observable
|
| 1277 |
+
# creation shape: at least one visible workbook is filed into the submitted new
|
| 1278 |
+
# tree. In that legacy shape every unknown submitted id is treated as temporary
|
| 1279 |
+
# and replaced, including empty siblings and ancestors. A bare undeclared id is
|
| 1280 |
+
# still refused identically whether it is hidden or nonexistent, so the repair does
|
| 1281 |
+
# not re-open the hidden-folder oracle while the old D-639 save door keeps working.
|
| 1282 |
+
declared_creates = "createdFolderIds" in raw
|
| 1283 |
+
created = raw.get("createdFolderIds", [])
|
| 1284 |
+
if (not isinstance(created, list)
|
| 1285 |
+
or any(not isinstance(fid, str) or not fid for fid in created)):
|
| 1286 |
+
raise sheets.SheetRefusal(
|
| 1287 |
+
"folder_create_ids_invalid",
|
| 1288 |
+
"New folders must declare their temporary identifiers.")
|
| 1289 |
+
created_ids = set(created)
|
| 1290 |
+
incoming = [dict(row) if isinstance(row, dict) else row
|
| 1291 |
+
for row in folders] if isinstance(folders, list) else []
|
| 1292 |
+
incoming_ids = {str(row.get("id")) for row in incoming
|
| 1293 |
+
if isinstance(row, dict) and row.get("id")}
|
| 1294 |
+
unknown = incoming_ids - visible_folder_ids
|
| 1295 |
+
if declared_creates:
|
| 1296 |
+
if unknown - created_ids or created_ids - unknown:
|
| 1297 |
+
raise sheets.SheetRefusal("folder_missing",
|
| 1298 |
+
"That folder no longer exists.")
|
| 1299 |
+
elif unknown:
|
| 1300 |
+
filed_into_unknown = {
|
| 1301 |
+
str(fid) for wid, fid in (item_folders.items()
|
| 1302 |
+
if isinstance(item_folders, dict) else [])
|
| 1303 |
+
if str(wid) in visible_ids and fid is not None and str(fid) in unknown
|
| 1304 |
+
}
|
| 1305 |
+
if not filed_into_unknown:
|
| 1306 |
+
raise sheets.SheetRefusal("folder_missing",
|
| 1307 |
+
"That folder no longer exists.")
|
| 1308 |
+
created_ids = unknown
|
| 1309 |
+
replacements = {fid: "f" + uuid.uuid4().hex for fid in created_ids}
|
| 1310 |
+
if replacements:
|
| 1311 |
+
for row in incoming:
|
| 1312 |
+
if not isinstance(row, dict):
|
| 1313 |
+
continue
|
| 1314 |
+
row["id"] = replacements.get(str(row.get("id")), row.get("id"))
|
| 1315 |
+
if row.get("parent") is not None:
|
| 1316 |
+
row["parent"] = replacements.get(str(row["parent"]), row["parent"])
|
| 1317 |
+
if isinstance(item_folders, dict):
|
| 1318 |
+
item_folders = {
|
| 1319 |
+
wid: replacements.get(str(fid), fid)
|
| 1320 |
+
for wid, fid in item_folders.items()
|
| 1321 |
+
}
|
| 1322 |
+
folders = incoming
|
| 1323 |
+
out = sheets.save_folders(_store(session), folders,
|
| 1324 |
+
item_folders, starred=raw.get("starred"),
|
| 1325 |
+
expect_rev=raw.get("rev"),
|
| 1326 |
+
writable_ids=None if session.admin else visible_ids,
|
| 1327 |
+
writable_folder_ids=(None if session.admin
|
| 1328 |
+
else visible_folder_ids))
|
| 1329 |
except sheets.SheetRefusal as exc:
|
| 1330 |
raise _refuse(exc)
|
| 1331 |
+
return {"index": _wire_index({**_index_for(session), "repairs": out.get("repairs", [])})}
|
| 1332 |
|
| 1333 |
|
| 1334 |
# ── C4: the .xlsx IMPORT, which is a JOB and not a request (wave 51, W51-T24) ────────────────
|
|
|
|
| 1424 |
_job_set(job_id, report=report)
|
| 1425 |
|
| 1426 |
stamp = _now_iso()
|
| 1427 |
+
made = sheets.create_workbook(runtime, name=book.get("name") or "", created_by=uname,
|
| 1428 |
+
on_duplicate="suffix")
|
| 1429 |
saved = sheets.put_workbook(runtime, made["id"],
|
| 1430 |
+
dict(book, refreshed_at=stamp, rev=made["rev"]), actor=uname,
|
| 1431 |
+
lossless=True)
|
| 1432 |
_job_set(job_id, state="done", progress=1.0, workbook_id=saved["id"],
|
| 1433 |
sentence=_import_sentence(report))
|
| 1434 |
except sheet_import.ImportRefusal as exc:
|
|
|
|
| 1553 |
from core import sheets
|
| 1554 |
try:
|
| 1555 |
book = sheets.get_workbook(_store(session), workbook_id)
|
| 1556 |
+
_require_reader(session, book)
|
| 1557 |
except sheets.SheetRefusal as exc:
|
| 1558 |
raise _refuse(exc)
|
| 1559 |
if str(tabs or "").strip().lower() == "meta":
|
|
|
|
| 1590 |
from core import sheets
|
| 1591 |
try:
|
| 1592 |
book = sheets.get_workbook(_store(session), workbook_id)
|
| 1593 |
+
_require_reader(session, book)
|
| 1594 |
served = _read_for(session, book)
|
| 1595 |
found = sheets.find_tab(served, tab)
|
| 1596 |
except sheets.SheetRefusal as exc:
|
|
|
|
| 1616 |
same patch has landed. A CAP breach is the opposite: it is a whole request 413, because the
|
| 1617 |
document as a whole cannot be stored and there is no partial version of that.
|
| 1618 |
|
| 1619 |
+
With retained history, a stale revision merges by address and the later value wins;
|
| 1620 |
+
`conflicts` names other people's overlapping writes, and only missing history refuses.
|
|
|
|
| 1621 |
|
| 1622 |
⛔⛔ C5's READ-ONLY CHECK RUNS BEFORE THAT COMPARISON AND THE ORDER IS THE CONTRACT. A viewer
|
| 1623 |
is refused 403 `workbook_read_only`; a viewer holding a stale rev is ALSO refused
|
|
|
|
| 1643 |
expect = raw.get("rev")
|
| 1644 |
try:
|
| 1645 |
out = sheets.patch_cells(_store(session), workbook_id, tab, raw.get("cells"),
|
| 1646 |
+
expect_rev=expect, actor=session.uname,
|
| 1647 |
+
actor_name=session.user.get("name") or session.uname)
|
| 1648 |
except sheets.SheetRefusal as exc:
|
| 1649 |
raise _refuse(exc)
|
| 1650 |
accepted = out.pop("accepted", [])
|
|
|
|
| 1693 |
(found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes}
|
| 1694 |
_recompute_formulas(book, fr, sheet_mod,
|
| 1695 |
cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
|
| 1696 |
+
# Date and filter-cell edits also change database totals. Recompute all readers, then formulas.
|
| 1697 |
+
ending_cells = set()
|
| 1698 |
+
for source in book.get("sheets") or []:
|
| 1699 |
+
for addr, cell in (source.get("cells") or {}).items():
|
| 1700 |
+
binding = cell.get("binding") or {}
|
| 1701 |
+
if cell.get("kind") != "bound" or not _has_cell_inputs(binding):
|
| 1702 |
+
continue
|
| 1703 |
+
try:
|
| 1704 |
+
result = _resolve_cell_binding(session, binding, book, source.get("id"))
|
| 1705 |
+
cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
|
| 1706 |
+
cell.pop("refusal", None)
|
| 1707 |
+
except HTTPException as exc:
|
| 1708 |
+
info = ((exc.detail or {}).get("error") or {})
|
| 1709 |
+
cell.update(_binding_refusal(
|
| 1710 |
+
info.get("code", "ending_cell_unreadable"),
|
| 1711 |
+
info.get("message", "The Ending date could not be read. Choose a valid date."),
|
| 1712 |
+
))
|
| 1713 |
+
ending_cells.add((source.get("id"), addr))
|
| 1714 |
+
if ending_cells:
|
| 1715 |
+
_recompute_formulas(book, fr, sheet_mod)
|
| 1716 |
found = sheets.find_tab(book, tab_ref)
|
| 1717 |
out = {}
|
| 1718 |
for addr, cell in (found.get("cells") or {}).items():
|
| 1719 |
if not isinstance(cell, dict) or (cell.get("kind") != "formula"
|
| 1720 |
+
and "spill" not in cell and "spillParent" not in cell
|
| 1721 |
+
and (found.get("id"), addr) not in ending_cells):
|
| 1722 |
continue
|
| 1723 |
row = {"value": cell.get("value"),
|
| 1724 |
"display": "" if cell.get("value") is None else str(cell.get("display") or ""),
|
|
|
|
| 1753 |
# `rev` inside the body IS the expectation; `core.sheets.put_workbook` reads it. This door
|
| 1754 |
# does not translate it into a second argument, because two ways to say the same thing is
|
| 1755 |
# how one of them ends up ignored.
|
| 1756 |
+
book = sheets.put_workbook(_store(session), workbook_id, raw,
|
| 1757 |
+
actor=session.uname,
|
| 1758 |
+
actor_name=session.user.get("name") or session.uname)
|
| 1759 |
except sheets.SheetRefusal as exc:
|
| 1760 |
raise _refuse(exc)
|
| 1761 |
+
return {"workbook": _response_workbook(book)}
|
| 1762 |
|
| 1763 |
|
| 1764 |
@router.delete("/sheets/{workbook_id}")
|
|
|
|
| 1829 |
rec = shares.set_grants("workbook", book["id"], entries,
|
| 1830 |
owner=(None if held.get("owner") else _workbook_owner(book)),
|
| 1831 |
granter=session.uname, st=_store(session), expect=prior)
|
| 1832 |
+
# A successful explicit share narrows legacy visibility; ordinary saves never do.
|
| 1833 |
+
if "access" not in book:
|
| 1834 |
+
# W60-T09, 2026-09-14: the first share decides VISIBILITY, making the owner gate load bearing.
|
| 1835 |
+
# A non-owner must not reach this narrowing on an ownerless legacy workbook
|
| 1836 |
+
# and lock themselves out by sharing it.
|
| 1837 |
+
sheets.narrow_workbook(_store(session), book["id"])
|
| 1838 |
+
except sheets.SheetRefusal as exc:
|
| 1839 |
+
raise _refuse(exc)
|
| 1840 |
except shares.GrantsChanged as exc:
|
| 1841 |
# ⚠ `grants_changed`, THE SAME CODE `routes_shares.py` ALREADY ANSWERS, and a plain 409
|
| 1842 |
# rather than the `_stale` shape: there is no revision to hand back here, and a
|
|
|
|
| 1929 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 1930 |
if not isinstance(cell, dict) or cell.get("kind") != "bound":
|
| 1931 |
continue
|
| 1932 |
+
if _is_view_binding(cell):
|
| 1933 |
+
continue # C2: a placed view refreshes through its own door
|
| 1934 |
binding = cell.get("binding") or {}
|
| 1935 |
db = str(binding.get("database") or "")
|
| 1936 |
if db not in seen_db:
|
|
|
|
| 1941 |
seen_db[db] = exc
|
| 1942 |
failure = seen_db.get(db)
|
| 1943 |
if failure is not None:
|
| 1944 |
+
cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}."))
|
| 1945 |
+
cell.pop("limits", None)
|
|
|
|
|
|
|
| 1946 |
continue
|
| 1947 |
try:
|
| 1948 |
+
out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
|
| 1949 |
except HTTPException as exc:
|
| 1950 |
detail = getattr(exc, "detail", None) or {}
|
| 1951 |
info = detail.get("error") if isinstance(detail, dict) else None
|
| 1952 |
+
cell.update(_binding_refusal(
|
| 1953 |
+
(info or {}).get("code", "sheet_unresolved"),
|
| 1954 |
+
(info or {}).get("message", "This cell could not be refreshed."),
|
| 1955 |
+
))
|
| 1956 |
+
cell.pop("limits", None)
|
|
|
|
| 1957 |
continue
|
| 1958 |
cell["state"] = "fresh"
|
| 1959 |
cell["value"] = out["value"]
|
|
|
|
| 1979 |
_recompute_formulas(book, fr, sheet_mod)
|
| 1980 |
recalc_ms = round((perf_counter() - started) * 1000.0, 1)
|
| 1981 |
try:
|
| 1982 |
+
# R19: book came from get_workbook; clamping on refresh would destroy imported widths.
|
| 1983 |
saved = sheets.put_workbook(_store(session), workbook_id,
|
| 1984 |
{"name": book.get("name"), "sheets": book.get("sheets"),
|
| 1985 |
+
"refreshed_at": stamp, "rev": book.get("rev")},
|
| 1986 |
+
actor=session.uname,
|
| 1987 |
+
actor_name=session.user.get("name") or session.uname,
|
| 1988 |
+
lossless=True)
|
| 1989 |
except sheets.SheetRefusal as exc:
|
| 1990 |
raise _refuse(exc)
|
| 1991 |
# ⭐⭐ R3 / W52-T12: REFRESH IS THE ONE EVENT ALLOWED TO INVALIDATE A READER'S CACHED SCOPE
|
| 1992 |
# VALUE, so it is the one door that clears the whole per-scope cache. Every reader's next GET
|
| 1993 |
# recomputes once, under their own scope, and fills its own slot again from there.
|
| 1994 |
sheets.clear_scope_cache(_store(session), workbook_id)
|
| 1995 |
+
return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms}
|
| 1996 |
|
| 1997 |
|
| 1998 |
def _now_iso():
|
|
|
|
| 2247 |
return column(raw)
|
| 2248 |
|
| 2249 |
|
| 2250 |
+
def _ending_window(session, binding, *, book=None, tab=None, today=None, seen=()):
|
| 2251 |
+
"""Resolve a period from authored cells. Submitted ending.date is never trusted for a ref."""
|
| 2252 |
+
from modules import sheet as sheet_mod
|
| 2253 |
+
from harness import windows as W
|
| 2254 |
+
period = binding.get("period")
|
| 2255 |
+
if not isinstance(period, dict) or (period.get("ending") or {}).get("kind") != "cell":
|
| 2256 |
+
return sheet_mod.period_window(period, today)
|
| 2257 |
+
ref = period.get("ref") or "(blank)"
|
| 2258 |
+
value = _read_binding_cell(session, ref, book=book, tab=tab, today=today, seen=seen)
|
| 2259 |
+
date = W.ending_cell_date(value, book.get("dateSystem", "1900"))
|
| 2260 |
+
if date is None:
|
| 2261 |
+
raise sheet_mod.SheetRefusal("ending_cell_unreadable", ref, "its value is not a real date.")
|
| 2262 |
+
return sheet_mod.period_window(period, today, cell_date=date)
|
| 2263 |
+
|
| 2264 |
+
|
| 2265 |
+
def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(),
|
| 2266 |
+
code="ending_cell_unreadable"):
|
| 2267 |
+
"""Shared cell read for Ending and Filter, including scoped bound and formula inputs."""
|
| 2268 |
+
from modules import sheet as sheet_mod
|
| 2269 |
+
import formula_runtime as fr
|
| 2270 |
+
|
| 2271 |
+
def refuse(reason):
|
| 2272 |
+
raise sheet_mod.SheetRefusal(code, ref or "(blank)", reason)
|
| 2273 |
+
|
| 2274 |
+
if not re.fullmatch(r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref):
|
| 2275 |
+
refuse("its reference is invalid.")
|
| 2276 |
+
if book is None:
|
| 2277 |
+
refuse("the source workbook is unavailable.")
|
| 2278 |
+
tabs = book.get("sheets") or []
|
| 2279 |
+
names = _tab_scope_names(tabs)
|
| 2280 |
+
home = _resolve_home_tab(book, tabs, names, tab)
|
| 2281 |
+
key = _asked_key(ref, home, fr)
|
| 2282 |
+
if key in seen:
|
| 2283 |
+
refuse("the reference is circular.")
|
| 2284 |
+
cells = {fr.a1_key(addr, name): cell for source, name in zip(tabs, names)
|
| 2285 |
+
for addr, cell in (source.get("cells") or {}).items()}
|
| 2286 |
+
cell = cells.get(key)
|
| 2287 |
+
if not isinstance(cell, dict) or cell.get("kind") == "empty":
|
| 2288 |
+
refuse("the cell or its sheet is missing or blank.")
|
| 2289 |
+
if cell.get("kind") == "bound":
|
| 2290 |
+
try:
|
| 2291 |
+
value = _resolve_binding(session, cell.get("binding"), today=today,
|
| 2292 |
+
book=book, tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"]
|
| 2293 |
+
except HTTPException as exc:
|
| 2294 |
+
reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
|
| 2295 |
+
refuse(reason)
|
| 2296 |
+
elif cell.get("kind") == "formula":
|
| 2297 |
+
# The existing formula graph discovers actual inputs, including dynamic references.
|
| 2298 |
+
# A database input starts refused, so a cached value can never bypass this reader's wall.
|
| 2299 |
+
bound = {k: c for k, c in cells.items() if c.get("kind") == "bound"}
|
| 2300 |
+
specs = {k: ({"refused": "ending_input_pending"} if k in bound
|
| 2301 |
+
else fr._document_spec(c, k.rpartition("!")[0])) for k, c in cells.items()}
|
| 2302 |
+
engine = fr.SheetCalculation(specs, sheets=names, today=today,
|
| 2303 |
+
named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
|
| 2304 |
+
attempted, failures = set(), {}
|
| 2305 |
+
while True:
|
| 2306 |
+
dependencies, pending = set(), [key]
|
| 2307 |
+
while pending:
|
| 2308 |
+
dependency = pending.pop()
|
| 2309 |
+
if dependency in dependencies:
|
| 2310 |
+
continue
|
| 2311 |
+
dependencies.add(dependency)
|
| 2312 |
+
pending.extend(engine.consumed.get(dependency, ()))
|
| 2313 |
+
needed = (dependencies & set(bound)) - attempted
|
| 2314 |
+
if not needed:
|
| 2315 |
+
break
|
| 2316 |
+
changes = {}
|
| 2317 |
+
for dependency in needed:
|
| 2318 |
+
attempted.add(dependency)
|
| 2319 |
+
try:
|
| 2320 |
+
result = _resolve_binding(session, bound[dependency].get("binding"), today=today,
|
| 2321 |
+
book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency))
|
| 2322 |
+
changes[dependency] = {"value": result["value"]}
|
| 2323 |
+
except HTTPException as exc:
|
| 2324 |
+
info = ((exc.detail or {}).get("error") or {})
|
| 2325 |
+
failures[dependency] = info.get("message", "a database input is refused.")
|
| 2326 |
+
changes[dependency] = {"refused": info.get("code", "ending_cell_unreadable")}
|
| 2327 |
+
# Each pass resolves previously unread database cells, so the workbook bounds the loop.
|
| 2328 |
+
engine.update(changes, revision=engine.revision + 1, today=today)
|
| 2329 |
+
if engine.errors.get(key):
|
| 2330 |
+
error_code = engine.errors[key]
|
| 2331 |
+
reason = next((failures[k] for k in dependencies if k in failures), None)
|
| 2332 |
+
if not reason:
|
| 2333 |
+
reason = (sheet_mod.SheetRefusal(error_code, ref, "a formula input is refused.").message
|
| 2334 |
+
if error_code in sheet_mod.REFUSALS else fr._sheet_error_message(error_code))
|
| 2335 |
+
refuse(reason)
|
| 2336 |
+
value = engine.values.get(key)
|
| 2337 |
+
else:
|
| 2338 |
+
if cell.get("state") == "refused":
|
| 2339 |
+
refuse((cell.get("refusal") or {}).get("message") or "the cell is refused.")
|
| 2340 |
+
if cell.get("state") == "stale":
|
| 2341 |
+
refuse("the cell needs to be refreshed.")
|
| 2342 |
+
value = cell.get("value", cell.get("text"))
|
| 2343 |
+
return value
|
| 2344 |
+
|
| 2345 |
+
|
| 2346 |
+
def _filter_cell_values(session, nodes, *, book=None, tab=None, today=None, seen=()):
|
| 2347 |
+
from modules import sheet as sheet_mod
|
| 2348 |
+
try:
|
| 2349 |
+
return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell(
|
| 2350 |
+
session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable"))
|
| 2351 |
+
except sheet_mod.SheetRefusal as exc:
|
| 2352 |
+
raise err(400, exc.code, exc.message)
|
| 2353 |
+
|
| 2354 |
+
|
| 2355 |
+
# ⛔⛔ W62-T12 / C2 / R29 — A `mode: "view"` BINDING IS SKIPPED BY EVERY METRIC RESOLVER, AND
|
| 2356 |
+
# SKIPPING IS NOT THE SAME AS REFUSING. A placed view has no scalar value: its rows are read through
|
| 2357 |
+
# `POST /sheets/{id}/view-rows` and PLACED by the client, because the placement is a structure op
|
| 2358 |
+
# only the client can take. Before this guard the refresh door handed a view binding to
|
| 2359 |
+
# `modules/sheet.py::clean_binding`, which REQUIRES an `aggregate`, so `Refresh All` marked every
|
| 2360 |
+
# placed view `refused` and painted "That cell's source is not set up correctly." over a view that
|
| 2361 |
+
# was perfectly well formed. Caught by this lane's own screenshot, not by a gate.
|
| 2362 |
+
def _is_view_binding(cell) -> bool:
|
| 2363 |
+
"""True for a C2 placed view, which the metric resolvers must leave exactly as they found it."""
|
| 2364 |
+
binding = (cell or {}).get("binding") if isinstance(cell, dict) else None
|
| 2365 |
+
return isinstance(binding, dict) and binding.get("mode") == "view"
|
| 2366 |
+
|
| 2367 |
+
|
| 2368 |
+
# R48: the persisted code is short; the original cause stays intact in the document.
|
| 2369 |
+
_BINDING_REFUSAL_DISPLAY = {
|
| 2370 |
+
"database_unreadable": "#NO ACCESS",
|
| 2371 |
+
"field_unknown": "#FIELD?",
|
| 2372 |
+
}
|
| 2373 |
+
|
| 2374 |
+
|
| 2375 |
+
def _binding_refusal(code, message):
|
| 2376 |
+
"""One refusal shape for compute, refresh and caller-scoped reads of metric cells."""
|
| 2377 |
+
return {
|
| 2378 |
+
"state": "refused",
|
| 2379 |
+
"value": None,
|
| 2380 |
+
"display": _BINDING_REFUSAL_DISPLAY.get(code, "#ERROR!"),
|
| 2381 |
+
"refusal": {"code": code, "message": message},
|
| 2382 |
+
}
|
| 2383 |
+
|
| 2384 |
+
|
| 2385 |
+
def _resolve_cell_binding(session, binding, book, tab, **kwargs):
|
| 2386 |
+
"""Keep the legacy resolver calling convention for bindings without workbook inputs."""
|
| 2387 |
+
if _has_cell_inputs(binding):
|
| 2388 |
+
return _resolve_binding(session, binding, book=book, tab=tab, **kwargs)
|
| 2389 |
+
return _resolve_binding(session, binding, **kwargs)
|
| 2390 |
+
|
| 2391 |
+
|
| 2392 |
+
def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
|
| 2393 |
+
book=None, tab=None, ending_seen=()):
|
| 2394 |
"""Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
|
| 2395 |
|
| 2396 |
The order below is the contract, not a preference:
|
|
|
|
| 2418 |
raise err(400, exc.code, exc.message)
|
| 2419 |
|
| 2420 |
database = _require_database(session, binding["database"])
|
| 2421 |
+
try:
|
| 2422 |
+
window = _ending_window(session, binding, book=book, tab=tab, today=today, seen=ending_seen)
|
| 2423 |
+
except sheet_mod.SheetRefusal as exc:
|
| 2424 |
+
raise err(400, exc.code, exc.message)
|
| 2425 |
# W59-T14 / R10: the per-database wall is contract step 1. A metric key must not be looked
|
| 2426 |
# up or acted on before the caller passes it; the ordinary column guards below still apply.
|
| 2427 |
if binding["aggregate"].get("metric_key"):
|
|
|
|
| 2443 |
scope.get("view_name", ""), memo=view_memo)
|
| 2444 |
else:
|
| 2445 |
scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
|
| 2446 |
+
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
|
| 2447 |
+
today=today, seen=ending_seen)
|
| 2448 |
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
| 2449 |
| sheet_mod.filter_columns_named(wall_nodes))
|
| 2450 |
if binding["aggregate"].get("field"):
|
| 2451 |
needed.add(str(binding["aggregate"]["field"]))
|
| 2452 |
|
| 2453 |
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 2454 |
+
cols = dict(cols)
|
| 2455 |
+
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
|
| 2456 |
|
| 2457 |
# ⛔ THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
|
| 2458 |
# real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
|
|
|
|
| 2550 |
field through the database's ordinary field writer; reusing a column needs no field write.
|
| 2551 |
"""
|
| 2552 |
raw = dict(body or {})
|
| 2553 |
+
binding = raw.get("binding") or {}
|
| 2554 |
+
book = None
|
| 2555 |
+
if _has_cell_inputs(binding):
|
| 2556 |
+
from core import sheets
|
| 2557 |
+
try:
|
| 2558 |
+
book = sheets.get_workbook(_store(session), workbook_id)
|
| 2559 |
+
except sheets.SheetRefusal as exc:
|
| 2560 |
+
raise _refuse(exc)
|
| 2561 |
+
out = _resolve_cell_binding(session, binding, book, raw.get("tab"))
|
| 2562 |
return {"value": out["value"], "display": "" if out["value"] is None else str(out["value"]),
|
| 2563 |
"limits": out["limits"]}
|
| 2564 |
|
|
|
|
| 2685 |
raise err(404, "cell_missing", f"Cell {key} is not filled in.")
|
| 2686 |
if cell.get("kind") != "bound":
|
| 2687 |
raise err(409, "cell_not_bound", f"Cell {key} is not bound to a database.")
|
| 2688 |
+
if _is_view_binding(cell):
|
| 2689 |
+
# C2: a placed view has rows, not a value. Refusing BY NAME beats resolving it as a metric,
|
| 2690 |
+
# which is what painted "That cell's source is not set up correctly." over a good view.
|
| 2691 |
+
raise err(409, "cell_is_a_view",
|
| 2692 |
+
f"Cell {key} holds a view, so it is read through its own rows door.")
|
| 2693 |
|
| 2694 |
tab_ref = str((tab or {}).get("id") or (tab or {}).get("name") or "")
|
| 2695 |
binding = cell.get("binding") or {}
|
|
|
|
| 2697 |
saved = dict(cell)
|
| 2698 |
limits, resolved = [], None
|
| 2699 |
try:
|
| 2700 |
+
resolved = _resolve_cell_binding(session, binding, book, tab_ref)
|
| 2701 |
except HTTPException as exc:
|
| 2702 |
#: The same per-cell refusal shape `refresh_sheet` writes, so a cell refused by this door
|
| 2703 |
#: and a cell refused by that one read identically. It is PERSISTED rather than raised:
|
| 2704 |
#: a cell left `stale` would shimmer forever, and the reason belongs on the cell.
|
| 2705 |
detail = getattr(exc, "detail", None) or {}
|
| 2706 |
info = detail.get("error") if isinstance(detail, dict) else None
|
| 2707 |
+
saved.update(_binding_refusal(
|
| 2708 |
+
(info or {}).get("code", "sheet_unresolved"),
|
| 2709 |
+
(info or {}).get("message", "This cell could not be refreshed."),
|
| 2710 |
+
))
|
|
|
|
|
|
|
|
|
|
| 2711 |
else:
|
| 2712 |
limits = resolved["limits"]
|
| 2713 |
saved["state"] = "fresh"
|
|
|
|
| 2717 |
saved.pop("refusal", None)
|
| 2718 |
|
| 2719 |
try:
|
| 2720 |
+
landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved},
|
| 2721 |
+
actor=session.uname,
|
| 2722 |
+
actor_name=session.user.get("name") or session.uname)
|
| 2723 |
except sheets.SheetRefusal as exc:
|
| 2724 |
raise _refuse(exc)
|
| 2725 |
|
|
|
|
| 2743 |
return {"cell": wire, "rev": landed.get("rev"), "tab": tab_ref, "addr": key}
|
| 2744 |
|
| 2745 |
|
| 2746 |
+
@router.post("/sheets/{workbook_id}/view-rows")
|
| 2747 |
+
def view_rows(workbook_id: str, body: object = Body(default=None),
|
| 2748 |
+
session: Session = Depends(require_session)):
|
| 2749 |
+
"""W62 C2 / R29: read the live rows of a placed view under this caller's walls."""
|
| 2750 |
+
from datetime import date
|
| 2751 |
+
from fastapi.responses import JSONResponse
|
| 2752 |
+
from core import perm_scope, sheets, user_tables
|
| 2753 |
+
from modules import sheet as sheet_mod
|
| 2754 |
+
|
| 2755 |
+
try:
|
| 2756 |
+
binding = sheets.clean_view_binding(body.get("binding") if isinstance(body, dict) else None)
|
| 2757 |
+
except sheets.SheetRefusal as exc:
|
| 2758 |
+
return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message})
|
| 2759 |
+
try:
|
| 2760 |
+
book = sheets.get_workbook(_store(session), workbook_id)
|
| 2761 |
+
except sheets.SheetRefusal as exc:
|
| 2762 |
+
raise _refuse(exc)
|
| 2763 |
+
_require_reader(session, book)
|
| 2764 |
+
database = _require_database(session, binding["database"])
|
| 2765 |
+
fields = list(_registry_assembly(session, database).get("fields") or [])
|
| 2766 |
+
hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime)
|
| 2767 |
+
visible = {field["key"]: field for field in fields if field.get("key") not in hidden}
|
| 2768 |
+
requested = binding["fields"]
|
| 2769 |
+
sorted_keys = {item["field"] for item in binding["sort"]}
|
| 2770 |
+
for key in dict.fromkeys([*requested, *sorted(sorted_keys)]):
|
| 2771 |
+
if key not in visible:
|
| 2772 |
+
return JSONResponse(status_code=400, content={
|
| 2773 |
+
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
|
| 2774 |
+
|
| 2775 |
+
# W62 C2: "Filter and sort apply to ALL rows BEFORE limit. total counts every matching row."
|
| 2776 |
+
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 2777 |
+
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 2778 |
+
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 2779 |
+
scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
|
| 2780 |
+
# W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
|
| 2781 |
+
for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
|
| 2782 |
+
if not key.startswith(sheet_mod.LINKED_FILTER_PREFIX) and key not in visible:
|
| 2783 |
+
return JSONResponse(status_code=400, content={
|
| 2784 |
+
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
|
| 2785 |
+
today = date.today()
|
| 2786 |
+
try:
|
| 2787 |
+
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, today=today)
|
| 2788 |
+
window = _ending_window(session, binding, book=book, today=today)
|
| 2789 |
+
needed = (set(requested) | sorted_keys | sheet_mod.filter_columns_named(scope_nodes)
|
| 2790 |
+
| sheet_mod.filter_columns_named(wall_nodes))
|
| 2791 |
+
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 2792 |
+
cols = dict(cols)
|
| 2793 |
+
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
|
| 2794 |
+
for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
|
| 2795 |
+
if key in hidden or not (cols.get(key) or {}).get("sql"):
|
| 2796 |
+
return JSONResponse(status_code=400, content={
|
| 2797 |
+
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
|
| 2798 |
+
# W62 C2: an unanswerable row wall must refuse before either SQL read can widen it.
|
| 2799 |
+
wall_compiled = sheet_mod.compile_scope(wall_nodes, wall_conj, cols, today=today)
|
| 2800 |
+
if sheet_mod.widen_limits(wall_nodes, cols, wall_compiled):
|
| 2801 |
+
raise ValueError("unapplied row wall")
|
| 2802 |
+
count_sql, params, limits = sheet_mod.build_aggregate_query(
|
| 2803 |
+
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 2804 |
+
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today)
|
| 2805 |
+
if limits:
|
| 2806 |
+
raise ValueError("unapplied view filter")
|
| 2807 |
+
except sheet_mod.SheetRefusal as exc:
|
| 2808 |
+
return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message})
|
| 2809 |
+
except ValueError:
|
| 2810 |
+
raise err(409, "scope_not_applicable",
|
| 2811 |
+
"This view uses a restriction that cannot be applied to this database's live connection.")
|
| 2812 |
+
|
| 2813 |
+
# W62 C2 / standing rule 1: "limit caps what is PLACED"; the matching count is unlimited.
|
| 2814 |
+
total = (cur.execute(count_sql, params).fetchone() or [0])[0]
|
| 2815 |
+
# ⭐⭐ W62-T14 — EVERY ROW CARRIES ITS RECORD ID, ALWAYS, AND IT IS NOT A PROJECTED FIELD.
|
| 2816 |
+
# A placed Grid over one of our own tables writes a record back, and the write door is Data's
|
| 2817 |
+
# own `PATCH /api/v1/tables/{database}/rows/{pid}`: without the id the client can only guess
|
| 2818 |
+
# from a row's POSITION in the sheet, which is wrong the moment anyone sorts, filters or
|
| 2819 |
+
# refreshes. So the id rides beside the values under the key `pid` while `fields` still names
|
| 2820 |
+
# only the columns the sheet paints. ⛔ It is the SAME `spec["id"]` expression `build_ids_query`
|
| 2821 |
+
# uses, so the identity here and the identity the drill lists can never be two different things.
|
| 2822 |
+
id_expr = str(spec.get("id") or "").strip()
|
| 2823 |
+
projection = ", ".join([*(cols[key]["sql"] for key in requested),
|
| 2824 |
+
*([id_expr] if id_expr else [])])
|
| 2825 |
+
rows_sql = count_sql.replace("SELECT COUNT(*) FROM", f"SELECT {projection} FROM", 1)
|
| 2826 |
+
if binding["sort"]:
|
| 2827 |
+
# W62 C2 / R30: expressions come only from resolved columns; directions are validated.
|
| 2828 |
+
order = ", ".join(cols[item["field"]]["sql"] + (" ASC" if item["dir"] == "asc" else " DESC")
|
| 2829 |
+
for item in binding["sort"])
|
| 2830 |
+
rows_sql += " ORDER BY " + order
|
| 2831 |
+
rows_sql += f" LIMIT {binding['limit']}"
|
| 2832 |
+
keys = [*requested, *(["pid"] if id_expr else [])]
|
| 2833 |
+
rows = [dict(zip(keys, row)) for row in cur.execute(rows_sql, params).fetchall()]
|
| 2834 |
+
# W62 C2 / R34: Data's _records_or_refuse uses database access (checked above) plus
|
| 2835 |
+
# W62 R34: records_mutable, with no separate edit-role grant; its row/field walls still apply.
|
| 2836 |
+
editable = user_tables.records_mutable(database, st=session.runtime)
|
| 2837 |
+
return {"fields": [{"key": key, "label": visible[key].get("label") or key,
|
| 2838 |
+
"kind": visible[key].get("type") or visible[key].get("kind") or "text"}
|
| 2839 |
+
for key in requested],
|
| 2840 |
+
"rows": rows, "total": int(total or 0), "limit": binding["limit"],
|
| 2841 |
+
"viewType": binding["viewType"], "editable": bool(editable)}
|
| 2842 |
+
|
| 2843 |
+
|
| 2844 |
@router.get("/sheets/{workbook_id}/cells/{addr}/drill")
|
| 2845 |
def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_session)):
|
| 2846 |
"""`{database, view_id?, filter?, record_ids, count}` for `LinkGridModal` (R6, R13, R17).
|
|
|
|
| 2876 |
if cell.get("kind") != "bound":
|
| 2877 |
raise err(409, "literal_no_drill",
|
| 2878 |
"A hand typed value has no rows behind it.")
|
| 2879 |
+
if _is_view_binding(cell):
|
| 2880 |
+
raise err(409, "cell_is_a_view",
|
| 2881 |
+
f"Cell {key} holds a view, so its rows are already on the sheet.")
|
| 2882 |
|
| 2883 |
try:
|
| 2884 |
binding = sheet_mod.clean_binding(cell.get("binding"))
|
|
|
|
| 2911 |
# `_metric_derived` at a month window is 27 ms of an 8,437 ms drill on `customer_data`. So the
|
| 2912 |
# window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from
|
| 2913 |
# the column set instead, where the time actually is.
|
| 2914 |
+
try:
|
| 2915 |
+
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab.get("id"))
|
| 2916 |
+
window = _ending_window(session, binding, book=book, tab=tab.get("id"))
|
| 2917 |
+
except sheet_mod.SheetRefusal as exc:
|
| 2918 |
+
raise err(400, exc.code, exc.message)
|
| 2919 |
cur, spec, cols = _mirror(session)(
|
| 2920 |
+
database, metric_window=window,
|
| 2921 |
needed=(sheet_mod.filter_columns_named(scope_nodes)
|
| 2922 |
| sheet_mod.filter_columns_named(wall_nodes)))
|
| 2923 |
+
cols = dict(cols)
|
| 2924 |
+
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols)
|
| 2925 |
|
| 2926 |
+
try:
|
| 2927 |
+
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
|
| 2928 |
+
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 2929 |
+
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj)
|
| 2930 |
+
except sheet_mod.SheetRefusal as exc:
|
| 2931 |
+
raise err(400, exc.code, exc.message)
|
| 2932 |
total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
|
| 2933 |
|
| 2934 |
ids_sql, ids_params, _ = sheet_mod.build_ids_query(
|
|
|
|
| 2992 |
@router.post("/sheets/{workbook_id}/resolve-bindings")
|
| 2993 |
def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
|
| 2994 |
session: Session = Depends(require_session)):
|
| 2995 |
+
"""Values or named refusals for the NAMED bound cells only (contract C1, rulings R2 and R48).
|
| 2996 |
|
| 2997 |
⭐⭐ THIS IS THE DOOR THE COMMIT PATH USES, AND ITS WHOLE POINT IS THAT IT IS SMALL.
|
| 2998 |
Under R2 a formula evaluates in the browser the instant it is committed, so the only thing the
|
|
|
|
| 3008 |
by THE EXACT STRING THE CALLER ASKED WITH, so a client looks its answer up under the
|
| 3009 |
reference it already holds and never has to re-derive this file's key spelling.
|
| 3010 |
|
| 3011 |
+
A KEY IS ABSENT WHEN NO BOUND CELL LIVES AT THAT ADDRESS. An address that is empty, holds a
|
| 3012 |
+
typed literal or holds a formula is not this door's business - lane A's evaluator already
|
| 3013 |
+
has those values in hand - so asking for one costs nothing and answers nothing.
|
| 3014 |
+
|
| 3015 |
+
R48: a refused binding answers `{value: null, refusal: {code, message}}`, so it cannot be
|
| 3016 |
+
mistaken for a legitimate empty aggregate. The flat shape is kept for every address that
|
| 3017 |
+
has an answer, including a genuine null, so no existing reader of those answers changes.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3018 |
|
| 3019 |
⛔ ONE BAD CELL NEVER FAILS THE BATCH, exactly as in `refresh_sheet`'s bound-cell loop: a
|
| 3020 |
refusal is caught per cell and the siblings still answer.
|
|
|
|
| 3076 |
if cell is None:
|
| 3077 |
continue
|
| 3078 |
binding = cell.get("binding") or {}
|
| 3079 |
+
own_tab = _asked_key(label, home, fr).rpartition("!")[0]
|
| 3080 |
+
slot = json.dumps([own_tab, binding], sort_keys=True, default=str)
|
| 3081 |
if slot not in by_binding:
|
| 3082 |
try:
|
| 3083 |
+
by_binding[slot] = _resolve_cell_binding(session, binding, book, own_tab,
|
| 3084 |
+
view_memo=view_memo)["value"]
|
| 3085 |
+
except HTTPException as exc:
|
| 3086 |
+
detail = getattr(exc, "detail", None) or {}
|
| 3087 |
+
info = detail.get("error") if isinstance(detail, dict) else None
|
| 3088 |
+
by_binding[slot] = {
|
| 3089 |
+
"value": None,
|
| 3090 |
+
"refusal": {
|
| 3091 |
+
"code": (info or {}).get("code", "sheet_unresolved"),
|
| 3092 |
+
"message": (info or {}).get("message", "This cell could not be refreshed."),
|
| 3093 |
+
},
|
| 3094 |
+
}
|
| 3095 |
answer[label] = by_binding[slot]
|
| 3096 |
return answer
|
api/routes_starred.py
CHANGED
|
@@ -28,9 +28,11 @@ disagree — a view deleted, a view unmarked from inside the database, a store w
|
|
| 28 |
and not the other — the product has two answers to one question and no way to tell which is right.
|
| 29 |
The scan is the price of one flag, and it is paid off the critical path.
|
| 30 |
"""
|
|
|
|
| 31 |
import hashlib
|
| 32 |
import json
|
| 33 |
import time
|
|
|
|
| 34 |
|
| 35 |
from fastapi import APIRouter, Body, Depends
|
| 36 |
|
|
@@ -39,17 +41,17 @@ from deps import Session, err, require_session
|
|
| 39 |
|
| 40 |
router = APIRouter(prefix="/api/v1")
|
| 41 |
|
| 42 |
-
#: The per-user bucket holding
|
| 43 |
#:
|
| 44 |
#: ⚠ PER USER (R6), like `nav_recents` and unlike `nav_meta`: what I care about is not a fact about
|
| 45 |
#: the object, so a tenant-wide star would put one person's shortlist on everybody's Home.
|
| 46 |
_STARRED_KEY = "starred"
|
| 47 |
|
| 48 |
-
#:
|
| 49 |
-
KINDS = ("database", "agent", "query", "view", "application")
|
| 50 |
|
| 51 |
-
#: The
|
| 52 |
-
_LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries"}
|
| 53 |
|
| 54 |
#: A bound on each id list. ⛔ REPORTED, NEVER SILENT: a POST that would cross it is refused with
|
| 55 |
#: the cause and the action (unstar something), rather than being accepted and truncated. A star
|
|
@@ -65,6 +67,88 @@ _MAX_ID = 120
|
|
| 65 |
#: container starts; warm it is not reached.
|
| 66 |
VIEW_SCAN_BUDGET_S = 2.5
|
| 67 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
|
| 69 |
def _clean_id(raw):
|
| 70 |
"""One id, or "" — the only shape this file stores."""
|
|
@@ -90,7 +174,7 @@ def _clean_list(raw):
|
|
| 90 |
|
| 91 |
|
| 92 |
def _read_lists(session):
|
| 93 |
-
"""This caller's
|
| 94 |
try:
|
| 95 |
mine = (session.runtime.get(_STARRED_KEY) or {}).get(session.uname) or {}
|
| 96 |
except Exception:
|
|
@@ -144,18 +228,56 @@ def _view_docs(session, order):
|
|
| 144 |
import core.view_templates as view_templates
|
| 145 |
docs, unread = {}, []
|
| 146 |
started = time.perf_counter()
|
|
|
|
| 147 |
for key in order:
|
| 148 |
ws_key = view_templates.workspace_key(key)
|
| 149 |
if not ws_key:
|
| 150 |
continue # a module surface or a folder head, not a table at all
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
try:
|
| 155 |
-
|
| 156 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
except Exception:
|
| 158 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
return docs, unread
|
| 160 |
|
| 161 |
|
|
@@ -207,8 +329,25 @@ def _scan_starred_views(session, keys=None, cache=None):
|
|
| 207 |
import core.table_store as table_store
|
| 208 |
import core.view_templates as view_templates
|
| 209 |
|
| 210 |
-
granted = routes_nav.
|
| 211 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
for key in order:
|
| 213 |
doc = docs.get(key)
|
| 214 |
if doc is None:
|
|
@@ -217,8 +356,43 @@ def _scan_starred_views(session, keys=None, cache=None):
|
|
| 217 |
if not ws_key:
|
| 218 |
continue
|
| 219 |
views = table_store.make(ws_key, st=session.runtime)
|
| 220 |
-
|
| 221 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 222 |
cfg = view.get("config") or {}
|
| 223 |
if cfg.get("important") is True:
|
| 224 |
out.append({"id": vid, "database": key,
|
|
@@ -289,13 +463,14 @@ def _query_names(session, qids):
|
|
| 289 |
|
| 290 |
|
| 291 |
def _payload(session, keys=None, cache=None):
|
| 292 |
-
"""The
|
| 293 |
|
| 294 |
⚠ `degraded` SHIPS AS `[]` RATHER THAN BEING OMITTED WHEN EMPTY — the same rule `/nav`'s
|
| 295 |
`omitted`/`degraded` follow, for the same reason: a key a consumer has to test for is a key a
|
| 296 |
consumer forgets to test for.
|
| 297 |
⚠ `cache` is a scan an earlier pass in THIS request already paid for — see `_scan_starred_views`.
|
| 298 |
"""
|
|
|
|
| 299 |
lists = _read_lists(session)
|
| 300 |
if keys is None:
|
| 301 |
keys, _ = _database_keys(session)
|
|
@@ -307,6 +482,8 @@ def _payload(session, keys=None, cache=None):
|
|
| 307 |
# validate, and pruning them here buys nothing: this list is the CALLER'S OWN writes, so it can
|
| 308 |
# disclose nothing they did not already know, and B/C render only objects they can already see
|
| 309 |
# (the way `groupRecents` drops an unreachable recent). Said out loud rather than left as a gap.
|
|
|
|
|
|
|
| 310 |
lists["databases"] = [k for k in lists["databases"] if k in keys]
|
| 311 |
# ⭐ ASK B-1 (2): the names, and — because the read that answers "what is it called" also answers
|
| 312 |
# "does it still exist" — the same prune the databases get. ⛔ ONLY WHEN THE READ SUCCEEDED: a
|
|
@@ -327,7 +504,7 @@ def _payload(session, keys=None, cache=None):
|
|
| 327 |
except Exception:
|
| 328 |
application_rows = []
|
| 329 |
out = {"databases": lists["databases"], "agents": lists["agents"],
|
| 330 |
-
"queries": lists["queries"], "queryNames": qnames,
|
| 331 |
"views": views, "applications": application_rows, "degraded": unread}
|
| 332 |
if unread:
|
| 333 |
# R6's second sentence, in the payload: the CAUSE and what it means, not a bare list.
|
|
@@ -503,7 +680,7 @@ def set_starred(body: dict = Body(default=None),
|
|
| 503 |
session: Session = Depends(require_session)):
|
| 504 |
"""C2 — star or unstar ONE object, and answer with the caller's whole new list.
|
| 505 |
|
| 506 |
-
Body: `{"kind": "database"|"agent"|"query"|"view", "id": str, "on": bool}`, plus an OPTIONAL
|
| 507 |
`"database"` when `kind` is `view` (the fast path — see `_find_view`).
|
| 508 |
|
| 509 |
⚠ THE WHOLE LIST COMES BACK, not an ack, because the client is optimistic-then-reconciled
|
|
|
|
| 28 |
and not the other — the product has two answers to one question and no way to tell which is right.
|
| 29 |
The scan is the price of one flag, and it is paid off the critical path.
|
| 30 |
"""
|
| 31 |
+
import copy
|
| 32 |
import hashlib
|
| 33 |
import json
|
| 34 |
import time
|
| 35 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 36 |
|
| 37 |
from fastapi import APIRouter, Body, Depends
|
| 38 |
|
|
|
|
| 41 |
|
| 42 |
router = APIRouter(prefix="/api/v1")
|
| 43 |
|
| 44 |
+
#: The per-user bucket holding ID-LIST kinds. `{username: {databases, agents, queries, sheets}}`
|
| 45 |
#:
|
| 46 |
#: ⚠ PER USER (R6), like `nav_recents` and unlike `nav_meta`: what I care about is not a fact about
|
| 47 |
#: the object, so a tenant-wide star would put one person's shortlist on everybody's Home.
|
| 48 |
_STARRED_KEY = "starred"
|
| 49 |
|
| 50 |
+
#: Accepted star kinds. Views and applications keep their own record-backed write paths.
|
| 51 |
+
KINDS = ("database", "agent", "query", "view", "application", "sheet")
|
| 52 |
|
| 53 |
+
#: The kinds this file's own bucket holds, and the payload key each answers under.
|
| 54 |
+
_LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries", "sheet": "sheets"}
|
| 55 |
|
| 56 |
#: A bound on each id list. ⛔ REPORTED, NEVER SILENT: a POST that would cross it is refused with
|
| 57 |
#: the cause and the action (unstar something), rather than being accepted and truncated. A star
|
|
|
|
| 67 |
#: container starts; warm it is not reached.
|
| 68 |
VIEW_SCAN_BUDGET_S = 2.5
|
| 69 |
|
| 70 |
+
# Four projected reads fit beneath the store pool's eight-connection ceiling while removing the
|
| 71 |
+
# serial N-read wall from Home. This is deliberately bounded: a tenant with many databases must
|
| 72 |
+
# not turn one Starred request into an unbounded connection burst.
|
| 73 |
+
VIEW_SCAN_WORKERS = 4
|
| 74 |
+
|
| 75 |
+
# One bounded process-wide pool, rather than one executor per authenticated request. A timed-out
|
| 76 |
+
# Python thread cannot be killed safely; a per-request executor therefore let repeated slow store
|
| 77 |
+
# calls grow one live worker apiece until the backend returned. The shared pool keeps that failure
|
| 78 |
+
# bounded to VIEW_SCAN_WORKERS and `future.cancel()` removes work that has not started yet.
|
| 79 |
+
_BATCH_READ_POOL = ThreadPoolExecutor(
|
| 80 |
+
max_workers=VIEW_SCAN_WORKERS,
|
| 81 |
+
thread_name_prefix="starred-batch",
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class _RequestReadMemo:
|
| 86 |
+
"""Read-through snapshot for one Starred payload assembly, never a cross-request cache."""
|
| 87 |
+
|
| 88 |
+
def __init__(self, runtime):
|
| 89 |
+
self._runtime = runtime
|
| 90 |
+
self._docs = {}
|
| 91 |
+
self._projections = {}
|
| 92 |
+
|
| 93 |
+
def preload(self, names):
|
| 94 |
+
reader = getattr(self._runtime, "get_many", None)
|
| 95 |
+
if not callable(reader):
|
| 96 |
+
return
|
| 97 |
+
try:
|
| 98 |
+
answer = reader(names)
|
| 99 |
+
except Exception:
|
| 100 |
+
return
|
| 101 |
+
if isinstance(answer, dict):
|
| 102 |
+
self._docs.update(answer)
|
| 103 |
+
|
| 104 |
+
def get(self, name, fresh=False):
|
| 105 |
+
key = str(name)
|
| 106 |
+
if not fresh and key in self._docs:
|
| 107 |
+
return copy.deepcopy(self._docs[key])
|
| 108 |
+
answer = self._runtime.get(name, fresh=fresh)
|
| 109 |
+
if not fresh:
|
| 110 |
+
self._docs[key] = copy.deepcopy(answer)
|
| 111 |
+
return answer
|
| 112 |
+
|
| 113 |
+
def get_projection(self, name, drop=()):
|
| 114 |
+
stamp = (str(name), tuple(str(key) for key in drop))
|
| 115 |
+
if stamp not in self._projections:
|
| 116 |
+
self._projections[stamp] = self._runtime.get_projection(name, drop=drop)
|
| 117 |
+
return copy.deepcopy(self._projections[stamp])
|
| 118 |
+
|
| 119 |
+
def get_projections(self, names, drop=(), timeout_s=None):
|
| 120 |
+
keys = [str(name) for name in names or ()]
|
| 121 |
+
stamps = [(key, tuple(str(field) for field in drop)) for key in keys]
|
| 122 |
+
if all(stamp in self._projections for stamp in stamps):
|
| 123 |
+
return {key: copy.deepcopy(self._projections[stamp])
|
| 124 |
+
for key, stamp in zip(keys, stamps)}
|
| 125 |
+
reader = getattr(self._runtime, "get_projections", None)
|
| 126 |
+
if not callable(reader):
|
| 127 |
+
return None
|
| 128 |
+
answer = reader(keys, drop=drop, timeout_s=timeout_s)
|
| 129 |
+
if answer is None:
|
| 130 |
+
return None
|
| 131 |
+
for key, stamp in zip(keys, stamps):
|
| 132 |
+
self._projections[stamp] = copy.deepcopy(answer.get(key, {}))
|
| 133 |
+
return {key: copy.deepcopy(self._projections[stamp])
|
| 134 |
+
for key, stamp in zip(keys, stamps)}
|
| 135 |
+
|
| 136 |
+
def __getattr__(self, name):
|
| 137 |
+
return getattr(self._runtime, name)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _memoized_session(session):
|
| 141 |
+
"""Preload the four small registries Starred rereads while assembling one response."""
|
| 142 |
+
import application_registry
|
| 143 |
+
from core import shares
|
| 144 |
+
|
| 145 |
+
runtime = _RequestReadMemo(session.runtime)
|
| 146 |
+
runtime.preload((_STARRED_KEY, routes_nav._NAV_RECENTS_KEY,
|
| 147 |
+
shares.SHARES_KEY, application_registry._KEY))
|
| 148 |
+
wrapped = copy.copy(session)
|
| 149 |
+
wrapped.runtime = runtime
|
| 150 |
+
return wrapped
|
| 151 |
+
|
| 152 |
|
| 153 |
def _clean_id(raw):
|
| 154 |
"""One id, or "" — the only shape this file stores."""
|
|
|
|
| 174 |
|
| 175 |
|
| 176 |
def _read_lists(session):
|
| 177 |
+
"""This caller's id lists. A store blip answers empty rather than taking the page down."""
|
| 178 |
try:
|
| 179 |
mine = (session.runtime.get(_STARRED_KEY) or {}).get(session.uname) or {}
|
| 180 |
except Exception:
|
|
|
|
| 228 |
import core.view_templates as view_templates
|
| 229 |
docs, unread = {}, []
|
| 230 |
started = time.perf_counter()
|
| 231 |
+
readable = []
|
| 232 |
for key in order:
|
| 233 |
ws_key = view_templates.workspace_key(key)
|
| 234 |
if not ws_key:
|
| 235 |
continue # a module surface or a folder head, not a table at all
|
| 236 |
+
readable.append((key, ws_key))
|
| 237 |
+
|
| 238 |
+
batch_reader = getattr(session.runtime, "get_projections", None)
|
| 239 |
+
if readable and callable(batch_reader):
|
| 240 |
+
# The Postgres implementation also installs a statement timeout, but the route keeps its
|
| 241 |
+
# own wall-clock guard: an optional store capability must not silently defeat the scan
|
| 242 |
+
# budget merely by ignoring that hint. The process-wide pool is deliberately bounded;
|
| 243 |
+
# creating one executor here would let repeated timeouts amplify live background workers.
|
| 244 |
+
remaining = max(0.001, VIEW_SCAN_BUDGET_S - (time.perf_counter() - started))
|
| 245 |
+
future = None
|
| 246 |
try:
|
| 247 |
+
future = _BATCH_READ_POOL.submit(
|
| 248 |
+
batch_reader,
|
| 249 |
+
[ws_key for _key, ws_key in readable],
|
| 250 |
+
drop=("overlays", "fields"),
|
| 251 |
+
timeout_s=remaining,
|
| 252 |
+
)
|
| 253 |
+
batch = future.result(timeout=remaining)
|
| 254 |
except Exception:
|
| 255 |
+
# One batch has one availability answer. Reporting every key as unread is honest;
|
| 256 |
+
# falling through to N retries would recreate the latency wall on an unhealthy store.
|
| 257 |
+
if future is not None:
|
| 258 |
+
future.cancel()
|
| 259 |
+
return {}, [key for key, _ws_key in readable]
|
| 260 |
+
if batch is not None:
|
| 261 |
+
for key, ws_key in readable:
|
| 262 |
+
docs[key] = batch.get(ws_key) or {}
|
| 263 |
+
return docs, unread
|
| 264 |
+
|
| 265 |
+
for offset in range(0, len(readable), VIEW_SCAN_WORKERS):
|
| 266 |
+
if time.perf_counter() - started > VIEW_SCAN_BUDGET_S:
|
| 267 |
+
unread.extend(key for key, _ in readable[offset:])
|
| 268 |
+
break
|
| 269 |
+
chunk = readable[offset:offset + VIEW_SCAN_WORKERS]
|
| 270 |
+
with ThreadPoolExecutor(max_workers=max(1, len(chunk))) as pool:
|
| 271 |
+
futures = [pool.submit(session.runtime.get_projection, ws_key,
|
| 272 |
+
drop=("overlays", "fields"))
|
| 273 |
+
for _key, ws_key in chunk]
|
| 274 |
+
# Consume in request order so recents-first remains the response order even when a
|
| 275 |
+
# later store read finishes first.
|
| 276 |
+
for (key, _ws_key), future in zip(chunk, futures):
|
| 277 |
+
try:
|
| 278 |
+
docs[key] = future.result() or {}
|
| 279 |
+
except Exception:
|
| 280 |
+
unread.append(key)
|
| 281 |
return docs, unread
|
| 282 |
|
| 283 |
|
|
|
|
| 329 |
import core.table_store as table_store
|
| 330 |
import core.view_templates as view_templates
|
| 331 |
|
| 332 |
+
granted = routes_nav._granted_view_sources(session)
|
| 333 |
+
# ⭐⭐ W62-T17 (D-756) — EVERY GRANTED VIEW ID WAS RESOLVED ONCE PER DATABASE, and that loop is
|
| 334 |
+
# where `/starred`'s budget went. MEASURED on staging's Royal Imports, 26 databases
|
| 335 |
+
# (`AutoWiki/waves/wave62/proto/T17-profile.md`): 4,844 ms total, of which `_visible_views` was
|
| 336 |
+
# 3,075 ms — `object_shares` read 28 times (1,117 ms) and `TableStore.find_view` 26 times
|
| 337 |
+
# (2,072 ms), for ONE granted id. Neither figure is about a slow read: the grant registry is
|
| 338 |
+
# 8 KB and each read is a single ~40 ms round trip. They are about the same question being
|
| 339 |
+
# asked once per database.
|
| 340 |
+
#
|
| 341 |
+
# ⭐ A VIEW LIVES IN EXACTLY ONE DATABASE AND ITS GRANT NAMES ITS OWNER. The modern fast path
|
| 342 |
+
# resolves that owner directly in the projected document this scan already paid for; it never
|
| 343 |
+
# re-reads the grant registry or calls `find_view` for every database. `resolved` still removes
|
| 344 |
+
# a found id from every later database's lookup work.
|
| 345 |
+
#
|
| 346 |
+
# ⛔ THIS IS NOT A LENT `object_shares` SNAPSHOT. `TableStore.find_view` reaches the legacy
|
| 347 |
+
# migration, which WRITES grants and reads them back; lending that key across the migration
|
| 348 |
+
# would serve a pre-write value and raise. The branch below detects retired shapes in the
|
| 349 |
+
# document already held, runs the unchanged strict migration, and refreshes that one document.
|
| 350 |
+
resolved, out = set(), []
|
| 351 |
for key in order:
|
| 352 |
doc = docs.get(key)
|
| 353 |
if doc is None:
|
|
|
|
| 356 |
if not ws_key:
|
| 357 |
continue
|
| 358 |
views = table_store.make(ws_key, st=session.runtime)
|
| 359 |
+
pending = set(granted) - resolved
|
| 360 |
+
|
| 361 |
+
# The old path called `find_view` once for every pending grant in every database. Each
|
| 362 |
+
# call strictly re-read this same workspace document and the whole share registry. The
|
| 363 |
+
# projected document above already contains every personal View, so modern grants can
|
| 364 |
+
# resolve directly through their canonical owner. A legacy document still takes the old
|
| 365 |
+
# strict migration door and is re-read afterwards; the optimization never skips a write.
|
| 366 |
+
if pending and views.prepare_view_projection(doc):
|
| 367 |
+
doc = session.runtime.get_projection(ws_key, drop=("overlays", "fields")) or {}
|
| 368 |
+
docs[key] = doc
|
| 369 |
+
|
| 370 |
+
found = {}
|
| 371 |
+
own = doc.get(session.uname)
|
| 372 |
+
for vid, view in ((own.get("views") or {}) if isinstance(own, dict) else {}).items():
|
| 373 |
+
if isinstance(view, dict):
|
| 374 |
+
found[str(vid)] = view
|
| 375 |
+
|
| 376 |
+
for vid in pending:
|
| 377 |
+
owner = granted.get(vid)
|
| 378 |
+
view = None
|
| 379 |
+
if owner:
|
| 380 |
+
owner_ws = doc.get(owner)
|
| 381 |
+
view = ((owner_ws.get("views") or {}).get(vid)
|
| 382 |
+
if isinstance(owner_ws, dict) else None)
|
| 383 |
+
else:
|
| 384 |
+
# Preserve the historical first-source lookup for a legacy ownerless grant.
|
| 385 |
+
for candidate_owner, workspace in doc.items():
|
| 386 |
+
if candidate_owner in table_store.RESERVED_MEMBERS or not isinstance(workspace, dict):
|
| 387 |
+
continue
|
| 388 |
+
candidate = (workspace.get("views") or {}).get(vid)
|
| 389 |
+
if isinstance(candidate, dict):
|
| 390 |
+
view = candidate
|
| 391 |
+
break
|
| 392 |
+
if isinstance(view, dict):
|
| 393 |
+
found.setdefault(vid, view)
|
| 394 |
+
resolved.add(vid)
|
| 395 |
+
for vid, view in found.items():
|
| 396 |
cfg = view.get("config") or {}
|
| 397 |
if cfg.get("important") is True:
|
| 398 |
out.append({"id": vid, "database": key,
|
|
|
|
| 463 |
|
| 464 |
|
| 465 |
def _payload(session, keys=None, cache=None):
|
| 466 |
+
"""The star answer: the caller's id lists, starred records, and what could not be read.
|
| 467 |
|
| 468 |
⚠ `degraded` SHIPS AS `[]` RATHER THAN BEING OMITTED WHEN EMPTY — the same rule `/nav`'s
|
| 469 |
`omitted`/`degraded` follow, for the same reason: a key a consumer has to test for is a key a
|
| 470 |
consumer forgets to test for.
|
| 471 |
⚠ `cache` is a scan an earlier pass in THIS request already paid for — see `_scan_starred_views`.
|
| 472 |
"""
|
| 473 |
+
session = _memoized_session(session)
|
| 474 |
lists = _read_lists(session)
|
| 475 |
if keys is None:
|
| 476 |
keys, _ = _database_keys(session)
|
|
|
|
| 482 |
# validate, and pruning them here buys nothing: this list is the CALLER'S OWN writes, so it can
|
| 483 |
# disclose nothing they did not already know, and B/C render only objects they can already see
|
| 484 |
# (the way `groupRecents` drops an unreachable recent). Said out loud rather than left as a gap.
|
| 485 |
+
# Sheets are also the CALLER'S OWN writes. R28 filters the sheet index; C2 forbids a second
|
| 486 |
+
# access check here, so sheet ids pass through unchanged.
|
| 487 |
lists["databases"] = [k for k in lists["databases"] if k in keys]
|
| 488 |
# ⭐ ASK B-1 (2): the names, and — because the read that answers "what is it called" also answers
|
| 489 |
# "does it still exist" — the same prune the databases get. ⛔ ONLY WHEN THE READ SUCCEEDED: a
|
|
|
|
| 504 |
except Exception:
|
| 505 |
application_rows = []
|
| 506 |
out = {"databases": lists["databases"], "agents": lists["agents"],
|
| 507 |
+
"queries": lists["queries"], "sheets": lists["sheets"], "queryNames": qnames,
|
| 508 |
"views": views, "applications": application_rows, "degraded": unread}
|
| 509 |
if unread:
|
| 510 |
# R6's second sentence, in the payload: the CAUSE and what it means, not a bare list.
|
|
|
|
| 680 |
session: Session = Depends(require_session)):
|
| 681 |
"""C2 — star or unstar ONE object, and answer with the caller's whole new list.
|
| 682 |
|
| 683 |
+
Body: `{"kind": "database"|"agent"|"query"|"view"|"application"|"sheet", "id": str, "on": bool}`, plus an OPTIONAL
|
| 684 |
`"database"` when `kind` is `view` (the fast path — see `_find_view`).
|
| 685 |
|
| 686 |
⚠ THE WHOLE LIST COMES BACK, not an ack, because the client is optimistic-then-reconciled
|
api/routes_tables.py
CHANGED
|
@@ -68,33 +68,45 @@ class _MetricReadRuntime:
|
|
| 68 |
raise AttributeError(name)
|
| 69 |
|
| 70 |
|
| 71 |
-
def _related_product_fields(session, rows):
|
| 72 |
"""Resolve authored Metric definitions through the existing workspace and field wall."""
|
| 73 |
import aios_grid
|
| 74 |
from core import grid_events, perm_scope, shared_overlay
|
| 75 |
from modules import product_data as pd
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
|
| 77 |
pids = frozenset(r["pid"] for r in rows)
|
| 78 |
ctx = grid_events.EventCtx(uname=session.uname, allowed_pids=pids, fields=[],
|
| 79 |
hidden_keys=set(), admin=session.admin, fallback_ws=None,
|
| 80 |
-
seen_ids={}, st=session.runtime, scope_key=
|
| 81 |
-
table=pd.table_ops(session.runtime))
|
| 82 |
ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False)
|
| 83 |
_, fields, _, _ = aios_grid.workspace_wire(
|
| 84 |
-
ws, session.uname, set(pids), defs={}, scope_key=
|
| 85 |
-
fields_base=
|
| 86 |
have = {f["key"] for f in fields}
|
| 87 |
fields = list(fields) + [dict(f, source="overlay", shared=True)
|
| 88 |
-
for key, f in shared_overlay.fields(
|
| 89 |
if key not in have]
|
| 90 |
-
hidden = perm_scope.hidden_keys(session.user,
|
| 91 |
return [f for f in fields if f.get("measure") and f["key"] not in hidden]
|
| 92 |
|
| 93 |
|
| 94 |
@router.post("/tables/{table_key}/related-metrics")
|
| 95 |
def related_metrics(table_key: str, body: dict = Body(default=None),
|
| 96 |
session: Session = Depends(require_session)):
|
| 97 |
-
"""
|
| 98 |
|
| 99 |
The request supplies only grid identities and a named relation. Fields, periods, source
|
| 100 |
identities and values are read from this tenant; nothing is authored by this endpoint.
|
|
@@ -106,66 +118,79 @@ def related_metrics(table_key: str, body: dict = Body(default=None),
|
|
| 106 |
import automation_engine
|
| 107 |
|
| 108 |
body = body if isinstance(body, dict) else {}
|
| 109 |
-
pin = body.get("related")
|
| 110 |
pids = body.get("pids")
|
| 111 |
-
if (
|
| 112 |
-
or pin.get("table") != "customer_data"
|
| 113 |
-
or type(pin.get("record")) is not int or pin["record"] <= 0
|
| 114 |
-
or not isinstance(pids, list) or not pids
|
| 115 |
or any(type(pid) is not int or pid <= 0 for pid in pids)):
|
| 116 |
-
raise err(400, "bad_metric_scope", "Name the origin
|
| 117 |
viewer = copy.copy(session)
|
| 118 |
viewer.runtime = _MetricReadRuntime(session.runtime)
|
| 119 |
try:
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
entries = routes_grid.related_path_entries(viewer, "customer_data", 1)
|
| 124 |
-
via = pin.get("via")
|
| 125 |
-
edge = relations.pick_path(entries, via) if isinstance(via, list) else relations.pick(entries, via)
|
| 126 |
-
if (edge is None or edge.get("target") != "product_data"
|
| 127 |
-
or edge.get("via", {}).get("field") != "products_ordered"):
|
| 128 |
-
raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.")
|
| 129 |
-
source_fields = routes_grid._source_fields(viewer, "customer_data")
|
| 130 |
-
hidden = perm_scope.hidden_keys(viewer.user, "customer_data", source_fields, st=viewer.runtime)
|
| 131 |
-
if "products_ordered" in hidden:
|
| 132 |
-
raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.")
|
| 133 |
-
customers = perm_scope.scoped_table(viewer.user, "customer_data", st=viewer.runtime)
|
| 134 |
-
if not any(r.get("pid") == pin["record"] for r in customers):
|
| 135 |
-
raise err(403, "metric_record_refused", "The origin record is not available to this account.")
|
| 136 |
-
products = perm_scope.scoped_table(viewer.user, "product_data", st=viewer.runtime)
|
| 137 |
requested = set(pids)
|
| 138 |
selected = [r for r in products if r.get("pid") in requested]
|
| 139 |
-
if
|
| 140 |
-
or any(type(r.get("product_id")) is not int or r["product_id"] <= 0 for r in selected)):
|
| 141 |
raise err(403, "metric_record_refused", "The destination records cannot be scoped for this account.")
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
values, reasons = automation_engine.related_product_metric_values(
|
| 157 |
-
viewer.runtime, pin["record"], selected, fields, routes_products.product_measures(viewer),
|
| 158 |
-
team_id=next(iter(teams), None), agent=next(iter(agents), None))
|
| 159 |
return {"ok": True, "values": values, "fields": [f["key"] for f in fields],
|
| 160 |
-
"
|
| 161 |
-
|
| 162 |
-
|
|
|
|
|
|
|
| 163 |
except (perm_scope.Denied, perm_scope.UnknownTable):
|
| 164 |
raise err(403, "metric_record_refused", "The related records are not available to this account.")
|
| 165 |
except (perm_scope.Unresolvable, RuntimeError, ValueError):
|
| 166 |
raise err(409, "metric_scope_unavailable", "Scoped Metrics could not be read. Reopen the related records after the database is ready.")
|
| 167 |
|
| 168 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
def _ut():
|
| 170 |
import core.user_tables as user_tables
|
| 171 |
return user_tables
|
|
@@ -1537,6 +1562,9 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
|
| 1537 |
# for both — the same sentence `routes_odoo_tables` writes over its own `overlays` dict.
|
| 1538 |
shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden}
|
| 1539 |
for pid, cells in shared_cells.items()}
|
|
|
|
|
|
|
|
|
|
| 1540 |
|
| 1541 |
# ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database
|
| 1542 |
# whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real
|
|
@@ -2277,7 +2305,8 @@ def _thin_json(fields, merged, table_key=""):
|
|
| 2277 |
|
| 2278 |
|
| 2279 |
@router.get("/tables/{table_key}/rows")
|
| 2280 |
-
def table_rows(table_key: str, session: Session = Depends(require_session)
|
|
|
|
| 2281 |
"""The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at,
|
| 2282 |
identity}` — so the client's generic topic fetch consumes it with zero new parsing.
|
| 2283 |
|
|
@@ -2299,7 +2328,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session)):
|
|
| 2299 |
# PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the
|
| 2300 |
# value it replaced (contract C5).
|
| 2301 |
_lent = _ut().lend(session.runtime)
|
| 2302 |
-
g = ut_assembly(session, table_key, st=_lent)
|
| 2303 |
# ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge
|
| 2304 |
# this line does not perform since the route was written (owner item 3, 2026-08-09:
|
| 2305 |
# *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*).
|
|
@@ -2361,6 +2390,12 @@ def table_rows(table_key: str, session: Session = Depends(require_session)):
|
|
| 2361 |
# ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/
|
| 2362 |
# `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`.
|
| 2363 |
_stamp_ai_states(table_key, g["fields"], rows, st=_lent)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2364 |
# ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you
|
| 2365 |
# need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this
|
| 2366 |
# database says so here, with its cause and the recommendation, rather than waiting to be
|
|
@@ -2382,16 +2417,24 @@ def table_rows(table_key: str, session: Session = Depends(require_session)):
|
|
| 2382 |
if str(pid) in _zone_pids])
|
| 2383 |
for zone in _ut().zones(table_key, st=_lent)]
|
| 2384 |
return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"],
|
|
|
|
| 2385 |
"zones": _zones,
|
| 2386 |
"docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname,
|
| 2387 |
admin=session.admin, st=session.runtime),
|
| 2388 |
"pulled_at": time.strftime("%Y-%m-%d %H:%M"),
|
| 2389 |
"identity": {"pid": "pid"},
|
| 2390 |
"scope": {"table": table_key, "rowCount": len(rows)},
|
| 2391 |
-
"limits": [_report] if _report else [],
|
| 2392 |
"recordsMutable": _ut().records_mutable(table_key, st=_lent)}
|
| 2393 |
|
| 2394 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2395 |
#: The per-cell provenance a row carries on the wire, one key per enrichment column.
|
| 2396 |
#: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic.
|
| 2397 |
#: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new
|
|
|
|
| 68 |
raise AttributeError(name)
|
| 69 |
|
| 70 |
|
| 71 |
+
def _related_product_fields(session, rows, table_key='product_data'):
|
| 72 |
"""Resolve authored Metric definitions through the existing workspace and field wall."""
|
| 73 |
import aios_grid
|
| 74 |
from core import grid_events, perm_scope, shared_overlay
|
| 75 |
from modules import product_data as pd
|
| 76 |
+
import routes_customers
|
| 77 |
+
|
| 78 |
+
scope = {'product_data': 'product', 'customer_data': 'customer'}.get(table_key, table_key)
|
| 79 |
+
if table_key == 'product_data':
|
| 80 |
+
table, base = pd.table_ops(session.runtime), aios_grid.product_fields()
|
| 81 |
+
shared_key = pd.TABLE_KEY
|
| 82 |
+
elif table_key == 'customer_data':
|
| 83 |
+
table, base = routes_customers._customer_table(session), aios_grid.FIELDS
|
| 84 |
+
shared_key = routes_customers._shared_key()
|
| 85 |
+
else:
|
| 86 |
+
table = _ops(session, table_key)
|
| 87 |
+
base = perm_scope.scoped_fields(session.user, table_key, st=session.runtime)
|
| 88 |
+
shared_key = table_key
|
| 89 |
|
| 90 |
pids = frozenset(r["pid"] for r in rows)
|
| 91 |
ctx = grid_events.EventCtx(uname=session.uname, allowed_pids=pids, fields=[],
|
| 92 |
hidden_keys=set(), admin=session.admin, fallback_ws=None,
|
| 93 |
+
seen_ids={}, st=session.runtime, scope_key=scope, table=table)
|
|
|
|
| 94 |
ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False)
|
| 95 |
_, fields, _, _ = aios_grid.workspace_wire(
|
| 96 |
+
ws, session.uname, set(pids), defs={}, scope_key=scope, with_cohorts=False,
|
| 97 |
+
fields_base=base, st=session.runtime)
|
| 98 |
have = {f["key"] for f in fields}
|
| 99 |
fields = list(fields) + [dict(f, source="overlay", shared=True)
|
| 100 |
+
for key, f in shared_overlay.fields(shared_key, st=session.runtime).items()
|
| 101 |
if key not in have]
|
| 102 |
+
hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
|
| 103 |
return [f for f in fields if f.get("measure") and f["key"] not in hidden]
|
| 104 |
|
| 105 |
|
| 106 |
@router.post("/tables/{table_key}/related-metrics")
|
| 107 |
def related_metrics(table_key: str, body: dict = Body(default=None),
|
| 108 |
session: Session = Depends(require_session)):
|
| 109 |
+
"""R5: any offered relation, with the rows door's exact walled membership.
|
| 110 |
|
| 111 |
The request supplies only grid identities and a named relation. Fields, periods, source
|
| 112 |
identities and values are read from this tenant; nothing is authored by this endpoint.
|
|
|
|
| 118 |
import automation_engine
|
| 119 |
|
| 120 |
body = body if isinstance(body, dict) else {}
|
| 121 |
+
pin = body.get("related")
|
| 122 |
pids = body.get("pids")
|
| 123 |
+
if (not isinstance(pin, dict) or not isinstance(pids, list)
|
|
|
|
|
|
|
|
|
|
| 124 |
or any(type(pid) is not int or pid <= 0 for pid in pids)):
|
| 125 |
+
raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.")
|
| 126 |
viewer = copy.copy(session)
|
| 127 |
viewer.runtime = _MetricReadRuntime(session.runtime)
|
| 128 |
try:
|
| 129 |
+
compiled, team_id = routes_grid.related_measure_pin(viewer, table_key, pin, pids)
|
| 130 |
+
# Read actual rows, not the fields-only door introduced by T24.
|
| 131 |
+
products = perm_scope.scoped_table(viewer.user, table_key, st=viewer.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
requested = set(pids)
|
| 133 |
selected = [r for r in products if r.get("pid") in requested]
|
| 134 |
+
if len(selected) != len(requested):
|
|
|
|
| 135 |
raise err(403, "metric_record_refused", "The destination records cannot be scoped for this account.")
|
| 136 |
+
fields = _related_product_fields(viewer, products, table_key)
|
| 137 |
+
from core import measure_resolve
|
| 138 |
+
if not perm_scope.may_metrics(viewer.user, table_key):
|
| 139 |
+
offer = []
|
| 140 |
+
elif table_key == 'customer_data':
|
| 141 |
+
offer = measure_resolve.offer(team_id)
|
| 142 |
+
elif table_key == 'product_data':
|
| 143 |
+
offer = routes_products.product_measures(viewer)
|
| 144 |
+
else:
|
| 145 |
+
offer = ut_measures(table_key, viewer)
|
| 146 |
+
admitted = {m['key'] for m in offer}
|
| 147 |
+
fields = [f for f in fields if f['measure'].get('key') in admitted]
|
| 148 |
+
values = automation_engine.related_metric_values(
|
| 149 |
+
viewer.runtime, table_key, selected, fields, offer, compiled, team_id=team_id)
|
|
|
|
|
|
|
|
|
|
| 150 |
return {"ok": True, "values": values, "fields": [f["key"] for f in fields],
|
| 151 |
+
"refusals": values.refusals,
|
| 152 |
+
"scope": {"table": table_key, "related": pin, "pids": sorted(requested),
|
| 153 |
+
"customer": pin.get('record'),
|
| 154 |
+
"products": {r['pid']: r.get('product_id') for r in selected}},
|
| 155 |
+
"reason": ' '.join(dict.fromkeys(r['message'] for r in values.refusals))}
|
| 156 |
except (perm_scope.Denied, perm_scope.UnknownTable):
|
| 157 |
raise err(403, "metric_record_refused", "The related records are not available to this account.")
|
| 158 |
except (perm_scope.Unresolvable, RuntimeError, ValueError):
|
| 159 |
raise err(409, "metric_scope_unavailable", "Scoped Metrics could not be read. Reopen the related records after the database is ready.")
|
| 160 |
|
| 161 |
|
| 162 |
+
def scope_linked_metrics(session, table_key, rows, fields, offer, related):
|
| 163 |
+
"""Replace metric cells before a pinned payload can be filtered or aggregated."""
|
| 164 |
+
if not related or not any(f.get('measure') for f in fields):
|
| 165 |
+
return rows, fields, []
|
| 166 |
+
from core import perm_scope
|
| 167 |
+
import routes_grid
|
| 168 |
+
import automation_engine
|
| 169 |
+
|
| 170 |
+
admitted = {m['key'] for m in offer} if perm_scope.may_metrics(session.user, table_key) else set()
|
| 171 |
+
hidden = {f['key'] for f in fields if f.get('measure')
|
| 172 |
+
and f['measure'].get('key') not in admitted}
|
| 173 |
+
fields = [f for f in fields if f['key'] not in hidden]
|
| 174 |
+
rows = [{k: v for k, v in row.items() if k not in hidden} for row in rows]
|
| 175 |
+
metrics = [f for f in fields if f.get('measure')]
|
| 176 |
+
if not metrics:
|
| 177 |
+
return rows, fields, []
|
| 178 |
+
pin, team_id = routes_grid.related_measure_pin(session, table_key, related,
|
| 179 |
+
[row['pid'] for row in rows])
|
| 180 |
+
answer = automation_engine.related_metric_values(session.runtime, table_key, rows, metrics,
|
| 181 |
+
offer, pin, team_id=team_id)
|
| 182 |
+
return [{**row, **answer[row['pid']]} for row in rows], fields, answer.refusals
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def linked_metric_limits(refusals):
|
| 186 |
+
"""The same refusal accompanies rows, filters and aggregate siblings."""
|
| 187 |
+
if not refusals:
|
| 188 |
+
return []
|
| 189 |
+
return [dict(subject='Linked metrics',
|
| 190 |
+
cause=' '.join(dict.fromkeys(marker['message'] for marker in refusals)),
|
| 191 |
+
recommendation='Check the metric source binding and period, then reopen this tab.')]
|
| 192 |
+
|
| 193 |
+
|
| 194 |
def _ut():
|
| 195 |
import core.user_tables as user_tables
|
| 196 |
return user_tables
|
|
|
|
| 1562 |
# for both — the same sentence `routes_odoo_tables` writes over its own `overlays` dict.
|
| 1563 |
shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden}
|
| 1564 |
for pid, cells in shared_cells.items()}
|
| 1565 |
+
# D-464: the saved views' column keys, on the `/workspace` wire only; `views` stays whole
|
| 1566 |
+
# for `_ut_measure_sets` below.
|
| 1567 |
+
workspace["views"] = aios_grid.narrow_view_keys(workspace.get("views"), hidden)
|
| 1568 |
|
| 1569 |
# ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database
|
| 1570 |
# whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real
|
|
|
|
| 2305 |
|
| 2306 |
|
| 2307 |
@router.get("/tables/{table_key}/rows")
|
| 2308 |
+
def table_rows(table_key: str, session: Session = Depends(require_session),
|
| 2309 |
+
related: str = None):
|
| 2310 |
"""The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at,
|
| 2311 |
identity}` — so the client's generic topic fetch consumes it with zero new parsing.
|
| 2312 |
|
|
|
|
| 2328 |
# PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the
|
| 2329 |
# value it replaced (contract C5).
|
| 2330 |
_lent = _ut().lend(session.runtime)
|
| 2331 |
+
g = ut_assembly(session, table_key, st=_lent, consume_corrections=not bool(related))
|
| 2332 |
# ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge
|
| 2333 |
# this line does not perform since the route was written (owner item 3, 2026-08-09:
|
| 2334 |
# *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*).
|
|
|
|
| 2390 |
# ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/
|
| 2391 |
# `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`.
|
| 2392 |
_stamp_ai_states(table_key, g["fields"], rows, st=_lent)
|
| 2393 |
+
from routes_odoo_tables import narrow_related_rows
|
| 2394 |
+
rows = narrow_related_rows(session, table_key, rows, related)
|
| 2395 |
+
rows, g['fields'], metric_refusals = scope_linked_metrics(
|
| 2396 |
+
session, table_key, rows, g['fields'], g['measures'], related)
|
| 2397 |
+
if related:
|
| 2398 |
+
g["pids"] = frozenset(row["pid"] for row in rows)
|
| 2399 |
# ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you
|
| 2400 |
# need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this
|
| 2401 |
# database says so here, with its cause and the recommendation, rather than waiting to be
|
|
|
|
| 2417 |
if str(pid) in _zone_pids])
|
| 2418 |
for zone in _ut().zones(table_key, st=_lent)]
|
| 2419 |
return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"],
|
| 2420 |
+
"refusals": metric_refusals,
|
| 2421 |
"zones": _zones,
|
| 2422 |
"docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname,
|
| 2423 |
admin=session.admin, st=session.runtime),
|
| 2424 |
"pulled_at": time.strftime("%Y-%m-%d %H:%M"),
|
| 2425 |
"identity": {"pid": "pid"},
|
| 2426 |
"scope": {"table": table_key, "rowCount": len(rows)},
|
| 2427 |
+
"limits": ([_report] if _report else []) + linked_metric_limits(metric_refusals),
|
| 2428 |
"recordsMutable": _ut().records_mutable(table_key, st=_lent)}
|
| 2429 |
|
| 2430 |
|
| 2431 |
+
@router.post("/tables/{table_key}/aggregates")
|
| 2432 |
+
def table_aggregates(table_key: str, body: dict = Body(default=None),
|
| 2433 |
+
session: Session = Depends(require_session)):
|
| 2434 |
+
from routes_odoo_tables import odoo_table_aggregates
|
| 2435 |
+
return odoo_table_aggregates(table_key, body=body, session=session)
|
| 2436 |
+
|
| 2437 |
+
|
| 2438 |
#: The per-cell provenance a row carries on the wire, one key per enrichment column.
|
| 2439 |
#: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic.
|
| 2440 |
#: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new
|
api/routes_templates.py
CHANGED
|
@@ -72,13 +72,17 @@ def _target_or_refuse(session, table_key):
|
|
| 72 |
|
| 73 |
|
| 74 |
@router.get("/templates")
|
| 75 |
-
def list_templates(table: str = "", session: Session = Depends(require_session)):
|
| 76 |
"""What can be applied to THIS database. `{table, templates: [...]}`.
|
| 77 |
|
| 78 |
Filtered by the target's real columns, so the picker cannot offer something the apply door
|
| 79 |
would refuse — one predicate, two callers (`view_templates.offer` wraps
|
| 80 |
`missing_columns`, which is the same function the POST below re-runs).
|
| 81 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
fields, source = _target_or_refuse(session, table)
|
| 83 |
vt = _templates()
|
| 84 |
return {"table": table, "templates": vt.offer(fields, source or None)}
|
|
@@ -172,6 +176,134 @@ def _store(session: Session):
|
|
| 172 |
return session.runtime
|
| 173 |
|
| 174 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
@router.get("/sheet-templates")
|
| 176 |
def list_sheet_templates(session: Session = Depends(require_session)):
|
| 177 |
"""The workbook gallery, including sources this tenant has not connected."""
|
|
@@ -210,28 +342,35 @@ def apply_sheet_template(key: str, body: dict = Body(default=None),
|
|
| 210 |
from routes_sheet import _require_database, refresh_sheet
|
| 211 |
|
| 212 |
descriptor = sheet_templates.template(key)
|
| 213 |
-
if descriptor
|
|
|
|
|
|
|
| 214 |
raise err(404, "unknown_sheet_template", "That sheet template does not exist.")
|
| 215 |
body = body if isinstance(body, dict) else {}
|
| 216 |
database_key = body.get("database")
|
| 217 |
name = body.get("name", "")
|
| 218 |
-
if not isinstance(database_key, str) or not database_key.strip():
|
| 219 |
raise err(404, "unknown_database", "That database does not exist.")
|
| 220 |
if not isinstance(name, str):
|
| 221 |
raise err(400, "bad_request", "The workbook name must be text.")
|
| 222 |
-
database_key = database_key.strip()
|
| 223 |
st = _store(session)
|
| 224 |
-
if database_key not in user_tables.all_defs(st):
|
| 225 |
raise err(404, "unknown_database", "That database does not exist.")
|
| 226 |
-
|
| 227 |
-
|
|
|
|
| 228 |
raise err(400, "template_source_mismatch",
|
| 229 |
"That database uses a different connector from this template.")
|
| 230 |
|
| 231 |
book = None
|
| 232 |
try:
|
| 233 |
-
|
| 234 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
# Owners read their persisted values. Use the same permission-scoped
|
| 236 |
# refresh as Sheet so the first open already contains computed cells.
|
| 237 |
refreshed = refresh_sheet(book["id"], session=session)["workbook"]
|
|
@@ -256,3 +395,34 @@ def apply_sheet_template(key: str, body: dict = Body(default=None),
|
|
| 256 |
if isinstance(exc, (sheets.SheetRefusal, BindingRefusal)):
|
| 257 |
raise err(409, exc.code, exc.message) from exc
|
| 258 |
raise
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
|
| 73 |
|
| 74 |
@router.get("/templates")
|
| 75 |
+
def list_templates(table: str = "", session: Session = Depends(require_session), module: str = ""):
|
| 76 |
"""What can be applied to THIS database. `{table, templates: [...]}`.
|
| 77 |
|
| 78 |
Filtered by the target's real columns, so the picker cannot offer something the apply door
|
| 79 |
would refuse — one predicate, two callers (`view_templates.offer` wraps
|
| 80 |
`missing_columns`, which is the same function the POST below re-runs).
|
| 81 |
"""
|
| 82 |
+
if module:
|
| 83 |
+
if module not in ("sheet", "data", "app"):
|
| 84 |
+
raise err(400, "unknown_module", "Choose Sheet, Data or App.")
|
| 85 |
+
return _module_templates(session)
|
| 86 |
fields, source = _target_or_refuse(session, table)
|
| 87 |
vt = _templates()
|
| 88 |
return {"table": table, "templates": vt.offer(fields, source or None)}
|
|
|
|
| 176 |
return session.runtime
|
| 177 |
|
| 178 |
|
| 179 |
+
def _saved_sheet_templates(st):
|
| 180 |
+
"""C4 metadata only. An absent or unrecognised document offers no saved templates."""
|
| 181 |
+
raw = st.get("sheet_templates_saved")
|
| 182 |
+
if not isinstance(raw, dict) or raw.get("schema") != 1:
|
| 183 |
+
return []
|
| 184 |
+
rows = raw.get("templates")
|
| 185 |
+
if not isinstance(rows, dict):
|
| 186 |
+
return []
|
| 187 |
+
return [row for key, row in rows.items()
|
| 188 |
+
if isinstance(row, dict) and row.get("key") == key
|
| 189 |
+
and isinstance(row.get("name"), str)
|
| 190 |
+
and isinstance(row.get("description"), str)
|
| 191 |
+
and isinstance(row.get("workbook"), dict)
|
| 192 |
+
and isinstance(row.get("databases"), list)
|
| 193 |
+
and all(isinstance(db, str) and db.strip() for db in row["databases"])]
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def _database_states(session, definitions, databases):
|
| 197 |
+
"""Resolve switches once per request without rereading the stored table document."""
|
| 198 |
+
from core import registry, user_tables
|
| 199 |
+
|
| 200 |
+
# With no arguments the registrar returns its current keys without changing them.
|
| 201 |
+
# Match is_connected's three legs using the definitions projection already in hand.
|
| 202 |
+
registered = user_tables.register_connected()
|
| 203 |
+
prefixes = user_tables.connected_prefixes()
|
| 204 |
+
states = {}
|
| 205 |
+
for database in set(databases):
|
| 206 |
+
key = str(database or "").strip()
|
| 207 |
+
definition = definitions.get(key) or {}
|
| 208 |
+
if registry.BY_KEY.get(database):
|
| 209 |
+
states[database] = perms.tenant_has_module(session.runtime, database)
|
| 210 |
+
else:
|
| 211 |
+
# Process-wide registration alone must not admit another tenant's database.
|
| 212 |
+
states[database] = database in definitions and (
|
| 213 |
+
key in registered or definition.get("connected") is True
|
| 214 |
+
or (any(key.startswith(prefix) for prefix in prefixes)
|
| 215 |
+
and definition.get("source") == user_tables.AUTOMATION_SOURCE
|
| 216 |
+
and definition.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE))
|
| 217 |
+
return states
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _requirement(database, definitions, have, database_states, connector=""):
|
| 221 |
+
from core import registry
|
| 222 |
+
from routes_connectors import _module_connection_key, _table_connection_key
|
| 223 |
+
|
| 224 |
+
definition = definitions.get(database) or {}
|
| 225 |
+
module = registry.BY_KEY.get(database) or {}
|
| 226 |
+
connector = (connector or _table_connection_key(database, definition)
|
| 227 |
+
or _module_connection_key(module.get("subject")))
|
| 228 |
+
# These two compiled connector databases replaced the retired ut_odoo_* copies.
|
| 229 |
+
# Their switch is tenant admission; stored databases use the connector registrar.
|
| 230 |
+
switched_on = database_states[database] if module or connector else bool(definition)
|
| 231 |
+
label = definition.get("label") or module.get("label")
|
| 232 |
+
if not label:
|
| 233 |
+
parts = database.removeprefix("ut_").split("_")
|
| 234 |
+
brand = {"odoo": "Odoo", "amzn": "Amazon", "ig": "Instagram"}.get(parts[0], parts[0].capitalize())
|
| 235 |
+
label = brand + " " + " ".join(parts[1:])
|
| 236 |
+
return {"connector": connector, "database": database,
|
| 237 |
+
"label": str(label).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
|
| 238 |
+
"state": "disconnected" if connector and connector not in have
|
| 239 |
+
else "ready" if switched_on else "off"}
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def _catalog_row(key, name, description, requires):
|
| 243 |
+
# Prefer the missing connection even when an earlier database is merely off.
|
| 244 |
+
blocked = next((row for row in requires if row["state"] == "disconnected"), None)
|
| 245 |
+
blocked = blocked or next((row for row in requires if row["state"] != "ready"), None)
|
| 246 |
+
return {"key": key, "name": str(name).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
|
| 247 |
+
"description": str(description).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
|
| 248 |
+
"requires": requires, "creatable": all(row["state"] == "ready" for row in requires),
|
| 249 |
+
"action": None if blocked is None else {
|
| 250 |
+
"kind": "connect" if blocked["state"] == "disconnected" else "switch_on",
|
| 251 |
+
"connector": blocked["connector"], "database": blocked["database"]}}
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def _data_template_database(template, definitions):
|
| 255 |
+
fixed = {"odoo_customer": "customer_data", "odoo_product": "product_data"}
|
| 256 |
+
if template["source"] in fixed:
|
| 257 |
+
return fixed[template["source"]]
|
| 258 |
+
from routes_connectors import _table_connection_key
|
| 259 |
+
for key, definition in definitions.items():
|
| 260 |
+
fields = [field.get("key") for field in definition.get("fields", [])]
|
| 261 |
+
if (_table_connection_key(key, definition) == template["source"]
|
| 262 |
+
and not _templates().missing_columns(template, fields)):
|
| 263 |
+
return key
|
| 264 |
+
return ""
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def _module_templates(session):
|
| 268 |
+
from core import sheet_templates, user_tables
|
| 269 |
+
from routes_connectors import _entry_types
|
| 270 |
+
from routes_keychain import env_odoo_available
|
| 271 |
+
from routes_nav import application_templates
|
| 272 |
+
|
| 273 |
+
definitions = user_tables.all_defs(session.runtime)
|
| 274 |
+
have, _scopes, _failure = _entry_types(session.runtime, session.uname)
|
| 275 |
+
have = set(have)
|
| 276 |
+
if env_odoo_available(session.runtime):
|
| 277 |
+
have.add("odoo")
|
| 278 |
+
saved = _saved_sheet_templates(session.runtime)
|
| 279 |
+
data_templates = [(row, _data_template_database(row, definitions))
|
| 280 |
+
for row in _templates().TEMPLATES]
|
| 281 |
+
databases = {row["database_hint"] for row in sheet_templates.TEMPLATES}
|
| 282 |
+
databases.update(db for row in saved for db in row["databases"])
|
| 283 |
+
databases.update(database for _row, database in data_templates if database)
|
| 284 |
+
database_states = _database_states(session, definitions, databases)
|
| 285 |
+
requirement = lambda database, connector="": _requirement(
|
| 286 |
+
database, definitions, have, database_states, connector)
|
| 287 |
+
sheet = [_catalog_row(row["key"], row["title"], row["answers"],
|
| 288 |
+
[requirement(row["database_hint"], row["source"])])
|
| 289 |
+
for row in sheet_templates.TEMPLATES]
|
| 290 |
+
sheet.extend(_catalog_row(row["key"], row["name"], row["description"],
|
| 291 |
+
[requirement(db) for db in row["databases"]])
|
| 292 |
+
for row in saved)
|
| 293 |
+
data = []
|
| 294 |
+
for row, database in data_templates:
|
| 295 |
+
if database:
|
| 296 |
+
connector = "odoo" if row["source"].startswith("odoo_") else row["source"]
|
| 297 |
+
data.append(_catalog_row(row["key"], row["label"], row["desc"],
|
| 298 |
+
[requirement(database, connector)]))
|
| 299 |
+
# These App contracts map fields in the existing wizard and require no fixed connector.
|
| 300 |
+
app = [_catalog_row(row["kind"], row["label"], "Choose a database and map its fields.", [])
|
| 301 |
+
for row in application_templates(session=session)["templates"]]
|
| 302 |
+
return {"sections": [{"module": "sheet", "templates": sheet},
|
| 303 |
+
{"module": "data", "templates": data},
|
| 304 |
+
{"module": "app", "templates": app}]}
|
| 305 |
+
|
| 306 |
+
|
| 307 |
@router.get("/sheet-templates")
|
| 308 |
def list_sheet_templates(session: Session = Depends(require_session)):
|
| 309 |
"""The workbook gallery, including sources this tenant has not connected."""
|
|
|
|
| 342 |
from routes_sheet import _require_database, refresh_sheet
|
| 343 |
|
| 344 |
descriptor = sheet_templates.template(key)
|
| 345 |
+
saved = (None if descriptor else
|
| 346 |
+
next((row for row in _saved_sheet_templates(_store(session)) if row["key"] == key), None))
|
| 347 |
+
if descriptor is None and saved is None:
|
| 348 |
raise err(404, "unknown_sheet_template", "That sheet template does not exist.")
|
| 349 |
body = body if isinstance(body, dict) else {}
|
| 350 |
database_key = body.get("database")
|
| 351 |
name = body.get("name", "")
|
| 352 |
+
if saved is None and (not isinstance(database_key, str) or not database_key.strip()):
|
| 353 |
raise err(404, "unknown_database", "That database does not exist.")
|
| 354 |
if not isinstance(name, str):
|
| 355 |
raise err(400, "bad_request", "The workbook name must be text.")
|
| 356 |
+
database_key = database_key.strip() if isinstance(database_key, str) else ""
|
| 357 |
st = _store(session)
|
| 358 |
+
if saved is None and database_key not in user_tables.all_defs(st):
|
| 359 |
raise err(404, "unknown_database", "That database does not exist.")
|
| 360 |
+
for required in saved["databases"] if saved else [database_key]:
|
| 361 |
+
_require_database(session, required)
|
| 362 |
+
if descriptor and sheet_templates.source_of_database(database_key) != descriptor["source"]:
|
| 363 |
raise err(400, "template_source_mismatch",
|
| 364 |
"That database uses a different connector from this template.")
|
| 365 |
|
| 366 |
book = None
|
| 367 |
try:
|
| 368 |
+
if saved:
|
| 369 |
+
# Lane G owns C4 materialisation; never reconstruct its workbook here.
|
| 370 |
+
book = sheet_templates.apply_saved_template(st, key, name=name, created_by=session.uname)
|
| 371 |
+
else:
|
| 372 |
+
book = sheet_templates.apply_template(st, key, database_key, name=name,
|
| 373 |
+
created_by=session.uname)
|
| 374 |
# Owners read their persisted values. Use the same permission-scoped
|
| 375 |
# refresh as Sheet so the first open already contains computed cells.
|
| 376 |
refreshed = refresh_sheet(book["id"], session=session)["workbook"]
|
|
|
|
| 395 |
if isinstance(exc, (sheets.SheetRefusal, BindingRefusal)):
|
| 396 |
raise err(409, exc.code, exc.message) from exc
|
| 397 |
raise
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
# Declaration order is load bearing: keep this AFTER /templates/{key}/apply.
|
| 401 |
+
# Both paths have three segments; reversing them steals /templates/x/apply.
|
| 402 |
+
@router.post("/templates/{module}/{key}")
|
| 403 |
+
def create_module_template(module: str, key: str, body: dict = Body(default=None),
|
| 404 |
+
session: Session = Depends(require_session)):
|
| 405 |
+
from routes_nav import create_nav_application
|
| 406 |
+
from urllib.parse import quote
|
| 407 |
+
|
| 408 |
+
body = body if isinstance(body, dict) else {}
|
| 409 |
+
section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None)
|
| 410 |
+
template = next((row for row in section["templates"] if row["key"] == key), None) if section else None
|
| 411 |
+
if template is None:
|
| 412 |
+
raise err(404, "unknown_template", "That template does not exist.")
|
| 413 |
+
if not template["creatable"]:
|
| 414 |
+
raise err(409, "template_unavailable", "Connect the required source and switch on its databases.")
|
| 415 |
+
if module == "sheet":
|
| 416 |
+
database = template["requires"][0]["database"] if template["requires"] else ""
|
| 417 |
+
result = apply_sheet_template(key, {"database": database, "name": body.get("name", "")}, session)
|
| 418 |
+
return {"address": "#/sheet?wb=" + quote(result["workbook"]["id"], safe=""), **result}
|
| 419 |
+
if module == "data":
|
| 420 |
+
database = template["requires"][0]["database"]
|
| 421 |
+
result = apply_template(key, {"table": database}, session)
|
| 422 |
+
return {"address": "#/" + quote(database, safe=""), **result}
|
| 423 |
+
# The existing wizard still owns its POST-then-PATCH share sequence. API clients may
|
| 424 |
+
# dispatch creation here; the existing App writer retains schema and permission walls.
|
| 425 |
+
result = create_nav_application({**body, "templateKind": key}, session=session)
|
| 426 |
+
app = result["application"]
|
| 427 |
+
return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""),
|
| 428 |
+
**result}
|
api/semantic_registry.py
CHANGED
|
@@ -9,6 +9,7 @@ import copy
|
|
| 9 |
import hashlib
|
| 10 |
import json
|
| 11 |
from collections import Counter
|
|
|
|
| 12 |
|
| 13 |
|
| 14 |
SEMANTIC_STATES = ("draft", "approved", "disabled")
|
|
@@ -17,6 +18,19 @@ VALUE_PROFILE_CAP = 50
|
|
| 17 |
VALUE_SAMPLE_CAP = 12
|
| 18 |
_KEY = "semantic_registry_v1"
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
class SemanticError(ValueError):
|
| 22 |
"""A semantic contract was refused before persistence or retrieval."""
|
|
@@ -42,36 +56,48 @@ def field_key(database, field):
|
|
| 42 |
|
| 43 |
|
| 44 |
def model_record(database, topic, field):
|
| 45 |
-
"""Build
|
| 46 |
-
|
| 47 |
-
Existing topic files predate approval. They therefore enter as ``draft`` unless the
|
| 48 |
-
declaration explicitly says otherwise; merely having a label never grants retrieval.
|
| 49 |
-
"""
|
| 50 |
field = field if isinstance(field, dict) else {}
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
state = str(semantic.get("state") or "draft").lower()
|
| 53 |
sensitivity = str(semantic.get("sensitivity") or "internal").lower()
|
| 54 |
if state not in SEMANTIC_STATES:
|
| 55 |
raise SemanticError(f"unknown semantic state {state!r}")
|
| 56 |
if sensitivity not in SENSITIVITY:
|
| 57 |
raise SemanticError(f"unknown sensitivity {sensitivity!r}")
|
| 58 |
-
return {
|
| 59 |
"database": _text(database, 120),
|
| 60 |
"topic": _text((topic or {}).get("key"), 120),
|
| 61 |
"field": _text(field.get("key"), 120),
|
| 62 |
"label": _text(field.get("label") or field.get("key"), 120),
|
| 63 |
-
"purpose": _text(semantic.get("purpose")
|
| 64 |
"dataType": _text(field.get("type") or "text", 40),
|
| 65 |
"role": _text(semantic.get("role") or "attribute", 60),
|
| 66 |
"synonyms": _strings(semantic.get("synonyms")),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
"sensitivity": sensitivity,
|
| 68 |
"state": state,
|
| 69 |
"valueProfile": None,
|
| 70 |
"revision": int(semantic.get("revision") or 1),
|
| 71 |
-
"approvedBy": _text(semantic.get("approvedBy"), 120) or
|
|
|
|
|
|
|
| 72 |
"disabledBy": _text(semantic.get("disabledBy"), 120) or None,
|
| 73 |
"profiledBy": _text(semantic.get("profiledBy"), 120) or None,
|
| 74 |
-
}
|
| 75 |
|
| 76 |
|
| 77 |
def topic_records(database, topic):
|
|
@@ -82,7 +108,39 @@ def topic_records(database, topic):
|
|
| 82 |
}
|
| 83 |
|
| 84 |
|
| 85 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
if not is_admin:
|
| 87 |
raise SemanticError("only a tenant admin can change semantic approval")
|
| 88 |
raw = raw if isinstance(raw, dict) else {}
|
|
@@ -111,6 +169,7 @@ def _clean_override(raw, baseline, *, actor, is_admin):
|
|
| 111 |
raise SemanticError("semantic state is not recognized")
|
| 112 |
out["state"] = state
|
| 113 |
out["approvedBy"] = _text(actor, 120) if state == "approved" else None
|
|
|
|
| 114 |
out["disabledBy"] = _text(actor, 120) if state == "disabled" else None
|
| 115 |
out["revision"] = int(baseline.get("revision") or 0) + 1
|
| 116 |
return out
|
|
@@ -127,7 +186,7 @@ def merge_records(baseline, persisted):
|
|
| 127 |
if key not in out or not isinstance(raw, dict):
|
| 128 |
continue
|
| 129 |
for name in ("purpose", "role", "synonyms", "sensitivity", "state", "valueProfile",
|
| 130 |
-
"revision", "approvedBy", "disabledBy", "profiledBy"):
|
| 131 |
if name in raw:
|
| 132 |
out[key][name] = copy.deepcopy(raw[name])
|
| 133 |
out[key] = enforce_redaction(out[key])
|
|
@@ -142,12 +201,12 @@ def load(runtime, baseline):
|
|
| 142 |
return merge_records(baseline, raw)
|
| 143 |
|
| 144 |
|
| 145 |
-
def update(runtime, baseline, database, field, patch, *, actor, is_admin):
|
| 146 |
key = field_key(database, field)
|
| 147 |
if key not in baseline:
|
| 148 |
raise SemanticError("that field is not in the database contract")
|
| 149 |
current = load(runtime, baseline)[key]
|
| 150 |
-
changed = _clean_override(patch, current, actor=actor, is_admin=is_admin)
|
| 151 |
|
| 152 |
def apply(raw):
|
| 153 |
doc = copy.deepcopy(raw) if isinstance(raw, dict) else {}
|
|
@@ -240,6 +299,85 @@ def value_profile(values, *, sensitivity="internal", distinct_cap=VALUE_PROFILE_
|
|
| 240 |
return {"redacted": False, "distinctCount": count, "values": [], "sample": sample}
|
| 241 |
|
| 242 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
def generated_markdown(record):
|
| 244 |
"""A deterministic projection. Sensitive profiles carry no values by construction."""
|
| 245 |
row = enforce_redaction(record)
|
|
|
|
| 9 |
import hashlib
|
| 10 |
import json
|
| 11 |
from collections import Counter
|
| 12 |
+
from datetime import datetime, timezone
|
| 13 |
|
| 14 |
|
| 15 |
SEMANTIC_STATES = ("draft", "approved", "disabled")
|
|
|
|
| 18 |
VALUE_SAMPLE_CAP = 12
|
| 19 |
_KEY = "semantic_registry_v1"
|
| 20 |
|
| 21 |
+
# The catalogue fields_get attribute list belongs to odoo_relational.py (lane N); add selection there to replace profile-derived choice values with Odoo choices.
|
| 22 |
+
ODOO_SEMANTIC_TYPES = {
|
| 23 |
+
"char": "text", "text": "text", "html": "text", "selection": "select",
|
| 24 |
+
"boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency",
|
| 25 |
+
"date": "date", "datetime": "datetime", "many2one": "link", "one2many": "link",
|
| 26 |
+
"many2many": "link", "binary": "text",
|
| 27 |
+
}
|
| 28 |
+
_ODOO_PII_NAMES = (
|
| 29 |
+
"email", "phone", "mobile", "fax", "vat", "street", "zip", "bank", "acc_number",
|
| 30 |
+
"iban", "password", "passwd", "token", "api_key", "secret", "signature", "ssn",
|
| 31 |
+
"birthday", "private",
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
|
| 35 |
class SemanticError(ValueError):
|
| 36 |
"""A semantic contract was refused before persistence or retrieval."""
|
|
|
|
| 56 |
|
| 57 |
|
| 58 |
def model_record(database, topic, field):
|
| 59 |
+
"""Build a baseline from topic curation, with per-field declarations taking precedence."""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
field = field if isinstance(field, dict) else {}
|
| 61 |
+
key = field.get("key")
|
| 62 |
+
curated = (topic or {}).get("semantic") or {}
|
| 63 |
+
defaults = {
|
| 64 |
+
"state": "approved" if key in (curated.get("approved") or ()) else "draft",
|
| 65 |
+
"sensitivity": "sensitive" if key in (curated.get("sensitive") or ()) else "internal",
|
| 66 |
+
"purpose": (curated.get("purposes") or {}).get(key, field.get("means") or ""),
|
| 67 |
+
"synonyms": (curated.get("synonyms") or {}).get(key, []),
|
| 68 |
+
"choices": (curated.get("choices") or {}).get(key, []),
|
| 69 |
+
}
|
| 70 |
+
field_meta = field.get("semantic") if isinstance(field.get("semantic"), dict) else {}
|
| 71 |
+
semantic = {**defaults, **field_meta} # Per-field curation wins.
|
| 72 |
state = str(semantic.get("state") or "draft").lower()
|
| 73 |
sensitivity = str(semantic.get("sensitivity") or "internal").lower()
|
| 74 |
if state not in SEMANTIC_STATES:
|
| 75 |
raise SemanticError(f"unknown semantic state {state!r}")
|
| 76 |
if sensitivity not in SENSITIVITY:
|
| 77 |
raise SemanticError(f"unknown sensitivity {sensitivity!r}")
|
| 78 |
+
return enforce_redaction({
|
| 79 |
"database": _text(database, 120),
|
| 80 |
"topic": _text((topic or {}).get("key"), 120),
|
| 81 |
"field": _text(field.get("key"), 120),
|
| 82 |
"label": _text(field.get("label") or field.get("key"), 120),
|
| 83 |
+
"purpose": _text(semantic.get("purpose"), 500),
|
| 84 |
"dataType": _text(field.get("type") or "text", 40),
|
| 85 |
"role": _text(semantic.get("role") or "attribute", 60),
|
| 86 |
"synonyms": _strings(semantic.get("synonyms")),
|
| 87 |
+
"relation": "",
|
| 88 |
+
"choices": copy.deepcopy(semantic.get("choices") or []),
|
| 89 |
+
"choicesFrom": "curated" if key in (curated.get("choices") or {}) or
|
| 90 |
+
"choices" in field_meta else ("profile" if field.get("type") == "select" else ""),
|
| 91 |
"sensitivity": sensitivity,
|
| 92 |
"state": state,
|
| 93 |
"valueProfile": None,
|
| 94 |
"revision": int(semantic.get("revision") or 1),
|
| 95 |
+
"approvedBy": (_text(semantic.get("approvedBy"), 120) or "platform baseline")
|
| 96 |
+
if state == "approved" else None,
|
| 97 |
+
"approvedAt": None,
|
| 98 |
"disabledBy": _text(semantic.get("disabledBy"), 120) or None,
|
| 99 |
"profiledBy": _text(semantic.get("profiledBy"), 120) or None,
|
| 100 |
+
})
|
| 101 |
|
| 102 |
|
| 103 |
def topic_records(database, topic):
|
|
|
|
| 108 |
}
|
| 109 |
|
| 110 |
|
| 111 |
+
def generated_records(database, model_row, *, model=""):
|
| 112 |
+
"""Pre-approved Odoo schema from the cached model's own fields, never inferred fields."""
|
| 113 |
+
out = {}
|
| 114 |
+
model = model or model_row.get("model") or ""
|
| 115 |
+
for name, metadata in (model_row.get("fields") or {}).items():
|
| 116 |
+
if not isinstance(metadata, dict) or not name:
|
| 117 |
+
continue
|
| 118 |
+
odoo_type = metadata.get("type")
|
| 119 |
+
relation = str(metadata.get("relation") or "")
|
| 120 |
+
sensitive = any(part in str(name).casefold() for part in _ODOO_PII_NAMES)
|
| 121 |
+
choices = metadata.get("selection")
|
| 122 |
+
choices = copy.deepcopy(list(choices)) if isinstance(choices, (list, tuple)) else []
|
| 123 |
+
role = ("link" if relation else "measure" if odoo_type in ("monetary", "float") else
|
| 124 |
+
"date" if odoo_type in ("date", "datetime") else "attribute")
|
| 125 |
+
row = model_record(database, {}, {
|
| 126 |
+
"key": name, "label": metadata.get("string") or name,
|
| 127 |
+
"type": ODOO_SEMANTIC_TYPES.get(odoo_type, "text"),
|
| 128 |
+
"semantic": {"state": "approved", "approvedBy": "odoo connector baseline",
|
| 129 |
+
"sensitivity": "sensitive" if sensitive else "internal", "role": role,
|
| 130 |
+
"purpose": f"The {name} field on Odoo model {model}."},
|
| 131 |
+
})
|
| 132 |
+
row.update({"relation": relation, "choices": choices,
|
| 133 |
+
"choicesFrom": "odoo" if choices else
|
| 134 |
+
("profile" if odoo_type == "selection" else "")})
|
| 135 |
+
out[field_key(database, name)] = row
|
| 136 |
+
return out
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _approval_time(now=None):
|
| 140 |
+
return (now or datetime.now(timezone.utc)).astimezone(timezone.utc).isoformat()
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _clean_override(raw, baseline, *, actor, is_admin, now=None):
|
| 144 |
if not is_admin:
|
| 145 |
raise SemanticError("only a tenant admin can change semantic approval")
|
| 146 |
raw = raw if isinstance(raw, dict) else {}
|
|
|
|
| 169 |
raise SemanticError("semantic state is not recognized")
|
| 170 |
out["state"] = state
|
| 171 |
out["approvedBy"] = _text(actor, 120) if state == "approved" else None
|
| 172 |
+
out["approvedAt"] = _approval_time(now) if state == "approved" else None
|
| 173 |
out["disabledBy"] = _text(actor, 120) if state == "disabled" else None
|
| 174 |
out["revision"] = int(baseline.get("revision") or 0) + 1
|
| 175 |
return out
|
|
|
|
| 186 |
if key not in out or not isinstance(raw, dict):
|
| 187 |
continue
|
| 188 |
for name in ("purpose", "role", "synonyms", "sensitivity", "state", "valueProfile",
|
| 189 |
+
"revision", "approvedBy", "approvedAt", "disabledBy", "profiledBy"):
|
| 190 |
if name in raw:
|
| 191 |
out[key][name] = copy.deepcopy(raw[name])
|
| 192 |
out[key] = enforce_redaction(out[key])
|
|
|
|
| 201 |
return merge_records(baseline, raw)
|
| 202 |
|
| 203 |
|
| 204 |
+
def update(runtime, baseline, database, field, patch, *, actor, is_admin, now=None):
|
| 205 |
key = field_key(database, field)
|
| 206 |
if key not in baseline:
|
| 207 |
raise SemanticError("that field is not in the database contract")
|
| 208 |
current = load(runtime, baseline)[key]
|
| 209 |
+
changed = _clean_override(patch, current, actor=actor, is_admin=is_admin, now=now)
|
| 210 |
|
| 211 |
def apply(raw):
|
| 212 |
doc = copy.deepcopy(raw) if isinstance(raw, dict) else {}
|
|
|
|
| 299 |
return {"redacted": False, "distinctCount": count, "values": [], "sample": sample}
|
| 300 |
|
| 301 |
|
| 302 |
+
def draft_fields(records, snapshot, *, limit=8):
|
| 303 |
+
"""Rank only source-declared, visible fields using the caller's real scoped rows."""
|
| 304 |
+
rows = [row for row in snapshot.get("records") or () if isinstance(row, dict)]
|
| 305 |
+
visible = {row["key"] for row in snapshot.get("fields") or () if row.get("key")}
|
| 306 |
+
ranked = []
|
| 307 |
+
for record in (records or {}).values():
|
| 308 |
+
field = record.get("field")
|
| 309 |
+
if (record.get("database") != snapshot.get("database") or field not in visible
|
| 310 |
+
or record.get("state") == "disabled"):
|
| 311 |
+
continue
|
| 312 |
+
values = [row.get(field) for row in rows]
|
| 313 |
+
populated = [value for value in values if value is not None and value != ""
|
| 314 |
+
and value != [] and value != {}]
|
| 315 |
+
role, kind = record.get("role") or "attribute", record.get("dataType") or "text"
|
| 316 |
+
linked = bool(record.get("relation")) or role == "link" or kind == "link"
|
| 317 |
+
dated = role == "date" or kind in ("date", "datetime", "created_time", "modified_time")
|
| 318 |
+
identifier = field.lower() in ("id", "pid", "uuid") or field.lower().endswith("_id")
|
| 319 |
+
evidence = f"Filled on {len(populated)} of {len(rows)} rows"
|
| 320 |
+
detail = (", and it links this database to related records" if linked else
|
| 321 |
+
", and it is a date this database can be scoped by" if dated else
|
| 322 |
+
", and it identifies a record" if identifier else "")
|
| 323 |
+
rank = len(populated) / max(1, len(rows)) + (0.4 if linked or dated else 0)
|
| 324 |
+
rank -= 1 if identifier and not linked else 0
|
| 325 |
+
sensitivity = record.get("sensitivity") or "internal"
|
| 326 |
+
ranked.append((rank, field, {
|
| 327 |
+
"field": field, "label": record.get("label") or field,
|
| 328 |
+
"why": evidence + detail + ".", "role": "link" if linked else "date" if dated else role,
|
| 329 |
+
"dataType": kind, "sensitivity": sensitivity,
|
| 330 |
+
"sample": value_profile(populated, sensitivity=sensitivity),
|
| 331 |
+
}))
|
| 332 |
+
ranked.sort(key=lambda item: (-item[0], item[1]))
|
| 333 |
+
return [item[2] for item in ranked[:max(0, min(int(limit), 8))]]
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def pending_approval_key():
|
| 337 |
+
return "semantic_draft_requests_v1"
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def record_draft_request(runtime, database, question, fields, *, actor, now=None):
|
| 341 |
+
"""Queue field keys only; shared requests never persist samples from a member's rows."""
|
| 342 |
+
request = {"database": database, "question": _text(question, 500),
|
| 343 |
+
"fields": list(dict.fromkeys(fields))[:8], "requestedBy": _text(actor, 120),
|
| 344 |
+
"requestedAt": _approval_time(now), "state": "pending",
|
| 345 |
+
"approvedBy": None, "approvedAt": None}
|
| 346 |
+
|
| 347 |
+
def apply(raw):
|
| 348 |
+
doc = copy.deepcopy(raw) if isinstance(raw, dict) else {}
|
| 349 |
+
doc["schema"] = 1
|
| 350 |
+
doc.setdefault("requests", {})[database] = request
|
| 351 |
+
return doc
|
| 352 |
+
|
| 353 |
+
runtime.update(pending_approval_key(), apply)
|
| 354 |
+
return copy.deepcopy(request)
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
def draft_request(runtime, database):
|
| 358 |
+
raw = runtime.get(pending_approval_key()) or {}
|
| 359 |
+
row = (raw.get("requests") or {}).get(database)
|
| 360 |
+
return copy.deepcopy(row) if isinstance(row, dict) else None
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
def approve_draft_request(runtime, database, fields, *, actor, is_admin, now=None):
|
| 364 |
+
if not is_admin:
|
| 365 |
+
raise SemanticError("only a tenant admin can change semantic approval")
|
| 366 |
+
landed = {}
|
| 367 |
+
|
| 368 |
+
def apply(raw):
|
| 369 |
+
doc = copy.deepcopy(raw) if isinstance(raw, dict) else {"schema": 1, "requests": {}}
|
| 370 |
+
row = doc.setdefault("requests", {}).get(database)
|
| 371 |
+
if row:
|
| 372 |
+
row.update({"fields": list(dict.fromkeys(fields))[:8], "state": "approved",
|
| 373 |
+
"approvedBy": _text(actor, 120), "approvedAt": _approval_time(now)})
|
| 374 |
+
landed.update(row)
|
| 375 |
+
return doc
|
| 376 |
+
|
| 377 |
+
runtime.update(pending_approval_key(), apply)
|
| 378 |
+
return copy.deepcopy(landed)
|
| 379 |
+
|
| 380 |
+
|
| 381 |
def generated_markdown(record):
|
| 382 |
"""A deterministic projection. Sensitive profiles carry no values by construction."""
|
| 383 |
row = enforce_redaction(record)
|
api/test_odoo_rank_refresh.py
CHANGED
|
@@ -5,7 +5,9 @@ import copy
|
|
| 5 |
import io
|
| 6 |
import os
|
| 7 |
from pathlib import Path
|
|
|
|
| 8 |
import socket
|
|
|
|
| 9 |
import sys
|
| 10 |
import tempfile
|
| 11 |
from types import SimpleNamespace
|
|
@@ -17,6 +19,59 @@ import duckdb
|
|
| 17 |
from harness import datastore as ds
|
| 18 |
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
class RankRefreshTests(unittest.TestCase):
|
| 21 |
def setUp(self):
|
| 22 |
self.stack = ExitStack()
|
|
@@ -146,16 +201,26 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 146 |
# tests died on it -- in the MERGED tree only: lane G does not run this
|
| 147 |
# gate and lane D's run predates lane G. The real `main.py` defines the
|
| 148 |
# function at module scope and the deployed Space boots fine.
|
|
|
|
|
|
|
| 149 |
_resync_mirror_pass=sync, _pull_meta=lambda *_: True,
|
| 150 |
_pull_amazon=lambda *_: True,
|
| 151 |
_rebuild_odoo_relational=rebuild,
|
| 152 |
_rebuild_meta_relational=lambda *_: {},
|
| 153 |
_rebuild_amazon_relational=lambda *_: {})
|
|
|
|
|
|
|
|
|
|
| 154 |
exec(compile(ast.Module(body=nodes, type_ignores=[]),
|
| 155 |
'main.py:resident-loop', 'exec'), ns)
|
| 156 |
runtime = SimpleNamespace(get_runtime=lambda *_: object())
|
|
|
|
|
|
|
|
|
|
| 157 |
with patch.dict(sys.modules, {'time': clock, 'harness.runtime': runtime}), \
|
| 158 |
patch.object(harness, 'runtime', runtime, create=True), \
|
|
|
|
|
|
|
| 159 |
redirect_stdout(io.StringIO()):
|
| 160 |
with self.assertRaisesRegex(StopIteration, 'bounded synthetic'):
|
| 161 |
ns['_store_resync_loop']()
|
|
|
|
| 5 |
import io
|
| 6 |
import os
|
| 7 |
from pathlib import Path
|
| 8 |
+
import builtins
|
| 9 |
import socket
|
| 10 |
+
import symtable
|
| 11 |
import sys
|
| 12 |
import tempfile
|
| 13 |
from types import SimpleNamespace
|
|
|
|
| 19 |
from harness import datastore as ds
|
| 20 |
|
| 21 |
|
| 22 |
+
class FixtureGap(AssertionError):
|
| 23 |
+
"""The synthetic namespace is missing a collaborator. A fixture defect, never a loop failure."""
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _free_globals(node):
|
| 27 |
+
"""Global names `node` reads, from Python's own symbol table rather than a name list."""
|
| 28 |
+
names = set()
|
| 29 |
+
def walk(table):
|
| 30 |
+
for sym in table.get_symbols():
|
| 31 |
+
if sym.is_referenced() and (
|
| 32 |
+
table.get_type() == 'module' and not (sym.is_assigned() or sym.is_imported())
|
| 33 |
+
or table.get_type() != 'module' and sym.is_global()):
|
| 34 |
+
names.add(sym.get_name())
|
| 35 |
+
for child in table.get_children():
|
| 36 |
+
walk(child)
|
| 37 |
+
walk(symtable.symtable(ast.unparse(node), 'main.py', 'exec'))
|
| 38 |
+
return names - set(dir(builtins))
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _with_collaborators(source, nodes, ns):
|
| 42 |
+
"""`nodes` plus every top-level main.py def/class/assign they transitively call.
|
| 43 |
+
|
| 44 |
+
W-228: the namespace below was hand-maintained, so `_run_sync_requests` arrived as a bare
|
| 45 |
+
NameError that named the resident loop. Explicit stubs in `ns` still win. A name that is
|
| 46 |
+
neither stubbed nor a top-level def/class/assign (an import, say) raises a FixtureGap.
|
| 47 |
+
"""
|
| 48 |
+
top = {}
|
| 49 |
+
for node in source.body:
|
| 50 |
+
if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
|
| 51 |
+
top.setdefault(node.name, node)
|
| 52 |
+
elif isinstance(node, ast.Assign):
|
| 53 |
+
for target in node.targets:
|
| 54 |
+
if isinstance(target, ast.Name):
|
| 55 |
+
top.setdefault(target.id, node)
|
| 56 |
+
have = list(nodes)
|
| 57 |
+
defined = {name for name, node in top.items() if node in have}
|
| 58 |
+
todo = [name for node in have for name in _free_globals(node)]
|
| 59 |
+
while todo:
|
| 60 |
+
name = todo.pop()
|
| 61 |
+
if name in ns or name in defined:
|
| 62 |
+
continue
|
| 63 |
+
node = top.get(name)
|
| 64 |
+
if node is None:
|
| 65 |
+
raise FixtureGap(
|
| 66 |
+
f'FIXTURE GAP, not a resident-loop failure: main.py:_store_resync_loop reads '
|
| 67 |
+
f'{name!r}, which this test neither stubs nor can extract as a top-level '
|
| 68 |
+
f'def/class/assign of main.py. Stub it in observe_resident_recovery.')
|
| 69 |
+
have.append(node)
|
| 70 |
+
defined |= {n for n, candidate in top.items() if candidate is node}
|
| 71 |
+
todo.extend(_free_globals(node))
|
| 72 |
+
return have
|
| 73 |
+
|
| 74 |
+
|
| 75 |
class RankRefreshTests(unittest.TestCase):
|
| 76 |
def setUp(self):
|
| 77 |
self.stack = ExitStack()
|
|
|
|
| 201 |
# tests died on it -- in the MERGED tree only: lane G does not run this
|
| 202 |
# gate and lane D's run predates lane G. The real `main.py` defines the
|
| 203 |
# function at module scope and the deployed Space boots fine.
|
| 204 |
+
# W-228: no longer a bare NameError. Unstubbed collaborators are extracted
|
| 205 |
+
# from main.py by `_with_collaborators`, or fail as a named FixtureGap.
|
| 206 |
_resync_mirror_pass=sync, _pull_meta=lambda *_: True,
|
| 207 |
_pull_amazon=lambda *_: True,
|
| 208 |
_rebuild_odoo_relational=rebuild,
|
| 209 |
_rebuild_meta_relational=lambda *_: {},
|
| 210 |
_rebuild_amazon_relational=lambda *_: {})
|
| 211 |
+
# W-228: the real helpers the loop calls (`_run_sync_requests` and what it needs) are
|
| 212 |
+
# extracted too; anything unresolvable fails as a named FixtureGap.
|
| 213 |
+
nodes = _with_collaborators(source, nodes, ns)
|
| 214 |
exec(compile(ast.Module(body=nodes, type_ignores=[]),
|
| 215 |
'main.py:resident-loop', 'exec'), ns)
|
| 216 |
runtime = SimpleNamespace(get_runtime=lambda *_: object())
|
| 217 |
+
from core import connector_sync
|
| 218 |
+
# The cadence wait is now an interruptible event, so it is routed to the synthetic clock.
|
| 219 |
+
wake = SimpleNamespace(wait=sleep, clear=lambda: None)
|
| 220 |
with patch.dict(sys.modules, {'time': clock, 'harness.runtime': runtime}), \
|
| 221 |
patch.object(harness, 'runtime', runtime, create=True), \
|
| 222 |
+
patch.object(connector_sync, 'sync_wake_event', return_value=wake), \
|
| 223 |
+
patch.object(connector_sync, 'take_sync_requests', return_value=[]), \
|
| 224 |
redirect_stdout(io.StringIO()):
|
| 225 |
with self.assertRaisesRegex(StopIteration, 'bounded synthetic'):
|
| 226 |
ns['_store_resync_loop']()
|
platform/aios_grid.py
CHANGED
|
@@ -471,6 +471,11 @@ def _clean_permissions(raw):
|
|
| 471 |
#: # $ % TEXT number formats already accepted by the client; `$` is also a regex anchor.
|
| 472 |
#:
|
| 473 |
#: `;` stays refused: no legal formula token needs a statement separator.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 474 |
_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"'\[\]|?:\\#$%@]*$")
|
| 475 |
_FORMULA_REF = re.compile(r"\{([^{}]*)\}")
|
| 476 |
|
|
@@ -3783,6 +3788,49 @@ def views_from_defs(defs, saved_views, fields, system_name="All customers", lock
|
|
| 3783 |
return views
|
| 3784 |
|
| 3785 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3786 |
def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None,
|
| 3787 |
fields_base=None, with_cohorts=True, st=None):
|
| 3788 |
"""The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE
|
|
|
|
| 471 |
#: # $ % TEXT number formats already accepted by the client; `$` is also a regex anchor.
|
| 472 |
#:
|
| 473 |
#: `;` stays refused: no legal formula token needs a statement separator.
|
| 474 |
+
#:
|
| 475 |
+
#: ⛔ D-547: A CHARACTER FILTER, NOT A SAFETY BOUNDARY. `__import__("os")` passes it (word
|
| 476 |
+
#: characters, parens, quotes). Such a source is inert because the ENGINE refuses it: the server
|
| 477 |
+
#: parser door in `aios-web/api/formula_runtime.py` is an AST interpreter over a function allowlist
|
| 478 |
+
#: with no Python eval. `api/verify_formula_storage.py` records both halves; never read this as the wall.
|
| 479 |
_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"'\[\]|?:\\#$%@]*$")
|
| 480 |
_FORMULA_REF = re.compile(r"\{([^{}]*)\}")
|
| 481 |
|
|
|
|
| 3788 |
return views
|
| 3789 |
|
| 3790 |
|
| 3791 |
+
def narrow_view_keys(views, hidden):
|
| 3792 |
+
"""D-464: a WIRE COPY of `views` whose column-bearing config names no key in `hidden`.
|
| 3793 |
+
|
| 3794 |
+
`views_from_defs` merges saved configs verbatim and runs before the field wall exists, so a
|
| 3795 |
+
hidden field's KEY rode `/workspace` in `order`/`visible`/`widths`/`sorts`/`groupBy`/`colorBy`
|
| 3796 |
+
(and `stackField`, `summary`, `display.*Field`) to the very reader the wall targets. Each
|
| 3797 |
+
assembly calls this beside its overlay narrowing,
|
| 3798 |
+
on `workspace["views"]` only: `g["views"]` stays whole for server-side work (exports, measure
|
| 3799 |
+
sets), and a walled reader's own autosave is already cleaned against their walled keys.
|
| 3800 |
+
⚠ FILTER LEAVES (and `secondaryView`) ARE LEFT ALONE: dropping a leaf would change which rows
|
| 3801 |
+
a view shows, so a hidden key named only in a filter still rides this wire."""
|
| 3802 |
+
hidden = set(hidden or ())
|
| 3803 |
+
if not hidden:
|
| 3804 |
+
return views
|
| 3805 |
+
out = []
|
| 3806 |
+
for view in views or []:
|
| 3807 |
+
cfg = view.get("config") if isinstance(view, dict) else None
|
| 3808 |
+
if not isinstance(cfg, dict):
|
| 3809 |
+
out.append(view)
|
| 3810 |
+
continue
|
| 3811 |
+
cfg = dict(cfg)
|
| 3812 |
+
for name in ("order", "visible"):
|
| 3813 |
+
if isinstance(cfg.get(name), list):
|
| 3814 |
+
cfg[name] = [key for key in cfg[name] if key not in hidden]
|
| 3815 |
+
if isinstance(cfg.get("widths"), dict):
|
| 3816 |
+
cfg["widths"] = {key: w for key, w in cfg["widths"].items() if key not in hidden}
|
| 3817 |
+
if isinstance(cfg.get("sorts"), list):
|
| 3818 |
+
cfg["sorts"] = [s for s in cfg["sorts"]
|
| 3819 |
+
if not (isinstance(s, dict) and s.get("colId") in hidden)]
|
| 3820 |
+
for name in ("groupBy", "colorBy", "stackField"):
|
| 3821 |
+
if isinstance(cfg.get(name), str) and cfg[name] in hidden:
|
| 3822 |
+
cfg[name] = None
|
| 3823 |
+
if isinstance(cfg.get("summary"), dict):
|
| 3824 |
+
cfg["summary"] = {key: s for key, s in cfg["summary"].items() if key not in hidden}
|
| 3825 |
+
if isinstance(cfg.get("display"), dict):
|
| 3826 |
+
# `DisplaySpec`'s `dateField`/`stackField`/`titleField`/`colorField`/... members.
|
| 3827 |
+
cfg["display"] = {name: value for name, value in cfg["display"].items()
|
| 3828 |
+
if not (name.endswith("Field") and isinstance(value, str)
|
| 3829 |
+
and value in hidden)}
|
| 3830 |
+
out.append(dict(view, config=cfg))
|
| 3831 |
+
return out
|
| 3832 |
+
|
| 3833 |
+
|
| 3834 |
def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None,
|
| 3835 |
fields_base=None, with_cohorts=True, st=None):
|
| 3836 |
"""The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE
|
platform/core/connector_sync.py
CHANGED
|
@@ -8,6 +8,7 @@ from __future__ import annotations
|
|
| 8 |
import datetime as _dt
|
| 9 |
import hashlib
|
| 10 |
import json
|
|
|
|
| 11 |
import time
|
| 12 |
import uuid
|
| 13 |
|
|
@@ -20,6 +21,77 @@ _ERROR_LIMIT = 50
|
|
| 20 |
_DEFAULT_CONNECTION = "default"
|
| 21 |
_PROVIDERS = {}
|
| 22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
# One receipt per immutable database identity. No credentials, session, or source values.
|
| 24 |
JOBS_KEY = "connector_materialization_jobs"
|
| 25 |
MAX_ATTEMPTS = 3
|
|
@@ -34,6 +106,9 @@ def job_key(provider, connection_id, database_key):
|
|
| 34 |
|
| 35 |
def _job_public(row):
|
| 36 |
answer = {k: v for k, v in row.items() if k != "leaseToken"}
|
|
|
|
|
|
|
|
|
|
| 37 |
answer["retryable"] = (row.get("state") == "failed"
|
| 38 |
and int(row.get("attempts", 0)) < MAX_ATTEMPTS)
|
| 39 |
answer["maxAttempts"] = MAX_ATTEMPTS
|
|
@@ -104,6 +179,36 @@ def enqueue(rt, provider, connection_id, database_key, *, model="", retry=False,
|
|
| 104 |
return _job_public(landed[key])
|
| 105 |
|
| 106 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
def claim(rt, candidate, *, now=None):
|
| 108 |
"""Atomically claim queued or abandoned work; replay callbacks have no side effects."""
|
| 109 |
key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
|
|
@@ -117,6 +222,7 @@ def claim(rt, candidate, *, now=None):
|
|
| 117 |
if row.get("state") != "queued" and not (
|
| 118 |
row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp):
|
| 119 |
return doc
|
|
|
|
| 120 |
if int(row.get("attempts", 0)) >= MAX_ATTEMPTS:
|
| 121 |
row.update(state="failed", at=stamp, leaseToken="", leaseUntil=0,
|
| 122 |
error="Sync attempts exhausted. Check the connection and update its settings.")
|
|
@@ -139,7 +245,27 @@ def owns(rt, job, *, now=None):
|
|
| 139 |
and row.get("leaseUntil", 0) > (time.time() if now is None else now))
|
| 140 |
|
| 141 |
|
| 142 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
"""A stale worker cannot acknowledge, renew, or overwrite a newer receipt."""
|
| 144 |
key = job_key(job["provider"], job["connectionId"], job["databaseKey"])
|
| 145 |
stamp = time.time() if now is None else now
|
|
@@ -160,7 +286,8 @@ def finish(rt, job, *, success=False, renew=False, interrupted=False, now=None):
|
|
| 160 |
else:
|
| 161 |
row.update(state="ready" if success else "failed", at=stamp,
|
| 162 |
leaseUntil=0, leaseToken="", workspaceChanged=bool(success),
|
| 163 |
-
error="" if success else
|
|
|
|
| 164 |
doc[key] = row
|
| 165 |
return doc
|
| 166 |
|
|
@@ -178,6 +305,8 @@ def cancel(rt, provider, connection_id, database_key):
|
|
| 178 |
doc[key] = {**doc[key], "state": "configured", "at": stamp,
|
| 179 |
"leaseToken": "", "leaseUntil": 0, "workspaceChanged": False,
|
| 180 |
"error": ""}
|
|
|
|
|
|
|
| 181 |
return doc
|
| 182 |
|
| 183 |
rt.update(JOBS_KEY, apply, flush="sync")
|
|
@@ -363,3 +492,5 @@ register_provider(
|
|
| 363 |
label="Amazon",
|
| 364 |
retry_hint="No sync runner is wired for this connector yet.",
|
| 365 |
)
|
|
|
|
|
|
|
|
|
| 8 |
import datetime as _dt
|
| 9 |
import hashlib
|
| 10 |
import json
|
| 11 |
+
import threading
|
| 12 |
import time
|
| 13 |
import uuid
|
| 14 |
|
|
|
|
| 21 |
_DEFAULT_CONNECTION = "default"
|
| 22 |
_PROVIDERS = {}
|
| 23 |
|
| 24 |
+
# One in-flight Sync now per (provider, tenant). Only the resident resync thread drains this
|
| 25 |
+
# intent desk. Nothing here imports a provider, calls a runner, or writes to the mirror/store.
|
| 26 |
+
_SYNC_REQUESTS = {}
|
| 27 |
+
_SYNC_REQUEST_LOCK = threading.Lock()
|
| 28 |
+
_SYNC_WAKE = threading.Event()
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def request_sync(provider, tenant, *, requested_by="", now=None):
|
| 32 |
+
"""Queue an intent. A duplicate queued/running request returns accepted=False."""
|
| 33 |
+
if provider not in _PROVIDERS:
|
| 34 |
+
raise ValueError("This connector is not registered for sync.")
|
| 35 |
+
tenant = str(tenant or "").strip()
|
| 36 |
+
if not tenant:
|
| 37 |
+
raise ValueError("A tenant is required for sync.")
|
| 38 |
+
with _SYNC_REQUEST_LOCK:
|
| 39 |
+
previous = _SYNC_REQUESTS.get((provider, tenant))
|
| 40 |
+
if previous and previous["state"] in ("queued", "running"):
|
| 41 |
+
return {**previous, "accepted": False}
|
| 42 |
+
receipt = {"provider": provider, "tenant": tenant, "state": "queued",
|
| 43 |
+
"requestedAt": time.time() if now is None else now,
|
| 44 |
+
"startedAt": None, "finishedAt": None, "detail": "",
|
| 45 |
+
"requestedBy": str(requested_by or ""), "accepted": True}
|
| 46 |
+
if requested_by == "first load" and previous and previous.get("firstLoad"):
|
| 47 |
+
# A continuation keeps the safe numeric progress visible while waiting for the writer.
|
| 48 |
+
receipt.update(detail=previous["detail"], firstLoad=dict(previous["firstLoad"]))
|
| 49 |
+
_SYNC_REQUESTS[(provider, tenant)] = receipt
|
| 50 |
+
_SYNC_WAKE.set()
|
| 51 |
+
return dict(receipt)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def sync_request(provider, tenant):
|
| 55 |
+
"""Return a detached current receipt, or None."""
|
| 56 |
+
with _SYNC_REQUEST_LOCK:
|
| 57 |
+
receipt = _SYNC_REQUESTS.get((provider, tenant))
|
| 58 |
+
if receipt is None:
|
| 59 |
+
return None
|
| 60 |
+
result = {**receipt, "accepted": False}
|
| 61 |
+
if "firstLoad" in result:
|
| 62 |
+
result["firstLoad"] = dict(result["firstLoad"])
|
| 63 |
+
return result
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def take_sync_requests():
|
| 67 |
+
"""Claim every queued intent atomically, on the resident resync thread."""
|
| 68 |
+
with _SYNC_REQUEST_LOCK:
|
| 69 |
+
_SYNC_WAKE.clear()
|
| 70 |
+
claimed = []
|
| 71 |
+
for receipt in _SYNC_REQUESTS.values():
|
| 72 |
+
if receipt["state"] == "queued":
|
| 73 |
+
receipt.update(state="running", startedAt=time.time(), accepted=False)
|
| 74 |
+
claimed.append(dict(receipt))
|
| 75 |
+
return claimed
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def finish_sync_request(provider, tenant, *, ok, detail="", now=None, first_load=None):
|
| 79 |
+
"""Finish a claimed intent. The API runner supplies a safe cause sentence, never raw errors."""
|
| 80 |
+
with _SYNC_REQUEST_LOCK:
|
| 81 |
+
receipt = _SYNC_REQUESTS.get((provider, tenant))
|
| 82 |
+
if receipt is None or receipt["state"] != "running":
|
| 83 |
+
return None
|
| 84 |
+
receipt.update(state="done" if ok else "failed", detail=str(detail or "")[:300],
|
| 85 |
+
finishedAt=time.time() if now is None else now, accepted=False)
|
| 86 |
+
if first_load is not None:
|
| 87 |
+
receipt["firstLoad"] = dict(first_load)
|
| 88 |
+
return dict(receipt)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def sync_wake_event():
|
| 92 |
+
"""The event the resident thread waits on between passes."""
|
| 93 |
+
return _SYNC_WAKE
|
| 94 |
+
|
| 95 |
# One receipt per immutable database identity. No credentials, session, or source values.
|
| 96 |
JOBS_KEY = "connector_materialization_jobs"
|
| 97 |
MAX_ATTEMPTS = 3
|
|
|
|
| 106 |
|
| 107 |
def _job_public(row):
|
| 108 |
answer = {k: v for k, v in row.items() if k != "leaseToken"}
|
| 109 |
+
# Older status readers only know error. Keep one persisted cause and project it there.
|
| 110 |
+
if row.get("state") == "queued" and row.get("stall"):
|
| 111 |
+
answer["error"] = row["stall"]["cause"]
|
| 112 |
answer["retryable"] = (row.get("state") == "failed"
|
| 113 |
and int(row.get("attempts", 0)) < MAX_ATTEMPTS)
|
| 114 |
answer["maxAttempts"] = MAX_ATTEMPTS
|
|
|
|
| 179 |
return _job_public(landed[key])
|
| 180 |
|
| 181 |
|
| 182 |
+
def stall(rt, candidate, cause, *, now=None):
|
| 183 |
+
"""Record WHY queued work cannot start, without consuming an attempt."""
|
| 184 |
+
key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
|
| 185 |
+
stamp = time.time() if now is None else now
|
| 186 |
+
sentence = str(cause or "")[:300]
|
| 187 |
+
|
| 188 |
+
def eligible(row):
|
| 189 |
+
return (row.get("id") == candidate.get("id") and
|
| 190 |
+
(row.get("state") == "queued" or
|
| 191 |
+
row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp))
|
| 192 |
+
|
| 193 |
+
previous = (rt.get(JOBS_KEY, fresh=True) or {}).get(key) or {}
|
| 194 |
+
# Avoid even submitting a store update on the unchanged 15-second drain pass.
|
| 195 |
+
if not eligible(previous) or (previous.get("stall") or {}).get("cause") == sentence:
|
| 196 |
+
return _job_public(previous)
|
| 197 |
+
|
| 198 |
+
def apply(current):
|
| 199 |
+
doc = dict(current or {})
|
| 200 |
+
row = dict(doc.get(key) or {})
|
| 201 |
+
if not eligible(row) or (row.get("stall") or {}).get("cause") == sentence:
|
| 202 |
+
return doc
|
| 203 |
+
row.update(state="queued", leaseUntil=0, leaseToken="",
|
| 204 |
+
stall={"at": stamp, "cause": sentence})
|
| 205 |
+
doc[key] = row
|
| 206 |
+
return doc
|
| 207 |
+
|
| 208 |
+
landed = rt.update(JOBS_KEY, apply, flush="sync")
|
| 209 |
+
return _job_public(landed.get(key) or {})
|
| 210 |
+
|
| 211 |
+
|
| 212 |
def claim(rt, candidate, *, now=None):
|
| 213 |
"""Atomically claim queued or abandoned work; replay callbacks have no side effects."""
|
| 214 |
key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
|
|
|
|
| 222 |
if row.get("state") != "queued" and not (
|
| 223 |
row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp):
|
| 224 |
return doc
|
| 225 |
+
row.pop("stall", None)
|
| 226 |
if int(row.get("attempts", 0)) >= MAX_ATTEMPTS:
|
| 227 |
row.update(state="failed", at=stamp, leaseToken="", leaseUntil=0,
|
| 228 |
error="Sync attempts exhausted. Check the connection and update its settings.")
|
|
|
|
| 245 |
and row.get("leaseUntil", 0) > (time.time() if now is None else now))
|
| 246 |
|
| 247 |
|
| 248 |
+
def progress(rt, job, *, rows, total):
|
| 249 |
+
"""Record published rows only while this worker still owns its unexpired lease."""
|
| 250 |
+
key = job_key(job["provider"], job["connectionId"], job["databaseKey"])
|
| 251 |
+
stamp = time.time()
|
| 252 |
+
|
| 253 |
+
def apply(current):
|
| 254 |
+
doc = dict(current or {})
|
| 255 |
+
row = dict(doc.get(key) or {})
|
| 256 |
+
if (row.get("id") != job.get("id") or row.get("state") != "syncing"
|
| 257 |
+
or row.get("leaseToken") != job.get("leaseToken")
|
| 258 |
+
or row.get("leaseUntil", 0) <= stamp):
|
| 259 |
+
return doc
|
| 260 |
+
row["progress"] = {"rows": rows, "total": total, "at": stamp}
|
| 261 |
+
doc[key] = row
|
| 262 |
+
return doc
|
| 263 |
+
|
| 264 |
+
landed = rt.update(JOBS_KEY, apply, flush="sync")
|
| 265 |
+
return _job_public(landed.get(key) or {})
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def finish(rt, job, *, success=False, renew=False, interrupted=False, cause="", now=None):
|
| 269 |
"""A stale worker cannot acknowledge, renew, or overwrite a newer receipt."""
|
| 270 |
key = job_key(job["provider"], job["connectionId"], job["databaseKey"])
|
| 271 |
stamp = time.time() if now is None else now
|
|
|
|
| 286 |
else:
|
| 287 |
row.update(state="ready" if success else "failed", at=stamp,
|
| 288 |
leaseUntil=0, leaseToken="", workspaceChanged=bool(success),
|
| 289 |
+
error="" if success else (str(cause or "")[:300] or
|
| 290 |
+
"Rows could not be synced. Check the connection and retry."))
|
| 291 |
doc[key] = row
|
| 292 |
return doc
|
| 293 |
|
|
|
|
| 305 |
doc[key] = {**doc[key], "state": "configured", "at": stamp,
|
| 306 |
"leaseToken": "", "leaseUntil": 0, "workspaceChanged": False,
|
| 307 |
"error": ""}
|
| 308 |
+
doc[key].pop("stall", None)
|
| 309 |
+
doc[key].pop("progress", None)
|
| 310 |
return doc
|
| 311 |
|
| 312 |
rt.update(JOBS_KEY, apply, flush="sync")
|
|
|
|
| 492 |
label="Amazon",
|
| 493 |
retry_hint="No sync runner is wired for this connector yet.",
|
| 494 |
)
|
| 495 |
+
register_provider("meta_ads", label="Meta Ads",
|
| 496 |
+
retry_hint="Use Sync now to retry the Meta Ads pull.")
|
platform/core/dependents.py
CHANGED
|
@@ -47,7 +47,13 @@ walked directly instead, exactly as the shape allows.
|
|
| 47 |
FIELD KEY (a string) of the link the rollup traverses. A rollup may also carry no `link` at all
|
| 48 |
(the `{'source': {...}}` shape, which joins through a topic dim). Every walker below branches on
|
| 49 |
`field['type']` first and on which key is present second.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
"""
|
|
|
|
|
|
|
| 51 |
import core.chart_system as chart_system
|
| 52 |
import core.route_system as route_system
|
| 53 |
import core.shares as shares
|
|
@@ -60,6 +66,13 @@ from core.table_store import ASSIGNED_KEY, SHARED_KEY
|
|
| 60 |
#: happened to register first (W49-T11 registers `sheet_cell` after the merge).
|
| 61 |
KIND_ORDER = ('view', 'route', 'link', 'rollup', 'grant', 'chart', 'sheet_cell')
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
#: Singular / plural noun phrase per kind. ⛔ THE CASING IS R23's OWN — lowercase `views`, capital
|
| 64 |
#: `Route`, capital `Sheet cells` — because the ruling quotes the sentence the user reads.
|
| 65 |
_NOUNS = {
|
|
@@ -75,6 +88,36 @@ _NOUNS = {
|
|
| 75 |
_PROVIDERS = {}
|
| 76 |
|
| 77 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
def noun(kind, count):
|
| 79 |
"""The noun phrase for `count` of `kind`, agreeing in number."""
|
| 80 |
pair = _NOUNS.get(str(kind or ''))
|
|
@@ -120,7 +163,7 @@ def clean_target(target):
|
|
| 120 |
return {'kind': kind, 'key': key, 'database': database}
|
| 121 |
|
| 122 |
|
| 123 |
-
def _dependent(kind, ids):
|
| 124 |
"""A Dependent, or None when nothing depends. ⛔ NONE IS THE ZERO CASE — never `count: 0`."""
|
| 125 |
seen, out = set(), []
|
| 126 |
for raw in ids or ():
|
|
@@ -130,7 +173,13 @@ def _dependent(kind, ids):
|
|
| 130 |
out.append(value)
|
| 131 |
if not out:
|
| 132 |
return None
|
| 133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
|
| 135 |
|
| 136 |
def dependents(st, target):
|
|
@@ -162,10 +211,36 @@ def dependents_report(st, target):
|
|
| 162 |
count = int(item.get('count') or len(ids) or 0)
|
| 163 |
if count <= 0:
|
| 164 |
continue
|
| 165 |
-
|
| 166 |
'label': str(item.get('label') or noun(kind, count)),
|
| 167 |
'count': count,
|
| 168 |
-
'ids': [str(i) for i in ids]}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
order = {kind: index for index, kind in enumerate(KIND_ORDER)}
|
| 170 |
found.sort(key=lambda item: (order.get(item['kind'], len(order)), item['kind']))
|
| 171 |
return {'dependents': found, 'errors': errors}
|
|
@@ -334,16 +409,27 @@ def view_dependents(st, target):
|
|
| 334 |
"""
|
| 335 |
kind, key, database = target['kind'], target['key'], target['database']
|
| 336 |
if kind == 'table':
|
| 337 |
-
|
|
|
|
|
|
|
| 338 |
if kind == 'view':
|
| 339 |
if not database:
|
| 340 |
return None
|
| 341 |
-
|
| 342 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 343 |
for _owner, view_id, view in walk_views(st, database):
|
| 344 |
-
if _view_names_field(view, key):
|
| 345 |
-
|
| 346 |
-
|
|
|
|
| 347 |
|
| 348 |
|
| 349 |
def _view_names_field(view, field_key):
|
|
@@ -383,21 +469,31 @@ def route_dependents(st, target):
|
|
| 383 |
a `geographicField`, which is what makes them a FIELD dependent too.
|
| 384 |
"""
|
| 385 |
kind, key, database = target['kind'], target['key'], target['database']
|
| 386 |
-
routes,
|
| 387 |
if kind == 'view':
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 401 |
|
| 402 |
|
| 403 |
def link_dependents(st, target):
|
|
@@ -410,18 +506,18 @@ def link_dependents(st, target):
|
|
| 410 |
kind, key = target['kind'], target['key']
|
| 411 |
if kind == 'view':
|
| 412 |
return None
|
| 413 |
-
|
| 414 |
for table_key, definition in _defs(st).items():
|
| 415 |
for field in _fields(definition):
|
| 416 |
bag = _link_bag(field)
|
| 417 |
if bag is None:
|
| 418 |
continue
|
| 419 |
if kind == 'table' and str(bag.get('table') or '') == key and str(table_key) != key:
|
| 420 |
-
|
| 421 |
elif kind == 'field' and str(bag.get('inverse') or '') == key \
|
| 422 |
and str(bag.get('table') or '') == str(target['database']):
|
| 423 |
-
|
| 424 |
-
return _dependent('link',
|
| 425 |
|
| 426 |
|
| 427 |
def rollup_dependents(st, target):
|
|
@@ -435,7 +531,7 @@ def rollup_dependents(st, target):
|
|
| 435 |
kind, key = target['kind'], target['key']
|
| 436 |
if kind == 'view':
|
| 437 |
return None
|
| 438 |
-
|
| 439 |
for table_key, definition in _defs(st).items():
|
| 440 |
fields = _fields(definition)
|
| 441 |
by_key = {str(f.get('key') or ''): f for f in fields}
|
|
@@ -449,18 +545,18 @@ def rollup_dependents(st, target):
|
|
| 449 |
hop_bag = _link_bag(hop_field) if hop_field else None
|
| 450 |
if kind == 'table':
|
| 451 |
if source and str(source.get('topic') or '') == key:
|
| 452 |
-
|
| 453 |
elif hop_bag is not None and str(hop_bag.get('table') or '') == key:
|
| 454 |
-
|
| 455 |
continue
|
| 456 |
if str(target['database']) and str(table_key) != str(target['database']):
|
| 457 |
continue
|
| 458 |
if hop == key or str(bag.get('field') or '') == key:
|
| 459 |
-
|
| 460 |
elif source and key in (str(source.get('on') or ''), str(source.get('groupBy') or ''),
|
| 461 |
str(source.get('measure') or '')):
|
| 462 |
-
|
| 463 |
-
return _dependent('rollup',
|
| 464 |
|
| 465 |
|
| 466 |
def grant_dependents(st, target):
|
|
@@ -471,13 +567,21 @@ def grant_dependents(st, target):
|
|
| 471 |
* field -> the `field` grant on `<database>:<key>`
|
| 472 |
"""
|
| 473 |
kind, key, database = target['kind'], target['key'], target['database']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 474 |
if kind == 'view':
|
| 475 |
-
return
|
| 476 |
if kind == 'field':
|
| 477 |
if not database:
|
| 478 |
return None
|
| 479 |
record = shares.grants('field', shares.field_oid(database, key), st=st)
|
| 480 |
-
return
|
| 481 |
holders = [str(e.get('user') or '')
|
| 482 |
for e in (shares.grants('database', key, st=st).get('entries') or ())]
|
| 483 |
try:
|
|
@@ -490,21 +594,110 @@ def grant_dependents(st, target):
|
|
| 490 |
continue
|
| 491 |
for entry in (shares.grants('field', oid, st=st).get('entries') or ()):
|
| 492 |
holders.append(str(entry.get('user') or ''))
|
| 493 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 494 |
|
| 495 |
|
| 496 |
def chart_dependents(st, target):
|
| 497 |
-
"""Saved charts filed against the target DATABASE.
|
| 498 |
|
| 499 |
⛔ A CHART NEVER DEPENDS ON A VIEW, and that is `chart_system`'s stated contract rather than an
|
| 500 |
omission here: a chart's `config` is OPAQUE and this file must not start reading keys out of
|
| 501 |
it to guess at a reference. Deleting a view cannot delete a chart, so it must not disclose one.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 502 |
"""
|
| 503 |
kind, key = target['kind'], target['key']
|
| 504 |
if kind != 'table':
|
| 505 |
return None
|
| 506 |
-
|
| 507 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 508 |
|
| 509 |
|
| 510 |
register('view', view_dependents)
|
|
|
|
| 47 |
FIELD KEY (a string) of the link the rollup traverses. A rollup may also carry no `link` at all
|
| 48 |
(the `{'source': {...}}` shape, which joins through a topic dim). Every walker below branches on
|
| 49 |
`field['type']` first and on which key is present second.
|
| 50 |
+
|
| 51 |
+
W60-T31 / C3: grouped kind/label/count/ids retain their meanings. Additive `items` names each
|
| 52 |
+
id in order and gives its address or a reason it cannot be opened. Data addresses use only
|
| 53 |
+
the existing #/data? prefix; workbook addresses belong to the Sheet provider (C2).
|
| 54 |
"""
|
| 55 |
+
from urllib.parse import parse_qs, urlencode
|
| 56 |
+
|
| 57 |
import core.chart_system as chart_system
|
| 58 |
import core.route_system as route_system
|
| 59 |
import core.shares as shares
|
|
|
|
| 66 |
#: happened to register first (W49-T11 registers `sheet_cell` after the merge).
|
| 67 |
KIND_ORDER = ('view', 'route', 'link', 'rollup', 'grant', 'chart', 'sheet_cell')
|
| 68 |
|
| 69 |
+
#: Built-in Default views are not dependents, including their saved owner copies. Their ids are
|
| 70 |
+
#: shared by EVERY database, so counting them charges one database's Default view to another.
|
| 71 |
+
#: `aios_grid.py` mints `all-customers` on every topic; its `IG_OVERVIEW_ID` is `tpl_overview`.
|
| 72 |
+
#: Hardcoded here because `core/` may not reach UP into `aios_grid`; `verify_shared_views.py`
|
| 73 |
+
#: pins this pair against that module so the declarations cannot drift.
|
| 74 |
+
PINNED_VIEW_IDS = frozenset({'all-customers', 'tpl_overview'})
|
| 75 |
+
|
| 76 |
#: Singular / plural noun phrase per kind. ⛔ THE CASING IS R23's OWN — lowercase `views`, capital
|
| 77 |
#: `Route`, capital `Sheet cells` — because the ruling quotes the sentence the user reads.
|
| 78 |
_NOUNS = {
|
|
|
|
| 88 |
_PROVIDERS = {}
|
| 89 |
|
| 90 |
|
| 91 |
+
DATA_HASH_PREFIX = '#/data?'
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def data_hash(database, **params):
|
| 95 |
+
"""Build a #/data?database=... address. The only hash builder in this module."""
|
| 96 |
+
database = str(database or '').strip()
|
| 97 |
+
if not database:
|
| 98 |
+
return None
|
| 99 |
+
query = {'database': database}
|
| 100 |
+
query.update({key: str(value) for key, value in params.items()
|
| 101 |
+
if key != 'database' and value is not None and str(value) != ''})
|
| 102 |
+
return DATA_HASH_PREFIX + urlencode(query)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _dependent_item(item_id, label, href=None, reason=''):
|
| 106 |
+
"""A named thing with an existing hash address or an explicit explanation."""
|
| 107 |
+
item_id = str(item_id or '')
|
| 108 |
+
label = str(label or item_id)
|
| 109 |
+
href = href if isinstance(href, str) and href.startswith((DATA_HASH_PREFIX, '#/sheet?')) else None
|
| 110 |
+
return {'id': item_id, 'label': label, 'href': href,
|
| 111 |
+
'reason': str(reason or '') if href else
|
| 112 |
+
str(reason or f'{label} has no direct address available.')}
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _field_item(table_key, field):
|
| 116 |
+
key = str(field.get('key') or '')
|
| 117 |
+
return _dependent_item(f'{table_key}:{key}', field.get('label') or key,
|
| 118 |
+
data_hash(table_key, field=key) if key else None)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
def noun(kind, count):
|
| 122 |
"""The noun phrase for `count` of `kind`, agreeing in number."""
|
| 123 |
pair = _NOUNS.get(str(kind or ''))
|
|
|
|
| 163 |
return {'kind': kind, 'key': key, 'database': database}
|
| 164 |
|
| 165 |
|
| 166 |
+
def _dependent(kind, ids, items=None):
|
| 167 |
"""A Dependent, or None when nothing depends. ⛔ NONE IS THE ZERO CASE — never `count: 0`."""
|
| 168 |
seen, out = set(), []
|
| 169 |
for raw in ids or ():
|
|
|
|
| 173 |
out.append(value)
|
| 174 |
if not out:
|
| 175 |
return None
|
| 176 |
+
result = {'kind': kind, 'label': noun(kind, len(out)), 'count': len(out), 'ids': out}
|
| 177 |
+
if items is not None:
|
| 178 |
+
by_id = {}
|
| 179 |
+
for item in items:
|
| 180 |
+
by_id.setdefault(str(item.get('id') or ''), item)
|
| 181 |
+
result['items'] = [by_id.get(value) or _dependent_item(value, value) for value in out]
|
| 182 |
+
return result
|
| 183 |
|
| 184 |
|
| 185 |
def dependents(st, target):
|
|
|
|
| 211 |
count = int(item.get('count') or len(ids) or 0)
|
| 212 |
if count <= 0:
|
| 213 |
continue
|
| 214 |
+
normalized = {'kind': str(item.get('kind') or kind),
|
| 215 |
'label': str(item.get('label') or noun(kind, count)),
|
| 216 |
'count': count,
|
| 217 |
+
'ids': [str(i) for i in ids]}
|
| 218 |
+
# Lane A may supply a single Sheet address on the dependent itself. Preserve
|
| 219 |
+
# both that shape and per-id items; neither changes the legacy count or label.
|
| 220 |
+
if 'href' in item:
|
| 221 |
+
normalized['href'] = item['href'] if isinstance(item['href'], str) else None
|
| 222 |
+
if 'reason' in item:
|
| 223 |
+
normalized['reason'] = str(item.get('reason') or '')
|
| 224 |
+
# W60-T12 (combine-owned), 2026-09-14: `refusal` is the OTHER half of the same fact
|
| 225 |
+
# and was being dropped at this seam. A provider says "no address" with `href: None`
|
| 226 |
+
# AND says WHY in `refusal`; carrying the null while dropping the cause turns an
|
| 227 |
+
# explained dead end into an unexplained one, which is the defect R23's sentence
|
| 228 |
+
# exists to prevent. Both keys are optional and neither changes count or label.
|
| 229 |
+
if 'refusal' in item:
|
| 230 |
+
normalized['refusal'] = str(item.get('refusal') or '')
|
| 231 |
+
if isinstance(item.get('items'), list):
|
| 232 |
+
normalized['items'] = []
|
| 233 |
+
for entry in item['items']:
|
| 234 |
+
entry = entry if isinstance(entry, dict) else {'id': entry}
|
| 235 |
+
href = entry.get('href') if isinstance(entry.get('href'), str) else None
|
| 236 |
+
normalized['items'].append({
|
| 237 |
+
'id': str(entry.get('id') or ''),
|
| 238 |
+
'label': str(entry.get('label') or entry.get('id') or ''),
|
| 239 |
+
'href': href,
|
| 240 |
+
'reason': str(entry.get('reason') or ('' if href else
|
| 241 |
+
'This dependent has no direct address available.')),
|
| 242 |
+
})
|
| 243 |
+
found.append(normalized)
|
| 244 |
order = {kind: index for index, kind in enumerate(KIND_ORDER)}
|
| 245 |
found.sort(key=lambda item: (order.get(item['kind'], len(order)), item['kind']))
|
| 246 |
return {'dependents': found, 'errors': errors}
|
|
|
|
| 409 |
"""
|
| 410 |
kind, key, database = target['kind'], target['key'], target['database']
|
| 411 |
if kind == 'table':
|
| 412 |
+
items = [_dependent_item(vid, view.get('name') or vid, data_hash(key, view=vid))
|
| 413 |
+
for _owner, vid, view in walk_views(st, key) if vid not in PINNED_VIEW_IDS]
|
| 414 |
+
return _dependent('view', [item['id'] for item in items], items=items)
|
| 415 |
if kind == 'view':
|
| 416 |
if not database:
|
| 417 |
return None
|
| 418 |
+
ids = _assigned_ids(st, database, key)
|
| 419 |
+
assignments = _table_doc(st, database).get(ASSIGNED_KEY) or {}
|
| 420 |
+
items = []
|
| 421 |
+
for item_id in ids:
|
| 422 |
+
recipient, _, view_id = item_id.partition(':')
|
| 423 |
+
view = assignments[recipient]['views'][view_id]
|
| 424 |
+
label = f'{recipient}: {view.get("name") or view_id}'
|
| 425 |
+
items.append(_dependent_item(item_id, label, data_hash(database, view=view_id)))
|
| 426 |
+
return _dependent('view', ids, items=items)
|
| 427 |
+
items = []
|
| 428 |
for _owner, view_id, view in walk_views(st, database):
|
| 429 |
+
if view_id not in PINNED_VIEW_IDS and _view_names_field(view, key):
|
| 430 |
+
items.append(_dependent_item(view_id, view.get('name') or view_id,
|
| 431 |
+
data_hash(database, view=view_id)))
|
| 432 |
+
return _dependent('view', [item['id'] for item in items], items=items)
|
| 433 |
|
| 434 |
|
| 435 |
def _view_names_field(view, field_key):
|
|
|
|
| 469 |
a `geographicField`, which is what makes them a FIELD dependent too.
|
| 470 |
"""
|
| 471 |
kind, key, database = target['kind'], target['key'], target['database']
|
| 472 |
+
routes, matched_routes, matched_zones = _routes(st), [], []
|
| 473 |
if kind == 'view':
|
| 474 |
+
matched_routes = [r for r in routes if str(r.get('sourceViewId') or '') == key]
|
| 475 |
+
matched_zones = [z for z in _zones(st, database or None)
|
| 476 |
+
if str(z.get('sourceViewId') or '') == key]
|
| 477 |
+
elif kind == 'table':
|
| 478 |
+
view_ids = {vid for _owner, vid, _v in walk_views(st, key)
|
| 479 |
+
if vid not in PINNED_VIEW_IDS}
|
| 480 |
+
# A-QI7: Routes store no source database and are created only from customer_data.
|
| 481 |
+
# The database test must therefore use the target table key, not the tenant-wide Route
|
| 482 |
+
# list or a shared view id. Zones already carry their own table scope.
|
| 483 |
+
if str(key) == route_system.DEFAULT_ZONE_TABLE:
|
| 484 |
+
matched_routes = [r for r in routes if str(r.get('sourceViewId') or '') in view_ids]
|
| 485 |
+
matched_zones = _zones(st, key)
|
| 486 |
+
else:
|
| 487 |
+
matched_zones = [z for z in _zones(st, database)
|
| 488 |
+
if str(z.get('geographicField') or '') == key]
|
| 489 |
+
items = [_dependent_item(str(r.get('id')), r.get('name') or r.get('id'),
|
| 490 |
+
data_hash(route_system.DEFAULT_ZONE_TABLE, route=r.get('id')))
|
| 491 |
+
for r in matched_routes]
|
| 492 |
+
for zone in matched_zones:
|
| 493 |
+
name = zone.get('name') or zone.get('id')
|
| 494 |
+
items.append(_dependent_item(str(zone.get('id')), name, reason=
|
| 495 |
+
f'{name} has no direct address. Edit this zone in the Map view Zones menu.'))
|
| 496 |
+
return _dependent('route', [item['id'] for item in items], items=items)
|
| 497 |
|
| 498 |
|
| 499 |
def link_dependents(st, target):
|
|
|
|
| 506 |
kind, key = target['kind'], target['key']
|
| 507 |
if kind == 'view':
|
| 508 |
return None
|
| 509 |
+
items = []
|
| 510 |
for table_key, definition in _defs(st).items():
|
| 511 |
for field in _fields(definition):
|
| 512 |
bag = _link_bag(field)
|
| 513 |
if bag is None:
|
| 514 |
continue
|
| 515 |
if kind == 'table' and str(bag.get('table') or '') == key and str(table_key) != key:
|
| 516 |
+
items.append(_field_item(table_key, field))
|
| 517 |
elif kind == 'field' and str(bag.get('inverse') or '') == key \
|
| 518 |
and str(bag.get('table') or '') == str(target['database']):
|
| 519 |
+
items.append(_field_item(table_key, field))
|
| 520 |
+
return _dependent('link', [item['id'] for item in items], items=items)
|
| 521 |
|
| 522 |
|
| 523 |
def rollup_dependents(st, target):
|
|
|
|
| 531 |
kind, key = target['kind'], target['key']
|
| 532 |
if kind == 'view':
|
| 533 |
return None
|
| 534 |
+
items = []
|
| 535 |
for table_key, definition in _defs(st).items():
|
| 536 |
fields = _fields(definition)
|
| 537 |
by_key = {str(f.get('key') or ''): f for f in fields}
|
|
|
|
| 545 |
hop_bag = _link_bag(hop_field) if hop_field else None
|
| 546 |
if kind == 'table':
|
| 547 |
if source and str(source.get('topic') or '') == key:
|
| 548 |
+
items.append(_field_item(table_key, field))
|
| 549 |
elif hop_bag is not None and str(hop_bag.get('table') or '') == key:
|
| 550 |
+
items.append(_field_item(table_key, field))
|
| 551 |
continue
|
| 552 |
if str(target['database']) and str(table_key) != str(target['database']):
|
| 553 |
continue
|
| 554 |
if hop == key or str(bag.get('field') or '') == key:
|
| 555 |
+
items.append(_field_item(table_key, field))
|
| 556 |
elif source and key in (str(source.get('on') or ''), str(source.get('groupBy') or ''),
|
| 557 |
str(source.get('measure') or '')):
|
| 558 |
+
items.append(_field_item(table_key, field))
|
| 559 |
+
return _dependent('rollup', [item['id'] for item in items], items=items)
|
| 560 |
|
| 561 |
|
| 562 |
def grant_dependents(st, target):
|
|
|
|
| 567 |
* field -> the `field` grant on `<database>:<key>`
|
| 568 |
"""
|
| 569 |
kind, key, database = target['kind'], target['key'], target['database']
|
| 570 |
+
|
| 571 |
+
def named_grants(names):
|
| 572 |
+
subject = {'view': 'view', 'table': 'database', 'field': 'field'}[kind]
|
| 573 |
+
items = [_dependent_item(name, name, reason=
|
| 574 |
+
f'{name} has no direct access address. Change their access from this '
|
| 575 |
+
f'{subject}\'s share dialog.') for name in names]
|
| 576 |
+
return _dependent('grant', names, items=items)
|
| 577 |
+
|
| 578 |
if kind == 'view':
|
| 579 |
+
return named_grants(share_audience(st, key)['names'])
|
| 580 |
if kind == 'field':
|
| 581 |
if not database:
|
| 582 |
return None
|
| 583 |
record = shares.grants('field', shares.field_oid(database, key), st=st)
|
| 584 |
+
return named_grants([str(e.get('user') or '') for e in (record.get('entries') or ())])
|
| 585 |
holders = [str(e.get('user') or '')
|
| 586 |
for e in (shares.grants('database', key, st=st).get('entries') or ())]
|
| 587 |
try:
|
|
|
|
| 594 |
continue
|
| 595 |
for entry in (shares.grants('field', oid, st=st).get('entries') or ()):
|
| 596 |
holders.append(str(entry.get('user') or ''))
|
| 597 |
+
return named_grants(holders)
|
| 598 |
+
|
| 599 |
+
|
| 600 |
+
def _sheet_chart_items(st, target):
|
| 601 |
+
"""Sheet charts drawn over a cell that depends on `target` (W60-T12, contract C2).
|
| 602 |
+
|
| 603 |
+
⛔⛔ THE DEPENDENCY DECISION IS NOT REMADE HERE, AND THAT IS THE WHOLE DESIGN. Whether a CELL
|
| 604 |
+
depends on the target is `modules/sheet.py`'s answer, and `core/` may not reach UP into
|
| 605 |
+
`modules/` to ask it directly -- the same layering rule `PINNED_VIEW_IDS` is hardcoded for. So
|
| 606 |
+
this reads the ALREADY-REGISTERED `sheet_cell` provider's own rows and takes the cells it
|
| 607 |
+
named. One source of truth for the predicate, so a chart can never disagree with the cell it
|
| 608 |
+
is drawn over; and if nothing registered `sheet_cell` there are no sheet charts to report and
|
| 609 |
+
this is silent, exactly as an absent provider is everywhere else in this file.
|
| 610 |
+
|
| 611 |
+
⚠ A CELL WITH NO ADDRESS CONTRIBUTES NO CHART ADDRESS. `sheet_cell` answers `href: None` plus a
|
| 612 |
+
cause when the destination is hidden or unreadable, and a link built anyway would open some
|
| 613 |
+
OTHER cell -- which is the defect W60-T12 exists to close, not one to reintroduce one layer up.
|
| 614 |
+
"""
|
| 615 |
+
provider = _PROVIDERS.get('sheet_cell')
|
| 616 |
+
if provider is None:
|
| 617 |
+
return []
|
| 618 |
+
try:
|
| 619 |
+
rows = provider(st, dict(target))
|
| 620 |
+
except Exception:
|
| 621 |
+
return [] # a provider that raises omits its kind; so does this
|
| 622 |
+
wanted = {}
|
| 623 |
+
for row in rows if isinstance(rows, (list, tuple)) else ():
|
| 624 |
+
if not isinstance(row, dict) or row.get('kind') != 'sheet_cell':
|
| 625 |
+
continue
|
| 626 |
+
href = row.get('href')
|
| 627 |
+
if not isinstance(href, str) or not href.startswith('#/sheet?'):
|
| 628 |
+
continue
|
| 629 |
+
params = parse_qs(href.split('?', 1)[1])
|
| 630 |
+
workbook_id = (params.get('wb') or [''])[0]
|
| 631 |
+
tab_id = (params.get('tab') or [''])[0]
|
| 632 |
+
addr = (params.get('cell') or [''])[0]
|
| 633 |
+
if workbook_id and tab_id and addr:
|
| 634 |
+
wanted.setdefault((workbook_id, tab_id), []).append(addr)
|
| 635 |
+
if not wanted:
|
| 636 |
+
return []
|
| 637 |
+
|
| 638 |
+
from core import sheets as sheet_store
|
| 639 |
+
items, seen = [], set()
|
| 640 |
+
for workbook_id in dict.fromkeys(book for book, _ in wanted):
|
| 641 |
+
try:
|
| 642 |
+
book = sheet_store.get_workbook(st, workbook_id)
|
| 643 |
+
except Exception:
|
| 644 |
+
continue # an unreadable workbook is `sheet_cell`'s to report
|
| 645 |
+
for tab in book.get('sheets') or ():
|
| 646 |
+
tab_id = str(tab.get('id') or '')
|
| 647 |
+
addrs = wanted.get((workbook_id, tab_id)) or []
|
| 648 |
+
if not addrs:
|
| 649 |
+
continue
|
| 650 |
+
# ⚠ THE ADDRESS MATHS IS `core/sheets.py`'s, not a second copy. `_clean_range` accepts
|
| 651 |
+
# both `"B4"` and the chart's `{col0,row0,col1,row1}` and answers the same zero-based
|
| 652 |
+
# rectangle for each, so a cell and the chart over it cannot drift apart.
|
| 653 |
+
cells = [rect for rect in (sheet_store._clean_range(a) for a in addrs) if rect]
|
| 654 |
+
for chart in tab.get('charts') or ():
|
| 655 |
+
if not isinstance(chart, dict):
|
| 656 |
+
continue
|
| 657 |
+
chart_id = str(chart.get('id') or '')
|
| 658 |
+
rect = sheet_store._clean_range(chart.get('range'))
|
| 659 |
+
if not chart_id or chart_id in seen or not rect:
|
| 660 |
+
continue
|
| 661 |
+
if not any(rect['col0'] <= c['col0'] <= rect['col1']
|
| 662 |
+
and rect['row0'] <= c['row0'] <= rect['row1'] for c in cells):
|
| 663 |
+
continue
|
| 664 |
+
seen.add(chart_id)
|
| 665 |
+
label = str(chart.get('name') or chart.get('title') or '').strip() or (
|
| 666 |
+
f"Chart in {book.get('name') or workbook_id} / {tab.get('name') or tab_id}")
|
| 667 |
+
items.append(_dependent_item(chart_id, label, '#/sheet?' + urlencode(
|
| 668 |
+
{'wb': workbook_id, 'tab': tab_id, 'chart': chart_id})))
|
| 669 |
+
return items
|
| 670 |
|
| 671 |
|
| 672 |
def chart_dependents(st, target):
|
| 673 |
+
"""Saved charts filed against the target DATABASE, and Sheet charts drawn over its cells.
|
| 674 |
|
| 675 |
⛔ A CHART NEVER DEPENDS ON A VIEW, and that is `chart_system`'s stated contract rather than an
|
| 676 |
omission here: a chart's `config` is OPAQUE and this file must not start reading keys out of
|
| 677 |
it to guess at a reference. Deleting a view cannot delete a chart, so it must not disclose one.
|
| 678 |
+
|
| 679 |
+
⭐ W60-T12 (combine-owned) ADDED THE SECOND SOURCE. A Sheet chart is not reachable from
|
| 680 |
+
`sheet_dependents` at all -- it is a sibling of the cells, not one of them -- so before this a
|
| 681 |
+
database could be switched off while a chart still drew from it, and R23's refusal would not
|
| 682 |
+
have said so. Charts are not read from the sheet document here; see `_sheet_chart_items`.
|
| 683 |
"""
|
| 684 |
kind, key = target['kind'], target['key']
|
| 685 |
if kind != 'table':
|
| 686 |
return None
|
| 687 |
+
items = [_dependent_item(str(c.get('id')), c.get('name') or c.get('id'), c.get('href'),
|
| 688 |
+
'' if c.get('href') else
|
| 689 |
+
'This chart has no direct address. Open it from Charts.')
|
| 690 |
+
for c in _charts(st) if str(c.get('database') or '') == key]
|
| 691 |
+
items += _sheet_chart_items(st, target)
|
| 692 |
+
row = _dependent('chart', [item['id'] for item in items], items=items)
|
| 693 |
+
# ⭐ WHEN EXACTLY ONE CHART DEPENDS ON THE TARGET, THE ROW ADDRESSES ONE THING, so it carries
|
| 694 |
+
# that address itself -- the single-address-on-the-dependent shape `dependents_report` already
|
| 695 |
+
# preserves. ⛔ With several it carries none: a row-level link would have to pick one chart and
|
| 696 |
+
# would silently mean the others, which is the naming-by-display defect W60-T12 closed.
|
| 697 |
+
if row and len(items) == 1:
|
| 698 |
+
row['href'] = items[0]['href']
|
| 699 |
+
row['reason'] = items[0]['reason']
|
| 700 |
+
return row
|
| 701 |
|
| 702 |
|
| 703 |
register('view', view_dependents)
|
platform/core/external_access.py
CHANGED
|
@@ -1,8 +1,9 @@
|
|
| 1 |
-
"""Per-user credentials
|
|
|
|
| 2 |
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
credential, user, bucket, or tenant runtime.
|
| 7 |
"""
|
| 8 |
import calendar
|
|
@@ -17,6 +18,11 @@ from harness import runtime
|
|
| 17 |
|
| 18 |
|
| 19 |
BUCKET = "external_access"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
ALGORITHM = "sha256-salt-v1"
|
| 21 |
AUTH_ERROR = "invalid_external_credential"
|
| 22 |
_DUMMY_SALT = bytes(32)
|
|
@@ -48,6 +54,50 @@ class ExternalAuthError(ExternalAccessError):
|
|
| 48 |
"""A uniform authentication refusal that reveals no credential or tenant detail."""
|
| 49 |
|
| 50 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
@dataclass(frozen=True)
|
| 52 |
class VerifiedCredential:
|
| 53 |
"""The authenticated public principal plus the request-scoped tenant runtime."""
|
|
@@ -56,6 +106,8 @@ class VerifiedCredential:
|
|
| 56 |
key_id: str
|
| 57 |
user: dict
|
| 58 |
runtime: object
|
|
|
|
|
|
|
| 59 |
|
| 60 |
|
| 61 |
def _now() -> str:
|
|
@@ -130,17 +182,33 @@ def _active_public_user(username: str, tenant: str) -> dict:
|
|
| 130 |
|
| 131 |
def _metadata(record: dict) -> dict:
|
| 132 |
"""Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam."""
|
|
|
|
| 133 |
return {
|
| 134 |
"id": record.get("id"),
|
| 135 |
"label": record.get("label"),
|
| 136 |
"created": record.get("created"),
|
| 137 |
"last_used": record.get("last_used"),
|
| 138 |
"revoked_at": record.get("revoked_at"),
|
|
|
|
|
|
|
| 139 |
}
|
| 140 |
|
| 141 |
|
| 142 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
"""Create one credential and return ``(metadata, token)``; the token is never persisted."""
|
|
|
|
|
|
|
| 144 |
owner, tenant_key = _normalise_owner(username, tenant)
|
| 145 |
_assert_bound_runtime(st, tenant_key)
|
| 146 |
_active_public_user(owner, tenant_key)
|
|
@@ -168,6 +236,8 @@ def mint(st, username: str, tenant: str, label: str) -> tuple[dict, str]:
|
|
| 168 |
"username": owner,
|
| 169 |
"tenant": tenant_key,
|
| 170 |
"label": clean_label,
|
|
|
|
|
|
|
| 171 |
"salt": salt.hex(),
|
| 172 |
"hash": _digest(salt, secret),
|
| 173 |
"algo": ALGORITHM,
|
|
@@ -295,8 +365,10 @@ def verify(token: str) -> VerifiedCredential:
|
|
| 295 |
except ExternalAccessError as exc:
|
| 296 |
raise ExternalAuthError(AUTH_ERROR) from exc
|
| 297 |
|
|
|
|
| 298 |
if not _stamp_is_due(record.get("last_used"), time.time()):
|
| 299 |
-
return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st
|
|
|
|
| 300 |
|
| 301 |
used_at = _now()
|
| 302 |
|
|
@@ -321,7 +393,8 @@ def verify(token: str) -> VerifiedCredential:
|
|
| 321 |
raise
|
| 322 |
except Exception as exc:
|
| 323 |
raise ExternalAuthError(AUTH_ERROR) from exc
|
| 324 |
-
return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st
|
|
|
|
| 325 |
except ExternalAuthError:
|
| 326 |
raise
|
| 327 |
except Exception as exc:
|
|
|
|
| 1 |
+
"""Per-user credentials carrying an access level and database subset for governed reads and
|
| 2 |
+
builds, with the maker's live grants still applying on top.
|
| 3 |
|
| 4 |
+
Verification reconstructs the same PUBLIC user record the normal API session carries, so every
|
| 5 |
+
later database read still goes through ``perm_scope`` and the existing Assistant mirror.
|
| 6 |
+
This module never reads product rows and never caches a
|
| 7 |
credential, user, bucket, or tenant runtime.
|
| 8 |
"""
|
| 9 |
import calendar
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
BUCKET = "external_access"
|
| 21 |
+
ACCESS_READ = "read"
|
| 22 |
+
ACCESS_READWRITE = "readwrite"
|
| 23 |
+
ACCESS_LEVELS = (ACCESS_READ, ACCESS_READWRITE)
|
| 24 |
+
ALL_DATABASES = "all"
|
| 25 |
+
MAX_SCOPE_DATABASES = 200
|
| 26 |
ALGORITHM = "sha256-salt-v1"
|
| 27 |
AUTH_ERROR = "invalid_external_credential"
|
| 28 |
_DUMMY_SALT = bytes(32)
|
|
|
|
| 54 |
"""A uniform authentication refusal that reveals no credential or tenant detail."""
|
| 55 |
|
| 56 |
|
| 57 |
+
def normalise_access(value) -> str:
|
| 58 |
+
if value is None or value == "":
|
| 59 |
+
return ACCESS_READWRITE
|
| 60 |
+
if value in ACCESS_LEVELS:
|
| 61 |
+
return value
|
| 62 |
+
raise ExternalAccessError("external_access_scope_invalid")
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def normalise_databases(value):
|
| 66 |
+
if value is None or (isinstance(value, str) and value.strip().lower() == ALL_DATABASES):
|
| 67 |
+
return ALL_DATABASES
|
| 68 |
+
if not isinstance(value, (list, tuple)) or not 1 <= len(value) <= MAX_SCOPE_DATABASES:
|
| 69 |
+
raise ExternalAccessError("external_databases_invalid")
|
| 70 |
+
if any(not isinstance(item, str) or not item.strip() or len(item.strip()) > 200
|
| 71 |
+
for item in value):
|
| 72 |
+
raise ExternalAccessError("external_databases_invalid")
|
| 73 |
+
return tuple(dict.fromkeys(item.strip() for item in value))
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def scope_of(record):
|
| 77 |
+
"""Read legacy defaults, but preserve a stored empty subset as no database access."""
|
| 78 |
+
record = record if isinstance(record, dict) else {}
|
| 79 |
+
access = record.get("access")
|
| 80 |
+
if access not in ACCESS_LEVELS:
|
| 81 |
+
access = ACCESS_READWRITE
|
| 82 |
+
databases = record.get("databases")
|
| 83 |
+
if isinstance(databases, (list, tuple)):
|
| 84 |
+
databases = tuple(item.strip() for item in databases
|
| 85 |
+
if isinstance(item, str) and item.strip())
|
| 86 |
+
else:
|
| 87 |
+
databases = ALL_DATABASES
|
| 88 |
+
return access, databases
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def may_write(access) -> bool:
|
| 92 |
+
return access == ACCESS_READWRITE
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def database_allowed(databases, database) -> bool:
|
| 96 |
+
if databases == ALL_DATABASES:
|
| 97 |
+
return True
|
| 98 |
+
return isinstance(database, str) and bool(database.strip()) and database.strip() in databases
|
| 99 |
+
|
| 100 |
+
|
| 101 |
@dataclass(frozen=True)
|
| 102 |
class VerifiedCredential:
|
| 103 |
"""The authenticated public principal plus the request-scoped tenant runtime."""
|
|
|
|
| 106 |
key_id: str
|
| 107 |
user: dict
|
| 108 |
runtime: object
|
| 109 |
+
access: str = ACCESS_READWRITE
|
| 110 |
+
databases: object = ALL_DATABASES
|
| 111 |
|
| 112 |
|
| 113 |
def _now() -> str:
|
|
|
|
| 182 |
|
| 183 |
def _metadata(record: dict) -> dict:
|
| 184 |
"""Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam."""
|
| 185 |
+
access, databases = scope_of(record)
|
| 186 |
return {
|
| 187 |
"id": record.get("id"),
|
| 188 |
"label": record.get("label"),
|
| 189 |
"created": record.get("created"),
|
| 190 |
"last_used": record.get("last_used"),
|
| 191 |
"revoked_at": record.get("revoked_at"),
|
| 192 |
+
"access": access,
|
| 193 |
+
"databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases),
|
| 194 |
}
|
| 195 |
|
| 196 |
|
| 197 |
+
def label_for(st, key_id: str) -> str:
|
| 198 |
+
"""Read only a verified key's label from its tenant runtime, never credential material."""
|
| 199 |
+
record = _book(st).get(key_id)
|
| 200 |
+
if not isinstance(record, dict) or record.get("revoked_at"):
|
| 201 |
+
raise ExternalAuthError(AUTH_ERROR)
|
| 202 |
+
label = record.get("label")
|
| 203 |
+
if not isinstance(label, str) or not label:
|
| 204 |
+
raise ExternalAccessError("external_access_unavailable")
|
| 205 |
+
return label
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def mint(st, username: str, tenant: str, label: str, access=None, databases=None) -> tuple[dict, str]:
|
| 209 |
"""Create one credential and return ``(metadata, token)``; the token is never persisted."""
|
| 210 |
+
access = normalise_access(access)
|
| 211 |
+
databases = normalise_databases(databases)
|
| 212 |
owner, tenant_key = _normalise_owner(username, tenant)
|
| 213 |
_assert_bound_runtime(st, tenant_key)
|
| 214 |
_active_public_user(owner, tenant_key)
|
|
|
|
| 236 |
"username": owner,
|
| 237 |
"tenant": tenant_key,
|
| 238 |
"label": clean_label,
|
| 239 |
+
"access": access,
|
| 240 |
+
"databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases),
|
| 241 |
"salt": salt.hex(),
|
| 242 |
"hash": _digest(salt, secret),
|
| 243 |
"algo": ALGORITHM,
|
|
|
|
| 365 |
except ExternalAccessError as exc:
|
| 366 |
raise ExternalAuthError(AUTH_ERROR) from exc
|
| 367 |
|
| 368 |
+
access, databases = scope_of(record)
|
| 369 |
if not _stamp_is_due(record.get("last_used"), time.time()):
|
| 370 |
+
return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st,
|
| 371 |
+
access=access, databases=databases)
|
| 372 |
|
| 373 |
used_at = _now()
|
| 374 |
|
|
|
|
| 393 |
raise
|
| 394 |
except Exception as exc:
|
| 395 |
raise ExternalAuthError(AUTH_ERROR) from exc
|
| 396 |
+
return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st,
|
| 397 |
+
access=access, databases=databases)
|
| 398 |
except ExternalAuthError:
|
| 399 |
raise
|
| 400 |
except Exception as exc:
|
platform/core/external_tools.py
ADDED
|
@@ -0,0 +1,372 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""External build tools over the app's Sheet, saved View and Application writers.
|
| 2 |
+
|
| 3 |
+
The API supplies its existing permission, read-projection and binding-resolution functions as
|
| 4 |
+
request-scoped callbacks. Core never imports API modules or chooses another tenant runtime.
|
| 5 |
+
Adding a tool means adding a registry entry; writes also require the router's literal allow-list.
|
| 6 |
+
"""
|
| 7 |
+
import copy
|
| 8 |
+
import json
|
| 9 |
+
import uuid
|
| 10 |
+
from dataclasses import dataclass
|
| 11 |
+
from typing import Callable
|
| 12 |
+
|
| 13 |
+
from core import sheets, sheet_structure, sheet_templates
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@dataclass(frozen=True)
|
| 17 |
+
class ToolContext:
|
| 18 |
+
session: object
|
| 19 |
+
actor_name: str
|
| 20 |
+
require_reader: Callable
|
| 21 |
+
require_editor: Callable
|
| 22 |
+
project_book: Callable
|
| 23 |
+
resolve_binding: Callable
|
| 24 |
+
cache_binding: Callable
|
| 25 |
+
list_databases: Callable
|
| 26 |
+
describe_database: Callable
|
| 27 |
+
view_event: Callable
|
| 28 |
+
app_upsert: Callable
|
| 29 |
+
app_delete: Callable
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _object(properties=None, required=(), **extra):
|
| 33 |
+
return {"type": "object", "properties": properties or {}, "required": list(required), **extra}
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
_STRING = {"type": "string"}
|
| 37 |
+
_ADDRESS = {"type": "string", "description": "An A1 cell address on the named tab."}
|
| 38 |
+
_FORMAT = _object({"kind": _STRING, "code": _STRING, "decimals": {"type": "integer"},
|
| 39 |
+
"currency": _STRING, "thousands": {"type": "boolean"}}, ("kind",))
|
| 40 |
+
_STYLE = _object({
|
| 41 |
+
**{key: {"type": "boolean"} for key in ("bold", "italic", "underline", "strike", "wrap")},
|
| 42 |
+
**{key: _STRING for key in ("fill", "color", "font", "align", "valign")},
|
| 43 |
+
"size": {"type": ["number", "string"]}, "indent": {"type": "integer"},
|
| 44 |
+
"textRotation": {"type": "number"}, "format": _FORMAT,
|
| 45 |
+
"border": _object({side: {"type": "boolean"} for side in ("top", "right", "bottom", "left")}),
|
| 46 |
+
})
|
| 47 |
+
_VALIDATION = _object({"list": {"type": "array", "items": _STRING}}, ("list",))
|
| 48 |
+
_FILTER = _object({"conj": {"enum": ["and", "or"]}, "nodes": {"type": "array", "items": _object({
|
| 49 |
+
"colId": _STRING, "op": _STRING, "value": {},
|
| 50 |
+
"valueRef": _object({"ref": _ADDRESS}, ("ref",)),
|
| 51 |
+
"conj": _STRING, "children": {"type": "array", "items": {"type": "object"}},
|
| 52 |
+
})}})
|
| 53 |
+
_BINDING = _object({
|
| 54 |
+
"database": _STRING,
|
| 55 |
+
"aggregate": _object({"fn": _STRING, "field": _STRING, "metric_key": _STRING}),
|
| 56 |
+
"scope": _object({"filter": _FILTER, "view_id": _STRING, "view_name": _STRING}),
|
| 57 |
+
"period": _object({"unit": _STRING, "start": _STRING}),
|
| 58 |
+
"ending": {"type": "object"},
|
| 59 |
+
}, ("database", "aggregate", "scope"))
|
| 60 |
+
_CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]},
|
| 61 |
+
"value": {}, "text": _STRING, "formula": _STRING, "state": _STRING,
|
| 62 |
+
"style": _STYLE, "validation": _VALIDATION, "binding": _BINDING}, ("kind",))
|
| 63 |
+
_RANGE = {"oneOf": [_STRING, _object({key: {"type": "integer", "minimum": 0}
|
| 64 |
+
for key in ("col0", "row0", "col1", "row1")},
|
| 65 |
+
("col0", "row0", "col1", "row1"))]}
|
| 66 |
+
_RULE = _object({"id": _STRING, "range": _RANGE, "kind": _STRING, "op": _STRING,
|
| 67 |
+
"value": {}, "value2": {}, "formula": _STRING, "style": _STYLE,
|
| 68 |
+
"scale": {"type": "object"}}, ("range", "op"))
|
| 69 |
+
_CHART = _object({"id": _STRING, "kind": _STRING, "range": _RANGE, "title": _STRING,
|
| 70 |
+
"stacking": _STRING, "source": {"type": "object"},
|
| 71 |
+
"series": {"type": "array", "items": {"type": "object"}},
|
| 72 |
+
"geometry": {"type": "object"}, "anchor": {"type": "object"},
|
| 73 |
+
"settings": {"type": "object"}}, ("kind", "range"))
|
| 74 |
+
_TARGET = {"workbook_id": _STRING,
|
| 75 |
+
"tab": {"type": "string", "description": "Tab id or tab name."},
|
| 76 |
+
"rev": {"type": "integer", "description": "Optional expected workbook revision."}}
|
| 77 |
+
_VIEW_CONFIG = _object({
|
| 78 |
+
"filters": {"oneOf": [_FILTER, {"type": "array", "items": {"type": "object"}}]},
|
| 79 |
+
"filterConj": {"enum": ["and", "or"]},
|
| 80 |
+
"sorts": {"type": "array", "items": _object({
|
| 81 |
+
"colId": _STRING, "dir": {"enum": ["asc", "desc"]},
|
| 82 |
+
"locked": {"type": "boolean"}}, ("colId", "dir"))},
|
| 83 |
+
"groupBy": {"type": ["string", "null"]},
|
| 84 |
+
"visible": {"type": "array", "items": _STRING,
|
| 85 |
+
"description": "Fields to show. Fields omitted from this list are hidden."},
|
| 86 |
+
"order": {"type": "array", "items": _STRING},
|
| 87 |
+
"display": _object({"mode": _STRING}, ("mode",)),
|
| 88 |
+
}, description="The grid's native config, passed unchanged to its saved View validator.")
|
| 89 |
+
_APPLICATION = {
|
| 90 |
+
"app_id": {"type": "string", "minLength": 1,
|
| 91 |
+
"description": "Omit to create. Supply an id to update mutable properties."},
|
| 92 |
+
"name": _STRING,
|
| 93 |
+
"type": {"enum": ["chart", "route", "map", "timeseries"]},
|
| 94 |
+
"compatibleDatabases": {"type": "array", "items": _STRING},
|
| 95 |
+
"sourceDatabase": {"type": "string", "description": "Source for a template Application only."},
|
| 96 |
+
"templateKind": {"enum": ["route", "catalog"]},
|
| 97 |
+
"fieldMap": {"type": "object", "additionalProperties": _STRING},
|
| 98 |
+
"navPlacement": _object({"section": {"const": "app"},
|
| 99 |
+
"order": {"type": "integer", "minimum": 0, "maximum": 999}},
|
| 100 |
+
("section", "order"), additionalProperties=False),
|
| 101 |
+
"semanticRoleBindings": {"type": "object"},
|
| 102 |
+
"config": _object({"schemaVersion": {"const": 1},
|
| 103 |
+
"display": _object({"mode": _STRING}, ("mode",))},
|
| 104 |
+
("schemaVersion", "display"), additionalProperties=False),
|
| 105 |
+
"share": _object({"mode": {"enum": ["private", "users", "tenant"]},
|
| 106 |
+
"users": {"type": "array", "items": _STRING}}, ("mode",)),
|
| 107 |
+
"starred": {"type": "boolean"},
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _schema(properties, required=(), *, tab=True):
|
| 112 |
+
return _object({**_TARGET, **properties},
|
| 113 |
+
("workbook_id", *(("tab",) if tab else ()), *required),
|
| 114 |
+
additionalProperties=False)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _book(ctx, args, *, write=False):
|
| 118 |
+
book = sheets.get_workbook(ctx.session.runtime, args["workbook_id"])
|
| 119 |
+
(ctx.require_editor if write else ctx.require_reader)(book)
|
| 120 |
+
return book
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _saved(book, tab=None):
|
| 124 |
+
out = {"id": book["id"], "rev": book["rev"],
|
| 125 |
+
"tabs": [{"id": row["id"], "name": row["name"]} for row in book["sheets"]]}
|
| 126 |
+
if tab is not None:
|
| 127 |
+
out["tab"] = tab
|
| 128 |
+
return out
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _put(ctx, args, book, tab=None):
|
| 132 |
+
# Carry the revision read for this document patch even when the client omitted one.
|
| 133 |
+
# Never overwrite an intervening UI save with the older snapshot we just patched.
|
| 134 |
+
if "rev" in args:
|
| 135 |
+
book["rev"] = args["rev"]
|
| 136 |
+
return _saved(sheets.put_workbook(ctx.session.runtime, book["id"], book,
|
| 137 |
+
actor=ctx.session.uname, actor_name=ctx.actor_name), tab)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _create(ctx, args):
|
| 141 |
+
return _saved(sheets.create_workbook(ctx.session.runtime, name=args["name"],
|
| 142 |
+
created_by=ctx.session.uname,
|
| 143 |
+
on_duplicate="refuse"))
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _add_tab(ctx, args):
|
| 147 |
+
book = _book(ctx, args, write=True)
|
| 148 |
+
#: ⛔ A DUPLICATE TAB NAME IS REFUSED, because every other tool addresses a tab by `tab`,
|
| 149 |
+
#: which `sheets.find_tab` resolves BY NAME to the FIRST match. Two tabs sharing a name make
|
| 150 |
+
#: every later write land on one of them and the other unreachable, with nothing reported.
|
| 151 |
+
#: Measured at SDLC 61's C7 build: a retried `sheet_add_tab` produced a second
|
| 152 |
+
#: `Summary (All)` and the workbook's cells split silently across the pair.
|
| 153 |
+
wanted = str(args["name"]).strip()
|
| 154 |
+
if any(str(sheet.get("name") or "").strip().casefold() == wanted.casefold()
|
| 155 |
+
for sheet in (book.get("sheets") or ())):
|
| 156 |
+
raise sheets.SheetRefusal(
|
| 157 |
+
"tab_name_taken",
|
| 158 |
+
"Tab " + wanted + " is already in this workbook. Choose another name.")
|
| 159 |
+
tab = {"id": uuid.uuid4().hex[:12], "name": args["name"], "cells": {}}
|
| 160 |
+
book["sheets"].append(tab)
|
| 161 |
+
return _put(ctx, args, book, tab["id"])
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _save_template(ctx, args):
|
| 165 |
+
book = ctx.project_book(_book(ctx, args))
|
| 166 |
+
record = sheet_templates.save_sheet_template(
|
| 167 |
+
ctx.session.runtime, book, args["key"], name=args["name"],
|
| 168 |
+
description=args["description"], created_by=ctx.session.uname, actor_name=ctx.actor_name)
|
| 169 |
+
return {"template": record}
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _read_cells(ctx, args):
|
| 173 |
+
book = ctx.project_book(_book(ctx, args))
|
| 174 |
+
tab = sheets.find_tab(book, args["tab"])
|
| 175 |
+
if "addresses" in args:
|
| 176 |
+
wanted = {addr.strip().upper() for addr in args["addresses"]}
|
| 177 |
+
tab = {**tab, "cells": {addr: cell for addr, cell in tab["cells"].items() if addr in wanted}}
|
| 178 |
+
return {"id": book["id"], "rev": book["rev"], "tab": tab}
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def _patch(ctx, args, cells, *, resolved=None):
|
| 182 |
+
out = sheets.patch_cells(ctx.session.runtime, args["workbook_id"], args["tab"], cells,
|
| 183 |
+
expect_rev=args.get("rev"), actor=ctx.session.uname,
|
| 184 |
+
actor_name=ctx.actor_name)
|
| 185 |
+
if resolved is not None:
|
| 186 |
+
for addr in out["accepted"]:
|
| 187 |
+
cell = cells[addr]
|
| 188 |
+
ctx.cache_binding(args["workbook_id"], out["tab"], addr, cell["binding"], resolved)
|
| 189 |
+
# The UI computes after its patch too. Project before computing so a shared workbook
|
| 190 |
+
# never returns its owner's cached database numbers through an editor's write response.
|
| 191 |
+
book = ctx.project_book(_book(ctx, args))
|
| 192 |
+
tab = sheets.find_tab(book, args["tab"])
|
| 193 |
+
out["computed"] = {addr: cell for addr, cell in tab["cells"].items()
|
| 194 |
+
if cell.get("kind") in ("formula", "bound") or "spillParent" in cell}
|
| 195 |
+
return out
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def _write_cells(ctx, args):
|
| 199 |
+
_book(ctx, args, write=True)
|
| 200 |
+
return _patch(ctx, args, args["cells"])
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def _set_binding(ctx, args):
|
| 204 |
+
book = _book(ctx, args, write=True)
|
| 205 |
+
tab = sheets.find_tab(book, args["tab"])
|
| 206 |
+
resolved = ctx.resolve_binding(args["binding"], book, tab["id"])
|
| 207 |
+
addr = args["addr"].strip().upper()
|
| 208 |
+
before = tab["cells"].get(addr) or {}
|
| 209 |
+
cell = {key: copy.deepcopy(before[key]) for key in ("style", "validation", "link", "note")
|
| 210 |
+
if key in before}
|
| 211 |
+
cell.update(kind="bound", binding=copy.deepcopy(args["binding"]), state="fresh",
|
| 212 |
+
value=resolved["value"],
|
| 213 |
+
display="" if resolved["value"] is None else str(resolved["value"]))
|
| 214 |
+
out = _patch(ctx, args, {addr: cell}, resolved=resolved)
|
| 215 |
+
out["limits"] = resolved.get("limits") or []
|
| 216 |
+
return out
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def _cell_property(ctx, args, property_name, values):
|
| 220 |
+
book = _book(ctx, args, write=True)
|
| 221 |
+
tab = sheets.find_tab(book, args["tab"])
|
| 222 |
+
cells = {}
|
| 223 |
+
for addr, value in values.items():
|
| 224 |
+
cell = copy.deepcopy(tab["cells"].get(addr.strip().upper()) or {"kind": "empty"})
|
| 225 |
+
cell[property_name] = value
|
| 226 |
+
cells[addr] = cell
|
| 227 |
+
return _patch(ctx, args, cells)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _dimensions(ctx, args):
|
| 231 |
+
book = _book(ctx, args, write=True)
|
| 232 |
+
tab = sheets.find_tab(book, args["tab"])
|
| 233 |
+
tab.update(copy.deepcopy(args["dimensions"]))
|
| 234 |
+
return _put(ctx, args, book, tab["id"])
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def _freeze(ctx, args):
|
| 238 |
+
book = _book(ctx, args, write=True)
|
| 239 |
+
tab = sheets.find_tab(book, args["tab"])
|
| 240 |
+
tab["freeze"] = args["freeze"]
|
| 241 |
+
return _put(ctx, args, book, tab["id"])
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def _rule(ctx, args):
|
| 245 |
+
book = _book(ctx, args, write=True)
|
| 246 |
+
tab = sheets.find_tab(book, args["tab"])
|
| 247 |
+
tab["rules"] = [*(tab.get("rules") or []), args["rule"]]
|
| 248 |
+
return _put(ctx, args, book, tab["id"])
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def _chart(ctx, args):
|
| 252 |
+
book = _book(ctx, args, write=True)
|
| 253 |
+
tab = sheets.find_tab(book, args["tab"])
|
| 254 |
+
chart = copy.deepcopy(args["chart"])
|
| 255 |
+
chart.setdefault("id", uuid.uuid4().hex[:12])
|
| 256 |
+
tab["charts"] = [row for row in tab.get("charts", []) if row["id"] != chart["id"]] + [chart]
|
| 257 |
+
return {**_put(ctx, args, book, tab["id"]), "chart_id": chart["id"]}
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _structure(ctx, args):
|
| 261 |
+
book = _book(ctx, args, write=True)
|
| 262 |
+
tab = sheets.find_tab(book, args["tab"])
|
| 263 |
+
shifted = sheet_structure.apply_structure(book, tab["id"], args["op"])
|
| 264 |
+
return _put(ctx, args, shifted, tab["id"])
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def _tool(description, schema, handler, *, write=False, grants=("analyst", "sheets")):
|
| 268 |
+
return {"description": description, "inputSchema": schema, "write": write,
|
| 269 |
+
"grants": grants, "handler": handler}
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
TOOLS = {
|
| 273 |
+
"sheet_create_workbook": _tool("Create a private workbook with one initial tab.",
|
| 274 |
+
_object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
|
| 275 |
+
"sheet_add_tab": _tool("Add a blank tab to a workbook.",
|
| 276 |
+
_schema({"name": _STRING}, ("name",), tab=False), _add_tab, write=True),
|
| 277 |
+
"sheet_read_cells": _tool("Read a tab with styles, geometry and figures under the maker's grants.",
|
| 278 |
+
_schema({"addresses": {"type": "array", "items": _ADDRESS}}), _read_cells),
|
| 279 |
+
"sheet_write_cells": _tool("Write cell documents with values or formulas; null deletes a cell.",
|
| 280 |
+
_schema({"cells": {"type": "object", "additionalProperties": {"anyOf": [_CELL, {"type": "null"}]}}},
|
| 281 |
+
("cells",)), _write_cells, write=True),
|
| 282 |
+
"sheet_set_binding": _tool("Set and resolve a Cell database binding, including filter valueRef cells.",
|
| 283 |
+
_schema({"addr": _ADDRESS, "binding": _BINDING}, ("addr", "binding")), _set_binding, write=True),
|
| 284 |
+
"sheet_set_styles": _tool("Replace styles by address, preserving cell content; null clears a style.",
|
| 285 |
+
_schema({"styles": {"type": "object", "additionalProperties": {"anyOf": [_STYLE, {"type": "null"}]}}},
|
| 286 |
+
("styles",)), lambda ctx, args: _cell_property(ctx, args, "style", args["styles"]), write=True),
|
| 287 |
+
"sheet_set_dimensions": _tool("Replace supplied geometry fields; widths use letters and row indices start at one.",
|
| 288 |
+
_schema({"dimensions": _object({
|
| 289 |
+
"cols": {"type": "integer"}, "rows": {"type": "integer"},
|
| 290 |
+
"colWidths": {"type": "object", "additionalProperties": {"type": "number"}},
|
| 291 |
+
"rowHeight": {"type": "number"},
|
| 292 |
+
"rowHeights": {"type": "object", "additionalProperties": {"type": "number"}},
|
| 293 |
+
"hiddenRows": {"type": "array", "items": {"type": "integer"}},
|
| 294 |
+
"hiddenCols": {"type": "array", "items": {"type": "integer"}},
|
| 295 |
+
}, additionalProperties=False)}, ("dimensions",)), _dimensions, write=True),
|
| 296 |
+
"sheet_set_validation": _tool("Replace dropdown lists by address, preserving content; null clears validation.",
|
| 297 |
+
_schema({"validations": {"type": "object", "additionalProperties": {
|
| 298 |
+
"anyOf": [_VALIDATION, {"type": "null"}]}}}, ("validations",)),
|
| 299 |
+
lambda ctx, args: _cell_property(ctx, args, "validation", args["validations"]), write=True),
|
| 300 |
+
"sheet_set_freeze": _tool("Set frozen row and column counts, using zero to unfreeze.",
|
| 301 |
+
_schema({"freeze": _object({"rows": {"type": "integer"}, "cols": {"type": "integer"}})},
|
| 302 |
+
("freeze",)), _freeze, write=True),
|
| 303 |
+
"sheet_add_rule": _tool("Append a conditional format rule accepted by the Sheet document cleaner.",
|
| 304 |
+
_schema({"rule": _RULE}, ("rule",)), _rule, write=True),
|
| 305 |
+
"sheet_set_chart": _tool("Add a chart or replace a chart with the same id using the app's chart document.",
|
| 306 |
+
_schema({"chart": _CHART}, ("chart",)), _chart, write=True),
|
| 307 |
+
"sheet_structure": _tool("Insert or delete rows or columns and reanchor references; positive delta inserts.",
|
| 308 |
+
_schema({"op": _object({"axis": {"enum": ["row", "col"]},
|
| 309 |
+
"at": {"type": "integer", "minimum": 1}, "delta": {"type": "integer"}},
|
| 310 |
+
("axis", "at", "delta"))}, ("op",)), _structure, write=True),
|
| 311 |
+
"sheet_save_template": _tool("Save a readable workbook as a tenant Sheet template with its Cell databases.",
|
| 312 |
+
_object({key: _STRING for key in ("workbook_id", "key", "name", "description")},
|
| 313 |
+
("workbook_id", "key", "name", "description"), additionalProperties=False),
|
| 314 |
+
_save_template, write=True),
|
| 315 |
+
"list_databases": _tool("List the databases permitted or visible to the key's maker.",
|
| 316 |
+
_object(additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("analyst",)),
|
| 317 |
+
"describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
|
| 318 |
+
_object({"database": _STRING}, ("database",), additionalProperties=False),
|
| 319 |
+
lambda ctx, args: ctx.describe_database(args["database"]), grants=("analyst",)),
|
| 320 |
+
"view_upsert": _tool(
|
| 321 |
+
"Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
|
| 322 |
+
_object({"database": _STRING, "view": _object({
|
| 323 |
+
"id": {"type": "string", "minLength": 1, "maxLength": 120},
|
| 324 |
+
"name": {"type": "string", "minLength": 1, "maxLength": 120},
|
| 325 |
+
"config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"))},
|
| 326 |
+
("database", "view"), additionalProperties=False),
|
| 327 |
+
lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("analyst",)),
|
| 328 |
+
"view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
|
| 329 |
+
_object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
|
| 330 |
+
additionalProperties=False),
|
| 331 |
+
lambda ctx, args: ctx.view_event("view_delete", args), write=True, grants=("analyst",)),
|
| 332 |
+
"app_upsert": _tool(
|
| 333 |
+
"Create an App with name, type, compatibleDatabases and config, or update it by app_id. Identity fields are immutable.",
|
| 334 |
+
_object(_APPLICATION, additionalProperties=False,
|
| 335 |
+
anyOf=[{"required": ["app_id"]}, {"required": ["name", "type", "config"]},
|
| 336 |
+
{"required": ["name", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"]}]),
|
| 337 |
+
lambda ctx, args: ctx.app_upsert(args), write=True, grants=("analyst",)),
|
| 338 |
+
"app_delete": _tool("Delete an App. confirmName must match its current name exactly.",
|
| 339 |
+
_object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
|
| 340 |
+
additionalProperties=False),
|
| 341 |
+
lambda ctx, args: ctx.app_delete(args), write=True, grants=("analyst",)),
|
| 342 |
+
}
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
def dispatch(ctx, name, arguments):
|
| 346 |
+
"""Dispatch one declared tool. The API translates core and grant refusals to MCP content."""
|
| 347 |
+
tool = TOOLS.get(name) if isinstance(name, str) else None
|
| 348 |
+
if tool is None:
|
| 349 |
+
raise sheets.SheetRefusal("tool_not_found", "The requested tool is not available.")
|
| 350 |
+
for grant in tool["grants"]:
|
| 351 |
+
ctx.session.require(grant)
|
| 352 |
+
schema = tool["inputSchema"]
|
| 353 |
+
if not isinstance(arguments, dict):
|
| 354 |
+
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.")
|
| 355 |
+
missing = set(schema["required"]) - arguments.keys()
|
| 356 |
+
unknown = arguments.keys() - schema["properties"].keys()
|
| 357 |
+
if missing or (unknown and schema.get("additionalProperties") is False):
|
| 358 |
+
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema.")
|
| 359 |
+
# Validate only the transport shape here. Document semantics belong to the app writers.
|
| 360 |
+
for key, value in arguments.items():
|
| 361 |
+
kind = schema["properties"].get(key, {}).get("type")
|
| 362 |
+
types = {"object": dict, "array": list, "string": str, "integer": int}
|
| 363 |
+
if kind in types and (not isinstance(value, types[kind]) or (kind == "integer" and isinstance(value, bool))):
|
| 364 |
+
raise sheets.SheetRefusal("tool_arguments_invalid", f"Tool argument {key} must be an {kind}.")
|
| 365 |
+
# A dimension patch must never become an arbitrary tab-document replacement.
|
| 366 |
+
if name == "sheet_set_dimensions":
|
| 367 |
+
allowed = schema["properties"]["dimensions"]["properties"]
|
| 368 |
+
if arguments["dimensions"].keys() - allowed.keys():
|
| 369 |
+
raise sheets.SheetRefusal("tool_arguments_invalid", "Dimensions may contain only the published geometry fields.")
|
| 370 |
+
if "lpb1." in json.dumps(arguments, ensure_ascii=False, allow_nan=False):
|
| 371 |
+
raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.")
|
| 372 |
+
return tool["handler"](ctx, arguments)
|
platform/core/measure_resolve.py
CHANGED
|
@@ -152,8 +152,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No
|
|
| 152 |
|
| 153 |
`related` is the row-window {table, record, via} pin. The host binds its scoped readers
|
| 154 |
with relations.measure_pin_context and publishes the returned mapping's `refusals` beside
|
| 155 |
-
its cells. A refused pin
|
| 156 |
-
|
| 157 |
"""
|
| 158 |
mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)]
|
| 159 |
if not mfields:
|
|
@@ -197,6 +197,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No
|
|
| 197 |
if related is not None:
|
| 198 |
out.refusals.extend(dict(marker, field=fkey)
|
| 199 |
for marker in getattr(vals, 'refusals', ()))
|
|
|
|
|
|
|
| 200 |
for pid, v in vals.items():
|
| 201 |
out.setdefault(pid, {})[fkey] = v
|
| 202 |
if len(memo) > 100:
|
|
|
|
| 152 |
|
| 153 |
`related` is the row-window {table, record, via} pin. The host binds its scoped readers
|
| 154 |
with relations.measure_pin_context and publishes the returned mapping's `refusals` beside
|
| 155 |
+
its cells. A refused pin publishes a per-field marker and no values, on memo hits too.
|
| 156 |
+
Every measure and its conditions use the same query scope and caller's pool.
|
| 157 |
"""
|
| 158 |
mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)]
|
| 159 |
if not mfields:
|
|
|
|
| 197 |
if related is not None:
|
| 198 |
out.refusals.extend(dict(marker, field=fkey)
|
| 199 |
for marker in getattr(vals, 'refusals', ()))
|
| 200 |
+
if getattr(vals, 'refusals', ()):
|
| 201 |
+
continue
|
| 202 |
for pid, v in vals.items():
|
| 203 |
out.setdefault(pid, {})[fkey] = v
|
| 204 |
if len(memo) > 100:
|
platform/core/odoo.py
CHANGED
|
@@ -103,7 +103,9 @@ def tenant_scope(slug, creds=None, team_ids=None, team_names=None, exclude_partn
|
|
| 103 |
Unbound (the default) = tenant #0 via env, which is every existing Streamlit/API caller.
|
| 104 |
The connector layer (harness/connectors/odoo.py) wraps each query in this, so a second
|
| 105 |
tenant's keychain credentials reach the wire without any module knowing. Scope constants
|
| 106 |
-
are PUSHED here from Tenant.config because core/ may not import harness/ (layering
|
|
|
|
|
|
|
| 107 |
prev = getattr(_tlocal, 'tenant_ctx', None)
|
| 108 |
_tlocal.tenant_ctx = {
|
| 109 |
'slug': str(slug or _DEFAULT_SLUG),
|
|
|
|
| 103 |
Unbound (the default) = tenant #0 via env, which is every existing Streamlit/API caller.
|
| 104 |
The connector layer (harness/connectors/odoo.py) wraps each query in this, so a second
|
| 105 |
tenant's keychain credentials reach the wire without any module knowing. Scope constants
|
| 106 |
+
are PUSHED here from Tenant.config because core/ may not import harness/ (layering; enforced
|
| 107 |
+
forward by platform/verify_layering.py, which grandfathers the older core -> harness edges
|
| 108 |
+
by name, D-460)."""
|
| 109 |
prev = getattr(_tlocal, 'tenant_ctx', None)
|
| 110 |
_tlocal.tenant_ctx = {
|
| 111 |
'slug': str(slug or _DEFAULT_SLUG),
|
platform/core/perm_scope.py
CHANGED
|
@@ -40,6 +40,7 @@ twice in this codebase (`modules: []` and `bus: [<unknown id>]` both read as UNR
|
|
| 40 |
perms block; without this clause an explicit-marker scheme locks the owner out of their own
|
| 41 |
product at exactly the moment the store is broken.
|
| 42 |
"""
|
|
|
|
| 43 |
import re
|
| 44 |
|
| 45 |
import core.perms as perms
|
|
@@ -1400,7 +1401,7 @@ class Unresolvable(RuntimeError):
|
|
| 1400 |
|
| 1401 |
|
| 1402 |
#: Row readers DECLARED by the app layer, keyed by EXACT database key.
|
| 1403 |
-
#: `reader(table_key, user, st) -> (fields, rows)`.
|
| 1404 |
#:
|
| 1405 |
#: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and
|
| 1406 |
#: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache
|
|
@@ -1409,6 +1410,7 @@ class Unresolvable(RuntimeError):
|
|
| 1409 |
#: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being
|
| 1410 |
#: interrogated by it."*
|
| 1411 |
_ROW_SOURCES = {}
|
|
|
|
| 1412 |
|
| 1413 |
#: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror.
|
| 1414 |
#: `reader(table_key, field_keys, st) -> rows`.
|
|
@@ -1423,10 +1425,18 @@ def register_rows(reader, *table_keys):
|
|
| 1423 |
says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write
|
| 1424 |
door's return keeps one construction site of the set instead of two.
|
| 1425 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1426 |
for key in table_keys:
|
| 1427 |
k = str(key or '').strip()
|
| 1428 |
if k:
|
| 1429 |
_ROW_SOURCES[k] = reader
|
|
|
|
| 1430 |
return frozenset(_ROW_SOURCES)
|
| 1431 |
|
| 1432 |
|
|
@@ -1568,8 +1578,8 @@ def scoped_fields(user, table_key, st=None):
|
|
| 1568 |
second path to the store (W1). So both come from here, off one wall.
|
| 1569 |
|
| 1570 |
⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a
|
| 1571 |
-
registry topic it
|
| 1572 |
-
|
| 1573 |
"""
|
| 1574 |
fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False)
|
| 1575 |
return fields
|
|
@@ -1613,7 +1623,10 @@ def _read(table_key, user, st, want_rows=True):
|
|
| 1613 |
"""`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`."""
|
| 1614 |
reader = _ROW_SOURCES.get(table_key)
|
| 1615 |
if reader is not None:
|
| 1616 |
-
|
|
|
|
|
|
|
|
|
|
| 1617 |
return list(fields or ()), list(rows or ())
|
| 1618 |
ut = _ut()
|
| 1619 |
if not table_key.startswith(ut.KEY_PREFIX):
|
|
|
|
| 40 |
perms block; without this clause an explicit-marker scheme locks the owner out of their own
|
| 41 |
product at exactly the moment the store is broken.
|
| 42 |
"""
|
| 43 |
+
import inspect
|
| 44 |
import re
|
| 45 |
|
| 46 |
import core.perms as perms
|
|
|
|
| 1401 |
|
| 1402 |
|
| 1403 |
#: Row readers DECLARED by the app layer, keyed by EXACT database key.
|
| 1404 |
+
#: `reader(table_key, user, st[, want_rows=True]) -> (fields, rows)`.
|
| 1405 |
#:
|
| 1406 |
#: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and
|
| 1407 |
#: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache
|
|
|
|
| 1410 |
#: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being
|
| 1411 |
#: interrogated by it."*
|
| 1412 |
_ROW_SOURCES = {}
|
| 1413 |
+
_ROW_SOURCE_WANT_ROWS = {}
|
| 1414 |
|
| 1415 |
#: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror.
|
| 1416 |
#: `reader(table_key, field_keys, st) -> rows`.
|
|
|
|
| 1425 |
says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write
|
| 1426 |
door's return keeps one construction site of the set instead of two.
|
| 1427 |
"""
|
| 1428 |
+
# Decide the calling convention once. Never retry a reader's own TypeError as an old
|
| 1429 |
+
# signature: doing so can silently replace a failed read with a different answer.
|
| 1430 |
+
parameters = inspect.signature(reader).parameters
|
| 1431 |
+
accepts_want_rows = any(p.kind == inspect.Parameter.VAR_KEYWORD
|
| 1432 |
+
for p in parameters.values()) or (
|
| 1433 |
+
'want_rows' in parameters and parameters['want_rows'].kind in (
|
| 1434 |
+
inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY))
|
| 1435 |
for key in table_keys:
|
| 1436 |
k = str(key or '').strip()
|
| 1437 |
if k:
|
| 1438 |
_ROW_SOURCES[k] = reader
|
| 1439 |
+
_ROW_SOURCE_WANT_ROWS[k] = accepts_want_rows
|
| 1440 |
return frozenset(_ROW_SOURCES)
|
| 1441 |
|
| 1442 |
|
|
|
|
| 1578 |
second path to the store (W1). So both come from here, off one wall.
|
| 1579 |
|
| 1580 |
⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a
|
| 1581 |
+
registry topic it asks the registered reader for fields only when supported. Legacy
|
| 1582 |
+
three-argument readers retain their existing call contract.
|
| 1583 |
"""
|
| 1584 |
fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False)
|
| 1585 |
return fields
|
|
|
|
| 1623 |
"""`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`."""
|
| 1624 |
reader = _ROW_SOURCES.get(table_key)
|
| 1625 |
if reader is not None:
|
| 1626 |
+
if _ROW_SOURCE_WANT_ROWS.get(table_key, False):
|
| 1627 |
+
fields, rows = reader(table_key, user, st, want_rows=want_rows)
|
| 1628 |
+
else:
|
| 1629 |
+
fields, rows = reader(table_key, user, st)
|
| 1630 |
return list(fields or ()), list(rows or ())
|
| 1631 |
ut = _ut()
|
| 1632 |
if not table_key.startswith(ut.KEY_PREFIX):
|
platform/core/relations.py
CHANGED
|
@@ -265,6 +265,20 @@ def _link_bag(field):
|
|
| 265 |
return bag if isinstance(bag, dict) else None
|
| 266 |
|
| 267 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
def _entry(key, label, target, direction, field=None, on=None, frm=None, served=True,
|
| 269 |
where=None, cardinality=None):
|
| 270 |
"""ONE `/related` entry, in C6's shape.
|
|
@@ -745,6 +759,10 @@ def measure_pin_key(related):
|
|
| 745 |
if related is None:
|
| 746 |
return None
|
| 747 |
ctx = _MEASURE_PIN_CONTEXT.get() or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 748 |
return json.dumps(related, sort_keys=True, separators=(',', ':')), ctx.get('cache_key')
|
| 749 |
|
| 750 |
|
|
@@ -754,11 +772,17 @@ class MeasurePin:
|
|
| 754 |
rows: dict
|
| 755 |
bindings: dict
|
| 756 |
refusal: dict = None
|
|
|
|
| 757 |
|
| 758 |
def predicate(self, store):
|
| 759 |
"""AND every declared, traversed member set into this metric's raw rows."""
|
| 760 |
if self.refusal is not None:
|
| 761 |
return None, []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 762 |
parts, params = [], []
|
| 763 |
for binding in self.bindings.get(store['table'], ()):
|
| 764 |
table, column = binding['table'], binding['column']
|
|
@@ -1126,6 +1150,15 @@ def one_hop(table, fields=None, defs=None, include_retired=False):
|
|
| 1126 |
out = []
|
| 1127 |
for entry in base:
|
| 1128 |
entry = dict(entry)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1129 |
got = cards.get(entry['key']) or bridge_cards.get(entry['key'])
|
| 1130 |
entry['cardinality'] = str(got or 'many_to_many')
|
| 1131 |
out.append(entry)
|
|
|
|
| 265 |
return bag if isinstance(bag, dict) else None
|
| 266 |
|
| 267 |
|
| 268 |
+
def declared_through(link):
|
| 269 |
+
"""R50 / C4: return the intermediate of a complete declared two-hop link, or ''.
|
| 270 |
+
|
| 271 |
+
The intermediate is a join, not a destination. R50 says grants on both ends suffice;
|
| 272 |
+
intermediate fields are never projected, returned or rendered. An incomplete through
|
| 273 |
+
bag is not a declared two-hop link and earns no traversal admission. Manual links do
|
| 274 |
+
not declare a traversal either.
|
| 275 |
+
"""
|
| 276 |
+
if (isinstance(link, dict) and link.get('through') and link.get('throughOn')
|
| 277 |
+
and link.get('throughTo') and link.get('selectionMode') != 'manual'):
|
| 278 |
+
return str(link['through'])
|
| 279 |
+
return ''
|
| 280 |
+
|
| 281 |
+
|
| 282 |
def _entry(key, label, target, direction, field=None, on=None, frm=None, served=True,
|
| 283 |
where=None, cardinality=None):
|
| 284 |
"""ONE `/related` entry, in C6's shape.
|
|
|
|
| 759 |
if related is None:
|
| 760 |
return None
|
| 761 |
ctx = _MEASURE_PIN_CONTEXT.get() or {}
|
| 762 |
+
if isinstance(related, MeasurePin):
|
| 763 |
+
# Compiled pins contain the server's walled membership, not just a wire path.
|
| 764 |
+
related = dict(path=related.path, rows=related.rows, bindings=related.bindings,
|
| 765 |
+
refusal=related.refusal, required=related.required)
|
| 766 |
return json.dumps(related, sort_keys=True, separators=(',', ':')), ctx.get('cache_key')
|
| 767 |
|
| 768 |
|
|
|
|
| 772 |
rows: dict
|
| 773 |
bindings: dict
|
| 774 |
refusal: dict = None
|
| 775 |
+
required: tuple = ()
|
| 776 |
|
| 777 |
def predicate(self, store):
|
| 778 |
"""AND every declared, traversed member set into this metric's raw rows."""
|
| 779 |
if self.refusal is not None:
|
| 780 |
return None, []
|
| 781 |
+
bound = {b['table'] for b in self.bindings.get(store['table'], ())}
|
| 782 |
+
if set(self.required) - bound:
|
| 783 |
+
raise PinRefusal('relation_binding_unavailable',
|
| 784 |
+
'This metric cannot be restricted to every database in this record path. '
|
| 785 |
+
'Ask an administrator to add the missing metric binding.', self.path)
|
| 786 |
parts, params = [], []
|
| 787 |
for binding in self.bindings.get(store['table'], ()):
|
| 788 |
table, column = binding['table'], binding['column']
|
|
|
|
| 1150 |
out = []
|
| 1151 |
for entry in base:
|
| 1152 |
entry = dict(entry)
|
| 1153 |
+
# Keep related()'s legacy shape. Depth-aware pins also need the declared manual,
|
| 1154 |
+
# inverse or through configuration, which remains schema-only cache input.
|
| 1155 |
+
via = entry.get('via') or {}
|
| 1156 |
+
owner_fields = (((defs or {}).get(entry['target']) or {}).get('fields') or []
|
| 1157 |
+
if via.get('direction') == 'reverse' else fields or [])
|
| 1158 |
+
declared = next((field for field in owner_fields
|
| 1159 |
+
if field.get('key') == via.get('field')), None)
|
| 1160 |
+
if declared and _link_bag(declared):
|
| 1161 |
+
entry['link'] = dict(_link_bag(declared))
|
| 1162 |
got = cards.get(entry['key']) or bridge_cards.get(entry['key'])
|
| 1163 |
entry['cardinality'] = str(got or 'many_to_many')
|
| 1164 |
out.append(entry)
|
platform/core/shares.py
CHANGED
|
@@ -553,11 +553,28 @@ def role_for(kind, oid, user, is_admin=False, st=None):
|
|
| 553 |
user = str(user or '').strip().lower()
|
| 554 |
if not user:
|
| 555 |
return None
|
| 556 |
-
|
| 557 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 558 |
return 'owner'
|
| 559 |
best = None
|
| 560 |
-
for e in rec
|
| 561 |
if e['user'] == user or e['user'] == EVERYONE:
|
| 562 |
# The STRONGER of the two wins when both a personal and an everyone grant exist:
|
| 563 |
# naming somebody explicitly is how you RAISE them above the room, so an
|
|
@@ -568,6 +585,37 @@ def role_for(kind, oid, user, is_admin=False, st=None):
|
|
| 568 |
return best
|
| 569 |
|
| 570 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 571 |
def may_see(kind, oid, user, is_admin=False, st=None):
|
| 572 |
return role_for(kind, oid, user, is_admin=is_admin, st=st) is not None
|
| 573 |
|
|
|
|
| 553 |
user = str(user or '').strip().lower()
|
| 554 |
if not user:
|
| 555 |
return None
|
| 556 |
+
return role_from(grants(kind, oid, st=st), user, is_admin=is_admin)
|
| 557 |
+
|
| 558 |
+
|
| 559 |
+
def role_from(rec, user, is_admin=False):
|
| 560 |
+
"""The SAME answer as `role_for`, from a record the caller already holds. PURE.
|
| 561 |
+
|
| 562 |
+
⭐⭐ W61-T17 — ONE EVALUATOR, TWO READ PATHS. `grants_bulk` exists so a list of N objects
|
| 563 |
+
costs ONE bucket read instead of N, and the moment a batched caller re-implements "which role
|
| 564 |
+
does this grant record give?" the two answers can drift — and the one that drifts is a
|
| 565 |
+
PERMISSION answer. So the loop lives here, `role_for` is now a thin read-then-call, and a
|
| 566 |
+
batched caller passes the record it already has. [[one-evaluator-per-question]]
|
| 567 |
+
|
| 568 |
+
⚠ `rec` may be None (an object with no grant record at all), which reads as no role.
|
| 569 |
+
"""
|
| 570 |
+
user = str(user or '').strip().lower()
|
| 571 |
+
if not user:
|
| 572 |
+
return None
|
| 573 |
+
rec = rec or {'owner': None, 'entries': []}
|
| 574 |
+
if is_admin or (rec.get('owner') and rec['owner'] == user):
|
| 575 |
return 'owner'
|
| 576 |
best = None
|
| 577 |
+
for e in rec.get('entries') or ():
|
| 578 |
if e['user'] == user or e['user'] == EVERYONE:
|
| 579 |
# The STRONGER of the two wins when both a personal and an everyone grant exist:
|
| 580 |
# naming somebody explicitly is how you RAISE them above the room, so an
|
|
|
|
| 585 |
return best
|
| 586 |
|
| 587 |
|
| 588 |
+
def grants_bulk(kind, st=None):
|
| 589 |
+
"""`{oid: {'owner', 'entries'}}` for every object of one `kind` — in ONE bucket read.
|
| 590 |
+
|
| 591 |
+
⭐⭐ W61-T17 — THE SAME SHAPE `granted_oids` EXISTS FOR, AND THE SAME REASON, stated there in
|
| 592 |
+
full: `grants()` re-reads and re-copies the WHOLE registry on every call, so a list of N
|
| 593 |
+
objects that asks per object pays N full reads of the tenant's entire grant registry. This
|
| 594 |
+
reads it once. `granted_oids` answers "does it have any grants"; this answers "what are they",
|
| 595 |
+
which is what a list joining `shares` onto rows actually needs.
|
| 596 |
+
|
| 597 |
+
⛔ IT NORMALISES THROUGH `_clean_entries`, exactly as `grants` does, so a junk entry cannot
|
| 598 |
+
reach a caller by the batched door that the single door would have dropped. An owner husk left
|
| 599 |
+
by a full revoke keeps its owner and an EMPTY entry list, which is what `grants` answers too.
|
| 600 |
+
⛔ FAIL-CLOSED: an unreadable bucket answers `{}` — no roles, no shares — never a default.
|
| 601 |
+
An unknown KIND still RAISES, as everywhere else in this module.
|
| 602 |
+
"""
|
| 603 |
+
kind = _check_kind(kind)
|
| 604 |
+
try:
|
| 605 |
+
by_kind = (_st(st).get(SHARES_KEY) or {}).get(kind) or {}
|
| 606 |
+
rows = list(by_kind.items())
|
| 607 |
+
except Exception:
|
| 608 |
+
return {}
|
| 609 |
+
out = {}
|
| 610 |
+
for oid, rec in rows:
|
| 611 |
+
if not isinstance(rec, dict):
|
| 612 |
+
continue
|
| 613 |
+
out[str(oid)] = {'owner': (str(rec.get('owner')).strip().lower()
|
| 614 |
+
if rec.get('owner') else None),
|
| 615 |
+
'entries': _clean_entries(rec.get('entries'))}
|
| 616 |
+
return out
|
| 617 |
+
|
| 618 |
+
|
| 619 |
def may_see(kind, oid, user, is_admin=False, st=None):
|
| 620 |
return role_for(kind, oid, user, is_admin=is_admin, st=st) is not None
|
| 621 |
|
platform/core/sheet_binding.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pure document cleaners shared by Sheet storage and Sheet query resolution.
|
| 2 |
+
|
| 3 |
+
These functions decide only what binding metadata may be stored. They do not resolve data,
|
| 4 |
+
permissions, or SQL. Keeping them in core lets the workbook document validate view bindings
|
| 5 |
+
without importing the higher ``modules.sheet`` layer.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import datetime
|
| 11 |
+
import re
|
| 12 |
+
|
| 13 |
+
from core import windows as W
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
PERIOD_UNITS = ("day", "week", "month", "quarter", "year")
|
| 17 |
+
PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
|
| 18 |
+
PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$")
|
| 19 |
+
PERIOD_REF = re.compile(r"^[A-Z]{1,3}[1-9][0-9]{0,6}$")
|
| 20 |
+
DRAG_ACROSS = ("period", "none")
|
| 21 |
+
DRAG_DOWN_NONE = "none"
|
| 22 |
+
|
| 23 |
+
_MESSAGES = {
|
| 24 |
+
"period_invalid": "{subject} is not a period this cell can use. Choose a valid start for the selected period unit.",
|
| 25 |
+
"ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.",
|
| 26 |
+
"ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year to date, this month or this quarter.",
|
| 27 |
+
"ending_cell_unreadable": "Ending from cell {subject} cannot be read. Use a cell containing a date in YYYY-MM-DD format.",
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class BindingMetadataRefusal(Exception):
|
| 32 |
+
"""A structured metadata refusal which each caller translates to its route exception."""
|
| 33 |
+
|
| 34 |
+
def __init__(self, code: str, subject: str = "", detail: str = ""):
|
| 35 |
+
super().__init__(code)
|
| 36 |
+
self.code = code
|
| 37 |
+
self.subject = str(subject or "")
|
| 38 |
+
self.detail = str(detail or "")
|
| 39 |
+
self.message = _MESSAGES.get(code, "{subject} could not be answered.").format(
|
| 40 |
+
subject=self.subject or "this database")
|
| 41 |
+
if code in ("ending_invalid", "ending_cell_unreadable") and self.detail:
|
| 42 |
+
verb = "used" if code == "ending_invalid" else "read"
|
| 43 |
+
self.message = self.message.replace(
|
| 44 |
+
f"cannot be {verb}.", f"cannot be {verb} because {self.detail}")
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def clean_period(raw):
|
| 48 |
+
"""Return one canonical Sheet period, or ``None`` when no window was named."""
|
| 49 |
+
if not isinstance(raw, dict):
|
| 50 |
+
return None
|
| 51 |
+
out = {}
|
| 52 |
+
|
| 53 |
+
if "ending" in raw:
|
| 54 |
+
window = W.normalize(raw.get("window"))
|
| 55 |
+
kind = (window or {}).get("kind")
|
| 56 |
+
if kind not in W.TO_DATE_KINDS:
|
| 57 |
+
raise BindingMetadataRefusal("ending_not_supported", kind or "This period")
|
| 58 |
+
ending = raw["ending"]
|
| 59 |
+
mode = ending.get("kind") if isinstance(ending, dict) else None
|
| 60 |
+
if mode not in W.ENDING_KINDS:
|
| 61 |
+
raise BindingMetadataRefusal("ending_invalid", str(mode or "selection"),
|
| 62 |
+
"the choice is unknown.")
|
| 63 |
+
step = raw.get("endingStep", W.ending_step_default(kind))
|
| 64 |
+
offset = raw.get("endingOffset", 0)
|
| 65 |
+
if (step not in W.ENDING_STEPS or isinstance(offset, bool)
|
| 66 |
+
or not isinstance(offset, int) or offset < 0):
|
| 67 |
+
raise BindingMetadataRefusal("ending_invalid", "step",
|
| 68 |
+
"its size or copy offset is invalid.")
|
| 69 |
+
out = {"window": window, "ending": {"kind": mode}, "endingStep": step}
|
| 70 |
+
if offset:
|
| 71 |
+
out["endingOffset"] = offset
|
| 72 |
+
if raw.get("locked") is True:
|
| 73 |
+
out["locked"] = True
|
| 74 |
+
if mode == "fixed":
|
| 75 |
+
day = W.ending_date(ending.get("date"))
|
| 76 |
+
if day is None:
|
| 77 |
+
raise BindingMetadataRefusal("ending_invalid", "fixed date",
|
| 78 |
+
"it is not a real ISO date.")
|
| 79 |
+
out["ending"]["date"] = day
|
| 80 |
+
if mode == "n_before_today":
|
| 81 |
+
n, unit = ending.get("n"), ending.get("unit")
|
| 82 |
+
if isinstance(n, bool) or not isinstance(n, int) or not 1 <= n <= W.MAX_N:
|
| 83 |
+
raise BindingMetadataRefusal(
|
| 84 |
+
"ending_invalid", "number", f"it must be a whole number from 1 to {W.MAX_N}.")
|
| 85 |
+
if unit not in W.ENDING_STEPS:
|
| 86 |
+
raise BindingMetadataRefusal("ending_invalid", "unit", "the unit is unknown.")
|
| 87 |
+
out["ending"].update({"n": n, "unit": unit})
|
| 88 |
+
if mode == "cell":
|
| 89 |
+
ref = str(raw.get("ref") or "").strip()
|
| 90 |
+
if not re.fullmatch(
|
| 91 |
+
r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref):
|
| 92 |
+
raise BindingMetadataRefusal("ending_cell_unreadable", ref or "(blank)",
|
| 93 |
+
"its reference is invalid.")
|
| 94 |
+
out["ref"] = ref
|
| 95 |
+
return out
|
| 96 |
+
|
| 97 |
+
unit = str(raw.get("unit") or "").strip().lower()
|
| 98 |
+
if unit:
|
| 99 |
+
if unit not in PERIOD_UNITS:
|
| 100 |
+
raise BindingMetadataRefusal("period_invalid", unit)
|
| 101 |
+
out["unit"] = unit
|
| 102 |
+
|
| 103 |
+
start = str(raw.get("start") or "").strip()
|
| 104 |
+
if start:
|
| 105 |
+
pattern = PERIOD_DAY if unit in ("day", "week") else PERIOD_START
|
| 106 |
+
if not pattern.fullmatch(start):
|
| 107 |
+
raise BindingMetadataRefusal("period_invalid", start)
|
| 108 |
+
if unit in ("day", "week"):
|
| 109 |
+
try:
|
| 110 |
+
day = datetime.date.fromisoformat(start)
|
| 111 |
+
except ValueError as exc:
|
| 112 |
+
raise BindingMetadataRefusal("period_invalid", start) from exc
|
| 113 |
+
if unit == "week":
|
| 114 |
+
start = (day - datetime.timedelta(days=day.weekday())).isoformat()
|
| 115 |
+
out["start"] = start
|
| 116 |
+
|
| 117 |
+
if raw.get("frozen") is True:
|
| 118 |
+
dates = {}
|
| 119 |
+
for key in ("from", "to"):
|
| 120 |
+
value = str(raw.get(key) or "").strip()
|
| 121 |
+
if not PERIOD_DAY.fullmatch(value):
|
| 122 |
+
raise BindingMetadataRefusal("period_invalid", value or key)
|
| 123 |
+
dates[key] = value
|
| 124 |
+
if dates["to"] < dates["from"]:
|
| 125 |
+
raise BindingMetadataRefusal("period_invalid", dates["to"])
|
| 126 |
+
out.update({"frozen": True, **dates})
|
| 127 |
+
|
| 128 |
+
if raw.get("locked") is True:
|
| 129 |
+
out["locked"] = True
|
| 130 |
+
ref = str(raw.get("ref") or "").strip().upper().replace("$", "")
|
| 131 |
+
if ref:
|
| 132 |
+
if not PERIOD_REF.match(ref):
|
| 133 |
+
raise BindingMetadataRefusal("period_invalid", ref)
|
| 134 |
+
if not out.get("frozen"):
|
| 135 |
+
out["ref"] = ref
|
| 136 |
+
if (raw.get("window") is not None and "unit" not in out and "start" not in out
|
| 137 |
+
and not out.get("frozen")):
|
| 138 |
+
out["window"] = raw["window"]
|
| 139 |
+
if not ({"start", "window", "ref", "frozen"} & set(out)):
|
| 140 |
+
return None
|
| 141 |
+
return out
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def clean_on_drag(raw):
|
| 145 |
+
"""Return canonical two-axis drag metadata, or ``None`` when absent."""
|
| 146 |
+
if not isinstance(raw, dict):
|
| 147 |
+
return None
|
| 148 |
+
across = str(raw.get("across") or "").strip().lower()
|
| 149 |
+
if across not in DRAG_ACROSS:
|
| 150 |
+
across = "period"
|
| 151 |
+
down = raw.get("down")
|
| 152 |
+
if isinstance(down, dict) and str(down.get("field") or "").strip():
|
| 153 |
+
out_down = {"field": str(down["field"]).strip()}
|
| 154 |
+
if down.get("mode") in ("value", "presence", "member"):
|
| 155 |
+
out_down["mode"] = down["mode"]
|
| 156 |
+
else:
|
| 157 |
+
out_down = DRAG_DOWN_NONE
|
| 158 |
+
if isinstance(raw.get("steps"), list):
|
| 159 |
+
steps = []
|
| 160 |
+
for step in raw["steps"]:
|
| 161 |
+
if not isinstance(step, dict):
|
| 162 |
+
continue
|
| 163 |
+
field = step.get("field")
|
| 164 |
+
if not isinstance(field, str) or not field.strip() or step.get("axis") not in ("across", "down"):
|
| 165 |
+
continue
|
| 166 |
+
if "size" in step and step["size"] not in ("day", "week", "month", "quarter", "year"):
|
| 167 |
+
continue
|
| 168 |
+
cleaned = {"field": field.strip(), "axis": step["axis"]}
|
| 169 |
+
if step.get("mode") in ("value", "presence", "member"):
|
| 170 |
+
cleaned["mode"] = step["mode"]
|
| 171 |
+
if "size" in step:
|
| 172 |
+
cleaned["size"] = step["size"]
|
| 173 |
+
steps.append(cleaned)
|
| 174 |
+
first_down = next((step for step in steps if step["axis"] == "down"), None)
|
| 175 |
+
out_down = ({key: first_down[key] for key in ("field", "mode") if key in first_down}
|
| 176 |
+
if first_down else DRAG_DOWN_NONE)
|
| 177 |
+
across = ("period" if any(step["axis"] == "across" and step["field"]
|
| 178 |
+
in ("__period", "__ending") for step in steps) else "none")
|
| 179 |
+
return {"across": across, "down": out_down, "steps": steps}
|
| 180 |
+
return {"across": across, "down": out_down}
|
platform/core/sheet_import.py
CHANGED
|
@@ -71,6 +71,13 @@ PALETTE = {
|
|
| 71 |
"--lp-ink": "#202433",
|
| 72 |
"--lp-muted": "#6b7280",
|
| 73 |
"--lp-line": "#e6e9ef",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
"--lp-wash": "#fafbfc",
|
| 75 |
"--lp-blue": "#9dbff2",
|
| 76 |
"--lp-blue-tint": "#edf3fd",
|
|
@@ -89,6 +96,23 @@ PALETTE = {
|
|
| 89 |
"--lp-purple": "#c3b3ee",
|
| 90 |
"--lp-purple-tint": "#f1eefb",
|
| 91 |
"--lp-purple-deep": "#6b57a8",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
"--lp-brand": "#6e56cf",
|
| 93 |
"--lp-primary": "#5433cc",
|
| 94 |
"--lp-primary-hover": "#472bad",
|
|
|
|
| 71 |
"--lp-ink": "#202433",
|
| 72 |
"--lp-muted": "#6b7280",
|
| 73 |
"--lp-line": "#e6e9ef",
|
| 74 |
+
# ⭐ ADDED AT THE WAVE-62 COMBINE. W62-T31 / D-684 declares `--lp-line-strong` in
|
| 75 |
+
# `index.css` as a darker step than `--lp-line` for stronger separators, and
|
| 76 |
+
# `verify_sheet.py` C4's other direction went red the moment lane H's branch merged:
|
| 77 |
+
# a colour `:root` declares that the importer cannot quantize onto is a colour an
|
| 78 |
+
# imported cell can never keep. `--lp-line` was already here, so a line colour belongs
|
| 79 |
+
# in this table by the same rule; the size tokens the head note excludes are unaffected.
|
| 80 |
+
"--lp-line-strong": "#cbd1dc",
|
| 81 |
"--lp-wash": "#fafbfc",
|
| 82 |
"--lp-blue": "#9dbff2",
|
| 83 |
"--lp-blue-tint": "#edf3fd",
|
|
|
|
| 96 |
"--lp-purple": "#c3b3ee",
|
| 97 |
"--lp-purple-tint": "#f1eefb",
|
| 98 |
"--lp-purple-deep": "#6b57a8",
|
| 99 |
+
# ⭐ W61-T08 / THE COMBINE — THE THIRD SEAM. W61-T08 gives chart series eight distinct brand
|
| 100 |
+
# hues and declares each in the two seams its `done-when` names, `index.css` `:root` and
|
| 101 |
+
# `customer-grid/theme.ts`. This copy is a THIRD, and nothing pointed lane A at it: it lives in
|
| 102 |
+
# `platform/core`, which may not read the web tree at all, so only `verify_sheet.py` section 4
|
| 103 |
+
# relates the two — and it is an API gate, not one of that ticket's. It caught the drift on the
|
| 104 |
+
# merged tree exactly as designed ("a colour ADDED to :root cannot sit unreachable to an
|
| 105 |
+
# import"), which is why these are ADDED here rather than the leg being narrowed: an .xlsx
|
| 106 |
+
# painted one of the new hues must quantize onto it, not onto the nearest older colour.
|
| 107 |
+
"--lp-orange": "#edbf91",
|
| 108 |
+
"--lp-orange-tint": "#fcf1e6",
|
| 109 |
+
"--lp-orange-deep": "#ba6c27",
|
| 110 |
+
"--lp-pink": "#e6afcc",
|
| 111 |
+
"--lp-pink-tint": "#f9edf4",
|
| 112 |
+
"--lp-pink-deep": "#a54c78",
|
| 113 |
+
"--lp-teal": "#9bcfce",
|
| 114 |
+
"--lp-teal-tint": "#eaf5f4",
|
| 115 |
+
"--lp-teal-deep": "#00938d",
|
| 116 |
"--lp-brand": "#6e56cf",
|
| 117 |
"--lp-primary": "#5433cc",
|
| 118 |
"--lp-primary-hover": "#472bad",
|
platform/core/sheet_structure.py
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pure workbook structure edits using Sheet's A1 scanner and the Wave 61 C1 rules.
|
| 2 |
+
|
| 3 |
+
Addresses, hidden lines and row-height keys are 1-based; rectangles are 0-based.
|
| 4 |
+
Structural edits move absolute references too. The dollar marks only pin a copy/fill.
|
| 5 |
+
The caller owns document validation, persistence and recalculation.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import copy
|
| 10 |
+
import math
|
| 11 |
+
import re
|
| 12 |
+
from collections.abc import Callable
|
| 13 |
+
|
| 14 |
+
from core.sheets import SheetRefusal
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
REF_SOURCE = r"(?:(?:'([^']{1,64})'|([A-Za-z_][A-Za-z0-9_]{0,63}))!)?(\$?)([A-Za-z]{1,3})(\$?)([1-9][0-9]{0,6})"
|
| 18 |
+
_REF = re.compile(REF_SOURCE)
|
| 19 |
+
_BARE_SHEET = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,63}")
|
| 20 |
+
_PREVIOUS = re.compile(r"[A-Za-z0-9_$]")
|
| 21 |
+
_START = re.compile(r"[A-Za-z$']")
|
| 22 |
+
_FUNCTION = re.compile(r"\s*\(")
|
| 23 |
+
BINDING_REF_KEYS = frozenset({"ref", "valueRef", "cellRef", "endingRef"})
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def col_to_index(letters: str) -> int:
|
| 27 |
+
"""Spreadsheet base-26: A -> 1, Z -> 26, AA -> 27."""
|
| 28 |
+
index = 0
|
| 29 |
+
for letter in letters.upper():
|
| 30 |
+
index = index * 26 + ord(letter) - 64
|
| 31 |
+
return index
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def index_to_col(index: int) -> str:
|
| 35 |
+
"""The inverse of col_to_index; an index below 1 yields an empty string."""
|
| 36 |
+
index = math.floor(index)
|
| 37 |
+
letters = []
|
| 38 |
+
while index > 0:
|
| 39 |
+
index, remainder = divmod(index - 1, 26)
|
| 40 |
+
letters.append(chr(65 + remainder))
|
| 41 |
+
return "".join(reversed(letters))
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def parse_ref(text: str) -> dict | None:
|
| 45 |
+
"""Parse one A1 reference, retaining the client's CellRef dictionary keys."""
|
| 46 |
+
match = _REF.fullmatch(str(text or "").strip())
|
| 47 |
+
if not match:
|
| 48 |
+
return None
|
| 49 |
+
sheet_quoted, sheet_bare, col_abs, col, row_abs, row = match.groups()
|
| 50 |
+
return {"sheet": sheet_quoted or sheet_bare or "", "colAbs": col_abs == "$",
|
| 51 |
+
"col": col.upper(), "rowAbs": row_abs == "$", "row": int(row)}
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def format_ref(ref: dict | str) -> str:
|
| 55 |
+
"""Format a CellRef, preserving dollar marks and quoting non-bare sheet names."""
|
| 56 |
+
if isinstance(ref, str):
|
| 57 |
+
return ref
|
| 58 |
+
body = (f"{'$' if ref['colAbs'] else ''}{ref['col']}"
|
| 59 |
+
f"{'$' if ref['rowAbs'] else ''}{ref['row']}")
|
| 60 |
+
sheet = ref["sheet"]
|
| 61 |
+
if not sheet:
|
| 62 |
+
return body
|
| 63 |
+
name = sheet if _BARE_SHEET.fullmatch(sheet) else "'" + sheet.replace("'", "") + "'"
|
| 64 |
+
return name + "!" + body
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def rewrite_refs(text: str, rewrite: Callable[[dict], dict | str],
|
| 68 |
+
rewrite_range: Callable[[dict, dict, str], str] | None = None) -> str:
|
| 69 |
+
"""Walk text exactly as a1.ts::rewriteRefs; strings and function names survive."""
|
| 70 |
+
src = str(text or "")
|
| 71 |
+
out = []
|
| 72 |
+
i = 0
|
| 73 |
+
quote = None
|
| 74 |
+
while i < len(src):
|
| 75 |
+
char = src[i]
|
| 76 |
+
if quote:
|
| 77 |
+
out.append(char)
|
| 78 |
+
if char == quote:
|
| 79 |
+
quote = None
|
| 80 |
+
i += 1
|
| 81 |
+
continue
|
| 82 |
+
if char in ('"', "'"):
|
| 83 |
+
close = src.find(char, i + 1)
|
| 84 |
+
is_sheet = char == "'" and close > i and src[close + 1:close + 2] == "!"
|
| 85 |
+
if not is_sheet:
|
| 86 |
+
quote = char
|
| 87 |
+
out.append(char)
|
| 88 |
+
i += 1
|
| 89 |
+
continue
|
| 90 |
+
previous = src[i - 1] if i else ""
|
| 91 |
+
if not _PREVIOUS.search(previous) and _START.fullmatch(char):
|
| 92 |
+
match = _REF.match(src, i)
|
| 93 |
+
if match and not _FUNCTION.match(src, match.end()):
|
| 94 |
+
ref = parse_ref(match[0])
|
| 95 |
+
if ref:
|
| 96 |
+
separator = re.match(r"\s*:\s*", src[match.end():]) if rewrite_range else None
|
| 97 |
+
if separator:
|
| 98 |
+
end = _REF.match(src, match.end() + separator.end())
|
| 99 |
+
if end and not _FUNCTION.match(src, end.end()):
|
| 100 |
+
last = parse_ref(end[0])
|
| 101 |
+
if last:
|
| 102 |
+
out.append(rewrite_range(ref, last, separator[0]))
|
| 103 |
+
i = end.end()
|
| 104 |
+
continue
|
| 105 |
+
out.append(format_ref(rewrite(ref)))
|
| 106 |
+
i = match.end()
|
| 107 |
+
continue
|
| 108 |
+
out.append(char)
|
| 109 |
+
i += 1
|
| 110 |
+
return "".join(out)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def shift_index(value: int, at: int, delta: int) -> int | None:
|
| 114 |
+
"""R49 retires the clamp: both grammars already parse #REF! as an error token.
|
| 115 |
+
|
| 116 |
+
The feared bad_formula cannot occur. Deleted indices return None; only a range
|
| 117 |
+
with surviving cells may clamp its deleted endpoint and shrink.
|
| 118 |
+
"""
|
| 119 |
+
return _moved_index(value, at, delta)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _same_name(left: str, right: str) -> bool:
|
| 123 |
+
return str(left or "").strip().lower() == str(right or "").strip().lower()
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def shift_for_structure(text: str, shift: dict, own_sheet: str, *, bare: bool = False) -> str:
|
| 127 |
+
"""Re-anchor formula text or, with bare=True, a single reference field.
|
| 128 |
+
|
| 129 |
+
shift names the edited sheet and contains axis (row/col), at and delta.
|
| 130 |
+
A bare reference belongs to own_sheet. Invalid bare fields are left verbatim.
|
| 131 |
+
"""
|
| 132 |
+
if not shift["delta"] or (bare and parse_ref(text) is None):
|
| 133 |
+
return str(text or "")
|
| 134 |
+
|
| 135 |
+
def index(ref):
|
| 136 |
+
return ref["row"] if shift["axis"] == "row" else col_to_index(ref["col"])
|
| 137 |
+
|
| 138 |
+
def at_index(ref, value):
|
| 139 |
+
return {**ref, shift["axis"]: value if shift["axis"] == "row" else index_to_col(value)}
|
| 140 |
+
|
| 141 |
+
def rewrite(ref):
|
| 142 |
+
if not _same_name(ref["sheet"] or own_sheet, shift["sheet"]):
|
| 143 |
+
return ref
|
| 144 |
+
moved = shift_index(index(ref), shift["at"], shift["delta"])
|
| 145 |
+
return "#REF!" if moved is None else at_index(ref, moved)
|
| 146 |
+
|
| 147 |
+
def rewrite_range(first, last, separator):
|
| 148 |
+
sheet = first["sheet"] or own_sheet
|
| 149 |
+
if last["sheet"] and not _same_name(sheet, last["sheet"]):
|
| 150 |
+
return format_ref(rewrite(first)) + separator + format_ref(rewrite(last))
|
| 151 |
+
# A bare right endpoint inherits the left endpoint's sheet, not the formula's tab.
|
| 152 |
+
if not _same_name(sheet, shift["sheet"]):
|
| 153 |
+
return format_ref(first) + separator + format_ref(last)
|
| 154 |
+
start, end = index(first), index(last)
|
| 155 |
+
low, high = min(start, end), max(start, end)
|
| 156 |
+
if low >= shift["at"] and high < shift["at"] - shift["delta"]:
|
| 157 |
+
return "#REF!"
|
| 158 |
+
|
| 159 |
+
def move(value):
|
| 160 |
+
moved = shift_index(value, shift["at"], shift["delta"])
|
| 161 |
+
return moved if moved is not None else shift["at"] if value == low else shift["at"] - 1
|
| 162 |
+
|
| 163 |
+
return format_ref(at_index(first, move(start))) + separator + format_ref(at_index(last, move(end)))
|
| 164 |
+
|
| 165 |
+
# Insertions keep the existing per-reference scan; range handling is deletion-only.
|
| 166 |
+
return rewrite_refs(text, rewrite, rewrite_range if shift["delta"] < 0 else None)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _moved_index(value: int, at: int, delta: int) -> int | None:
|
| 170 |
+
if value < at:
|
| 171 |
+
return value
|
| 172 |
+
if delta > 0:
|
| 173 |
+
return value + delta
|
| 174 |
+
return None if value < at - delta else value + delta
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def _reanchor_binding(value, shift: dict, own_sheet: str) -> bool:
|
| 178 |
+
"""Rewrite only parseable ref-bearing fields in the copied binding's dict/list tree."""
|
| 179 |
+
changed = False
|
| 180 |
+
if isinstance(value, list):
|
| 181 |
+
for child in value:
|
| 182 |
+
changed = _reanchor_binding(child, shift, own_sheet) or changed
|
| 183 |
+
elif isinstance(value, dict):
|
| 184 |
+
for key, child in value.items():
|
| 185 |
+
if key in BINDING_REF_KEYS and isinstance(child, str) and parse_ref(child):
|
| 186 |
+
value[key] = shift_for_structure(child, shift, own_sheet, bare=True)
|
| 187 |
+
changed = value[key] != child or changed
|
| 188 |
+
elif isinstance(child, (dict, list)):
|
| 189 |
+
changed = _reanchor_binding(child, shift, own_sheet) or changed
|
| 190 |
+
return changed
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _reanchor_cells(tab: dict, shift: dict) -> None:
|
| 194 |
+
for cell in tab.get("cells", {}).values():
|
| 195 |
+
changed = False
|
| 196 |
+
if cell.get("kind") == "formula" and isinstance(cell.get("formula"), str):
|
| 197 |
+
previous = cell["formula"]
|
| 198 |
+
cell["formula"] = shift_for_structure(previous, shift, tab["name"])
|
| 199 |
+
changed = cell["formula"] != previous
|
| 200 |
+
if changed and cell.get("display") == previous:
|
| 201 |
+
cell["display"] = cell["formula"]
|
| 202 |
+
if cell.get("kind") == "bound":
|
| 203 |
+
changed = _reanchor_binding(cell.get("binding"), shift, tab["name"]) or changed
|
| 204 |
+
link = cell.get("link")
|
| 205 |
+
if isinstance(link, dict) and link.get("kind") in ("cell", "sheet"):
|
| 206 |
+
href = link.get("href")
|
| 207 |
+
if isinstance(href, str) and parse_ref(href):
|
| 208 |
+
link["href"] = shift_for_structure(href, shift, tab["name"], bare=True)
|
| 209 |
+
changed = link["href"] != href or changed
|
| 210 |
+
if changed and cell.get("kind") in ("formula", "bound"):
|
| 211 |
+
cell["state"] = "stale"
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def _move_range(rect: dict, op: dict, *, drop_deleted: bool = False) -> dict | None:
|
| 215 |
+
"""Keep surviving rectangle cells; collapse a fully deleted chart at the gap."""
|
| 216 |
+
start, end = op["axis"] + "0", op["axis"] + "1"
|
| 217 |
+
lo, hi = rect[start], rect[end]
|
| 218 |
+
point, delta = op["at"] - 1, op["delta"]
|
| 219 |
+
if delta > 0:
|
| 220 |
+
lo += delta if point <= lo else 0
|
| 221 |
+
hi += delta if point <= hi else 0
|
| 222 |
+
elif delta < 0:
|
| 223 |
+
past = point - delta
|
| 224 |
+
before, after = lo < point, hi >= past
|
| 225 |
+
if not before and not after and drop_deleted:
|
| 226 |
+
return None
|
| 227 |
+
lo = lo if before else max(lo, past) + delta if after else point
|
| 228 |
+
hi = hi + delta if after else min(hi, point - 1) if before else point
|
| 229 |
+
return {**rect, start: lo, end: hi}
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _move_tab(tab: dict, op: dict) -> None:
|
| 233 |
+
axis, at, delta = op["axis"], op["at"], op["delta"]
|
| 234 |
+
cells = {}
|
| 235 |
+
for address, cell in tab.get("cells", {}).items():
|
| 236 |
+
ref = parse_ref(address)
|
| 237 |
+
if not ref:
|
| 238 |
+
cells[address] = cell
|
| 239 |
+
continue
|
| 240 |
+
moved = _moved_index(ref["row"] if axis == "row" else col_to_index(ref["col"]), at, delta)
|
| 241 |
+
if moved is not None:
|
| 242 |
+
ref[axis] = moved if axis == "row" else index_to_col(moved)
|
| 243 |
+
cells[format_ref(ref)] = cell
|
| 244 |
+
tab["cells"] = cells
|
| 245 |
+
dimension = "rowHeights" if axis == "row" else "colWidths"
|
| 246 |
+
if dimension in tab:
|
| 247 |
+
sizes = {}
|
| 248 |
+
for key, size in tab[dimension].items():
|
| 249 |
+
moved = _moved_index(int(key) if axis == "row" else col_to_index(key), at, delta)
|
| 250 |
+
if moved is not None:
|
| 251 |
+
sizes[str(moved) if axis == "row" else index_to_col(moved)] = size
|
| 252 |
+
tab[dimension] = sizes
|
| 253 |
+
hidden = "hiddenRows" if axis == "row" else "hiddenCols"
|
| 254 |
+
if hidden in tab:
|
| 255 |
+
tab[hidden] = [moved for value in tab[hidden]
|
| 256 |
+
if (moved := _moved_index(value, at, delta)) is not None]
|
| 257 |
+
frozen = tab.get("freeze", {})
|
| 258 |
+
key = "rows" if axis == "row" else "cols"
|
| 259 |
+
if key in frozen and at <= frozen[key]:
|
| 260 |
+
# Only deleted frozen lines shrink the count, even when the band extends beyond it.
|
| 261 |
+
frozen[key] = max(0, frozen[key] + (delta if delta > 0 else -min(-delta, frozen[key] - at + 1)))
|
| 262 |
+
if "rules" in tab:
|
| 263 |
+
rules = []
|
| 264 |
+
for rule in tab["rules"]:
|
| 265 |
+
rect = _move_range(rule["range"], op, drop_deleted=True)
|
| 266 |
+
if rect is not None:
|
| 267 |
+
rules.append({**rule, "range": rect})
|
| 268 |
+
tab["rules"] = rules
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def _move_charts(tab: dict, tab_id: str, op: dict) -> None:
|
| 272 |
+
for chart in tab.get("charts", []):
|
| 273 |
+
source = chart.get("source") or {}
|
| 274 |
+
if source.get("sheetId", tab["id"]) == tab_id:
|
| 275 |
+
chart["range"] = _move_range(chart["range"], op)
|
| 276 |
+
if "ranges" in source:
|
| 277 |
+
source["ranges"] = [_move_range(rect, op) for rect in source["ranges"]]
|
| 278 |
+
if tab["id"] == tab_id:
|
| 279 |
+
# Stored geometry anchors are zero-based too; pixel offsets and sizes stay intact.
|
| 280 |
+
for key in ("geometry", "anchor"):
|
| 281 |
+
geometry = chart.get(key)
|
| 282 |
+
if isinstance(geometry, dict) and op["axis"] in geometry:
|
| 283 |
+
moved = shift_index(geometry[op["axis"]] + 1, op["at"], op["delta"])
|
| 284 |
+
geometry[op["axis"]] = (op["at"] if moved is None else moved) - 1
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def apply_structure(book: dict, tab_id: str, op: dict) -> dict:
|
| 288 |
+
"""Return a new document after an insert/delete on tab_id; never access persistence.
|
| 289 |
+
|
| 290 |
+
op requires axis (row/col), a positive integer at and an integer delta. Zero delta
|
| 291 |
+
returns an independent unchanged copy. An optional sheet name must match tab_id.
|
| 292 |
+
Sheet dimensions are viewport extents, retained as in the client applyStructure.
|
| 293 |
+
"""
|
| 294 |
+
if not isinstance(op, dict) or op.get("axis") not in ("row", "col"):
|
| 295 |
+
raise SheetRefusal("sheet_structure_invalid", "A structure change needs a row or column axis.")
|
| 296 |
+
if type(op.get("at")) is not int or op["at"] < 1:
|
| 297 |
+
raise SheetRefusal("sheet_structure_invalid", "A structure change needs a positive whole starting index.")
|
| 298 |
+
if type(op.get("delta")) is not int:
|
| 299 |
+
raise SheetRefusal("sheet_structure_invalid", "A structure change needs a whole insert or delete count.")
|
| 300 |
+
if not isinstance(book, dict) or not isinstance(book.get("sheets"), list):
|
| 301 |
+
raise SheetRefusal("sheet_structure_invalid", "A structure change needs a workbook with sheets.")
|
| 302 |
+
target = next((tab for tab in book["sheets"] if tab.get("id") == tab_id), None)
|
| 303 |
+
if target is None:
|
| 304 |
+
raise SheetRefusal("sheet_structure_tab_missing", "The sheet to insert or delete on does not exist.")
|
| 305 |
+
if "sheet" in op and not _same_name(op["sheet"], target["name"]):
|
| 306 |
+
raise SheetRefusal("sheet_structure_invalid", "The structure change names a different sheet.")
|
| 307 |
+
result = copy.deepcopy(book)
|
| 308 |
+
if not op["delta"]:
|
| 309 |
+
return result
|
| 310 |
+
shift = {**op, "sheet": target["name"]}
|
| 311 |
+
for tab in result["sheets"]:
|
| 312 |
+
_reanchor_cells(tab, shift)
|
| 313 |
+
_move_charts(tab, tab_id, op)
|
| 314 |
+
if tab["id"] == tab_id:
|
| 315 |
+
_move_tab(tab, op)
|
| 316 |
+
return result
|
platform/core/sheet_templates.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Loopable's
|
| 2 |
|
| 3 |
Descriptors are data; applying them uses the existing workbook persistence engine.
|
| 4 |
The API owns permission checks and calculation under the calling user's row scope.
|
|
@@ -6,7 +6,7 @@ The API owns permission checks and calculation under the calling user's row scop
|
|
| 6 |
from __future__ import annotations
|
| 7 |
|
| 8 |
from copy import deepcopy
|
| 9 |
-
from datetime import date, timedelta
|
| 10 |
import re
|
| 11 |
|
| 12 |
from core import sheets, user_tables
|
|
@@ -154,6 +154,103 @@ def templates_for_sources(sources):
|
|
| 154 |
return [row for row in TEMPLATES if row["source"] in allowed]
|
| 155 |
|
| 156 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
class TemplateRefusal(Exception):
|
| 158 |
def __init__(self, code, *, missing=(), detail=""):
|
| 159 |
self.code = code
|
|
@@ -205,8 +302,15 @@ def build_cells(template, resolved, database_key):
|
|
| 205 |
|
| 206 |
for entry in template["bindings"]:
|
| 207 |
cells, addr = target(entry)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 208 |
binding = {"database": database_key,
|
| 209 |
-
"aggregate":
|
| 210 |
"scope": _resolved_scope(entry.get("scope", {"filter": []}), resolved)}
|
| 211 |
if "period" in entry:
|
| 212 |
binding["period"] = deepcopy(entry["period"])
|
|
@@ -233,12 +337,14 @@ def apply_template(st, key, database_key, *, name="", created_by=""):
|
|
| 233 |
raise TemplateRefusal("unknown_database", detail="That database does not exist.")
|
| 234 |
if source_of_database(database_key) != descriptor["source"]:
|
| 235 |
raise TemplateRefusal("template_source_mismatch", detail="That database uses a different connector from this template.")
|
| 236 |
-
|
|
|
|
| 237 |
prepared = build_cells(descriptor, resolved, database_key)
|
| 238 |
# Validate tab names, extents and every cell before the first store write.
|
| 239 |
checked = sheets.migrate_workbook({"sheets": [
|
| 240 |
{"name": tab_name, "cells": cells} for _, tab_name, cells in prepared]})
|
| 241 |
-
book = sheets.create_workbook(st, name=name or descriptor["title"], created_by=created_by
|
|
|
|
| 242 |
try:
|
| 243 |
tabs = [{**tab, "cells": {}} for tab in checked["sheets"]]
|
| 244 |
book = sheets.put_workbook(st, book["id"], {"name": book["name"], "sheets": tabs, "rev": book["rev"]})
|
|
|
|
| 1 |
+
"""Loopable's built-in and tenant-saved connector workbooks.
|
| 2 |
|
| 3 |
Descriptors are data; applying them uses the existing workbook persistence engine.
|
| 4 |
The API owns permission checks and calculation under the calling user's row scope.
|
|
|
|
| 6 |
from __future__ import annotations
|
| 7 |
|
| 8 |
from copy import deepcopy
|
| 9 |
+
from datetime import date, datetime, timedelta, timezone
|
| 10 |
import re
|
| 11 |
|
| 12 |
from core import sheets, user_tables
|
|
|
|
| 154 |
return [row for row in TEMPLATES if row["source"] in allowed]
|
| 155 |
|
| 156 |
|
| 157 |
+
SAVED_TEMPLATES_KEY = "sheet_templates_saved"
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def saved_templates(st):
|
| 161 |
+
"""Pure, detached C4 document reader; absent storage is an empty schema-1 document."""
|
| 162 |
+
return deepcopy(st.get(SAVED_TEMPLATES_KEY) or {"schema": 1, "templates": {}})
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def saved_template(st, key):
|
| 166 |
+
"""One detached saved record, or None. Built-in descriptors remain separate."""
|
| 167 |
+
return saved_templates(st)["templates"].get(key)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _template_workbook(book):
|
| 171 |
+
clean = deepcopy(book)
|
| 172 |
+
for field in ("id", "rev", "writes", "writesFrom"):
|
| 173 |
+
clean.pop(field, None)
|
| 174 |
+
for tab in clean.get("sheets", []):
|
| 175 |
+
tab.pop("writes", None)
|
| 176 |
+
tab.pop("writesFrom", None)
|
| 177 |
+
return clean
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def save_sheet_template(st, workbook, key, *, name="", description="", created_by="", actor_name=""):
|
| 181 |
+
"""Save a workbook already admitted by the caller's reader wall, using C4's tenant document."""
|
| 182 |
+
if not isinstance(key, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", key):
|
| 183 |
+
raise sheets.SheetRefusal("sheet_template_key_invalid",
|
| 184 |
+
"A Sheet template key must use letters, numbers, underscores or hyphens.")
|
| 185 |
+
if template(key) is not None:
|
| 186 |
+
raise sheets.SheetRefusal("sheet_template_key_exists",
|
| 187 |
+
f"Sheet template {key} already names a built-in template.")
|
| 188 |
+
if saved_template(st, key) is not None:
|
| 189 |
+
raise sheets.SheetRefusal("sheet_template_key_exists",
|
| 190 |
+
f"Sheet template {key} already exists. Choose another key.")
|
| 191 |
+
if not isinstance(name, str) or not isinstance(description, str):
|
| 192 |
+
raise sheets.SheetRefusal("sheet_template_text_invalid",
|
| 193 |
+
f"Sheet template {key} needs a text name and description.")
|
| 194 |
+
checked = sheets.migrate_workbook(workbook)
|
| 195 |
+
book = _template_workbook(checked)
|
| 196 |
+
databases = set()
|
| 197 |
+
for tab in book["sheets"]:
|
| 198 |
+
for addr, cell in tab["cells"].items():
|
| 199 |
+
if cell.get("kind") != "bound":
|
| 200 |
+
continue
|
| 201 |
+
database = (cell.get("binding") or {}).get("database")
|
| 202 |
+
if not isinstance(database, str) or not database.strip():
|
| 203 |
+
raise sheets.SheetRefusal("sheet_template_database_missing",
|
| 204 |
+
f"Sheet template {key} has no Cell database at {addr}.")
|
| 205 |
+
databases.add(database)
|
| 206 |
+
record = {"key": key, "name": name.strip() or book.get("name") or key,
|
| 207 |
+
"description": description, "databases": sorted(databases), "workbook": book,
|
| 208 |
+
"created_by": created_by, "created_at": datetime.now(timezone.utc).isoformat(),
|
| 209 |
+
"byName": actor_name or created_by}
|
| 210 |
+
|
| 211 |
+
def save(state):
|
| 212 |
+
document = deepcopy(state or {"schema": 1, "templates": {}})
|
| 213 |
+
if document.get("schema") != 1 or not isinstance(document.get("templates"), dict):
|
| 214 |
+
raise sheets.SheetRefusal("sheet_templates_schema_unknown",
|
| 215 |
+
"Saved Sheet templates use a document version this app cannot save.")
|
| 216 |
+
if key in document["templates"]:
|
| 217 |
+
raise sheets.SheetRefusal("sheet_template_key_exists",
|
| 218 |
+
f"Sheet template {key} already exists. Choose another key.")
|
| 219 |
+
document["templates"][key] = deepcopy(record)
|
| 220 |
+
return document
|
| 221 |
+
|
| 222 |
+
st.update(SAVED_TEMPLATES_KEY, save, flush="sync")
|
| 223 |
+
return deepcopy(record)
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def apply_saved_template(st, key, *, name="", created_by=""):
|
| 227 |
+
"""Materialise a saved workbook through the same create/put/patch engine as built-ins."""
|
| 228 |
+
record = saved_template(st, key)
|
| 229 |
+
if record is None:
|
| 230 |
+
# Keys from a URL can contain user text; refusal prose follows standing rule 2.
|
| 231 |
+
label = str(key).replace(chr(0x2014), "-").replace(chr(0x2013), "-")
|
| 232 |
+
raise sheets.SheetRefusal("unknown_sheet_template", f"Sheet template {label} does not exist.")
|
| 233 |
+
# Validate the whole document before the first write, including tabs after the first.
|
| 234 |
+
checked = sheets.migrate_workbook(_template_workbook(record["workbook"]))
|
| 235 |
+
book = sheets.create_workbook(st, name=name or record["name"], created_by=created_by,
|
| 236 |
+
on_duplicate="suffix")
|
| 237 |
+
try:
|
| 238 |
+
tabs = [{**tab, "cells": {}} for tab in checked["sheets"]]
|
| 239 |
+
book = sheets.put_workbook(st, book["id"], {
|
| 240 |
+
**checked, "name": book["name"], "sheets": tabs, "rev": book["rev"]}, actor=created_by)
|
| 241 |
+
for index, tab in enumerate(checked["sheets"]):
|
| 242 |
+
result = sheets.patch_cells(st, book["id"], book["sheets"][index]["id"],
|
| 243 |
+
tab["cells"], actor=created_by)
|
| 244 |
+
if result.get("refused"):
|
| 245 |
+
addresses = [str(item["addr"]) for item in result["refused"]]
|
| 246 |
+
raise sheets.SheetRefusal("template_cells_refused",
|
| 247 |
+
"The template could not apply cells: " + ", ".join(addresses) + ".")
|
| 248 |
+
return sheets.get_workbook(st, book["id"])
|
| 249 |
+
except Exception:
|
| 250 |
+
sheets.delete_workbook(st, book["id"])
|
| 251 |
+
raise
|
| 252 |
+
|
| 253 |
+
|
| 254 |
class TemplateRefusal(Exception):
|
| 255 |
def __init__(self, code, *, missing=(), detail=""):
|
| 256 |
self.code = code
|
|
|
|
| 302 |
|
| 303 |
for entry in template["bindings"]:
|
| 304 |
cells, addr = target(entry)
|
| 305 |
+
if set(entry) - {"sheet", "addr", "fn", "field", "metric_key", "scope", "period"}:
|
| 306 |
+
raise TemplateRefusal("template_invalid", detail="The template binding has an unknown setting.")
|
| 307 |
+
aggregate = ({"metric_key": entry["metric_key"]} if "metric_key" in entry else
|
| 308 |
+
{"fn": entry["fn"], "field": resolved[entry["field"]]})
|
| 309 |
+
if "metric_key" in entry and (not entry["metric_key"] or "field" in entry or "fn" in entry
|
| 310 |
+
or not entry.get("period")):
|
| 311 |
+
raise TemplateRefusal("template_invalid", detail="A template metric needs a period and no field aggregate.")
|
| 312 |
binding = {"database": database_key,
|
| 313 |
+
"aggregate": aggregate,
|
| 314 |
"scope": _resolved_scope(entry.get("scope", {"filter": []}), resolved)}
|
| 315 |
if "period" in entry:
|
| 316 |
binding["period"] = deepcopy(entry["period"])
|
|
|
|
| 337 |
raise TemplateRefusal("unknown_database", detail="That database does not exist.")
|
| 338 |
if source_of_database(database_key) != descriptor["source"]:
|
| 339 |
raise TemplateRefusal("template_source_mismatch", detail="That database uses a different connector from this template.")
|
| 340 |
+
field_keys = [field["key"] for field in definition.get("fields", [])]
|
| 341 |
+
resolved = resolve_fields(descriptor, field_keys)
|
| 342 |
prepared = build_cells(descriptor, resolved, database_key)
|
| 343 |
# Validate tab names, extents and every cell before the first store write.
|
| 344 |
checked = sheets.migrate_workbook({"sheets": [
|
| 345 |
{"name": tab_name, "cells": cells} for _, tab_name, cells in prepared]})
|
| 346 |
+
book = sheets.create_workbook(st, name=name or descriptor["title"], created_by=created_by,
|
| 347 |
+
on_duplicate="suffix")
|
| 348 |
try:
|
| 349 |
tabs = [{**tab, "cells": {}} for tab in checked["sheets"]]
|
| 350 |
book = sheets.put_workbook(st, book["id"], {"name": book["name"], "sheets": tabs, "rev": book["rev"]})
|
platform/core/sheets.py
CHANGED
|
@@ -63,6 +63,12 @@ import re
|
|
| 63 |
import uuid
|
| 64 |
from datetime import datetime, timezone
|
| 65 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
#: C1's document key. `<id>` is the workbook id.
|
| 67 |
WORKBOOK_KEY_PREFIX = "sheet_workbook_"
|
| 68 |
|
|
@@ -81,6 +87,7 @@ AMAZON_DATA_ORIGIN_NAME = "Amazon Services API"
|
|
| 81 |
SHEET_SCHEMA_VERSION = 1
|
| 82 |
DOCUMENT_MAX_COLS = 16384
|
| 83 |
DOCUMENT_MAX_ROWS = 1048576
|
|
|
|
| 84 |
|
| 85 |
#: ⛔ THE FROZEN VOCABULARY (contract C1, amended by C2). `verify_api.py` and `verify_sheet.py`
|
| 86 |
#: assert `web/src/sheet/types.ts` carries exactly these, name for name.
|
|
@@ -92,6 +99,11 @@ DOCUMENT_MAX_ROWS = 1048576
|
|
| 92 |
CELL_KINDS = ("literal", "formula", "bound", "empty")
|
| 93 |
CELL_STATES = ("fresh", "stale", "typed", "refused")
|
| 94 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
#: A literal is ALWAYS this state (R19). Stated once, here, so no caller has to remember it.
|
| 96 |
#: ⚠ AN `empty` CELL TAKES THE SAME STATE, deliberately: it holds no binding, so `stale` would say
|
| 97 |
#: a refresh is owed for a value that does not exist, and a reader counting stale cells would
|
|
@@ -192,7 +204,9 @@ SHEET_RULE_OPS = ("gt", "lt", "gte", "lte", "eq", "neq", "between", "contains",
|
|
| 192 |
#: ⭐ `cell` IS STORED AS AN ABSENCE, exactly as `wrap: false` and `indent: 0` are: it is what
|
| 193 |
#: every rule written before this ticket already is, so writing the word onto them would make a
|
| 194 |
#: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`.
|
| 195 |
-
SHEET_RULE_KINDS = ("cell", "scale", "bar")
|
|
|
|
|
|
|
| 196 |
|
| 197 |
#: The style keys a conditional format rule may repaint. A rule is a PAINT, never a format: it
|
| 198 |
#: cannot change how a number reads, so it carries no `format`.
|
|
@@ -212,7 +226,11 @@ _RULE_STYLE_COLOURS = ("fill", "color")
|
|
| 212 |
#: but not stored here, is the defect, and it is what widening this tuple prevents.
|
| 213 |
SHEET_CHART_KINDS = ("bar", "line", "area", "donut", "column", "pie", "scatter", "combo",
|
| 214 |
"waterfall", "histogram", "bubble", "bullet", "funnel", "gauge",
|
| 215 |
-
"polar", "radar", "treemap", "stock", "gantt")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
|
| 217 |
#: ⛔⛔ D-640's MECHANISM, STATED. THIS IS EVERY KEY A CLEANED TAB MAY CARRY, and a key that is
|
| 218 |
#: not named here is DELETED on the next save, in silence, with nothing going red: `_clean_sheet`
|
|
@@ -230,7 +248,7 @@ SHEET_CHART_KINDS = ("bar", "line", "area", "donut", "column", "pie", "scatter",
|
|
| 230 |
#: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
|
| 231 |
SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
|
| 232 |
"rules", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
|
| 233 |
-
"visibility", "print", "excel")
|
| 234 |
|
| 235 |
#: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
|
| 236 |
#: `SHEET_TAB_KEYS` is written only when it holds something, for `_clean_style`'s reason: a tab
|
|
@@ -254,6 +272,7 @@ _MAX_DECIMALS = 10
|
|
| 254 |
#: constants, which is why nothing could grow a sheet or remember a dragged column: there was
|
| 255 |
#: nowhere to put the answer. These are the DEFAULTS a sheet is created with and the ceiling it may
|
| 256 |
#: grow to, and `web/src/sheet/types.ts` mirrors the three defaults.
|
|
|
|
| 257 |
#:
|
| 258 |
#: ⚠ THE CEILING IS A RENDERING BUDGET, NOT A GRAMMAR LIMIT. `_ADDR` below already admits three
|
| 259 |
#: column letters and seven row digits, and `a1.ts` agrees with it; 702 is the last TWO letter
|
|
@@ -266,7 +285,7 @@ MIN_SHEET_COLS = 1
|
|
| 266 |
MIN_SHEET_ROWS = 1
|
| 267 |
MAX_SHEET_COLS = 702
|
| 268 |
MAX_SHEET_ROWS = 100000
|
| 269 |
-
MIN_COL_WIDTH =
|
| 270 |
MAX_COL_WIDTH = 1000
|
| 271 |
|
| 272 |
#: A column key in `colWidths`. Letters only: a WIDTH belongs to a column, and a row number there
|
|
@@ -683,6 +702,54 @@ def _clean_sparkline(raw):
|
|
| 683 |
return {"kind": kind, "range": rect}
|
| 684 |
|
| 685 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 686 |
def clean_cell(raw, addr=""):
|
| 687 |
"""Normalize ONE cell, or raise `SheetRefusal` with a named code.
|
| 688 |
|
|
@@ -728,7 +795,17 @@ def clean_cell(raw, addr=""):
|
|
| 728 |
if meta and meta.get("kind", "normal") not in ("normal", "array", "shared", "dataTable"):
|
| 729 |
raise SheetRefusal("cell_formula_kind_unknown", "That formula representation is not supported.")
|
| 730 |
binding = raw.get("binding")
|
| 731 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 732 |
for tag in ("kind", "mode"):
|
| 733 |
if tag in binding and binding[tag] not in ("number", "column"):
|
| 734 |
raise SheetRefusal("cell_binding_kind_unknown", "A cell binding must choose Number or Column.")
|
|
@@ -778,7 +855,7 @@ def clean_cell(raw, addr=""):
|
|
| 778 |
cell["value"] = _json_safe(raw.get("value"), "cell value")
|
| 779 |
|
| 780 |
for key in ("binding", "source", "refusal"):
|
| 781 |
-
blob = raw.get(key)
|
| 782 |
if blob is None:
|
| 783 |
continue
|
| 784 |
if not isinstance(blob, dict):
|
|
@@ -881,12 +958,19 @@ def _clean_col_widths(raw, cols, *, lossless=False):
|
|
| 881 |
"Column " + (letters or str(key)) + " is not a column letter.")
|
| 882 |
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
| 883 |
if lossless:
|
| 884 |
-
raise SheetRefusal("col_width_invalid", "A column width must be a finite
|
| 885 |
continue
|
| 886 |
if not math.isfinite(value):
|
| 887 |
raise SheetRefusal("col_width_invalid", "A column width must be a finite size.")
|
| 888 |
-
|
| 889 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 890 |
width = value if lossless else max(MIN_COL_WIDTH, min(MAX_COL_WIDTH, int(value)))
|
| 891 |
if lossless or width != DEFAULT_COL_WIDTH:
|
| 892 |
out[letters] = width
|
|
@@ -1045,6 +1129,18 @@ def _clean_rules(raw, *, lossless=False):
|
|
| 1045 |
raise SheetRefusal(
|
| 1046 |
"sheet_rule_kind_unknown",
|
| 1047 |
"A sheet rule kind must be one of " + ", ".join(SHEET_RULE_KINDS) + ".")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1048 |
rect = _clean_range(row.get("range"))
|
| 1049 |
if rect is None:
|
| 1050 |
continue
|
|
@@ -1113,6 +1209,12 @@ def _clean_charts(raw):
|
|
| 1113 |
rect = _clean_range(row.get("range"))
|
| 1114 |
if kind not in SHEET_CHART_KINDS:
|
| 1115 |
raise SheetRefusal("sheet_chart_kind_unknown", "That chart kind is not supported.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1116 |
if rect is None:
|
| 1117 |
raise SheetRefusal("sheet_chart_range_invalid", "A chart needs a valid source range.")
|
| 1118 |
# Settings and source/series bags are authored data. Validate the discriminants we
|
|
@@ -1120,6 +1222,11 @@ def _clean_charts(raw):
|
|
| 1120 |
chart = _document_object(row, "sheet_chart")
|
| 1121 |
chart.update({"id": _text(row.get("id"), _MAX_ID) or _migration_id("chart", index, row),
|
| 1122 |
"range": rect, "kind": kind})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1123 |
for key in ("geometry", "anchor"):
|
| 1124 |
if chart.get(key) is not None:
|
| 1125 |
box = _document_object(chart[key], "sheet_chart_" + key)
|
|
@@ -1174,15 +1281,18 @@ def _clean_base_row_height(raw):
|
|
| 1174 |
return raw
|
| 1175 |
|
| 1176 |
|
| 1177 |
-
def _clean_row_heights(raw):
|
| 1178 |
if raw is None:
|
| 1179 |
return {}
|
| 1180 |
out = _document_object(raw, "sheet_row_heights")
|
| 1181 |
for row, height in out.items():
|
| 1182 |
if not re.fullmatch(r"[1-9][0-9]*", str(row)) or int(row) > DOCUMENT_MAX_ROWS \
|
| 1183 |
or isinstance(height, bool) or not isinstance(height, (int, float)) \
|
| 1184 |
-
or not math.isfinite(height) or height <= 0:
|
| 1185 |
raise SheetRefusal("sheet_row_height_invalid", "A row height needs a row number and a positive size.")
|
|
|
|
|
|
|
|
|
|
| 1186 |
return out
|
| 1187 |
|
| 1188 |
|
|
@@ -1197,6 +1307,38 @@ def _clean_print(raw):
|
|
| 1197 |
return out
|
| 1198 |
|
| 1199 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1200 |
def _clean_imported(raw):
|
| 1201 |
"""C4's import record: when, from what, and WHAT DID NOT SURVIVE.
|
| 1202 |
|
|
@@ -1267,8 +1409,9 @@ def _clean_sheet(raw, index=0, *, lossless=False):
|
|
| 1267 |
"charts": _clean_charts(raw.get("charts")),
|
| 1268 |
"imported": _clean_imported(raw.get("imported")),
|
| 1269 |
"rowHeight": _clean_base_row_height(raw.get("rowHeight")),
|
| 1270 |
-
"rowHeights": _clean_row_heights(raw.get("rowHeights")),
|
| 1271 |
"visibility": visibility,
|
|
|
|
| 1272 |
"print": _clean_print(raw.get("print")),
|
| 1273 |
"excel": excel,
|
| 1274 |
}
|
|
@@ -1287,7 +1430,7 @@ def _clean_sheet(raw, index=0, *, lossless=False):
|
|
| 1287 |
if value:
|
| 1288 |
sheet[key] = value
|
| 1289 |
extensions = {key: copy.deepcopy(value) for key, value in raw.items()
|
| 1290 |
-
if key not in set(optional) | set(SHEET_TAB_ALWAYS_KEYS) | {"cellCount"}}
|
| 1291 |
sheet.update(_json_safe(extensions, "sheet extensions"))
|
| 1292 |
if lossless:
|
| 1293 |
sheet["cols"] = max(cols, (sheet.get("freeze") or {}).get("cols", 0))
|
|
@@ -1308,7 +1451,7 @@ def _clean_sheets(raw, *, lossless=False):
|
|
| 1308 |
|
| 1309 |
# ── the workbook ─────────────────────────────────────────────────────────────────────────────
|
| 1310 |
|
| 1311 |
-
def migrate_workbook(raw):
|
| 1312 |
"""Pure unversioned/v0 -> v1 migration; validate current documents without recalculating.
|
| 1313 |
|
| 1314 |
A read never writes or changes `rev`, timestamps or cached results. Unknown versions
|
|
@@ -1332,7 +1475,7 @@ def migrate_workbook(raw):
|
|
| 1332 |
if book.get("print") is not None:
|
| 1333 |
book["print"] = _clean_print(book["print"])
|
| 1334 |
book["schemaVersion"] = SHEET_SCHEMA_VERSION
|
| 1335 |
-
book["sheets"] = _clean_sheets(raw.get("sheets"), lossless=
|
| 1336 |
return _json_safe(book, "workbook")
|
| 1337 |
|
| 1338 |
|
|
@@ -1346,10 +1489,11 @@ def new_workbook(name="", created_by="", workbook_id=""):
|
|
| 1346 |
"name": _text(name, _MAX_NAME) or "Untitled workbook",
|
| 1347 |
"sheets": [{"id": uuid.uuid4().hex[:12], "name": "Sheet1",
|
| 1348 |
"cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {},
|
| 1349 |
-
"usedExtent": None}],
|
| 1350 |
"rev": 1,
|
| 1351 |
"refreshed_at": None,
|
| 1352 |
"created_by": _text(created_by, _MAX_ID),
|
|
|
|
| 1353 |
"created_at": stamp,
|
| 1354 |
"updated_at": stamp,
|
| 1355 |
}
|
|
@@ -1416,8 +1560,45 @@ def _merge_data_origins(doc, *, sheets=(), databases=()):
|
|
| 1416 |
return origins
|
| 1417 |
|
| 1418 |
|
| 1419 |
-
def create_workbook(st, name="", created_by=""):
|
| 1420 |
-
"""Write a new `sheet_workbook_<id>` document and register it in the index.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1421 |
book = new_workbook(name=name, created_by=created_by)
|
| 1422 |
_json_safe(book, "workbook")
|
| 1423 |
|
|
@@ -1429,6 +1610,30 @@ def create_workbook(st, name="", created_by=""):
|
|
| 1429 |
return book
|
| 1430 |
|
| 1431 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1432 |
def get_workbook(st, workbook_id):
|
| 1433 |
"""One workbook, or a named refusal when the document is not there."""
|
| 1434 |
wid = _slug_id(workbook_id)
|
|
@@ -1436,13 +1641,13 @@ def get_workbook(st, workbook_id):
|
|
| 1436 |
if not doc or not doc.get("id"):
|
| 1437 |
raise SheetRefusal("workbook_missing",
|
| 1438 |
"That workbook no longer exists.")
|
| 1439 |
-
return migrate_workbook(doc)
|
| 1440 |
|
| 1441 |
|
| 1442 |
-
def put_workbook(st, workbook_id, raw, expect_rev=None):
|
| 1443 |
"""Replace a workbook's name and sheets, bumping `rev`.
|
| 1444 |
|
| 1445 |
-
⛔ `id`, `created_by` and `created_at` are PRIOR-WINS: a client may not reassign a workbook by
|
| 1446 |
sending a different id, and a client that never read the document cannot blank its provenance.
|
| 1447 |
|
| 1448 |
⚠ `expect_rev` makes the write CONDITIONAL, and when it is not passed explicitly the `rev`
|
|
@@ -1461,7 +1666,8 @@ def put_workbook(st, workbook_id, raw, expect_rev=None):
|
|
| 1461 |
if expect_rev is False:
|
| 1462 |
expect_rev = None
|
| 1463 |
|
| 1464 |
-
|
|
|
|
| 1465 |
sheets = incoming["sheets"]
|
| 1466 |
name = _text(raw.get("name"), _MAX_NAME)
|
| 1467 |
refreshed_at = raw.get("refreshed_at")
|
|
@@ -1472,9 +1678,10 @@ def put_workbook(st, workbook_id, raw, expect_rev=None):
|
|
| 1472 |
if prior is None:
|
| 1473 |
raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
|
| 1474 |
if expect_rev is not None and int(prior.get("rev") or 0) != int(expect_rev):
|
| 1475 |
-
raise
|
| 1476 |
"workbook_revision_stale",
|
| 1477 |
-
"That workbook changed somewhere else, so this save was not applied."
|
|
|
|
| 1478 |
book = migrate_workbook(prior)
|
| 1479 |
# Immutable ownership and server read projections cannot be reassigned by a PUT.
|
| 1480 |
# ``dataOrigins`` is SERVER-MAINTAINED (DPP 1.8) and therefore protected: a PUT body may
|
|
@@ -1483,16 +1690,24 @@ def put_workbook(st, workbook_id, raw, expect_rev=None):
|
|
| 1483 |
# ``[]`` erases a genuine Amazon origin, and sending a well-formed tag plants one on an
|
| 1484 |
# unrelated workbook. The origin list is rebuilt from the PRIOR document plus the cleaned
|
| 1485 |
# cells by ``_merge_data_origins`` immediately below.
|
| 1486 |
-
protected = {"id", "created_by", "created_at", "updated_at", "rev", "refreshed_at",
|
| 1487 |
"sheets", "name", "refreshedFor", "shares", "role", "permissions",
|
| 1488 |
"dataOrigins"}
|
| 1489 |
book.update({key: value for key, value in incoming.items() if key not in protected})
|
| 1490 |
book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook"
|
| 1491 |
-
book["sheets"] = sheets
|
|
|
|
| 1492 |
# DPP 1.8: merge from the PRIOR document and the cleaned cells. ``raw.dataOrigins`` is
|
| 1493 |
# ignored, so a client cannot erase or replace the server-maintained attribution list.
|
| 1494 |
_merge_data_origins(book, sheets=sheets)
|
| 1495 |
book["rev"] = int(prior.get("rev") or 0) + 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1496 |
if refreshed_at is not None:
|
| 1497 |
book["refreshed_at"] = _text(refreshed_at, 120) or None
|
| 1498 |
book["updated_at"] = _now()
|
|
@@ -1574,7 +1789,7 @@ def workbook_meta(book):
|
|
| 1574 |
return meta
|
| 1575 |
|
| 1576 |
|
| 1577 |
-
def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
|
| 1578 |
"""Merge a RANGE of cells into ONE tab. Returns `{rev, tab, refused}` (contract C3).
|
| 1579 |
|
| 1580 |
⭐⭐ ONE BAD CELL DOES NOT THROW THE COMMIT AWAY. Every address is cleaned on its own and a
|
|
@@ -1640,13 +1855,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
|
|
| 1640 |
if prior is None:
|
| 1641 |
raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
|
| 1642 |
current = int(prior.get("rev") or 0)
|
| 1643 |
-
|
| 1644 |
-
raise SheetStaleRefusal(
|
| 1645 |
-
"sheet_revision_stale",
|
| 1646 |
-
"This sheet changed somewhere else while you had it open, so this save was not "
|
| 1647 |
-
"applied. Reload the tab and try again.",
|
| 1648 |
-
rev_current=current)
|
| 1649 |
-
book = migrate_workbook(prior)
|
| 1650 |
tabs = book.get("sheets") or []
|
| 1651 |
at = _tab_index(tabs, tab_ref)
|
| 1652 |
if at is None:
|
|
@@ -1654,6 +1863,21 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
|
|
| 1654 |
"tab_missing",
|
| 1655 |
"Tab " + (str(tab_ref) or "(unnamed)") + " is not in this workbook.")
|
| 1656 |
tab = tabs[at]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1657 |
cells = dict(tab.get("cells") or {})
|
| 1658 |
for key in deleted:
|
| 1659 |
cells.pop(key, None)
|
|
@@ -1670,6 +1894,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
|
|
| 1670 |
book["sheets"] = tabs
|
| 1671 |
_merge_data_origins(book, sheets=tabs)
|
| 1672 |
book["rev"] = current + 1
|
|
|
|
| 1673 |
book["updated_at"] = _now()
|
| 1674 |
_json_safe(book, "workbook")
|
| 1675 |
out["rev"] = book["rev"]
|
|
@@ -1677,6 +1902,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
|
|
| 1677 |
out["cells"] = len(cells)
|
| 1678 |
out["schemaVersion"] = SHEET_SCHEMA_VERSION
|
| 1679 |
out["usedExtent"] = tab["usedExtent"]
|
|
|
|
| 1680 |
# Server-cleaned keys only. The route consumes this delta for calculation context.
|
| 1681 |
out["accepted"] = list(dict.fromkeys([*merged, *deleted]))
|
| 1682 |
saved.update(book)
|
|
@@ -1832,6 +2058,7 @@ def _index_row(book):
|
|
| 1832 |
"id": book.get("id") or "",
|
| 1833 |
"name": book.get("name") or "",
|
| 1834 |
"created_by": book.get("created_by") or "",
|
|
|
|
| 1835 |
"created_at": book.get("created_at") or "",
|
| 1836 |
"updated_at": book.get("updated_at") or "",
|
| 1837 |
"rev": int(book.get("rev") or 0),
|
|
@@ -1883,6 +2110,21 @@ def _index_upsert(st, book):
|
|
| 1883 |
#: registry's words, so a second spelling cannot be introduced here by accident.
|
| 1884 |
|
| 1885 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1886 |
def workbook_shares(st, workbook_id):
|
| 1887 |
"""C5's `shares` list for ONE workbook, computed from `core.shares` and never stored.
|
| 1888 |
|
|
@@ -1901,24 +2143,112 @@ def workbook_shares(st, workbook_id):
|
|
| 1901 |
rec = shares.grants("workbook", str(workbook_id), st=st)
|
| 1902 |
except Exception:
|
| 1903 |
return []
|
| 1904 |
-
return
|
|
|
|
| 1905 |
|
|
|
|
|
|
|
| 1906 |
|
| 1907 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1908 |
"""Join `shares` onto a list of index rows. Returns NEW rows; the stored ones are untouched.
|
| 1909 |
|
| 1910 |
-
|
| 1911 |
-
|
|
|
|
| 1912 |
`iter_workbooks` walks it to answer W49-T10's dependents question, and paying a grant-registry
|
| 1913 |
read per workbook there would put a permission lookup inside a scan that asks no permission
|
| 1914 |
question at all.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1915 |
"""
|
| 1916 |
out = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1917 |
for row in rows or ():
|
| 1918 |
if not isinstance(row, dict) or not row.get("id"):
|
| 1919 |
continue
|
| 1920 |
joined = dict(row)
|
| 1921 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1922 |
out.append(joined)
|
| 1923 |
return out
|
| 1924 |
|
|
@@ -2374,7 +2704,31 @@ def _index_rev(doc):
|
|
| 2374 |
return 0
|
| 2375 |
|
| 2376 |
|
| 2377 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2378 |
"""The whole rail document: `{workbooks, folders, itemFolders, starred, rev}`, plus
|
| 2379 |
`sharedWithMe` when a `viewer` is named.
|
| 2380 |
|
|
@@ -2388,8 +2742,8 @@ def read_index(st, viewer=None):
|
|
| 2388 |
so a `rev` here without the check there (or the check without this) is a guard nothing can
|
| 2389 |
reach ([[a-rev-guard-inert-on-the-events-channel]]). The pair lands together or not at all.
|
| 2390 |
|
| 2391 |
-
|
| 2392 |
-
|
| 2393 |
`_index_upsert` deliberately does NOT, because it is called by `create_workbook`,
|
| 2394 |
`put_workbook` AND `patch_cells` — bumping there would make every cell edit 409 a folder drag
|
| 2395 |
that touched nothing the guard is about.
|
|
@@ -2399,6 +2753,20 @@ def read_index(st, viewer=None):
|
|
| 2399 |
doc = _read(st, INDEX_KEY)
|
| 2400 |
rows = [dict(r) for r in (doc.get("workbooks") or [])
|
| 2401 |
if isinstance(r, dict) and r.get("id")]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2402 |
ids = {r["id"] for r in rows}
|
| 2403 |
placements = doc.get("itemFolders")
|
| 2404 |
try:
|
|
@@ -2407,37 +2775,45 @@ def read_index(st, viewer=None):
|
|
| 2407 |
flat = [dict(f, parent=None) for f in (doc.get("folders") or [])
|
| 2408 |
if isinstance(f, dict) and f.get("id")]
|
| 2409 |
folders, placed, _repairs = validate_folder_tree(flat, placements)
|
|
|
|
|
|
|
|
|
|
| 2410 |
out = {
|
| 2411 |
-
"workbooks":
|
| 2412 |
"folders": folders,
|
| 2413 |
"itemFolders": {k: v for k, v in placed.items() if k in ids},
|
| 2414 |
-
"starred":
|
| 2415 |
"rev": _index_rev(doc),
|
| 2416 |
}
|
| 2417 |
if viewer:
|
| 2418 |
-
# C5's `sharedWithMe`
|
| 2419 |
-
#
|
| 2420 |
-
#
|
| 2421 |
-
# second definition of what being shared means.
|
| 2422 |
try:
|
| 2423 |
import core.shares as shares
|
| 2424 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2425 |
except Exception:
|
| 2426 |
granted = []
|
| 2427 |
out["sharedWithMe"] = [w for w in (granted or []) if w in ids]
|
| 2428 |
return out
|
| 2429 |
|
| 2430 |
|
| 2431 |
-
def save_folders(st, folders, item_folders, starred=None, expect_rev=None
|
|
|
|
| 2432 |
"""Replace the folder, placement and star strata WHOLESALE, once the tree is proven legal.
|
| 2433 |
|
| 2434 |
Wholesale rather than per folder because the three are interdependent: a placement is only
|
| 2435 |
legal while its folder exists, so committing them separately would leave a window where a
|
| 2436 |
reader sees a workbook filed into a folder that is not there yet. One write, one state.
|
| 2437 |
|
| 2438 |
-
⛔ VALIDATION RUNS BEFORE THE UPDATE
|
| 2439 |
-
|
| 2440 |
-
|
| 2441 |
|
| 2442 |
RETURNS `{workbooks, folders, itemFolders, starred, rev, repairs}`. RAISES
|
| 2443 |
`SheetFolderRefusal` on a cycle or an over deep chain, having written nothing.
|
|
@@ -2455,9 +2831,16 @@ def save_folders(st, folders, item_folders, starred=None, expect_rev=None):
|
|
| 2455 |
impossible.
|
| 2456 |
"""
|
| 2457 |
tree, placed, repairs = validate_folder_tree(folders, item_folders)
|
|
|
|
|
|
|
|
|
|
| 2458 |
out = {}
|
| 2459 |
|
| 2460 |
def _apply(state):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2461 |
if expect_rev is not None and _index_rev(state) != int(expect_rev):
|
| 2462 |
raise SheetStaleRefusal(
|
| 2463 |
"index_revision_stale",
|
|
@@ -2467,25 +2850,76 @@ def save_folders(st, folders, item_folders, starred=None, expect_rev=None):
|
|
| 2467 |
rows = [r for r in (state.get("workbooks") or [])
|
| 2468 |
if isinstance(r, dict) and r.get("id")]
|
| 2469 |
ids = {r["id"] for r in rows}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2470 |
kept = {}
|
| 2471 |
-
for item_id, fid in
|
| 2472 |
if item_id in ids:
|
| 2473 |
kept[item_id] = fid
|
| 2474 |
else:
|
| 2475 |
-
|
| 2476 |
-
state["folders"] =
|
| 2477 |
state["itemFolders"] = kept
|
| 2478 |
-
if
|
| 2479 |
-
state["starred"] = _clean_starred(
|
| 2480 |
# D-639: this write IS the one the guard protects, so it moves the revision.
|
| 2481 |
state["rev"] = _index_rev(state) + 1
|
| 2482 |
out["value"] = {
|
| 2483 |
"workbooks": [dict(r) for r in rows],
|
| 2484 |
-
"folders": [dict(f) for f in
|
| 2485 |
"itemFolders": dict(kept),
|
| 2486 |
"starred": list(state.get("starred") or []),
|
| 2487 |
"rev": _index_rev(state),
|
| 2488 |
}
|
|
|
|
| 2489 |
return state, None
|
| 2490 |
|
| 2491 |
_persist(st, INDEX_KEY, _apply, seed={"workbooks": []})
|
|
@@ -2495,5 +2929,54 @@ def save_folders(st, folders, item_folders, starred=None, expect_rev=None):
|
|
| 2495 |
# store, and reading one document from inside another document's update callback puts a second
|
| 2496 |
# store call under a lock this one already holds.
|
| 2497 |
result["workbooks"] = project_shares(st, result.get("workbooks"))
|
| 2498 |
-
result["repairs"] = repairs
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2499 |
return result
|
|
|
|
| 63 |
import uuid
|
| 64 |
from datetime import datetime, timezone
|
| 65 |
|
| 66 |
+
from core.sheet_binding import (
|
| 67 |
+
BindingMetadataRefusal,
|
| 68 |
+
clean_on_drag as clean_binding_on_drag,
|
| 69 |
+
clean_period as clean_binding_period,
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
#: C1's document key. `<id>` is the workbook id.
|
| 73 |
WORKBOOK_KEY_PREFIX = "sheet_workbook_"
|
| 74 |
|
|
|
|
| 87 |
SHEET_SCHEMA_VERSION = 1
|
| 88 |
DOCUMENT_MAX_COLS = 16384
|
| 89 |
DOCUMENT_MAX_ROWS = 1048576
|
| 90 |
+
WRITE_LOG_REVS = 200
|
| 91 |
|
| 92 |
#: ⛔ THE FROZEN VOCABULARY (contract C1, amended by C2). `verify_api.py` and `verify_sheet.py`
|
| 93 |
#: assert `web/src/sheet/types.ts` carries exactly these, name for name.
|
|
|
|
| 99 |
CELL_KINDS = ("literal", "formula", "bound", "empty")
|
| 100 |
CELL_STATES = ("fresh", "stale", "typed", "refused")
|
| 101 |
|
| 102 |
+
# W62 C2 / R31: "every view type can be placed" inside the binding.
|
| 103 |
+
VIEW_BINDING_TYPES = ("grid", "kanban", "map", "gallery", "timeline", "list", "calendar", "catalog", "swipe")
|
| 104 |
+
VIEW_BINDING_LIMIT_DEFAULT = 100
|
| 105 |
+
VIEW_BINDING_LIMIT_MAX = 10000
|
| 106 |
+
|
| 107 |
#: A literal is ALWAYS this state (R19). Stated once, here, so no caller has to remember it.
|
| 108 |
#: ⚠ AN `empty` CELL TAKES THE SAME STATE, deliberately: it holds no binding, so `stale` would say
|
| 109 |
#: a refresh is owed for a value that does not exist, and a reader counting stale cells would
|
|
|
|
| 204 |
#: ⭐ `cell` IS STORED AS AN ABSENCE, exactly as `wrap: false` and `indent: 0` are: it is what
|
| 205 |
#: every rule written before this ticket already is, so writing the word onto them would make a
|
| 206 |
#: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`.
|
| 207 |
+
SHEET_RULE_KINDS = ("cell", "scale", "bar", "formula")
|
| 208 |
+
# The client compiler accepts 500 characters per condition.
|
| 209 |
+
MAX_RULE_FORMULA_LENGTH = 500
|
| 210 |
|
| 211 |
#: The style keys a conditional format rule may repaint. A rule is a PAINT, never a format: it
|
| 212 |
#: cannot change how a number reads, so it carries no `format`.
|
|
|
|
| 226 |
#: but not stored here, is the defect, and it is what widening this tuple prevents.
|
| 227 |
SHEET_CHART_KINDS = ("bar", "line", "area", "donut", "column", "pie", "scatter", "combo",
|
| 228 |
"waterfall", "histogram", "bubble", "bullet", "funnel", "gauge",
|
| 229 |
+
"polar", "radar", "treemap", "stock", "gantt", "sunburst", "boxWhisker")
|
| 230 |
+
|
| 231 |
+
# C5 / R21: a spec-level sub-type, never a presentation setting or another chart kind.
|
| 232 |
+
SHEET_CHART_STACKING_KINDS = ("column", "bar", "line", "area")
|
| 233 |
+
SHEET_CHART_STACKING = ("clustered", "stacked", "percent")
|
| 234 |
|
| 235 |
#: ⛔⛔ D-640's MECHANISM, STATED. THIS IS EVERY KEY A CLEANED TAB MAY CARRY, and a key that is
|
| 236 |
#: not named here is DELETED on the next save, in silence, with nothing going red: `_clean_sheet`
|
|
|
|
| 248 |
#: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
|
| 249 |
SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
|
| 250 |
"rules", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
|
| 251 |
+
"visibility", "view", "print", "excel")
|
| 252 |
|
| 253 |
#: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
|
| 254 |
#: `SHEET_TAB_KEYS` is written only when it holds something, for `_clean_style`'s reason: a tab
|
|
|
|
| 272 |
#: constants, which is why nothing could grow a sheet or remember a dragged column: there was
|
| 273 |
#: nowhere to put the answer. These are the DEFAULTS a sheet is created with and the ceiling it may
|
| 274 |
#: grow to, and `web/src/sheet/types.ts` mirrors the three defaults.
|
| 275 |
+
#: W62 C1 / R14 also mirrors the width bounds there; the client half lands in W62-T05.
|
| 276 |
#:
|
| 277 |
#: ⚠ THE CEILING IS A RENDERING BUDGET, NOT A GRAMMAR LIMIT. `_ADDR` below already admits three
|
| 278 |
#: column letters and seven row digits, and `a1.ts` agrees with it; 702 is the last TWO letter
|
|
|
|
| 285 |
MIN_SHEET_ROWS = 1
|
| 286 |
MAX_SHEET_COLS = 702
|
| 287 |
MAX_SHEET_ROWS = 100000
|
| 288 |
+
MIN_COL_WIDTH = 0
|
| 289 |
MAX_COL_WIDTH = 1000
|
| 290 |
|
| 291 |
#: A column key in `colWidths`. Letters only: a WIDTH belongs to a column, and a row number there
|
|
|
|
| 702 |
return {"kind": kind, "range": rect}
|
| 703 |
|
| 704 |
|
| 705 |
+
def clean_view_binding(raw):
|
| 706 |
+
"""W62-T10 / C2: normalize a PLACED VIEW binding, or raise `SheetRefusal` with a named code.
|
| 707 |
+
|
| 708 |
+
`{database, mode: "view", viewType, fields: [key], filter?, sort: [{field, dir}], period?,`
|
| 709 |
+
`limit (1..10000, default 100), onDrag?, block?: {cols, rows}}`.
|
| 710 |
+
|
| 711 |
+
⛔ IT IS A SEPARATE VALIDATOR FROM `modules/sheet.py::clean_binding` RATHER THAN A WIDENING OF
|
| 712 |
+
IT. That one REQUIRES an `aggregate`, because a metric binding IS an aggregate; a view has
|
| 713 |
+
none. Widening it would make `aggregate` optional for every caller, and the day a metric
|
| 714 |
+
binding arrives without one it would resolve to nothing instead of refusing by name.
|
| 715 |
+
"""
|
| 716 |
+
if not isinstance(raw, dict) or not isinstance(raw.get("database"), str) or not raw["database"].strip():
|
| 717 |
+
raise SheetRefusal("view_binding_invalid", "A view binding must be an object naming a database.")
|
| 718 |
+
if raw.get("mode", raw.get("kind")) != "view":
|
| 719 |
+
raise SheetRefusal("view_binding_mode_unknown", "A view binding must choose view mode.")
|
| 720 |
+
if raw.get("viewType") not in VIEW_BINDING_TYPES:
|
| 721 |
+
raise SheetRefusal("view_binding_type_unknown", "That view display type is not supported.")
|
| 722 |
+
fields = raw.get("fields")
|
| 723 |
+
if not isinstance(fields, list) or not fields or any(
|
| 724 |
+
not isinstance(key, str) or not key.strip() for key in fields):
|
| 725 |
+
raise SheetRefusal("view_binding_fields_invalid", "A view needs a non-empty list of field keys.")
|
| 726 |
+
sort = raw.get("sort", [])
|
| 727 |
+
if not isinstance(sort, list) or any(
|
| 728 |
+
not isinstance(item, dict) or not isinstance(item.get("field"), str)
|
| 729 |
+
or not item["field"].strip() or item.get("dir") not in ("asc", "desc") for item in sort):
|
| 730 |
+
raise SheetRefusal("view_binding_sort_invalid", "A view sort needs a field key and asc or desc direction.")
|
| 731 |
+
limit = raw.get("limit", VIEW_BINDING_LIMIT_DEFAULT)
|
| 732 |
+
if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= VIEW_BINDING_LIMIT_MAX:
|
| 733 |
+
raise SheetRefusal("view_binding_limit_invalid", "A view limit must be a whole number from 1 to 10000.")
|
| 734 |
+
out = {"database": raw["database"].strip(), "mode": "view", "viewType": raw["viewType"],
|
| 735 |
+
"fields": list(fields), "sort": [{"field": item["field"], "dir": item["dir"]} for item in sort],
|
| 736 |
+
"limit": limit}
|
| 737 |
+
for key in ("filter", "block"):
|
| 738 |
+
if key in raw:
|
| 739 |
+
out[key] = _document_object(raw[key], "view_binding_" + key)
|
| 740 |
+
# W62 C2: one lower-layer metadata cleaner serves both view and metric bindings.
|
| 741 |
+
if "period" in raw or "onDrag" in raw:
|
| 742 |
+
for key, cleaner in (("period", clean_binding_period), ("onDrag", clean_binding_on_drag)):
|
| 743 |
+
if key in raw:
|
| 744 |
+
try:
|
| 745 |
+
value = cleaner(_json_safe(copy.deepcopy(raw[key]), "view binding " + key))
|
| 746 |
+
except BindingMetadataRefusal as exc:
|
| 747 |
+
raise SheetRefusal(exc.code, exc.message) from exc
|
| 748 |
+
if value is not None:
|
| 749 |
+
out[key] = value
|
| 750 |
+
return out
|
| 751 |
+
|
| 752 |
+
|
| 753 |
def clean_cell(raw, addr=""):
|
| 754 |
"""Normalize ONE cell, or raise `SheetRefusal` with a named code.
|
| 755 |
|
|
|
|
| 795 |
if meta and meta.get("kind", "normal") not in ("normal", "array", "shared", "dataTable"):
|
| 796 |
raise SheetRefusal("cell_formula_kind_unknown", "That formula representation is not supported.")
|
| 797 |
binding = raw.get("binding")
|
| 798 |
+
# ⭐⭐ W62-T10 / C2 — A PLACED VIEW LIVES INSIDE `kind: "bound"`, AND `CELL_KINDS` STAYS
|
| 799 |
+
# FROZEN. Wave 49's cell-kind contract is the reason: the vocabulary is pinned here and two
|
| 800 |
+
# gates prove the client agrees with it, so a fifth kind would be a change to a frozen
|
| 801 |
+
# contract for a feature that does not need one. The discriminator is `binding.mode == "view"`,
|
| 802 |
+
# and a view binding carries no `aggregate` at all, which is why it cannot go through the
|
| 803 |
+
# metric validator below.
|
| 804 |
+
if isinstance(binding, dict) and (binding.get("mode") == "view" or binding.get("kind") == "view"):
|
| 805 |
+
if kind != "bound":
|
| 806 |
+
raise SheetRefusal("view_binding_invalid", "A view binding needs a bound cell.")
|
| 807 |
+
binding = clean_view_binding(binding)
|
| 808 |
+
elif isinstance(binding, dict):
|
| 809 |
for tag in ("kind", "mode"):
|
| 810 |
if tag in binding and binding[tag] not in ("number", "column"):
|
| 811 |
raise SheetRefusal("cell_binding_kind_unknown", "A cell binding must choose Number or Column.")
|
|
|
|
| 855 |
cell["value"] = _json_safe(raw.get("value"), "cell value")
|
| 856 |
|
| 857 |
for key in ("binding", "source", "refusal"):
|
| 858 |
+
blob = binding if key == "binding" else raw.get(key)
|
| 859 |
if blob is None:
|
| 860 |
continue
|
| 861 |
if not isinstance(blob, dict):
|
|
|
|
| 958 |
"Column " + (letters or str(key)) + " is not a column letter.")
|
| 959 |
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
| 960 |
if lossless:
|
| 961 |
+
raise SheetRefusal("col_width_invalid", "A column width must be a finite size.")
|
| 962 |
continue
|
| 963 |
if not math.isfinite(value):
|
| 964 |
raise SheetRefusal("col_width_invalid", "A column width must be a finite size.")
|
| 965 |
+
# ⭐ W62-T04 / R14 — A ZERO IS NOW A WIDTH, AND A NEGATIVE ONE STILL IS NOT. Before this
|
| 966 |
+
# ticket the lossless door refused every `value <= 0`, which is the same test for both; R14
|
| 967 |
+
# made 0 Excel's own hidden column, so an xlsx carrying one must import rather than refuse.
|
| 968 |
+
# A negative width is still nothing a file or a person can mean, so it keeps its refusal on
|
| 969 |
+
# the lossless door and is clamped to the floor on a live save.
|
| 970 |
+
if lossless and value < 0:
|
| 971 |
+
raise SheetRefusal("col_width_invalid", "A column width cannot be negative.")
|
| 972 |
+
#: W62 C1 / R14 / R19: reads retain hidden zero widths and exact imported values;
|
| 973 |
+
#: only a live save applies the bounds a person can set.
|
| 974 |
width = value if lossless else max(MIN_COL_WIDTH, min(MAX_COL_WIDTH, int(value)))
|
| 975 |
if lossless or width != DEFAULT_COL_WIDTH:
|
| 976 |
out[letters] = width
|
|
|
|
| 1129 |
raise SheetRefusal(
|
| 1130 |
"sheet_rule_kind_unknown",
|
| 1131 |
"A sheet rule kind must be one of " + ", ".join(SHEET_RULE_KINDS) + ".")
|
| 1132 |
+
if kind == "formula":
|
| 1133 |
+
condition = row.get("value")
|
| 1134 |
+
if not isinstance(condition, str) or not condition.lstrip("= \t\r\n"):
|
| 1135 |
+
raise SheetRefusal("sheet_rule_formula_missing",
|
| 1136 |
+
"A formula rule needs a condition.")
|
| 1137 |
+
if len(condition) > MAX_RULE_FORMULA_LENGTH:
|
| 1138 |
+
raise SheetRefusal("sheet_rule_formula_too_long",
|
| 1139 |
+
"A formula rule condition can hold at most "
|
| 1140 |
+
+ str(MAX_RULE_FORMULA_LENGTH) + " characters.")
|
| 1141 |
+
# A formula has one operand. Normalize even an imported empty/notEmpty op so
|
| 1142 |
+
# the existing value persistence path cannot silently drop its condition.
|
| 1143 |
+
op = "eq"
|
| 1144 |
rect = _clean_range(row.get("range"))
|
| 1145 |
if rect is None:
|
| 1146 |
continue
|
|
|
|
| 1209 |
rect = _clean_range(row.get("range"))
|
| 1210 |
if kind not in SHEET_CHART_KINDS:
|
| 1211 |
raise SheetRefusal("sheet_chart_kind_unknown", "That chart kind is not supported.")
|
| 1212 |
+
if "stacking" in row:
|
| 1213 |
+
if row["stacking"] not in SHEET_CHART_STACKING:
|
| 1214 |
+
raise SheetRefusal("sheet_chart_stacking_unknown", "That chart sub-type is not supported.")
|
| 1215 |
+
if kind not in SHEET_CHART_STACKING_KINDS: # Includes sunburst and boxWhisker (C5).
|
| 1216 |
+
raise SheetRefusal("sheet_chart_stacking_kind_invalid",
|
| 1217 |
+
"Chart sub-types need a column, bar, line or area chart.")
|
| 1218 |
if rect is None:
|
| 1219 |
raise SheetRefusal("sheet_chart_range_invalid", "A chart needs a valid source range.")
|
| 1220 |
# Settings and source/series bags are authored data. Validate the discriminants we
|
|
|
|
| 1222 |
chart = _document_object(row, "sheet_chart")
|
| 1223 |
chart.update({"id": _text(row.get("id"), _MAX_ID) or _migration_id("chart", index, row),
|
| 1224 |
"range": rect, "kind": kind})
|
| 1225 |
+
# Keep the C5 spec field explicitly across cleaning (D-640). Clustered is an absence,
|
| 1226 |
+
# including when an external writer sends the default by name.
|
| 1227 |
+
chart.pop("stacking", None)
|
| 1228 |
+
if row.get("stacking") in ("stacked", "percent"):
|
| 1229 |
+
chart["stacking"] = row["stacking"]
|
| 1230 |
for key in ("geometry", "anchor"):
|
| 1231 |
if chart.get(key) is not None:
|
| 1232 |
box = _document_object(chart[key], "sheet_chart_" + key)
|
|
|
|
| 1281 |
return raw
|
| 1282 |
|
| 1283 |
|
| 1284 |
+
def _clean_row_heights(raw, *, lossless=False):
|
| 1285 |
if raw is None:
|
| 1286 |
return {}
|
| 1287 |
out = _document_object(raw, "sheet_row_heights")
|
| 1288 |
for row, height in out.items():
|
| 1289 |
if not re.fullmatch(r"[1-9][0-9]*", str(row)) or int(row) > DOCUMENT_MAX_ROWS \
|
| 1290 |
or isinstance(height, bool) or not isinstance(height, (int, float)) \
|
| 1291 |
+
or not math.isfinite(height) or (lossless and height <= 0):
|
| 1292 |
raise SheetRefusal("sheet_row_height_invalid", "A row height needs a row number and a positive size.")
|
| 1293 |
+
#: W62 C1 / R19: a live size is clamped, while imported positive sizes stay exact.
|
| 1294 |
+
if not lossless:
|
| 1295 |
+
out[row] = max(18, min(200, height))
|
| 1296 |
return out
|
| 1297 |
|
| 1298 |
|
|
|
|
| 1307 |
return out
|
| 1308 |
|
| 1309 |
|
| 1310 |
+
#: Wave 60 base kit: `SheetTabView` in `sheet/types.ts`. Display settings, so a bad value is
|
| 1311 |
+
#: DROPPED rather than refused: a sheet must never fail to save over a zoom level.
|
| 1312 |
+
_VIEW_ZOOM_MIN, _VIEW_ZOOM_MAX = 10, 400
|
| 1313 |
+
_VIEW_PAGE_VIEWS = ("normal", "pageBreak", "pageLayout")
|
| 1314 |
+
|
| 1315 |
+
|
| 1316 |
+
def _clean_view(raw):
|
| 1317 |
+
"""A tab's display settings: gridlines, headings, zoom, pageView and split, nothing else.
|
| 1318 |
+
|
| 1319 |
+
Booleans stay booleans, zoom clamps to 10..400 (a non-number is dropped), pageView keeps only
|
| 1320 |
+
the three words, split keeps non-negative integer rows and cols or is dropped. An empty
|
| 1321 |
+
result is None, so the tab stores no empty stratum.
|
| 1322 |
+
"""
|
| 1323 |
+
if not isinstance(raw, dict):
|
| 1324 |
+
return None
|
| 1325 |
+
out = {}
|
| 1326 |
+
for key in ("gridlines", "headings"):
|
| 1327 |
+
if isinstance(raw.get(key), bool):
|
| 1328 |
+
out[key] = raw[key]
|
| 1329 |
+
zoom = raw.get("zoom")
|
| 1330 |
+
if not isinstance(zoom, bool) and isinstance(zoom, (int, float)) and math.isfinite(zoom):
|
| 1331 |
+
out["zoom"] = max(_VIEW_ZOOM_MIN, min(_VIEW_ZOOM_MAX, zoom))
|
| 1332 |
+
if raw.get("pageView") in _VIEW_PAGE_VIEWS:
|
| 1333 |
+
out["pageView"] = raw["pageView"]
|
| 1334 |
+
split = raw.get("split")
|
| 1335 |
+
if isinstance(split, dict):
|
| 1336 |
+
parts = [split.get("rows"), split.get("cols")]
|
| 1337 |
+
if all(isinstance(n, int) and not isinstance(n, bool) and n >= 0 for n in parts):
|
| 1338 |
+
out["split"] = {"rows": parts[0], "cols": parts[1]}
|
| 1339 |
+
return out or None
|
| 1340 |
+
|
| 1341 |
+
|
| 1342 |
def _clean_imported(raw):
|
| 1343 |
"""C4's import record: when, from what, and WHAT DID NOT SURVIVE.
|
| 1344 |
|
|
|
|
| 1409 |
"charts": _clean_charts(raw.get("charts")),
|
| 1410 |
"imported": _clean_imported(raw.get("imported")),
|
| 1411 |
"rowHeight": _clean_base_row_height(raw.get("rowHeight")),
|
| 1412 |
+
"rowHeights": _clean_row_heights(raw.get("rowHeights"), lossless=lossless),
|
| 1413 |
"visibility": visibility,
|
| 1414 |
+
"view": _clean_view(raw.get("view")),
|
| 1415 |
"print": _clean_print(raw.get("print")),
|
| 1416 |
"excel": excel,
|
| 1417 |
}
|
|
|
|
| 1430 |
if value:
|
| 1431 |
sheet[key] = value
|
| 1432 |
extensions = {key: copy.deepcopy(value) for key, value in raw.items()
|
| 1433 |
+
if key not in set(optional) | set(SHEET_TAB_ALWAYS_KEYS) | {"cellCount", "writes", "writesFrom"}}
|
| 1434 |
sheet.update(_json_safe(extensions, "sheet extensions"))
|
| 1435 |
if lossless:
|
| 1436 |
sheet["cols"] = max(cols, (sheet.get("freeze") or {}).get("cols", 0))
|
|
|
|
| 1451 |
|
| 1452 |
# ── the workbook ─────────────────────────────────────────────────────────────────────────────
|
| 1453 |
|
| 1454 |
+
def migrate_workbook(raw, *, lossless=True):
|
| 1455 |
"""Pure unversioned/v0 -> v1 migration; validate current documents without recalculating.
|
| 1456 |
|
| 1457 |
A read never writes or changes `rev`, timestamps or cached results. Unknown versions
|
|
|
|
| 1475 |
if book.get("print") is not None:
|
| 1476 |
book["print"] = _clean_print(book["print"])
|
| 1477 |
book["schemaVersion"] = SHEET_SCHEMA_VERSION
|
| 1478 |
+
book["sheets"] = _clean_sheets(raw.get("sheets"), lossless=lossless)
|
| 1479 |
return _json_safe(book, "workbook")
|
| 1480 |
|
| 1481 |
|
|
|
|
| 1489 |
"name": _text(name, _MAX_NAME) or "Untitled workbook",
|
| 1490 |
"sheets": [{"id": uuid.uuid4().hex[:12], "name": "Sheet1",
|
| 1491 |
"cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {},
|
| 1492 |
+
"usedExtent": None, "writesFrom": 0}],
|
| 1493 |
"rev": 1,
|
| 1494 |
"refreshed_at": None,
|
| 1495 |
"created_by": _text(created_by, _MAX_ID),
|
| 1496 |
+
"access": "private",
|
| 1497 |
"created_at": stamp,
|
| 1498 |
"updated_at": stamp,
|
| 1499 |
}
|
|
|
|
| 1560 |
return origins
|
| 1561 |
|
| 1562 |
|
| 1563 |
+
def create_workbook(st, name="", created_by="", on_duplicate="allow"):
|
| 1564 |
+
"""Write a new `sheet_workbook_<id>` document and register it in the index.
|
| 1565 |
+
|
| 1566 |
+
`on_duplicate` decides what a name already in this workspace means:
|
| 1567 |
+
`allow` (the default) writes it anyway, `refuse` raises `workbook_name_taken`, and `suffix`
|
| 1568 |
+
writes `Name (2)`, `Name (3)` instead.
|
| 1569 |
+
|
| 1570 |
+
⛔ THE DUPLICATE-NAME RULE BELONGS TO THE DOORS, NOT TO THIS MECHANISM (owner ruling
|
| 1571 |
+
2026-09-16, shaped by what the gate then measured). A workbook's identity is its id, so
|
| 1572 |
+
duplicates were always legal and nothing ever checked; the rail shows N identical rows with no
|
| 1573 |
+
date, size or owner and no reader can tell which is theirs. SDLC 61's C7 build left five
|
| 1574 |
+
workbooks named `Daily Sales Performance`, which is what made it visible.
|
| 1575 |
+
⚠ **Where the rule lives matters.** A person names a workbook at exactly two doors, and both
|
| 1576 |
+
pass `refuse`: `POST /sheets` (Create new) and the gateway's `sheet_create_workbook`. Every
|
| 1577 |
+
other caller creates a workbook as a MECHANISM, not as a naming act: applying a template and
|
| 1578 |
+
importing a file both INHERIT a name, so they pass `suffix` and a template stays usable more
|
| 1579 |
+
than once. Putting `refuse` on this function instead made a template single use, made an
|
| 1580 |
+
import single use, and broke thirteen gate fixtures whose names are incidental. A rule that
|
| 1581 |
+
fires on a mechanism fires on everything that uses it.
|
| 1582 |
+
"""
|
| 1583 |
+
wanted = str(name or "").strip()
|
| 1584 |
+
if wanted and on_duplicate != "allow":
|
| 1585 |
+
taken = {str((row or {}).get("name") or "").strip().casefold()
|
| 1586 |
+
for row in list_workbooks(st)}
|
| 1587 |
+
if wanted.casefold() in taken:
|
| 1588 |
+
if on_duplicate != "suffix":
|
| 1589 |
+
raise SheetRefusal(
|
| 1590 |
+
"workbook_name_taken",
|
| 1591 |
+
"A workbook named " + wanted + " is already in this workspace. "
|
| 1592 |
+
"Choose another name.")
|
| 1593 |
+
for nth in range(2, 1000):
|
| 1594 |
+
candidate = wanted + " (" + str(nth) + ")"
|
| 1595 |
+
if candidate.casefold() not in taken:
|
| 1596 |
+
name = candidate
|
| 1597 |
+
break
|
| 1598 |
+
else:
|
| 1599 |
+
raise SheetRefusal(
|
| 1600 |
+
"workbook_name_taken",
|
| 1601 |
+
"Too many workbooks are already named " + wanted + ".")
|
| 1602 |
book = new_workbook(name=name, created_by=created_by)
|
| 1603 |
_json_safe(book, "workbook")
|
| 1604 |
|
|
|
|
| 1610 |
return book
|
| 1611 |
|
| 1612 |
|
| 1613 |
+
def _carry_write_logs(book, prior):
|
| 1614 |
+
"""Only stored tabs may supply write history; client metadata never owns it."""
|
| 1615 |
+
held = {tab.get("id"): tab for tab in prior.get("sheets") or []}
|
| 1616 |
+
for tab in book.get("sheets") or []:
|
| 1617 |
+
source = held.get(tab.get("id")) or {}
|
| 1618 |
+
tab.pop("writes", None)
|
| 1619 |
+
tab["writesFrom"] = int(source.get("writesFrom") or 0)
|
| 1620 |
+
if "writes" in source:
|
| 1621 |
+
tab["writes"] = copy.deepcopy(source["writes"])
|
| 1622 |
+
return book
|
| 1623 |
+
|
| 1624 |
+
|
| 1625 |
+
def _append_write_log(tab, rev, addrs, *, actor="", actor_name=""):
|
| 1626 |
+
"""Record accepted addresses and the oldest revision with complete retained history."""
|
| 1627 |
+
if not addrs:
|
| 1628 |
+
return
|
| 1629 |
+
writes = [*(tab.get("writes") or []), {"rev": rev, "by": actor,
|
| 1630 |
+
"byName": actor_name or actor, "addrs": list(addrs)}]
|
| 1631 |
+
dropped = writes[:-WRITE_LOG_REVS]
|
| 1632 |
+
if dropped:
|
| 1633 |
+
tab["writesFrom"] = max(int(tab.get("writesFrom") or 0), int(dropped[-1]["rev"]))
|
| 1634 |
+
tab["writes"] = writes[-WRITE_LOG_REVS:]
|
| 1635 |
+
|
| 1636 |
+
|
| 1637 |
def get_workbook(st, workbook_id):
|
| 1638 |
"""One workbook, or a named refusal when the document is not there."""
|
| 1639 |
wid = _slug_id(workbook_id)
|
|
|
|
| 1641 |
if not doc or not doc.get("id"):
|
| 1642 |
raise SheetRefusal("workbook_missing",
|
| 1643 |
"That workbook no longer exists.")
|
| 1644 |
+
return _carry_write_logs(migrate_workbook(doc), doc)
|
| 1645 |
|
| 1646 |
|
| 1647 |
+
def put_workbook(st, workbook_id, raw, expect_rev=None, *, actor="", actor_name="", lossless=False):
|
| 1648 |
"""Replace a workbook's name and sheets, bumping `rev`.
|
| 1649 |
|
| 1650 |
+
⛔ `id`, `created_by`, `access` and `created_at` are PRIOR-WINS: a client may not reassign a workbook by
|
| 1651 |
sending a different id, and a client that never read the document cannot blank its provenance.
|
| 1652 |
|
| 1653 |
⚠ `expect_rev` makes the write CONDITIONAL, and when it is not passed explicitly the `rev`
|
|
|
|
| 1666 |
if expect_rev is False:
|
| 1667 |
expect_rev = None
|
| 1668 |
|
| 1669 |
+
#: W62 C1 / R19: app and AI saves share bounded defaults; import and refresh opt out.
|
| 1670 |
+
incoming = migrate_workbook(raw, lossless=lossless)
|
| 1671 |
sheets = incoming["sheets"]
|
| 1672 |
name = _text(raw.get("name"), _MAX_NAME)
|
| 1673 |
refreshed_at = raw.get("refreshed_at")
|
|
|
|
| 1678 |
if prior is None:
|
| 1679 |
raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
|
| 1680 |
if expect_rev is not None and int(prior.get("rev") or 0) != int(expect_rev):
|
| 1681 |
+
raise SheetStaleRefusal(
|
| 1682 |
"workbook_revision_stale",
|
| 1683 |
+
"That workbook changed somewhere else, so this save was not applied.",
|
| 1684 |
+
rev_current=int(prior.get("rev") or 0))
|
| 1685 |
book = migrate_workbook(prior)
|
| 1686 |
# Immutable ownership and server read projections cannot be reassigned by a PUT.
|
| 1687 |
# ``dataOrigins`` is SERVER-MAINTAINED (DPP 1.8) and therefore protected: a PUT body may
|
|
|
|
| 1690 |
# ``[]`` erases a genuine Amazon origin, and sending a well-formed tag plants one on an
|
| 1691 |
# unrelated workbook. The origin list is rebuilt from the PRIOR document plus the cleaned
|
| 1692 |
# cells by ``_merge_data_origins`` immediately below.
|
| 1693 |
+
protected = {"id", "created_by", "access", "created_at", "updated_at", "rev", "refreshed_at",
|
| 1694 |
"sheets", "name", "refreshedFor", "shares", "role", "permissions",
|
| 1695 |
"dataOrigins"}
|
| 1696 |
book.update({key: value for key, value in incoming.items() if key not in protected})
|
| 1697 |
book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook"
|
| 1698 |
+
book["sheets"] = copy.deepcopy(sheets)
|
| 1699 |
+
_carry_write_logs(book, prior)
|
| 1700 |
# DPP 1.8: merge from the PRIOR document and the cleaned cells. ``raw.dataOrigins`` is
|
| 1701 |
# ignored, so a client cannot erase or replace the server-maintained attribution list.
|
| 1702 |
_merge_data_origins(book, sheets=sheets)
|
| 1703 |
book["rev"] = int(prior.get("rev") or 0) + 1
|
| 1704 |
+
prior_tabs = {tab.get("id"): tab for tab in prior.get("sheets") or []}
|
| 1705 |
+
for tab in book["sheets"]:
|
| 1706 |
+
before = (prior_tabs.get(tab.get("id")) or {}).get("cells") or {}
|
| 1707 |
+
after = tab.get("cells") or {}
|
| 1708 |
+
changed = sorted(addr for addr in before.keys() | after.keys()
|
| 1709 |
+
if before.get(addr) != after.get(addr))
|
| 1710 |
+
_append_write_log(tab, book["rev"], changed, actor=actor, actor_name=actor_name)
|
| 1711 |
if refreshed_at is not None:
|
| 1712 |
book["refreshed_at"] = _text(refreshed_at, 120) or None
|
| 1713 |
book["updated_at"] = _now()
|
|
|
|
| 1789 |
return meta
|
| 1790 |
|
| 1791 |
|
| 1792 |
+
def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None, *, actor="", actor_name=""):
|
| 1793 |
"""Merge a RANGE of cells into ONE tab. Returns `{rev, tab, refused}` (contract C3).
|
| 1794 |
|
| 1795 |
⭐⭐ ONE BAD CELL DOES NOT THROW THE COMMIT AWAY. Every address is cleaned on its own and a
|
|
|
|
| 1855 |
if prior is None:
|
| 1856 |
raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
|
| 1857 |
current = int(prior.get("rev") or 0)
|
| 1858 |
+
book = _carry_write_logs(migrate_workbook(prior), prior)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1859 |
tabs = book.get("sheets") or []
|
| 1860 |
at = _tab_index(tabs, tab_ref)
|
| 1861 |
if at is None:
|
|
|
|
| 1863 |
"tab_missing",
|
| 1864 |
"Tab " + (str(tab_ref) or "(unnamed)") + " is not in this workbook.")
|
| 1865 |
tab = tabs[at]
|
| 1866 |
+
writes = tab.get("writes") or []
|
| 1867 |
+
touched = list(dict.fromkeys([*merged, *deleted]))
|
| 1868 |
+
conflicts = {}
|
| 1869 |
+
if touched and expect_rev is not None and int(expect_rev) < current:
|
| 1870 |
+
revision = int(expect_rev)
|
| 1871 |
+
if revision < int(tab.get("writesFrom") or 0):
|
| 1872 |
+
raise SheetStaleRefusal(
|
| 1873 |
+
"sheet_revision_stale",
|
| 1874 |
+
"This tab's retained change history was truncated after your revision. Reload the tab and try again.",
|
| 1875 |
+
rev_current=current)
|
| 1876 |
+
for write in writes:
|
| 1877 |
+
if int(write["rev"]) > revision and write["by"] != actor:
|
| 1878 |
+
for addr in sorted(set(touched).intersection(write["addrs"])):
|
| 1879 |
+
conflicts[addr] = {"addr": addr, "by": write["by"],
|
| 1880 |
+
"byName": write.get("byName") or write["by"]}
|
| 1881 |
cells = dict(tab.get("cells") or {})
|
| 1882 |
for key in deleted:
|
| 1883 |
cells.pop(key, None)
|
|
|
|
| 1894 |
book["sheets"] = tabs
|
| 1895 |
_merge_data_origins(book, sheets=tabs)
|
| 1896 |
book["rev"] = current + 1
|
| 1897 |
+
_append_write_log(tab, book["rev"], touched, actor=actor, actor_name=actor_name)
|
| 1898 |
book["updated_at"] = _now()
|
| 1899 |
_json_safe(book, "workbook")
|
| 1900 |
out["rev"] = book["rev"]
|
|
|
|
| 1902 |
out["cells"] = len(cells)
|
| 1903 |
out["schemaVersion"] = SHEET_SCHEMA_VERSION
|
| 1904 |
out["usedExtent"] = tab["usedExtent"]
|
| 1905 |
+
out["conflicts"] = list(conflicts.values())
|
| 1906 |
# Server-cleaned keys only. The route consumes this delta for calculation context.
|
| 1907 |
out["accepted"] = list(dict.fromkeys([*merged, *deleted]))
|
| 1908 |
saved.update(book)
|
|
|
|
| 2058 |
"id": book.get("id") or "",
|
| 2059 |
"name": book.get("name") or "",
|
| 2060 |
"created_by": book.get("created_by") or "",
|
| 2061 |
+
"has_access": "access" in book,
|
| 2062 |
"created_at": book.get("created_at") or "",
|
| 2063 |
"updated_at": book.get("updated_at") or "",
|
| 2064 |
"rev": int(book.get("rev") or 0),
|
|
|
|
| 2110 |
#: registry's words, so a second spelling cannot be introduced here by accident.
|
| 2111 |
|
| 2112 |
|
| 2113 |
+
def narrow_workbook(st, workbook_id):
|
| 2114 |
+
"""Persist explicit sharing's one way visibility decision on the latest document."""
|
| 2115 |
+
wid = _slug_id(workbook_id)
|
| 2116 |
+
|
| 2117 |
+
def _apply(state):
|
| 2118 |
+
if not state.get("id"):
|
| 2119 |
+
raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
|
| 2120 |
+
state.setdefault("access", "private")
|
| 2121 |
+
return state, state
|
| 2122 |
+
|
| 2123 |
+
# R28 / W62-T25: explicit sharing changes the fact used by the index's visibility filter.
|
| 2124 |
+
book = _persist(st, workbook_key(wid), _apply)
|
| 2125 |
+
_index_upsert(st, book)
|
| 2126 |
+
|
| 2127 |
+
|
| 2128 |
def workbook_shares(st, workbook_id):
|
| 2129 |
"""C5's `shares` list for ONE workbook, computed from `core.shares` and never stored.
|
| 2130 |
|
|
|
|
| 2143 |
rec = shares.grants("workbook", str(workbook_id), st=st)
|
| 2144 |
except Exception:
|
| 2145 |
return []
|
| 2146 |
+
return shares_from_record(rec)
|
| 2147 |
+
|
| 2148 |
|
| 2149 |
+
def shares_from_record(rec):
|
| 2150 |
+
"""C5's `shares` list from a grant record the caller already read. PURE.
|
| 2151 |
|
| 2152 |
+
W61-T17: `project_shares` reads the whole registry once and calls this per row, so the list
|
| 2153 |
+
a batched caller builds and the list `workbook_shares` builds are the same list by
|
| 2154 |
+
construction rather than by two copies of one comprehension.
|
| 2155 |
+
"""
|
| 2156 |
+
return [{"who": e["user"], "role": e["role"]} for e in ((rec or {}).get("entries") or [])]
|
| 2157 |
+
|
| 2158 |
+
|
| 2159 |
+
def workbook_role(st, book, viewer, *, is_admin=False, rec=None):
|
| 2160 |
+
"""Document ownership plus the existing grant reader, shared with the route's role helper.
|
| 2161 |
+
|
| 2162 |
+
⭐ W61-T17 — `book` NEED ONLY ANSWER `id` AND `created_by`, and that is what lets the index
|
| 2163 |
+
stop reading whole workbook documents to build a list. An index row already carries both
|
| 2164 |
+
(`_index_row` rebuilds them off the document on every save), so the caller may pass the ROW.
|
| 2165 |
+
⭐ `rec` is an already-read grant record, from `shares.grants_bulk`. Passing it skips a
|
| 2166 |
+
registry read; omitting it reads exactly as before. The role logic itself is
|
| 2167 |
+
`shares.role_from` either way, so the two paths cannot answer differently.
|
| 2168 |
+
"""
|
| 2169 |
+
from core import shares
|
| 2170 |
+
me = str(viewer or "").strip().lower()
|
| 2171 |
+
if me and str(book.get("created_by") or "").strip().lower() == me:
|
| 2172 |
+
return "owner"
|
| 2173 |
+
if rec is not None:
|
| 2174 |
+
return shares.role_from(rec, me, is_admin=is_admin)
|
| 2175 |
+
return shares.role_for("workbook", book.get("id"), me, is_admin=is_admin, st=st)
|
| 2176 |
+
|
| 2177 |
+
|
| 2178 |
+
def may_read_workbook(book, role):
|
| 2179 |
+
"""R28: a role admits a reader; unmarked or creatorless documents remain legacy-visible.
|
| 2180 |
+
|
| 2181 |
+
R28 / W62-T25: creation and explicit sharing stamp access. The index records its presence;
|
| 2182 |
+
workbook reads still use the document, and PUT must preserve marker absence too.
|
| 2183 |
+
Legacy visibility grants no ownership or editing role.
|
| 2184 |
+
"""
|
| 2185 |
+
return bool(role) or "access" not in book or not book.get("created_by")
|
| 2186 |
+
|
| 2187 |
+
|
| 2188 |
+
def project_shares(st, rows, viewer=None, *, is_admin=False, role_on_workbook=None,
|
| 2189 |
+
grant_records=None):
|
| 2190 |
"""Join `shares` onto a list of index rows. Returns NEW rows; the stored ones are untouched.
|
| 2191 |
|
| 2192 |
+
W62-T25: a named viewer filters rows using document facts and receives their role.
|
| 2193 |
+
The route supplies its existing role helper; core callers use workbook_role directly.
|
| 2194 |
+
`list_workbooks` itself stays a bare pointer list on purpose:
|
| 2195 |
`iter_workbooks` walks it to answer W49-T10's dependents question, and paying a grant-registry
|
| 2196 |
read per workbook there would put a permission lookup inside a scan that asks no permission
|
| 2197 |
question at all.
|
| 2198 |
+
|
| 2199 |
+
⭐⭐ W61-T17 — IT NO LONGER READS EVERY WORKBOOK DOCUMENT TO BUILD A LIST, and that was the
|
| 2200 |
+
whole cost of opening the Sheet home. Measured on staging's Royal Imports with 50 workbooks:
|
| 2201 |
+
12,485 ms and 269,946,170 characters read, to produce a 12 KB answer. Every one of those
|
| 2202 |
+
documents was pulled in full — all its tabs, all its cells — so that three fields could be
|
| 2203 |
+
consulted, and the list got slower with every cell anybody typed.
|
| 2204 |
+
|
| 2205 |
+
⭐ THE ROW ALREADY ANSWERS TWO OF THE THREE. `_index_row` rebuilds `id` and `created_by` off
|
| 2206 |
+
the document on every save, and `workbook_role` reads nothing else; so does the route's
|
| 2207 |
+
injected helper. The third is `may_read_workbook`'s legacy-visibility clause, `"access" not in
|
| 2208 |
+
book`, and it is consulted ONLY when the viewer has no role at all AND the row names a
|
| 2209 |
+
creator - the rows that are otherwise about to be REFUSED. W62-T25 records that third fact
|
| 2210 |
+
as `has_access`; only rows written before that fact existed still read their document.
|
| 2211 |
+
|
| 2212 |
+
R28 / W62-T25: admission uses the document facts rebuilt on save and explicit sharing.
|
| 2213 |
+
`_index_drop` removes a deleted document's row. Older rows still check document existence
|
| 2214 |
+
when taking the fallback; opening any workbook always reads its document for real.
|
| 2215 |
"""
|
| 2216 |
out = []
|
| 2217 |
+
bulk = grant_records
|
| 2218 |
+
if bulk is None and rows:
|
| 2219 |
+
try:
|
| 2220 |
+
from core import shares as _shares
|
| 2221 |
+
bulk = _shares.grants_bulk("workbook", st=st)
|
| 2222 |
+
except Exception:
|
| 2223 |
+
bulk = {}
|
| 2224 |
+
bulk = bulk or {}
|
| 2225 |
for row in rows or ():
|
| 2226 |
if not isinstance(row, dict) or not row.get("id"):
|
| 2227 |
continue
|
| 2228 |
joined = dict(row)
|
| 2229 |
+
# ⛔ NEVER None: an object with NO grants has no record in the bucket, and passing
|
| 2230 |
+
# None to `workbook_role` would read "caller supplied nothing" and send it back to
|
| 2231 |
+
# `shares.role_for` for a per-row read of the whole registry — which is one of the
|
| 2232 |
+
# two costs this change exists to remove. An empty record IS the answer.
|
| 2233 |
+
rec = bulk.get(str(row["id"])) or {"owner": None, "entries": []}
|
| 2234 |
+
if viewer:
|
| 2235 |
+
# The two fields the role question actually asks of a "book".
|
| 2236 |
+
light = {"id": row["id"], "created_by": row.get("created_by") or ""}
|
| 2237 |
+
role = (role_on_workbook(light) if role_on_workbook is not None
|
| 2238 |
+
else workbook_role(st, light, viewer, is_admin=is_admin, rec=rec))
|
| 2239 |
+
if not role and light["created_by"]:
|
| 2240 |
+
if "has_access" in row:
|
| 2241 |
+
if row["has_access"]:
|
| 2242 |
+
continue
|
| 2243 |
+
else:
|
| 2244 |
+
# R28 / W62-T25: old rows lack this fact, so retain their document check.
|
| 2245 |
+
# R28: this fallback set is expected to be empty after every workbook
|
| 2246 |
+
# has been saved once; guessing would change R28's legacy admission.
|
| 2247 |
+
book = _read(st, workbook_key(row["id"]))
|
| 2248 |
+
if not book or not book.get("id") or not may_read_workbook(book, role):
|
| 2249 |
+
continue
|
| 2250 |
+
joined["role"] = role or "view"
|
| 2251 |
+
joined["shares"] = shares_from_record(rec)
|
| 2252 |
out.append(joined)
|
| 2253 |
return out
|
| 2254 |
|
|
|
|
| 2704 |
return 0
|
| 2705 |
|
| 2706 |
|
| 2707 |
+
def _visible_rail(folders, placed, starred, ids):
|
| 2708 |
+
"""Prune hidden placements, stars and folders occupied only by hidden workbooks.
|
| 2709 |
+
|
| 2710 |
+
Keep visible folders' ancestors and intentionally empty folders. Otherwise creating an
|
| 2711 |
+
empty folder would immediately hide it from the person who needs to put a workbook in it.
|
| 2712 |
+
"""
|
| 2713 |
+
parents = {f["id"]: f.get("parent") for f in folders}
|
| 2714 |
+
|
| 2715 |
+
def ancestors(folder_ids):
|
| 2716 |
+
found = set()
|
| 2717 |
+
for fid in folder_ids:
|
| 2718 |
+
while fid in parents and fid not in found:
|
| 2719 |
+
found.add(fid)
|
| 2720 |
+
fid = parents[fid]
|
| 2721 |
+
return found
|
| 2722 |
+
|
| 2723 |
+
visible = {wid: fid for wid, fid in placed.items() if wid in ids}
|
| 2724 |
+
occupied = ancestors(placed.values())
|
| 2725 |
+
needed = ancestors(set(visible.values()) | (parents.keys() - occupied))
|
| 2726 |
+
hidden_only = occupied - needed
|
| 2727 |
+
return ([f for f in folders if f["id"] not in hidden_only], visible,
|
| 2728 |
+
_clean_starred(starred, ids, []))
|
| 2729 |
+
|
| 2730 |
+
|
| 2731 |
+
def read_index(st, viewer=None, *, is_admin=False, role_on_workbook=None):
|
| 2732 |
"""The whole rail document: `{workbooks, folders, itemFolders, starred, rev}`, plus
|
| 2733 |
`sharedWithMe` when a `viewer` is named.
|
| 2734 |
|
|
|
|
| 2742 |
so a `rev` here without the check there (or the check without this) is a guard nothing can
|
| 2743 |
reach ([[a-rev-guard-inert-on-the-events-channel]]). The pair lands together or not at all.
|
| 2744 |
|
| 2745 |
+
D-639 / W62-T25: `save_folders`, `merge_placements` and `_index_drop` move the revision
|
| 2746 |
+
because they write the folder, placement and star strata this guard protects.
|
| 2747 |
`_index_upsert` deliberately does NOT, because it is called by `create_workbook`,
|
| 2748 |
`put_workbook` AND `patch_cells` — bumping there would make every cell edit 409 a folder drag
|
| 2749 |
that touched nothing the guard is about.
|
|
|
|
| 2753 |
doc = _read(st, INDEX_KEY)
|
| 2754 |
rows = [dict(r) for r in (doc.get("workbooks") or [])
|
| 2755 |
if isinstance(r, dict) and r.get("id")]
|
| 2756 |
+
# W62 release repair: both the role join and Shared with me ask the SAME grant-registry
|
| 2757 |
+
# question. Read it once and hand the normalized records to both consumers; otherwise a
|
| 2758 |
+
# 50-row index pays one bulk read here, N more through an injected route helper, then one
|
| 2759 |
+
# final read through `shared_with`.
|
| 2760 |
+
grant_records = {}
|
| 2761 |
+
if rows:
|
| 2762 |
+
try:
|
| 2763 |
+
from core import shares as _shares
|
| 2764 |
+
grant_records = _shares.grants_bulk("workbook", st=st)
|
| 2765 |
+
except Exception:
|
| 2766 |
+
grant_records = {}
|
| 2767 |
+
rows = project_shares(st, rows, viewer, is_admin=is_admin,
|
| 2768 |
+
role_on_workbook=role_on_workbook,
|
| 2769 |
+
grant_records=grant_records)
|
| 2770 |
ids = {r["id"] for r in rows}
|
| 2771 |
placements = doc.get("itemFolders")
|
| 2772 |
try:
|
|
|
|
| 2775 |
flat = [dict(f, parent=None) for f in (doc.get("folders") or [])
|
| 2776 |
if isinstance(f, dict) and f.get("id")]
|
| 2777 |
folders, placed, _repairs = validate_folder_tree(flat, placements)
|
| 2778 |
+
starred = _clean_starred(doc.get("starred"), ids, [])
|
| 2779 |
+
if viewer:
|
| 2780 |
+
folders, placed, starred = _visible_rail(folders, placed, doc.get("starred"), ids)
|
| 2781 |
out = {
|
| 2782 |
+
"workbooks": rows,
|
| 2783 |
"folders": folders,
|
| 2784 |
"itemFolders": {k: v for k, v in placed.items() if k in ids},
|
| 2785 |
+
"starred": starred,
|
| 2786 |
"rev": _index_rev(doc),
|
| 2787 |
}
|
| 2788 |
if viewer:
|
| 2789 |
+
# C5's `sharedWithMe`, derived from the SAME normalized grant records as the role join.
|
| 2790 |
+
# Match `shares.shared_with`: exclude owned objects, admit personal/everyone grants, sort,
|
| 2791 |
+
# and finally intersect with the already-visible index ids. No second store read.
|
|
|
|
| 2792 |
try:
|
| 2793 |
import core.shares as shares
|
| 2794 |
+
me = str(viewer or "").strip().lower()
|
| 2795 |
+
granted = sorted(
|
| 2796 |
+
oid for oid, rec in grant_records.items()
|
| 2797 |
+
if str(rec.get("owner") or "").strip().lower() != me
|
| 2798 |
+
and shares.role_from(rec, me) is not None
|
| 2799 |
+
)
|
| 2800 |
except Exception:
|
| 2801 |
granted = []
|
| 2802 |
out["sharedWithMe"] = [w for w in (granted or []) if w in ids]
|
| 2803 |
return out
|
| 2804 |
|
| 2805 |
|
| 2806 |
+
def save_folders(st, folders, item_folders, starred=None, expect_rev=None, *,
|
| 2807 |
+
writable_ids=None, writable_folder_ids=None):
|
| 2808 |
"""Replace the folder, placement and star strata WHOLESALE, once the tree is proven legal.
|
| 2809 |
|
| 2810 |
Wholesale rather than per folder because the three are interdependent: a placement is only
|
| 2811 |
legal while its folder exists, so committing them separately would leave a window where a
|
| 2812 |
reader sees a workbook filed into a folder that is not there yet. One write, one state.
|
| 2813 |
|
| 2814 |
+
⛔ STRUCTURAL VALIDATION RUNS BEFORE THE UPDATE. A permission-scoped save also reconciles
|
| 2815 |
+
the submitted projection with hidden stored strata inside the callback, under the store lock;
|
| 2816 |
+
any refusal still abandons the callback atomically, so no half-touched rail can persist.
|
| 2817 |
|
| 2818 |
RETURNS `{workbooks, folders, itemFolders, starred, rev, repairs}`. RAISES
|
| 2819 |
`SheetFolderRefusal` on a cycle or an over deep chain, having written nothing.
|
|
|
|
| 2831 |
impossible.
|
| 2832 |
"""
|
| 2833 |
tree, placed, repairs = validate_folder_tree(folders, item_folders)
|
| 2834 |
+
writable = None if writable_ids is None else {str(wid) for wid in writable_ids}
|
| 2835 |
+
writable_folders = (None if writable_folder_ids is None
|
| 2836 |
+
else {str(fid) for fid in writable_folder_ids})
|
| 2837 |
out = {}
|
| 2838 |
|
| 2839 |
def _apply(state):
|
| 2840 |
+
tree_now = [dict(row) for row in tree]
|
| 2841 |
+
placed_now = dict(placed)
|
| 2842 |
+
repairs_now = list(repairs)
|
| 2843 |
+
starred_now = starred
|
| 2844 |
if expect_rev is not None and _index_rev(state) != int(expect_rev):
|
| 2845 |
raise SheetStaleRefusal(
|
| 2846 |
"index_revision_stale",
|
|
|
|
| 2850 |
rows = [r for r in (state.get("workbooks") or [])
|
| 2851 |
if isinstance(r, dict) and r.get("id")]
|
| 2852 |
ids = {r["id"] for r in rows}
|
| 2853 |
+
saved_tree, saved_placed, _saved_repairs = validate_folder_tree(
|
| 2854 |
+
state.get("folders"), state.get("itemFolders"))
|
| 2855 |
+
|
| 2856 |
+
if writable is not None:
|
| 2857 |
+
# An id absent from this caller's projection is indistinguishable from a missing one.
|
| 2858 |
+
# Refuse explicit guesses, then preserve every hidden placement/star against omission
|
| 2859 |
+
# from the caller's necessarily partial document.
|
| 2860 |
+
if any(item_id not in writable for item_id in placed_now):
|
| 2861 |
+
raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
|
| 2862 |
+
incoming_stars = (_clean_starred(starred_now, None, [])
|
| 2863 |
+
if starred_now is not None else [])
|
| 2864 |
+
if any(item_id not in writable for item_id in incoming_stars):
|
| 2865 |
+
raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
|
| 2866 |
+
|
| 2867 |
+
saved_by_id = {row["id"]: row for row in saved_tree}
|
| 2868 |
+
incoming_ids = {row["id"] for row in tree_now}
|
| 2869 |
+
if writable_folders is not None:
|
| 2870 |
+
hidden_folder_ids = set(saved_by_id) - writable_folders
|
| 2871 |
+
if incoming_ids & hidden_folder_ids:
|
| 2872 |
+
raise SheetRefusal("folder_missing", "That folder no longer exists.")
|
| 2873 |
+
|
| 2874 |
+
hidden_ids = ids - writable
|
| 2875 |
+
hidden_placed = {wid: fid for wid, fid in saved_placed.items()
|
| 2876 |
+
if wid in hidden_ids}
|
| 2877 |
+
parents = {row["id"]: row.get("parent") for row in saved_tree}
|
| 2878 |
+
required = set()
|
| 2879 |
+
for fid in hidden_placed.values():
|
| 2880 |
+
while fid in parents and fid not in required:
|
| 2881 |
+
required.add(fid)
|
| 2882 |
+
fid = parents[fid]
|
| 2883 |
+
# Missing hidden-only branches are restored from the stored tree. A shared folder the
|
| 2884 |
+
# caller edited keeps the submitted row; deleting one that still holds a hidden book
|
| 2885 |
+
# preserves the stored row only for that hidden stratum.
|
| 2886 |
+
combined_tree = list(tree_now)
|
| 2887 |
+
# Preserve the stored order of every restored hidden branch. Iterating the set
|
| 2888 |
+
# difference made an honest partial save reorder another user's hidden folders by
|
| 2889 |
+
# Python hash seed: the contents survived, but their rail order did not.
|
| 2890 |
+
combined_tree.extend(
|
| 2891 |
+
dict(row) for row in saved_tree
|
| 2892 |
+
if row["id"] in required and row["id"] not in incoming_ids)
|
| 2893 |
+
combined_placed = {**hidden_placed, **placed_now}
|
| 2894 |
+
tree_now, placed_now, combined_repairs = validate_folder_tree(
|
| 2895 |
+
combined_tree, combined_placed)
|
| 2896 |
+
repairs_now.extend(combined_repairs)
|
| 2897 |
+
|
| 2898 |
+
if starred_now is not None:
|
| 2899 |
+
hidden_stars = [wid for wid in _clean_starred(state.get("starred"), ids, [])
|
| 2900 |
+
if wid in hidden_ids]
|
| 2901 |
+
starred_now = hidden_stars + [wid for wid in incoming_stars if wid in writable]
|
| 2902 |
+
|
| 2903 |
kept = {}
|
| 2904 |
+
for item_id, fid in placed_now.items():
|
| 2905 |
if item_id in ids:
|
| 2906 |
kept[item_id] = fid
|
| 2907 |
else:
|
| 2908 |
+
repairs_now.append({"reason": "placement_missing_workbook", "item_id": item_id})
|
| 2909 |
+
state["folders"] = tree_now
|
| 2910 |
state["itemFolders"] = kept
|
| 2911 |
+
if starred_now is not None:
|
| 2912 |
+
state["starred"] = _clean_starred(starred_now, ids, repairs_now)
|
| 2913 |
# D-639: this write IS the one the guard protects, so it moves the revision.
|
| 2914 |
state["rev"] = _index_rev(state) + 1
|
| 2915 |
out["value"] = {
|
| 2916 |
"workbooks": [dict(r) for r in rows],
|
| 2917 |
+
"folders": [dict(f) for f in tree_now],
|
| 2918 |
"itemFolders": dict(kept),
|
| 2919 |
"starred": list(state.get("starred") or []),
|
| 2920 |
"rev": _index_rev(state),
|
| 2921 |
}
|
| 2922 |
+
out["repairs"] = repairs_now
|
| 2923 |
return state, None
|
| 2924 |
|
| 2925 |
_persist(st, INDEX_KEY, _apply, seed={"workbooks": []})
|
|
|
|
| 2929 |
# store, and reading one document from inside another document's update callback puts a second
|
| 2930 |
# store call under a lock this one already holds.
|
| 2931 |
result["workbooks"] = project_shares(st, result.get("workbooks"))
|
| 2932 |
+
result["repairs"] = list(out.get("repairs") or repairs)
|
| 2933 |
+
return result
|
| 2934 |
+
|
| 2935 |
+
|
| 2936 |
+
def merge_placements(st, placements, *, starred=None):
|
| 2937 |
+
"""D-639 / W62-T25: apply only named moves to the current rail under the store's lock.
|
| 2938 |
+
|
| 2939 |
+
None unfiles a workbook. Disjoint moves need no expected revision; every accepted merge
|
| 2940 |
+
advances it so a stale wholesale save still refuses. Stars change only if explicitly supplied.
|
| 2941 |
+
"""
|
| 2942 |
+
if not isinstance(placements, dict):
|
| 2943 |
+
raise SheetRefusal("placements_not_a_map", "Sheet placements must be a map of workbook ids.")
|
| 2944 |
+
|
| 2945 |
+
def _apply(state):
|
| 2946 |
+
rows = [r for r in (state.get("workbooks") or [])
|
| 2947 |
+
if isinstance(r, dict) and r.get("id")]
|
| 2948 |
+
ids = {r["id"] for r in rows}
|
| 2949 |
+
merged = dict(state.get("itemFolders") or {})
|
| 2950 |
+
for wid, fid in placements.items():
|
| 2951 |
+
if wid not in ids:
|
| 2952 |
+
raise SheetRefusal("placement_missing_workbook",
|
| 2953 |
+
'The workbook "' + str(wid) + '" no longer exists in the Sheet rail.')
|
| 2954 |
+
if fid is None:
|
| 2955 |
+
merged.pop(wid, None)
|
| 2956 |
+
else:
|
| 2957 |
+
merged[wid] = fid
|
| 2958 |
+
# D-639: validate the merged tree inside the same update, before changing any stratum.
|
| 2959 |
+
# D-639: use wholesale repair names, but refuse a malformed delta instead of losing it.
|
| 2960 |
+
_tree, placed, repairs = validate_folder_tree(state.get("folders"), merged)
|
| 2961 |
+
if repairs:
|
| 2962 |
+
repair = repairs[0]
|
| 2963 |
+
raise SheetFolderRefusal(
|
| 2964 |
+
repair["reason"],
|
| 2965 |
+
'The placement for workbook "' + str(repair.get("item_id") or "")
|
| 2966 |
+
+ '" cannot be saved. Use an existing folder id or null to unfile it.',
|
| 2967 |
+
folder_id=str(repair.get("folder_id") or ""), reason=repair["reason"])
|
| 2968 |
+
for wid, fid in placements.items():
|
| 2969 |
+
if fid is not None:
|
| 2970 |
+
merged[wid] = placed[wid]
|
| 2971 |
+
state["itemFolders"] = merged
|
| 2972 |
+
if starred is not None:
|
| 2973 |
+
state["starred"] = _clean_starred(starred, ids, repairs)
|
| 2974 |
+
state["rev"] = _index_rev(state) + 1
|
| 2975 |
+
return state, {"workbooks": rows, "folders": state.get("folders") or [],
|
| 2976 |
+
"itemFolders": merged, "starred": state.get("starred") or [],
|
| 2977 |
+
"rev": _index_rev(state), "repairs": repairs}
|
| 2978 |
+
|
| 2979 |
+
result = _persist(st, INDEX_KEY, _apply, seed={"workbooks": []})
|
| 2980 |
+
# D-639: grant reads happen after the store releases its update lock, as in save_folders.
|
| 2981 |
+
result["workbooks"] = project_shares(st, result["workbooks"])
|
| 2982 |
return result
|
platform/core/store_pg.py
CHANGED
|
@@ -347,6 +347,66 @@ def get_projection(name, drop=(), tenant_slug=None):
|
|
| 347 |
return _hf._project(data, frozenset(drops))
|
| 348 |
|
| 349 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
def exists(name, tenant_slug=None):
|
| 351 |
"""True/False — and unlike the HF backend this is NEVER "True on uncertainty".
|
| 352 |
|
|
@@ -610,6 +670,15 @@ class PgStore:
|
|
| 610 |
"""
|
| 611 |
return get_projection(name, drop=drop, tenant_slug=self.tenant_slug)
|
| 612 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 613 |
def _read_strict(self, name):
|
| 614 |
"""Interface parity with the HF Store. There, `get` is lenient (swallows a transient
|
| 615 |
failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
|
|
|
|
| 347 |
return _hf._project(data, frozenset(drops))
|
| 348 |
|
| 349 |
|
| 350 |
+
def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
|
| 351 |
+
"""Several projected documents in one round trip, keyed by their requested store key.
|
| 352 |
+
|
| 353 |
+
Private on purpose: the public module interface is the backend-parity surface, while this is
|
| 354 |
+
an optimization exposed only by a bound ``PgStore`` handle. Callers must retain a per-key
|
| 355 |
+
fallback for stores that do not offer it.
|
| 356 |
+
"""
|
| 357 |
+
import core.store as _hf # noqa: PLC0415 — projection shape lives with Store
|
| 358 |
+
|
| 359 |
+
keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
|
| 360 |
+
if not keys:
|
| 361 |
+
return {}
|
| 362 |
+
drops = tuple(str(d) for d in drop)
|
| 363 |
+
with _pool().connection() as con:
|
| 364 |
+
if timeout_s is not None:
|
| 365 |
+
timeout_ms = max(1, int(float(timeout_s) * 1000))
|
| 366 |
+
# Transaction-local: a pooled connection must not lend this caller's budget to the
|
| 367 |
+
# next request. PgStore connections are non-autocommit, so this setting covers the
|
| 368 |
+
# following projection and is discarded when the context commits or rolls back.
|
| 369 |
+
con.execute("SELECT set_config('statement_timeout', %s, true)",
|
| 370 |
+
(str(timeout_ms),))
|
| 371 |
+
rows = con.execute(
|
| 372 |
+
_q('SELECT kv.key, CASE '
|
| 373 |
+
"WHEN jsonb_typeof(kv.value) = 'object' THEN "
|
| 374 |
+
"COALESCE((SELECT jsonb_object_agg(e.entry_key, "
|
| 375 |
+
"CASE WHEN jsonb_typeof(e.entry_value) = 'object' "
|
| 376 |
+
'THEN e.entry_value - %s::text[] ELSE e.entry_value END) '
|
| 377 |
+
'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), '
|
| 378 |
+
"'{{}}'::jsonb) ELSE kv.value END FROM {tbl} AS kv "
|
| 379 |
+
'WHERE kv.key = ANY(%s::text[])', 'store_kv', tenant_slug),
|
| 380 |
+
(list(drops), keys)).fetchall()
|
| 381 |
+
found = {str(key): value for key, value in rows}
|
| 382 |
+
dropped = frozenset(drops)
|
| 383 |
+
return {
|
| 384 |
+
key: _hf._project(
|
| 385 |
+
dict(found[key]) if isinstance(found.get(key), dict)
|
| 386 |
+
else (found[key] if key in found else {}),
|
| 387 |
+
dropped)
|
| 388 |
+
for key in keys
|
| 389 |
+
}
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def _get_many(names, tenant_slug=None):
|
| 393 |
+
"""Several ordinary documents in one round trip, keyed by requested store key."""
|
| 394 |
+
keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
|
| 395 |
+
if not keys:
|
| 396 |
+
return {}
|
| 397 |
+
with _pool().connection() as con:
|
| 398 |
+
rows = con.execute(
|
| 399 |
+
_q('SELECT key, value FROM {tbl} WHERE key = ANY(%s::text[])',
|
| 400 |
+
'store_kv', tenant_slug),
|
| 401 |
+
(keys,)).fetchall()
|
| 402 |
+
found = {str(key): value for key, value in rows}
|
| 403 |
+
return {
|
| 404 |
+
key: (dict(found[key]) if isinstance(found.get(key), dict)
|
| 405 |
+
else (found[key] if key in found else {}))
|
| 406 |
+
for key in keys
|
| 407 |
+
}
|
| 408 |
+
|
| 409 |
+
|
| 410 |
def exists(name, tenant_slug=None):
|
| 411 |
"""True/False — and unlike the HF backend this is NEVER "True on uncertainty".
|
| 412 |
|
|
|
|
| 670 |
"""
|
| 671 |
return get_projection(name, drop=drop, tenant_slug=self.tenant_slug)
|
| 672 |
|
| 673 |
+
def get_projections(self, names, drop=(), timeout_s=None):
|
| 674 |
+
"""Bound batch projection; absent keys keep the single-read empty-document shape."""
|
| 675 |
+
return _get_projections(names, drop=drop, tenant_slug=self.tenant_slug,
|
| 676 |
+
timeout_s=timeout_s)
|
| 677 |
+
|
| 678 |
+
def get_many(self, names):
|
| 679 |
+
"""Bound multi-key read used by request-scoped read snapshots."""
|
| 680 |
+
return _get_many(names, tenant_slug=self.tenant_slug)
|
| 681 |
+
|
| 682 |
def _read_strict(self, name):
|
| 683 |
"""Interface parity with the HF Store. There, `get` is lenient (swallows a transient
|
| 684 |
failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
|
platform/core/table_store.py
CHANGED
|
@@ -1331,6 +1331,34 @@ class TableStore:
|
|
| 1331 |
})
|
| 1332 |
return plans
|
| 1333 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1334 |
def migrate_legacy_shared_views(self):
|
| 1335 |
"""Convert ``__shared__.views`` to personal source records plus explicit grants.
|
| 1336 |
|
|
|
|
| 1331 |
})
|
| 1332 |
return plans
|
| 1333 |
|
| 1334 |
+
def legacy_view_migration_needed(self, source=None):
|
| 1335 |
+
"""Whether a preloaded workspace document still carries either retired View shape.
|
| 1336 |
+
|
| 1337 |
+
This is the read-only half of :meth:`migrate_legacy_shared_views`. A caller that already
|
| 1338 |
+
paid for a rows-free workspace projection can avoid a second store read for the common
|
| 1339 |
+
clean document while still routing every legacy document through the full strict,
|
| 1340 |
+
write-and-read-back migration below. Malformed legacy data raises here exactly as it does
|
| 1341 |
+
there; it is never interpreted as clean.
|
| 1342 |
+
"""
|
| 1343 |
+
data = self._strict_document() if source is None else source
|
| 1344 |
+
if not isinstance(data, dict):
|
| 1345 |
+
raise LegacySharedViewMigrationError(
|
| 1346 |
+
'legacy shared-view migration found a non-object workspace document')
|
| 1347 |
+
has_blank_legacy, _blank = self._blank_legacy_view_bucket(data)
|
| 1348 |
+
return bool(has_blank_legacy or self._legacy_shared_plans(data)
|
| 1349 |
+
or self._personal_permission_plans(data))
|
| 1350 |
+
|
| 1351 |
+
def prepare_view_projection(self, source):
|
| 1352 |
+
"""Migrate a preloaded projection when needed; return whether its caller must refresh.
|
| 1353 |
+
|
| 1354 |
+
Keeping this decision beside the migration prevents read routes from acquiring a second,
|
| 1355 |
+
partial definition of which retired shapes are write-bearing.
|
| 1356 |
+
"""
|
| 1357 |
+
if not self.legacy_view_migration_needed(source):
|
| 1358 |
+
return False
|
| 1359 |
+
self.migrate_legacy_shared_views()
|
| 1360 |
+
return True
|
| 1361 |
+
|
| 1362 |
def migrate_legacy_shared_views(self):
|
| 1363 |
"""Convert ``__shared__.views`` to personal source records plus explicit grants.
|
| 1364 |
|
platform/core/user_tables.py
CHANGED
|
@@ -218,7 +218,20 @@ def is_connected(table_key, st=None):
|
|
| 218 |
key = str(table_key or '').strip()
|
| 219 |
if key in _CONNECTED:
|
| 220 |
return True
|
| 221 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 222 |
if not table:
|
| 223 |
return False
|
| 224 |
if table.get('connected') is True:
|
|
@@ -927,7 +940,7 @@ def _slug(label):
|
|
| 927 |
return (s or 'table')[:40]
|
| 928 |
|
| 929 |
|
| 930 |
-
def _ag_formula(raw):
|
| 931 |
"""`aios_grid._clean_formula`, reached the way this module reaches everything one layer up.
|
| 932 |
|
| 933 |
⚠ ONE call site for the function-local import, shared by BOTH field doors (`clean_fields` and
|
|
@@ -936,12 +949,11 @@ def _ag_formula(raw):
|
|
| 936 |
cost twice. A third and fourth copy of `import aios_grid as _agX` is how one of them ends up
|
| 937 |
calling a different validator.
|
| 938 |
|
| 939 |
-
⚠ `valid_keys`
|
| 940 |
-
|
| 941 |
-
worse than leaving refs to the read-time behaviour `_clean_formula` already documents.
|
| 942 |
"""
|
| 943 |
import aios_grid as _agf
|
| 944 |
-
return _agf._clean_formula(raw)
|
| 945 |
|
| 946 |
|
| 947 |
def _ag_geocode(raw):
|
|
@@ -2874,9 +2886,13 @@ def note_human_edit(table_key, values, row_id, st=None):
|
|
| 2874 |
return sorted(touched)
|
| 2875 |
|
| 2876 |
|
| 2877 |
-
def _clean_field(raw, previous=None, recurrence_validator=None):
|
| 2878 |
"""One field dict → the stored shape, or None. The single validator for create AND patch, so
|
| 2879 |
-
a column cannot be typed one way on the way in and another on the way back.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2880 |
prev = previous or {}
|
| 2881 |
raw = raw if isinstance(raw, dict) else {}
|
| 2882 |
label = ' '.join(str(raw.get('label') or prev.get('label') or '').split())[:80]
|
|
@@ -3149,21 +3165,21 @@ def _clean_field(raw, previous=None, recurrence_validator=None):
|
|
| 3149 |
# formula column with no expression is a column that can never show a value, so the pairing is
|
| 3150 |
# enforced BOTH WAYS and either half alone refuses the field.
|
| 3151 |
#
|
| 3152 |
-
#
|
| 3153 |
-
#
|
| 3154 |
-
#
|
| 3155 |
-
# `add_field`
|
| 3156 |
-
#
|
| 3157 |
-
#
|
| 3158 |
-
#
|
| 3159 |
-
#
|
| 3160 |
-
#
|
| 3161 |
-
#
|
| 3162 |
# ⚠ Reached through `_ag_formula` so BOTH doors call one validator through one import.
|
| 3163 |
formula_raw = (raw.get('formula') if 'formula' in raw
|
| 3164 |
else (prev.get('formula') if prev_type == ftype else None))
|
| 3165 |
if formula_raw is not None:
|
| 3166 |
-
fx = _ag_formula(formula_raw)
|
| 3167 |
if fx is None or ftype != 'formula':
|
| 3168 |
return None
|
| 3169 |
out['formula'] = fx
|
|
@@ -3780,7 +3796,8 @@ def add_field(table_key, raw, st=None, recurrence_validator=None):
|
|
| 3780 |
have = [f for f in (defn.get('fields') or [])]
|
| 3781 |
if len(have) >= MAX_FIELDS:
|
| 3782 |
return None
|
| 3783 |
-
field = _clean_field(raw, recurrence_validator=recurrence_validator
|
|
|
|
| 3784 |
if not field or any(f.get('key') == field['key'] for f in have):
|
| 3785 |
return None
|
| 3786 |
if not dynamic_link_filter_valid(field, have):
|
|
@@ -3988,7 +4005,13 @@ def patch_field(table_key, fkey, raw, st=None, recurrence_validator=None):
|
|
| 3988 |
if 'description' in merged and isinstance(fields[idx].get('automation'), dict) \
|
| 3989 |
and fields[idx]['automation'].get('preset') is True:
|
| 3990 |
merged.pop('description')
|
| 3991 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3992 |
if not field:
|
| 3993 |
return None
|
| 3994 |
if not dynamic_link_filter_valid(field, fields):
|
|
@@ -4473,10 +4496,13 @@ def may_edit_database(table_key, viewer, is_admin=False, st=None):
|
|
| 4473 |
#:
|
| 4474 |
#: ⚠ THE ALLOW-LIST IS THE SAFETY ARGUMENT, not the read-only-ness. Any other key passes through
|
| 4475 |
#: to the real runtime uncached, `fresh=True` always bypasses, and every WRITE verb reaches the
|
| 4476 |
-
#: runtime through `__getattr__` — so this can serve a stale value for
|
|
|
|
| 4477 |
#: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level
|
| 4478 |
#: before anybody touches it; `grants`/`_clean_entries` build new objects; `may_open` only READS
|
| 4479 |
-
#: `createdBy`).
|
|
|
|
|
|
|
| 4480 |
_LENDABLE = (STORE_KEY, 'object_shares')
|
| 4481 |
|
| 4482 |
|
|
@@ -4487,7 +4513,7 @@ class _Lent:
|
|
| 4487 |
self._runtime, self._docs = runtime, dict(docs or {})
|
| 4488 |
|
| 4489 |
def get(self, name, fresh=False):
|
| 4490 |
-
if fresh or name not in _LENDABLE:
|
| 4491 |
return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
|
| 4492 |
if name not in self._docs:
|
| 4493 |
self._docs[name] = self._runtime.get(name) # lazily, once, for this pass
|
|
@@ -4507,7 +4533,7 @@ class _Lent:
|
|
| 4507 |
A re-projection would cost a second copy to convert the first case and change nothing
|
| 4508 |
about the second.
|
| 4509 |
"""
|
| 4510 |
-
if name not in _LENDABLE:
|
| 4511 |
return self._runtime.get_projection(name, drop=drop)
|
| 4512 |
if name not in self._docs:
|
| 4513 |
self._docs[name] = self._runtime.get_projection(name, drop=drop)
|
|
@@ -4518,7 +4544,7 @@ class _Lent:
|
|
| 4518 |
|
| 4519 |
|
| 4520 |
def lend(st=None, **docs):
|
| 4521 |
-
"""
|
| 4522 |
|
| 4523 |
`lend(rt, user_tables=doc)` — keyword names are STORE KEYS. Call it once per read pass and
|
| 4524 |
hand the result to every predicate in that pass; see the note above for what makes it safe.
|
|
|
|
| 218 |
key = str(table_key or '').strip()
|
| 219 |
if key in _CONNECTED:
|
| 220 |
return True
|
| 221 |
+
# ⭐⭐ W62-T17 (D-739) — `all_defs`, NOT `get`, AND THE THREE KEYS BELOW ARE THE WHOLE ARGUMENT.
|
| 222 |
+
# This function asks `connected`, `source` and `recordMode`. Every one of them is a DEFINITION
|
| 223 |
+
# key; not one of them is a row. `get` reaches `all_tables`, which is the whole tenant
|
| 224 |
+
# document — MEASURED at 97,859,052 bytes and 2,274-2,816 ms PER CALL on staging's Royal
|
| 225 |
+
# Imports (`AutoWiki/waves/wave62/proto/T17-profile.md`), against 106,779 bytes for the
|
| 226 |
+
# rows-free projection that answers identically. D-739 called this "a platform-wide hazard,
|
| 227 |
+
# not a templates one: anything that calls it in a loop has this defect", and the loop callers
|
| 228 |
+
# were never the only victims — one call was already over the whole 2,000 ms module budget.
|
| 229 |
+
#
|
| 230 |
+
# ⚠ THE PROJECTION IS SAFE HERE FOR THE SAME REASON IT IS SAFE IN `/nav`: `all_defs` drops
|
| 231 |
+
# exactly `rows`, raises loudly (or falls back to the whole read) rather than answering empty,
|
| 232 |
+
# and a lent handle already serves whichever document the pass is holding. A caller that lends
|
| 233 |
+
# the WHOLE document gets MORE than asked for, which is the one direction this may be wrong in.
|
| 234 |
+
table = all_defs(st).get(key) or {}
|
| 235 |
if not table:
|
| 236 |
return False
|
| 237 |
if table.get('connected') is True:
|
|
|
|
| 940 |
return (s or 'table')[:40]
|
| 941 |
|
| 942 |
|
| 943 |
+
def _ag_formula(raw, valid_keys=None):
|
| 944 |
"""`aios_grid._clean_formula`, reached the way this module reaches everything one layer up.
|
| 945 |
|
| 946 |
⚠ ONE call site for the function-local import, shared by BOTH field doors (`clean_fields` and
|
|
|
|
| 949 |
cost twice. A third and fourth copy of `import aios_grid as _agX` is how one of them ends up
|
| 950 |
calling a different validator.
|
| 951 |
|
| 952 |
+
⚠ `valid_keys` is forwarded as given, `None` by default. D-540: only `add_field`/`patch_field`
|
| 953 |
+
supply it; see the note at `_clean_field`'s formula branch.
|
|
|
|
| 954 |
"""
|
| 955 |
import aios_grid as _agf
|
| 956 |
+
return _agf._clean_formula(raw, valid_keys)
|
| 957 |
|
| 958 |
|
| 959 |
def _ag_geocode(raw):
|
|
|
|
| 2886 |
return sorted(touched)
|
| 2887 |
|
| 2888 |
|
| 2889 |
+
def _clean_field(raw, previous=None, recurrence_validator=None, valid_keys=None):
|
| 2890 |
"""One field dict → the stored shape, or None. The single validator for create AND patch, so
|
| 2891 |
+
a column cannot be typed one way on the way in and another on the way back.
|
| 2892 |
+
|
| 2893 |
+
`valid_keys` (D-540): the table's other column keys, handed in ONLY by the two write doors
|
| 2894 |
+
that have table context (`add_field`, and `patch_field` when the caller sends a formula).
|
| 2895 |
+
Every other caller keeps `None`, which is the read-time/list-construction posture."""
|
| 2896 |
prev = previous or {}
|
| 2897 |
raw = raw if isinstance(raw, dict) else {}
|
| 2898 |
label = ' '.join(str(raw.get('label') or prev.get('label') or '').split())[:80]
|
|
|
|
| 3165 |
# formula column with no expression is a column that can never show a value, so the pairing is
|
| 3166 |
# enforced BOTH WAYS and either half alone refuses the field.
|
| 3167 |
#
|
| 3168 |
+
# ⛔ D-540 (2026-09-17): `valid_keys` IS PASSED WHEN THE CALLER HAS TABLE CONTEXT, and only
|
| 3169 |
+
# then. `grid_events` passes the workspace's own key set, and `formulaEngine.ts::
|
| 3170 |
+
# validateFormula` refuses a ref outside `knownKeys`, so the `ut_*` write door was the one
|
| 3171 |
+
# door that stored `{notacolumn}`. `add_field` and `patch_field` (when the caller sends a
|
| 3172 |
+
# formula) now hand in the table's other column keys and an unknown ref is refused there.
|
| 3173 |
+
# `clean_fields` (a list mid-construction, where half the referenced columns do not exist
|
| 3174 |
+
# yet), `clean_machine_fields` (a list with no table at all) and the link reciprocal keep the
|
| 3175 |
+
# `None` default: STRUCTURE only, and an unresolvable ref blanks the CELL client-side, which
|
| 3176 |
+
# is `_clean_formula`'s documented read-time behaviour. A patch that does not send a formula
|
| 3177 |
+
# keeps its stored one unjudged, so deleting a referenced column never vaporises a column.
|
| 3178 |
# ⚠ Reached through `_ag_formula` so BOTH doors call one validator through one import.
|
| 3179 |
formula_raw = (raw.get('formula') if 'formula' in raw
|
| 3180 |
else (prev.get('formula') if prev_type == ftype else None))
|
| 3181 |
if formula_raw is not None:
|
| 3182 |
+
fx = _ag_formula(formula_raw, valid_keys)
|
| 3183 |
if fx is None or ftype != 'formula':
|
| 3184 |
return None
|
| 3185 |
out['formula'] = fx
|
|
|
|
| 3796 |
have = [f for f in (defn.get('fields') or [])]
|
| 3797 |
if len(have) >= MAX_FIELDS:
|
| 3798 |
return None
|
| 3799 |
+
field = _clean_field(raw, recurrence_validator=recurrence_validator,
|
| 3800 |
+
valid_keys={str(f.get('key')) for f in have if f.get('key')})
|
| 3801 |
if not field or any(f.get('key') == field['key'] for f in have):
|
| 3802 |
return None
|
| 3803 |
if not dynamic_link_filter_valid(field, have):
|
|
|
|
| 4005 |
if 'description' in merged and isinstance(fields[idx].get('automation'), dict) \
|
| 4006 |
and fields[idx]['automation'].get('preset') is True:
|
| 4007 |
merged.pop('description')
|
| 4008 |
+
# D-540: a patch that SENDS a formula is judged against the table's other columns; one that
|
| 4009 |
+
# does not keeps its stored formula unjudged (a later-deleted ref blanks cells, not the column).
|
| 4010 |
+
field = _clean_field(
|
| 4011 |
+
merged, fields[idx], recurrence_validator=recurrence_validator,
|
| 4012 |
+
valid_keys=({str(f.get('key')) for f in fields
|
| 4013 |
+
if f.get('key') and str(f.get('key')) != str(fkey)}
|
| 4014 |
+
if isinstance(raw, dict) and 'formula' in raw else None))
|
| 4015 |
if not field:
|
| 4016 |
return None
|
| 4017 |
if not dynamic_link_filter_valid(field, fields):
|
|
|
|
| 4496 |
#:
|
| 4497 |
#: ⚠ THE ALLOW-LIST IS THE SAFETY ARGUMENT, not the read-only-ness. Any other key passes through
|
| 4498 |
#: to the real runtime uncached, `fresh=True` always bypasses, and every WRITE verb reaches the
|
| 4499 |
+
#: runtime through `__getattr__` — so this can serve a stale value for the two named keys and
|
| 4500 |
+
#: dynamically named `<key>__shared` overlay buckets, for
|
| 4501 |
#: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level
|
| 4502 |
#: before anybody touches it; `grants`/`_clean_entries` build new objects; `may_open` only READS
|
| 4503 |
+
#: `createdBy`). Shared overlays are also read-only on this path: `_shared_value_keys` only
|
| 4504 |
+
#: reads their field keys, never mutates the document. Their first read is lent for one pass;
|
| 4505 |
+
#: `fresh=True` and write verbs still reach the runtime, never the snapshot.
|
| 4506 |
_LENDABLE = (STORE_KEY, 'object_shares')
|
| 4507 |
|
| 4508 |
|
|
|
|
| 4513 |
self._runtime, self._docs = runtime, dict(docs or {})
|
| 4514 |
|
| 4515 |
def get(self, name, fresh=False):
|
| 4516 |
+
if fresh or (name not in _LENDABLE and not str(name).endswith('__shared')):
|
| 4517 |
return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
|
| 4518 |
if name not in self._docs:
|
| 4519 |
self._docs[name] = self._runtime.get(name) # lazily, once, for this pass
|
|
|
|
| 4533 |
A re-projection would cost a second copy to convert the first case and change nothing
|
| 4534 |
about the second.
|
| 4535 |
"""
|
| 4536 |
+
if name not in _LENDABLE and not str(name).endswith('__shared'):
|
| 4537 |
return self._runtime.get_projection(name, drop=drop)
|
| 4538 |
if name not in self._docs:
|
| 4539 |
self._docs[name] = self._runtime.get_projection(name, drop=drop)
|
|
|
|
| 4544 |
|
| 4545 |
|
| 4546 |
def lend(st=None, **docs):
|
| 4547 |
+
"""Lend `_LENDABLE` keys and `__shared` suffix buckets; defer other keys to `st`.
|
| 4548 |
|
| 4549 |
`lend(rt, user_tables=doc)` — keyword names are STORE KEYS. Call it once per read pass and
|
| 4550 |
hand the result to every predicate in that pass; see the note above for what makes it safe.
|
platform/core/windows.py
ADDED
|
@@ -0,0 +1,492 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/windows.py — the DATE WINDOW vocabulary (CG-7, owner item 2, 2026-07-26).
|
| 2 |
+
|
| 3 |
+
A filter condition on a measure reads:
|
| 4 |
+
|
| 5 |
+
Where [Sales] [in the last 90 days] [>] [5,000]
|
| 6 |
+
|
| 7 |
+
The middle bracket is this module. It turns a window SPEC — a small JSON object a saved view can
|
| 8 |
+
persist — into a concrete `(date_from, date_to)` pair of ISO dates.
|
| 9 |
+
|
| 10 |
+
WHY IT IS ITS OWN MODULE, AND WHY IT TAKES `today` AS AN ARGUMENT
|
| 11 |
+
-----------------------------------------------------------------------------------------------
|
| 12 |
+
Two engines must agree on what "this quarter" means: this one (which compiles the SQL) and
|
| 13 |
+
`customer-grid/windows.ts` (which renders the label the user reads and the bounds the client
|
| 14 |
+
engine would use). If they disagree by one day, the grid shows a number the label denies, and
|
| 15 |
+
nothing errors. `aios-web/verify_windows.py` holds them in lock-step over a fixed set of probe
|
| 16 |
+
dates, exactly as verify_filter_engine.py does for the filter tree.
|
| 17 |
+
|
| 18 |
+
`today` is a PARAMETER, never `date.today()` read inside. Three reasons, all learned:
|
| 19 |
+
- a gate cannot compare two engines on "now" — the two runs are milliseconds apart and land on
|
| 20 |
+
different sides of midnight roughly once every few thousand runs, which is the worst kind of
|
| 21 |
+
flake because it looks like a real divergence
|
| 22 |
+
- a window resolved during a request must not shift between the count query and the row query
|
| 23 |
+
- the tenant's day boundary is the tenant's, not the server's
|
| 24 |
+
|
| 25 |
+
DESIGN NOTES THAT ARE EASY TO GET WRONG
|
| 26 |
+
-----------------------------------------------------------------------------------------------
|
| 27 |
+
- Bounds are INCLUSIVE at both ends and are DATES, not timestamps. `store_query` casts both
|
| 28 |
+
sides to TIMESTAMP and appends 23:59:59 to the upper bound, so a bare date is correct there;
|
| 29 |
+
the client compares the 10-char ISO prefix, so a bare date is correct there too.
|
| 30 |
+
- "Last N days" INCLUDES today. Airtable's "the past week" means the last 7 days up to and
|
| 31 |
+
including now, not the 7 days before yesterday. Off-by-one here silently drops today's
|
| 32 |
+
orders from every "recent" filter — invisible until someone asks why a sale they just
|
| 33 |
+
entered is missing.
|
| 34 |
+
- Week starts MONDAY (ISO 8601). Stated rather than defaulted: Python's weekday() is
|
| 35 |
+
Monday=0 and JavaScript's getDay() is Sunday=0, so the two engines disagree unless one of
|
| 36 |
+
them is explicitly corrected. That correction is the single most likely divergence in this
|
| 37 |
+
file, and the gate probes a Sunday and a Monday for exactly that reason.
|
| 38 |
+
- A `custom` window with only one bound is legal and means open-ended on the other side.
|
| 39 |
+
- An UNRESOLVABLE window returns None rather than a default. A window that quietly becomes
|
| 40 |
+
"all time" would widen a filter while still reporting an authoritative count.
|
| 41 |
+
The historical ``harness.windows`` import remains a compatibility re-export. The vocabulary
|
| 42 |
+
lives in core because workbook document validation also consumes it and core must never import
|
| 43 |
+
upward from harness.
|
| 44 |
+
"""
|
| 45 |
+
import datetime as _dt
|
| 46 |
+
import re
|
| 47 |
+
import math
|
| 48 |
+
|
| 49 |
+
#: Every window kind the UI may offer. Mirrors WINDOW_KINDS in customer-grid/windows.ts;
|
| 50 |
+
#: verify_windows.py asserts the two lists are identical, because a kind the client can emit and
|
| 51 |
+
#: the server cannot resolve is a filter that silently stops narrowing.
|
| 52 |
+
WINDOW_KINDS = (
|
| 53 |
+
"all_time",
|
| 54 |
+
"today",
|
| 55 |
+
"yesterday",
|
| 56 |
+
"this_week",
|
| 57 |
+
"last_week",
|
| 58 |
+
"this_month",
|
| 59 |
+
"last_month",
|
| 60 |
+
"this_quarter",
|
| 61 |
+
"last_quarter",
|
| 62 |
+
"this_year",
|
| 63 |
+
"last_year",
|
| 64 |
+
"ytd",
|
| 65 |
+
"ytd_last_year",
|
| 66 |
+
"ltm",
|
| 67 |
+
"past_week",
|
| 68 |
+
"past_month",
|
| 69 |
+
"past_year",
|
| 70 |
+
"last_n_days",
|
| 71 |
+
"next_n_days",
|
| 72 |
+
"custom",
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
#: Kinds that carry an integer `n`.
|
| 76 |
+
N_KINDS = frozenset({"last_n_days", "next_n_days"})
|
| 77 |
+
|
| 78 |
+
#: How each kind reads in a sentence. The UI renders these; they are here so the label and the
|
| 79 |
+
#: arithmetic cannot drift apart in the one place a user would never think to check.
|
| 80 |
+
WINDOW_LABELS = {
|
| 81 |
+
"all_time": "all time",
|
| 82 |
+
"today": "today",
|
| 83 |
+
"yesterday": "yesterday",
|
| 84 |
+
"this_week": "this week",
|
| 85 |
+
"last_week": "last week",
|
| 86 |
+
"this_month": "this month",
|
| 87 |
+
"last_month": "last month",
|
| 88 |
+
"this_quarter": "this quarter",
|
| 89 |
+
"last_quarter": "last quarter",
|
| 90 |
+
"this_year": "this year",
|
| 91 |
+
"last_year": "last year",
|
| 92 |
+
"ytd": "year to date",
|
| 93 |
+
# "last year to date" (LYTD), not "year to date, last year": the longer phrasing CLIPPED to
|
| 94 |
+
# "year to date, last ye" in the 146px window select — and this vocabulary is CLOSED, so
|
| 95 |
+
# the rule for it is that it stays readable rather than that it fits. Seen in a live
|
| 96 |
+
# screenshot of the owner's own comparison; every assertion in that run was green.
|
| 97 |
+
"ytd_last_year": "last year to date",
|
| 98 |
+
"ltm": "the last 12 months",
|
| 99 |
+
"past_week": "the past week",
|
| 100 |
+
"past_month": "the past month",
|
| 101 |
+
"past_year": "the past year",
|
| 102 |
+
"last_n_days": "the last {n} days",
|
| 103 |
+
"next_n_days": "the next {n} days",
|
| 104 |
+
"custom": "a custom range",
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
MAX_N = 3650 # ten years; a bound, not a business rule
|
| 108 |
+
|
| 109 |
+
# --- the DATE-VALUE anchors (owner item 3, 2026-07-26) ----------------------------------------
|
| 110 |
+
# The second half of a date CONDITION, as distinct from a measure's window:
|
| 111 |
+
#
|
| 112 |
+
# Where [Last order] [is before] [one month ago]
|
| 113 |
+
# ^ op ^ THIS
|
| 114 |
+
#
|
| 115 |
+
# A window answers "over what period do I sum"; an anchor answers "which single date am I
|
| 116 |
+
# comparing against". Both live in this module for one reason: they are the same two-engine
|
| 117 |
+
# contract, resolved from the same `today`, and a gate that holds one in lock-step and not the
|
| 118 |
+
# other would leave half the sentence free to drift.
|
| 119 |
+
|
| 120 |
+
#: Anchor modes, in the order the picker offers them. `exact` is the historical behaviour — a
|
| 121 |
+
#: rule with NO mode is an `exact` rule whose value is an ISO date, which is what every view
|
| 122 |
+
#: saved before this change carries.
|
| 123 |
+
ANCHOR_MODES = (
|
| 124 |
+
"today",
|
| 125 |
+
"yesterday",
|
| 126 |
+
"one_week_ago",
|
| 127 |
+
"one_month_ago",
|
| 128 |
+
"n_days_ago",
|
| 129 |
+
"exact",
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
#: Modes that carry NO value. ⚠ This set is load-bearing far outside this module: a rule whose
|
| 133 |
+
#: value is blank normally reads as INACTIVE, and an inactive rule is IGNORED — which WIDENS the
|
| 134 |
+
#: result under a count nobody would doubt. Every activeness check (`isRuleActive` in TS,
|
| 135 |
+
#: `filter_sql.is_rule_active`, and the validator's value handling) has to know these four are
|
| 136 |
+
#: active with an empty value.
|
| 137 |
+
ANCHOR_VALUE_FREE = frozenset({"today", "yesterday", "one_week_ago", "one_month_ago"})
|
| 138 |
+
|
| 139 |
+
ANCHOR_LABELS = {
|
| 140 |
+
"today": "today",
|
| 141 |
+
"yesterday": "yesterday",
|
| 142 |
+
"one_week_ago": "one week ago",
|
| 143 |
+
"one_month_ago": "one month ago",
|
| 144 |
+
"n_days_ago": "{n} days ago",
|
| 145 |
+
"exact": "an exact date",
|
| 146 |
+
}
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def _iso(d):
|
| 150 |
+
return d.isoformat()
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def _month_start(d):
|
| 154 |
+
return d.replace(day=1)
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _month_end(d):
|
| 158 |
+
return _next_month(d.replace(day=1)) - _dt.timedelta(days=1)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def _next_month(d):
|
| 162 |
+
return (d.replace(day=28) + _dt.timedelta(days=4)).replace(day=1)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _quarter_start(d):
|
| 166 |
+
return _dt.date(d.year, 3 * ((d.month - 1) // 3) + 1, 1)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _days_in_month(y, m):
|
| 170 |
+
return (_dt.date(y + (m == 12), 1 if m == 12 else m + 1, 1) - _dt.timedelta(days=1)).day
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _shift_months(d, n):
|
| 174 |
+
"""`d` moved `n` months, CLAMPING the day to the target month's length.
|
| 175 |
+
|
| 176 |
+
Jan 31 back one month is Dec 31, but Mar 31 back one month is Feb 28 (or 29) — there is no
|
| 177 |
+
Feb 31 to land on. Both engines must clamp identically or "the past month" differs by up to
|
| 178 |
+
three days for a third of the calendar; the gate probes a 31st and a leap day for exactly it.
|
| 179 |
+
"""
|
| 180 |
+
total = (d.year * 12 + (d.month - 1)) + n
|
| 181 |
+
y, m = divmod(total, 12)
|
| 182 |
+
m += 1
|
| 183 |
+
return _dt.date(y, m, min(d.day, _days_in_month(y, m)))
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _parse_date(v):
|
| 187 |
+
"""Accept 'YYYY-MM-DD' (and tolerate a longer ISO timestamp by taking its date part)."""
|
| 188 |
+
if isinstance(v, _dt.date):
|
| 189 |
+
return v
|
| 190 |
+
s = str(v or "").strip()[:10]
|
| 191 |
+
if not s:
|
| 192 |
+
return None
|
| 193 |
+
try:
|
| 194 |
+
return _dt.date.fromisoformat(s)
|
| 195 |
+
except ValueError:
|
| 196 |
+
return None
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
# Sheet Ending metadata belongs to the binding period, never to a window spec.
|
| 200 |
+
ENDING_KINDS = ("today", "yesterday", "last_week", "last_month", "last_quarter",
|
| 201 |
+
"last_year", "fixed", "cell", "n_before_today")
|
| 202 |
+
ENDING_LABELS = dict(zip(ENDING_KINDS, ("Today (live)", "Yesterday", "End of last week",
|
| 203 |
+
"End of last month", "End of last quarter", "End of last year", "A fixed date", "From a cell",
|
| 204 |
+
"N periods before today")))
|
| 205 |
+
TO_DATE_KINDS = ("ytd", "ytd_last_year", "this_month", "this_quarter")
|
| 206 |
+
ENDING_STEPS = ("day", "week", "month", "quarter", "year")
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def ending_step_default(kind):
|
| 210 |
+
return "quarter" if kind == "this_quarter" else "month"
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def ending_date(value):
|
| 214 |
+
"""A real ISO day only. No truncation, calendar rollover, or blank fallback."""
|
| 215 |
+
if not isinstance(value, str) or not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value.strip()):
|
| 216 |
+
return None
|
| 217 |
+
d = _parse_date(value)
|
| 218 |
+
return _iso(d) if d else None
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def resolve_ending(mode, value, today, step="month", offset=0, n=0, unit="month"):
|
| 222 |
+
"""Inclusive ending date; cell values are supplied by the caller's trusted read."""
|
| 223 |
+
d = _parse_date(today)
|
| 224 |
+
if d is None:
|
| 225 |
+
raise ValueError("Ending needs a valid today date")
|
| 226 |
+
if mode not in ENDING_KINDS or step not in ENDING_STEPS:
|
| 227 |
+
return None
|
| 228 |
+
if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
|
| 229 |
+
return None
|
| 230 |
+
if mode in ("fixed", "cell"):
|
| 231 |
+
parsed = ending_date(value)
|
| 232 |
+
if parsed is None:
|
| 233 |
+
return None
|
| 234 |
+
end = _parse_date(parsed)
|
| 235 |
+
elif mode == "n_before_today":
|
| 236 |
+
if (isinstance(n, bool) or not isinstance(n, int) or not 1 <= n <= MAX_N
|
| 237 |
+
or unit not in ENDING_STEPS):
|
| 238 |
+
return None
|
| 239 |
+
try:
|
| 240 |
+
if unit in ("day", "week"):
|
| 241 |
+
end = d - _dt.timedelta(days=n * (7 if unit == "week" else 1))
|
| 242 |
+
else:
|
| 243 |
+
end = _shift_months(d, -n * {"month": 1, "quarter": 3, "year": 12}[unit])
|
| 244 |
+
except (ValueError, OverflowError):
|
| 245 |
+
return None
|
| 246 |
+
elif mode == "today":
|
| 247 |
+
end = d
|
| 248 |
+
elif mode == "yesterday":
|
| 249 |
+
end = d - _dt.timedelta(days=1)
|
| 250 |
+
elif mode == "last_week":
|
| 251 |
+
end = d - _dt.timedelta(days=d.weekday() + 1)
|
| 252 |
+
elif mode == "last_month":
|
| 253 |
+
end = _month_start(d) - _dt.timedelta(days=1)
|
| 254 |
+
elif mode == "last_quarter":
|
| 255 |
+
end = _quarter_start(d) - _dt.timedelta(days=1)
|
| 256 |
+
else:
|
| 257 |
+
end = _dt.date(d.year - 1, 12, 31)
|
| 258 |
+
try:
|
| 259 |
+
if step in ("day", "week"):
|
| 260 |
+
end += _dt.timedelta(days=offset * (7 if step == "week" else 1))
|
| 261 |
+
else:
|
| 262 |
+
end = _shift_months(end, offset * {"month": 1, "quarter": 3, "year": 12}[step])
|
| 263 |
+
return _iso(end)
|
| 264 |
+
except (ValueError, OverflowError):
|
| 265 |
+
return None
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def ending_cell_date(value, date_system="1900"):
|
| 269 |
+
"""Sheet dates may be ISO days or Excel serials in the workbook's declared epoch."""
|
| 270 |
+
if not isinstance(value, (int, float)) or isinstance(value, bool):
|
| 271 |
+
return ending_date(value)
|
| 272 |
+
if not math.isfinite(value) or value < (0 if date_system == "1904" else 1):
|
| 273 |
+
return None
|
| 274 |
+
days = math.floor(value)
|
| 275 |
+
if date_system != "1904" and days == 60:
|
| 276 |
+
return None # Excel's fictional February 29, 1900 is not a calendar date.
|
| 277 |
+
epoch = (_dt.date(1904, 1, 1) if date_system == "1904" else
|
| 278 |
+
_dt.date(1899, 12, 31) if days < 60 else _dt.date(1899, 12, 30))
|
| 279 |
+
try:
|
| 280 |
+
return _iso(epoch + _dt.timedelta(days=days))
|
| 281 |
+
except (ValueError, OverflowError):
|
| 282 |
+
return None
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def ending_bounds(kind, mode, value, today, step="month", offset=0, n=0, unit="month"):
|
| 286 |
+
"""Keep the kind's own start. An ending before it refuses instead of reversing bounds."""
|
| 287 |
+
if kind not in TO_DATE_KINDS:
|
| 288 |
+
return None
|
| 289 |
+
end = resolve_ending(mode, value, today, step, offset, n, unit)
|
| 290 |
+
if end is None:
|
| 291 |
+
return None
|
| 292 |
+
start = resolve({"kind": kind}, today)[0]
|
| 293 |
+
return (start, end) if start <= end else None
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def normalize(spec):
|
| 297 |
+
"""Coerce an untrusted window spec to `{kind, n?, from?, to?}` or None.
|
| 298 |
+
|
| 299 |
+
Fail-closed on the KIND (an unknown kind is not a window), tolerant on the rest — the same
|
| 300 |
+
split `clean_filter_tree` uses, so one malformed window cannot cost a user their saved view.
|
| 301 |
+
"""
|
| 302 |
+
if not isinstance(spec, dict):
|
| 303 |
+
return None
|
| 304 |
+
kind = spec.get("kind")
|
| 305 |
+
if kind not in WINDOW_KINDS:
|
| 306 |
+
return None
|
| 307 |
+
out = {"kind": kind}
|
| 308 |
+
if kind in N_KINDS:
|
| 309 |
+
try:
|
| 310 |
+
n = int(spec.get("n"))
|
| 311 |
+
except (TypeError, ValueError):
|
| 312 |
+
return None
|
| 313 |
+
if n < 1 or n > MAX_N:
|
| 314 |
+
return None
|
| 315 |
+
out["n"] = n
|
| 316 |
+
if kind == "custom":
|
| 317 |
+
f, t = _parse_date(spec.get("from")), _parse_date(spec.get("to"))
|
| 318 |
+
if f is None and t is None:
|
| 319 |
+
return None # a custom range with no bounds is not a window
|
| 320 |
+
if f is not None and t is not None and f > t:
|
| 321 |
+
f, t = t, f # the builder cannot enforce order; the engine can
|
| 322 |
+
if f is not None:
|
| 323 |
+
out["from"] = _iso(f)
|
| 324 |
+
if t is not None:
|
| 325 |
+
out["to"] = _iso(t)
|
| 326 |
+
return out
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def resolve(spec, today):
|
| 330 |
+
"""Window spec + the tenant's today -> `(date_from, date_to)`, both inclusive ISO dates.
|
| 331 |
+
|
| 332 |
+
Either side may be None, meaning open-ended. Returns None when the spec is not a window at
|
| 333 |
+
all — callers MUST treat that as "this condition cannot be evaluated" and refuse, never as
|
| 334 |
+
"no window", which would silently widen the result to all time.
|
| 335 |
+
"""
|
| 336 |
+
spec = normalize(spec)
|
| 337 |
+
if spec is None:
|
| 338 |
+
return None
|
| 339 |
+
kind = spec["kind"]
|
| 340 |
+
d = _parse_date(today)
|
| 341 |
+
if d is None:
|
| 342 |
+
raise ValueError("resolve() needs an explicit `today`; see this module's docstring")
|
| 343 |
+
|
| 344 |
+
if kind == "all_time":
|
| 345 |
+
return (None, None)
|
| 346 |
+
if kind == "today":
|
| 347 |
+
return (_iso(d), _iso(d))
|
| 348 |
+
if kind == "yesterday":
|
| 349 |
+
y = d - _dt.timedelta(days=1)
|
| 350 |
+
return (_iso(y), _iso(y))
|
| 351 |
+
|
| 352 |
+
# ISO 8601: the week starts MONDAY. Python's weekday() is already Monday=0; the TS mirror
|
| 353 |
+
# has to correct getDay(), which is Sunday=0. The gate probes both a Sunday and a Monday.
|
| 354 |
+
if kind == "this_week":
|
| 355 |
+
start = d - _dt.timedelta(days=d.weekday())
|
| 356 |
+
return (_iso(start), _iso(start + _dt.timedelta(days=6)))
|
| 357 |
+
if kind == "last_week":
|
| 358 |
+
start = d - _dt.timedelta(days=d.weekday() + 7)
|
| 359 |
+
return (_iso(start), _iso(start + _dt.timedelta(days=6)))
|
| 360 |
+
|
| 361 |
+
if kind == "this_month":
|
| 362 |
+
return (_iso(_month_start(d)), _iso(_month_end(d)))
|
| 363 |
+
if kind == "last_month":
|
| 364 |
+
prev = _month_start(d) - _dt.timedelta(days=1)
|
| 365 |
+
return (_iso(_month_start(prev)), _iso(prev))
|
| 366 |
+
|
| 367 |
+
if kind == "this_quarter":
|
| 368 |
+
qs = _quarter_start(d)
|
| 369 |
+
qe = _month_end(_dt.date(qs.year, qs.month + 2, 1))
|
| 370 |
+
return (_iso(qs), _iso(qe))
|
| 371 |
+
if kind == "last_quarter":
|
| 372 |
+
prev_end = _quarter_start(d) - _dt.timedelta(days=1)
|
| 373 |
+
qs = _quarter_start(prev_end)
|
| 374 |
+
return (_iso(qs), _iso(prev_end))
|
| 375 |
+
|
| 376 |
+
if kind == "this_year":
|
| 377 |
+
return (_iso(_dt.date(d.year, 1, 1)), _iso(_dt.date(d.year, 12, 31)))
|
| 378 |
+
if kind == "last_year":
|
| 379 |
+
return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_dt.date(d.year - 1, 12, 31)))
|
| 380 |
+
if kind == "ytd":
|
| 381 |
+
# year-to-date ENDS TODAY, unlike this_year which runs to Dec 31. The distinction is the
|
| 382 |
+
# whole reason both exist: comparing "this year" against last year double-counts the
|
| 383 |
+
# months that have not happened yet.
|
| 384 |
+
return (_iso(_dt.date(d.year, 1, 1)), _iso(d))
|
| 385 |
+
if kind == "ytd_last_year":
|
| 386 |
+
# SAME PERIOD last year — Jan 1 LY through today's month/day LY. This is the honest
|
| 387 |
+
# partner of `ytd`, and it is not a nicety: it mirrors `core.periods.ytd_last_year`
|
| 388 |
+
# EXACTLY (including the Feb 29 -> Feb 28 clamp), which is what `pool()` computes the
|
| 389 |
+
# `revenue_ly` column from. `Sales[ytd] < Sales[ytd_last_year]` therefore reproduces the
|
| 390 |
+
# retired `at_risk > 0` condition rather than approximating it. Comparing `ytd` against
|
| 391 |
+
# `last_year` instead would pit seven months against twelve.
|
| 392 |
+
return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_shift_months(d, -12)))
|
| 393 |
+
if kind == "ltm":
|
| 394 |
+
# the last twelve months INCLUDING today: 365 days back, both ends inclusive
|
| 395 |
+
return (_iso(d - _dt.timedelta(days=364)), _iso(d))
|
| 396 |
+
|
| 397 |
+
# "The past week/month/year" are ROLLING and end TODAY — Airtable's wording, and NOT the
|
| 398 |
+
# calendar kinds above: `last_week` is the previous Monday-Sunday, `past_week` is the seven
|
| 399 |
+
# days ending today. Both are offered because a user means different things by them, and
|
| 400 |
+
# picking one to serve both would silently answer the other question.
|
| 401 |
+
# past_week = the same rule as last_n_days(7) — one period back, PLUS ONE DAY, so today
|
| 402 |
+
# is included and the span is exactly 7 days rather than 8.
|
| 403 |
+
# past_year agrees with `ltm` except across a leap day, where the calendar shift keeps the
|
| 404 |
+
# same month/day and the 364-day subtraction cannot.
|
| 405 |
+
if kind == "past_week":
|
| 406 |
+
return (_iso(d - _dt.timedelta(days=6)), _iso(d))
|
| 407 |
+
if kind == "past_month":
|
| 408 |
+
return (_iso(_shift_months(d, -1) + _dt.timedelta(days=1)), _iso(d))
|
| 409 |
+
if kind == "past_year":
|
| 410 |
+
return (_iso(_shift_months(d, -12) + _dt.timedelta(days=1)), _iso(d))
|
| 411 |
+
|
| 412 |
+
if kind == "last_n_days":
|
| 413 |
+
# INCLUSIVE of today — "the last 7 days" is today and the 6 before it, not the 7 before
|
| 414 |
+
# yesterday. Getting this wrong drops today's orders from every recent-activity filter.
|
| 415 |
+
return (_iso(d - _dt.timedelta(days=spec["n"] - 1)), _iso(d))
|
| 416 |
+
if kind == "next_n_days":
|
| 417 |
+
return (_iso(d), _iso(d + _dt.timedelta(days=spec["n"] - 1)))
|
| 418 |
+
|
| 419 |
+
if kind == "custom":
|
| 420 |
+
return (spec.get("from"), spec.get("to"))
|
| 421 |
+
raise AssertionError(f"unhandled window kind {kind!r}") # unreachable; WINDOW_KINDS is closed
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def label(spec):
|
| 425 |
+
"""How a window reads in the condition sentence ('in the last 90 days')."""
|
| 426 |
+
spec = normalize(spec)
|
| 427 |
+
if spec is None:
|
| 428 |
+
return "an invalid range"
|
| 429 |
+
return WINDOW_LABELS[spec["kind"]].format(n=spec.get("n"))
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
def resolve_anchor(mode, value, today):
|
| 433 |
+
"""A date condition's right-hand side -> ONE inclusive ISO date, or None.
|
| 434 |
+
|
| 435 |
+
`mode` is an entry of ANCHOR_MODES; None/'' means `exact`, which is what every view saved
|
| 436 |
+
before anchors existed carries (op + an ISO date in `value`). `value` supplies the number for
|
| 437 |
+
`n_days_ago` and the date for `exact`, and is ignored by the four value-free modes.
|
| 438 |
+
|
| 439 |
+
Returns None when the anchor cannot be resolved — an unknown mode, a non-numeric N, a value
|
| 440 |
+
that is not a date. The caller must then treat the CONDITION as unanswerable and match
|
| 441 |
+
nothing. It must NOT fall through to "no condition": that widens the result while the count
|
| 442 |
+
beside it still looks authoritative, which is the whole reason this returns None rather than
|
| 443 |
+
a best guess (same contract as `resolve()` above).
|
| 444 |
+
|
| 445 |
+
`n_days_ago` accepts n = 0 where a WINDOW requires n >= 1. A zero-day window is empty and
|
| 446 |
+
could only be a mistake; "0 days ago" is today, which is a date a person can mean.
|
| 447 |
+
"""
|
| 448 |
+
m = mode or "exact"
|
| 449 |
+
if m not in ANCHOR_MODES:
|
| 450 |
+
return None
|
| 451 |
+
d = _parse_date(today)
|
| 452 |
+
if d is None:
|
| 453 |
+
raise ValueError("resolve_anchor() needs an explicit `today`; see this module's "
|
| 454 |
+
"docstring on why the clock is never read here")
|
| 455 |
+
if m == "today":
|
| 456 |
+
return _iso(d)
|
| 457 |
+
if m == "yesterday":
|
| 458 |
+
return _iso(d - _dt.timedelta(days=1))
|
| 459 |
+
if m == "one_week_ago":
|
| 460 |
+
# A week ago is a DATE (today minus 7), not the past-week RANGE. The two read almost
|
| 461 |
+
# identically in English and mean different things in a comparison.
|
| 462 |
+
return _iso(d - _dt.timedelta(days=7))
|
| 463 |
+
if m == "one_month_ago":
|
| 464 |
+
return _iso(_shift_months(d, -1))
|
| 465 |
+
if m == "n_days_ago":
|
| 466 |
+
s = str("" if value is None else value).strip()
|
| 467 |
+
# `[0-9]` not `\d`: a Python \d matches every Unicode decimal digit, so a fullwidth
|
| 468 |
+
# '30' would resolve here and be rejected by the TS mirror's Number() — the exact
|
| 469 |
+
# two-engine divergence filter_sql.to_num was rewritten to avoid.
|
| 470 |
+
if not s or not all("0" <= ch <= "9" for ch in s):
|
| 471 |
+
return None
|
| 472 |
+
n = int(s)
|
| 473 |
+
# No `n < 0` guard: the digit test above already excludes a sign, so it would be
|
| 474 |
+
# unreachable. The negative control proved that — removing it changed nothing, which is
|
| 475 |
+
# how dead code hides in a defensive-looking line. The CAP is the live guard.
|
| 476 |
+
if n > MAX_N:
|
| 477 |
+
return None
|
| 478 |
+
return _iso(d - _dt.timedelta(days=n))
|
| 479 |
+
parsed = _parse_date(value) # exact
|
| 480 |
+
return None if parsed is None else _iso(parsed)
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
def _anchor_label(mode, value):
|
| 484 |
+
"""How an anchor reads in the condition sentence ('one month ago', '30 days ago')."""
|
| 485 |
+
m = mode or "exact"
|
| 486 |
+
if m not in ANCHOR_MODES:
|
| 487 |
+
return "an invalid date"
|
| 488 |
+
if m == "n_days_ago":
|
| 489 |
+
return ANCHOR_LABELS[m].format(n=str("" if value is None else value).strip() or "N")
|
| 490 |
+
if m == "exact":
|
| 491 |
+
return str("" if value is None else value).strip() or ANCHOR_LABELS[m]
|
| 492 |
+
return ANCHOR_LABELS[m]
|
platform/harness/amazon_client.py
CHANGED
|
@@ -245,8 +245,10 @@ REPORT_TYPES = {
|
|
| 245 |
# reads whatever V2 actually sends and nothing here needs to know the column list -- which is
|
| 246 |
# exactly the property that makes swapping the report type a one-line change.
|
| 247 |
# · https://developer-docs.amazon.com/sp-api/docs/report-type-values-settlement
|
|
|
|
|
|
|
| 248 |
"settlements": {
|
| 249 |
-
"
|
| 250 |
"validates_against": "settlement summary vs settlement lines",
|
| 251 |
"role": "Finance and Accounting",
|
| 252 |
},
|
|
@@ -324,6 +326,13 @@ OPERATIONS = {
|
|
| 324 |
"name": "createReport", "roles": _BULK_REPORT_ROLES, "method": "POST",
|
| 325 |
"path": "reports/2021-06-30/reports", "report_types": _APPROVED_REPORT_TYPES,
|
| 326 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 327 |
"getReport": {
|
| 328 |
"name": "getReport", "roles": _BULK_REPORT_ROLES, "method": "GET",
|
| 329 |
"path": "reports/2021-06-30/reports/{reportId}",
|
|
@@ -967,6 +976,69 @@ def report_document(fields, report_type, start=None, end=None, options=None,
|
|
| 967 |
"compression": meta.get("compressionAlgorithm")}
|
| 968 |
|
| 969 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 970 |
def _download_document(url, compression="", timeout=120):
|
| 971 |
"""Fetch one presigned report document and decode it. -> text.
|
| 972 |
|
|
|
|
| 245 |
# reads whatever V2 actually sends and nothing here needs to know the column list -- which is
|
| 246 |
# exactly the property that makes swapping the report type a one-line change.
|
| 247 |
# · https://developer-docs.amazon.com/sp-api/docs/report-type-values-settlement
|
| 248 |
+
# Amazon creates settlements on its own schedule and refuses createReport for them.
|
| 249 |
+
# This type is a getReports discovery filter, never a requestable report flavor.
|
| 250 |
"settlements": {
|
| 251 |
+
"report_discover": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2",
|
| 252 |
"validates_against": "settlement summary vs settlement lines",
|
| 253 |
"role": "Finance and Accounting",
|
| 254 |
},
|
|
|
|
| 326 |
"name": "createReport", "roles": _BULK_REPORT_ROLES, "method": "POST",
|
| 327 |
"path": "reports/2021-06-30/reports", "report_types": _APPROVED_REPORT_TYPES,
|
| 328 |
},
|
| 329 |
+
# Discovery is approved only for Amazon-issued settlements and their finance role.
|
| 330 |
+
# Sharing the Reports path grants neither the other bulk types nor their roles.
|
| 331 |
+
"getReports": {
|
| 332 |
+
"name": "getReports", "roles": ("Finance and Accounting",), "method": "GET",
|
| 333 |
+
"path": "reports/2021-06-30/reports",
|
| 334 |
+
"report_types": (REPORT_TYPES["settlements"]["report_discover"],),
|
| 335 |
+
},
|
| 336 |
"getReport": {
|
| 337 |
"name": "getReport", "roles": _BULK_REPORT_ROLES, "method": "GET",
|
| 338 |
"path": "reports/2021-06-30/reports/{reportId}",
|
|
|
|
| 976 |
"compression": meta.get("compressionAlgorithm")}
|
| 977 |
|
| 978 |
|
| 979 |
+
SETTLEMENT_REPORT_PAGE_CAP = 1000
|
| 980 |
+
|
| 981 |
+
|
| 982 |
+
def settlement_reports(fields, *, since=None, timeout=_TIMEOUT, log=None):
|
| 983 |
+
"""List settlements Amazon has already created, oldest first.
|
| 984 |
+
|
| 985 |
+
Amazon issues these on its own fortnightly cycle, so this lists rather than requests.
|
| 986 |
+
Only safe, named metadata leaves this function; vendor extras and tokens stay here.
|
| 987 |
+
"""
|
| 988 |
+
require_operation("GET", _REPORTS_CREATE_PATH,
|
| 989 |
+
report_type=REPORT_TYPES["settlements"]["report_discover"])
|
| 990 |
+
import re
|
| 991 |
+
|
| 992 |
+
rtype = REPORT_TYPES["settlements"]["report_discover"]
|
| 993 |
+
token = access_token(fields, timeout=timeout)
|
| 994 |
+
region = str((fields or {}).get("region") or DEFAULT_REGION)
|
| 995 |
+
params = {"reportTypes": rtype, "processingStatuses": "DONE", "pageSize": 100}
|
| 996 |
+
if since is not None:
|
| 997 |
+
params["createdSince"] = str(since)
|
| 998 |
+
reports = []
|
| 999 |
+
for _page in range(SETTLEMENT_REPORT_PAGE_CAP):
|
| 1000 |
+
# _request owns the same _retry policy as the existing report/document reads.
|
| 1001 |
+
got = _request(_REPORTS_CREATE_PATH, token, region=region, timeout=timeout,
|
| 1002 |
+
sandbox=is_sandbox(fields), **params)
|
| 1003 |
+
for raw in got.get("reports") or ():
|
| 1004 |
+
row = {key: str(raw[key]) if raw.get(key) is not None else "" for key in (
|
| 1005 |
+
"reportId", "reportDocumentId", "reportType", "dataStartTime",
|
| 1006 |
+
"dataEndTime", "createdTime")}
|
| 1007 |
+
# Match the API status door without importing upward into the API layer.
|
| 1008 |
+
# Refuse unsafe values with a fixed sentence that cannot echo the credential.
|
| 1009 |
+
if any(len(value) > 512 or "Atzr|" in value
|
| 1010 |
+
or re.search(r"\bbearer\s+\S+", value, flags=re.IGNORECASE)
|
| 1011 |
+
or any(ch in value for ch in ("\u2014", "\u2013"))
|
| 1012 |
+
for value in row.values()):
|
| 1013 |
+
raise AmazonError("Settlement discovery returned unsafe metadata; no status was emitted.")
|
| 1014 |
+
if row["reportType"] != rtype or not row["reportDocumentId"]:
|
| 1015 |
+
raise AmazonError("Settlement discovery returned an unexpected type or no document ID.")
|
| 1016 |
+
reports.append(row)
|
| 1017 |
+
next_token = got.get("nextToken")
|
| 1018 |
+
if not next_token:
|
| 1019 |
+
if log:
|
| 1020 |
+
log(f"getReports discovered {len(reports)} settlement reports")
|
| 1021 |
+
return sorted(reports, key=lambda row: (row["createdTime"], row["reportId"]))
|
| 1022 |
+
# Amazon continuation tokens replace the original filters, which the token carries.
|
| 1023 |
+
params = {"nextToken": str(next_token)}
|
| 1024 |
+
raise AmazonError(
|
| 1025 |
+
f"Settlement discovery hit the {SETTLEMENT_REPORT_PAGE_CAP}-page cap because Amazon kept "
|
| 1026 |
+
"returning nextToken; retry with a narrower createdSince window.")
|
| 1027 |
+
|
| 1028 |
+
|
| 1029 |
+
def settlement_document(fields, document_id, *, timeout=_TIMEOUT):
|
| 1030 |
+
"""One settlement document's text through getReportDocument and the presigned URL."""
|
| 1031 |
+
require_operation("GET", f"reports/2021-06-30/documents/{document_id}")
|
| 1032 |
+
token = access_token(fields, timeout=timeout)
|
| 1033 |
+
region = str((fields or {}).get("region") or DEFAULT_REGION)
|
| 1034 |
+
meta = _request(f"reports/2021-06-30/documents/{document_id}", token, region=region,
|
| 1035 |
+
timeout=timeout, sandbox=is_sandbox(fields))
|
| 1036 |
+
url = str(meta.get("url") or "")
|
| 1037 |
+
if not url:
|
| 1038 |
+
raise AmazonError("The settlement document carried no download URL.")
|
| 1039 |
+
return _download_document(url, str(meta.get("compressionAlgorithm") or ""))
|
| 1040 |
+
|
| 1041 |
+
|
| 1042 |
def _download_document(url, compression="", timeout=120):
|
| 1043 |
"""Fetch one presigned report document and decode it. -> text.
|
| 1044 |
|
platform/harness/amazon_store.py
CHANGED
|
@@ -70,6 +70,9 @@ PREFIX = "amzn_"
|
|
| 70 |
TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
|
| 71 |
"pricing", "brand_analytics")
|
| 72 |
|
|
|
|
|
|
|
|
|
|
| 73 |
# C1 is a tenant-store document, not an API dependency. verify_amazon gates reader parity.
|
| 74 |
ENABLEMENT_KEY = "amazon_connector_enablement"
|
| 75 |
# Settlements are issued roughly fortnightly; spending a report on every hourly tick adds nothing.
|
|
@@ -149,8 +152,20 @@ def entity_key(table):
|
|
| 149 |
#: How far back a first sync reaches when the caller names no window. Amazon's flat-file order
|
| 150 |
#: reports accept at most ~31 days per request, so a backfill is sliced (`_windows`).
|
| 151 |
DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730)
|
|
|
|
|
|
|
| 152 |
#: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one.
|
| 153 |
MAX_WINDOW_DAYS = 30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
|
| 155 |
# Connector metadata, NOT measured Amazon fields. TSV normalization strips leading underscores;
|
| 156 |
# JSON flattening rejects these exact reserved keys. Neither path can overwrite metadata.
|
|
@@ -447,7 +462,7 @@ def _cell(value, kind):
|
|
| 447 |
return None
|
| 448 |
|
| 449 |
|
| 450 |
-
def _ensure(con, table, cols, rows=None, declared=None):
|
| 451 |
"""Create or widen the mirror table. A column whose type is now wrong is REBUILT, not patched.
|
| 452 |
|
| 453 |
⚠ Same ruling as `meta_store._ensure`: DuckDB cannot retype in place, and this is derived data
|
|
@@ -479,7 +494,15 @@ def _ensure(con, table, cols, rows=None, declared=None):
|
|
| 479 |
clash = [c for c, ty in want.items()
|
| 480 |
if c in have and have[c] != ty
|
| 481 |
and not (have[c].startswith("VARCHAR") and ty == "VARCHAR")]
|
| 482 |
-
if clash:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 483 |
con.execute(f"DROP TABLE {table}")
|
| 484 |
have = {}
|
| 485 |
if not have:
|
|
@@ -508,6 +531,47 @@ def _replace(con, table, cols, types, rows):
|
|
| 508 |
return len(rows)
|
| 509 |
|
| 510 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 511 |
def _windows(start, end, size=MAX_WINDOW_DAYS):
|
| 512 |
"""Slice a backfill into report-sized windows. Amazon's ceiling, reported not hidden."""
|
| 513 |
out, cur = [], start
|
|
@@ -518,6 +582,49 @@ def _windows(start, end, size=MAX_WINDOW_DAYS):
|
|
| 518 |
return out
|
| 519 |
|
| 520 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 521 |
def _iso(dt):
|
| 522 |
return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 523 |
|
|
@@ -1071,6 +1178,94 @@ def _read_state(con, key):
|
|
| 1071 |
return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]}
|
| 1072 |
|
| 1073 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1074 |
def _sync_event(kind, tenant, table, detail, rt, report, log):
|
| 1075 |
"""Sync is nondestructive: preserve C2's loud failure, but continue the data pull."""
|
| 1076 |
try:
|
|
@@ -1108,7 +1303,7 @@ def _sync_problem(exc, table, operation, tenant, rt, report, log):
|
|
| 1108 |
|
| 1109 |
|
| 1110 |
def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
|
| 1111 |
-
max_wait=900, *, scheduled=False):
|
| 1112 |
"""Pull every declared table into the tenant's mirror. -> a report dict.
|
| 1113 |
|
| 1114 |
Idempotent: a re-sync replaces each window rather than appending. Refuses, loudly and with the
|
|
@@ -1142,7 +1337,6 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 1142 |
|
| 1143 |
want = [t for t in (TABLES if tables is None else tables) if t in TABLES]
|
| 1144 |
end = datetime.now(timezone.utc)
|
| 1145 |
-
start = since or (end - timedelta(days=DEFAULT_BACKFILL_DAYS))
|
| 1146 |
report = {"tenant": tenant_key, "tables": {}, "started": _iso(end),
|
| 1147 |
"problems": [], "audit_problems": []}
|
| 1148 |
enablement = read_database_enablement(rt)
|
|
@@ -1171,6 +1365,16 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 1171 |
if obligation["earliestTriggerAt"]:
|
| 1172 |
con.close()
|
| 1173 |
raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1174 |
if scheduled:
|
| 1175 |
plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table))
|
| 1176 |
for table in TABLES}, want, now=end)
|
|
@@ -1179,9 +1383,60 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 1179 |
report["tables"][table] = {"state": "deferred", "reason": reason}
|
| 1180 |
log(f"{table}: DEFERRED: {reason}")
|
| 1181 |
|
| 1182 |
-
|
| 1183 |
-
|
| 1184 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1185 |
key, mirror = entity_key(table), table_name(table)
|
| 1186 |
if not rtype:
|
| 1187 |
report["tables"][table] = {"rows": 0, "state": "no confirmed bulk report"}
|
|
@@ -1189,15 +1444,89 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 1189 |
continue
|
| 1190 |
prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
|
| 1191 |
root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1192 |
try:
|
| 1193 |
# Attempt first, so a refusal or process interruption cannot starve the other tables.
|
| 1194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1195 |
written, cols, cleared = 0, [], False
|
| 1196 |
pricing_asins = []
|
| 1197 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1198 |
# W59-T43 / C2 has no amazon.access kind. Record each credential-backed
|
| 1199 |
# vendor attempt as a sync ACCESS phase, distinct from the completion event.
|
| 1200 |
# Lane F/combine own the vocabulary decision; no credential or row enters C2.
|
|
|
|
| 1201 |
_sync_event("amazon.sync", tenant_key, table,
|
| 1202 |
{"state": "access", "operation": "report_document"}, rt, report, log)
|
| 1203 |
text, meta = cx.report_document(
|
|
@@ -1237,7 +1566,25 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 1237 |
text, document_id=meta.get("reportDocumentId"), report_type=rtype)
|
| 1238 |
declared = {}
|
| 1239 |
if not cols:
|
| 1240 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1241 |
if table == "orders":
|
| 1242 |
rows = _dedupe_orders(rows)
|
| 1243 |
if table == "inventory":
|
|
@@ -1246,16 +1593,32 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 1246 |
cols.append("inventory_source")
|
| 1247 |
if table == "pricing":
|
| 1248 |
pricing_asins.extend(_pricing_asins(rows))
|
| 1249 |
-
types = _ensure(con, mirror, cols, rows, declared=declared
|
| 1250 |
-
|
| 1251 |
-
#
|
| 1252 |
-
if not cleared:
|
| 1253 |
con.execute(f"DELETE FROM {mirror} WHERE 1=1")
|
| 1254 |
cleared = True
|
| 1255 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1256 |
|
| 1257 |
-
operations = {}
|
| 1258 |
if table == "pricing":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1259 |
state_key = ENTITY_STATE_KEYS[table]
|
| 1260 |
if not pricing_asins:
|
| 1261 |
_record_state(con, state_key, "unproven", 0,
|
|
@@ -1291,11 +1654,11 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 1291 |
awd_rows = _awd_rows(inventory)
|
| 1292 |
if awd_rows:
|
| 1293 |
awd_cols = sorted({c for row in awd_rows for c in row})
|
| 1294 |
-
awd_types = _ensure(con, mirror, awd_cols, awd_rows)
|
| 1295 |
-
if not cleared:
|
| 1296 |
con.execute(f"DELETE FROM {mirror} WHERE 1=1")
|
| 1297 |
cleared = True
|
| 1298 |
-
written +=
|
| 1299 |
_record_state(con, state_key, "done", len(awd_rows))
|
| 1300 |
operations["listInventory"] = {
|
| 1301 |
"state": "done", "rows": len(awd_rows)}
|
|
@@ -1312,21 +1675,67 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 1312 |
column_count, total_rows = len(cols), written
|
| 1313 |
report["tables"][table] = {
|
| 1314 |
"rows": total_rows, "columns": column_count, "provenance": prov,
|
| 1315 |
-
"report": rtype, "
|
|
|
|
| 1316 |
_record_state(con, key, "done", total_rows)
|
| 1317 |
_record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
|
|
|
|
|
|
|
| 1318 |
_sync_event("amazon.sync", tenant_key, table,
|
| 1319 |
{"state": "done", "rows": total_rows,
|
| 1320 |
"operations_ok": all(v["state"] == "done" for v in operations.values())},
|
| 1321 |
rt, report, log)
|
| 1322 |
log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1323 |
except Exception as e: # noqa: BLE001
|
|
|
|
| 1324 |
# ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
|
| 1325 |
# ticket answering itself, and it must not cost the six tables that did work.
|
| 1326 |
-
problem = _sync_problem(e, table,
|
|
|
|
| 1327 |
report["tables"][table] = {"rows": 0, **problem}
|
| 1328 |
_sync_event("amazon.sync", tenant_key, table,
|
| 1329 |
{"state": problem["state"], "rows": 0}, rt, report, log)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1330 |
report["finished"] = _iso(datetime.now(timezone.utc))
|
| 1331 |
con.close()
|
| 1332 |
return report
|
|
|
|
| 70 |
TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
|
| 71 |
"pricing", "brand_analytics")
|
| 72 |
|
| 73 |
+
#: Head -> lines: both tables read the same document; a second report buys no new rows.
|
| 74 |
+
SHARED_REPORT_SIBLINGS = {"orders": "order_items"}
|
| 75 |
+
|
| 76 |
# C1 is a tenant-store document, not an API dependency. verify_amazon gates reader parity.
|
| 77 |
ENABLEMENT_KEY = "amazon_connector_enablement"
|
| 78 |
# Settlements are issued roughly fortnightly; spending a report on every hourly tick adds nothing.
|
|
|
|
| 152 |
#: How far back a first sync reaches when the caller names no window. Amazon's flat-file order
|
| 153 |
#: reports accept at most ~31 days per request, so a backfill is sliced (`_windows`).
|
| 154 |
DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730)
|
| 155 |
+
#: Per-table marker whose detail is the ISO UTC end of the last successfully landed window.
|
| 156 |
+
WATERMARK_PREFIX = "sync_mark_"
|
| 157 |
#: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one.
|
| 158 |
MAX_WINDOW_DAYS = 30
|
| 159 |
+
#: R18: the most recent month lands before any history, so a newly connected account has usable
|
| 160 |
+
#: rows in minutes instead of hours. One window, because Amazon's own ceiling is 30 days.
|
| 161 |
+
FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS
|
| 162 |
+
#: F4: Amazon paces createReport (burst 15, one per 60 s, 5 s gap), so 7 tables x 25 windows is
|
| 163 |
+
#: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
|
| 164 |
+
#: thread for hours and the Odoo resync behind it would never run.
|
| 165 |
+
FIRST_LOAD_REPORTS_PER_PASS = 15
|
| 166 |
+
BACKFILL_FLOOR_PREFIX = "sync_floor_"
|
| 167 |
+
FIRST_LOAD_END_KEY = "sync_first_load_end"
|
| 168 |
+
FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
|
| 169 |
|
| 170 |
# Connector metadata, NOT measured Amazon fields. TSV normalization strips leading underscores;
|
| 171 |
# JSON flattening rejects these exact reserved keys. Neither path can overwrite metadata.
|
|
|
|
| 462 |
return None
|
| 463 |
|
| 464 |
|
| 465 |
+
def _ensure(con, table, cols, rows=None, declared=None, *, preserve_rows=False):
|
| 466 |
"""Create or widen the mirror table. A column whose type is now wrong is REBUILT, not patched.
|
| 467 |
|
| 468 |
⚠ Same ruling as `meta_store._ensure`: DuckDB cannot retype in place, and this is derived data
|
|
|
|
| 494 |
clash = [c for c, ty in want.items()
|
| 495 |
if c in have and have[c] != ty
|
| 496 |
and not (have[c].startswith("VARCHAR") and ty == "VARCHAR")]
|
| 497 |
+
if clash and preserve_rows:
|
| 498 |
+
# Settlement document markers preclude replay: widening must retain prior lines.
|
| 499 |
+
# The measured types are VARCHAR, BIGINT and DOUBLE; never narrow historical data.
|
| 500 |
+
for c in clash:
|
| 501 |
+
widened = "VARCHAR" if "VARCHAR" in (have[c], want[c]) else "DOUBLE"
|
| 502 |
+
if have[c] != widened:
|
| 503 |
+
con.execute(f'ALTER TABLE "{table}" ALTER COLUMN "{c}" TYPE {widened}')
|
| 504 |
+
have[c] = want[c] = widened
|
| 505 |
+
elif clash:
|
| 506 |
con.execute(f"DROP TABLE {table}")
|
| 507 |
have = {}
|
| 508 |
if not have:
|
|
|
|
| 531 |
return len(rows)
|
| 532 |
|
| 533 |
|
| 534 |
+
def replace_keys_for(table):
|
| 535 |
+
"""Replace on declared identity excluding lineage, which changes on every fetch."""
|
| 536 |
+
return tuple(c for c in (amazon_schema.TABLE_SPECS.get(table) or {}).get("row_identity", ())
|
| 537 |
+
if c not in _ORIGIN_TYPES)
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
def replace_window(con, table, cols, types, rows, keys):
|
| 541 |
+
"""Replace incoming key groups atomically, or report why only insertion is possible."""
|
| 542 |
+
missing = [c for c in keys if c not in cols]
|
| 543 |
+
if missing:
|
| 544 |
+
note = (f"{table}: window replacement unavailable; missing identity columns: "
|
| 545 |
+
f"{', '.join(missing)}; rows inserted; use a report carrying these columns")
|
| 546 |
+
return _replace(con, table, cols, types, rows), note
|
| 547 |
+
if not keys or not rows:
|
| 548 |
+
return _replace(con, table, cols, types, rows), None
|
| 549 |
+
quote = lambda name: '"' + str(name).replace('"', '""') + '"'
|
| 550 |
+
target = quote(table)
|
| 551 |
+
temporary = quote("amazon_window_keys_" + uuid.uuid4().hex)
|
| 552 |
+
columns = ", ".join(quote(c) for c in keys)
|
| 553 |
+
# Use the mirror's types and the insert's cell conversion so NULL and numeric keys agree.
|
| 554 |
+
incoming = list(dict.fromkeys(tuple(_cell(row.get(c), types.get(c, "VARCHAR"))
|
| 555 |
+
for c in keys) for row in rows))
|
| 556 |
+
con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE")
|
| 557 |
+
try:
|
| 558 |
+
con.executemany(f"INSERT INTO {temporary} VALUES ({', '.join('?' for _ in keys)})",
|
| 559 |
+
incoming)
|
| 560 |
+
predicate = " AND ".join(
|
| 561 |
+
f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
|
| 562 |
+
con.execute("BEGIN TRANSACTION")
|
| 563 |
+
try:
|
| 564 |
+
con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})')
|
| 565 |
+
count = _replace(con, table, cols, types, rows)
|
| 566 |
+
con.execute("COMMIT")
|
| 567 |
+
except Exception:
|
| 568 |
+
con.execute("ROLLBACK")
|
| 569 |
+
raise
|
| 570 |
+
return count, None
|
| 571 |
+
finally:
|
| 572 |
+
con.execute(f"DROP TABLE {temporary}")
|
| 573 |
+
|
| 574 |
+
|
| 575 |
def _windows(start, end, size=MAX_WINDOW_DAYS):
|
| 576 |
"""Slice a backfill into report-sized windows. Amazon's ceiling, reported not hidden."""
|
| 577 |
out, cur = [], start
|
|
|
|
| 582 |
return out
|
| 583 |
|
| 584 |
|
| 585 |
+
def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAYS):
|
| 586 |
+
"""R18's request order: every recent month, then each table's history, oldest last.
|
| 587 |
+
|
| 588 |
+
Settlements have one discovery slot (None, None), never a createReport window.
|
| 589 |
+
This planner is pure; durable floors filter its fixed windows when a pass resumes.
|
| 590 |
+
"""
|
| 591 |
+
start, end = _utc(start), _utc(end)
|
| 592 |
+
if start >= end:
|
| 593 |
+
return []
|
| 594 |
+
size = min(float(size), MAX_WINDOW_DAYS)
|
| 595 |
+
recent_days = FIRST_LOAD_RECENT_DAYS if recent_days is None else recent_days
|
| 596 |
+
recent_days = min(float(recent_days), size)
|
| 597 |
+
if size <= 0 or recent_days <= 0:
|
| 598 |
+
raise ValueError("Amazon report windows must have a positive size.")
|
| 599 |
+
recent_start = max(start, end - timedelta(days=recent_days))
|
| 600 |
+
tables = list(dict.fromkeys(tables))
|
| 601 |
+
recent = [(table, None, None) if table == "settlements" else (table, recent_start, end)
|
| 602 |
+
for table in tables]
|
| 603 |
+
history = [(table, w0, w1) for table in tables if table != "settlements"
|
| 604 |
+
for w0, w1 in reversed(_windows(start, recent_start, size))]
|
| 605 |
+
return recent + history
|
| 606 |
+
|
| 607 |
+
|
| 608 |
+
def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None):
|
| 609 |
+
"""Amazon createReport pacing floor in seconds; polling and other work can add time."""
|
| 610 |
+
from harness.amazon_client import REPORT_STAGGER_SECONDS, REPORT_BURST, REPORT_SUSTAINED_SECONDS
|
| 611 |
+
if report_count <= 0:
|
| 612 |
+
return 0
|
| 613 |
+
stagger = REPORT_STAGGER_SECONDS if stagger_seconds is None else stagger_seconds
|
| 614 |
+
tokens = REPORT_BURST if burst is None else burst
|
| 615 |
+
rate = 60 / REPORT_SUSTAINED_SECONDS if per_minute is None else per_minute
|
| 616 |
+
seconds = max(0, (report_count - 1) * stagger, (report_count - tokens) * 60 / rate)
|
| 617 |
+
return int(seconds) + (seconds > int(seconds))
|
| 618 |
+
|
| 619 |
+
|
| 620 |
+
def first_load_eta_cause(report_count):
|
| 621 |
+
"""Screen-safe explanation of the vendor ceiling and the automatic continuation."""
|
| 622 |
+
from harness.amazon_client import REPORT_BURST, REPORT_SUSTAINED_SECONDS
|
| 623 |
+
return (f"Amazon's report rate allows a burst of {REPORT_BURST}, then one report every "
|
| 624 |
+
f"{REPORT_SUSTAINED_SECONDS:g} seconds. Keep the connection enabled while history "
|
| 625 |
+
"loads automatically. Report processing and other syncs can add time.")
|
| 626 |
+
|
| 627 |
+
|
| 628 |
def _iso(dt):
|
| 629 |
return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 630 |
|
|
|
|
| 1178 |
return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]}
|
| 1179 |
|
| 1180 |
|
| 1181 |
+
def settlement_is_new(con, document_id):
|
| 1182 |
+
"""A completed document is durable even when it contained no mirror lines."""
|
| 1183 |
+
state = _read_state(con, entity_key("settlement_doc_" + str(document_id)))
|
| 1184 |
+
return not state or state["state"] != "done"
|
| 1185 |
+
|
| 1186 |
+
|
| 1187 |
+
def read_watermark(con, table):
|
| 1188 |
+
"""Missing or malformed detail means no successful window, never the current time."""
|
| 1189 |
+
state = _read_state(con, entity_key(WATERMARK_PREFIX + table))
|
| 1190 |
+
detail = (state or {}).get("detail")
|
| 1191 |
+
if not detail:
|
| 1192 |
+
return None
|
| 1193 |
+
try:
|
| 1194 |
+
return _utc(detail)
|
| 1195 |
+
except (ValueError, TypeError, AttributeError, OverflowError):
|
| 1196 |
+
return None
|
| 1197 |
+
|
| 1198 |
+
|
| 1199 |
+
def record_watermark(con, table, when, rows=0):
|
| 1200 |
+
"""Persist only after this table's pass succeeds."""
|
| 1201 |
+
_record_state(con, entity_key(WATERMARK_PREFIX + table), "done", rows, _iso(_utc(when)))
|
| 1202 |
+
|
| 1203 |
+
|
| 1204 |
+
def read_backfill_floor(con, table):
|
| 1205 |
+
"""Oldest contiguous landed window; missing or corrupt markers restart safely."""
|
| 1206 |
+
state = _read_state(con, entity_key(BACKFILL_FLOOR_PREFIX + table))
|
| 1207 |
+
try:
|
| 1208 |
+
return _utc(state["detail"]) if state and state.get("detail") else None
|
| 1209 |
+
except (ValueError, TypeError, AttributeError, OverflowError):
|
| 1210 |
+
return None
|
| 1211 |
+
|
| 1212 |
+
|
| 1213 |
+
def record_backfill_floor(con, table, when):
|
| 1214 |
+
_record_state(con, entity_key(BACKFILL_FLOOR_PREFIX + table), "done", detail=_iso(_utc(when)))
|
| 1215 |
+
|
| 1216 |
+
|
| 1217 |
+
def needs_first_load(tenant_key, rt=None):
|
| 1218 |
+
"""A fresh account or an unfinished durable load uses bounded passes, even after one table finishes."""
|
| 1219 |
+
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 1220 |
+
try:
|
| 1221 |
+
load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
|
| 1222 |
+
return bool(load and load["state"] != "done") or not any(
|
| 1223 |
+
read_watermark(con, table) for table in TABLES)
|
| 1224 |
+
finally:
|
| 1225 |
+
con.close()
|
| 1226 |
+
|
| 1227 |
+
|
| 1228 |
+
def first_load_status(tenant_key, rt=None):
|
| 1229 |
+
"""Durable numeric progress for the API's safe-status projection; no vendor payloads."""
|
| 1230 |
+
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 1231 |
+
try:
|
| 1232 |
+
state = _read_state(con, entity_key(FIRST_LOAD_STATUS_KEY))
|
| 1233 |
+
if not state:
|
| 1234 |
+
return None
|
| 1235 |
+
values = json.loads(state["detail"])
|
| 1236 |
+
return {"state": state["state"], "remaining": int(values["remaining"]),
|
| 1237 |
+
"etaSeconds": int(values["etaSeconds"]), "backfillDays": DEFAULT_BACKFILL_DAYS,
|
| 1238 |
+
"cause": first_load_eta_cause(values["remaining"])}
|
| 1239 |
+
finally:
|
| 1240 |
+
con.close()
|
| 1241 |
+
|
| 1242 |
+
|
| 1243 |
+
def sync_window(table, mark, *, now, since=None, backfill_days=None):
|
| 1244 |
+
"""Pure request decision for one table: start, end, report, incremental and cause."""
|
| 1245 |
+
end = _utc(now)
|
| 1246 |
+
days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days
|
| 1247 |
+
try:
|
| 1248 |
+
mark = _utc(mark) if mark else None
|
| 1249 |
+
except (ValueError, TypeError, AttributeError, OverflowError):
|
| 1250 |
+
mark = None
|
| 1251 |
+
if since is not None:
|
| 1252 |
+
start, incremental = _utc(since), False
|
| 1253 |
+
cause = f"caller named the window starting {_iso(start)}"
|
| 1254 |
+
elif mark is not None:
|
| 1255 |
+
start, incremental = mark, True
|
| 1256 |
+
cause = f"incremental window since watermark {_iso(mark)}"
|
| 1257 |
+
else:
|
| 1258 |
+
start, incremental = end - timedelta(days=days), False
|
| 1259 |
+
cause = f"first full load: {days} days of backfill"
|
| 1260 |
+
spec = _client().REPORT_TYPES.get(table) or {}
|
| 1261 |
+
rtype = spec.get("report_incremental") if incremental and spec.get("report_incremental") else spec.get("report")
|
| 1262 |
+
if table == "settlements":
|
| 1263 |
+
# The watermark filters issued reports by creation time; it never requests one.
|
| 1264 |
+
rtype = spec.get("report_discover")
|
| 1265 |
+
return {"start": start, "end": end, "report": rtype,
|
| 1266 |
+
"incremental": incremental, "cause": cause}
|
| 1267 |
+
|
| 1268 |
+
|
| 1269 |
def _sync_event(kind, tenant, table, detail, rt, report, log):
|
| 1270 |
"""Sync is nondestructive: preserve C2's loud failure, but continue the data pull."""
|
| 1271 |
try:
|
|
|
|
| 1303 |
|
| 1304 |
|
| 1305 |
def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
|
| 1306 |
+
max_wait=900, *, scheduled=False, first_load=False, report_budget=None):
|
| 1307 |
"""Pull every declared table into the tenant's mirror. -> a report dict.
|
| 1308 |
|
| 1309 |
Idempotent: a re-sync replaces each window rather than appending. Refuses, loudly and with the
|
|
|
|
| 1337 |
|
| 1338 |
want = [t for t in (TABLES if tables is None else tables) if t in TABLES]
|
| 1339 |
end = datetime.now(timezone.utc)
|
|
|
|
| 1340 |
report = {"tenant": tenant_key, "tables": {}, "started": _iso(end),
|
| 1341 |
"problems": [], "audit_problems": []}
|
| 1342 |
enablement = read_database_enablement(rt)
|
|
|
|
| 1365 |
if obligation["earliestTriggerAt"]:
|
| 1366 |
con.close()
|
| 1367 |
raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data")
|
| 1368 |
+
load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
|
| 1369 |
+
if scheduled and load and load["state"] != "done":
|
| 1370 |
+
# The pinned inline producer still calls sync(scheduled=True). Do not let it start an
|
| 1371 |
+
# unbounded history pull between the resident writer's bounded first-load passes.
|
| 1372 |
+
for table in want:
|
| 1373 |
+
report["tables"][table] = {"state": "deferred", "reason":
|
| 1374 |
+
"The first load is continuing in bounded passes on the resident sync thread."}
|
| 1375 |
+
report["finished"] = _iso(datetime.now(timezone.utc))
|
| 1376 |
+
con.close()
|
| 1377 |
+
return report
|
| 1378 |
if scheduled:
|
| 1379 |
plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table))
|
| 1380 |
for table in TABLES}, want, now=end)
|
|
|
|
| 1383 |
report["tables"][table] = {"state": "deferred", "reason": reason}
|
| 1384 |
log(f"{table}: DEFERRED: {reason}")
|
| 1385 |
|
| 1386 |
+
# Heads go first even when the caller listed lines first. Scheduled plans still select one.
|
| 1387 |
+
pending = list(dict.fromkeys([head for head, sibling in SHARED_REPORT_SIBLINGS.items()
|
| 1388 |
+
if head in want and sibling in want] + want))
|
| 1389 |
+
handled, noted = set(), set()
|
| 1390 |
+
|
| 1391 |
+
def write_window(table, cols, types, rows):
|
| 1392 |
+
count, note = replace_window(con, table_name(table), cols, types, rows,
|
| 1393 |
+
replace_keys_for(table))
|
| 1394 |
+
if note and table not in noted:
|
| 1395 |
+
noted.add(table)
|
| 1396 |
+
report["problems"].append(note)
|
| 1397 |
+
log(note)
|
| 1398 |
+
return count
|
| 1399 |
+
|
| 1400 |
+
work, plans, paired, contexts = [], {}, {}, {}
|
| 1401 |
+
documents, settlement_remaining = 0, 0
|
| 1402 |
+
failed, landed = set(), set()
|
| 1403 |
+
if first_load:
|
| 1404 |
+
# Freeze the cohort end across passes. Recomputing it would move every window boundary.
|
| 1405 |
+
end = _utc(load["detail"]) if load else _utc(_iso(end))
|
| 1406 |
+
_record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end))
|
| 1407 |
+
plans = {table: sync_window(table, None, now=end) for table in pending}
|
| 1408 |
+
floors = {table: read_backfill_floor(con, table) for table in pending}
|
| 1409 |
+
wanted = [table for table in pending if not read_watermark(con, table)
|
| 1410 |
+
or read_watermark(con, table) < end]
|
| 1411 |
+
for task in first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end):
|
| 1412 |
+
table, w0, w1 = task
|
| 1413 |
+
if w1 is None or floors[table] is None or w1 <= floors[table]:
|
| 1414 |
+
work.append(task)
|
| 1415 |
+
# A failed enrichment after the final report retries completion without fetching again.
|
| 1416 |
+
for table in wanted:
|
| 1417 |
+
if not any(task[0] == table for task in work):
|
| 1418 |
+
work.append((table, None, None))
|
| 1419 |
+
# Reuse a document for head and lines only when BOTH durable cursors agree.
|
| 1420 |
+
for head, sibling in SHARED_REPORT_SIBLINGS.items():
|
| 1421 |
+
head_tasks = [(w0, w1) for table, w0, w1 in work if table == head]
|
| 1422 |
+
line_tasks = [(w0, w1) for table, w0, w1 in work if table == sibling]
|
| 1423 |
+
if head_tasks and head_tasks == line_tasks:
|
| 1424 |
+
paired[head] = sibling
|
| 1425 |
+
work = [task for task in work if task[0] != sibling]
|
| 1426 |
+
else:
|
| 1427 |
+
work = [(table, None, None) for table in pending]
|
| 1428 |
+
|
| 1429 |
+
for index, (table, task_start, task_end) in enumerate(work):
|
| 1430 |
+
if first_load and table in failed:
|
| 1431 |
+
continue
|
| 1432 |
+
if (first_load and isinstance(report_budget, int) and documents >= max(0, report_budget)
|
| 1433 |
+
and (task_end is not None or table == "settlements")):
|
| 1434 |
+
break
|
| 1435 |
+
if table in handled:
|
| 1436 |
+
continue
|
| 1437 |
+
plan = plans[table] if first_load else sync_window(
|
| 1438 |
+
table, read_watermark(con, table), now=end, since=since)
|
| 1439 |
+
rtype = plan["report"]
|
| 1440 |
key, mirror = entity_key(table), table_name(table)
|
| 1441 |
if not rtype:
|
| 1442 |
report["tables"][table] = {"rows": 0, "state": "no confirmed bulk report"}
|
|
|
|
| 1444 |
continue
|
| 1445 |
prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
|
| 1446 |
root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1]
|
| 1447 |
+
log(f"{table}: {plan['cause']}")
|
| 1448 |
+
sibling = SHARED_REPORT_SIBLINGS.get(table)
|
| 1449 |
+
sibling_plan = None
|
| 1450 |
+
if first_load:
|
| 1451 |
+
sibling = paired.get(table)
|
| 1452 |
+
sibling_plan = plans.get(sibling)
|
| 1453 |
+
elif sibling in want and sibling not in handled:
|
| 1454 |
+
sibling_plan = sync_window(sibling, read_watermark(con, sibling), now=end, since=since)
|
| 1455 |
+
differs = [part for part in ("start", "end", "report")
|
| 1456 |
+
if plan[part] != sibling_plan[part]]
|
| 1457 |
+
if differs:
|
| 1458 |
+
log(f"{table} and {sibling}: cannot share this pass; differing {', '.join(differs)}")
|
| 1459 |
+
sibling = None
|
| 1460 |
+
else:
|
| 1461 |
+
handled.add(sibling)
|
| 1462 |
+
log(f"{sibling}: {sibling_plan['cause']}")
|
| 1463 |
+
else:
|
| 1464 |
+
sibling = None
|
| 1465 |
+
operation = "report_document"
|
| 1466 |
try:
|
| 1467 |
# Attempt first, so a refusal or process interruption cannot starve the other tables.
|
| 1468 |
+
if not first_load:
|
| 1469 |
+
_record_state(con, entity_key("sync_tick_" + table), "attempted")
|
| 1470 |
+
if sibling and not first_load:
|
| 1471 |
+
_record_state(con, entity_key("sync_tick_" + sibling), "attempted")
|
| 1472 |
+
sibling_written, sibling_cleared = 0, False
|
| 1473 |
written, cols, cleared = 0, [], False
|
| 1474 |
pricing_asins = []
|
| 1475 |
+
operations = {}
|
| 1476 |
+
if first_load:
|
| 1477 |
+
cleared = floors[table] is not None
|
| 1478 |
+
sibling_cleared = bool(sibling and floors[sibling] is not None)
|
| 1479 |
+
pricing_asins = contexts.setdefault(table, [])
|
| 1480 |
+
if table == "settlements":
|
| 1481 |
+
operation = "getReports"
|
| 1482 |
+
_sync_event("amazon.sync", tenant_key, table,
|
| 1483 |
+
{"state": "access", "operation": operation}, rt, report, log)
|
| 1484 |
+
discovered = cx.settlement_reports(
|
| 1485 |
+
fields, since=_iso(plan["start"]) if plan["incremental"] else None,
|
| 1486 |
+
log=lambda message: log(f"settlements: {message}"))
|
| 1487 |
+
downloaded, skipped = 0, 0
|
| 1488 |
+
for issued_index, issued in enumerate(discovered):
|
| 1489 |
+
document_id = str(issued.get("reportDocumentId") or "")
|
| 1490 |
+
if not document_id:
|
| 1491 |
+
raise cx.AmazonError("Settlement discovery returned no document ID.")
|
| 1492 |
+
if not settlement_is_new(con, document_id):
|
| 1493 |
+
skipped += 1
|
| 1494 |
+
continue
|
| 1495 |
+
if first_load and isinstance(report_budget, int) and documents >= max(0, report_budget):
|
| 1496 |
+
settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or ""))
|
| 1497 |
+
for item in discovered[issued_index:])
|
| 1498 |
+
break
|
| 1499 |
+
operation = "getReportDocument"
|
| 1500 |
+
documents += 1
|
| 1501 |
+
_sync_event("amazon.sync", tenant_key, table,
|
| 1502 |
+
{"state": "access", "operation": operation}, rt, report, log)
|
| 1503 |
+
text = cx.settlement_document(fields, document_id)
|
| 1504 |
+
cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype)
|
| 1505 |
+
count = 0
|
| 1506 |
+
if cols and rows:
|
| 1507 |
+
types = _ensure(con, mirror, cols, rows, preserve_rows=True)
|
| 1508 |
+
# Never clear settlement history: durable markers prevent its replay.
|
| 1509 |
+
# A newly issued document still replaces its own settlement's lines.
|
| 1510 |
+
count = write_window(table, cols, types, rows)
|
| 1511 |
+
written += count
|
| 1512 |
+
# Record completion after the write, including header-only and blank files.
|
| 1513 |
+
_record_state(con, entity_key("settlement_doc_" + document_id), "done",
|
| 1514 |
+
count, issued.get("dataEndTime"))
|
| 1515 |
+
downloaded += 1
|
| 1516 |
+
operations["getReports"] = {
|
| 1517 |
+
"state": "done", "reports": len(discovered),
|
| 1518 |
+
"downloaded": downloaded, "skipped": skipped}
|
| 1519 |
+
if first_load and settlement_remaining:
|
| 1520 |
+
break
|
| 1521 |
+
windows = ()
|
| 1522 |
+
if table != "settlements":
|
| 1523 |
+
windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
|
| 1524 |
+
else _windows(plan["start"], plan["end"]))
|
| 1525 |
+
for w0, w1 in (() if table == "settlements" else windows):
|
| 1526 |
# W59-T43 / C2 has no amazon.access kind. Record each credential-backed
|
| 1527 |
# vendor attempt as a sync ACCESS phase, distinct from the completion event.
|
| 1528 |
# Lane F/combine own the vocabulary decision; no credential or row enters C2.
|
| 1529 |
+
documents += 1
|
| 1530 |
_sync_event("amazon.sync", tenant_key, table,
|
| 1531 |
{"state": "access", "operation": "report_document"}, rt, report, log)
|
| 1532 |
text, meta = cx.report_document(
|
|
|
|
| 1566 |
text, document_id=meta.get("reportDocumentId"), report_type=rtype)
|
| 1567 |
declared = {}
|
| 1568 |
if not cols:
|
| 1569 |
+
if first_load:
|
| 1570 |
+
# Even an empty recent window clears old rows exactly once.
|
| 1571 |
+
for empty_table in (table, sibling):
|
| 1572 |
+
if empty_table and floors[empty_table] is None:
|
| 1573 |
+
if con.execute("SELECT 1 FROM information_schema.tables WHERE table_name=?",
|
| 1574 |
+
[table_name(empty_table)]).fetchone():
|
| 1575 |
+
con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1")
|
| 1576 |
+
cleared = True
|
| 1577 |
+
else:
|
| 1578 |
+
continue
|
| 1579 |
+
if sibling:
|
| 1580 |
+
sibling_mirror = table_name(sibling)
|
| 1581 |
+
sibling_types = _ensure(con, sibling_mirror, cols, rows, declared=declared,
|
| 1582 |
+
preserve_rows=first_load) if cols else {}
|
| 1583 |
+
if cols and not sibling_cleared and not sibling_plan["incremental"]:
|
| 1584 |
+
con.execute(f"DELETE FROM {sibling_mirror} WHERE 1=1")
|
| 1585 |
+
sibling_cleared = True
|
| 1586 |
+
if cols:
|
| 1587 |
+
sibling_written += write_window(sibling, cols, sibling_types, rows)
|
| 1588 |
if table == "orders":
|
| 1589 |
rows = _dedupe_orders(rows)
|
| 1590 |
if table == "inventory":
|
|
|
|
| 1593 |
cols.append("inventory_source")
|
| 1594 |
if table == "pricing":
|
| 1595 |
pricing_asins.extend(_pricing_asins(rows))
|
| 1596 |
+
types = _ensure(con, mirror, cols, rows, declared=declared,
|
| 1597 |
+
preserve_rows=first_load) if cols else {}
|
| 1598 |
+
# Explicit windows and first loads replace the mirror; incremental passes keep history.
|
| 1599 |
+
if not cleared and not plan["incremental"]:
|
| 1600 |
con.execute(f"DELETE FROM {mirror} WHERE 1=1")
|
| 1601 |
cleared = True
|
| 1602 |
+
if cols:
|
| 1603 |
+
written += write_window(table, cols, types, rows)
|
| 1604 |
+
if first_load:
|
| 1605 |
+
for reached in (table, sibling):
|
| 1606 |
+
if reached:
|
| 1607 |
+
record_backfill_floor(con, reached, w0)
|
| 1608 |
+
floors[reached] = w0
|
| 1609 |
+
|
| 1610 |
+
if first_load:
|
| 1611 |
+
landed.add((table, task_start, task_end))
|
| 1612 |
+
if any(next_table == table for next_table, _w0, _w1 in work[index + 1:]):
|
| 1613 |
+
continue
|
| 1614 |
|
|
|
|
| 1615 |
if table == "pricing":
|
| 1616 |
+
if first_load:
|
| 1617 |
+
# Completion can be in another process/pass than the recent ASIN report.
|
| 1618 |
+
available = {row[1] for row in con.execute(f"PRAGMA table_info('{mirror}')").fetchall()}
|
| 1619 |
+
pricing_asins = list(dict.fromkeys(pricing_asins + [str(row[0])
|
| 1620 |
+
for col in ("child_asin", "parent_asin") if col in available
|
| 1621 |
+
for row in con.execute(f'SELECT DISTINCT "{col}" FROM {mirror} WHERE "{col}" IS NOT NULL').fetchall()]))
|
| 1622 |
state_key = ENTITY_STATE_KEYS[table]
|
| 1623 |
if not pricing_asins:
|
| 1624 |
_record_state(con, state_key, "unproven", 0,
|
|
|
|
| 1654 |
awd_rows = _awd_rows(inventory)
|
| 1655 |
if awd_rows:
|
| 1656 |
awd_cols = sorted({c for row in awd_rows for c in row})
|
| 1657 |
+
awd_types = _ensure(con, mirror, awd_cols, awd_rows, preserve_rows=first_load)
|
| 1658 |
+
if not cleared and not plan["incremental"]:
|
| 1659 |
con.execute(f"DELETE FROM {mirror} WHERE 1=1")
|
| 1660 |
cleared = True
|
| 1661 |
+
written += write_window(table, awd_cols, awd_types, awd_rows)
|
| 1662 |
_record_state(con, state_key, "done", len(awd_rows))
|
| 1663 |
operations["listInventory"] = {
|
| 1664 |
"state": "done", "rows": len(awd_rows)}
|
|
|
|
| 1675 |
column_count, total_rows = len(cols), written
|
| 1676 |
report["tables"][table] = {
|
| 1677 |
"rows": total_rows, "columns": column_count, "provenance": prov,
|
| 1678 |
+
"report": rtype, "window": plan["cause"], "incremental": plan["incremental"],
|
| 1679 |
+
"operations": operations}
|
| 1680 |
_record_state(con, key, "done", total_rows)
|
| 1681 |
_record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
|
| 1682 |
+
if not any(v["state"] in ("refused", "throttled") for v in operations.values()):
|
| 1683 |
+
record_watermark(con, table, plan["end"], total_rows)
|
| 1684 |
_sync_event("amazon.sync", tenant_key, table,
|
| 1685 |
{"state": "done", "rows": total_rows,
|
| 1686 |
"operations_ok": all(v["state"] == "done" for v in operations.values())},
|
| 1687 |
rt, report, log)
|
| 1688 |
log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}")
|
| 1689 |
+
if sibling:
|
| 1690 |
+
sibling_mirror = table_name(sibling)
|
| 1691 |
+
try:
|
| 1692 |
+
sibling_columns = len(con.execute(f"PRAGMA table_info('{sibling_mirror}')").fetchall())
|
| 1693 |
+
sibling_total = int(con.execute(f"SELECT COUNT(*) FROM {sibling_mirror}").fetchone()[0])
|
| 1694 |
+
except Exception:
|
| 1695 |
+
sibling_columns, sibling_total = len(cols), sibling_written
|
| 1696 |
+
sibling_prov = amazon_schema.PROVENANCE.get(sibling, "measured_at_sync")
|
| 1697 |
+
report["tables"][sibling] = {
|
| 1698 |
+
"rows": sibling_total, "columns": sibling_columns, "provenance": sibling_prov,
|
| 1699 |
+
"report": rtype, "window": sibling_plan["cause"],
|
| 1700 |
+
"incremental": sibling_plan["incremental"], "operations": {}}
|
| 1701 |
+
_record_state(con, entity_key(sibling), "done", sibling_total)
|
| 1702 |
+
_record_state(con, entity_key("sync_tick_" + sibling), "done", sibling_total)
|
| 1703 |
+
record_watermark(con, sibling, sibling_plan["end"], sibling_total)
|
| 1704 |
+
_sync_event("amazon.sync", tenant_key, sibling,
|
| 1705 |
+
{"state": "done", "rows": sibling_total, "operations_ok": True},
|
| 1706 |
+
rt, report, log)
|
| 1707 |
+
log(f" {sibling:<16} {sibling_total:>7,} rows {sibling_columns:>3} cols {sibling_prov}")
|
| 1708 |
except Exception as e: # noqa: BLE001
|
| 1709 |
+
# A failed pass retains its previous watermark so its window will be retried.
|
| 1710 |
# ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
|
| 1711 |
# ticket answering itself, and it must not cost the six tables that did work.
|
| 1712 |
+
problem = _sync_problem(e, table, operation, tenant_key, rt, report, log)
|
| 1713 |
+
failed.add(table)
|
| 1714 |
report["tables"][table] = {"rows": 0, **problem}
|
| 1715 |
_sync_event("amazon.sync", tenant_key, table,
|
| 1716 |
{"state": problem["state"], "rows": 0}, rt, report, log)
|
| 1717 |
+
if sibling:
|
| 1718 |
+
problem = _sync_problem(e, sibling, "report_document", tenant_key, rt, report, log)
|
| 1719 |
+
report["tables"][sibling] = {"rows": 0, **problem}
|
| 1720 |
+
_sync_event("amazon.sync", tenant_key, sibling,
|
| 1721 |
+
{"state": problem["state"], "rows": 0}, rt, report, log)
|
| 1722 |
+
if first_load:
|
| 1723 |
+
remaining = sum(task not in landed for task in work)
|
| 1724 |
+
if settlement_remaining:
|
| 1725 |
+
remaining += settlement_remaining - 1 # Replace the discovery slot with known documents.
|
| 1726 |
+
# Keep healthy tables moving after an isolated refusal. Once only failed work remains,
|
| 1727 |
+
# stop automatic continuation instead of endlessly retrying the same vendor refusal.
|
| 1728 |
+
continuable = any(task not in landed and task[0] not in failed for task in work)
|
| 1729 |
+
state = ("partial" if remaining and continuable else
|
| 1730 |
+
"failed" if remaining or report["problems"] or report["audit_problems"] else "done")
|
| 1731 |
+
progress = {"state": state, "remaining": remaining,
|
| 1732 |
+
"etaSeconds": first_load_eta(remaining, burst=0),
|
| 1733 |
+
"cause": first_load_eta_cause(remaining), "backfillDays": DEFAULT_BACKFILL_DAYS}
|
| 1734 |
+
report.update(progress)
|
| 1735 |
+
report["firstLoad"] = dict(progress)
|
| 1736 |
+
_record_state(con, entity_key(FIRST_LOAD_END_KEY), state, detail=_iso(end))
|
| 1737 |
+
_record_state(con, entity_key(FIRST_LOAD_STATUS_KEY), state,
|
| 1738 |
+
detail=json.dumps({key: progress[key] for key in ("remaining", "etaSeconds")}))
|
| 1739 |
report["finished"] = _iso(datetime.now(timezone.utc))
|
| 1740 |
con.close()
|
| 1741 |
return report
|
platform/harness/measure_filter.py
CHANGED
|
@@ -302,7 +302,7 @@ def _base_topic(key):
|
|
| 302 |
|
| 303 |
def _related_scope(related, keys):
|
| 304 |
"""Prepare all bases before querying: a refused composite must be wholly unscoped."""
|
| 305 |
-
if related is None or isinstance(related, _relations.MeasurePin):
|
| 306 |
return related
|
| 307 |
try:
|
| 308 |
pin = _relations.measure_pin(related)
|
|
@@ -673,6 +673,10 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=
|
|
| 673 |
con = _sem._store_con()
|
| 674 |
try:
|
| 675 |
totals = _totals(con, key, window, today, team_id, related)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 676 |
finally:
|
| 677 |
con.close()
|
| 678 |
scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
|
|
@@ -682,7 +686,8 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=
|
|
| 682 |
# round(x, None) returns an int — exactly the pool's `_round` convention for money/counts.
|
| 683 |
# The pin narrows SQL input only. Keep the caller's entire pool for zero reconstruction.
|
| 684 |
return _related_answer(
|
| 685 |
-
{int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe
|
|
|
|
| 686 |
|
| 687 |
|
| 688 |
#: The series channel's calendar buckets. `date_trunc` names in DuckDB; 'week' truncates to
|
|
|
|
| 302 |
|
| 303 |
def _related_scope(related, keys):
|
| 304 |
"""Prepare all bases before querying: a refused composite must be wholly unscoped."""
|
| 305 |
+
if related is None or (isinstance(related, _relations.MeasurePin) and related.refusal):
|
| 306 |
return related
|
| 307 |
try:
|
| 308 |
pin = _relations.measure_pin(related)
|
|
|
|
| 673 |
con = _sem._store_con()
|
| 674 |
try:
|
| 675 |
totals = _totals(con, key, window, today, team_id, related)
|
| 676 |
+
guard = None
|
| 677 |
+
metric = _sem.metrics().get(key, {})
|
| 678 |
+
if related is not None and not related.refusal and metric.get('agg') == 'ratio':
|
| 679 |
+
guard = _totals(con, metric['denominator'], window, today, team_id, related)
|
| 680 |
finally:
|
| 681 |
con.close()
|
| 682 |
scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
|
|
|
|
| 686 |
# round(x, None) returns an int — exactly the pool's `_round` convention for money/counts.
|
| 687 |
# The pin narrows SQL input only. Keep the caller's entire pool for zero reconstruction.
|
| 688 |
return _related_answer(
|
| 689 |
+
{int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe
|
| 690 |
+
if guard is None or guard.get(p)}, related)
|
| 691 |
|
| 692 |
|
| 693 |
#: The series channel's calendar buckets. `date_trunc` names in DuckDB; 'week' truncates to
|
platform/harness/runtime.py
CHANGED
|
@@ -126,6 +126,8 @@ class TenantRuntime:
|
|
| 126 |
tenant: object # harness.base.Tenant
|
| 127 |
store_namespace: str = ""
|
| 128 |
pool_cache: dict = field(default_factory=dict)
|
|
|
|
|
|
|
| 129 |
#: EXIT-5 — the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps
|
| 130 |
#: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on
|
| 131 |
#: (stamp, scope, pool identity, question) so nothing user-shaped lives in them.
|
|
@@ -179,6 +181,29 @@ class TenantRuntime:
|
|
| 179 |
"""
|
| 180 |
return self._store().get_projection(self.store_key(name), drop=drop)
|
| 181 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
def revision(self, name):
|
| 183 |
"""Return the tiny durable change token for this tenant's addressed bucket."""
|
| 184 |
return self._store().revision(self.store_key(name))
|
|
|
|
| 126 |
tenant: object # harness.base.Tenant
|
| 127 |
store_namespace: str = ""
|
| 128 |
pool_cache: dict = field(default_factory=dict)
|
| 129 |
+
# Names/edges only, bounded and revision-keyed by routes_grid; no row pools live here.
|
| 130 |
+
related_cache: dict = field(default_factory=dict)
|
| 131 |
#: EXIT-5 — the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps
|
| 132 |
#: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on
|
| 133 |
#: (stamp, scope, pool identity, question) so nothing user-shaped lives in them.
|
|
|
|
| 181 |
"""
|
| 182 |
return self._store().get_projection(self.store_key(name), drop=drop)
|
| 183 |
|
| 184 |
+
def get_projections(self, names, drop=(), timeout_s=None):
|
| 185 |
+
"""Batch projected reads when the active bound store has a native batch operation.
|
| 186 |
+
|
| 187 |
+
``None`` means the backend has no batch primitive; callers retain their ordinary per-key
|
| 188 |
+
path. That keeps this an optional optimization rather than widening the store contract.
|
| 189 |
+
"""
|
| 190 |
+
reader = getattr(self._store(), "get_projections", None)
|
| 191 |
+
if not callable(reader):
|
| 192 |
+
return None
|
| 193 |
+
keyed = [(str(name), self.store_key(name)) for name in (names or ())]
|
| 194 |
+
answer = reader([store_key for _name, store_key in keyed], drop=drop,
|
| 195 |
+
timeout_s=timeout_s)
|
| 196 |
+
return {name: answer.get(store_key, {}) for name, store_key in keyed}
|
| 197 |
+
|
| 198 |
+
def get_many(self, names):
|
| 199 |
+
"""Batch ordinary reads when the active bound store offers that optimization."""
|
| 200 |
+
reader = getattr(self._store(), "get_many", None)
|
| 201 |
+
if not callable(reader):
|
| 202 |
+
return None
|
| 203 |
+
keyed = [(str(name), self.store_key(name)) for name in (names or ())]
|
| 204 |
+
answer = reader([store_key for _name, store_key in keyed])
|
| 205 |
+
return {name: answer.get(store_key, {}) for name, store_key in keyed}
|
| 206 |
+
|
| 207 |
def revision(self, name):
|
| 208 |
"""Return the tiny durable change token for this tenant's addressed bucket."""
|
| 209 |
return self._store().revision(self.store_key(name))
|