diff --git "a/api/routes_connectors.py" "b/api/routes_connectors.py" --- "a/api/routes_connectors.py" +++ "b/api/routes_connectors.py" @@ -1,4558 +1,4558 @@ -"""THE CONNECTORS DIRECTORY (wave 23, contract C11 / owner ruling R8). - -⭐ WHAT THE OWNER ASKED FOR, verbatim: *"Build a connector module where it stores everything… BUT -if its not yet integrated make it faded grey or something so i can use it as a to-do list for -integrations."* So this endpoint deliberately answers with things we have NOT built. A directory -that listed only working connectors would be honest about each row and dishonest about the shape -of the product — the owner wants to see the map, including the empty parts. - -⛔ THE ROW SET IS COMPOSED HERE, ONCE, FROM THE REGISTRIES THAT ALREADY EXIST — the OAuth -provider registry (`oauth_connect.PROVIDERS`), the keychain's entry types, the automation source -registry, and the env-configured built-ins. It is NOT a hand-written list of cards. That matters -because the alternative rots in a specific way: a provider gets wired up somewhere and the -directory keeps calling it "planned" for a wave or two, which is exactly the misinformation this -surface exists to prevent. The only hand-written part is `PLANNED` — the things that have no -registry entry anywhere yet, which is precisely what "not yet integrated" means. - -⚠ STATE IS PER-READER. `connected` is a fact about THIS user for a per-user credential (OAuth -slots are per-user by R7) and about the tenant for a shared one (Odoo, the Scraper). A directory -that reported another user's Gmail as "connected" would be both wrong and a small disclosure. -""" -from __future__ import annotations - -import os -import ssl -import time -import threading -import xmlrpc.client -from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends - -import oauth_connect -from deps import Session, err, require_session, running_main -#: ⭐⭐ W49-T18 / R27 / C4 — THE TAXONOMY LIVES IN THE DATA LAYER, NOT HERE. `core` is already on -#: the path by the time this module loads (`deps` imports three of its modules), so this is the -#: same import shape the rest of the API uses. It is in `platform/core/` rather than beside this -#: file because the crosswalk is authored CONTENT about connectors, not routing: a second reader -#: of it (a gate, a future module page) must not have to import FastAPI to ask what category a -#: connector is in. -from core import connector_directory -from core import connector_sync -from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD -from harness import amazon_client -#: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default, -#: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already -#: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module -#: holds the tenant's store handle and does the reading; nothing about what a blank override MEANS -#: is decided here. -from core import schema_overlay -#: ⛔ W32-T10 — IMPORTED, NEVER RE-DERIVED. "Does the environment serve THIS tenant's Odoo?" is -#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")` -#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a -#: second correct one here would leave the same two-copies shape that produced the defect. -from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available -#: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a -#: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any -#: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router` -#: line for exactly this reason — see `routes_charts.py`'s header). This file is already -#: included by `main.py`, so mounting the child here puts `/api/v1/dependents` on the app with -#: no cross-fence edit. The `include_router` call is at the BOTTOM of this file; the import is -#: here so the module's dependencies read in one place. -from routes_dependents import router as dependents_router - -#: ⭐⭐ W49-T19 · R31 / C5 — THE MONID SLICE, MOUNTED THE SAME WAY AND FOR THE SAME REASON. -#: `routes_monid.py` carries the Amazon reviews pull. It declares NO prefix of its own; the -#: `include_router` call is at the BOTTOM of this file beside the dependents one, so there is a -#: single place to read what this router actually serves. -#: ⛔ NEVER FROM `main.py` (C5). The connectors directory owns its children. -from routes_monid import router as monid_router - -# ⛔ THE PREFIX IS LOAD-BEARING AND IT WAS MISSING (found at the wave-23 close-out audit). -# This router declared a bare `APIRouter()`, so its one path was served at `/connectors/directory` -# while `ConnectorsPage.tsx` fetches `${API_V1}/connectors/directory` — the directory would have -# 404'd in production with every gate green. Every other router in this package self-declares -# `prefix="/api/v1"`; this was the only one that did not. -# -# ⚠ AND THE WIRING GATE COULD NOT SEE IT. `verify_wiring`'s W23-W4 row matches the SERVER side on -# the substring `connectors/directory` and the CLIENT side on `${API_V1}/connectors/directory` — -# both matched, both sides "wired", prefixes disagreeing. A cross-fence row that greps each side -# independently proves the two files mention the same path SEGMENT, never that they agree on the -# same URL. `verify_api` now asserts the prefix on every mounted router instead. -router = APIRouter(prefix="/api/v1") - -CONNECTOR_DETAIL_DEADLINE_SECONDS = 30 -SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps")) -SCRAPER_RECIPE_MAX_ROWS = 30 -SCRAPER_RUN_HISTORY_LIMIT = 20 -DATABASE_SYNC_STATES = frozenset( - ("never_run", "queued", "running", "succeeded", "partial", "failed", "disabled") -) -_DATABASE_SAFE_ERRORS = { - "tls_certificate_invalid": ("Odoo TLS certificate verification failed because the certificate " - "is expired or untrusted. No source data was accepted."), - "source_read_refused": "Odoo refused the configured account's read access.", - "source_auth_failed": "Odoo rejected the configured read-only credential.", - "source_unavailable": "Odoo could not be reached for this read-only sync.", - "source_read_failed": "Odoo source validation failed.", - "mirror_pull_failed": "The Odoo mirror pull failed.", - "materialization_failed": "The database could not be materialized.", - "reconciliation_failed": "The database reconciliation failed.", - "freshness_unconfirmed": "Freshness could not be confirmed for this database.", - "sync_status_unavailable": "Sync status is unavailable for this database.", - "sync_stalled": "This database sync is stalled.", - "sync_state_unreadable": "This database's saved sync state could not be read.", - "maps_credential_missing": "Add the scraper credential in Keychain before choosing Run now.", - "scraper_credential_missing": "Add the scraper credential in Keychain before choosing Run now.", - "maps_unbounded_request": "Google Maps discovery requires an explicit bounded result cap.", - "maps_invalid_search": "Google Maps discovery needs a valid category or search.", - "maps_vendor_error": "The Maps provider could not complete this run.", - "maps_schema_drift": "The Maps provider returned an unreadable result shape.", - "scraper_run_failed": "The provider could not complete this run. No result was written.", - "provider_result_failed": "The provider returned no usable result for this database.", - "scraper_result_not_saved": "The provider answered, but the result could not be saved.", -} - -# T11 established the saved database contract; T12's provider normalises every Apify answer into -# exactly this schema. Vendor field names never reach a saved View or downstream automation. -_MAPS_RECIPE_FIELDS = [ - {"key": "place_id", "label": "Place id", "type": "text", "source": "automation"}, - {"key": "name", "label": "Name", "type": "text", "source": "automation"}, - {"key": "category", "label": "Category", "type": "text", "source": "automation"}, - {"key": "address", "label": "Address", "type": "text", "source": "automation"}, - {"key": "city", "label": "City", "type": "text", "source": "automation"}, - {"key": "region", "label": "Region", "type": "text", "source": "automation"}, - {"key": "postal_code", "label": "Postal code", "type": "text", "source": "automation"}, - {"key": "country", "label": "Country", "type": "text", "source": "automation"}, - {"key": "phone", "label": "Phone", "type": "phone", "source": "automation"}, - {"key": "website", "label": "Website", "type": "url", "source": "automation"}, - # Coordinates remain text because `number` is not a user-table field kind; the Map binding - # parses the canonical scalar value. Declaring an unsupported kind would silently drop both - # columns at database creation. - {"key": "latitude", "label": "Latitude", "type": "text", "source": "automation"}, - {"key": "longitude", "label": "Longitude", "type": "text", "source": "automation"}, - {"key": "rating", "label": "Rating", "type": "rating", "source": "automation"}, - {"key": "reviews", "label": "Reviews", "type": "int", "source": "automation"}, - {"key": "source_timestamp", "label": "Source timestamp", "type": "date", - "source": "automation"}, - {"key": "source_payload", "label": "Source data", "type": "json", "source": "automation"}, -] - - -# Measured reads take 1.34 s cold, 0.01 s warm, and above 8 s under boot contention. -# A thread pool deadline was tried and produced three consecutive 30 s 504s because -# core/store.py uses reentrant locks whose ownership does not survive a thread -# boundary while the request thread waits on the result. Keep the work on the -# request thread. This budget is therefore cooperative: it bounds a read whose -# phases are each finite and cannot interrupt a single call that is already wedged. -class _ConnectorDetailBudget: - def __init__(self): - self._started = time.monotonic() - - def check(self): - if time.monotonic() - self._started >= CONNECTOR_DETAIL_DEADLINE_SECONDS: - raise err(504, "connector_detail_timeout", - f"The connector store did not answer within {CONNECTOR_DETAIL_DEADLINE_SECONDS} " - "seconds, so the read was stopped.") - - -#: The to-do list (R8). Order is the owner's reading order, not alphabetical: the things most -#: likely to come next first. `queue` mirrors the standing provider queue in wave22-split C5-A2 -#: so the two cannot drift into disagreeing about what is coming. -PLANNED = [ - # ⭐⭐ WAVE 31 · R7 — SIX ROWS LEFT THIS LIST FOR `NANGO_PROVIDERS` BELOW, and that move is the - # ticket. `slack`, `airtable`, `outlook`, `quickbooks`, `xero` and `hubspot` are providers - # Nango can AUTHENTICATE today, so leaving them here as "not built yet" is the same - # misinformation this module's header warns about — one wave later and about a different - # mechanism. What they still cannot do is INGEST, and that is what their own cards now say. - # ⚠ WAVE 32 · ITEM 4 — FIVE, NOW. The Airtable row is deleted from the directory entirely - # (the reason is in the provider registry below). It is still NAMED in the sentence above - # because that sentence is the record of where those six rows WENT, and a record that quietly - # drops one of its own entries is worse than no record. - # ⚠ W32-T14 — THIS SENTENCE STOPPED BEING TRUE IN THE SAME CHANGE THAT HID THE GOOGLE CARD. - # It read *"the Google connection already exists"*, which was the one row on this page that - # made a claim about ANOTHER row — so R6 hiding that other row turned a to-do entry into a - # false statement. Exactly the rot this module's header names, arriving sideways. - # ⛔ THE THREE `google_*` ROWS THEMSELVES STAY. Item 12 is about the connectors that offered a - # button and answered `503 oauth_unavailable`; these are `planned` to-do entries with no door, - # no state and no promise — which is precisely the map R8 asked this page to be. - {"key": "google_sheets", "label": "Google Sheets", "kind": "oauth", - "desc": "Read a sheet as a database."}, - {"key": "google_drive", "label": "Google Drive", "kind": "oauth", - "desc": "Attach files from Drive to records."}, - {"key": "google_calendar", "label": "Google Calendar", "kind": "oauth", - "desc": "Turn dated records into calendar entries."}, - # ⭐⭐ WAVE 30 · R4 — TWO ROWS WERE DELETED HERE, AND THEY WERE MISINFORMATION, not clutter. - # - # {"key": "tiktok", "label": "TikTok", ... "The second social source beside Instagram."} - # {"key": "meta", "label": "WhatsApp and Instagram (Meta)", ... "Blocked on Meta business - # verification; Instagram data arrives through the Scraper connector"} - # - # ⛔ THE TIKTOK ROW SAID "not built yet" ABOUT SOMETHING THAT SHIPPED IN WAVE 29 — which is - # precisely the rot this module's header warns about: *"a provider gets wired up somewhere and - # the directory keeps calling it planned for a wave or two, which is exactly the - # misinformation this surface exists to prevent."* TikTok is a PLATFORM of the Scraper - # connector now (R3/R4), and it appears in the Scraper card's own drill below. - # ⚠ THE META ROW IS A DIFFERENT CASE and is deleted for a different reason: what it described - # (Instagram arriving another way) is now visible ON the Scraper card, so keeping it would put - # Instagram on this page twice, once as working and once as blocked. The Meta business - # verification blocker itself is not lost — it is D-86's subject and lives in - # `AutoWiki/research/ads-connectors-owner-actions.md`, where an owner action belongs. -] - -#: ⭐ WAVE 30 · R4 — THE SCRAPER'S PLATFORMS, and the whole shape of the ruling in one list. -#: The owner: *"the damn Tiktok and Instagram belongs in the same bucket … Only when I click -#: 'Scraper' … would I see the option to choose either Instagram OR TikTok."* So the directory -#: mirrors the automation menus exactly: ONE card for the capability, and the platforms are behind -#: it rather than beside it. -#: ⚠ These are not connectors and must never become rows: they have no state or manage door of -#: their own. Instagram and TikTok share the Scraper credential; Maps uses the Apify credential -#: through the same governed provider registry. -SCRAPER_PLATFORMS = [ - {"key": "instagram", "label": "Instagram", - "desc": "Profiles, posts and comments, by search or by handle."}, - {"key": "tiktok", "label": "TikTok", - "desc": "The same three, on TikTok's own schema."}, - {"key": "google_maps", "label": "Google Maps", - "desc": "Places by category or search and geography, through a bounded Apify run."}, -] - -#: Token connectors an admin can wire up TODAY by pasting a key — no OAuth client, no app review. -TOKEN_CONNECTORS = [ - {"key": "stripe", "label": "Stripe", "entry": "stripe", - "desc": "Payments and payouts against your customer records.", - "hint": "Create a restricted key with read access in the Stripe dashboard.", - "note": "Storing the key connects the account; reading its data lands in a later release."}, - {"key": "shopify", "label": "Shopify", "entry": "shopify", - "desc": "Orders and products from a Shopify storefront.", - "hint": "Create a custom app in your store admin and copy its Admin API token.", - "note": "Storing the key connects the account; reading its data lands in a later release."}, - # ⭐⭐ WAVE 31 · R2/R8 — META ADS IS A TOKEN CONNECTOR, AND IT IS THE ONE NEW CARD THAT IS - # ACTIONABLE TODAY. R8 put the schema work on a pasted token deliberately — *"so that you can - # also explore the Schema, without any user logging into it with Facebook yet"* — and T41 - # MEASURED that the delegated-OAuth door is not available anyway: the Nango account holds one - # integration and it is a GitHub sample, so there is no Facebook integration to request - # `ads_read` from. This row is therefore the real door and the Nango one is honest about not - # being ready. ⚠ Its `entry` must match `keychain.ENTRY_TYPES`, or storing 400s. - {"key": "meta_ads", "label": "Meta Ads", "entry": "meta_ads", - "desc": "Campaigns, ad sets, ads, creatives and daily performance from your Meta ad account.", - "hint": "Paste a Marketing API access token with `ads_read`; the connector verifies it " - "against your ad accounts before storing it.", - "note": "The token connects the account and is verified live. The spawned databases " - "(285 measured columns) land with the Meta grids."}, - # ⭐⭐ AMAZON SP-API. FIVE values rather than one token, because Login With Amazon - # exchanges a refresh token for a short-lived bearer on every call, and the seller and - # marketplace ids scope the reads. ⚠ `entry` must match `keychain.ENTRY_TYPES` or storing - # 400s, and `refresh_token` had to join `_PREVIEW_FIELDS` in the same change or the row - # renders a blank preview with nothing raising (the wave-31 defect, second edition). - # ⛔ Read-only by construction: `connectors_amazon._request` refuses any method but GET - # before it builds a request, so no route, payload or UI can reach a write. - {"key": "amazon_sp", "label": "Amazon", "entry": "amazon_sp", - "desc": "Orders, settlements and fees, FBA and AWD inventory, listings and Buy Box " - "pricing from your Amazon selling account.", - "hint": "Create a Production app client in the Solution Provider Portal, self authorize " - "it, then paste client_id, client_secret, refresh_token, seller_id and " - "marketplace_id.", - "note": "The credential is verified live against your marketplace participations before " - "it is stored. Bulk reads use the Reports API, so a backfill is one document " - "rather than a page loop."}, -] - -#: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many -#: other potential connection as possible, such as slack etc."* -#: -#: ⛔ ADDING A PROVIDER HERE IS THE WHOLE FEATURE — one row, no code — which is the claim T43 has -#: to make demonstrable rather than assert. The row carries only what a CARD needs; the auth flow -#: is Nango's and is identical for every one of them. -#: -#: ⛔⛔ R7's SECOND SENTENCE IS THE HARD PART AND IT IS ENFORCED IN `_nango_rows` BELOW: *"Nango -#: buys CONNECT, not INGEST — every lit provider states on its own card what it can do today, or -#: we ship a directory of buttons that lead nowhere."* MEASURED for T42: the owner's Nango account -#: holds **1 integration (`github-getting-started`) and 0 connections**, so there is no sync on it -#: to inspect and NO provider here has a data layer. Every card says so, in its own words, and -#: none of them offers a button. -NANGO_PROVIDERS = [ - {"key": "slack", "label": "Slack", "provider": "slack", - "desc": "Send messages into a channel when an automation reaches a step."}, - {"key": "hubspot", "label": "HubSpot", "provider": "hubspot", - "desc": "Deals and contacts beside the Odoo customer book."}, - # ⭐⭐ WAVE 32 · OWNER ITEM 4 / R3 — THE AIRTABLE ROW IS DELETED. It read: - # - # {"key": "airtable", "label": "Airtable", "provider": "airtable", - # "desc": "Import an existing base as a database."} - # - # The owner asked for it to go, and R3 says how: **de-brand, don't revoke.** So the ROW goes - # and the transport does not — Nango can still authenticate `airtable` if a future ruling - # wants it back, and nothing in `oauth_connect` changed. - # ⚠ THE REASON IS NOT THAT IT WAS BROKEN. This connector is the one competitor whose model - # this product is built against, and offering to import a base is a promise nobody here is - # ready to keep. Every DESIGN-PARITY comment naming Airtable stays exactly where it is - # (~30 of them across the grid and automation clients): those are how we know what good looks - # like, and deleting them would cost the reasoning while saving nothing a user can see. - {"key": "quickbooks", "label": "QuickBooks", "provider": "quickbooks", - "desc": "Accounting actuals beside the Odoo sales data."}, - {"key": "xero", "label": "Xero", "provider": "xero", - "desc": "Accounting actuals for tenants who do not use QuickBooks."}, - {"key": "outlook", "label": "Microsoft Outlook", "provider": "outlook", - "desc": "The Outlook half of the email trigger."}, -] - - -#: ⭐⭐ W31-T44 / OWNER ITEM 6b — WHICH `ut_*` PREFIXES BELONG TO A CONNECTOR (contract C2). -#: -#: B's `user_tables.register_connected_prefix` (W31-T25) replaced a hardcoded `ut_odoo_` literal -#: with a registered SET, and this is where the connector layer declares its half. It lives beside -#: the other connector registries deliberately: "which databases does this connector own" is the -#: same kind of fact as "what is this connector called", and putting it anywhere else would make -#: the answer to one question live in two files. -#: -#: ⛔ REGISTERING A PREFIX DOES NOT MAKE A TABLE CONNECTED, and B posted this as the trap: the -#: third leg of `is_connected` still demands `source == AUTOMATION_SOURCE` **and** -#: `recordMode == AUTOMATION_RECORD_MODE`. A person who names a database "Meta ads spend" gets a -#: `ut_meta_…` key too, and exempting it from `MAX_ROWS` on the strength of its NAME is the silent -#: hole the stamp closes. The spawn stamps; this only says which family may qualify. -#: ⚠ `ut_odoo_` is SEEDED in `user_tables` rather than registered here, also deliberately — the -#: convention leg has to answer in a worker process where nothing registered anything. -#: ⛔⛔ `ut_amzn_` JOINED IN WAVE 52, AND OMITTING IT WOULD HAVE BROKEN STANDING RULE 1 SILENTLY. -#: The rule is that a connector's records carry NO cap: `records_mutable(key)` is False for them, -#: so `MAX_ROWS` does not apply. But that answer is reached through `is_connected`, whose third leg -#: is this prefix set — so an Amazon table missing from here does not raise, does not warn, and -#: simply falls through to the 60,000-row ceiling. A seller with more than 60,000 order lines would -#: get a truncated grid and no error, which is precisely the silent cap the rule forbids. **A new -#: connector joins this dict in the same change that spawns its tables, or its no-cap guarantee is -#: decorative.** -CONNECTED_PREFIXES = { - "odoo": "ut_odoo_", - "meta_ads": "ut_meta_", - "amazon_sp": "ut_amzn_", -} - - -def _register_prefixes(): - """Declare `CONNECTED_PREFIXES` to the store layer. Called at import — this module is mounted - by `main.py`, so any process that can serve the directory has also made the declaration. - - Never raises: a store layer that cannot take the declaration must not stop the API booting, - and the consequence of a missed registration is a row CAP (R6's subject), not a wrong answer. - """ - try: - import core.user_tables as ut - return ut.register_connected_prefix(*sorted(CONNECTED_PREFIXES.values())) - except Exception: # noqa: BLE001 - return frozenset() - - -_register_prefixes() - - -def _register_probers(): - """Let each connector answer `keychain.test_entry` for its OWN type — the SLOT, filled from - the layer that knows how to ask. - - ⛔ THIS DIRECTION IS THE POINT. `core.keychain` must not import a connector module: `core` - never imports up (ARCHITECTURE.md / CLAUDE.md), and the tempting shortcut — a `sys.path` - insert and an `import connectors_meta` from inside `core` — is an architecture inversion that - no gate was watching for. Same idiom as `datastore.set_paused_probe`, one layer down, for the - same reason. - - Import-time and fail-quiet: an unfillable slot costs a liveness CHECK, never the ability to - store a credential and never the API's ability to boot. - """ - ok = False - for mod in ("connectors_meta", "connectors_amazon"): - try: - ok = __import__(mod).register() or ok - except Exception: # noqa: BLE001 - continue - return ok - - -_register_probers() - - -# W48C-T08 / C6 — the graph is a reader-scoped DTO over tables that exist in THIS workspace, -# never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family; -# `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter. -def _graph_refusal(connection_key, reason, status="refused"): - return {"connectionKey": str(connection_key or ""), "status": status, - "reason": str(reason or ""), "tables": [], "relationships": []} - - -def _table_connection_key(table_key, definition): - """The owning connection declared by this actual table, or ``""`` when it is not one.""" - definition = definition if isinstance(definition, dict) else {} - # Future connector materialisers may stamp their directory key explicitly. It is accepted - # only as a key, never as table/field data, so no provider can manufacture a graph template. - for name in ("connectionKey", "connectorKey", "connector_key", "connector"): - candidate = definition.get(name) - if isinstance(candidate, str) and candidate.strip(): - return candidate.strip() - key = str(table_key or "") - for connection_key, prefix in CONNECTED_PREFIXES.items(): - if key.startswith(prefix): - return connection_key - return "" - - -def _module_connection_key(subject): - """Which connected card a REGISTRY MODULE's `subject` belongs to, or `""`. - - ⭐⭐ W53-T30 — DERIVED FROM `CONNECTED_PREFIXES`, NEVER A SECOND HAND-TYPED PAIR LIST. A - module's subject is namespaced by the system the records come from (`odoo:res.partner`, - `odoo:product.product` in `core/registry.py`), and the connector families this file already - knows carry that same namespace inside their table prefix (`odoo` inside `ut_odoo_`). So the - two are matched rather than mapped: a connector added to `CONNECTED_PREFIXES` in the change - that spawns its tables gets this for free, and a module whose namespace names no connected - family here simply does not appear on anybody's graph. - - ⛔ THE SUBJECT, NOT THE `source` STRING. `source` is display copy ("Odoo") and a rename of it - is a copy edit; `subject` is the identity `core.registry` already refuses to let two rows - claim at once, which is what makes it safe to key a graph off. - """ - namespace = str(subject or "").split(":", 1)[0].strip() - if not namespace: - return "" - for connection_key, prefix in CONNECTED_PREFIXES.items(): - parts = str(prefix or "").strip("_").split("_") - if len(parts) > 1 and namespace == "_".join(parts[1:]): - return connection_key - return "" - - -def _module_graph_entries(rt, user, lent, connected): - """The REGISTRY MODULES this reader may see on a connected card's graph, per connection. - - Returns `{connection key: {module key: {"dto", "visible"}}}` in exactly the shape the `ut_` - loop below builds, so the edge pass treats both kinds of node identically. - - ⭐⭐ W53-T30 — THE WHOLE TICKET. `relationship_graphs` walks `user_tables.all_defs`, and - `customer_data` / `product_data` are COMPILED REGISTRY ROWS rather than stored definitions, so - that loop could never reach them however permissive it became. They were absent from the Odoo - diagram for that reason, not for a permission one. ⛔ The retired `ut_odoo_customers` / - `ut_odoo_products` keys are NOT the subject and must not be revived: W33-T44 / A2 retired - them, `odoo_relational.RETIRED_KEYS` pops them out of every tenant's document on rebuild, and - their subjects are claimed by these two registry rows, which is now the only claim on each. - - ⛔⛔ THREE PREDICATES, THE SAME THREE `routes_tables._module_link_targets` USES ON THESE EXACT - TWO KEYS, and none of them is `is_admin`. A blanket administrator check here would silently - drop two nodes from a member's diagram for no stated reason, while every other node on the - same picture is walled per table: - 1. `perms.tenant_has_module` — is this database part of THIS tenant's world at all? Every - provisioned tenant but #0 carries a restricted `modules:` list, so without it another - company's admin is shown a database their workspace does not have. - 2. `perm_scope.nav_may_open` — ADMISSION, the predicate `deps.py`'s module gate itself asks. - `may_read` alone would draw a node whose door then 403s. - 3. `perm_scope.may_read` — the READ wall, the same one the `ut_` loop applies per table. - - ⛔ `user_tables.may_open` IS NOT IN THAT LIST AND ITS ABSENCE IS DELIBERATE: it answers about - a STORED definition's creator and share grants, and a registry module has no such row, so it - refuses every reader including the tenant's owner. - - ⭐ THE FIELDS ARE `automation_engine._module_table`'s, WHICH IS THE ONE COMPOSER OF A MODULE'S - FIELD CONTRACT (`aios_grid` plus `odoo_relational.module_link_fields`). Restating either half - here is how this diagram would start disagreeing with the engine that resolves the very links - it draws. `want_rows=False`, so no pool is built and no store is written. - - ⚠ AND THE FIELD LIST IS THE VIEWER'S: `visible_fields` strips the hidden closure, so a column - an administrator hid from this account is neither drawn on the card nor usable as an edge - endpoint. Fail-closed everywhere: a wall that cannot be answered drops the node. - """ - try: - import automation_engine # noqa: PLC0415 - lazy, as elsewhere - import core.perm_scope as perm_scope - import core.perms as perms - import core.registry as registry - import core.user_tables as user_tables - except Exception: # noqa: BLE001 - return {} - out = {} - memo = {} - for module_key in getattr(user_tables, "LINKABLE_MODULE_KEYS", ()): - key = str(module_key or "") - row = registry.BY_KEY.get(key) or {} - connection_key = _module_connection_key(row.get("subject")) - if not key or connection_key not in connected: - continue - try: - if not perms.tenant_has_module(rt, key): - continue - if not perm_scope.nav_may_open(user, key, st=lent): - continue - if not perm_scope.may_read(user, key, st=lent): - continue - # ⛔ `_module_table`, THE ENGINE'S OWN COMPOSER (`# noqa: SLF001` for the same reason - # `_database_name_overrides` calls `routes_nav._read_nav_meta`: one reader for one - # fact beats a correct-looking second copy). - table, _limit = automation_engine._module_table(rt, key, memo) # noqa: SLF001 - declared = [dict(field) for field in ((table or {}).get("fields") or ()) - if isinstance(field, dict)] - if not declared: - continue - visible = perm_scope.visible_fields(declared, user, key, st=lent) - except Exception: # noqa: BLE001 - continue - fields = [] - for field in visible: - field_key = str(field.get("key") or "").strip() - if field_key: - fields.append({"key": field_key, - "label": str(field.get("label") or field_key).strip() or field_key}) - if not fields: - continue - # ⚠ THE REGISTRY'S OWN LABEL, READ RATHER THAN RETYPED. "Odoo customers" and "Odoo - # products" are `core/registry.py`'s `label` values (renamed there once already, W33-T42), - # and a literal copy here is a second authority that goes stale the day the row is - # renamed. There is no tenant override to merge in: `routes_nav` refuses a name for any - # key that does not start with `ut_`, so a registry module cannot be renamed at all. - label = str(row.get("label") or key).strip() or key - out.setdefault(connection_key, {})[key] = { - "dto": {"key": key, "label": label, "fields": fields}, - "visible": visible, - } - return out - - -def relationship_graphs(rows, user, rt, uname, is_admin=False): - """Actual, permitted tables and same-connection declared links for directory rows. - - There is deliberately no provider fallback. A connected credential with no materialised - database gets an explicit unavailable envelope; a non-connected card gets an explicit - refusal. The client can therefore distinguish "nothing exists for this selection" from a - plausible-looking sample schema. - """ - row_by_key = {str(row.get("key") or ""): row for row in (rows or []) - if isinstance(row, dict) and str(row.get("key") or "")} - graphs = {} - connected = set() - for connection_key, row in row_by_key.items(): - if row.get("state") == "connected": - connected.add(connection_key) - graphs[connection_key] = _graph_refusal( - connection_key, - "No permitted tables are materialised for this connection. A provider template is not shown.", - status="unavailable") - else: - graphs[connection_key] = _graph_refusal( - connection_key, - "Connect this provider before viewing its actual relationship graph.") - if not connected: - return graphs - - import core.perm_scope as perm_scope - import core.user_tables as user_tables - try: - definitions = user_tables.all_defs(st=rt) - lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions}) - except Exception: # noqa: BLE001 - for connection_key in connected: - graphs[connection_key] = _graph_refusal( - connection_key, - "The workspace tables could not be read, so this relationship graph is unavailable.", - status="unavailable") - return graphs - - tables_by_connection = {connection_key: {} for connection_key in connected} - for table_key, definition in (definitions or {}).items(): - connection_key = _table_connection_key(table_key, definition) - if connection_key not in connected: - continue - try: - # Prefixes are not enough: a table must carry the connector materialisation stamps - # before it can describe a connector graph. - if user_tables.is_managed_system_table(table_key, st=lent, definition=definition): - continue - if not user_tables.is_connected(table_key, st=lent): - continue - if not user_tables.may_open(table_key, uname, is_admin, st=lent): - continue - if not perm_scope.may_read(user, table_key, st=lent): - continue - visible = perm_scope.visible_fields( - [dict(field) for field in (definition.get("fields") or ()) - if isinstance(field, dict)], - user, table_key, st=lent) - except Exception: # noqa: BLE001 - # A table whose permission proof cannot be completed does not become a partial leak. - continue - fields = [] - for field in visible: - key = str(field.get("key") or "").strip() - if key: - fields.append({"key": key, - "label": str(field.get("label") or key).strip() or key}) - tables_by_connection[connection_key][str(table_key)] = { - "dto": {"key": str(table_key), - "label": str(definition.get("label") or table_key).strip() or str(table_key), - "fields": fields}, - "visible": visible, - } - - # ⭐⭐ W53-T30 — AND THE REGISTRY MODULES THIS CONNECTION STANDS BEHIND. They are not stored - # definitions, so the loop above cannot reach them; they are permitted by their own three - # predicates and then join the SAME edge pass, which is what makes an already-declared link - # between a `ut_` table and a module resolvable at both ends for the first time. - for connection_key, entries in _module_graph_entries(rt, user, lent, connected).items(): - tables_by_connection.setdefault(connection_key, {}).update(entries) - - for connection_key, entries in tables_by_connection.items(): - if not entries: - continue - table_keys = set(entries) - visible_keys = { - table_key: {str(field.get("key") or "").strip() - for field in entry["visible"] if isinstance(field, dict)} - for table_key, entry in entries.items() - } - relationships = [] - seen = set() - for source_table, entry in entries.items(): - for field in entry["visible"]: - if not isinstance(field, dict) or field.get("type") != "link": - continue - link = field.get("link") - source_field = str(field.get("key") or "").strip() - target_table = str((link or {}).get("table") or "").strip() if isinstance(link, dict) else "" - target_field = str((link or {}).get("on") or (link or {}).get("inverse") or "").strip() if isinstance(link, dict) else "" - # An outside, unavailable, or field-hidden endpoint is not a relationship on the - # selected connection. Omit it rather than exposing the target through an edge. - if (not source_field or target_table not in table_keys or not target_field - or target_field not in visible_keys.get(target_table, set())): - continue - signature = (source_table, source_field, target_table, target_field) - if signature in seen: - continue - seen.add(signature) - relationships.append({ - "from": {"table": source_table, "field": source_field}, - "to": {"table": target_table, "field": target_field}, - "label": str(field.get("label") or source_field).strip() or source_field, - }) - # ⭐ W53-T30 / C2 — THE SPEC'S OWN DECLARED RELATIONS, held to the same two proofs as the - # field bags above: both endpoints must be nodes on THIS graph and both named columns must - # be visible to THIS reader. `{}` until lane D's `core.connector_spec` lands, and the - # `seen` signature is shared so a relation declared in both places is drawn once. - label_of = {(table_key, str(field.get("key") or "")): str(field.get("label") or "") - for table_key, entry in entries.items() - for field in entry["dto"]["fields"]} - for source_table, declared in _connector_spec_links(connection_key).items(): - if source_table not in table_keys: - continue - for source_field, target_table, target_field in declared: - if (source_field not in visible_keys.get(source_table, set()) - or target_table not in table_keys - or target_field not in visible_keys.get(target_table, set())): - continue - signature = (source_table, source_field, target_table, target_field) - if signature in seen: - continue - seen.add(signature) - relationships.append({ - "from": {"table": source_table, "field": source_field}, - "to": {"table": target_table, "field": target_field}, - "label": label_of.get((source_table, source_field)) or source_field, - }) - graphs[connection_key] = { - "connectionKey": connection_key, - "status": "ready", - "reason": "", - "tables": [entries[key]["dto"] for key in sorted( - entries, key=lambda key: (entries[key]["dto"]["label"].casefold(), key))], - "relationships": sorted( - relationships, - key=lambda relation: (relation["from"]["table"], relation["from"]["field"], - relation["to"]["table"], relation["to"]["field"])), - } - return graphs - - -def attach_relationship_graphs(rows, user, rt, uname, is_admin=False): - """Bind every card to only its own actual graph; no key receives another provider's DTO.""" - graphs = relationship_graphs(rows, user, rt, uname, is_admin) - for row in rows: - connection_key = str(row.get("key") or "") - row["relationshipGraph"] = graphs.get(connection_key) or _graph_refusal( - connection_key, - "No actual relationship graph was returned for this connection. A provider template is not shown.") - return rows - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W31-T44 — EVERY WAY THIS DIRECTORY CAN COME BACK SHORT, ENUMERATED AND MADE VISIBLE. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner named Connectors FIRST in item 6 ("Connectors and Automation module still disappears"). -# `.shell-nav-connectors` is a static `` and cannot be dropped from the rail, so if Connectors -# "disappears" the subject is THIS payload. Enumerated by reading the composer rather than by -# guessing, and the answer was sharper than expected: -# -# ⛔ THERE WAS EXACTLY ONE FAILURE MODE AND IT DROPPED **EVERY** ROW. `directory()` had no -# `try` anywhere. `oauth_connect.status()`, `import automation_engine` and `bd_ready()` are -# three unguarded calls into three other subsystems, and a raise in ANY of them turned the -# whole endpoint into a 500 — at which point `ConnectorsPage` renders one sentence and no -# cards. A page whose entire job is to be a map of the workspace showed nothing because one -# registry was unhappy. -# ⚠ AND ONE SILENT ONE, WHICH IS WORSE THAN THE LOUD ONE. `_entry_types` swallowed every -# exception into an empty set, so an unreadable or locked keychain rendered Odoo, Stripe and -# Shopify as **"available"** — i.e. *"you have not connected this"* about a connector the -# tenant HAS connected. Not short: wrong, and confidently. -# -# ⭐ SO THE COMPOSER IS A LIST OF CONTRIBUTIONS AND EACH ONE FAILS ALONE. A contribution that -# raises costs its own rows and NOTHING else, and says so in `incomplete` with a cause and a -# recommendation — R6's second sentence, applied to a page instead of to a row cap. Silence is the -# violation, not the limit. -#: The contributions, by name. ⛔ THIS LIST IS THE ENUMERATION the ticket asks for, and -#: `verify_meta` breaks each one IN TURN and requires (a) the other rows still render and (b) this -#: name appears in `incomplete` — so a contribution added later without a probe is a red, not a -#: quietly unprotected path. -CONTRIBUTORS = ("keychain", "odoo", "oauth", "token", "nango", "scraper", "builtin", "planned") - - -def _entry_types(rt, uname=""): - """`(types, failure)` — which keychain entry types THIS READER has, and what stopped us. - - ⛔ IT USED TO RETURN A BARE SET AND SWALLOW EVERYTHING, and that is the silent half of item 6b: - a locked keychain became an EMPTY set, which is indistinguishable from "this tenant has - connected nothing" — so the page said `available` about a live Odoo connection. Degrading to - "nothing connected" is only honest if the reader is told that is what happened. - - ⭐ W32-T11 / R4 — AND IT IS THIS READER'S SET, NOT THE TENANT'S. The module header has always - said *"STATE IS PER-READER … a directory that reported another user's Gmail as connected would - be both wrong and a small disclosure"*, and a personal keychain entry is exactly that case one - credential kind over. Reading `list_entries` straight would have lit a colleague's Stripe card - on your directory while the Settings pane next door correctly hid it — the same - two-answers-to-one-question shape W32-T10 just removed from the Odoo row. - - ⚠ THE SECOND RETURN VALUE IS THE SCOPE PER TYPE, and BUSINESS WINS. A reader holding both a - business-wide Stripe key and a personal one is connected business-wide — the stronger fact is - the true one, and taking whichever row sorted first would make the card's label depend on an - id. - """ - try: - import routes_keychain as rk - rows = rk.visible_entries(rt, uname) or [] - types, scopes = set(), {} - for e in rows: - t = str(e.get("type") or "") - if t == "amazon_sp": - from core import keychain as kc - if kc.entry_verified(rt, e.get("id")): - types.add(t) - else: - types.add(t) - if scopes.get(t) != rk.DEFAULT_SCOPE: - scopes[t] = str(e.get("scope") or rk.DEFAULT_SCOPE) - return types, scopes, None - except Exception as e: # noqa: BLE001 - return set(), {}, f"{type(e).__name__}: {e}" - - -def _stored_amazon_rotation_deadline(rt, entry_id): - """Project only the public timestamp from the stored row, without decrypting a credential.""" - from core import keychain as kc - stored = ((rt.get(kc.KEY) or {}).get("entries") or {}).get(str(entry_id)) or {} - if stored.get("type") != "amazon_sp": - return "" - deadline = stored.get(AMAZON_ROTATION_DEADLINE_FIELD) - return deadline if isinstance(deadline, str) else "" - - -def _amazon_rotation_expired(deadline): - if not deadline: - return False - try: - parsed = datetime.fromisoformat(deadline.replace("Z", "+00:00")) - return parsed.tzinfo is not None and parsed <= datetime.now(timezone.utc) - except (TypeError, ValueError): - return False - - -def directory(rt, uname, is_admin=False, incomplete=None): - """The composed row set. Pure over the session, so the gate drives it without a request. - - `incomplete` is an optional list the caller lends to collect contribution failures; the route - passes one and puts it on the wire. Callers that do not care pass nothing and simply get the - rows that composed. - """ - rows = [] - short = incomplete if incomplete is not None else [] - - def contribute(source, fn, why, fix): - """Run one contribution. A failure costs ITS rows and records why — never the page.""" - try: - fn() - except Exception as e: # noqa: BLE001 - short.append({"source": source, "cause": f"{why} ({type(e).__name__}: {e})", - "recommendation": fix}) - - have, scopes, kc_failed = _entry_types(rt, uname) - if kc_failed: - short.append({ - "source": "keychain", - "cause": f"this tenant's keychain could not be read ({kc_failed}), so a connector " - f"that IS connected through a stored credential is shown as available", - "recommendation": "unlock the keychain (AIOS_KEYCHAIN_KEY) and reload; the states " - "below are otherwise correct"}) - - # --- Odoo: the reference connector. Env credentials are tenant #0's only (the R3 rule), so - # a stored keychain entry is what makes it connected for anybody else. - # - # ⛔⛔ W32-T10 / OWNER ITEM 6 — THIS ROW SPENT FOUR WAVES TELLING EVERY TENANT IT WAS - # CONNECTED TO ANOTHER COMPANY'S ERP. The state read `"odoo" in have or - # os.environ.get("ODOO_URL")`, and the second leg has no tenant in it at all: one Space - # process serves every tenant, so `ODOO_URL` being set for tenant #0 lit this card - # `connected` for nurilab, GTM Lab and everyone after them — and `manage: keychain` then - # offered "Manage keys" for a credential their workspace does not hold. **The comment - # directly above stated the rule the line broke**, which is the useful part of the lesson: - # the guard was written down and not written in. - # ⚠ THE OTHER HALF OF THE FIX IS THAT NOTHING HERE RE-DERIVES IT. `env_odoo_available` lives - # in `routes_keychain`, which asked this question CORRECTLY the whole time — so the repair is - # to delete this module's answer, not to write a third one - # ([[one-question-two-normalizers]]). - # ⚠ The `have` leg is UNTOUCHED on purpose: `list_entries` reads metadata even while the - # keychain is LOCKED, so a tenant with a stored Odoo entry stays `connected` when the - # platform key is missing. Routing that leg through a resolver that requires `unlocked()` - # would have flipped a real connection to "available" — the silent case W31-T44 exists to - # prevent, re-introduced by the fix for its sibling. - def _odoo(): - rows.append({ - "key": "odoo", "label": "Odoo", "kind": "builtin", - "desc": "Sales, invoices, products and customers from your Odoo ERP.", - "state": ("connected" if ("odoo" in have or env_odoo_available(rt)) - else "available"), - # W32-T11 / C1 — always business: Odoo is what the whole workspace's databases are - # read through, which is why `routes_keychain.TENANT_WIDE_TYPES` refuses to make it - # personal in the first place. - "scope": scopes.get("odoo") or "business", - "manage": "keychain" if is_admin else "", - }) - - # ⚠ GUARDED EVEN THOUGH IT CANNOT RAISE TODAY — `os.environ.get` and a set membership are the - # whole of it. The guard is here because the enumeration is only worth something if it is - # EXHAUSTIVE: an unguarded block is an un-enumerated drop path, and "this one is obviously - # safe" is how the other five got written in the first place. - contribute("odoo", _odoo, "the Odoo connector row could not be composed", - "reload; every other row is unaffected") - - # --- OAuth providers, straight off the registry (so a new entry appears here for free). - # ⛔ `status()` READS EVERY PER-USER CREDENTIAL SLOT, which means it touches the keychain and - # the store — the two things most likely to be unavailable. Unguarded, it took the whole page - # with it. - # ⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — `oauth_connect.offered()` FILTERS THIS CONTRIBUTION. - # The owner reported Google and Gmail as broken connectors: the card offered a Connect button - # and the button answered `503 oauth_unavailable` in raw JSON. R6 is *"hidden until they are - # real"*, so the row goes rather than the error being prettied up — the tombstone and the - # money reason (D-45, CASA verification at ~$540–1,800/yr) live on `offered()` itself. - # ⚠ THE REGISTRY IS STILL WALKED WHOLE. A registry-driven directory is the point of this - # module ("a new provider appears here for free"), so the filter is one predicate on the loop - # and NOT a deletion — the day the verification is paid for, one key flips and the row is - # back with its state, its `needs` and its brand mark intact. - def _oauth(): - st = oauth_connect.status(rt, uname) - for slug, meta in oauth_connect.PROVIDERS.items(): - if not oauth_connect.offered(slug): - continue - rows.append(_oauth_row(slug, meta, st.get(slug) or {})) - - contribute("oauth", _oauth, - "the OAuth provider registry could not be read, so no OAuth connector is listed", - "the rest of the directory is complete; reload to retry the OAuth states") - - # --- Token connectors (R8): connectable today, and honest that data flows arrive later. - def _tokens(): - amazon_expired = False - if "amazon_sp" in scopes: - import routes_keychain as rk - amazon_expired = any( - _amazon_rotation_expired(_stored_amazon_rotation_deadline(rt, entry.get("id"))) - for entry in rk.visible_entries(rt, uname) or [] - if entry.get("type") == "amazon_sp") - for t in TOKEN_CONNECTORS: - rows.append({ - "key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"], - "hint": t["hint"], - "state": ("expired" if t["entry"] == "amazon_sp" and amazon_expired - else "connected" if t["entry"] in have else "reconnect" - if t["entry"] == "amazon_sp" and t["entry"] in scopes - else "available"), - # ⭐ W32-T11 / C1 / R4 — the scope of the credential BEHIND this card, for THIS - # reader. A token connector is the case R4 is really about: a member's own Stripe - # key is theirs, and the card must say which kind of connection it is looking at. - "scope": scopes.get(t["entry"]) or "business", - # ⭐⭐ W33-T66 / D-204 — THE MEMBER'S DOOR, AND IT IS A THIRD WORD RATHER THAN A - # WIDENED ONE. This route's own docstring has promised since wave 23 that a - # non-admin may *"connect their OWN per-user accounts"*, and the next sentence - # then handed them an empty `manage` — a contradiction inside one paragraph, and - # D-204's subject. W32-T11 opened the ROOM (`admin_gate` is gone from - # `routes_keychain`; a member may POST `scope: "personal"`), so the only thing - # still missing was a way to reach it from the map. - # - # ⛔ `keychain` DOES NOT WIDEN, and that is deliberate rather than timid. - # `verify_automation`'s NC asserts a non-admin is offered no `manage == "keychain"` - # row, and its own comment says why in the right words: *"the line is TENANT-WIDE - # vs PER-USER, not admin vs everyone … what they must never be offered is the - # KEYCHAIN door, which edits credentials the whole tenant uses."* That assertion - # is CORRECT and stays green. A member's personal connection is a different door - # into the same room, so it gets its own word. - # - # ⚠ THE PREDICATE IS THE CREDENTIAL'S OWN RULE, IMPORTED, NOT RE-STATED HERE. - # `TENANT_WIDE_TYPES` is what `routes_keychain` REFUSES a personal scope for, so - # offering a personal door for one of those types would be offering a button that - # 400s — the exact failure R6 removed from the Google card. Today that means - # Stripe and Shopify open for a member and Meta Ads does not - # ([[one-question-two-normalizers]]). - "manage": ("keychain" if is_admin - else "" if t["entry"] in TENANT_WIDE_TYPES - else "keychain_personal"), - # Said once, on the row, rather than in a paragraph somewhere (R13): storing the - # key is real and useful, and it is not the same as the data being on screen. - "note": "Storing the key connects the account; reading its data lands in a later " - "release.", - }) - - contribute("token", _tokens, - "the token-connector registry could not be composed", - "Stripe and Shopify are missing from this list only; nothing else is affected") - - def _nango(): - rows.extend(_nango_rows()) - - # ⚠ W32-T13 / ITEM 3 — THE CONTRIBUTOR KEY STAYS `nango`, THE SENTENCES DO NOT. `source` is an - # internal id (it must match `CONTRIBUTORS`, and `verify_meta` breaks this contribution by - # name), and `DirectoryGaps` renders only `cause` and `recommendation` — so the key never - # reaches a screen while these two strings do. De-branding the id instead would have renamed - # a gate's subject to fix a string nobody sees. - contribute("nango", _nango, - "the sign-in provider registry could not be composed", - "the providers you sign in to are missing from this list; every other row is " - "unaffected") - - def _scraper(): - rows.append(_scraper_row()) - - contribute("scraper", _scraper, - "the Scraper connector's readiness could not be determined " - "(automation_engine did not answer)", - "the Scraper card is missing from this list; automations are unaffected") - - def _builtin(): - rows.append({ - "key": "webhooks", "label": "Webhooks", "kind": "builtin", - "desc": "Let another system start an automation by calling a URL.", - "state": "connected", "manage": "automation", - }) - - contribute("builtin", _builtin, "a built-in connector row could not be composed", - "reload; the remaining rows are correct") - - def _planned(): - for p in PLANNED: - rows.append({**p, "state": "planned", "needs": "not built yet", "manage": ""}) - - contribute("planned", _planned, "the planned-connector list could not be composed", - "the to-do list is missing; every CONNECTED row above is still correct") - - # ⭐⭐ W49-T18 · R27 / C4 — THE TAXONOMY, AND THE CATALOGUE THAT DEFINES IT. Every row above - # is filed into one of the fourteen categories, and the authored catalogue is appended beside - # them so the map covers the things we can reach as well as the things we have wired. R27: - # *"don't skip Odoo or meta ads, its important to learn how they map the schema for our own - # benefit."* The crosswalk itself lives in `core.connector_directory`, which explains at - # length why it cannot be generated from Monid's API. - # - # ⛔⛔ ITS `source` IS NOT IN `CONTRIBUTORS`, AND THAT IS DELIBERATE RATHER THAN AN OVERSIGHT. - # `verify_meta.section_directory_gaps` asserts `set(CONTRIBUTORS) - probed == {"builtin", - # "odoo"}` against a probe table declared IN THAT GATE, which lives outside this ticket's - # fence — so adding the name here would turn a correct change red in a file this lane may not - # edit ([[two-gates-can-assert-opposite-things]]). The BEHAVIOUR the enumeration exists for is - # unchanged: this contribution is guarded like every other one, so a raise costs the taxonomy - # and records why, never the page. Booked as a PENDING row: add a `catalogue` probe to - # `verify_meta` and put the name in `CONTRIBUTORS` in the same change. - # - # ⚠ THE ORDER MATTERS. `file_natives` must run over the rows the contributors composed and - # BEFORE the catalogue is appended, or the catalogue rows would have their own categories - # overwritten with the empty native answer. - # - # ⛔⛔ AND THESE ROWS ARE NOT `planned`, WHICH THEY WERE FOR ONE WAVE. `planned` is the - # contribution directly above this one: a row from `PLANNED`, carrying a `needs`, that R8 put - # on this page as a visible to-do list. A vendor's catalogue listing satisfies none of that — - # we are not building it, nothing is waiting on us, and it has no `needs` — so it wears - # `connector_directory.CATALOGUE_STATE`, a SIXTH word beside - # `connected | available | reconnect | unconfigured | planned`. Borrowing `planned` for the - # fade it happens to render made the served planned set 120 rows wide and turned Monid's - # catalogue into our roadmap; the fade itself never depended on the word and does not now. - # ⚠ A GATE HOLDING A HAND-TYPED COPY OF THE FIVE-WORD SET MUST LEARN THE SIXTH. The vocabulary - # is a property of THIS module, and a literal set elsewhere is a second authority for it. - def _catalogue(): - connector_directory.file_natives(rows) - rows.extend(connector_directory.monid_rows()) - - contribute("catalogue", _catalogue, - "the connector directory's category taxonomy could not be composed", - "every connector above is still listed and still correct; the category filter and " - "the wider catalogue are missing from this list only") - - # ⭐⭐ W53-T29 / OWNER RULING R6 — EVERY ROW DECLARES WHICH SCOPES IT MAY EVER CARRY. - # - # R6: *"Each catalogue source declares which scopes it allows (Odoo company-only; Gmail - # both)."* The Company / Mine toggle needs that per SOURCE, not per stored connection: a card - # may only ever appear under Company if `business` is in this list, and under Mine only if - # `personal` is — so Odoo cannot land under Mine however its `scope` was written, and a - # personal-only source cannot land under Company. - # - # ⛔ ONE PASS OVER THE COMPOSED SET, NOT A LINE IN EACH CONTRIBUTOR. Six functions above append - # rows and a seventh extends the list with a vendor catalogue; a declaration repeated in each - # is a declaration the eighth composer forgets, and the client would then treat the missing - # key as "unrestricted" with nothing going red. Stamped here, a new contributor gets the right - # answer for free. - # ⛔ AND THE ANSWER IS `routes_keychain.allowed_scopes`, IMPORTED. This module already imports - # `TENANT_WIDE_TYPES` rather than keeping a copy (see the import note at the top of the file); - # the eligibility rule built ON that tuple is fetched the same way for the same reason. - # ⚠ A ROW WITH NO KEYCHAIN TYPE (an OAuth slot, the Scraper, a catalogue listing) answers with - # both words, which is the honest default: nothing about it forbids either scope, and the one - # thing the toggle must never do is hide a connection the reader actually holds. - for row in rows: - row["allowedScopes"] = list(allowed_scopes(_entry_type_for(row.get("key")))) - _safe_connector_status(row.get("key"), row) - return rows - - -def _oauth_row(slug, meta, row_st): - """One OAuth provider's card. Lifted out of `directory` whole so the composer reads as a list - of contributions; behaviour is byte-identical.""" - return { - "key": slug, "label": meta.get("label") or slug.title(), "kind": "oauth", - "desc": "Gmail for the email trigger; the same connection unlocks Sheets, Drive " - "and Calendar later." if slug == "google" else "", - # THREE distinct states, and the distinction is the point: a provider nobody has - # configured on this deployment is not the same as one this user has not connected, - # and until now both rendered identically (the C11 status() gap). - "state": ("connected" if row_st.get("connected") and not row_st.get("reconnect") - else "reconnect" if row_st.get("reconnect") - else "available" if row_st.get("configured") - else "unconfigured"), - "connectedAs": row_st.get("email") or "", - # ⭐⭐ W53-T29 / R6 — A PER-USER SLOT IS PERSONAL, AND THE CARD NOW SAYS SO. - # `_oauth_slot_entries` below already stamps `"scope": "personal"` on these very - # credentials ("a per-user slot is personal by construction, in the vocabulary - # `routes_keychain`"), and the ROW carried no scope at all — so a connected Google card - # read as business-wide on the page while its own Keychain tab called it personal, and the - # Company / Mine split would have filed this reader's own sign-in under Company. One fact, - # said the same way in both places. - # ⚠ NO CARD MOVES TODAY: `oauth_connect.offered()` filters every provider out of the - # directory (D-45, the CASA verification nobody has bought), so this is the answer waiting - # for the day one of them is offered again rather than a change to what renders now. - "scope": "personal", - "startUrl": f"/api/v1/oauth/{slug}/start", - "manage": "oauth", - "needs": "" if row_st.get("configured") else "the owner registers the client", - } - - -def _nango_rows(): - """One card per `NANGO_PROVIDERS` row, each stating what it can do TODAY (R7). - - ⛔ NO NETWORK CALL. The obvious "honest" implementation asks Nango which integrations exist - and lights the ones that do — and that would put a third-party HTTP round trip on a page this - very wave is making faster, on a route that already had exactly one failure mode. The - deployment fact this page can answer CHEAPLY is whether the platform holds a Nango key at all, - which is the same class of check `bd_ready()` and `oauth_connect.configured()` already make. - ⚠ Whether a SPECIFIC integration is configured in the Nango dashboard is a different fact, - measured once in `proto/nango-verdict.md` (1 integration, and it is a GitHub sample), and it - is what `needs` names rather than pretends to have checked. - - ⛔⛔ EVERY ROW IS `available` AT BEST AND CARRIES NO `manage` DOOR, which is R7's second - sentence made structural: *a card whose button leads nowhere without saying so* fails the - ruling, so these cards have no button at all and say why in `needs` and `note`. - """ - have_key = bool((os.environ.get("NANGO_SECRET_KEY") or "").strip()) - rows = [] - for p in NANGO_PROVIDERS: - rows.append({ - "key": p["key"], "label": p["label"], "kind": "oauth", "desc": p["desc"], - "provider": p["provider"], - "state": "available" if have_key else "unconfigured", - # ⭐⭐ WAVE 32 · OWNER ITEM 3 / R3 — DE-BRANDED. These two sentences used to read: - # - # needs: "this provider's integration must be created in the Nango dashboard first" - # / "the owner registers NANGO_SECRET_KEY on this deployment" - # note: "Connects through Nango. Reading this provider's data into a database is - # not built yet — measured: no sync is configured on this account." - # - # ⛔ R3 IS "DE-BRAND, DON'T REVOKE": the transport is unchanged and still Nango's — - # `oauth_connect.nango_probe`, `NANGO_SECRET_KEY` and every constant keep their names, - # because that is what the person setting the secret has to type. What goes is the - # supplier's name on a CUSTOMER's screen. Which vendor carries an OAuth handshake is - # our problem, exactly as `AIOS_BRIGHTDATA_KEY` sits behind a card labelled "Scraper". - # ⚠ THE HONESTY THE SENTENCE CARRIED IS NOT LOST, which is the half a de-brand usually - # drops: R7's rule is that a lit provider states what it can do TODAY, so the note - # still says "not built yet" in as many words, and `needs` still names the remaining - # step — it just no longer names the company doing it. - "needs": ("this connection has not been set up for this provider yet" - if have_key else - "the owner finishes connection setup on this deployment"), - # ⛔ THE SENTENCE R7 DEMANDS, on the row rather than in a paragraph somewhere (R13). - "note": "Signing in works. Reading this provider's data into a database is not " - "built yet.", - "manage": "", - }) - return rows - - -def _scraper_row(): - """The Scraper card. Its own function because `bd_ready()` reaches into `automation_engine` — - an import and a call into another subsystem, i.e. the single most likely thing in this file to - raise, and until W31-T44 it could take the whole page with it.""" - import automation_engine as _eng - # ⭐ WAVE 25 · C2 / owner ruling R9 — THE ROW IS "SCRAPER", NOT THE VENDOR'S NAME. - # - # ⛔ THE KEY CHANGED TOO (`brightdata` -> `scraper`), which is the half worth being careful - # about: it is what the trigger picker's Connector sub-group is named after, so the two must - # agree. Grepped before flipping — NO client file switches on this key (the six client hits - # for "brightdata" are all `CaptureTier`, which is the VENDOR and is deliberately untouched), - # and the one server reader was `verify_automation`'s directory-key check, retargeted in the - # same change. - # - # ⚠ THE VENDOR IS REAL AND STAYS REAL. `AIOS_BRIGHTDATA_KEY`, `bd_ready()` and every `BD_*` - # constant are the supplier and keep their names — this renames what a CUSTOMER sees, because - # which supplier answers a scrape is our problem and not something to make them learn. The - # `needs` string still names the env var, deliberately: it is read by whoever has to set it. - ready = _eng.bd_ready() - return { - "key": "scraper", "label": "Scraper", "kind": "builtin", - # ⭐ WAVE 30 · R4 — the sentence names the CAPABILITY; the platforms are the drill below. - # It used to say "Instagram profile and post capture", which stopped being true the day - # TikTok discovery shipped and would have to be re-edited for every platform after it. - "desc": "Create saved Instagram, TikTok and Google Maps database recipes.", - "state": "connected" if ready else "unconfigured", - "needs": "" if ready else "AIOS_BRIGHTDATA_KEY is not set on this deployment", - "manage": "", - # The directory taxonomy normally derives this flag from connected table prefixes. - # Scraper recipes are user databases, so the route stamps it again after normalisation. - "spawns_tables": True, - # Platforms remain a drill behind one recipe/database capability, never directory rows. - # `bd_ready()` describes today's social runners only; Maps' own sentence names that its - # execution provider is intentionally absent until T12. - "platforms": [dict(p) for p in SCRAPER_PLATFORMS], - } - - -@router.get("/connectors/directory") -def connectors_directory(session: Session = Depends(require_session)): - """C11: every connector this platform knows about, with its state for THIS reader. - - Session-gated rather than admin-gated on purpose: a non-admin should be able to see what the - workspace is connected to, and connect their OWN per-user accounts — which is the whole point - of R7's per-user slots. - - ⭐⭐ W33-T66 / D-204 — AND THAT SECOND CLAUSE IS NOW TRUE. This docstring used to promise it and - then say, one sentence later, that `manage` *"is empty for a non-admin"* — a contradiction - inside a single paragraph, which is how the promise survived three waves without a door behind - it. `manage` carries WHICH ROOM, and there are three answers for a keychain-backed row: - - · `keychain` — the tenant-wide credential door. **Admins only**, and it must stay - that way: it edits the key the whole workspace reads through. - · `keychain_personal` — the same room, for a connection that is this reader's own. Offered - to a non-admin **only** where a personal credential is actually - allowed (`routes_keychain.TENANT_WIDE_TYPES` is the refusal list, so - Odoo and Meta Ads never get it). - · `""` — nothing this reader can do here; the client renders no control - rather than a door that would 403. - """ - # ⚠ `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as - # careful and is silently False for every admin on earth — the manage actions would simply - # never appear, with nothing to catch it. Attribute name checked against `deps.Session`. - # ⭐ W31-T44 — `incomplete` IS PART OF THE ANSWER, not an error channel. The page's job is to - # be a map of the workspace; a map missing a region must say which region, or the reader - # concludes the region does not exist. That is the owner's item 6b in one sentence, and it is - # R6's second sentence applied to a payload: a limit that cannot be removed is REPORTED with - # its cause and a recommended fix. `[]` on a healthy read, so the client can test it directly. - incomplete = [] - rows = directory(session.runtime, session.uname, bool(session.admin), incomplete=incomplete) - # ⚠ W49-T18 — GRAPHS ARE ATTACHED TO OUR OWN ROWS ONLY, and the reason is copy rather than - # cost. `relationship_graphs` does no I/O for a row that is not `connected` (it returns a - # refusal envelope and the expensive `user_tables.all_defs` read is keyed on the connected - # set), so running it over the catalogue would be cheap. What it would produce is 118 copies - # of the sentence *"Connect this provider before viewing its actual relationship graph"* about - # rows that carry no credential and no door, which is a promise the product cannot keep. - # ⚠ THE CLIENT ALREADY HANDLES THE ABSENCE: `parseRelationshipGraph` answers a missing - # `connectionKey` with `graphRefusal(selectedKey, ...)`, so a card with no envelope on the - # wire renders exactly as it did before. The list comprehension holds the SAME dicts, so the - # in-place attachment still lands on the native rows. - attach_relationship_graphs([r for r in rows if r.get("provider") != "monid"], - session.user, session.runtime, session.uname, - bool(session.admin)) - # ⛔⛔ W49-T18 / C4 — THE ONE PLACE THE SERVER ROW BECOMES THE CLIENT DTO. `provider` - # (`native` | `monid`) is how we reason about where a row came from; R27's design rule is that - # it never renders, in any form, so it is stripped HERE rather than left to each reader to - # remember not to paint. `verify_catalog.py` reads this payload's shape and its negative - # control leaks the key on purpose to prove the leg can bite. - # ⚠ THE CHIP COUNTS ARE COMPUTED FROM THE ROWS ACTUALLY SERVED, so the number on a chip is the - # number of cards behind it by construction, and never a sum of endpoint counts (the ticket's - # trap; `chip_counts` carries the argument). - served = connector_directory.to_client_rows(rows) - for row in served: - if str(row.get("key") or "") == "scraper": - row["spawns_tables"] = True - _safe_connector_status(row.get("key"), row) - _safe_connector_status("amazon_sp", {"incomplete": incomplete}) - return {"connectors": served, - "incomplete": incomplete, - # ⭐⭐ W53-T29 / OWNER RULING R6 — MAY THIS READER SEE THE COMPANY SIDE AT ALL. - # - # R6: *"A non-admin does not see the Company side at all; the toggle itself is hidden - # for them."* That is a PERMISSION, so it is the server's answer and not something the - # page infers from whether some row happened to carry `manage: "keychain"`. - # ⛔ THE WORD IS `routes_keychain`'s OWN. `GET /admin/keychain` has answered - # `"canBusiness": bool(session.admin)` since W32-T11, for exactly this purpose ("the - # vocabulary and the permission, so the client renders a picker it can honour"), and - # the same fact under a second name would be two vocabularies for one boolean. - # ⚠ CARRIED HERE RATHER THAN FETCHED FROM THAT ROUTE. The keychain door lists every - # visible ENTRY, so a page that wanted one boolean would pull the workspace's whole - # credential list on every visit; this route already holds the session that decides it. - "canBusiness": bool(session.admin), - "categories": connector_directory.chip_counts(served), - # ⛔⛔ `planned` COUNTS THE ROADMAP AGAIN, AND FOR ONE WAVE IT COUNTED 120. W49-T18 - # gave every Monid listing the `planned` state because that state renders faded, so - # this number stopped being "things we intend to build" and became "things we intend - # to build, plus a vendor's whole catalogue". The listings wear - # `connector_directory.CATALOGUE_STATE` now and are counted as what they are; the - # constant is read rather than re-typed so the two cannot drift apart. - "counts": {"connected": sum(1 for r in served if r["state"] == "connected"), - "available": sum(1 for r in served if r["state"] == "available"), - "planned": sum(1 for r in served if r["state"] == "planned"), - "catalogue": sum(1 for r in served - if r["state"] == connector_directory.CATALOGUE_STATE)}} - - -_ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements" -_ODOO_SYNC_STATUS = "odoo_sync_status" -_ODOO_HEALTH_STATUS = "odoo_health_status_v1" -_ODOO_HEALTH_STAGES = ("source", "mirrorPull", "materialization", "reconciliation", "freshness") -_SYNC_REQUEST_CONNECTIONS = {} -_SYNC_REQUEST_CONNECTION_LOCK = threading.Lock() - - -def _odoo_connection_slot(rt, connection_id=None): - """Return the durable Odoo connection id used by jobs, status, and health receipts.""" - requested = str(connection_id or "").strip() - if requested and requested != "default": - return requested - try: - import odoo_relational - return str(odoo_relational.default_connection_id(rt) or "default") - except Exception: # noqa: BLE001 - return "default" - - -def sync_request_connection(provider, tenant): - """Return the connection bound to a process-local Sync now intent.""" - with _SYNC_REQUEST_CONNECTION_LOCK: - return _SYNC_REQUEST_CONNECTIONS.get((str(provider), str(tenant))) - - -def _odoo_failure(exc, stage="source"): - """Classify a source/database exception without persisting its potentially sensitive text.""" - stage = stage if stage in _ODOO_HEALTH_STAGES else "source" - text = str(exc or "").casefold() - exc_type = type(exc).__name__ if isinstance(exc, BaseException) else "SourceError" - if (isinstance(exc, ssl.SSLCertVerificationError) - or "certificate_verify_failed" in text - or "certificate verify failed" in text - or "certificate has expired" in text - or "self signed certificate" in text): - return {"stage": "source", "code": "tls_certificate_invalid", - "type": exc_type, - "message": ("Odoo TLS certificate verification failed because the certificate " - "is expired or untrusted. No source data was accepted.")} - if isinstance(exc, xmlrpc.client.Fault) and any( - token in text for token in ("accesserror", "access denied", "security restrictions")): - return {"stage": "source", "code": "source_read_refused", "type": exc_type, - "message": "Odoo refused the configured account's read access."} - if any(token in text for token in ("authentication", "access denied", "invalid api key")): - return {"stage": "source", "code": "source_auth_failed", "type": exc_type, - "message": "Odoo rejected the configured read-only credential."} - if isinstance(exc, (TimeoutError, ConnectionError)) or any( - token in text for token in ("timed out", "timeout", "name resolution", "connection refused")): - return {"stage": "source", "code": "source_unavailable", "type": exc_type, - "message": "Odoo could not be reached for this read-only sync."} - labels = { - "source": ("source_read_failed", "Odoo source validation failed."), - "mirrorPull": ("mirror_pull_failed", "The Odoo mirror pull failed."), - "materialization": ("materialization_failed", "The Odoo database could not be materialized."), - "reconciliation": ("reconciliation_failed", "The Odoo database reconciliation failed."), - "freshness": ("freshness_unconfirmed", "Freshness could not be confirmed for this database."), - } - code, message = labels[stage] - return {"stage": stage, "code": code, "type": exc_type, "message": message} - - -def _clean_odoo_failure(failure, stage): - if failure is None: - return None - if not isinstance(failure, dict): - return _odoo_failure(failure, stage) - return {"stage": str(failure.get("stage") or stage), - "code": str(failure.get("code") or f"{stage}_failed"), - "type": str(failure.get("type") or "SourceError")[:80], - "message": str(failure.get("message") or "Odoo sync failed.")[:300]} - - -def _odoo_stage_failure(stages): - """Return the earliest unresolved lifecycle failure, preserving its named cause.""" - return next((stages.get(name, {}).get("failure") for name in _ODOO_HEALTH_STAGES - if stages.get(name, {}).get("state") == "failed" - and stages.get(name, {}).get("failure")), None) - - -def _clean_odoo_database_receipt(receipt, database_key): - """Allow-list one data-free reconciliation receipt for durable health storage.""" - if not isinstance(receipt, dict): - return None - key = str(database_key or "").strip() - if str(receipt.get("databaseKey") or "").strip() != key: - raise ValueError("the reconciliation receipt belongs to a different database") - out = {"databaseKey": key} - for name in ("schemaKey", "topicKey", "domainKey", "model", "countModel", "state", - "sourceCountMode", "sourceReconciliation"): - value = receipt.get(name) - if value is not None: - out[name] = str(value)[:160] - for name in ("sourceCount", "mirrorCount", "materializedCount", "canonicalCount", - "servingCount", "duplicateCount", "explainedMergeCount", - "unexplainedMissingCount", "unexplainedExtraCount", - "unexplainedCanonicalMissingCount", "unexplainedCanonicalExtraCount", - "unexplainedCount"): - value = receipt.get(name) - out[name] = None if value is None else max(0, int(value)) - if "readThrough" in receipt: - out["readThrough"] = bool(receipt.get("readThrough")) - selection = receipt.get("selection") - if isinstance(selection, dict): - out["selection"] = { - name: (bool(selection.get(name)) if name == "enabled" - else str(selection.get(name) or "")[:160]) - for name in ("bucket", "rule", "enabled") if name in selection} - domain = receipt.get("domain") - if isinstance(domain, list): - # Curated domains contain public field/operator/value predicates. Refuse nested objects, - # which could otherwise turn this metadata seam into durable source-row storage. - clean_domain = [] - for predicate in domain: - if not isinstance(predicate, (list, tuple)) or len(predicate) != 3: - continue - field, operator, value = predicate - if isinstance(value, (list, tuple)): - clean_value = [item for item in value - if isinstance(item, (str, int, float, bool))][:50] - elif isinstance(value, (str, int, float, bool)) or value is None: - clean_value = value - else: - continue - clean_domain.append([str(field)[:100], str(operator)[:40], clean_value]) - out["domain"] = clean_domain - reconciliation = receipt.get("reconciliation") - if isinstance(reconciliation, dict): - out["reconciliation"] = { - name: str(reconciliation.get(name) or "")[:80] - for name in ("sourceToMirror", "mirrorToMaterialized", "mirrorToCanonical") - if name in reconciliation} - freshness = receipt.get("freshness") - if isinstance(freshness, dict): - out["freshness"] = { - name: str(freshness.get(name) or "")[:100] - for name in ("state", "capturedAt") if name in freshness} - for name in ("sourceError", "error"): - if receipt.get(name) is not None: - out[name] = _clean_odoo_failure(receipt.get(name), "reconciliation") - return out - - -def record_odoo_stage(rt, connection_id, stage, state, *, database_key=None, row_count=None, - refreshed_at=None, cadence_seconds=None, failure=None, receipt=None, - at=None): - """Persist one data-free Odoo stage transition for one connection/database. - - This is additive observability over the established ``connector_sync`` and materialization - job documents. It stores identifiers, counts, timestamps, and classified failures only; - credentials and source rows never cross this seam. - """ - if stage not in _ODOO_HEALTH_STAGES: - raise ValueError(f"unknown Odoo health stage: {stage}") - if state not in ("unknown", "queued", "running", "ready", "failed", "paused"): - raise ValueError(f"unknown Odoo health state: {state}") - cid = _odoo_connection_slot(rt, connection_id) - stamp = str(at or datetime.now(timezone.utc).isoformat()) - failed = _clean_odoo_failure(failure, stage) if state == "failed" else None - db_key = str(database_key or "").strip() - safe_receipt = _clean_odoo_database_receipt(receipt, db_key) if receipt is not None else None - - def _update(current): - document = dict(current or {}) - connections = dict(document.get("connections") or {}) - connection = dict(connections.get(cid) or {}) - stages = dict(connection.get("stages") or {}) - target_stages = stages - database = None - databases = None - if db_key: - databases = dict(connection.get("databases") or {}) - database = dict(databases.get(db_key) or {"databaseKey": db_key}) - target_stages = dict(database.get("stages") or {}) - previous = dict(target_stages.get(stage) or {}) - stage_row = {**previous, "state": state, "at": stamp, "failure": failed} - if state == "running": - stage_row["startedAt"] = stamp - stage_row["finishedAt"] = None - elif state in ("ready", "failed", "paused"): - stage_row["finishedAt"] = stamp - if state == "ready": - stage_row["lastSuccess"] = stamp - target_stages[stage] = stage_row - connection.update({"connectionId": cid, "stages": stages, "lastAttempt": stamp}) - if cadence_seconds is not None: - connection["cadenceSeconds"] = int(cadence_seconds) - - if db_key: - database.update({"databaseKey": db_key, "stages": target_stages, - "lastAttempt": stamp}) - if row_count is not None: - database["rowCount"] = max(0, int(row_count)) - if refreshed_at: - database["recordRefreshed"] = str(refreshed_at) - if safe_receipt is not None: - database["reconciliationReceipt"] = safe_receipt - db_failure = _odoo_stage_failure(target_stages) - if db_failure: - database["failure"] = db_failure - else: - database.pop("failure", None) - if state == "ready" and stage == "freshness": - database["lastSuccess"] = stamp - databases[db_key] = database - connection["databases"] = databases - else: - stages[stage] = stage_row - connection["stages"] = stages - connection_failure = _odoo_stage_failure(stages) - if connection_failure: - connection["failure"] = connection_failure - else: - connection.pop("failure", None) - if state == "ready" and stage == "freshness": - connection["lastSuccess"] = stamp - connections[cid] = connection - document["connections"] = connections - return document - - rt.update(_ODOO_HEALTH_STATUS, _update, flush="sync") - return odoo_health_receipt(rt, cid) - - -def _lag_seconds(stamp, now=None): - if not stamp: - return None - try: - parsed = datetime.fromisoformat(str(stamp).replace("Z", "+00:00")) - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - current = now or datetime.now(timezone.utc) - return max(0, int((current - parsed.astimezone(timezone.utc)).total_seconds())) - except (TypeError, ValueError): - return None - - -def _empty_odoo_stages(): - return {stage: {"state": "unknown", "at": None, "startedAt": None, - "finishedAt": None, "lastSuccess": None, "failure": None} - for stage in _ODOO_HEALTH_STAGES} - - -def odoo_health_receipt(rt, connection_id=None, *, now=None): - """Project the truthful Odoo receipt for one connection and each of its databases.""" - cid = _odoo_connection_slot(rt, connection_id) - try: - document = rt.get(_ODOO_HEALTH_STATUS, fresh=True) or {} - except TypeError: - document = rt.get(_ODOO_HEALTH_STATUS) or {} - except Exception: # noqa: BLE001 - document = {} - stored = dict(((document.get("connections") or {}).get(cid)) or {}) - stages = _empty_odoo_stages() - stages.update({key: dict(value) for key, value in (stored.get("stages") or {}).items() - if key in stages and isinstance(value, dict)}) - databases = {str(key): dict(value) for key, value in (stored.get("databases") or {}).items() - if isinstance(value, dict)} - - # Durable materialization receipts are the authority for database work and survive restarts. - for job in connector_sync.jobs(rt, "odoo", cid): - key = str(job.get("databaseKey") or "") - if not key: - continue - database = dict(databases.get(key) or {"databaseKey": key}) - db_stages = _empty_odoo_stages() - db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items() - if name in db_stages and isinstance(value, dict)}) - job_state = str(job.get("state") or "unknown") - material_state = {"configured": "queued", "catalogue-ready": "queued", - "queued": "queued", "syncing": "running", "ready": "ready", - "failed": "failed"}.get(job_state, "unknown") - job_at = None - if job.get("at") is not None: - try: - job_at = datetime.fromtimestamp(float(job["at"]), timezone.utc).isoformat() - except (TypeError, ValueError, OSError): - job_at = None - job_failure = (_odoo_failure(job.get("error"), "materialization") - if material_state == "failed" else None) - db_stages["materialization"] = { - **db_stages["materialization"], "state": material_state, - "at": job_at, "finishedAt": job_at if material_state in ("ready", "failed") else None, - "lastSuccess": (job_at if material_state == "ready" else - db_stages["materialization"].get("lastSuccess")), - "failure": job_failure, - } - progress = job.get("progress") if isinstance(job.get("progress"), dict) else {} - if progress.get("rows") is not None: - database["rowCount"] = max(0, int(progress["rows"])) - database.update({"databaseKey": key, "model": str(job.get("model") or ""), - "state": material_state, "stages": db_stages}) - persisted_failure = database.get("failure") or _odoo_stage_failure(db_stages) - if persisted_failure or job_failure: - database["failure"] = persisted_failure or job_failure - elif material_state == "ready" and not database.get("failure"): - database["lastSuccess"] = database.get("lastSuccess") or job_at - databases[key] = database - - # The resident mirror state is process-local by design; overlay it only on the connection - # that owns this tenant's mirror. A named sibling must never inherit default's fresh badge. - cadence = stored.get("cadenceSeconds") - try: - _main = running_main() # D-781: resolve, never import - resident = dict(_main.RESYNC_STATE) - resident_cid = str(resident.get("connectionId") or "") - if resident_cid == cid: - cadence = resident.get("resolvedSeconds") or cadence - mirror_failure = resident.get("lastError") - mirror_state = ("paused" if resident.get("manual") else - "failed" if resident.get("lastPassOk") is False else - "ready" if resident.get("lastPassOk") is True else "unknown") - stages["mirrorPull"] = {**stages["mirrorPull"], "state": mirror_state, - "at": resident.get("lastPassAt"), - "finishedAt": resident.get("lastPassAt"), - "lastSuccess": resident.get("lastSuccessAt"), - "failure": (_clean_odoo_failure(mirror_failure, "mirrorPull") - if mirror_failure else None)} - except Exception: # noqa: BLE001 - pass - - for database in databases.values(): - db_stages = _empty_odoo_stages() - db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items() - if name in db_stages and isinstance(value, dict)}) - # Connection source/mirror/reconciliation facts apply to each database, while its own - # materialization/freshness facts remain independent. - for stage in ("source", "mirrorPull", "reconciliation"): - if db_stages[stage].get("state") == "unknown": - db_stages[stage] = dict(stages[stage]) - database["stages"] = db_stages - success = database.get("lastSuccess") or db_stages["freshness"].get("lastSuccess") - database["lastSuccess"] = success - database["currentLagSeconds"] = _lag_seconds(success, now) - database.setdefault("rowCount", None) - database.setdefault("recordRefreshed", None) - stage_failure = _odoo_stage_failure(db_stages) - if stage_failure: - database["failure"] = stage_failure - elif not database.get("failure"): - database["failure"] = None - - successes = [row.get("lastSuccess") for row in databases.values() if row.get("lastSuccess")] - last_success = max(successes, default=stored.get("lastSuccess")) - row_counts = [row.get("rowCount") for row in databases.values() - if row.get("rowCount") is not None] - failure = (_odoo_stage_failure(stages) or stored.get("failure") or next( - (row.get("failure") for row in databases.values() if row.get("failure")), None)) - return {"connectionId": cid, "cadenceSeconds": cadence, - "lastSuccess": last_success, "currentLagSeconds": _lag_seconds(last_success, now), - "rowCount": sum(row_counts) if row_counts else None, - "failure": failure, "stages": stages, - "databases": [databases[key] for key in sorted(databases)]} - - -def _odoo_admin(session): - if not session.admin: - raise err(403, "not_admin", "only an administrator can inspect the Odoo schema") - - -def _sync_admin(session): - if not session.admin: - raise err(403, "not_admin", "only an administrator can retry a connector sync") - - -def _connection_jobs(rt, provider, connection_id): - """A status endpoint selects one connection; the jobs inventory may enumerate them all.""" - cid = str(connection_id or "").strip() or "default" - if provider == "odoo" and cid == "default": - cid = _odoo_connection_slot(rt, connection_id) - return connector_sync.jobs(rt, provider, cid) - - -def _sync_payload(connector_key, connection_id, row=None): - """One wire shape for every registered connector, including one not yet run.""" - meta = connector_sync.providers()[connector_key] - slot = str(connection_id or "").strip() or "default" - if row is None: - return _safe_connector_status(connector_key, {"provider": connector_key, "label": meta["label"], - "connectionId": slot, "recorded": False, "state": None, - "at": None, "stateAt": None, "from": None, - "detail": "no sync has been recorded", "errors": [], "errorsTotal": 0, - "databases": [], "retryable": False, "retryHint": meta["retryHint"]}) - answer = dict(row) - answer.update({"provider": connector_key, "connectionId": slot, - "label": meta["label"], "recorded": True, - "stateAt": row.get("at"), "retryHint": meta["retryHint"]}) - return _safe_connector_status(connector_key, answer) - - -def _odoo_result_database_failures(rt, connection_id, result): - """Map provider-reported model/table failures to their durable database keys.""" - import odoo_relational as rel - try: - choices = rel.grid_choices(rt, connection_id) - except Exception: # noqa: BLE001 - choices = [] - by_model = {str(row.get("model") or ""): str(row.get("key") or "") - for row in choices if row.get("model") and row.get("key")} - failures = {str(key): _clean_odoo_failure(value, "materialization") - for key, value in (result.get("databaseErrors") or {}).items()} - receipt_groups = [] - for name in ("databases", "canonicalDatabases"): - group = result.get(name) if isinstance(result, dict) else None - if isinstance(group, dict): - receipt_groups.append(group) - for group in receipt_groups: - for raw_key, receipt in group.items(): - if not isinstance(receipt, dict) or receipt.get("state") == "succeeded": - continue - key = str(raw_key or "") - if not key or key in failures: - continue - raw_failure = receipt.get("error") or receipt.get("sourceError") or { - "stage": "reconciliation", "code": "reconciliation_incomplete", - "type": "ReconciliationError", - "message": "The Odoo database reconciliation is incomplete.", - } - failures[key] = _clean_odoo_failure(raw_failure, "reconciliation") - unmatched = [] - discovered = result.get("discovered") if isinstance(result, dict) else {} - for raw in ((discovered or {}).get("errors") or ()): - model = str(raw).split(":", 1)[0].strip() - key = by_model.get(model) - if key: - failures[key] = _odoo_failure(raw, "materialization") - else: - unmatched.append(str(raw)) - limits = result.get("limits") if isinstance(result, dict) else None - if isinstance(limits, dict): - for raw_key, raw in limits.items(): - key = by_model.get(str(raw_key or ""), str(raw_key or "")) - if key: - failures[key] = _odoo_failure(raw, "materialization") - elif limits: - unmatched.append(str(limits)) - return failures, unmatched - - -def _odoo_sync_runner(session, connection_id): - """Run Odoo through C6 without moving provider imports into the data layer.""" - cid = _odoo_connection_slot(session.runtime, connection_id) - connector_sync.record(session.runtime, "odoo", cid, "syncing") - record_odoo_stage(session.runtime, cid, "source", "running") - record_odoo_stage(session.runtime, cid, "materialization", "queued") - import odoo_relational - def _materializing(): - connector_sync.record(session.runtime, "odoo", cid, "materializing") - record_odoo_stage(session.runtime, cid, "materialization", "running") - try: - result = odoo_relational.refresh(session.runtime, session.tenant, - username=session.uname or "automation", - connection_id=connection_id, - on_apply=_materializing) - except Exception as exc: - failure = _odoo_failure(exc, "source") - record_odoo_stage(session.runtime, cid, failure["stage"], "failed", failure=failure) - record_odoo_stage(session.runtime, cid, "freshness", "failed", - failure={**failure, "stage": "freshness"}) - connector_sync.record(session.runtime, "odoo", cid, "error", - errors=[f"{failure['code']}: {failure['message']}"]) - raise - tables = result.get("tables") if isinstance(result, dict) else {} - published = dict(tables or {}) - database_receipts = result.get("databases") if isinstance(result, dict) else {} - database_receipts = database_receipts if isinstance(database_receipts, dict) else {} - canonical_receipts = result.get("canonicalDatabases") if isinstance(result, dict) else {} - canonical_receipts = canonical_receipts if isinstance(canonical_receipts, dict) else {} - reconciliation_receipts = {**database_receipts, **canonical_receipts} - databases = sorted(str(key) for key in published) - sync_errors = [] - if isinstance(result, dict): - if result.get("failed"): - sync_errors.append(str(result["failed"])) - discovered = result.get("discovered") - if isinstance(discovered, dict): - sync_errors.extend(str(value) for value in (discovered.get("errors") or ())) - discovered_tables = discovered.get("tables") - if isinstance(discovered_tables, dict): - published.update(discovered_tables) - databases.extend(str(key) for key in discovered_tables) - if result.get("frozen"): - sync_errors.append(str(result.get("note") or "the connector is frozen")) - database_failures, unmatched_errors = _odoo_result_database_failures( - session.runtime, connection_id, result if isinstance(result, dict) else {}) - databases.extend(reconciliation_receipts) - databases.extend(database_failures) - sync_errors.extend(f"{key}: {failure['code']}" for key, failure in database_failures.items()) - sync_errors.extend(raw for raw in unmatched_errors if raw not in sync_errors) - state = ("partial" if published else "error") if sync_errors else "ready" - record_odoo_stage(session.runtime, cid, "source", "ready") - refreshed_at = datetime.now(timezone.utc).isoformat() - database_keys = list(dict.fromkeys([*reconciliation_receipts, *published])) - for key in database_keys: - if key in database_failures: - continue - counts = published.get(key) - receipt = reconciliation_receipts.get(key) - receipt = receipt if isinstance(receipt, dict) else None - nrows = (receipt.get("servingCount") if receipt is not None - else counts.get("rows") if isinstance(counts, dict) else None) - record_odoo_stage(session.runtime, cid, "source", "ready", database_key=key, - receipt=receipt) - record_odoo_stage(session.runtime, cid, "materialization", "ready", - database_key=key, row_count=nrows, receipt=receipt) - record_odoo_stage(session.runtime, cid, "reconciliation", "ready", - database_key=key, row_count=nrows, receipt=receipt) - record_odoo_stage(session.runtime, cid, "freshness", "ready", - database_key=key, row_count=nrows, refreshed_at=refreshed_at, - receipt=receipt) - for key, failure in database_failures.items(): - receipt = reconciliation_receipts.get(key) - receipt = receipt if isinstance(receipt, dict) else None - failed_stage = str(failure.get("stage") or "materialization") - failed_stage = failed_stage if failed_stage in _ODOO_HEALTH_STAGES else "materialization" - record_odoo_stage(session.runtime, cid, failed_stage, "failed", - database_key=key, failure=failure, receipt=receipt) - record_odoo_stage(session.runtime, cid, "freshness", "failed", database_key=key, - failure={**failure, "stage": "freshness"}, receipt=receipt) - if sync_errors: - failure = _odoo_failure("provider result contained errors", "materialization") - record_odoo_stage(session.runtime, cid, "materialization", "failed", failure=failure) - record_odoo_stage(session.runtime, cid, "freshness", "failed", - failure={**failure, "stage": "freshness"}) - else: - record_odoo_stage(session.runtime, cid, "materialization", "ready") - record_odoo_stage(session.runtime, cid, "reconciliation", "ready") - record_odoo_stage(session.runtime, cid, "freshness", "ready") - return connector_sync.record( - session.runtime, "odoo", cid, state, - detail=str(result.get("note") or "") if isinstance(result, dict) else "", - errors=sync_errors, databases=sorted(set(databases)), - ) - - -# Provider runners are an API-layer capability. The data-layer registration proves Amazon uses -# the same contract while this table honestly leaves it without an executable sync. -_SYNC_RUNNERS = {"odoo": _odoo_sync_runner} - - -ODOO_DISCONNECTED_CAUSE = ( - "Odoo is disconnected, so rows cannot load. " - "Reconnect Odoo in the Keychain and this resumes on its own.") - - -class _SupersededMaterialization(Exception): - pass - - -def _invalidate_materialization(rt, key): - """Fence a publisher in the same atomic document that holds the records.""" - import core.user_tables as ut - def apply(current): - doc = dict(current or {}) - if key in doc: - doc[key] = {**doc[key], "_connectorJob": None} - return doc - rt.update(ut.STORE_KEY, apply, flush="sync") - - -def _queue_odoo_database(rt, key, *, model="", retry=False, changed=False): - import odoo_relational as rel - cid = rel.connection_for_table_key(key, rt) - revision = connector_sync.configuration_revision( - "odoo", cid, key, model, rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}) - previous = (rt.get(connector_sync.JOBS_KEY, fresh=True) or {}).get( - connector_sync.job_key("odoo", cid, key)) or {} - if previous.get("state") == "syncing" and (changed or previous.get("revision") != revision): - _invalidate_materialization(rt, key) - job = connector_sync.enqueue( - rt, "odoo", cid, key, model=model, retry=retry, changed=changed, revision=revision) - # The runtime service is started by ASGI lifespan, never by a handler or a fixture import. - _materialization_wake.set() - return job - - -class _MaterializationRuntime: - """Delegate tenant reads; fence each row publication inside the store's atomic updater.""" - def __init__(self, rt, job): - self.rt, self.job = rt, job - - def __getattr__(self, name): - return getattr(self.rt, name) - - def valid(self): - import odoo_relational as rel - key = self.job["databaseKey"] - if not connector_sync.owns(self.rt, self.job): - raise _SupersededMaterialization() - cfg = self.rt.get(rel.CONFIG_KEY, fresh=True) or {} - if cfg.get("frozen") or (cfg.get("grids") or {}).get(key) is False: - raise _SupersededMaterialization() - # Save commits field choices before the queue update. The lease alone cannot - # detect that newer configuration if enqueue fails or has not run yet. - revision = self.job.get("revision") - if revision is not None and revision != connector_sync.configuration_revision( - self.job["provider"], self.job["connectionId"], key, self.job["model"], - self.rt.get(_ODOO_SCHEMA_ENABLEMENTS, fresh=True) or {}): - raise _SupersededMaterialization() - - def update(self, name, fn, flush="sync", **kwargs): - import core.user_tables as ut - key = self.job["databaseKey"] - marker = [self.job["id"], self.job["leaseToken"]] - def apply(current): - self.valid() - missing = name == ut.STORE_KEY and key not in current - if (name == ut.STORE_KEY and not missing - and (current.get(key) or {}).get("_connectorJob") != marker): - raise _SupersededMaterialization() - answer = fn(current) - if missing and key in answer: - # A reconciled receipt may precede its definition. Its first schema write - # installs the marker under the same lease and configuration fence. - answer[key]["_connectorJob"] = marker - # Lease expiry during a long merge cannot acknowledge a stale write either. - self.valid() - return answer - return self.rt.update(name, apply, flush=flush, **kwargs) - - -def _odoo_read_access_refused(exc): - """Match only XML-RPC faults with explicit access/read refusal markers.""" - if not isinstance(exc, xmlrpc.client.Fault): - return False - message = str(exc).casefold() - return "security restrictions" in message or "operation: read" in message - - -def _run_database_job(rt, candidate): - """Run one persisted receipt. Repeating a completed publication reconciles the same rows.""" - import core.user_tables as ut - import odoo_relational as rel - # A disconnected source is paused work, not a failed attempt. Preserve the receipt so - # reconnect can resume it; the publication fence still rechecks a mid-run disconnect. - if (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"): - connector_sync.stall(rt, candidate, ODOO_DISCONNECTED_CAUSE) - return - job = connector_sync.claim(rt, candidate) - if job is None: - return - fenced = _MaterializationRuntime(rt, job) - key = job["databaseKey"] - cid = job["connectionId"] - record_odoo_stage(rt, cid, "source", "running", database_key=key) - record_odoo_stage(rt, cid, "materialization", "running", database_key=key) - def finish(success=False, cause=""): - # The pre-claim check cannot cover a disconnect during source I/O or publication. - # Only the current lease may requeue; disable and a newer Save still win. - interrupted = bool((rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen")) - if success and not interrupted: - # A newer field Save can persist after rows commit but before this acknowledgement. - # The old rows remain valid history; they cannot establish the new settings ready. - fenced.valid() - # Only provider-authored sentences reach this seam, never raw exception messages. - cause = str(cause or "").replace("\u2014", ",").replace("\u2013", "-") - result = connector_sync.finish( - rt, job, success=success, interrupted=interrupted, cause=cause) - if interrupted: - connector_sync.stall(rt, job, ODOO_DISCONNECTED_CAUSE) - return result - - stopped = threading.Event() - def heartbeat(): - while not stopped.wait(connector_sync.LEASE_SECONDS / 3): - try: - connector_sync.finish(rt, job, renew=True) - except Exception: - return # An unrenewed lease cannot publish. - heartbeat_thread = None - try: - def install(current): - fenced.valid() - if key in (current or {}): - current[key]["_connectorJob"] = [job["id"], job["leaseToken"]] - return current - rt.update(ut.STORE_KEY, install, flush="sync") - heartbeat_thread = threading.Thread(target=heartbeat, daemon=True, - name="connector-lease") - heartbeat_thread.start() - cid = job["connectionId"] - source_id = None if cid == "default" else cid - if key not in (rt.get(ut.STORE_KEY, fresh=True) or {}): - schema = rel.publish_database_schema(fenced, key, connection_id=source_id, - username="automation") - if not job["model"]: - # Reconciliation cannot infer a missing definition's model. Now the provider - # has resolved it, queue its real field fingerprint before reading any rows. - fenced.valid() - _queue_odoo_database(rt, key, model=schema["identity"]["model"], changed=True) - return - if rel.base_table_key(key) in rel.ODOO_TABLE_MODELS: - result = rel.refresh( - fenced, getattr(rt, "tenant", None), username="automation", - connection_id=source_id, table_keys={key}) - else: - # A source-only model must not depend on an unrelated DuckDB mirror being warm. - _cid, namespace = rel.connection_namespace(rt, source_id) - result = {"discovered": rel.sync_discovered_models( - fenced, source_connection_id=source_id, namespace=namespace, - username="automation", table_keys={key}, - on_progress=lambda rows, total: connector_sync.progress( - fenced, job, rows=rows, total=total))} - discovered = result.get("discovered") or {} - published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})} - receipt = (result.get("databases") or {}).get(key) - receipt = receipt if isinstance(receipt, dict) else None - database_errors = result.get("databaseErrors") or {} - success = (key in published and not result.get("failed") and not result.get("frozen") - and not result.get("limits") and not discovered.get("errors") - and key not in database_errors - and (receipt is None or receipt.get("state") == "succeeded")) - counts = published.get(key) if isinstance(published.get(key), dict) else {} - nrows = (receipt.get("servingCount") if receipt is not None - else counts.get("rows") if isinstance(counts, dict) else None) - if success: - refreshed_at = datetime.now(timezone.utc).isoformat() - record_odoo_stage(rt, cid, "source", "ready", database_key=key, - receipt=receipt) - record_odoo_stage(rt, cid, "materialization", "ready", - database_key=key, row_count=nrows, receipt=receipt) - record_odoo_stage(rt, cid, "reconciliation", "ready", - database_key=key, row_count=nrows, receipt=receipt) - record_odoo_stage(rt, cid, "freshness", "ready", database_key=key, - row_count=nrows, refreshed_at=refreshed_at, receipt=receipt) - finish(success=True) - else: - raw_failure = ((database_errors.get(key) if isinstance(database_errors, dict) else None) - or (receipt or {}).get("error") or (receipt or {}).get("sourceError")) - failure = (_clean_odoo_failure(raw_failure, "reconciliation") if raw_failure - else _odoo_failure("provider result was incomplete", "materialization")) - failed_stage = str(failure.get("stage") or "materialization") - failed_stage = (failed_stage if failed_stage in _ODOO_HEALTH_STAGES - else "materialization") - record_odoo_stage(rt, cid, failed_stage, "failed", - database_key=key, failure=failure, receipt=receipt) - record_odoo_stage(rt, cid, "freshness", "failed", database_key=key, - failure={**failure, "stage": "freshness"}, receipt=receipt) - finish(cause=f"{failure['code']}: {failure['message']}") - except _SupersededMaterialization: - # Disable/new Save already invalidated the table marker; never restore it here. - finish() - except rel.Refused as exc: - failure = _odoo_failure(exc, "materialization") - record_odoo_stage(rt, cid, failure["stage"], "failed", - database_key=key, failure=failure) - record_odoo_stage(rt, cid, "freshness", "failed", database_key=key, - failure={**failure, "stage": "freshness"}) - finish(cause=f"{failure['code']}: {failure['message']}") - except Exception as exc: - # Raw provider exceptions may carry credentials or source values. Store only a named, - # data-free classification, including the fail-closed TLS case. - failure = _odoo_failure(exc, "source") - if _odoo_read_access_refused(exc): - failure = {"stage": "source", "code": "source_read_refused", - "type": type(exc).__name__, - "message": (f"Odoo refused read access to {job['model']}. Give the " - "connected account read access, or switch this database off.")} - record_odoo_stage(rt, cid, failure["stage"], "failed", - database_key=key, failure=failure) - record_odoo_stage(rt, cid, "freshness", "failed", database_key=key, - failure={**failure, "stage": "freshness"}) - finish(cause=f"{failure['code']}: {failure['message']}") - finally: - stopped.set() - if heartbeat_thread: - heartbeat_thread.join(timeout=1) - - -def _reconcile_database_jobs(rt): - """Repair enabled Odoo grids with no receipt using only tenant-local definitions.""" - import core.user_tables as ut - import odoo_relational as rel - grids = (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("grids") or {} - receipts = {job["databaseKey"] for job in connector_sync.jobs(rt, "odoo")} - missing = [key for key, enabled in grids.items() if enabled is True and key not in receipts] - if not missing: - return - definitions = rt.get_projection(ut.STORE_KEY, drop=("rows",)) or {} - for key in missing: - try: - model = str(((definitions.get(key) or {}).get("connector") or {}).get("model") or "") - # The queue's identity validator also decides whether an unknown model is legal. - _queue_odoo_database(rt, key, model=model) - except Exception: - continue # One unresolvable definition cannot starve other queued work. - - -def _resume_database_jobs(rt): - """One serial pass, also usable by isolated synthetic fixtures.""" - try: - _reconcile_database_jobs(rt) - except Exception: - pass # A reconciliation failure must still fall through to the receipt scan. - for candidate in connector_sync.jobs(rt): - if candidate.get("provider") == "odoo" and ( - candidate.get("state") == "queued" or - candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()): - try: - _run_database_job(rt, candidate) - except Exception: - # A failed claim/status write leaves durable work for a later scan. It must - # not starve sibling databases; never log raw store/provider exceptions. - continue - - -_materialization_wake = threading.Event() -_materialization_stop = threading.Event() -_materialization_thread = None -_materialization_lifecycle = threading.Lock() -# 3 is a starting point, not tuned: each worker holds an Odoo read and a growing store document in a small Space container. -MATERIALIZATION_WORKERS = 3 -_materialization_pool = None -_materialization_inflight = set() -_materialization_inflight_lock = threading.Lock() - - -def _release_database_job(key): - # Completion and cancellation both release the reservation, including failed claims. - with _materialization_inflight_lock: - _materialization_inflight.discard(key) - - -def _dispatch_database_jobs(rt): - """Submit a resident scan without waiting for source reads or row publication.""" - global _materialization_pool - try: - _reconcile_database_jobs(rt) - except Exception: - pass # Match the serial fixture pass even when reconciliation is unavailable. - for candidate in connector_sync.jobs(rt): - if candidate.get("provider") == "odoo" and ( - candidate.get("state") == "queued" or - candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()): - key = (rt.key, candidate["provider"], candidate["connectionId"], candidate["databaseKey"]) - with _materialization_inflight_lock: - if key in _materialization_inflight: - continue - _materialization_inflight.add(key) - try: - with _materialization_lifecycle: - if _materialization_stop.is_set(): - _release_database_job(key) - return - if _materialization_pool is None: - _materialization_pool = ThreadPoolExecutor( - max_workers=MATERIALIZATION_WORKERS, - thread_name_prefix="connector-materialization-job") - pool = _materialization_pool - future = pool.submit(_run_database_job, rt, candidate) - # Register outside both locks: an already finished future calls back inline. - future.add_done_callback(lambda _future, key=key: _release_database_job(key)) - except Exception: - _release_database_job(key) - # A concurrent shutdown or failed submit leaves the receipt for the next pass. - continue - - -@router.on_event("startup") -def _start_database_jobs(): - """Discover durable work at boot, including tenants that have no active browser session.""" - global _materialization_thread - def run(): - global _materialization_thread - from harness import runtime - while True: - with _materialization_lifecycle: - if _materialization_stop.is_set(): - # Retire under the startup lock, before this thread exits. A new lifespan - # either revives this worker or starts its replacement, never a dying one. - _materialization_thread = None - return - _materialization_wake.clear() - try: - tenants = runtime.known_tenants() - for tenant in tenants: - if _materialization_stop.is_set(): - break - try: - _dispatch_database_jobs(runtime.get_runtime(tenant)) - except Exception: - continue # Durable queued/leased work is retried by the next scan. - except Exception: - pass - _materialization_wake.wait(15) - with _materialization_lifecycle: - _materialization_stop.clear() - _materialization_wake.set() - if _materialization_thread and _materialization_thread.is_alive(): - return - _materialization_thread = threading.Thread(target=run, daemon=True, - name="connector-materialization") - _materialization_thread.start() - - -@router.on_event("shutdown") -def _stop_database_jobs(): - global _materialization_pool - with _materialization_lifecycle: - _materialization_stop.set() - _materialization_wake.set() - pool, _materialization_pool = _materialization_pool, None - if pool is not None: - # Running reads retain their reservation until done; lease expiry permits recovery. - pool.shutdown(wait=False, cancel_futures=True) - - -def _sync_refusals(): - """The provider refusal types whose OWN sentence is the answer, as a tuple for `except`. - - ⚠ Resolved lazily and defensively: `odoo_relational` is imported inside functions throughout - this module to avoid an import cycle, and a refusal handler that could itself raise on import - would turn every failed sync into a different failure. - """ - try: - import odoo_relational # noqa: PLC0415 - return (odoo_relational.Refused,) - except Exception: # noqa: BLE001 - return () - - -def _schema_catalog(session): - """Rows-free Odoo field catalogue; source fields remain discoverable when every value is empty.""" - return _schema_catalog_rt(session.runtime) - - -def _odoo_raw_models(rt, snapshot=None, connection_id=None): - """Raw fields from stamped discovery, or the legacy snapshot before discovery exists.""" - if snapshot is None: - import core.user_tables as user_tables - snapshot = user_tables.odoo_schema_snapshot(rt) - legacy = dict((snapshot or {}).get("models") or {}) if connection_id is None else {} - try: - import odoo_relational - discovered = odoo_relational.readable_model_catalog(rt, connection_id) - except Exception: # noqa: BLE001 - discovered = {} - if not discovered.get("capturedAt"): - return legacy - # A stamp means this tenant-bound account has spoken. Even a partial stamped set is safer - # than adding snapshot-only models that the current account never proved it may read. - return {str(item.get("model") or ""): dict(item.get("fields") or {}) - for item in (discovered.get("models") or ()) - if str(item.get("model") or "")} - - -def _schema_catalog_rt(rt, database_key=None, connection_id=None): - """The same catalogue, over a RUNTIME rather than a session. - - ⚠ SPLIT IN W49-T12 SO THE DETAIL ROUTE CAN REUSE IT RATHER THAN RE-DERIVE IT. The connector - modal's Databases half needs exactly this catalogue, and writing a second reader of - `odoo_schema_snapshot` here would be the two-answers-to-one-question shape this module has - already paid for twice ([[one-question-two-normalizers]]). `_schema_catalog` keeps its - signature so every existing caller is untouched. - """ - import core.user_tables as user_tables - enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} - snapshot = user_tables.odoo_schema_snapshot(rt) - raw_models = _odoo_raw_models(rt, snapshot, connection_id) - if raw_models: - rows = [] - for model, raw_fields in raw_models.items(): - for raw_name, meta in (raw_fields or {}).items(): - legacy_key = f"{model}.{raw_name}" - key = (_odoo_field_key(rt, database_key, model, raw_name) - if database_key else legacy_key) - rows.append({"key": key, "model": model, "rawName": raw_name, - "label": str(meta.get("string") or raw_name), - "type": str(meta.get("type") or "text"), "source": "Odoo", - "emptyDiscoverable": True, "defaultEnabled": False, - # Legacy choices seed default-connection databases until each has - # its own explicit override. Scoped writes never change this seed. - "enabled": enabled.get(key, enabled.get(legacy_key) - if connection_id is None else False) is True}) - return sorted(rows, key=lambda row: row["label"].casefold()) - # Never present compiled mirror declarations as an authoritative fields_get catalogue. - # Until a sync captures metadata, the schema is unavailable and enablement stays refused. - return [] - - -def _odoo_field_key(rt, database_key, model, raw_name, connection_id=...): - # An explicitly resolved None must not trigger another lookup. - if connection_id is ...: - import odoo_relational - connection_id = odoo_relational.connection_for_table_key(database_key, rt) - return connector_sync.field_key( - "odoo", connection_id, database_key, model, raw_name) - - -@router.get("/connectors/odoo/status") -def odoo_status(connection_id: str = None, session: Session = Depends(require_session)): - _odoo_admin(session) - import odoo_relational - resolved_connection = _odoo_connection_slot(session.runtime, connection_id) - cfg = odoo_relational.read_config(session.runtime) - stored = session.runtime.get(_ODOO_SYNC_STATUS) or {} - errors = stored.get("errors") if isinstance(stored.get("errors"), list) else [] - phase = str(stored.get("phase") or "unknown") - lifecycle = connector_sync.read(session.runtime, "odoo", resolved_connection) - durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id)) - receipt = odoo_health_receipt(session.runtime, resolved_connection) - if durable: - return {"schedule": cfg.get("syncEvery"), - "health": "paused" if cfg.get("frozen") else durable["state"], - "lastRun": durable["at"], "frozen": cfg.get("frozen") is True, - **durable, "stateAt": durable["at"], "connectionId": resolved_connection, - "receipt": receipt} - # The old keys stay the old default-connection answer. A caller that names a connection gets - # that slot's lifecycle instead of the tenant-wide legacy document that caused the C6 defect. - named = connection_id is not None - shown_errors = (list(lifecycle.get("errors") or ()) if named and lifecycle else - [] if named else errors) - shown_phase = ((lifecycle or {}).get("state") or "unknown") if named else phase - shown_last_run = ((lifecycle or {}).get("at") if named else stored.get("lastRun")) - state = ((lifecycle or {}).get("state") - or {"syncing": "syncing", "ok": "ready", "error": "error"}.get(shown_phase) - or "queued") - errors_total = ((lifecycle or {}).get("errorsTotal", 0) if named or lifecycle - else len(errors)) - return {"schedule": cfg.get("syncEvery"), - "health": "paused" if cfg.get("frozen") else - shown_phase if named else "error" if shown_errors else shown_phase, - "lastRun": shown_last_run, "errors": shown_errors[:20], - "frozen": cfg.get("frozen") is True, - "state": state, - "stateAt": (lifecycle or {}).get("at") or shown_last_run, - "databases": list((lifecycle or {}).get("databases") or ()), - "retryable": state in ("error", "partial"), - "errorsTotal": int(errors_total or 0), - "connectionId": (lifecycle or {}).get("connectionId") - or resolved_connection, - "receipt": receipt} - - -@router.get("/connectors/{connector_key}/sync") -def connector_sync_status(connector_key: str, connection_id: str = None, - session: Session = Depends(require_session)): - registered = connector_sync.providers() - if connector_key not in registered: - raise err(404, "sync_provider_unknown", - f"{connector_key} is not registered for connector sync") - row = (connector_sync.job_summary(_connection_jobs(session.runtime, connector_key, connection_id)) - or connector_sync.read(session.runtime, connector_key, connection_id)) - return _sync_payload(connector_key, connection_id, row) - - -@router.post("/connectors/{connector_key}/sync/now") -def connector_sync_now(connector_key: str, connection_id: str = None, - session: Session = Depends(require_session)): - _sync_admin(session) - if connector_key not in connector_sync.providers(): - raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") - try: - requested_connection = (_odoo_connection_slot(session.runtime, connection_id) - if connector_key == "odoo" else - str(connection_id or "default").strip() or "default") - key = (connector_key, session.runtime.key) - with _SYNC_REQUEST_CONNECTION_LOCK: - # Bind under one API-side lock so a duplicate request cannot race ahead of the - # accepted request and attach its different connection id to the queued intent. - receipt = connector_sync.request_sync(connector_key, session.runtime.key, - requested_by=session.uname) - if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS: - _SYNC_REQUEST_CONNECTIONS[key] = requested_connection - receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key] - return receipt - except ValueError: - raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None - - -@router.get("/connectors/{connector_key}/sync/now") -def connector_sync_now_status(connector_key: str, session: Session = Depends(require_session)): - _sync_admin(session) - if connector_key not in connector_sync.providers(): - raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") - receipt = connector_sync.sync_request(connector_key, session.runtime.key) - if receipt is not None: - receipt["connectionId"] = sync_request_connection(connector_key, session.runtime.key) - return receipt - return { - "provider": connector_key, "tenant": session.runtime.key, "state": "idle", - "requestedAt": None, "startedAt": None, "finishedAt": None, - "requestedBy": "", "detail": "", "accepted": False, - } - - -@router.post("/connectors/{connector_key}/sync/retry") -def connector_sync_retry(connector_key: str, connection_id: str = None, - session: Session = Depends(require_session), database_key: str = None): - _sync_admin(session) - if connection_id == "default": - connection_id = None - registered = connector_sync.providers() - if connector_key not in registered: - raise err(404, "sync_provider_unknown", - f"{connector_key} is not registered for connector sync") - if connector_key != "odoo": - raise err(501, "sync_runner_unavailable", registered[connector_key]["retryHint"]) - import odoo_relational as rel - if database_key: - key, _ns, choice = _odoo_database_target(session.runtime, database_key, connection_id) - choices = [choice] - else: - choices = rel.grid_choices(session.runtime, connection_id) - receipts = [] - for choice in choices: - if not choice.get("enabled"): - continue - key = choice["key"] - existing = connector_sync.jobs(session.runtime, connector_key, - rel.connection_for_table_key(key, session.runtime)) - previous = next((j for j in existing if j["databaseKey"] == key), {}) - if previous.get("state") == "failed" and not previous.get("retryable"): - if database_key: - raise err(409, "sync_attempts_exhausted", previous["error"]) - continue - receipts.append(_queue_odoo_database(session.runtime, key, model=choice.get("model", ""), - retry=True)) - if not receipts: - raise err(409, "sync_not_retryable", "No enabled database can be retried. Check its settings.") - return {"provider": connector_key, "jobs": receipts, **connector_sync.job_summary(receipts)} - - -@router.get("/connectors/{connector_key}/jobs") -def connector_database_jobs(connector_key: str, session: Session = Depends(require_session), - connection_id: str = None): - _sync_admin(session) - if connector_key not in connector_sync.providers(): - raise err(404, "sync_provider_unknown", "This connector has no database sync runner.") - payload = {"provider": connector_key, "jobs": connector_sync.jobs( - session.runtime, connector_key, connection_id)} - if connector_key == "odoo": - payload["receipt"] = odoo_health_receipt(session.runtime, connection_id) - return _safe_connector_status(connector_key, payload) - - -@router.get("/connectors/odoo/schema") -def odoo_schema(session: Session = Depends(require_session)): - _odoo_admin(session) - fields = _schema_catalog(session) - import core.user_tables as user_tables - snapshot = user_tables.odoo_schema_snapshot(session.runtime) - try: - import odoo_relational - discovered = odoo_relational.readable_model_catalog(session.runtime) - except Exception: # noqa: BLE001 - discovered = {} - has_discovery = bool(discovered.get("capturedAt")) - complete = (discovered.get("complete") is True if has_discovery - else snapshot.get("complete") is True) - return {"fields": fields, "default": "off", "adminOnly": True, - "complete": complete, - "capturedAt": (discovered.get("capturedAt") if has_discovery - else snapshot.get("capturedAt")), - "errors": list((discovered.get("errors") if has_discovery - else snapshot.get("errors")) or ()), - "source": "Odoo"} - - -@router.post("/connectors/odoo/census") -def odoo_population_census(session: Session = Depends(require_session), exact: bool = False): - """Count up to 101 ids per model by default; exact=true requests the full census.""" - _odoo_admin(session) - import odoo_relational - census = odoo_relational.refresh_population_census(session.runtime, force=True, exact=exact) - return {"capturedAt": str(census.get("capturedAt") or ""), - "mode": census["mode"], - "counts": census["counts"], - "complete": census.get("complete") is True, - "durationMs": int(census.get("durationMs") or 0), - "modelsCounted": int(census.get("modelsCounted") or 0), - "modelsWithRows": int(census.get("modelsWithRows") or 0), - "errors": [str(value).replace("\u2014", ",").replace("\u2013", "-") - for value in (census.get("errors") or ())][:50]} - - -def _materialisation_failure_reason(result): - """The materialiser's own reason, made safe for a sentence shown in the connector modal.""" - result = result if isinstance(result, dict) else {} - candidates = [] - if result.get("note"): - candidates.append(result.get("note")) - if result.get("failed"): - candidates.append(result.get("failed")) - errors = result.get("errors") - candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,)) - discovered = result.get("discovered") - if isinstance(discovered, dict): - errors = discovered.get("errors") - candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,)) - reason = str(next((value for value in candidates if str(value or "").strip()), "")).strip() - # Provider errors can carry typographic punctuation. Screen strings use plain punctuation. - # ⛔ Standing rule 2 names the EM DASH and the EN DASH, never the ASCII hyphen. A blanket - # strip of every Unicode dash category also breaks "read-only" and "e-commerce", so the - # two characters the rule names are rewritten and nothing else is touched. - reason = reason.replace("\u2014", ",").replace("\u2013", "-") - return reason[:1].upper() + reason[1:] if reason else "" - - -def _evict_odoo_product_pools(rt): - """Drop every raw-field-sensitive row pool after field enablement changes.""" - for cache_key in list(getattr(rt, "pool_cache", {}) or {}): - if isinstance(cache_key, tuple) and cache_key[:1] == ("product_pool",): - rt.pool_cache.pop(cache_key, None) - - -# W57-T31 / R10 / C9: switching off starts a seven-day lock to prevent repeated toggling. -# The config document is shared with read_config, which ignores unknown keys: disabledAt -# belongs beside grids, not in a second document that could drift from enablement. -_RELOCK_DAYS = 7 -_RELOCK_SECONDS = _RELOCK_DAYS * 86400 - - -def _relock_stamps(rt): - """Read disable epochs without letting an absent or malformed map break a request.""" - try: - import odoo_relational - stamps = (rt.get(odoo_relational.CONFIG_KEY) or {}).get("disabledAt", {}) - if not isinstance(stamps, dict): - return {} - return {str(key): int(value) for key, value in stamps.items() - if isinstance(value, (int, float)) and not isinstance(value, bool)} - except Exception: # noqa: BLE001 - return {} - - -def _relock_days_left(stamp, now=None): - """Round any remaining part of a lock day up to a whole day.""" - if isinstance(stamp, bool) or not isinstance(stamp, (int, float)): - return 0 - try: - from math import ceil - remaining = stamp + _RELOCK_SECONDS - (time.time() if now is None else now) - return max(0, ceil(remaining / 86400)) - except Exception: # noqa: BLE001 - return 0 - - -def _odoo_database_target(rt, table_key, connection_id=None): - """Resolve a database once; explicit connection/key disagreement fails closed.""" - import odoo_relational - raw = str(table_key or "").strip() - base, namespace = odoo_relational.split_table_key(raw) - requested_connection = connection_id if connection_id is not None else namespace - try: - _cid, ns = odoo_relational.connection_namespace(rt, requested_connection) - choices = odoo_relational.grid_choices(rt, requested_connection) - except odoo_relational.Refused as exc: - raise err(404, "unknown_odoo_connection", str(exc)) - key = odoo_relational.table_key_for(base or raw, ns) - if namespace is not None and namespace != ns: - raise err(409, "database_connection_mismatch", - "the database belongs to a different connection") - choice = next((row for row in choices if row["key"] == key), None) - if choice is None: - raise err(404, "unknown_odoo_database", - f"Odoo does not declare a database with key {raw!r}") - return key, ns, choice - - -def _enable_odoo_database(table_key, session, connection_id=None): - """Turn on one database and publish its schema; row materialization is separate work.""" - if not session.admin: - raise err(403, "not_admin", "only an administrator can enable an Odoo database") - - import odoo_relational # noqa: PLC0415 - provider owns both paths - key, namespace, choice = _odoo_database_target(session.runtime, table_key, connection_id) - - # R10 / C9: only a transition back on is locked; an already-enabled database may retry - # materialisation. A successful enable clears its stamp so the next switch-off starts fresh. - # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail. - if choice.get("enabled") is not True: - days = _relock_days_left(_relock_stamps(session.runtime).get(key)) - if days > 0: - label = str(choice.get("label") or key) - unit = "day" if days == 1 else "days" - raise err(409, "database_relocked", - f"{label} was switched off less than {_RELOCK_DAYS} days ago. " - f"It cannot be switched back on for another {days} {unit}. " - f"The {_RELOCK_DAYS} day lock is what stops a database being switched " - "on and off repeatedly.") - - # Publishing the definition makes Data and rename usable before any rows have loaded. - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur.setdefault("grids", {}).update({key: True}) - if isinstance(cur.get("disabledAt"), dict): - cur["disabledAt"].pop(key, None) - return cur - - failure = None - try: - result = odoo_relational.publish_database_schema( - session.runtime, key, connection_id=namespace, username=session.uname) - except odoo_relational.Refused as exc: - # The provider owns this sentence. In particular it names missing credentials and the - # MAX_TABLES refusal without this route guessing which one happened. - failure = (409, "odoo_database_refused", str(exc)) - except Exception: # noqa: BLE001 - failure = (500, "database_schema_failed", - "The setting was not changed and the database was not created because the " - "field catalogue could not be published. Refresh the catalogue and try again.") - - # A refused publication must leave both the flag and its relock stamp untouched. - if failure is None: - try: - session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync") - except Exception: # noqa: BLE001 - raise err(503, "store_unavailable", "the database setting was not saved. Try again.") - - facts = (_rename_rights(session.runtime, session.uname, bool(session.admin), [key]).get(key) - or {"exists": False, "canRename": False}) - exists = facts.get("exists") is True - can_rename = facts.get("canRename") is True - # ⛔⛔ THE ERROR ENVELOPE STAYS THE ONE SHAPE. `deps.err` is documented as X2's single - # non-2xx body, `{"error": {"code", "message"}}`, and an earlier draft of this door attached - # `key`, `enabled` and the fresh facts BESIDE that key so the client could read them off a - # failure. Two shapes for one thing is what this module's own comments call the defect it has - # paid for twice, and a caller that learns to read a sibling off an error will keep doing it. - # The client re-reads `/detail` on both paths; publication precedes the flag write and row - # materialisation remains separate work. - if failure is not None: - raise err(*failure) - model = str(choice.get("model") or "").strip() - if not exists: - reason = _materialisation_failure_reason(result) - message = "The setting was enabled, but the database was not created." - if reason: - message = f"{message} {reason}" - else: - message = f"{message} The field catalogue could not be published." - try: - _queue_odoo_database(session.runtime, key, model=model) - finally: - # Preserve the original refusal envelope even if the queue store is unavailable. - raise err(409, "database_not_created", message) - - # Every permission-visible field is available from schema alone. Existing per-database - # choices survive a retry; population counts are optional background observations. - try: - schema_keys = {_odoo_field_key(session.runtime, key, model, name): True - for name in result["sourceFields"]} - session.runtime.update( - _ODOO_SCHEMA_ENABLEMENTS, - lambda current: { - **schema_keys, - **(current if isinstance(current, dict) else {}), - }, - flush="sync") - except Exception: # noqa: BLE001 - try: - _queue_odoo_database(session.runtime, key, model=model) - finally: - raise err(503, "field_auto_enable_failed", - "The database was created, but its field choices could not be saved. " - "Try again.") - _evict_odoo_product_pools(session.runtime) - - # W56-T36: there is deliberately no navigation projection eviction here. Data's database - # list comes from - # `GET /nav` -> `core.user_tables.all_defs()` -> `TenantRuntime.get_projection(user_tables)`. - # Schema publication synchronously updates that same `user_tables` document; the HF - # backend replaces its whole-document cache as part of that write, while Postgres performs a - # fresh SELECT for every projection. `pool_cache` contains row pools for built-in databases, - # not the `ut_*` definition list, and compiled registry rows never contain tenant databases. - # The table now exists, so clients must discard navigation-derived database lists. This is a - # wire instruction rather than a guessed client row: `/nav` remains the permission-filtered - # authority for which databases the current reader may see. - try: - job = _queue_odoo_database(session.runtime, key, model=model) - except Exception: - raise err(503, "sync_queue_unavailable", - "The database settings were saved, but row sync could not be queued. Save again.") - return {"key": key, "enabled": True, "exists": exists, "canRename": can_rename, - "identity": connector_sync.database_identity( - "odoo", odoo_relational.connection_for_table_key(key, session.runtime), key, model), - "workspaceChanged": True, "state": job["state"], "job": job, - "schemaState": result["state"], - "schemaFields": result["fieldCount"], "populatedFieldsEnabled": 0, - "fieldCensusComplete": False, "fieldCensusErrors": [], - "autoEnableAlsoAffects": []} - - -@router.post("/connectors/odoo/databases/{table_key}/enable") -def odoo_database_enable(table_key: str, session: Session = Depends(require_session), - connection_id: str = None): - """Enable and create one declared Odoo database in this tenant's governed store.""" - return _enable_odoo_database(table_key, session, connection_id) - - -def _disable_odoo_database(table_key, session, connection_id=None): - """Turn off one declared Odoo database and persist its re-enable lock.""" - if not session.admin: - raise err(403, "not_admin", "only an administrator can disable an Odoo database") - - import odoo_relational - key, namespace, choice = _odoo_database_target(session.runtime, table_key, connection_id) - - # W57-T31 / R10 / C9: one config-only write starts the lock, with no sync or row deletion. - # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail. - # Repeating a disable must preserve the original deadline. - was_enabled = choice.get("enabled") is True - _invalidate_materialization(session.runtime, key) - connector_sync.cancel(session.runtime, "odoo", - odoo_relational.connection_for_table_key(key, session.runtime), key) - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - grids = cur.get("grids") - grids = grids if isinstance(grids, dict) else {} - grids[key] = False - cur["grids"] = grids - if was_enabled: - if not isinstance(cur.get("disabledAt"), dict): - cur["disabledAt"] = {} - cur["disabledAt"][key] = int(time.time()) - return cur - - try: - session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync") - except Exception: # noqa: BLE001 - raise err(503, "store_unavailable", "the database setting was not saved. Try again.") - - stamp = _relock_stamps(session.runtime).get(key) - return {"key": key, "enabled": False, "relockDays": _RELOCK_DAYS, - "identity": connector_sync.database_identity( - "odoo", odoo_relational.connection_for_table_key(key, session.runtime), - key, choice.get("model")), - "disabledAt": stamp, - "canEnableAt": stamp + _RELOCK_SECONDS if stamp is not None else None} - - -@router.post("/connectors/odoo/databases/{table_key}/disable") -def odoo_database_disable(table_key: str, session: Session = Depends(require_session), - connection_id: str = None): - """Disable one declared Odoo database in this tenant's governed store.""" - return _disable_odoo_database(table_key, session, connection_id) - - -@router.patch("/connectors/odoo/schema/{field_key}") -def odoo_schema_enable(field_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - _odoo_admin(session) - import core.user_tables as user_tables - import odoo_relational - request = body if isinstance(body, dict) else {} - key = str(field_key or "").strip() - scoped = connector_sync.parse_field_key(key) - database_key = request.get("databaseKey") - connection_id = request.get("connectionId") - model, raw_name = key.rsplit(".", 1) if "." in key else ("", "") - if scoped: - provider, connection_id, database_key, model, raw_name = scoped - if provider != "odoo": - raise err(404, "unknown_odoo_field", "that field belongs to another connector") - if connection_id == "default": - connection_id = None - if database_key is None: - candidates = [row for row in odoo_relational.grid_choices(session.runtime, connection_id) - if row.get("model") == model] - if len(candidates) > 1: - raise err(400, "database_required", "choose the database for this field") - if candidates: - database_key = candidates[0]["key"] - namespace = None - if database_key: - database_key, namespace, choice = _odoo_database_target( - session.runtime, database_key, connection_id) - if model != choice.get("model"): - raise err(404, "unknown_odoo_field", "that model does not belong to this database") - canonical = _odoo_field_key(session.runtime, database_key, model, raw_name) - if scoped and key != canonical: - raise err(409, "database_connection_mismatch", - "the field belongs to a different connection") - key = canonical - _snapshot = user_tables.odoo_schema_snapshot(session.runtime) - try: - import odoo_relational - _discovered_complete = odoo_relational.readable_model_catalog( - session.runtime, namespace).get("complete") is True - except Exception: # noqa: BLE001 - _discovered_complete = False - if (namespace is not None or _snapshot.get("complete") is not True) and not _discovered_complete: - raise err(409, "odoo_schema_unavailable", - "run a successful schema sync before changing raw-field enablement") - known = {field["key"] for field in _schema_catalog_rt( - session.runtime, database_key, namespace)} - if key not in known: - raise err(404, "unknown_odoo_field", "that field is not in the Odoo schema catalogue") - enabled = request.get("enabled") - if not isinstance(enabled, bool): - raise err(400, "bad_enabled", "enabled must be true or false") - previous_enabled = (session.runtime.get(_ODOO_SCHEMA_ENABLEMENTS) or {}).get(key) is True - session.runtime.update( - _ODOO_SCHEMA_ENABLEMENTS, - lambda current: {**(current if isinstance(current, dict) else {}), key: enabled}, - flush="sync") - _evict_odoo_product_pools(session.runtime) - job = None - if database_key and choice.get("enabled"): - try: - job = _queue_odoo_database(session.runtime, database_key, model=model, - changed=previous_enabled != enabled) - except Exception: - raise err(503, "sync_queue_unavailable", - "The field settings were saved, but row sync could not be queued. Save again.") - return {"key": key, "enabled": enabled, "source": "Odoo", - "permissionWall": "normal field permissions", "job": job} - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W49-T12 · RULING R26 · CONTRACT C4 — ONE CONNECTOR, IN FULL, FOR THE MODAL. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# R26: *"The connector popup is a centered MODAL with tabs Overview, Keychain, Databases, -# Relationships, opened from the card. The Keychain moves INTO it and Settings > Keychains becomes -# a link to the modal (one door). Field enablement moves into the Databases tab."* -# -# ⛔ ALL FOUR HALVES ARE COMPOSED NOW even though only Overview renders this wave, and that is -# C4's own instruction. The alternative — ship `overview` and add the rest as each tab lands — -# would make the DTO's shape a moving target across four tickets and give T13/T14/T15 nothing to -# build against. Every half is read from the source that already answers the question: -# -# overview · the directory row this module already composes, plus the Databases labels -# keychain · `routes_keychain.visible_entries`, i.e. the exact rows the Keychains room -# shows THIS reader, with a member's masked preview already blanked by R4 -# databases · `odoo_relational.grid_choices` (derived from `TABLES`, never a second list) -# plus the Odoo field catalogue this module already serves at -# `/connectors/odoo/schema` -# relationships · `relationship_graphs` UNCHANGED — the same DTO the directory already binds -# -# ⛔ AND NOTHING IS FABRICATED. A connector with no introspectable schema gets `databases: []`; a -# connector with no credential door gets an empty `entries` list and `canConnect: false`; a -# connection with no materialised tables gets the graph's own `unavailable` envelope. The rule this -# module was built on is that a plausible-looking placeholder is worse than an honest empty. - - -def _entry_type_for(connector_key): - """The keychain entry type behind one card, or `""` when the card has no stored credential. - - ⛔ DERIVED FROM `TOKEN_CONNECTORS`, NOT RE-TYPED. This module's header rule is that the row set - is composed from the registries that already exist; a second hand-written - `{"stripe": "stripe", ...}` map here would rot in exactly the way the header describes, one - connector at a time. Odoo is the one built-in whose credential lives in the keychain, so it is - named once, beside the row that names it. - """ - key = str(connector_key or "") - if key == "odoo": - return "odoo" - # ⛔⛔ FOUND ONLY ON THE MERGED TREE, AT THE WAVE-53 COMBINE, AND IT IS A 500 ON A LIVE ROUTE. - # `directory` answers 200 with a NAMED gap when any one contributor registry raises (W32-T14 / - # R6: "it used to 500 and show nothing"), and every contributor is called inside that guard. - # The `allowedScopes` pass added this wave runs AFTER the composition, OUTSIDE the guard, and - # this loop is a second unguarded read of `TOKEN_CONNECTORS` - so a broken token registry took - # the whole page back to a 500. Neither lane could see it: one owns this route, the other owns - # the gate that drives the broken registry, and only the merged tree holds both. - # ⚠ `""` is not a swallow, it is this function's own documented answer for a row with no - # keychain type, and `allowed_scopes("")` offers BOTH scopes - the permissive default the - # composer already states, chosen because the toggle must never hide a connection the reader - # actually holds. A token connector's rows are already absent from the set in this state. - try: - for token in TOKEN_CONNECTORS: - if str(token.get("key") or "") == key: - return str(token.get("entry") or "") - except Exception: # noqa: BLE001 - a registry, not a bug - return "" - return "" - - -#: ⭐⭐ W49-T13 — WHAT A CONNECTOR ASKS FOR WHEN YOU CONNECT IT, declared server-side. -#: -#: ⛔ THE CLIENT MUST NOT OWN THIS MAP. R26 moves the add-a-key form into the connector modal, and -#: the obvious client implementation is `type === "odoo" ? [url, db, user, api_key] : [api_key]` — -#: a second copy of a credential SHAPE that already lives in `harness/connectors/odoo.py` and in -#: `routes_keychain._env_odoo_fields`. The wave-9 law is about vocabularies and this is one: a -#: connector whose credential grows a field would take a client edit to become storable. -#: ⚠ `secret` decides the input type, so a field marked secret is typed into a password box and -#: never echoed back — the masked preview is the only form the stored value takes afterwards. -#: Sentence case with the initialism capitalised, which is what the rest of this app does. -_AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret", - "refresh_token": "Refresh token", "seller_id": "Seller ID", - "marketplace_id": "Marketplace ID"} -_CONNECT_FIELDS = { - "odoo": [{"key": "url", "label": "Server URL (https://...)", "secret": False}, - {"key": "db", "label": "Database", "secret": False}, - {"key": "user", "label": "API user", "secret": False}, - {"key": "api_key", "label": "API key", "secret": True}], - # ⭐ THE KEYS AND THEIR ORDER ARE DERIVED, NEVER TYPED. `amazon_client.CREDENTIAL_FIELDS` is - # the one roster, and lane G's `verify_amazon` asserts this sequence against it, so a field - # added there appears here rather than drifting. Labels are the one thing NOT derived: - # `"client_id".replace("_", " ").capitalize()` renders "Client id", and this app writes the - # initialism. The map carries a derived fallback so a new credential field still renders. - "amazon_sp": [ - {"key": key, - "label": _AMAZON_FIELD_LABELS.get(key, key.replace("_", " ").capitalize()), - "secret": key in ("client_secret", "refresh_token"), - **({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})} - for key in amazon_client.CREDENTIAL_FIELDS - ], -} -#: Everything else is one pasted token. The field name is `api_key` because that is the first of -#: `core.keychain._PREVIEW_FIELDS`, so the stored entry gets a preview rather than a blank cell — -#: the wave-31 defect that comment records, avoided rather than repeated. -_CONNECT_TOKEN_FIELD = [{"key": "api_key", "label": "API key", "secret": True}] - - -def _safe_connector_status(connector_key, payload): - """Apply the provider's disclosure guard at the status boundary, outside fallback catches.""" - if connector_key == "amazon_sp": - import connectors_amazon as cx - return cx.assert_safe_status(payload) - return payload - - -def _sync_stamp(value): - """Return a parseable public timestamp, never a creation-time freshness substitute.""" - text = str(value or "").strip() - if not text: - return None - try: - parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - return parsed.astimezone(timezone.utc).isoformat() - except (TypeError, ValueError, OverflowError): - return None - - -def _database_sync_health(*, enabled=True, state="never_run", last_success=None, - source_timestamp=None, materialized_timestamp=None, row_count=None, - error=None, now=None): - """Normalize one database's durable facts into S63-T34's shared public vocabulary.""" - clean_state = str(state or "never_run") - if clean_state not in DATABASE_SYNC_STATES: - clean_state = "failed" - error = {"code": "sync_state_unreadable", - "message": "This database's saved sync state could not be read."} - if not enabled: - clean_state = "disabled" - count = row_count if type(row_count) is int and row_count >= 0 else None - success = _sync_stamp(last_success) - source = _sync_stamp(source_timestamp) - materialized = _sync_stamp(materialized_timestamp) - safe_error = None - if error is not None: - raw = error if isinstance(error, dict) else {} - requested_code = str(raw.get("code") or "") - code = requested_code if requested_code in _DATABASE_SAFE_ERRORS else "connector_sync_failed" - safe_error = { - "code": code, - # Raw provider and exception text never crosses this boundary. - "message": _DATABASE_SAFE_ERRORS.get(code, "This database could not be synced."), - } - # A current failure may retain the prior success timestamp, but it can never look current. - lag = _lag_seconds(success, now) if success else None - return {"state": clean_state, "lastSuccess": success, - "sourceTimestamp": source, "materializedTimestamp": materialized, - "rowCount": count, "lagSeconds": lag, "error": safe_error} - - -def _odoo_database_sync_health(raw, enabled=True, *, now=None): - row = raw if isinstance(raw, dict) else {} - stages = row.get("stages") if isinstance(row.get("stages"), dict) else {} - failure = row.get("failure") if isinstance(row.get("failure"), dict) else None - raw_state = str(row.get("state") or "unknown") - stage_states = {str((value or {}).get("state") or "unknown") - for value in stages.values() if isinstance(value, dict)} - last_success = row.get("lastSuccess") - if failure or "failed" in stage_states or raw_state == "failed": - state = "partial" if last_success else "failed" - elif raw_state == "running" or "running" in stage_states: - state = "running" - elif raw_state == "queued" or "queued" in stage_states: - state = "queued" - elif last_success or raw_state == "ready": - state = "succeeded" - else: - state = "never_run" - source_stage = stages.get("source") if isinstance(stages.get("source"), dict) else {} - material_stage = (stages.get("materialization") - if isinstance(stages.get("materialization"), dict) else {}) - return _database_sync_health( - enabled=enabled, state=state, last_success=last_success, - source_timestamp=source_stage.get("lastSuccess"), - materialized_timestamp=material_stage.get("lastSuccess"), - row_count=row.get("rowCount"), error=failure, now=now) - - -def _amazon_database_sync_health(raw, enabled=True, *, now=None): - if not enabled: - return _database_sync_health(enabled=False) - if not isinstance(raw, dict): - return _database_sync_health( - state="failed", error={"code": "sync_status_unavailable", - "message": "Sync status is unavailable for this database."}) - success = raw.get("lastSuccess") - if raw.get("stuck") is True: - state = "partial" if success else "failed" - error = {"code": "sync_stalled", - "message": "This database sync is stalled."} - elif raw.get("inProgress") is True: - state, error = "running", None - elif success: - state, error = "succeeded", None - elif raw.get("deferred"): - state, error = "queued", None - else: - state, error = "never_run", None - return _database_sync_health( - state=state, last_success=success, materialized_timestamp=success, - row_count=raw.get("rowsLoaded"), error=error, now=now) - - -def _stored_database_sync_health(raw, enabled=True, *, now=None): - row = raw if isinstance(raw, dict) else {} - return _database_sync_health( - enabled=enabled, state=row.get("state") or "never_run", - last_success=row.get("lastSuccess"), source_timestamp=row.get("sourceTimestamp"), - materialized_timestamp=row.get("materializedTimestamp"), row_count=row.get("rowCount"), - error=row.get("error"), now=now) - - -def _detail_connect(entry_type, is_admin): - """C4's `keychain.connect` half: what the Connect form must ask for, and what it may store. - - ⭐ AN ADDITION TO C4, DECLARED. The contract pins `entries`, `canConnect` and `canDisconnect`; - it does not say where the form gets its fields, and the only two answers are "the server says" - and "the client re-derives". This is the first. - - `scope` carries the credential's own rule rather than a permission: a tenant-wide type has - exactly one legal scope, so the form states it instead of offering a picker whose save 400s - (`routes_keychain.add_key` refuses it, and R7's banned shape is a button that leads nowhere). - """ - if not entry_type: - return None - fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD) - tenant_wide = entry_type in TENANT_WIDE_TYPES - return {"type": entry_type, - "fields": [dict(field) for field in fields], - # A member may always store a personal one; business-wide is an administrator's to - # give (R4), and a tenant-wide type is business-wide by construction. - "scopes": (["business"] if tenant_wide - else ["business", "personal"] if is_admin - else ["personal"]), - "tenantWide": tenant_wide} - - -def _replacement_field_specs(entry_type): - """The write-only fields that may change on an existing credential. - - This is deliberately a projection of the Connect descriptor rather than another provider - vocabulary. URL, database, account and every future non-secret field stay server-held during - replacement. Optional rotation metadata is accepted separately from this descriptor. - """ - spec = _detail_connect(entry_type, True) or {} - return [dict(field) for field in (spec.get("fields") or []) - if field.get("secret") is True and str(field.get("key") or "").strip()] - - -def _oauth_slot_entries(rt, uname, row): - """A per-user OAuth connection, rendered as C4's `keychain` entry shape. - - ⛔⛔ THE KEYCHAIN TAB HAS A SECOND SOURCE, AND WITHOUT IT AN OAUTH CARD SAYS THE WRONG THING. - `_entry_type_for` covers `odoo` and `TOKEN_CONNECTORS` only, because an OAuth credential is - NOT a keychain entry: it lives in `core/keychain`'s separate `user_secrets` bucket, which - `list_entries` deliberately does not walk (*"a refresh token is identity, not shared - infrastructure"*). So an OAuth card asked for its entries got `[]` — i.e. *"nobody has - connected this"* about a provider this very reader may be signed in to. That is the same - confidently-wrong shape `accessSentence` was corrected for one tab over. - ⚠ AND IT IS NOT WIDENED INTO `_entry_type_for`. These are different stores with different - owners and different doors; teaching one function to answer for both would put a keychain - entry type on a credential that has none. - - ⛔ `testable` IS FALSE HERE, AND THAT IS THE HONEST VALUE RATHER THAN THE CAUTIOUS ONE. - `POST /admin/keychain/{id}/test` resolves its id through `visible_entries` -> `list_entries`, - which walks the `entries` bucket ONLY, so a Test control on a user secret would 404 every - time. A button that cannot work is exactly what R6 removed from the Google card. - - ⚠ NO SECRET IS READ. `oauth_connect.status` returns the account email and two flags; the - refresh token itself is never opened, and `masked` is empty because there is nothing about - this credential a preview could honestly show. - """ - slug = str(row.get("key") or "") - try: - state = (oauth_connect.status(rt, uname) or {}).get(slug) or {} - except Exception: # noqa: BLE001 - return [] - if not state.get("connected"): - return [] - email = str(state.get("email") or "") - label = str(row.get("label") or slug) - return [{"id": f"oauth:{slug}", - "label": f"{label} ({email})" if email else label, - "masked": "", - # A per-user slot is personal by construction, in the vocabulary `routes_keychain` - # already declares. Never a third word for the same idea. - "scope": "personal", - "testable": False, - "replaceable": False}] - - -def _detail_keychain(rt, uname, is_admin, row): - """C4's `keychain` half: the credential rows THIS reader may see, and what they may do. - - ⚠ `visible_entries` IS CALLED WITH THE READER'S OWN ADMIN FLAG, deliberately. R4 blanks the - masked preview on a business entry a member did not create, and passing `True` to get a fuller - answer would hand a member four characters of the workspace's Odoo key. The restricted default - is the correct one and it is passed through rather than routed around. - - ⚠ A KEYCHAIN THAT CANNOT BE READ COSTS THE ENTRY LIST AND NOTHING ELSE, and it is not silent: - `directory()` records that failure in `incomplete`, and the page this modal opens from renders - that banner ABOVE the card grid. The reader has already been told before they click. - """ - entry_type = _entry_type_for(row.get("key")) - manage = str(row.get("manage") or "") - # The door this reader was offered on the CARD, not a second opinion about it: `directory` - # already decided admin vs member vs nothing, and deciding again here is how two answers to - # one question get born ([[one-question-two-normalizers]]). - can_connect = manage in ("keychain", "keychain_personal") - can_replace = bool(is_admin and can_connect and _replacement_field_specs(entry_type)) - entries = [] - if entry_type: - try: - import routes_keychain as rk - for entry in rk.visible_entries(rt, uname, bool(is_admin)) or []: - if str(entry.get("type") or "") != entry_type: - continue - entries.append({ - "id": str(entry.get("id") or ""), - "label": str(entry.get("label") or entry.get("id") or ""), - "masked": str(entry.get("preview") or ""), - "scope": str(entry.get("scope") or rk.DEFAULT_SCOPE), - # `keychain.test_entry` answers for every stored entry: a registered prober for - # the type when the connector layer filled that slot, and a read-only shape or - # xmlrpc check otherwise. The flag says whether a Test control would have - # something to call, and it always does. - "testable": True, - # W55-T18 / C10: credential replacement is an administrator action. The - # client reads this wall instead of inferring it from scope or from whether - # a masked preview happened to be visible. The replacement endpoint changes - # this row's encrypted payload in place; no secret is added to this DTO. - "replaceable": can_replace, - **({AMAZON_ROTATION_DEADLINE_FIELD: - _stored_amazon_rotation_deadline(rt, entry.get("id"))} - if entry_type == "amazon_sp" else {}), - }) - except Exception: # noqa: BLE001 - entries = [] - # ⭐ THE SECOND SOURCE. An OAuth card has no keychain entry type at all, so `entries` above is - # empty for it by construction and this is the only reading that can fill the tab. - if not entry_type and manage == "oauth": - entries = _oauth_slot_entries(rt, uname, row) - # `startUrl` is the Connect door for this kind, and `disconnect` is a per-user route, so - # both answers come from the ROW rather than from a credential door this card has none of. - return {"entries": entries, - "canConnect": bool(row.get("startUrl")), - "canDisconnect": bool(entries), - "connect": None} - return {"entries": entries, - "canConnect": can_connect, - "canDisconnect": bool(can_connect and entries), - # What the Connect form asks for. `None` when this card has no credential door at all, - # which is what stops the client inventing one (Webhooks, the Scraper, a planned row). - "connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None} - - -def _odoo_catalog_by_model(rt, connection_id=None): - """`{odoo model: (fields, excluded)}` from the schema catalogue this module already serves. - - `excluded` is a DIFF, not a guess: every raw field the snapshot captured that the catalogue did - not offer. Today that diff is empty, which is the honest answer, and it fills itself the moment - the catalogue starts filtering instead of needing somebody to remember. - """ - raw_models = _odoo_raw_models(rt, connection_id=connection_id) - offered = {} - for row in _schema_catalog_rt(rt, connection_id=connection_id): - offered.setdefault(str(row.get("model") or ""), []).append(row) - out = {} - for model, raw_fields in raw_models.items(): - rows = offered.get(str(model), []) - offered_raw = {str(row.get("rawName") or "") for row in rows} - out[str(model)] = ( - [{"key": str(row.get("key") or ""), - "label": str(row.get("label") or ""), - "type": str(row.get("type") or "text"), - "source_field": str(row.get("rawName") or ""), - "enabled": bool(row.get("enabled"))} - for row in rows], - [{"source_field": str(name), - "reason": "This source field was captured from Odoo but the field catalogue did not " - "offer it."} - for name in sorted(raw_fields or {}) if str(name) not in offered_raw], - ) - return out - - -def _spec_field(item, name): - """One attribute off a C2 table spec, whether lane D ships dataclasses or dicts.""" - if isinstance(item, dict): - return item.get(name) - return getattr(item, name, None) - - -def _connector_spec_items(connector_key): - """C2's raw table specs for one connector, as a sequence of items, or `()`. - - ⭐⭐ W53-T30 — EXTRACTED FROM `_connector_spec_tables` (W53-T27) SO THERE IS STILL EXACTLY ONE - IMPORT PATH TO LANE D'S MODULE. Two consumers now read the spec: the Databases tab wants - `native` and `label`, the relationship graph wants `links`. A second `from core import - connector_spec` with its own lookup order is how the two would start disagreeing about which - attribute of D's module is the spec, and only one of them would be exercised by any given - tenant. The lookup order, the failure posture and the reasons for both are unchanged and are - documented on `_connector_spec_tables` below. - """ - try: - from core import connector_spec # noqa: PLC0415 - lane D's module (C2) - except Exception: # noqa: BLE001 - return () - key = str(connector_key or "") - # The directory uses the keychain type; the table registry uses the provider name. - if key == "amazon_sp": - key = "amazon" - found = None - for name in ("tables_for", "for_connector", "tables", "spec", "spec_for"): - fn = getattr(connector_spec, name, None) - if not callable(fn): - continue - try: - got = fn(key) - except Exception: # noqa: BLE001 - continue - if got: - found = got - break - if not found: - for name in ("SPECS", "TABLES", "CONNECTOR_TABLES"): - blob = getattr(connector_spec, name, None) - if isinstance(blob, dict): - found = blob.get(key) or list(blob.values()) - elif isinstance(blob, (list, tuple)): - found = list(blob) - if found: - break - return list(found.values()) if isinstance(found, dict) else list(found or ()) - - -def _connector_spec_links(connector_key): - """C2's DECLARED relations for one connector: `{table key: [(from, target, to, label)]}`. - - ⭐⭐ W53-T30 / C2 — THE GRAPH'S SECOND SOURCE OF EDGES, and the contract's own words for it: - each spec row carries `links`, a list of `(from_col, target_key, to_col, cardinality)`. Today - every edge on this graph is read off a stored FIELD bag, which can only ever describe a - relation somebody wrote a column for; a spec-declared relation between two tables that carry - no link column has nowhere else to come from. - - ⛔ AN EDGE FROM HERE IS HELD TO THE SAME TWO PROOFS AS ONE FROM A FIELD BAG, in the caller: - both endpoints must be permitted NODES on this graph and both named columns must be VISIBLE - to this reader. A spec is a declaration about the shipped schema, never a permission decision - about a person, and an edge drawn on the strength of the declaration alone would name a column - an administrator hid from the account looking at it. - - ⛔ AND IT IS NOT AN OUTAGE WHEN LANE D'S MODULE IS ABSENT: `{}` means "no spec-declared - relations", the field bags still answer, and the graph is exactly what it is today. - - ⚠ `cardinality` IS NOT A LABEL and is not read as one. The wire's `label` is what the reader - sees on the line, and "one to many" is a fact about the join rather than a name for it, so the - caller labels a spec edge with the SOURCE COLUMN's own label instead. - """ - out = {} - for item in _connector_spec_items(connector_key): - table_key = str(_spec_field(item, "key") or "") - links = _spec_field(item, "links") - if not table_key or not isinstance(links, (list, tuple)): - continue - edges = [] - for link in links: - if not isinstance(link, (list, tuple)) or len(link) < 3: - continue - from_col = str(link[0] or "").strip() - target_key = str(link[1] or "").strip() - to_col = str(link[2] or "").strip() - if from_col and target_key and to_col: - edges.append((from_col, target_key, to_col)) - if edges: - out[table_key] = edges - return out - - -def _connector_spec_tables(connector_key): - """C2's spec for one connector as `{table key: {"native", "label"}}`, or `{}`. - - ⛔⛔ `core.connector_spec` IS LANE D'S MODULE AND IT IS NOT IN THIS TREE YET. C2 splits the - work: lane D declares the spec (`key`, `native`, `source`, `label`, `pinned`, `identity`, - `links`) and this lane CONSUMES it through `GET /connectors/{key}/detail`. So the import is - defensive and the fallback is the in-tree source each value already has — Odoo's native model - from `odoo_relational.ODOO_TABLE_MODELS`, its default label from `grid_choices`, Amazon's - report type from `connectors_amazon.REPORT_TYPES` and its label from `LABELS_AMAZON`. When D's - module lands, the spec WINS with no edit here. - - ⚠ D'S EXACT API IS NOT FROZEN, so the lookups tried are written down rather than guessed at by - the next reader: the callables `tables_for(key)`, `for_connector(key)`, `tables(key)`, - `spec(key)`, `spec_for(key)`, then the module attributes `SPECS`, `TABLES`, - `CONNECTOR_TABLES` (a dict keyed by connector, a dict keyed by table, or a flat sequence). - Whatever answers first is normalised by TABLE KEY, and only the keys this connector already - declares are ever consulted — so a spec carrying other connectors' tables is harmless. - ⚠ W53-T30 — THAT LOOKUP NOW LIVES IN `_connector_spec_items` ABOVE, unchanged, because the - relationship graph reads the same spec for its `links`. One import path, two readings. - - ⛔ A SPEC THAT CANNOT BE READ IS NOT AN OUTAGE. Every failure returns `{}` and the row keeps - the in-tree value it has today: C2 is a better source for these two strings, never the only - one, and a Databases tab that 500s because a sibling lane's module changed shape would be a - disclosure screen taken down by a naming question. - """ - out = {} - for item in _connector_spec_items(connector_key): - table_key = str(_spec_field(item, "key") or "") - if not table_key: - continue - out[table_key] = {"native": str(_spec_field(item, "native") or ""), - "label": str(_spec_field(item, "label") or "")} - return out - - -def _database_name_overrides(rt): - """This tenant's per-database name overrides: `{table key: {"name": ...}}`, or `{}`. - - ⛔⛔ `nav_meta` IS THE STORE, AND IT IS READ THROUGH THE NAVIGATION RAIL'S OWN READER. R9, - verbatim: *"the rename database under the Navigation rail also works, it should then sync to - the navigation in the connector too."* That rename has written `nav_meta` since wave 19 - (`routes_nav.save_nav_meta`), and `routes_tables.ut_label` already resolves a served name as - nav_meta's name, then the definition's label, then the key. Calling `routes_nav._read_nav_meta` - rather than re-deriving the document shape here is what makes the rail and this tab provably - ONE fact instead of two answers to one question. - - ⛔ AND IT IS WHY NOTHING HERE WRITES A TABLE DEFINITION'S LABEL. `user_tables.set_label` is - rewritten by a connector re-sync (`odoo_relational.apply_plan` rebuilds definitions), so an - override stored there would be silently clobbered on the next sync, which is precisely the - R9 violation this ticket exists to prevent. - - ⚠ ONE READ FOR THE WHOLE LIST, never one per row: `store.get` deep copies the tenant document - on every call, and this list is eight rows on Odoo and seven on Amazon. - ⚠ A store blip is "no overrides", never a failed page. The default name is still true. - """ - try: - import routes_nav # noqa: PLC0415 - lazy, avoids a cycle - meta = routes_nav._read_nav_meta(rt) or {} # noqa: SLF001 - the ONE reader (R9) - except Exception: # noqa: BLE001 - return {} - return {str(k): v for k, v in meta.items() if isinstance(v, dict)} - - -def _rename_rights(rt, uname, is_admin, keys): - """`{table key: {exists, canRename}}` for R9's write wall and its prerequisite. - - ⛔ THE SAME PREDICATE `routes_nav.save_nav_meta` APPLIES TO A `ut_` KEY, not a second one: - `user_tables.may_open`, i.e. the table's creator, a tenant admin, or a share grant. Two ideas - of who may rename a database would show one person a control that the other door refuses. - - ⛔ A DATABASE THIS CONNECTION HAS NOT CREATED YET IS NOT RENAMEABLE, and that is not a - technicality: `nav_meta` is keyed by TABLE key, so a name stored for a table nobody can open - would sit in the tenant's document naming nothing, and the rail would never show it. - - ⭐ W56-T31: `enabled` USED TO MEAN "will be included in the next sync", while the screen - read as though the database already existed. `canRename` was telling the truth about a table - that did not exist. Returning `exists` from this same projection lets the client distinguish - that state from a reader who may not open an existing table. - - ⚠ IT IS SERVED PER ROW because the answers ARE per row. One `lend_defs` read answers the - whole list; that projection carries definitions without rows (~0.1% of the document's bytes). - """ - try: - import core.user_tables as user_tables # noqa: PLC0415 - lazy, as elsewhere - definitions = user_tables.lend_defs(rt) - facts = {} - for key in keys: - exists = bool(user_tables.get(key, st=definitions)) - facts[key] = { - "exists": exists, - "canRename": bool(str(key).startswith("ut_") and exists - and user_tables.may_open( - key, uname, bool(is_admin), st=definitions)), - } - return facts - except Exception: # noqa: BLE001 - # Fail closed. A wall that could not be read is not a wall that admitted you. - return {key: {"exists": False, "canRename": False} for key in keys} - - -def _named_databases(rt, is_admin, uname, rows): - """The LAST step of every databases list, Odoo's and Amazon's alike: the three names. - - One place, deliberately. Each half of `_detail_databases` builds its own `native` and its own - shipped default from its own source, and then BOTH come through here — so "what is this - database called for this tenant" is answered once, by `schema_overlay.merge_database_labels`, - and a third connector cannot arrive with a fourth opinion about a blank override. - """ - named = schema_overlay.merge_database_labels(rows, _database_name_overrides(rt)) - rights = _rename_rights(rt, uname, is_admin, [str(row.get("key") or "") for row in named]) - for row in named: - facts = rights.get(str(row.get("key") or "")) or {} - row["exists"] = facts.get("exists") is True - row["canRename"] = facts.get("canRename") is True - return named - - -def _clean_scraper_recipe(raw): - """Return T11's persisted recipe shape and explicit structural validation errors. - - This function is intentionally pure. Preview and creation both call it, so opening a saved - recipe cannot reinterpret it differently, and validation cannot accidentally become a paid - provider probe. - """ - value = raw if isinstance(raw, dict) else {} - kind = str(value.get("kind") or "").strip().lower() - errors = [] - if kind not in SCRAPER_RECIPE_KINDS: - errors.append("Choose Instagram, TikTok, or Google Maps.") - return {}, errors - - limits = value.get("limits") if isinstance(value.get("limits"), dict) else {} - schedule = value.get("schedule") if isinstance(value.get("schedule"), dict) else {} - schedule_mode = str(schedule.get("mode") or "manual").strip().lower() - if schedule_mode != "manual": - errors.append("This recipe must use the manual Run now schedule.") - - if kind == "google_maps": - seed = value.get("seed") if isinstance(value.get("seed"), dict) else {} - search = str(seed.get("search") or value.get("search") or "").strip() - geography = str(seed.get("geography") or value.get("geography") or "").strip() - raw_cap = limits.get("records", value.get("limit")) - if not search: - errors.append("Enter a category or search term.") - if not geography: - errors.append("Enter a geography.") - if raw_cap in (None, ""): - errors.append("Enter a result cap.") - cap = 0 - else: - try: - cap = int(raw_cap) - except (TypeError, ValueError): - cap = 0 - errors.append("The result cap must be a whole number.") - if raw_cap not in (None, "") and (cap < 1 or cap > 500): - errors.append("The result cap must be between 1 and 500.") - return ({"kind": kind, "source": "places", - "seed": {"search": search, "geography": geography}, - "limits": {"records": cap}, "schedule": {"mode": "manual"}}, errors) - - seed = str(value.get("seed") or "").strip() - raw_limit = limits.get("posts", value.get("limit")) - post_type = str(limits.get("postType") or value.get("postType") or "video").strip().lower() - if not seed: - errors.append(f"Enter an {kind.title()} profile seed.") - if raw_limit in (None, ""): - errors.append("Enter a last-N post limit.") - limit = 0 - else: - try: - limit = int(raw_limit) - except (TypeError, ValueError): - limit = 0 - errors.append("The post limit must be a whole number.") - if raw_limit not in (None, "") and (limit < 1 or limit > SCRAPER_RECIPE_MAX_ROWS): - errors.append(f"The post limit must be between 1 and {SCRAPER_RECIPE_MAX_ROWS}.") - allowed_types = ("video", "image", "carousel") if kind == "instagram" else ("video",) - if post_type not in allowed_types: - errors.append("Choose video, image, or carousel posts." - if kind == "instagram" else - "TikTok's native profile window supports video posts only.") - return ({"kind": kind, "source": "profile", "seed": seed, - "limits": {"posts": limit, "postType": post_type}, - "schedule": {"mode": "manual"}}, errors) - - -def _scraper_recipe_fields(kind): - """Stable user-table schema for one recipe, derived from the shipped normalisers.""" - if kind == "google_maps": - return [dict(field) for field in _MAPS_RECIPE_FIELDS] - import automation_engine as engine # noqa: PLC0415 - existing schema owner - source = engine.POST_FIELDS if kind == "instagram" else engine.TT_POST_FIELDS - # Links and rollups target the shared automation tables. A recipe database is independent, - # so copying those references would create a plausible but broken relationship graph. - return [dict(field) for field in source - if isinstance(field, dict) and field.get("type") not in ("link", "rollup")] - - -def _scraper_plan(recipe): - """Network-free setup preview: declared provider, modelled spend, cap, and credential state.""" - kind = str((recipe or {}).get("kind") or "") - limit_key = "records" if kind == "google_maps" else "posts" - limit = int(((recipe or {}).get("limits") or {}).get(limit_key) or 0) - if kind == "google_maps": - import providers # noqa: PLC0415 - registry owns cost/capability - apify = providers.PROVIDERS["apify"] - capability = apify.cap("google_maps_stores") - per_record = float(capability.cost_per_record or 0.0) if capability else 0.0 - amount = round(per_record * limit, 4) - return {"provider": apify.label, "credentialReady": apify.configured(), - "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit, - "canRun": True, - "note": "Modelled from the declared per-place rate; the provider receipt " - "reconciles the bounded result count."} - - import providers # noqa: PLC0415 - registry is the authority - bright = providers.PROVIDERS["brightdata"] - apify = providers.PROVIDERS["apify"] - if kind == "tiktok": - per_record = float(bright.cap("tt_post_metrics").cost_per_record or 0.0) - profile_cost = float(bright.cap("tt_profile").cost_per_record or 0.0) - estimate_usd = profile_cost + (per_record * limit) - provider_label = bright.label - credential_ready = bright.configured() - else: - # The established Instagram runner can use Bright Data and its Apify fallback. The cap - # includes both post metrics and view enrichment, so it never hides the worst declared - # cost from the person authorising Run now. - bd_cost = float(bright.cap("ig_post_metrics").cost_per_record or 0.0) - profile_cost = float(bright.cap("ig_profile").cost_per_record or 0.0) - view_cost = float(apify.cap("ig_post_views").cost_per_record or 0.0) - estimate_usd = profile_cost + ((bd_cost + view_cost) * limit) - provider_label = (bright.label if bright.configured() or not apify.configured() - else apify.label) - credential_ready = bright.configured() or apify.configured() - amount = round(estimate_usd, 4) - return {"provider": provider_label, "credentialReady": bool(credential_ready), - "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit, - "canRun": True, "note": "Modelled from declared per-record list rates."} - - -def _scraper_databases(rt, is_admin, uname): - """Saved recipe databases this reader may open, with no row payloads or secrets.""" - try: - import core.user_tables as user_tables - definitions = user_tables.all_defs(rt) - lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions}) - except Exception: # noqa: BLE001 - return [] - rows = [] - for key, definition in (definitions or {}).items(): - recipe = definition.get("scraperRecipe") if isinstance(definition, dict) else None - if not isinstance(recipe, dict): - continue - try: - if not user_tables.may_open(key, uname, bool(is_admin), st=lent): - continue - except Exception: # noqa: BLE001 - continue - cleaned, errors = _clean_scraper_recipe(recipe) - if errors: - continue - plan = _scraper_plan(cleaned) - # Recipe existence alone is not enablement. The creation door stamps both ownership - # markers; a legacy/malformed row without them is disclosed as disabled and cannot run. - enabled = (definition.get("connectorKey") == "scraper" - and definition.get("recordMode") == "automation") - rows.append({"key": str(key), "label": str(definition.get("label") or key), - "defaultLabel": str(definition.get("label") or key), - "native": cleaned["kind"], "enabled": enabled, "exists": True, - "canRename": True, "sourceRows": None, "populated": None, - "fields": [dict(field) for field in (definition.get("fields") or ()) - if isinstance(field, dict)], - "excluded": [], "sharesFieldsWith": [], "scraperRecipe": cleaned, - "syncHealth": _stored_database_sync_health( - definition.get(user_tables.CONNECTOR_HEALTH_FIELD), enabled), - "scraperRuns": [dict(item) for item in (definition.get("scraperRuns") or ()) - if isinstance(item, dict)], - "runPath": f"/connectors/scraper/databases/{key}/run", - "canRun": enabled and plan["canRun"] is True}) - return _named_databases(rt, is_admin, uname, rows) - - -def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None): - """C4's `databases` half: what this connector materialises, and the fields behind each one. - - ⛔ ODOO IS THE ONLY INTROSPECTABLE CONNECTOR TODAY and every other card answers `[]`. That is - the contract's own instruction, and the alternative would be a list of database names no - connector actually creates. - - ⛔ THE ROW SURVIVES FOR A NON-ADMIN; ONLY THE CATALOGUE HALVES EMPTY. Returning `[]` to a - member would be indistinguishable from *"this connector creates no databases"*, which is the - confidently-wrong shape this module's header was written about. A database's NAME and whether - it is switched on are facts about their own workspace; the raw Odoo field catalogue is - administrator-only (`_odoo_admin` gates the route that serves it) and stays that way. - - Fields with the same source model have independent enablement keys in each database. - `sharesFieldsWith` remains an empty list for existing clients of the older shared switches. - - ⭐⭐ W53-T27 / R9 / C2 — THREE NAMES PER ROW, AND THEY ARE THREE DIFFERENT FACTS. `native` is - the VENDOR's own identifier (`res.partner`, or Amazon's report type) and is never editable by - anybody; `defaultLabel` is OUR shipped default; `override` is this tenant's own rename, held - in the `nav_meta` bucket the navigation rail already writes. `label` stays the EFFECTIVE name - because every existing reader of it — this tab's heading, `_detail_overview`'s `creates` list - — already means "what this database is called today", and serving the un-overridden default - there would be the drift R9 exists to close. - - ⛔ ALL THREE SURVIVE FOR A NON-ADMIN, like `key` and `enabled` above them and for the same - reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE - is walled, per row, by `canRename`. - """ - if str(connector_key) == "scraper": - return _scraper_databases(rt, is_admin, uname) - if not connector_directory.is_introspectable(connector_key): - return [] - spec = _connector_spec_tables(connector_key) - if str(connector_key) == "amazon_sp": - return _named_databases(rt, is_admin, uname, _amazon_databases(rt, is_admin, spec)) - try: - import odoo_relational - namespace = (odoo_relational.connection_namespace(rt, connection_id)[1] - if connection_id else None) - choices = odoo_relational.grid_choices(rt, connection_id) - models = dict(getattr(odoo_relational, "ODOO_TABLE_MODELS", {}) or {}) - # This read never contacts Odoo. A 499-model exact pass measured 67.87 s after the - # catalogue was warm, so modal detail must never turn a missing cache into inline work. - census = odoo_relational.population_census(rt, namespace) - except Exception: # noqa: BLE001 - return [] - health = odoo_health_receipt(rt, connection_id) - health_by_key = {str(item.get("databaseKey") or ""): item - for item in (health.get("databases") or ())} - census_at = str(census.get("capturedAt") or "") - counts = census.get("counts") if isinstance(census.get("counts"), dict) else {} - catalog = {} - if is_admin: - try: - catalog = _odoo_catalog_by_model(rt, namespace) - except Exception: # noqa: BLE001 - catalog = {} - enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {} - # Connection resolution reaches keychain.list_entries and therefore the store (Postgres). - # It ran 12,824 times for fields and database identities, then 501 with a per-key memo. - # Resolve the shared default lazily once instead. This local belongs only to this detail - # build: request scope keeps the answer per runtime and tenant, never process-wide. - default_connection_id = ... - - def resolve_connection_id(key): - nonlocal default_connection_id - base, cid = odoo_relational.split_table_key(key) - if base is None: - return None - if cid: - return cid - if default_connection_id is ...: - default_connection_id = (odoo_relational.default_connection_id(rt) - or odoo_relational.DEFAULT_CONNECTION) - return default_connection_id - - out = [] - for choice in choices: - key = str(choice.get("key") or "") - resolved_connection_id = resolve_connection_id(key) - model = str(choice.get("model") or models.get(key) or "") - fields, excluded = catalog.get(model, ([], [])) - scoped_fields = [] - for field in fields: - field_key = _odoo_field_key(rt, key, model, field["source_field"], - connection_id=resolved_connection_id) - scoped_fields.append({**field, "key": field_key, - "enabled": enabled_fields.get(field_key, - field.get("enabled") if namespace is None else False) is True}) - declared = spec.get(choice.get("baseKey") or key) or {} - # Counts are model-level facts. Databases backed by the same model deliberately show the - # same sourceRows number; inventing a per-database domain here would make the number lie. - source_rows = counts.get(model) if census_at and model in counts else None - database_health = health_by_key.get(key) or { - # Source rows is a model census. It is not this governed database's materialized - # row count, especially when two curated databases share one source model. - "databaseKey": key, "state": "unknown", "rowCount": None, - "lastSuccess": None, "currentLagSeconds": None, "recordRefreshed": None, - "failure": None, "stages": _empty_odoo_stages()} - out.append({"key": key, - "identity": connector_sync.database_identity( - "odoo", resolved_connection_id, key, model), - "label": str(declared.get("label") or choice.get("label") or key), - "defaultLabel": str(declared.get("label") or choice.get("label") or key), - "native": str(declared.get("native") or model), - "enabled": bool(choice.get("enabled")), - "sourceRows": source_rows, - "populated": None if source_rows is None else source_rows > 0, - # Data-free client receipt: source/mirror/materialization/reconciliation/ - # freshness are distinct, and one database's failure cannot make a sibling - # look newly successful. - "sync": database_health, - "syncHealth": _odoo_database_sync_health( - database_health, bool(choice.get("enabled"))), - "fields": scoped_fields, - "excluded": [dict(item) for item in excluded], - "sharesFieldsWith": []}) - return _named_databases(rt, is_admin, uname, out) - - -def _amazon_report_name(table): - """The report type this Amazon table reads, whichever key declares it. - - ⛔ SETTLEMENTS DECLARE `report_discover`, NOT `report` (W60-T36), because Amazon issues them on - its own schedule and refuses a createReport for that type: it is a getReports DISCOVERY filter - and is deliberately absent from `_APPROVED_REPORT_TYPES`. Reading only `report` here printed an - empty Source name for settlements on the connector screen, which reads as "this table has no - report" when the truth is "this table is not requested, it is listed". - """ - import connectors_amazon as cx # noqa: PLC0415 - provider owns state - spec = cx.REPORT_TYPES.get(table) or {} - return str(spec.get("report") or spec.get("report_discover") or "") - - -def _amazon_databases(rt, is_admin, spec=None): - """⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM. - - Fills the shared `ConnectorDatabase` shape and adds an administrator-only Amazon `sync` - block. Members retain the shared shape without the field catalogue or sync bookkeeping. - - ⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five - of the seven tables have NO published schema, because Amazon publishes artifacts only for its - JSON analytics reports and every flat-file report we depend on has none. Their columns are - measured from the report document's own header at first sync, so before that they genuinely - have zero fields. A screen that showed an empty field list with no explanation would say - *"this database has no fields"* when the truth is *"nobody has run it yet"* — the exact - confusion `_detail_introspection`'s header was written about, one layer in. - - ⛔ `sharesFieldsWith` IS REAL HERE, NOT COPIED. `orders` and `order_items` are BOTH filled from - ONE report document (`GET_FLAT_FILE_ALL_ORDERS_DATA_*`, which is one row per order ITEM), so - they share a header and therefore a field list. That is the same class of fact the Odoo pairs - carry, measured the same way: derived from the report map, never hand-typed. - - ⚠ NO FIELD TOGGLES. `canToggleFields` is False for Amazon in `_detail_introspection`; there is - no `PATCH /connectors/amazon_sp/schema/{field}` route and this wave does not build one. Fields - are disclosed, not switched. Database enablement is a separate provider capability and route, - never the generic Odoo-table flag consumed by `DatabasesTab`. - """ - try: - from harness import amazon_schema - import connectors_amazon as cx - except Exception: # noqa: BLE001 - return [] - enablement = cx.read_database_enablement(rt) - enabled = enablement.get("databases") or {} - cat = amazon_schema.catalogue() - tables = cat.get("tables") or {} - states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None - - # Which OTHER declared tables are filled from the SAME report document. Derived from the - # report map so a table added to `REPORT_TYPES` gets this disclosure for free. - by_report = {} - for name in amazon_schema.PROVENANCE: - rep = _amazon_report_name(name) - if rep: - by_report.setdefault(rep, []).append("ut_amzn_" + name) - - out = [] - for name, provenance in amazon_schema.PROVENANCE.items(): - key = "ut_amzn_" + name - spec = tables.get(name) or {} - cols = spec.get("columns") or [] - rep = _amazon_report_name(name) - fields = [{"key": str(c.get("name") or ""), - "label": str(c.get("name") or "").replace("_", " ").capitalize(), - "type": "text" if str(c.get("type")) == "VARCHAR" else "number", - "source_field": str(c.get("name") or ""), - # ⚠ EVERY DISCLOSED FIELD IS ON. There is no per-field switch for Amazon, and - # reporting `enabled: False` would describe a control that does not exist. - "enabled": True} - for c in cols] if is_admin else [] - excluded = [] - if provenance == "measured_at_sync" and not cols: - excluded.append({ - "source_field": "", - "reason": (amazon_schema.PROVENANCE_COPY["measured_at_sync"] - + f" The report is {rep}." if rep - else amazon_schema.PROVENANCE_COPY["measured_at_sync"])}) - # ⭐⭐ W53-T27 / C2 — THE SAME THREE FACTS AS THE ODOO ROWS, one source swapped. Amazon - # has no per-table model name; the identifier AMAZON itself uses for what fills a table is - # the REPORT TYPE (`rep`, already computed above for `sharesFieldsWith`). That is exactly - # what `native` carries: the vendor's own name for the thing, never one we invented. It is - # empty for a table whose report type is unknown, and the tab says so rather than guessing. - declared = (spec or {}).get(key) or {} - raw_sync = (states or {}).get(name) if is_admin else None - is_enabled = enabled.get(key) is True - out.append({"key": key, - "label": str(declared.get("label") or LABELS_AMAZON.get(key) or key), - "defaultLabel": str(declared.get("label") - or LABELS_AMAZON.get(key) or key), - "native": str(declared.get("native") or rep), - # W55-T19 / R7 — AMAZON'S OWN tenant-local answer. This is deliberately not - # read from Odoo's grid config, and disabling it leaves every stored row and - # definition untouched: only this provider config document changes. - "enabled": is_enabled, - **({"sync": raw_sync} if is_admin else {}), - "syncHealth": _amazon_database_sync_health(raw_sync, is_enabled), - "fields": fields, - "excluded": excluded, - "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]}) - return out - - -#: What an empty `excluded` list means for AMAZON, which is not what it means for Odoo. -#: -#: ⛔ A SEPARATE SENTENCE ON PURPOSE. Odoo's basis describes a DIFF of a live snapshot against a -#: catalogue. Amazon has no snapshot and no diff: a table either has a published schema or it -#: measures its columns at first sync. Reusing Odoo's wording would describe a computation that -#: does not happen here, which is the same class of error as reusing its toggles. -_AMAZON_EXCLUDED_BASIS = ( - "Amazon publishes a schema for its JSON analytics reports only. A table with no published " - "schema reads its columns from the report's own header the first time it syncs, so it lists " - "no fields until then. Nothing is being held back from you.") - - -#: Table labels for the Amazon disclosure. ⛔ NO EM DASH OR EN DASH: these reach a screen. -LABELS_AMAZON = { - "ut_amzn_orders": "Amazon orders", - "ut_amzn_order_items": "Amazon order items", - "ut_amzn_listings": "Amazon listings", - "ut_amzn_settlements": "Amazon settlements", - "ut_amzn_inventory": "Amazon inventory", - "ut_amzn_pricing": "Amazon pricing and Buy Box", - "ut_amzn_brand_analytics": "Amazon brand analytics", -} - - -#: What an EMPTY `excluded` list actually means, said once, on the server that computes it. -#: -#: ⛔⛔ THE TRAP THIS CLOSES, MEASURED: `_odoo_catalog_by_model` builds `excluded` as a DIFF of the -#: snapshot's raw fields against the catalogue, and the catalogue offers every raw field it -#: captured — so the diff is EMPTY TODAY BY CONSTRUCTION. A tab that rendered "Excluded: none" -#: over that would be telling an administrator that this workspace holds back nothing from Odoo, -#: which is a claim about POLICY that the number cannot support. The empty list means the -#: catalogue dropped nothing it saw; it says nothing at all about what Odoo never reported. -#: -#: ⚠ IT IS SERVED RATHER THAN WRITTEN IN THE CLIENT because the sentence is a description of a -#: server computation. A client copy would go stale the day the catalogue starts filtering, and -#: nothing would go red. -_EXCLUDED_BASIS = ("A field is excluded when this workspace captured it from Odoo and the field " - "catalogue did not offer it. Anything Odoo never reported cannot appear in " - "this list at all.") - - -def _detail_introspection(rt, is_admin, row): - """R24's half of the Databases tab: may this connector be disclosed, and to THIS reader? - - ⭐ AN ADDITION TO C4, DECLARED, and the same kind of addition `_detail_connect` is. C4 pins - the `databases` list and says nothing about how a client tells "this connector has no schema" - apart from "this connector's schema could not be read". Those two are the same empty list and - they mean opposite things, so the difference is stated rather than inferred. - - ⛔ EVERY FLAG HERE IS A CAPABILITY THE SERVER WILL HONOUR, not a guess the client makes. The - field toggle is `PATCH /connectors/odoo/schema/{field_key}`, which `_odoo_admin` gates and - which raises 409 `odoo_schema_unavailable` while no complete snapshot exists — so - `canToggleFields` carries BOTH conditions and the client renders no control that could only - ever 409. A button that always fails is R7's banned shape. - - ⚠ `fieldsVisible` IS SEPARATE FROM `canToggleFields` ON PURPOSE. A member sees the database - names and their on/off state (their own workspace's facts) and no field catalogue at all - (`_odoo_admin` gates the route that serves it); saying which of the two is missing is what - stops "administrator only" being rendered as "this database has no fields". - """ - key = str(row.get("key") or "") - if key == "scraper": - # Recipe databases publish their saved, stable schema to every reader who may open the - # database. They have no generic schema-toggle route, so no write capability is implied. - return {"introspectable": True, "reason": "", "fieldsVisible": True, - "canToggleFields": False, "canToggleTables": False, - "catalogueComplete": True, "capturedAt": "", "excludedBasis": "", - "censusAt": "", "censusComplete": False, "censusDurationMs": 0, - "censusModelsCounted": 0, "censusModelsWithRows": 0} - if not connector_directory.is_introspectable(key): - reason = ("Nothing is connected behind this listing yet, so there is no schema to read." - if str(row.get("state") or "") == connector_directory.CATALOGUE_STATE - else "This connector does not publish a field catalogue, so the databases it " - "creates cannot be listed field by field.") - return {"introspectable": False, "reason": reason, "fieldsVisible": False, - "canToggleFields": False, "canToggleTables": False, - "catalogueComplete": False, "capturedAt": "", "excludedBasis": "", - "censusAt": "", "censusComplete": False, "censusDurationMs": 0, - "censusModelsCounted": 0, "censusModelsWithRows": 0} - if key == "amazon_sp": - # ⭐⭐ W52 · R6. Introspectable under the AMENDED test (a real schema artifact, not a - # `fields_get`), and deliberately with NO generic toggles: there is no Amazon field-toggle - # route, and `canToggleTables` belongs to Odoo's config surface. W55-T19 serves Amazon's - # database switches through the separate `enablement` capability instead, so this flag - # stays False and can never send Amazon through an Odoo endpoint. - # - # ⛔ `catalogueComplete` IS FALSE AND SAYING SO IS THE HONEST ANSWER. Only two of the seven - # tables have a published artifact; the other five measure their columns from the report's - # own header at first sync. Reporting `True` would claim a completeness that does not - # exist, and reporting `introspectable: False` would hide the two that ARE published. - try: - from harness import amazon_schema - cat = amazon_schema.catalogue() - tables = cat.get("tables") or {} - complete = bool(tables) and all((t.get("columns") or []) for t in tables.values()) - captured = str(cat.get("fetched_utc") or "") - except Exception: # noqa: BLE001 - complete, captured = False, "" - return {"introspectable": True, - "reason": "", - "fieldsVisible": bool(is_admin), - "canToggleFields": False, - "canToggleTables": False, - "catalogueComplete": complete, - # ⚠ THE CATALOGUE'S OWN FETCH DATE, NEVER TODAY'S. R6 is explicit that an - # unstamped pinned catalogue is a stale seed waiting to happen, and this screen is - # the only place the staleness becomes visible. - "capturedAt": captured, - "excludedBasis": _AMAZON_EXCLUDED_BASIS, - "censusAt": "", "censusComplete": False, "censusDurationMs": 0, - "censusModelsCounted": 0, "censusModelsWithRows": 0} - complete = False - captured = "" - census = {} - try: - import core.user_tables as user_tables - snapshot = user_tables.odoo_schema_snapshot(rt) - import odoo_relational - discovered = odoo_relational.readable_model_catalog(rt) - census = odoo_relational.population_census(rt) - has_discovery = bool(discovered.get("capturedAt")) - complete = (discovered.get("complete") is True if has_discovery - else snapshot.get("complete") is True) - captured = str((discovered.get("capturedAt") if has_discovery - else snapshot.get("capturedAt")) or "") - except Exception: # noqa: BLE001 - complete, captured = False, "" - return {"introspectable": True, - "reason": "", - "fieldsVisible": bool(is_admin), - # BOTH conditions, because the write door enforces both. - "canToggleFields": bool(is_admin) and complete, - # ⭐ R22: ANY table can be switched off, not only the ones we mark optional. The write - # is `PUT /admin/connectors/odoo/config`, which is an administrator's door and accepts - # every key `grid_choices` names, including all of them at once. - "canToggleTables": bool(is_admin), - "catalogueComplete": complete, - "capturedAt": captured, - "excludedBasis": _EXCLUDED_BASIS, - "censusAt": str(census.get("capturedAt") or ""), - "censusComplete": census.get("complete") is True, - "censusDurationMs": int(census.get("durationMs") or 0), - "censusModelsCounted": int(census.get("modelsCounted") or 0), - "censusModelsWithRows": int(census.get("modelsWithRows") or 0)} - - -def _detail_overview(row, databases, graph): - """C4's `overview` half: what the modal's first tab reads, top to bottom. - - `creates` is the ENABLED table LABELS this connector stands behind: the declared grids when the - connector declares any, and otherwise the tables its own permitted graph actually found. Both - are real readings of THIS workspace; neither is a provider template. - """ - creates = [str(entry.get("label") or "") for entry in (databases or []) - if entry.get("enabled") is True and str(entry.get("label") or "")] - if not databases and isinstance(graph, dict) and graph.get("status") == "ready": - creates = [str(table.get("label") or "") for table in (graph.get("tables") or ()) - if str(table.get("label") or "")] - return {"key": str(row.get("key") or ""), - "label": str(row.get("label") or ""), - "status": str(row.get("state") or ""), - # Empty when the server has no account to name. The client says so in a sentence - # rather than printing a blank row. - "account": str(row.get("connectedAs") or ""), - # The system the data comes FROM, in the words a customer reads on the card. Never a - # provider slug: an internal identifier on screen is exactly what `parseDirectory` - # drops a row to avoid. - "source": str(row.get("label") or ""), - "creates": creates} - - -def _detail_enablement(rt, is_admin, row): - """A provider-owned enablement capability, or ``None`` when this connector has no door. - - Amazon is intentionally the first entry and is named here rather than inferred from whether a - database list happened to be non-empty. Odoo keeps its model-catalogue and config routes; - every other connector keeps no enablement affordance until its provider declares one. - """ - if str(row.get("key") or "") != "amazon_sp": - return None - try: - import connectors_amazon as cx - return cx.enablement_capability(rt, bool(is_admin)) - except Exception: # noqa: BLE001 - # Fail closed. An unreadable provider capability is no write door. - return {"provider": "amazon_sp", "mode": "database", "canManage": False, - "updatePath": "", "preservesData": True, "available": False} - - -def _scraper_preview(body): - request = body if isinstance(body, dict) else {} - raw_recipe = request.get("recipe") if isinstance(request.get("recipe"), dict) else request - recipe, errors = _clean_scraper_recipe(raw_recipe) - if errors: - raise err(400, "invalid_scraper_recipe", " ".join(errors)) - plan = _scraper_plan(recipe) - missing = [] if plan["credentialReady"] else ["credential"] - return {"recipe": recipe, "inputs": recipe["seed"], "provider": plan["provider"], - "credentialReady": plan["credentialReady"], "estimatedUsd": plan["estimatedUsd"], - "costCapUsd": plan["costCapUsd"], "recordsCap": plan["recordsCap"], - "canRun": plan["canRun"], "missing": missing, "note": plan["note"], - # This is a contract field, not a measurement. A preview and a create never call a - # provider, so both truthfully report zero incurred spend. - "incurredUsd": 0.0} - - -def _create_scraper_database(rt, uname, body): - """Persist one recipe database. Contains no provider import or runner call by construction.""" - if not rt.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - request = body if isinstance(body, dict) else {} - label = str(request.get("label") or "").strip() - if not label: - raise err(400, "missing_database_name", "Enter a database name.") - preview = _scraper_preview(request) - recipe = preview["recipe"] - import core.user_tables as user_tables # noqa: PLC0415 - existing create door - key = user_tables.create(label, uname, fields=_scraper_recipe_fields(recipe["kind"]), - source=user_tables.AUTOMATION_SOURCE, st=rt) - if not key: - raise err(409, "database_not_created", - "The database name is unavailable or this workspace reached its database limit.") - - def _attach(cur): - current = dict(cur or {}) - definition = dict(current.get(key) or {}) - if not definition: - raise RuntimeError("the new database disappeared before its recipe was attached") - definition.update({"connectorKey": "scraper", "recordMode": "automation", - "scraperRecipe": recipe, "scraperRuns": [], - user_tables.CONNECTOR_HEALTH_FIELD: - user_tables.connector_health_receipt("never_run")}) - current[key] = definition - return current - - try: - rt.update(user_tables.STORE_KEY, _attach, flush="sync") - except Exception: # noqa: BLE001 - user_tables.delete(key, st=rt) - raise err(503, "recipe_not_saved", - "The recipe could not be saved, so the new database was removed.") - return {"key": key, "label": label, "recipe": recipe, "preview": preview, - "createdWithoutProviderCall": True, "incurredUsd": 0.0} - - -def _scraper_database_run(rt, uname, is_admin, database_key): - """The one explicit paid-work boundary for any saved scraper recipe.""" - if not rt.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - import core.user_tables as user_tables - key = str(database_key or "").strip() - definitions = user_tables.lend_defs(rt) - definition = user_tables.get(key, st=definitions) or {} - if not definition: - raise err(404, "unknown_scraper_database", "This saved scraper database was not found.") - raw_recipe = definition.get("scraperRecipe") - recipe, errors = _clean_scraper_recipe(raw_recipe) - if errors: - raw_kind = str((raw_recipe or {}).get("kind") or "") if isinstance(raw_recipe, dict) else "" - raw_limits = ((raw_recipe or {}).get("limits") or {}) if isinstance(raw_recipe, dict) else {} - raw_cap = raw_limits.get("records") if isinstance(raw_limits, dict) else None - try: - bounded = 1 <= int(raw_cap) <= 500 - except (TypeError, ValueError): - bounded = False - if raw_kind == "google_maps" and not bounded: - raise err(400, "maps_unbounded_request", - "Google Maps discovery requires an explicit result cap between 1 and 500.") - raise err(409, "invalid_saved_scraper_recipe", - "This saved scraper recipe is no longer valid. Review its settings before running it.") - # Run now both spends against a provider account and mutates this database. A viewer grant - # may expose the rows without granting either power, so the ordinary open wall is too broad. - # Reuse the database editor authority that every other mutation surface already honours. - if not user_tables.may_edit_database(key, uname, bool(is_admin), st=definitions): - raise err(403, "database_forbidden", "You cannot run this database recipe.") - plan = _scraper_plan(recipe) - previous_health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD) - previous_health = previous_health if isinstance(previous_health, dict) else {} - - def record_attempt(state, error=None): - try: - return user_tables.record_connector_health( - key, state, st=rt, - last_success=previous_health.get("lastSuccess"), - source_timestamp=previous_health.get("sourceTimestamp"), - materialized_timestamp=previous_health.get("materializedTimestamp"), - row_count=previous_health.get("rowCount"), error=error) - except Exception: # noqa: BLE001 - raise err(503, "sync_status_not_saved", - "The run did not start because its durable status could not be saved.") from None - - if not plan["credentialReady"]: - code = "maps_credential_missing" if recipe["kind"] == "google_maps" else \ - "scraper_credential_missing" - message = "Add the scraper credential in Keychain before choosing Run now." - record_attempt("failed", {"code": code, "message": message}) - raise err(409, code, "Add the scraper credential in Keychain before choosing Run now.") - - # Imports and provider calls occur only below the explicit Run-now endpoint. Preview, create, - # detail, and page load never reach either runner. - started_at = datetime.now(timezone.utc).isoformat() - record_attempt("running") - key_field = "shortcode" - if recipe["kind"] == "google_maps": - import providers - try: - result = providers.discover_google_maps_stores( - recipe["seed"]["search"], recipe["seed"]["geography"], - recipe["limits"]["records"], captured_at=started_at) - except providers.MapsProviderRefusal as exc: - record_attempt("failed", {"code": exc.code, "message": exc.public_message}) - status = 409 if exc.code == "maps_credential_missing" else \ - (400 if exc.code in ("maps_unbounded_request", "maps_invalid_search") else 502) - raise err(status, exc.code, exc.public_message) - incoming_key = "rows" - key_field = "place_id" - else: - limit = recipe["limits"]["posts"] - post_type = recipe["limits"]["postType"] - try: - if recipe["kind"] == "instagram": - import connectors_ig as runner - run_recipe = runner.pull_ig_recipe - else: - import connectors_tt as runner - run_recipe = runner.pull_tt_recipe - result = run_recipe(recipe["seed"], limit, post_type=post_type, - log=lambda *_args: None) - except Exception: # noqa: BLE001 - record_attempt("failed", { - "code": "scraper_run_failed", - "message": "The provider could not complete this run. No result was written.", - }) - raise err(502, "scraper_run_failed", - "The provider could not complete this run. No result was written.") - incoming_key = "posts" - result = result if isinstance(result, dict) else {} - incoming = [dict(row) for row in (result.get(incoming_key) or ()) if isinstance(row, dict)] - state = str(result.get("state") or "error") - if state not in ("ok", "partial"): - incoming = [] - finished_at = datetime.now(timezone.utc).isoformat() - source_stamps = [_sync_stamp(row.get("source_timestamp")) for row in incoming] - source_timestamp = max((stamp for stamp in source_stamps if stamp), default=None) - provider_counts = dict(result.get("counts") or {}) - provider_counts.setdefault("returned", len(incoming)) - provider_cost = dict(result.get("cost") or {}) - provider_cost.setdefault("currency", "USD") - provider_cost.setdefault("estimatedUsd", plan["estimatedUsd"]) - provider_cost.setdefault("costCapUsd", plan["costCapUsd"]) - receipt = {"id": finished_at, "state": state, - "provider": str(result.get("via") or plan["provider"]), - "rows": len(incoming), "startedAt": started_at, "finishedAt": finished_at, - "estimatedUsd": plan["estimatedUsd"], "costCapUsd": plan["costCapUsd"], - "cost": provider_cost, "providerCounts": provider_counts, - "attempts": [dict(item) for item in (result.get("attempts") or ()) - if isinstance(item, dict)], - # Provider notes can echo a submitted handle. Run history stores operational - # metadata only; the saved recipe already owns the input and no customer value is - # duplicated into an audit-like receipt. - "note": f"Provider returned {state}."} - counts = {"inserted": 0, "updated": 0, "unchanged": 0, "skipped": 0, - "duplicates": 0, "orphans": 0, "capped": 0} - - def _write(cur): - nonlocal counts - current = dict(cur or {}) - stored = dict(current.get(key) or {}) - if not stored: - raise RuntimeError("the database disappeared before the run could be written") - if incoming: - import automation_engine as engine - rows, counts = engine.upsert_rows(stored.get("rows") or {}, incoming, key_field) - stored["rows"] = rows - processed = sum(int(counts.get(name) or 0) - for name in ("inserted", "updated", "unchanged", "skipped", "capped")) - receipt["writeCounts"] = dict(counts) - receipt["countReconciled"] = ( - int(provider_counts.get("returned") or 0) == len(incoming) == processed) - history = [dict(item) for item in (stored.get("scraperRuns") or ()) - if isinstance(item, dict)] - stored["scraperRuns"] = [receipt, *history][:SCRAPER_RUN_HISTORY_LIMIT] - final_state = "succeeded" if state == "ok" else "partial" if state == "partial" else "failed" - final_error = None if final_state != "failed" else { - "code": "provider_result_failed", - "message": "The provider returned no usable result for this database.", - } - stored[user_tables.CONNECTOR_HEALTH_FIELD] = user_tables.connector_health_receipt( - final_state, - last_success=finished_at if final_state in ("succeeded", "partial") - else previous_health.get("lastSuccess"), - source_timestamp=source_timestamp, - materialized_timestamp=finished_at if final_state in ("succeeded", "partial") else None, - row_count=len(stored.get("rows") or {}), error=final_error) - current[key] = stored - return current - - try: - rt.update(user_tables.STORE_KEY, _write, flush="sync") - except Exception: # noqa: BLE001 - try: - record_attempt("failed", { - "code": "scraper_result_not_saved", - "message": "The provider answered, but the result could not be saved.", - }) - except Exception: # noqa: BLE001 - pass - raise err(503, "scraper_result_not_saved", - "The provider answered, but the result could not be saved. Reload before retrying.") - return {"key": key, "state": state, "counts": counts, "receipt": receipt} - - -def connector_detail(rt, user, uname, is_admin, connector_key): - """The whole C4 detail DTO for one connector, or `None` when this reader has no such card. - - Pure over the session for the same reason `directory` is: a gate can drive it without a - request, so the modal's payload is provable without a browser. - """ - budget = _ConnectorDetailBudget() - rows = directory(rt, uname, bool(is_admin)) - budget.check() - key = str(connector_key or "") - row = next((candidate for candidate in rows - if str(candidate.get("key") or "") == key), None) - if row is None: - return None - graph = relationship_graphs(rows, user, rt, uname, bool(is_admin)).get(key) or _graph_refusal( - key, "No actual relationship graph was returned for this connection. A provider template " - "is not shown.") - budget.check() - # ⚠ `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a - # question about a PERSON and a table, not about an administrator flag alone. - databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or "")) - budget.check() - overview = _detail_overview(row, databases, graph) - budget.check() - keychain = _detail_keychain(rt, uname, bool(is_admin), row) - budget.check() - enablement = _detail_enablement(rt, bool(is_admin), row) - budget.check() - introspection = _detail_introspection(rt, bool(is_admin), row) - budget.check() - return _safe_connector_status(key, {"overview": overview, - "keychain": keychain, - "databases": databases, - "syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"} - if is_admin and key in connector_sync.providers() else None), - "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs", - "retryPath": f"/connectors/{key}/sync/retry"} - if is_admin and key == "odoo" else None), - # W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it - # from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints. - "enablement": enablement, - # ⭐ W49-T14 / R24 — WHETHER THE DATABASES TAB MAY DISCLOSE ANYTHING AT ALL, and what - # THIS reader may do there. An addition to C4, declared on `_detail_introspection`: - # without it an empty `databases` list is "this connector has no schema" and "this - # connector's schema could not be read" at the same time. - "introspection": introspection, - # ⛔ THE `relationship_graphs` SHAPE, UNCHANGED. C4 says so in as many words, and the - # client already has a reader for it (`parseRelationshipGraph`) that refuses a graph - # bound to another connection. A second shape here would need a second reader. - "relationships": graph}) - - -def _set_amazon_database_enablement(database_key, body, session): - """The testable core of Amazon's tenant-scoped administrator enablement door.""" - if not session.admin: - raise err(403, "not_admin", - "only an administrator can change Amazon database enablement") - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - request = body if isinstance(body, dict) else {} - if set(request) != {"enabled"} or type(request.get("enabled")) is not bool: - raise err(400, "bad_enabled", "enabled must be true or false") - - import connectors_amazon as cx # noqa: PLC0415 - provider owns state - key = str(database_key or "").strip() - try: - state = cx.set_database_enabled(session.runtime, key, request["enabled"]) - except KeyError: - # One refusal for blank, malformed and cross-provider keys. The response does not expose - # the rest of the tenant's Amazon roster. - raise err(404, "unknown_amazon_database", - "Amazon does not declare a database with that key") - except cx.EnablementConfigError: - raise err(409, "amazon_enablement_unavailable", - "the stored Amazon enablement settings are invalid. Nothing was changed.") - except ValueError: - raise err(400, "bad_enabled", "enabled must be true or false") - except Exception: # noqa: BLE001 - raise err(503, "enablement_unconfirmed", - "the server could not confirm the Amazon database setting. Reload before " - "trying again.") - return {"provider": cx.ENTRY_TYPE, - "key": key, - "enabled": state["databases"].get(key) is True, - # This route changes only the provider config document. It never deletes a mirror or - # a materialised table, and the wire states that invariant explicitly. - "preservesData": True} - - -# Literal and provider-owned. It appears before the generic connector detail route so future -# dynamic route additions cannot turn Amazon enablement into an Odoo-model dispatch by accident. -@router.patch("/connectors/amazon_sp/databases/{database_key}/enablement") -def amazon_database_enablement(database_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Enable or disable one declared Amazon database for this tenant, without deleting data.""" - return _set_amazon_database_enablement(database_key, body, session) - - -@router.post("/connectors/scraper/databases/preview") -def scraper_database_preview(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Validate and price a recipe locally. This endpoint makes zero provider calls.""" - return _scraper_preview(body) - - -@router.post("/connectors/scraper/databases") -def scraper_database_create(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Create a saved recipe database for $0; paid work begins only at its Run-now door.""" - return _create_scraper_database(session.runtime, session.uname, body) - - -@router.post("/connectors/scraper/databases/{database_key}/run") -def scraper_database_run(database_key: str, session: Session = Depends(require_session)): - """Explicitly execute one bounded saved scraper recipe through its provider seam.""" - return _scraper_database_run(session.runtime, session.uname, bool(session.admin), database_key) - - -# ⚠ DECLARED AFTER THE LITERAL `/connectors/odoo/*` ROUTES. FastAPI matches in declaration order, -# and while none of those ends in `/detail` — so no shadowing is possible today — the ordering is -# the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot -# collide with this three-segment path at all. -@router.get("/connectors/{connector_key}/detail") -def connectors_detail(connector_key: str, session: Session = Depends(require_session)): - """C4 / R26: everything the connector modal shows, for THIS reader, in one read. - - Session-gated exactly like the directory it is opened from: a member may look at what the - workspace is connected to. What a member does NOT get is the tenant-wide credential door - (`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue. - """ - detail = connector_detail( - session.runtime, session.user, session.uname, - bool(session.admin), connector_key) - if detail is None: - raise err(404, "unknown_connector", - "this workspace has no connector with that key") - return detail - - -class _ReplacementConflict(Exception): - """The addressed keychain row changed between replacement preparation and its store update.""" - - -def _encrypted_replacement_payload(kc, label, entry_type, fields, username): - """Validate and encrypt through ``core.keychain.add_entry`` without writing a tenant row. - - ``add_entry`` is the keychain's one validation, preview and Fernet implementation. A tiny - memory-only runtime lets this route reuse it while discarding the temporary identity it mints; - the tenant store sees only the later, single-row payload swap. - """ - class _Stage: - def __init__(self): - self.bucket = {} - - def get(self, name): - return self.bucket if name == kc.KEY else {} - - def update(self, name, fn, flush="sync"): - if name != kc.KEY: - raise RuntimeError("the keychain encryptor wrote an unexpected bucket") - updated = fn(self.bucket) - self.bucket = updated if updated is not None else self.bucket - return self.bucket - - stage = _Stage() - temporary = kc.add_entry(stage, label, entry_type, fields, username) - row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {}) - if not row.get("enc"): - raise RuntimeError("the keychain encryptor returned no encrypted payload") - payload = {"enc": str(row["enc"]), "preview": str(row.get("preview") or ""), - "envelope_version": row["envelope_version"]} - if entry_type == "amazon_sp": - payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD] - return payload - - -def _replace_connector_key(connector_key, entry_id, body, session): - """Replace only one connector entry's declared secrets while preserving its identity. - - The existing row remains untouched through validation, decryption, merge and encryption. The - final store update compares the complete old row before swapping the prepared envelope; - a concurrent rename or credential change therefore conflicts instead of being overwritten. - Scope and owner live in a separate bucket and are never written by this operation. - """ - if not session.admin: - raise err(403, "not_admin", - "only an administrator can replace a workspace connection key") - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - - key = str(connector_key or "").strip() - connector = next((row for row in directory(session.runtime, session.uname, True) - if str(row.get("key") or "") == key), None) - entry_type = _entry_type_for(key) - if connector is None or not entry_type: - raise err(404, "unknown_connector", "this workspace has no connector with that key") - if str(connector.get("manage") or "") not in ("keychain", "keychain_personal"): - raise err(403, "replacement_unavailable", - "this connector has no replaceable keychain credential") - - import routes_keychain as rk # noqa: PLC0415 - circular at import - from core import keychain as kc # noqa: PLC0415 - keep boot lean - visible = rk.visible_entries(session.runtime, session.uname, True) - entry = next((row for row in visible - if str(row.get("id") or "") == str(entry_id) - and str(row.get("type") or "") == entry_type), None) - if entry is None: - # One 404 covers missing, cross-tenant, another user's personal entry and a real id bound - # to another connector. The mismatch itself is information this route does not disclose. - raise err(404, "no_entry", "no such key for this connector") - - secret_specs = _replacement_field_specs(entry_type) - secret_keys = [str(field["key"]) for field in secret_specs if field.get("secret") is True] - optional_keys = {AMAZON_ROTATION_DEADLINE_FIELD} if entry_type == "amazon_sp" else set() - request = body if isinstance(body, dict) else {} - supplied = request.get("fields") - if set(request) != {"fields"} or not isinstance(supplied, dict): - raise err(400, "bad_replacement", - "send only the connector's new secret fields") - if not set(secret_keys) <= set(supplied) <= set(secret_keys) | optional_keys: - raise err(400, "bad_replacement", - "send every new secret field, optional rotation metadata, and no account or server settings") - replacements = {} - for field in secret_keys: - value = supplied.get(field) - if not isinstance(value, str) or not value.strip(): - raise err(400, "bad_replacement", "a replacement secret cannot be blank") - replacements[field] = value.strip() - - # Snapshot before decryption so a concurrent replacement cannot become our baseline later. - from copy import deepcopy - raw_bucket = session.runtime.get(kc.KEY) or {} - original = deepcopy((raw_bucket.get("entries") or {}).get(str(entry_id))) - if not isinstance(original, dict): - raise err(409, "credential_changed", - "the stored credential changed while replacement was prepared. Reload it.") - try: - current_fields = kc.read_fields(session.runtime, entry_id) - except kc.KeychainEnvelopeVersionError: - raise err(409, "credential_envelope_unsupported", - "This credential uses an unsupported encryption envelope version. Nothing was changed.") - except kc.KeychainLocked as exc: - raise err(503, "keychain_locked", - f"the keychain is locked: {exc}. Nothing was changed.") - except Exception: # noqa: BLE001 - raise err(503, "store_unavailable", - "the current key could not be read. Nothing was changed.") - if not isinstance(current_fields, dict) or not current_fields: - raise err(409, "credential_unreadable", - "the stored credential is incomplete. Nothing was changed.") - - merged_fields = {**current_fields, **replacements} - if entry_type == "amazon_sp": - # The previous secret's deadline does not describe its replacement. Absence is unknown. - merged_fields[AMAZON_ROTATION_DEADLINE_FIELD] = supplied.get(AMAZON_ROTATION_DEADLINE_FIELD, "") - try: - encrypted = _encrypted_replacement_payload( - kc, entry.get("label"), entry_type, merged_fields, session.uname) - except kc.KeychainLocked as exc: - raise err(503, "keychain_locked", - f"the keychain is locked: {exc}. Nothing was changed.") - except ValueError as exc: - raise err(400, "bad_replacement", str(exc)) - except Exception: # noqa: BLE001 - raise err(503, "replacement_not_stored", - "the replacement could not be encrypted. The current key is unchanged.") - - def _swap(cur): - entries = cur.get("entries") if isinstance(cur, dict) else None - stored = entries.get(str(entry_id)) if isinstance(entries, dict) else None - if not isinstance(stored, dict) or stored != original: - raise _ReplacementConflict() - replacement = dict(stored) - replacement.update(encrypted) - replacement.pop("probe", None) - # Copy on write: a store failure after this callback must not mutate its cached old row. - return {**cur, "entries": {**entries, str(entry_id): replacement}} - - try: - session.runtime.update(kc.KEY, _swap, flush="sync") - except _ReplacementConflict: - raise err(409, "credential_changed", - "the stored credential changed while replacement was prepared. Reload it.") - except Exception: # noqa: BLE001 - # A remote store can fail after accepting a write. Claiming either outcome would be a lie; - # the safe next act is a fresh detail read, never an automatic retry or delete. - raise err(503, "replacement_unconfirmed", - "the server could not confirm whether the replacement landed. Reload this " - "connector before trying again.") - return {"ok": True} - - -@router.post("/connectors/{connector_key}/keychain/{entry_id}/replace") -def replace_connector_key(connector_key: str, entry_id: str, - body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Admin-only, write-only replacement for one visible connector credential.""" - return _replace_connector_key(connector_key, entry_id, body, session) - - -# ⚠ FIVE SEGMENTS, so it can collide with nothing already declared here: the literal Odoo routes -# are four (`/connectors/odoo/schema/{field_key}`) and `/detail` is three. -@router.patch("/connectors/{connector_key}/databases/{database_key}/name") -def connector_database_name(connector_key: str, database_key: str, - body: dict = Body(default=None), - session: Session = Depends(require_session), - connection_id: str = None): - """W53-T27 / R9 — set or CLEAR this tenant's own name for one of a connector's databases. - - ⛔⛔ IT WRITES THE BUCKET THE NAVIGATION RAIL WRITES, `nav_meta`, IN THE SAME SHAPE AND WITH - THE SAME PER-KEY MERGE. That is the whole ruling: *"the rename database under the Navigation - rail also works, it should then sync to the navigation in the connector too."* A rename made - here is the one the rail shows, and a rename made on the rail is the one this connector's - Databases tab shows, because there is one document and one reader of it - ([[one-question-two-normalizers]]). The key names, the length cap and the merge semantics are - IMPORTED from `routes_nav` rather than copied, so the two doors cannot drift apart. - - ⛔ SO WHY A SECOND DOOR AT ALL? `POST /nav/meta` cannot CLEAR a name: an empty or null `name` - 400s there ("a database needs a name"), so "clearing the override falls back to our shipped - default" has nowhere to happen today. This route is that door. `name: null` and `name: ""` - both clear; any other string sets. The day `/nav/meta` grows the same clearing capability this - becomes a thin alias of it and should be folded in. - - ⛔ AND IT NEVER TOUCHES A TABLE DEFINITION. `user_tables.set_label` would be rewritten by the - next connector re-sync (`odoo_relational.apply_plan` rebuilds definitions), so an override - stored there would vanish on a sync nobody connected to a rename. R9 forbids exactly that, and - the store choice is what makes it impossible rather than merely unlikely. - - THE WALLS, in order, and each one names what it refused: - · the database must be one THIS connector declares (404) - the door is scoped to the card - the reader opened it from, so it cannot be used to rename an unrelated table; - · the key must be a `ut_` key (400 `name_not_allowed`, the same refusal `/nav/meta` gives) - - a built-in module's label is a compiled registry literal; - · the table must EXIST in this workspace (409) - a name for a table nobody can open would - sit in the document naming nothing; - · `may_open` - the creator, an admin, or a share grant, the same predicate `/nav/meta` uses - and the same one `canRename` reports, so the control and the door always agree. - """ - import routes_nav # noqa: PLC0415 - the ONE nav owner - rt = session.runtime - key = str(database_key or "").strip() - if connector_key == "odoo": - key, connection_id, _choice = _odoo_database_target(rt, key, connection_id) - rows = _detail_databases(rt, bool(session.admin), str(connector_key or ""), - uname=session.uname, connection_id=connection_id) - row = next((candidate for candidate in rows - if str(candidate.get("key") or "") == key), None) - if row is None: - raise err(404, "unknown_database", - "this connection does not create a database with that name") - if not key.startswith("ut_"): - raise err(400, "name_not_allowed", - "only a database this connection created can be renamed. This one's name comes " - "from the module registry") - import core.user_tables as user_tables # noqa: PLC0415 - lazy, as elsewhere - definitions = user_tables.lend_defs(rt) - if not user_tables.get(key, st=definitions): - raise err(409, "database_not_created", - "this connection has not created that database in this workspace yet, so there " - "is nothing to rename") - if not user_tables.may_open(key, session.uname, session.admin, st=definitions): - raise err(403, "forbidden", "that database belongs to another user") - - body = body if isinstance(body, dict) else {} - if "name" not in body: - raise err(400, "bad_request", - "no name was sent. Send a name to set one, or null to go back to the default") - # The same normalisation `/nav/meta` applies: whitespace collapsed, capped at its own limit. - # A blank result CLEARS, which is the half that route refuses and this one exists for. - name = " ".join(str(body.get("name") or "").split())[:routes_nav._MAX_NAV_NAME] - - if not rt.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - # `routes_nav.save_nav_meta._up`, clause for clause: an entry left empty is REMOVED rather - # than stored as `{}`, and an icon set from the rail survives a rename made here. - data = data if isinstance(data, dict) else {} - entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} - if name: - entry["name"] = name - else: - entry.pop("name", None) - if entry: - data[key] = entry - else: - data.pop(key, None) - return data - - try: - rt.update(routes_nav._NAV_META_KEY, _up) # noqa: SLF001 - the nav's own key - except Exception: # noqa: BLE001 - raise err(503, "store_unavailable", - "the change was not saved: the store refused the write.") - - # Read BACK through the same resolver the tab reads, so the answer cannot be the echo of a - # write that landed somewhere else. - default_label = str(row.get("defaultLabel") or "") - override = schema_overlay.override_name(_database_name_overrides(rt).get(key)) - return {"key": key, - "identity": row.get("identity"), - # ⛔ SERVED, NEVER ACCEPTED. `native` is the vendor's identifier; this door reads - # `name` and nothing else off the body (C2: `native` is never editable). - "native": str(row.get("native") or ""), - "defaultLabel": default_label, - "override": override, - "label": schema_overlay.database_label(default_label, override)} - - -# ⛔⛔ W49-T16 — THE MOUNT. `dependents_router` DECLARES NO PREFIX OF ITS OWN, so including it in -# this router (which carries `prefix="/api/v1"`) yields exactly `/api/v1/dependents`. Declaring -# the prefix on both would serve `/api/v1/api/v1/dependents` — the mirror image of the wave-23 -# defect this file's own header records, where the MISSING prefix 404'd a shipped route with -# every gate green. `verify_shared_views.py` asserts the assembled path off this very object -# rather than trusting the arithmetic. -router.include_router(dependents_router) - -# ⛔⛔ W49-T19 — THE SECOND MOUNT, IDENTICAL IN SHAPE. `monid_router` is a bare `APIRouter()`, -# so this yields `/api/v1/monid/amazon-reviews/meta`, `/api/v1/monid/amazon-reviews/run` and -# `/api/v1/monid/spend`. -# ⚠ THE ASSEMBLED PATH IS NOT VISIBLE IN `router.routes`. FastAPI 0.139.2 leaves an -# `_IncludedRouter` placeholder there, so a walk of that list finds nothing and a gate written -# against it would pass on a router that serves the paths twice-prefixed. `app.openapi()["paths"]` -# is the reliable read, and it is what `verify_meta.py::section_monid_mount` uses. -router.include_router(monid_router) +"""THE CONNECTORS DIRECTORY (wave 23, contract C11 / owner ruling R8). + +⭐ WHAT THE OWNER ASKED FOR, verbatim: *"Build a connector module where it stores everything… BUT +if its not yet integrated make it faded grey or something so i can use it as a to-do list for +integrations."* So this endpoint deliberately answers with things we have NOT built. A directory +that listed only working connectors would be honest about each row and dishonest about the shape +of the product — the owner wants to see the map, including the empty parts. + +⛔ THE ROW SET IS COMPOSED HERE, ONCE, FROM THE REGISTRIES THAT ALREADY EXIST — the OAuth +provider registry (`oauth_connect.PROVIDERS`), the keychain's entry types, the automation source +registry, and the env-configured built-ins. It is NOT a hand-written list of cards. That matters +because the alternative rots in a specific way: a provider gets wired up somewhere and the +directory keeps calling it "planned" for a wave or two, which is exactly the misinformation this +surface exists to prevent. The only hand-written part is `PLANNED` — the things that have no +registry entry anywhere yet, which is precisely what "not yet integrated" means. + +⚠ STATE IS PER-READER. `connected` is a fact about THIS user for a per-user credential (OAuth +slots are per-user by R7) and about the tenant for a shared one (Odoo, the Scraper). A directory +that reported another user's Gmail as "connected" would be both wrong and a small disclosure. +""" +from __future__ import annotations + +import os +import ssl +import time +import threading +import xmlrpc.client +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +import oauth_connect +from deps import Session, err, require_session, running_main +#: ⭐⭐ W49-T18 / R27 / C4 — THE TAXONOMY LIVES IN THE DATA LAYER, NOT HERE. `core` is already on +#: the path by the time this module loads (`deps` imports three of its modules), so this is the +#: same import shape the rest of the API uses. It is in `platform/core/` rather than beside this +#: file because the crosswalk is authored CONTENT about connectors, not routing: a second reader +#: of it (a gate, a future module page) must not have to import FastAPI to ask what category a +#: connector is in. +from core import connector_directory +from core import connector_sync +from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD +from harness import amazon_client +#: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default, +#: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already +#: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module +#: holds the tenant's store handle and does the reading; nothing about what a blank override MEANS +#: is decided here. +from core import schema_overlay +#: ⛔ W32-T10 — IMPORTED, NEVER RE-DERIVED. "Does the environment serve THIS tenant's Odoo?" is +#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")` +#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a +#: second correct one here would leave the same two-copies shape that produced the defect. +from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available +#: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a +#: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any +#: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router` +#: line for exactly this reason — see `routes_charts.py`'s header). This file is already +#: included by `main.py`, so mounting the child here puts `/api/v1/dependents` on the app with +#: no cross-fence edit. The `include_router` call is at the BOTTOM of this file; the import is +#: here so the module's dependencies read in one place. +from routes_dependents import router as dependents_router + +#: ⭐⭐ W49-T19 · R31 / C5 — THE MONID SLICE, MOUNTED THE SAME WAY AND FOR THE SAME REASON. +#: `routes_monid.py` carries the Amazon reviews pull. It declares NO prefix of its own; the +#: `include_router` call is at the BOTTOM of this file beside the dependents one, so there is a +#: single place to read what this router actually serves. +#: ⛔ NEVER FROM `main.py` (C5). The connectors directory owns its children. +from routes_monid import router as monid_router + +# ⛔ THE PREFIX IS LOAD-BEARING AND IT WAS MISSING (found at the wave-23 close-out audit). +# This router declared a bare `APIRouter()`, so its one path was served at `/connectors/directory` +# while `ConnectorsPage.tsx` fetches `${API_V1}/connectors/directory` — the directory would have +# 404'd in production with every gate green. Every other router in this package self-declares +# `prefix="/api/v1"`; this was the only one that did not. +# +# ⚠ AND THE WIRING GATE COULD NOT SEE IT. `verify_wiring`'s W23-W4 row matches the SERVER side on +# the substring `connectors/directory` and the CLIENT side on `${API_V1}/connectors/directory` — +# both matched, both sides "wired", prefixes disagreeing. A cross-fence row that greps each side +# independently proves the two files mention the same path SEGMENT, never that they agree on the +# same URL. `verify_api` now asserts the prefix on every mounted router instead. +router = APIRouter(prefix="/api/v1") + +CONNECTOR_DETAIL_DEADLINE_SECONDS = 30 +SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps")) +SCRAPER_RECIPE_MAX_ROWS = 30 +SCRAPER_RUN_HISTORY_LIMIT = 20 +DATABASE_SYNC_STATES = frozenset( + ("never_run", "queued", "running", "succeeded", "partial", "failed", "disabled") +) +_DATABASE_SAFE_ERRORS = { + "tls_certificate_invalid": ("Odoo TLS certificate verification failed because the certificate " + "is expired or untrusted. No source data was accepted."), + "source_read_refused": "Odoo refused the configured account's read access.", + "source_auth_failed": "Odoo rejected the configured read-only credential.", + "source_unavailable": "Odoo could not be reached for this read-only sync.", + "source_read_failed": "Odoo source validation failed.", + "mirror_pull_failed": "The Odoo mirror pull failed.", + "materialization_failed": "The database could not be materialized.", + "reconciliation_failed": "The database reconciliation failed.", + "freshness_unconfirmed": "Freshness could not be confirmed for this database.", + "sync_status_unavailable": "Sync status is unavailable for this database.", + "sync_stalled": "This database sync is stalled.", + "sync_state_unreadable": "This database's saved sync state could not be read.", + "maps_credential_missing": "Add the scraper credential in Keychain before choosing Run now.", + "scraper_credential_missing": "Add the scraper credential in Keychain before choosing Run now.", + "maps_unbounded_request": "Google Maps discovery requires an explicit bounded result cap.", + "maps_invalid_search": "Google Maps discovery needs a valid category or search.", + "maps_vendor_error": "The Maps provider could not complete this run.", + "maps_schema_drift": "The Maps provider returned an unreadable result shape.", + "scraper_run_failed": "The provider could not complete this run. No result was written.", + "provider_result_failed": "The provider returned no usable result for this database.", + "scraper_result_not_saved": "The provider answered, but the result could not be saved.", +} + +# T11 established the saved database contract; T12's provider normalises every Apify answer into +# exactly this schema. Vendor field names never reach a saved View or downstream automation. +_MAPS_RECIPE_FIELDS = [ + {"key": "place_id", "label": "Place id", "type": "text", "source": "automation"}, + {"key": "name", "label": "Name", "type": "text", "source": "automation"}, + {"key": "category", "label": "Category", "type": "text", "source": "automation"}, + {"key": "address", "label": "Address", "type": "text", "source": "automation"}, + {"key": "city", "label": "City", "type": "text", "source": "automation"}, + {"key": "region", "label": "Region", "type": "text", "source": "automation"}, + {"key": "postal_code", "label": "Postal code", "type": "text", "source": "automation"}, + {"key": "country", "label": "Country", "type": "text", "source": "automation"}, + {"key": "phone", "label": "Phone", "type": "phone", "source": "automation"}, + {"key": "website", "label": "Website", "type": "url", "source": "automation"}, + # Coordinates remain text because `number` is not a user-table field kind; the Map binding + # parses the canonical scalar value. Declaring an unsupported kind would silently drop both + # columns at database creation. + {"key": "latitude", "label": "Latitude", "type": "text", "source": "automation"}, + {"key": "longitude", "label": "Longitude", "type": "text", "source": "automation"}, + {"key": "rating", "label": "Rating", "type": "rating", "source": "automation"}, + {"key": "reviews", "label": "Reviews", "type": "int", "source": "automation"}, + {"key": "source_timestamp", "label": "Source timestamp", "type": "date", + "source": "automation"}, + {"key": "source_payload", "label": "Source data", "type": "json", "source": "automation"}, +] + + +# Measured reads take 1.34 s cold, 0.01 s warm, and above 8 s under boot contention. +# A thread pool deadline was tried and produced three consecutive 30 s 504s because +# core/store.py uses reentrant locks whose ownership does not survive a thread +# boundary while the request thread waits on the result. Keep the work on the +# request thread. This budget is therefore cooperative: it bounds a read whose +# phases are each finite and cannot interrupt a single call that is already wedged. +class _ConnectorDetailBudget: + def __init__(self): + self._started = time.monotonic() + + def check(self): + if time.monotonic() - self._started >= CONNECTOR_DETAIL_DEADLINE_SECONDS: + raise err(504, "connector_detail_timeout", + f"The connector store did not answer within {CONNECTOR_DETAIL_DEADLINE_SECONDS} " + "seconds, so the read was stopped.") + + +#: The to-do list (R8). Order is the owner's reading order, not alphabetical: the things most +#: likely to come next first. `queue` mirrors the standing provider queue in wave22-split C5-A2 +#: so the two cannot drift into disagreeing about what is coming. +PLANNED = [ + # ⭐⭐ WAVE 31 · R7 — SIX ROWS LEFT THIS LIST FOR `NANGO_PROVIDERS` BELOW, and that move is the + # ticket. `slack`, `airtable`, `outlook`, `quickbooks`, `xero` and `hubspot` are providers + # Nango can AUTHENTICATE today, so leaving them here as "not built yet" is the same + # misinformation this module's header warns about — one wave later and about a different + # mechanism. What they still cannot do is INGEST, and that is what their own cards now say. + # ⚠ WAVE 32 · ITEM 4 — FIVE, NOW. The Airtable row is deleted from the directory entirely + # (the reason is in the provider registry below). It is still NAMED in the sentence above + # because that sentence is the record of where those six rows WENT, and a record that quietly + # drops one of its own entries is worse than no record. + # ⚠ W32-T14 — THIS SENTENCE STOPPED BEING TRUE IN THE SAME CHANGE THAT HID THE GOOGLE CARD. + # It read *"the Google connection already exists"*, which was the one row on this page that + # made a claim about ANOTHER row — so R6 hiding that other row turned a to-do entry into a + # false statement. Exactly the rot this module's header names, arriving sideways. + # ⛔ THE THREE `google_*` ROWS THEMSELVES STAY. Item 12 is about the connectors that offered a + # button and answered `503 oauth_unavailable`; these are `planned` to-do entries with no door, + # no state and no promise — which is precisely the map R8 asked this page to be. + {"key": "google_sheets", "label": "Google Sheets", "kind": "oauth", + "desc": "Read a sheet as a database."}, + {"key": "google_drive", "label": "Google Drive", "kind": "oauth", + "desc": "Attach files from Drive to records."}, + {"key": "google_calendar", "label": "Google Calendar", "kind": "oauth", + "desc": "Turn dated records into calendar entries."}, + # ⭐⭐ WAVE 30 · R4 — TWO ROWS WERE DELETED HERE, AND THEY WERE MISINFORMATION, not clutter. + # + # {"key": "tiktok", "label": "TikTok", ... "The second social source beside Instagram."} + # {"key": "meta", "label": "WhatsApp and Instagram (Meta)", ... "Blocked on Meta business + # verification; Instagram data arrives through the Scraper connector"} + # + # ⛔ THE TIKTOK ROW SAID "not built yet" ABOUT SOMETHING THAT SHIPPED IN WAVE 29 — which is + # precisely the rot this module's header warns about: *"a provider gets wired up somewhere and + # the directory keeps calling it planned for a wave or two, which is exactly the + # misinformation this surface exists to prevent."* TikTok is a PLATFORM of the Scraper + # connector now (R3/R4), and it appears in the Scraper card's own drill below. + # ⚠ THE META ROW IS A DIFFERENT CASE and is deleted for a different reason: what it described + # (Instagram arriving another way) is now visible ON the Scraper card, so keeping it would put + # Instagram on this page twice, once as working and once as blocked. The Meta business + # verification blocker itself is not lost — it is D-86's subject and lives in + # `AutoWiki/research/ads-connectors-owner-actions.md`, where an owner action belongs. +] + +#: ⭐ WAVE 30 · R4 — THE SCRAPER'S PLATFORMS, and the whole shape of the ruling in one list. +#: The owner: *"the damn Tiktok and Instagram belongs in the same bucket … Only when I click +#: 'Scraper' … would I see the option to choose either Instagram OR TikTok."* So the directory +#: mirrors the automation menus exactly: ONE card for the capability, and the platforms are behind +#: it rather than beside it. +#: ⚠ These are not connectors and must never become rows: they have no state or manage door of +#: their own. Instagram and TikTok share the Scraper credential; Maps uses the Apify credential +#: through the same governed provider registry. +SCRAPER_PLATFORMS = [ + {"key": "instagram", "label": "Instagram", + "desc": "Profiles, posts and comments, by search or by handle."}, + {"key": "tiktok", "label": "TikTok", + "desc": "The same three, on TikTok's own schema."}, + {"key": "google_maps", "label": "Google Maps", + "desc": "Places by category or search and geography, through a bounded Apify run."}, +] + +#: Token connectors an admin can wire up TODAY by pasting a key — no OAuth client, no app review. +TOKEN_CONNECTORS = [ + {"key": "stripe", "label": "Stripe", "entry": "stripe", + "desc": "Payments and payouts against your customer records.", + "hint": "Create a restricted key with read access in the Stripe dashboard.", + "note": "Storing the key connects the account; reading its data lands in a later release."}, + {"key": "shopify", "label": "Shopify", "entry": "shopify", + "desc": "Orders and products from a Shopify storefront.", + "hint": "Create a custom app in your store admin and copy its Admin API token.", + "note": "Storing the key connects the account; reading its data lands in a later release."}, + # ⭐⭐ WAVE 31 · R2/R8 — META ADS IS A TOKEN CONNECTOR, AND IT IS THE ONE NEW CARD THAT IS + # ACTIONABLE TODAY. R8 put the schema work on a pasted token deliberately — *"so that you can + # also explore the Schema, without any user logging into it with Facebook yet"* — and T41 + # MEASURED that the delegated-OAuth door is not available anyway: the Nango account holds one + # integration and it is a GitHub sample, so there is no Facebook integration to request + # `ads_read` from. This row is therefore the real door and the Nango one is honest about not + # being ready. ⚠ Its `entry` must match `keychain.ENTRY_TYPES`, or storing 400s. + {"key": "meta_ads", "label": "Meta Ads", "entry": "meta_ads", + "desc": "Campaigns, ad sets, ads, creatives and daily performance from your Meta ad account.", + "hint": "Paste a Marketing API access token with `ads_read`; the connector verifies it " + "against your ad accounts before storing it.", + "note": "The token connects the account and is verified live. The spawned databases " + "(285 measured columns) land with the Meta grids."}, + # ⭐⭐ AMAZON SP-API. FIVE values rather than one token, because Login With Amazon + # exchanges a refresh token for a short-lived bearer on every call, and the seller and + # marketplace ids scope the reads. ⚠ `entry` must match `keychain.ENTRY_TYPES` or storing + # 400s, and `refresh_token` had to join `_PREVIEW_FIELDS` in the same change or the row + # renders a blank preview with nothing raising (the wave-31 defect, second edition). + # ⛔ Read-only by construction: `connectors_amazon._request` refuses any method but GET + # before it builds a request, so no route, payload or UI can reach a write. + {"key": "amazon_sp", "label": "Amazon", "entry": "amazon_sp", + "desc": "Orders, settlements and fees, FBA and AWD inventory, listings and Buy Box " + "pricing from your Amazon selling account.", + "hint": "Create a Production app client in the Solution Provider Portal, self authorize " + "it, then paste client_id, client_secret, refresh_token, seller_id and " + "marketplace_id.", + "note": "The credential is verified live against your marketplace participations before " + "it is stored. Bulk reads use the Reports API, so a backfill is one document " + "rather than a page loop."}, +] + +#: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many +#: other potential connection as possible, such as slack etc."* +#: +#: ⛔ ADDING A PROVIDER HERE IS THE WHOLE FEATURE — one row, no code — which is the claim T43 has +#: to make demonstrable rather than assert. The row carries only what a CARD needs; the auth flow +#: is Nango's and is identical for every one of them. +#: +#: ⛔⛔ R7's SECOND SENTENCE IS THE HARD PART AND IT IS ENFORCED IN `_nango_rows` BELOW: *"Nango +#: buys CONNECT, not INGEST — every lit provider states on its own card what it can do today, or +#: we ship a directory of buttons that lead nowhere."* MEASURED for T42: the owner's Nango account +#: holds **1 integration (`github-getting-started`) and 0 connections**, so there is no sync on it +#: to inspect and NO provider here has a data layer. Every card says so, in its own words, and +#: none of them offers a button. +NANGO_PROVIDERS = [ + {"key": "slack", "label": "Slack", "provider": "slack", + "desc": "Send messages into a channel when an automation reaches a step."}, + {"key": "hubspot", "label": "HubSpot", "provider": "hubspot", + "desc": "Deals and contacts beside the Odoo customer book."}, + # ⭐⭐ WAVE 32 · OWNER ITEM 4 / R3 — THE AIRTABLE ROW IS DELETED. It read: + # + # {"key": "airtable", "label": "Airtable", "provider": "airtable", + # "desc": "Import an existing base as a database."} + # + # The owner asked for it to go, and R3 says how: **de-brand, don't revoke.** So the ROW goes + # and the transport does not — Nango can still authenticate `airtable` if a future ruling + # wants it back, and nothing in `oauth_connect` changed. + # ⚠ THE REASON IS NOT THAT IT WAS BROKEN. This connector is the one competitor whose model + # this product is built against, and offering to import a base is a promise nobody here is + # ready to keep. Every DESIGN-PARITY comment naming Airtable stays exactly where it is + # (~30 of them across the grid and automation clients): those are how we know what good looks + # like, and deleting them would cost the reasoning while saving nothing a user can see. + {"key": "quickbooks", "label": "QuickBooks", "provider": "quickbooks", + "desc": "Accounting actuals beside the Odoo sales data."}, + {"key": "xero", "label": "Xero", "provider": "xero", + "desc": "Accounting actuals for tenants who do not use QuickBooks."}, + {"key": "outlook", "label": "Microsoft Outlook", "provider": "outlook", + "desc": "The Outlook half of the email trigger."}, +] + + +#: ⭐⭐ W31-T44 / OWNER ITEM 6b — WHICH `ut_*` PREFIXES BELONG TO A CONNECTOR (contract C2). +#: +#: B's `user_tables.register_connected_prefix` (W31-T25) replaced a hardcoded `ut_odoo_` literal +#: with a registered SET, and this is where the connector layer declares its half. It lives beside +#: the other connector registries deliberately: "which databases does this connector own" is the +#: same kind of fact as "what is this connector called", and putting it anywhere else would make +#: the answer to one question live in two files. +#: +#: ⛔ REGISTERING A PREFIX DOES NOT MAKE A TABLE CONNECTED, and B posted this as the trap: the +#: third leg of `is_connected` still demands `source == AUTOMATION_SOURCE` **and** +#: `recordMode == AUTOMATION_RECORD_MODE`. A person who names a database "Meta ads spend" gets a +#: `ut_meta_…` key too, and exempting it from `MAX_ROWS` on the strength of its NAME is the silent +#: hole the stamp closes. The spawn stamps; this only says which family may qualify. +#: ⚠ `ut_odoo_` is SEEDED in `user_tables` rather than registered here, also deliberately — the +#: convention leg has to answer in a worker process where nothing registered anything. +#: ⛔⛔ `ut_amzn_` JOINED IN WAVE 52, AND OMITTING IT WOULD HAVE BROKEN STANDING RULE 1 SILENTLY. +#: The rule is that a connector's records carry NO cap: `records_mutable(key)` is False for them, +#: so `MAX_ROWS` does not apply. But that answer is reached through `is_connected`, whose third leg +#: is this prefix set — so an Amazon table missing from here does not raise, does not warn, and +#: simply falls through to the 60,000-row ceiling. A seller with more than 60,000 order lines would +#: get a truncated grid and no error, which is precisely the silent cap the rule forbids. **A new +#: connector joins this dict in the same change that spawns its tables, or its no-cap guarantee is +#: decorative.** +CONNECTED_PREFIXES = { + "odoo": "ut_odoo_", + "meta_ads": "ut_meta_", + "amazon_sp": "ut_amzn_", +} + + +def _register_prefixes(): + """Declare `CONNECTED_PREFIXES` to the store layer. Called at import — this module is mounted + by `main.py`, so any process that can serve the directory has also made the declaration. + + Never raises: a store layer that cannot take the declaration must not stop the API booting, + and the consequence of a missed registration is a row CAP (R6's subject), not a wrong answer. + """ + try: + import core.user_tables as ut + return ut.register_connected_prefix(*sorted(CONNECTED_PREFIXES.values())) + except Exception: # noqa: BLE001 + return frozenset() + + +_register_prefixes() + + +def _register_probers(): + """Let each connector answer `keychain.test_entry` for its OWN type — the SLOT, filled from + the layer that knows how to ask. + + ⛔ THIS DIRECTION IS THE POINT. `core.keychain` must not import a connector module: `core` + never imports up (ARCHITECTURE.md / CLAUDE.md), and the tempting shortcut — a `sys.path` + insert and an `import connectors_meta` from inside `core` — is an architecture inversion that + no gate was watching for. Same idiom as `datastore.set_paused_probe`, one layer down, for the + same reason. + + Import-time and fail-quiet: an unfillable slot costs a liveness CHECK, never the ability to + store a credential and never the API's ability to boot. + """ + ok = False + for mod in ("connectors_meta", "connectors_amazon"): + try: + ok = __import__(mod).register() or ok + except Exception: # noqa: BLE001 + continue + return ok + + +_register_probers() + + +# W48C-T08 / C6 — the graph is a reader-scoped DTO over tables that exist in THIS workspace, +# never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family; +# `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter. +def _graph_refusal(connection_key, reason, status="refused"): + return {"connectionKey": str(connection_key or ""), "status": status, + "reason": str(reason or ""), "tables": [], "relationships": []} + + +def _table_connection_key(table_key, definition): + """The owning connection declared by this actual table, or ``""`` when it is not one.""" + definition = definition if isinstance(definition, dict) else {} + # Future connector materialisers may stamp their directory key explicitly. It is accepted + # only as a key, never as table/field data, so no provider can manufacture a graph template. + for name in ("connectionKey", "connectorKey", "connector_key", "connector"): + candidate = definition.get(name) + if isinstance(candidate, str) and candidate.strip(): + return candidate.strip() + key = str(table_key or "") + for connection_key, prefix in CONNECTED_PREFIXES.items(): + if key.startswith(prefix): + return connection_key + return "" + + +def _module_connection_key(subject): + """Which connected card a REGISTRY MODULE's `subject` belongs to, or `""`. + + ⭐⭐ W53-T30 — DERIVED FROM `CONNECTED_PREFIXES`, NEVER A SECOND HAND-TYPED PAIR LIST. A + module's subject is namespaced by the system the records come from (`odoo:res.partner`, + `odoo:product.product` in `core/registry.py`), and the connector families this file already + knows carry that same namespace inside their table prefix (`odoo` inside `ut_odoo_`). So the + two are matched rather than mapped: a connector added to `CONNECTED_PREFIXES` in the change + that spawns its tables gets this for free, and a module whose namespace names no connected + family here simply does not appear on anybody's graph. + + ⛔ THE SUBJECT, NOT THE `source` STRING. `source` is display copy ("Odoo") and a rename of it + is a copy edit; `subject` is the identity `core.registry` already refuses to let two rows + claim at once, which is what makes it safe to key a graph off. + """ + namespace = str(subject or "").split(":", 1)[0].strip() + if not namespace: + return "" + for connection_key, prefix in CONNECTED_PREFIXES.items(): + parts = str(prefix or "").strip("_").split("_") + if len(parts) > 1 and namespace == "_".join(parts[1:]): + return connection_key + return "" + + +def _module_graph_entries(rt, user, lent, connected): + """The REGISTRY MODULES this reader may see on a connected card's graph, per connection. + + Returns `{connection key: {module key: {"dto", "visible"}}}` in exactly the shape the `ut_` + loop below builds, so the edge pass treats both kinds of node identically. + + ⭐⭐ W53-T30 — THE WHOLE TICKET. `relationship_graphs` walks `user_tables.all_defs`, and + `customer_data` / `product_data` are COMPILED REGISTRY ROWS rather than stored definitions, so + that loop could never reach them however permissive it became. They were absent from the Odoo + diagram for that reason, not for a permission one. ⛔ The retired `ut_odoo_customers` / + `ut_odoo_products` keys are NOT the subject and must not be revived: W33-T44 / A2 retired + them, `odoo_relational.RETIRED_KEYS` pops them out of every tenant's document on rebuild, and + their subjects are claimed by these two registry rows, which is now the only claim on each. + + ⛔⛔ THREE PREDICATES, THE SAME THREE `routes_tables._module_link_targets` USES ON THESE EXACT + TWO KEYS, and none of them is `is_admin`. A blanket administrator check here would silently + drop two nodes from a member's diagram for no stated reason, while every other node on the + same picture is walled per table: + 1. `perms.tenant_has_module` — is this database part of THIS tenant's world at all? Every + provisioned tenant but #0 carries a restricted `modules:` list, so without it another + company's admin is shown a database their workspace does not have. + 2. `perm_scope.nav_may_open` — ADMISSION, the predicate `deps.py`'s module gate itself asks. + `may_read` alone would draw a node whose door then 403s. + 3. `perm_scope.may_read` — the READ wall, the same one the `ut_` loop applies per table. + + ⛔ `user_tables.may_open` IS NOT IN THAT LIST AND ITS ABSENCE IS DELIBERATE: it answers about + a STORED definition's creator and share grants, and a registry module has no such row, so it + refuses every reader including the tenant's owner. + + ⭐ THE FIELDS ARE `automation_engine._module_table`'s, WHICH IS THE ONE COMPOSER OF A MODULE'S + FIELD CONTRACT (`aios_grid` plus `odoo_relational.module_link_fields`). Restating either half + here is how this diagram would start disagreeing with the engine that resolves the very links + it draws. `want_rows=False`, so no pool is built and no store is written. + + ⚠ AND THE FIELD LIST IS THE VIEWER'S: `visible_fields` strips the hidden closure, so a column + an administrator hid from this account is neither drawn on the card nor usable as an edge + endpoint. Fail-closed everywhere: a wall that cannot be answered drops the node. + """ + try: + import automation_engine # noqa: PLC0415 - lazy, as elsewhere + import core.perm_scope as perm_scope + import core.perms as perms + import core.registry as registry + import core.user_tables as user_tables + except Exception: # noqa: BLE001 + return {} + out = {} + memo = {} + for module_key in getattr(user_tables, "LINKABLE_MODULE_KEYS", ()): + key = str(module_key or "") + row = registry.BY_KEY.get(key) or {} + connection_key = _module_connection_key(row.get("subject")) + if not key or connection_key not in connected: + continue + try: + if not perms.tenant_has_module(rt, key): + continue + if not perm_scope.nav_may_open(user, key, st=lent): + continue + if not perm_scope.may_read(user, key, st=lent): + continue + # ⛔ `_module_table`, THE ENGINE'S OWN COMPOSER (`# noqa: SLF001` for the same reason + # `_database_name_overrides` calls `routes_nav._read_nav_meta`: one reader for one + # fact beats a correct-looking second copy). + table, _limit = automation_engine._module_table(rt, key, memo) # noqa: SLF001 + declared = [dict(field) for field in ((table or {}).get("fields") or ()) + if isinstance(field, dict)] + if not declared: + continue + visible = perm_scope.visible_fields(declared, user, key, st=lent) + except Exception: # noqa: BLE001 + continue + fields = [] + for field in visible: + field_key = str(field.get("key") or "").strip() + if field_key: + fields.append({"key": field_key, + "label": str(field.get("label") or field_key).strip() or field_key}) + if not fields: + continue + # ⚠ THE REGISTRY'S OWN LABEL, READ RATHER THAN RETYPED. "Odoo customers" and "Odoo + # products" are `core/registry.py`'s `label` values (renamed there once already, W33-T42), + # and a literal copy here is a second authority that goes stale the day the row is + # renamed. There is no tenant override to merge in: `routes_nav` refuses a name for any + # key that does not start with `ut_`, so a registry module cannot be renamed at all. + label = str(row.get("label") or key).strip() or key + out.setdefault(connection_key, {})[key] = { + "dto": {"key": key, "label": label, "fields": fields}, + "visible": visible, + } + return out + + +def relationship_graphs(rows, user, rt, uname, is_admin=False): + """Actual, permitted tables and same-connection declared links for directory rows. + + There is deliberately no provider fallback. A connected credential with no materialised + database gets an explicit unavailable envelope; a non-connected card gets an explicit + refusal. The client can therefore distinguish "nothing exists for this selection" from a + plausible-looking sample schema. + """ + row_by_key = {str(row.get("key") or ""): row for row in (rows or []) + if isinstance(row, dict) and str(row.get("key") or "")} + graphs = {} + connected = set() + for connection_key, row in row_by_key.items(): + if row.get("state") == "connected": + connected.add(connection_key) + graphs[connection_key] = _graph_refusal( + connection_key, + "No permitted tables are materialised for this connection. A provider template is not shown.", + status="unavailable") + else: + graphs[connection_key] = _graph_refusal( + connection_key, + "Connect this provider before viewing its actual relationship graph.") + if not connected: + return graphs + + import core.perm_scope as perm_scope + import core.user_tables as user_tables + try: + definitions = user_tables.all_defs(st=rt) + lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions}) + except Exception: # noqa: BLE001 + for connection_key in connected: + graphs[connection_key] = _graph_refusal( + connection_key, + "The workspace tables could not be read, so this relationship graph is unavailable.", + status="unavailable") + return graphs + + tables_by_connection = {connection_key: {} for connection_key in connected} + for table_key, definition in (definitions or {}).items(): + connection_key = _table_connection_key(table_key, definition) + if connection_key not in connected: + continue + try: + # Prefixes are not enough: a table must carry the connector materialisation stamps + # before it can describe a connector graph. + if user_tables.is_managed_system_table(table_key, st=lent, definition=definition): + continue + if not user_tables.is_connected(table_key, st=lent): + continue + if not user_tables.may_open(table_key, uname, is_admin, st=lent): + continue + if not perm_scope.may_read(user, table_key, st=lent): + continue + visible = perm_scope.visible_fields( + [dict(field) for field in (definition.get("fields") or ()) + if isinstance(field, dict)], + user, table_key, st=lent) + except Exception: # noqa: BLE001 + # A table whose permission proof cannot be completed does not become a partial leak. + continue + fields = [] + for field in visible: + key = str(field.get("key") or "").strip() + if key: + fields.append({"key": key, + "label": str(field.get("label") or key).strip() or key}) + tables_by_connection[connection_key][str(table_key)] = { + "dto": {"key": str(table_key), + "label": str(definition.get("label") or table_key).strip() or str(table_key), + "fields": fields}, + "visible": visible, + } + + # ⭐⭐ W53-T30 — AND THE REGISTRY MODULES THIS CONNECTION STANDS BEHIND. They are not stored + # definitions, so the loop above cannot reach them; they are permitted by their own three + # predicates and then join the SAME edge pass, which is what makes an already-declared link + # between a `ut_` table and a module resolvable at both ends for the first time. + for connection_key, entries in _module_graph_entries(rt, user, lent, connected).items(): + tables_by_connection.setdefault(connection_key, {}).update(entries) + + for connection_key, entries in tables_by_connection.items(): + if not entries: + continue + table_keys = set(entries) + visible_keys = { + table_key: {str(field.get("key") or "").strip() + for field in entry["visible"] if isinstance(field, dict)} + for table_key, entry in entries.items() + } + relationships = [] + seen = set() + for source_table, entry in entries.items(): + for field in entry["visible"]: + if not isinstance(field, dict) or field.get("type") != "link": + continue + link = field.get("link") + source_field = str(field.get("key") or "").strip() + target_table = str((link or {}).get("table") or "").strip() if isinstance(link, dict) else "" + target_field = str((link or {}).get("on") or (link or {}).get("inverse") or "").strip() if isinstance(link, dict) else "" + # An outside, unavailable, or field-hidden endpoint is not a relationship on the + # selected connection. Omit it rather than exposing the target through an edge. + if (not source_field or target_table not in table_keys or not target_field + or target_field not in visible_keys.get(target_table, set())): + continue + signature = (source_table, source_field, target_table, target_field) + if signature in seen: + continue + seen.add(signature) + relationships.append({ + "from": {"table": source_table, "field": source_field}, + "to": {"table": target_table, "field": target_field}, + "label": str(field.get("label") or source_field).strip() or source_field, + }) + # ⭐ W53-T30 / C2 — THE SPEC'S OWN DECLARED RELATIONS, held to the same two proofs as the + # field bags above: both endpoints must be nodes on THIS graph and both named columns must + # be visible to THIS reader. `{}` until lane D's `core.connector_spec` lands, and the + # `seen` signature is shared so a relation declared in both places is drawn once. + label_of = {(table_key, str(field.get("key") or "")): str(field.get("label") or "") + for table_key, entry in entries.items() + for field in entry["dto"]["fields"]} + for source_table, declared in _connector_spec_links(connection_key).items(): + if source_table not in table_keys: + continue + for source_field, target_table, target_field in declared: + if (source_field not in visible_keys.get(source_table, set()) + or target_table not in table_keys + or target_field not in visible_keys.get(target_table, set())): + continue + signature = (source_table, source_field, target_table, target_field) + if signature in seen: + continue + seen.add(signature) + relationships.append({ + "from": {"table": source_table, "field": source_field}, + "to": {"table": target_table, "field": target_field}, + "label": label_of.get((source_table, source_field)) or source_field, + }) + graphs[connection_key] = { + "connectionKey": connection_key, + "status": "ready", + "reason": "", + "tables": [entries[key]["dto"] for key in sorted( + entries, key=lambda key: (entries[key]["dto"]["label"].casefold(), key))], + "relationships": sorted( + relationships, + key=lambda relation: (relation["from"]["table"], relation["from"]["field"], + relation["to"]["table"], relation["to"]["field"])), + } + return graphs + + +def attach_relationship_graphs(rows, user, rt, uname, is_admin=False): + """Bind every card to only its own actual graph; no key receives another provider's DTO.""" + graphs = relationship_graphs(rows, user, rt, uname, is_admin) + for row in rows: + connection_key = str(row.get("key") or "") + row["relationshipGraph"] = graphs.get(connection_key) or _graph_refusal( + connection_key, + "No actual relationship graph was returned for this connection. A provider template is not shown.") + return rows + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W31-T44 — EVERY WAY THIS DIRECTORY CAN COME BACK SHORT, ENUMERATED AND MADE VISIBLE. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner named Connectors FIRST in item 6 ("Connectors and Automation module still disappears"). +# `.shell-nav-connectors` is a static `` and cannot be dropped from the rail, so if Connectors +# "disappears" the subject is THIS payload. Enumerated by reading the composer rather than by +# guessing, and the answer was sharper than expected: +# +# ⛔ THERE WAS EXACTLY ONE FAILURE MODE AND IT DROPPED **EVERY** ROW. `directory()` had no +# `try` anywhere. `oauth_connect.status()`, `import automation_engine` and `bd_ready()` are +# three unguarded calls into three other subsystems, and a raise in ANY of them turned the +# whole endpoint into a 500 — at which point `ConnectorsPage` renders one sentence and no +# cards. A page whose entire job is to be a map of the workspace showed nothing because one +# registry was unhappy. +# ⚠ AND ONE SILENT ONE, WHICH IS WORSE THAN THE LOUD ONE. `_entry_types` swallowed every +# exception into an empty set, so an unreadable or locked keychain rendered Odoo, Stripe and +# Shopify as **"available"** — i.e. *"you have not connected this"* about a connector the +# tenant HAS connected. Not short: wrong, and confidently. +# +# ⭐ SO THE COMPOSER IS A LIST OF CONTRIBUTIONS AND EACH ONE FAILS ALONE. A contribution that +# raises costs its own rows and NOTHING else, and says so in `incomplete` with a cause and a +# recommendation — R6's second sentence, applied to a page instead of to a row cap. Silence is the +# violation, not the limit. +#: The contributions, by name. ⛔ THIS LIST IS THE ENUMERATION the ticket asks for, and +#: `verify_meta` breaks each one IN TURN and requires (a) the other rows still render and (b) this +#: name appears in `incomplete` — so a contribution added later without a probe is a red, not a +#: quietly unprotected path. +CONTRIBUTORS = ("keychain", "odoo", "oauth", "token", "nango", "scraper", "builtin", "planned") + + +def _entry_types(rt, uname=""): + """`(types, failure)` — which keychain entry types THIS READER has, and what stopped us. + + ⛔ IT USED TO RETURN A BARE SET AND SWALLOW EVERYTHING, and that is the silent half of item 6b: + a locked keychain became an EMPTY set, which is indistinguishable from "this tenant has + connected nothing" — so the page said `available` about a live Odoo connection. Degrading to + "nothing connected" is only honest if the reader is told that is what happened. + + ⭐ W32-T11 / R4 — AND IT IS THIS READER'S SET, NOT THE TENANT'S. The module header has always + said *"STATE IS PER-READER … a directory that reported another user's Gmail as connected would + be both wrong and a small disclosure"*, and a personal keychain entry is exactly that case one + credential kind over. Reading `list_entries` straight would have lit a colleague's Stripe card + on your directory while the Settings pane next door correctly hid it — the same + two-answers-to-one-question shape W32-T10 just removed from the Odoo row. + + ⚠ THE SECOND RETURN VALUE IS THE SCOPE PER TYPE, and BUSINESS WINS. A reader holding both a + business-wide Stripe key and a personal one is connected business-wide — the stronger fact is + the true one, and taking whichever row sorted first would make the card's label depend on an + id. + """ + try: + import routes_keychain as rk + rows = rk.visible_entries(rt, uname) or [] + types, scopes = set(), {} + for e in rows: + t = str(e.get("type") or "") + if t == "amazon_sp": + from core import keychain as kc + if kc.entry_verified(rt, e.get("id")): + types.add(t) + else: + types.add(t) + if scopes.get(t) != rk.DEFAULT_SCOPE: + scopes[t] = str(e.get("scope") or rk.DEFAULT_SCOPE) + return types, scopes, None + except Exception as e: # noqa: BLE001 + return set(), {}, f"{type(e).__name__}: {e}" + + +def _stored_amazon_rotation_deadline(rt, entry_id): + """Project only the public timestamp from the stored row, without decrypting a credential.""" + from core import keychain as kc + stored = ((rt.get(kc.KEY) or {}).get("entries") or {}).get(str(entry_id)) or {} + if stored.get("type") != "amazon_sp": + return "" + deadline = stored.get(AMAZON_ROTATION_DEADLINE_FIELD) + return deadline if isinstance(deadline, str) else "" + + +def _amazon_rotation_expired(deadline): + if not deadline: + return False + try: + parsed = datetime.fromisoformat(deadline.replace("Z", "+00:00")) + return parsed.tzinfo is not None and parsed <= datetime.now(timezone.utc) + except (TypeError, ValueError): + return False + + +def directory(rt, uname, is_admin=False, incomplete=None): + """The composed row set. Pure over the session, so the gate drives it without a request. + + `incomplete` is an optional list the caller lends to collect contribution failures; the route + passes one and puts it on the wire. Callers that do not care pass nothing and simply get the + rows that composed. + """ + rows = [] + short = incomplete if incomplete is not None else [] + + def contribute(source, fn, why, fix): + """Run one contribution. A failure costs ITS rows and records why — never the page.""" + try: + fn() + except Exception as e: # noqa: BLE001 + short.append({"source": source, "cause": f"{why} ({type(e).__name__}: {e})", + "recommendation": fix}) + + have, scopes, kc_failed = _entry_types(rt, uname) + if kc_failed: + short.append({ + "source": "keychain", + "cause": f"this tenant's keychain could not be read ({kc_failed}), so a connector " + f"that IS connected through a stored credential is shown as available", + "recommendation": "unlock the keychain (AIOS_KEYCHAIN_KEY) and reload; the states " + "below are otherwise correct"}) + + # --- Odoo: the reference connector. Env credentials are tenant #0's only (the R3 rule), so + # a stored keychain entry is what makes it connected for anybody else. + # + # ⛔⛔ W32-T10 / OWNER ITEM 6 — THIS ROW SPENT FOUR WAVES TELLING EVERY TENANT IT WAS + # CONNECTED TO ANOTHER COMPANY'S ERP. The state read `"odoo" in have or + # os.environ.get("ODOO_URL")`, and the second leg has no tenant in it at all: one Space + # process serves every tenant, so `ODOO_URL` being set for tenant #0 lit this card + # `connected` for nurilab, GTM Lab and everyone after them — and `manage: keychain` then + # offered "Manage keys" for a credential their workspace does not hold. **The comment + # directly above stated the rule the line broke**, which is the useful part of the lesson: + # the guard was written down and not written in. + # ⚠ THE OTHER HALF OF THE FIX IS THAT NOTHING HERE RE-DERIVES IT. `env_odoo_available` lives + # in `routes_keychain`, which asked this question CORRECTLY the whole time — so the repair is + # to delete this module's answer, not to write a third one + # ([[one-question-two-normalizers]]). + # ⚠ The `have` leg is UNTOUCHED on purpose: `list_entries` reads metadata even while the + # keychain is LOCKED, so a tenant with a stored Odoo entry stays `connected` when the + # platform key is missing. Routing that leg through a resolver that requires `unlocked()` + # would have flipped a real connection to "available" — the silent case W31-T44 exists to + # prevent, re-introduced by the fix for its sibling. + def _odoo(): + rows.append({ + "key": "odoo", "label": "Odoo", "kind": "builtin", + "desc": "Sales, invoices, products and customers from your Odoo ERP.", + "state": ("connected" if ("odoo" in have or env_odoo_available(rt)) + else "available"), + # W32-T11 / C1 — always business: Odoo is what the whole workspace's databases are + # read through, which is why `routes_keychain.TENANT_WIDE_TYPES` refuses to make it + # personal in the first place. + "scope": scopes.get("odoo") or "business", + "manage": "keychain" if is_admin else "", + }) + + # ⚠ GUARDED EVEN THOUGH IT CANNOT RAISE TODAY — `os.environ.get` and a set membership are the + # whole of it. The guard is here because the enumeration is only worth something if it is + # EXHAUSTIVE: an unguarded block is an un-enumerated drop path, and "this one is obviously + # safe" is how the other five got written in the first place. + contribute("odoo", _odoo, "the Odoo connector row could not be composed", + "reload; every other row is unaffected") + + # --- OAuth providers, straight off the registry (so a new entry appears here for free). + # ⛔ `status()` READS EVERY PER-USER CREDENTIAL SLOT, which means it touches the keychain and + # the store — the two things most likely to be unavailable. Unguarded, it took the whole page + # with it. + # ⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — `oauth_connect.offered()` FILTERS THIS CONTRIBUTION. + # The owner reported Google and Gmail as broken connectors: the card offered a Connect button + # and the button answered `503 oauth_unavailable` in raw JSON. R6 is *"hidden until they are + # real"*, so the row goes rather than the error being prettied up — the tombstone and the + # money reason (D-45, CASA verification at ~$540–1,800/yr) live on `offered()` itself. + # ⚠ THE REGISTRY IS STILL WALKED WHOLE. A registry-driven directory is the point of this + # module ("a new provider appears here for free"), so the filter is one predicate on the loop + # and NOT a deletion — the day the verification is paid for, one key flips and the row is + # back with its state, its `needs` and its brand mark intact. + def _oauth(): + st = oauth_connect.status(rt, uname) + for slug, meta in oauth_connect.PROVIDERS.items(): + if not oauth_connect.offered(slug): + continue + rows.append(_oauth_row(slug, meta, st.get(slug) or {})) + + contribute("oauth", _oauth, + "the OAuth provider registry could not be read, so no OAuth connector is listed", + "the rest of the directory is complete; reload to retry the OAuth states") + + # --- Token connectors (R8): connectable today, and honest that data flows arrive later. + def _tokens(): + amazon_expired = False + if "amazon_sp" in scopes: + import routes_keychain as rk + amazon_expired = any( + _amazon_rotation_expired(_stored_amazon_rotation_deadline(rt, entry.get("id"))) + for entry in rk.visible_entries(rt, uname) or [] + if entry.get("type") == "amazon_sp") + for t in TOKEN_CONNECTORS: + rows.append({ + "key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"], + "hint": t["hint"], + "state": ("expired" if t["entry"] == "amazon_sp" and amazon_expired + else "connected" if t["entry"] in have else "reconnect" + if t["entry"] == "amazon_sp" and t["entry"] in scopes + else "available"), + # ⭐ W32-T11 / C1 / R4 — the scope of the credential BEHIND this card, for THIS + # reader. A token connector is the case R4 is really about: a member's own Stripe + # key is theirs, and the card must say which kind of connection it is looking at. + "scope": scopes.get(t["entry"]) or "business", + # ⭐⭐ W33-T66 / D-204 — THE MEMBER'S DOOR, AND IT IS A THIRD WORD RATHER THAN A + # WIDENED ONE. This route's own docstring has promised since wave 23 that a + # non-admin may *"connect their OWN per-user accounts"*, and the next sentence + # then handed them an empty `manage` — a contradiction inside one paragraph, and + # D-204's subject. W32-T11 opened the ROOM (`admin_gate` is gone from + # `routes_keychain`; a member may POST `scope: "personal"`), so the only thing + # still missing was a way to reach it from the map. + # + # ⛔ `keychain` DOES NOT WIDEN, and that is deliberate rather than timid. + # `verify_automation`'s NC asserts a non-admin is offered no `manage == "keychain"` + # row, and its own comment says why in the right words: *"the line is TENANT-WIDE + # vs PER-USER, not admin vs everyone … what they must never be offered is the + # KEYCHAIN door, which edits credentials the whole tenant uses."* That assertion + # is CORRECT and stays green. A member's personal connection is a different door + # into the same room, so it gets its own word. + # + # ⚠ THE PREDICATE IS THE CREDENTIAL'S OWN RULE, IMPORTED, NOT RE-STATED HERE. + # `TENANT_WIDE_TYPES` is what `routes_keychain` REFUSES a personal scope for, so + # offering a personal door for one of those types would be offering a button that + # 400s — the exact failure R6 removed from the Google card. Today that means + # Stripe and Shopify open for a member and Meta Ads does not + # ([[one-question-two-normalizers]]). + "manage": ("keychain" if is_admin + else "" if t["entry"] in TENANT_WIDE_TYPES + else "keychain_personal"), + # Said once, on the row, rather than in a paragraph somewhere (R13): storing the + # key is real and useful, and it is not the same as the data being on screen. + "note": "Storing the key connects the account; reading its data lands in a later " + "release.", + }) + + contribute("token", _tokens, + "the token-connector registry could not be composed", + "Stripe and Shopify are missing from this list only; nothing else is affected") + + def _nango(): + rows.extend(_nango_rows()) + + # ⚠ W32-T13 / ITEM 3 — THE CONTRIBUTOR KEY STAYS `nango`, THE SENTENCES DO NOT. `source` is an + # internal id (it must match `CONTRIBUTORS`, and `verify_meta` breaks this contribution by + # name), and `DirectoryGaps` renders only `cause` and `recommendation` — so the key never + # reaches a screen while these two strings do. De-branding the id instead would have renamed + # a gate's subject to fix a string nobody sees. + contribute("nango", _nango, + "the sign-in provider registry could not be composed", + "the providers you sign in to are missing from this list; every other row is " + "unaffected") + + def _scraper(): + rows.append(_scraper_row()) + + contribute("scraper", _scraper, + "the Scraper connector's readiness could not be determined " + "(automation_engine did not answer)", + "the Scraper card is missing from this list; automations are unaffected") + + def _builtin(): + rows.append({ + "key": "webhooks", "label": "Webhooks", "kind": "builtin", + "desc": "Let another system start an automation by calling a URL.", + "state": "connected", "manage": "automation", + }) + + contribute("builtin", _builtin, "a built-in connector row could not be composed", + "reload; the remaining rows are correct") + + def _planned(): + for p in PLANNED: + rows.append({**p, "state": "planned", "needs": "not built yet", "manage": ""}) + + contribute("planned", _planned, "the planned-connector list could not be composed", + "the to-do list is missing; every CONNECTED row above is still correct") + + # ⭐⭐ W49-T18 · R27 / C4 — THE TAXONOMY, AND THE CATALOGUE THAT DEFINES IT. Every row above + # is filed into one of the fourteen categories, and the authored catalogue is appended beside + # them so the map covers the things we can reach as well as the things we have wired. R27: + # *"don't skip Odoo or meta ads, its important to learn how they map the schema for our own + # benefit."* The crosswalk itself lives in `core.connector_directory`, which explains at + # length why it cannot be generated from Monid's API. + # + # ⛔⛔ ITS `source` IS NOT IN `CONTRIBUTORS`, AND THAT IS DELIBERATE RATHER THAN AN OVERSIGHT. + # `verify_meta.section_directory_gaps` asserts `set(CONTRIBUTORS) - probed == {"builtin", + # "odoo"}` against a probe table declared IN THAT GATE, which lives outside this ticket's + # fence — so adding the name here would turn a correct change red in a file this lane may not + # edit ([[two-gates-can-assert-opposite-things]]). The BEHAVIOUR the enumeration exists for is + # unchanged: this contribution is guarded like every other one, so a raise costs the taxonomy + # and records why, never the page. Booked as a PENDING row: add a `catalogue` probe to + # `verify_meta` and put the name in `CONTRIBUTORS` in the same change. + # + # ⚠ THE ORDER MATTERS. `file_natives` must run over the rows the contributors composed and + # BEFORE the catalogue is appended, or the catalogue rows would have their own categories + # overwritten with the empty native answer. + # + # ⛔⛔ AND THESE ROWS ARE NOT `planned`, WHICH THEY WERE FOR ONE WAVE. `planned` is the + # contribution directly above this one: a row from `PLANNED`, carrying a `needs`, that R8 put + # on this page as a visible to-do list. A vendor's catalogue listing satisfies none of that — + # we are not building it, nothing is waiting on us, and it has no `needs` — so it wears + # `connector_directory.CATALOGUE_STATE`, a SIXTH word beside + # `connected | available | reconnect | unconfigured | planned`. Borrowing `planned` for the + # fade it happens to render made the served planned set 120 rows wide and turned Monid's + # catalogue into our roadmap; the fade itself never depended on the word and does not now. + # ⚠ A GATE HOLDING A HAND-TYPED COPY OF THE FIVE-WORD SET MUST LEARN THE SIXTH. The vocabulary + # is a property of THIS module, and a literal set elsewhere is a second authority for it. + def _catalogue(): + connector_directory.file_natives(rows) + rows.extend(connector_directory.monid_rows()) + + contribute("catalogue", _catalogue, + "the connector directory's category taxonomy could not be composed", + "every connector above is still listed and still correct; the category filter and " + "the wider catalogue are missing from this list only") + + # ⭐⭐ W53-T29 / OWNER RULING R6 — EVERY ROW DECLARES WHICH SCOPES IT MAY EVER CARRY. + # + # R6: *"Each catalogue source declares which scopes it allows (Odoo company-only; Gmail + # both)."* The Company / Mine toggle needs that per SOURCE, not per stored connection: a card + # may only ever appear under Company if `business` is in this list, and under Mine only if + # `personal` is — so Odoo cannot land under Mine however its `scope` was written, and a + # personal-only source cannot land under Company. + # + # ⛔ ONE PASS OVER THE COMPOSED SET, NOT A LINE IN EACH CONTRIBUTOR. Six functions above append + # rows and a seventh extends the list with a vendor catalogue; a declaration repeated in each + # is a declaration the eighth composer forgets, and the client would then treat the missing + # key as "unrestricted" with nothing going red. Stamped here, a new contributor gets the right + # answer for free. + # ⛔ AND THE ANSWER IS `routes_keychain.allowed_scopes`, IMPORTED. This module already imports + # `TENANT_WIDE_TYPES` rather than keeping a copy (see the import note at the top of the file); + # the eligibility rule built ON that tuple is fetched the same way for the same reason. + # ⚠ A ROW WITH NO KEYCHAIN TYPE (an OAuth slot, the Scraper, a catalogue listing) answers with + # both words, which is the honest default: nothing about it forbids either scope, and the one + # thing the toggle must never do is hide a connection the reader actually holds. + for row in rows: + row["allowedScopes"] = list(allowed_scopes(_entry_type_for(row.get("key")))) + _safe_connector_status(row.get("key"), row) + return rows + + +def _oauth_row(slug, meta, row_st): + """One OAuth provider's card. Lifted out of `directory` whole so the composer reads as a list + of contributions; behaviour is byte-identical.""" + return { + "key": slug, "label": meta.get("label") or slug.title(), "kind": "oauth", + "desc": "Gmail for the email trigger; the same connection unlocks Sheets, Drive " + "and Calendar later." if slug == "google" else "", + # THREE distinct states, and the distinction is the point: a provider nobody has + # configured on this deployment is not the same as one this user has not connected, + # and until now both rendered identically (the C11 status() gap). + "state": ("connected" if row_st.get("connected") and not row_st.get("reconnect") + else "reconnect" if row_st.get("reconnect") + else "available" if row_st.get("configured") + else "unconfigured"), + "connectedAs": row_st.get("email") or "", + # ⭐⭐ W53-T29 / R6 — A PER-USER SLOT IS PERSONAL, AND THE CARD NOW SAYS SO. + # `_oauth_slot_entries` below already stamps `"scope": "personal"` on these very + # credentials ("a per-user slot is personal by construction, in the vocabulary + # `routes_keychain`"), and the ROW carried no scope at all — so a connected Google card + # read as business-wide on the page while its own Keychain tab called it personal, and the + # Company / Mine split would have filed this reader's own sign-in under Company. One fact, + # said the same way in both places. + # ⚠ NO CARD MOVES TODAY: `oauth_connect.offered()` filters every provider out of the + # directory (D-45, the CASA verification nobody has bought), so this is the answer waiting + # for the day one of them is offered again rather than a change to what renders now. + "scope": "personal", + "startUrl": f"/api/v1/oauth/{slug}/start", + "manage": "oauth", + "needs": "" if row_st.get("configured") else "the owner registers the client", + } + + +def _nango_rows(): + """One card per `NANGO_PROVIDERS` row, each stating what it can do TODAY (R7). + + ⛔ NO NETWORK CALL. The obvious "honest" implementation asks Nango which integrations exist + and lights the ones that do — and that would put a third-party HTTP round trip on a page this + very wave is making faster, on a route that already had exactly one failure mode. The + deployment fact this page can answer CHEAPLY is whether the platform holds a Nango key at all, + which is the same class of check `bd_ready()` and `oauth_connect.configured()` already make. + ⚠ Whether a SPECIFIC integration is configured in the Nango dashboard is a different fact, + measured once in `proto/nango-verdict.md` (1 integration, and it is a GitHub sample), and it + is what `needs` names rather than pretends to have checked. + + ⛔⛔ EVERY ROW IS `available` AT BEST AND CARRIES NO `manage` DOOR, which is R7's second + sentence made structural: *a card whose button leads nowhere without saying so* fails the + ruling, so these cards have no button at all and say why in `needs` and `note`. + """ + have_key = bool((os.environ.get("NANGO_SECRET_KEY") or "").strip()) + rows = [] + for p in NANGO_PROVIDERS: + rows.append({ + "key": p["key"], "label": p["label"], "kind": "oauth", "desc": p["desc"], + "provider": p["provider"], + "state": "available" if have_key else "unconfigured", + # ⭐⭐ WAVE 32 · OWNER ITEM 3 / R3 — DE-BRANDED. These two sentences used to read: + # + # needs: "this provider's integration must be created in the Nango dashboard first" + # / "the owner registers NANGO_SECRET_KEY on this deployment" + # note: "Connects through Nango. Reading this provider's data into a database is + # not built yet — measured: no sync is configured on this account." + # + # ⛔ R3 IS "DE-BRAND, DON'T REVOKE": the transport is unchanged and still Nango's — + # `oauth_connect.nango_probe`, `NANGO_SECRET_KEY` and every constant keep their names, + # because that is what the person setting the secret has to type. What goes is the + # supplier's name on a CUSTOMER's screen. Which vendor carries an OAuth handshake is + # our problem, exactly as `AIOS_BRIGHTDATA_KEY` sits behind a card labelled "Scraper". + # ⚠ THE HONESTY THE SENTENCE CARRIED IS NOT LOST, which is the half a de-brand usually + # drops: R7's rule is that a lit provider states what it can do TODAY, so the note + # still says "not built yet" in as many words, and `needs` still names the remaining + # step — it just no longer names the company doing it. + "needs": ("this connection has not been set up for this provider yet" + if have_key else + "the owner finishes connection setup on this deployment"), + # ⛔ THE SENTENCE R7 DEMANDS, on the row rather than in a paragraph somewhere (R13). + "note": "Signing in works. Reading this provider's data into a database is not " + "built yet.", + "manage": "", + }) + return rows + + +def _scraper_row(): + """The Scraper card. Its own function because `bd_ready()` reaches into `automation_engine` — + an import and a call into another subsystem, i.e. the single most likely thing in this file to + raise, and until W31-T44 it could take the whole page with it.""" + import automation_engine as _eng + # ⭐ WAVE 25 · C2 / owner ruling R9 — THE ROW IS "SCRAPER", NOT THE VENDOR'S NAME. + # + # ⛔ THE KEY CHANGED TOO (`brightdata` -> `scraper`), which is the half worth being careful + # about: it is what the trigger picker's Connector sub-group is named after, so the two must + # agree. Grepped before flipping — NO client file switches on this key (the six client hits + # for "brightdata" are all `CaptureTier`, which is the VENDOR and is deliberately untouched), + # and the one server reader was `verify_automation`'s directory-key check, retargeted in the + # same change. + # + # ⚠ THE VENDOR IS REAL AND STAYS REAL. `AIOS_BRIGHTDATA_KEY`, `bd_ready()` and every `BD_*` + # constant are the supplier and keep their names — this renames what a CUSTOMER sees, because + # which supplier answers a scrape is our problem and not something to make them learn. The + # `needs` string still names the env var, deliberately: it is read by whoever has to set it. + ready = _eng.bd_ready() + return { + "key": "scraper", "label": "Scraper", "kind": "builtin", + # ⭐ WAVE 30 · R4 — the sentence names the CAPABILITY; the platforms are the drill below. + # It used to say "Instagram profile and post capture", which stopped being true the day + # TikTok discovery shipped and would have to be re-edited for every platform after it. + "desc": "Create saved Instagram, TikTok and Google Maps database recipes.", + "state": "connected" if ready else "unconfigured", + "needs": "" if ready else "AIOS_BRIGHTDATA_KEY is not set on this deployment", + "manage": "", + # The directory taxonomy normally derives this flag from connected table prefixes. + # Scraper recipes are user databases, so the route stamps it again after normalisation. + "spawns_tables": True, + # Platforms remain a drill behind one recipe/database capability, never directory rows. + # `bd_ready()` describes today's social runners only; Maps' own sentence names that its + # execution provider is intentionally absent until T12. + "platforms": [dict(p) for p in SCRAPER_PLATFORMS], + } + + +@router.get("/connectors/directory") +def connectors_directory(session: Session = Depends(require_session)): + """C11: every connector this platform knows about, with its state for THIS reader. + + Session-gated rather than admin-gated on purpose: a non-admin should be able to see what the + workspace is connected to, and connect their OWN per-user accounts — which is the whole point + of R7's per-user slots. + + ⭐⭐ W33-T66 / D-204 — AND THAT SECOND CLAUSE IS NOW TRUE. This docstring used to promise it and + then say, one sentence later, that `manage` *"is empty for a non-admin"* — a contradiction + inside a single paragraph, which is how the promise survived three waves without a door behind + it. `manage` carries WHICH ROOM, and there are three answers for a keychain-backed row: + + · `keychain` — the tenant-wide credential door. **Admins only**, and it must stay + that way: it edits the key the whole workspace reads through. + · `keychain_personal` — the same room, for a connection that is this reader's own. Offered + to a non-admin **only** where a personal credential is actually + allowed (`routes_keychain.TENANT_WIDE_TYPES` is the refusal list, so + Odoo and Meta Ads never get it). + · `""` — nothing this reader can do here; the client renders no control + rather than a door that would 403. + """ + # ⚠ `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as + # careful and is silently False for every admin on earth — the manage actions would simply + # never appear, with nothing to catch it. Attribute name checked against `deps.Session`. + # ⭐ W31-T44 — `incomplete` IS PART OF THE ANSWER, not an error channel. The page's job is to + # be a map of the workspace; a map missing a region must say which region, or the reader + # concludes the region does not exist. That is the owner's item 6b in one sentence, and it is + # R6's second sentence applied to a payload: a limit that cannot be removed is REPORTED with + # its cause and a recommended fix. `[]` on a healthy read, so the client can test it directly. + incomplete = [] + rows = directory(session.runtime, session.uname, bool(session.admin), incomplete=incomplete) + # ⚠ W49-T18 — GRAPHS ARE ATTACHED TO OUR OWN ROWS ONLY, and the reason is copy rather than + # cost. `relationship_graphs` does no I/O for a row that is not `connected` (it returns a + # refusal envelope and the expensive `user_tables.all_defs` read is keyed on the connected + # set), so running it over the catalogue would be cheap. What it would produce is 118 copies + # of the sentence *"Connect this provider before viewing its actual relationship graph"* about + # rows that carry no credential and no door, which is a promise the product cannot keep. + # ⚠ THE CLIENT ALREADY HANDLES THE ABSENCE: `parseRelationshipGraph` answers a missing + # `connectionKey` with `graphRefusal(selectedKey, ...)`, so a card with no envelope on the + # wire renders exactly as it did before. The list comprehension holds the SAME dicts, so the + # in-place attachment still lands on the native rows. + attach_relationship_graphs([r for r in rows if r.get("provider") != "monid"], + session.user, session.runtime, session.uname, + bool(session.admin)) + # ⛔⛔ W49-T18 / C4 — THE ONE PLACE THE SERVER ROW BECOMES THE CLIENT DTO. `provider` + # (`native` | `monid`) is how we reason about where a row came from; R27's design rule is that + # it never renders, in any form, so it is stripped HERE rather than left to each reader to + # remember not to paint. `verify_catalog.py` reads this payload's shape and its negative + # control leaks the key on purpose to prove the leg can bite. + # ⚠ THE CHIP COUNTS ARE COMPUTED FROM THE ROWS ACTUALLY SERVED, so the number on a chip is the + # number of cards behind it by construction, and never a sum of endpoint counts (the ticket's + # trap; `chip_counts` carries the argument). + served = connector_directory.to_client_rows(rows) + for row in served: + if str(row.get("key") or "") == "scraper": + row["spawns_tables"] = True + _safe_connector_status(row.get("key"), row) + _safe_connector_status("amazon_sp", {"incomplete": incomplete}) + return {"connectors": served, + "incomplete": incomplete, + # ⭐⭐ W53-T29 / OWNER RULING R6 — MAY THIS READER SEE THE COMPANY SIDE AT ALL. + # + # R6: *"A non-admin does not see the Company side at all; the toggle itself is hidden + # for them."* That is a PERMISSION, so it is the server's answer and not something the + # page infers from whether some row happened to carry `manage: "keychain"`. + # ⛔ THE WORD IS `routes_keychain`'s OWN. `GET /admin/keychain` has answered + # `"canBusiness": bool(session.admin)` since W32-T11, for exactly this purpose ("the + # vocabulary and the permission, so the client renders a picker it can honour"), and + # the same fact under a second name would be two vocabularies for one boolean. + # ⚠ CARRIED HERE RATHER THAN FETCHED FROM THAT ROUTE. The keychain door lists every + # visible ENTRY, so a page that wanted one boolean would pull the workspace's whole + # credential list on every visit; this route already holds the session that decides it. + "canBusiness": bool(session.admin), + "categories": connector_directory.chip_counts(served), + # ⛔⛔ `planned` COUNTS THE ROADMAP AGAIN, AND FOR ONE WAVE IT COUNTED 120. W49-T18 + # gave every Monid listing the `planned` state because that state renders faded, so + # this number stopped being "things we intend to build" and became "things we intend + # to build, plus a vendor's whole catalogue". The listings wear + # `connector_directory.CATALOGUE_STATE` now and are counted as what they are; the + # constant is read rather than re-typed so the two cannot drift apart. + "counts": {"connected": sum(1 for r in served if r["state"] == "connected"), + "available": sum(1 for r in served if r["state"] == "available"), + "planned": sum(1 for r in served if r["state"] == "planned"), + "catalogue": sum(1 for r in served + if r["state"] == connector_directory.CATALOGUE_STATE)}} + + +_ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements" +_ODOO_SYNC_STATUS = "odoo_sync_status" +_ODOO_HEALTH_STATUS = "odoo_health_status_v1" +_ODOO_HEALTH_STAGES = ("source", "mirrorPull", "materialization", "reconciliation", "freshness") +_SYNC_REQUEST_CONNECTIONS = {} +_SYNC_REQUEST_CONNECTION_LOCK = threading.Lock() + + +def _odoo_connection_slot(rt, connection_id=None): + """Return the durable Odoo connection id used by jobs, status, and health receipts.""" + requested = str(connection_id or "").strip() + if requested and requested != "default": + return requested + try: + import odoo_relational + return str(odoo_relational.default_connection_id(rt) or "default") + except Exception: # noqa: BLE001 + return "default" + + +def sync_request_connection(provider, tenant): + """Return the connection bound to a process-local Sync now intent.""" + with _SYNC_REQUEST_CONNECTION_LOCK: + return _SYNC_REQUEST_CONNECTIONS.get((str(provider), str(tenant))) + + +def _odoo_failure(exc, stage="source"): + """Classify a source/database exception without persisting its potentially sensitive text.""" + stage = stage if stage in _ODOO_HEALTH_STAGES else "source" + text = str(exc or "").casefold() + exc_type = type(exc).__name__ if isinstance(exc, BaseException) else "SourceError" + if (isinstance(exc, ssl.SSLCertVerificationError) + or "certificate_verify_failed" in text + or "certificate verify failed" in text + or "certificate has expired" in text + or "self signed certificate" in text): + return {"stage": "source", "code": "tls_certificate_invalid", + "type": exc_type, + "message": ("Odoo TLS certificate verification failed because the certificate " + "is expired or untrusted. No source data was accepted.")} + if isinstance(exc, xmlrpc.client.Fault) and any( + token in text for token in ("accesserror", "access denied", "security restrictions")): + return {"stage": "source", "code": "source_read_refused", "type": exc_type, + "message": "Odoo refused the configured account's read access."} + if any(token in text for token in ("authentication", "access denied", "invalid api key")): + return {"stage": "source", "code": "source_auth_failed", "type": exc_type, + "message": "Odoo rejected the configured read-only credential."} + if isinstance(exc, (TimeoutError, ConnectionError)) or any( + token in text for token in ("timed out", "timeout", "name resolution", "connection refused")): + return {"stage": "source", "code": "source_unavailable", "type": exc_type, + "message": "Odoo could not be reached for this read-only sync."} + labels = { + "source": ("source_read_failed", "Odoo source validation failed."), + "mirrorPull": ("mirror_pull_failed", "The Odoo mirror pull failed."), + "materialization": ("materialization_failed", "The Odoo database could not be materialized."), + "reconciliation": ("reconciliation_failed", "The Odoo database reconciliation failed."), + "freshness": ("freshness_unconfirmed", "Freshness could not be confirmed for this database."), + } + code, message = labels[stage] + return {"stage": stage, "code": code, "type": exc_type, "message": message} + + +def _clean_odoo_failure(failure, stage): + if failure is None: + return None + if not isinstance(failure, dict): + return _odoo_failure(failure, stage) + return {"stage": str(failure.get("stage") or stage), + "code": str(failure.get("code") or f"{stage}_failed"), + "type": str(failure.get("type") or "SourceError")[:80], + "message": str(failure.get("message") or "Odoo sync failed.")[:300]} + + +def _odoo_stage_failure(stages): + """Return the earliest unresolved lifecycle failure, preserving its named cause.""" + return next((stages.get(name, {}).get("failure") for name in _ODOO_HEALTH_STAGES + if stages.get(name, {}).get("state") == "failed" + and stages.get(name, {}).get("failure")), None) + + +def _clean_odoo_database_receipt(receipt, database_key): + """Allow-list one data-free reconciliation receipt for durable health storage.""" + if not isinstance(receipt, dict): + return None + key = str(database_key or "").strip() + if str(receipt.get("databaseKey") or "").strip() != key: + raise ValueError("the reconciliation receipt belongs to a different database") + out = {"databaseKey": key} + for name in ("schemaKey", "topicKey", "domainKey", "model", "countModel", "state", + "sourceCountMode", "sourceReconciliation"): + value = receipt.get(name) + if value is not None: + out[name] = str(value)[:160] + for name in ("sourceCount", "mirrorCount", "materializedCount", "canonicalCount", + "servingCount", "duplicateCount", "explainedMergeCount", + "unexplainedMissingCount", "unexplainedExtraCount", + "unexplainedCanonicalMissingCount", "unexplainedCanonicalExtraCount", + "unexplainedCount"): + value = receipt.get(name) + out[name] = None if value is None else max(0, int(value)) + if "readThrough" in receipt: + out["readThrough"] = bool(receipt.get("readThrough")) + selection = receipt.get("selection") + if isinstance(selection, dict): + out["selection"] = { + name: (bool(selection.get(name)) if name == "enabled" + else str(selection.get(name) or "")[:160]) + for name in ("bucket", "rule", "enabled") if name in selection} + domain = receipt.get("domain") + if isinstance(domain, list): + # Curated domains contain public field/operator/value predicates. Refuse nested objects, + # which could otherwise turn this metadata seam into durable source-row storage. + clean_domain = [] + for predicate in domain: + if not isinstance(predicate, (list, tuple)) or len(predicate) != 3: + continue + field, operator, value = predicate + if isinstance(value, (list, tuple)): + clean_value = [item for item in value + if isinstance(item, (str, int, float, bool))][:50] + elif isinstance(value, (str, int, float, bool)) or value is None: + clean_value = value + else: + continue + clean_domain.append([str(field)[:100], str(operator)[:40], clean_value]) + out["domain"] = clean_domain + reconciliation = receipt.get("reconciliation") + if isinstance(reconciliation, dict): + out["reconciliation"] = { + name: str(reconciliation.get(name) or "")[:80] + for name in ("sourceToMirror", "mirrorToMaterialized", "mirrorToCanonical") + if name in reconciliation} + freshness = receipt.get("freshness") + if isinstance(freshness, dict): + out["freshness"] = { + name: str(freshness.get(name) or "")[:100] + for name in ("state", "capturedAt") if name in freshness} + for name in ("sourceError", "error"): + if receipt.get(name) is not None: + out[name] = _clean_odoo_failure(receipt.get(name), "reconciliation") + return out + + +def record_odoo_stage(rt, connection_id, stage, state, *, database_key=None, row_count=None, + refreshed_at=None, cadence_seconds=None, failure=None, receipt=None, + at=None): + """Persist one data-free Odoo stage transition for one connection/database. + + This is additive observability over the established ``connector_sync`` and materialization + job documents. It stores identifiers, counts, timestamps, and classified failures only; + credentials and source rows never cross this seam. + """ + if stage not in _ODOO_HEALTH_STAGES: + raise ValueError(f"unknown Odoo health stage: {stage}") + if state not in ("unknown", "queued", "running", "ready", "failed", "paused"): + raise ValueError(f"unknown Odoo health state: {state}") + cid = _odoo_connection_slot(rt, connection_id) + stamp = str(at or datetime.now(timezone.utc).isoformat()) + failed = _clean_odoo_failure(failure, stage) if state == "failed" else None + db_key = str(database_key or "").strip() + safe_receipt = _clean_odoo_database_receipt(receipt, db_key) if receipt is not None else None + + def _update(current): + document = dict(current or {}) + connections = dict(document.get("connections") or {}) + connection = dict(connections.get(cid) or {}) + stages = dict(connection.get("stages") or {}) + target_stages = stages + database = None + databases = None + if db_key: + databases = dict(connection.get("databases") or {}) + database = dict(databases.get(db_key) or {"databaseKey": db_key}) + target_stages = dict(database.get("stages") or {}) + previous = dict(target_stages.get(stage) or {}) + stage_row = {**previous, "state": state, "at": stamp, "failure": failed} + if state == "running": + stage_row["startedAt"] = stamp + stage_row["finishedAt"] = None + elif state in ("ready", "failed", "paused"): + stage_row["finishedAt"] = stamp + if state == "ready": + stage_row["lastSuccess"] = stamp + target_stages[stage] = stage_row + connection.update({"connectionId": cid, "stages": stages, "lastAttempt": stamp}) + if cadence_seconds is not None: + connection["cadenceSeconds"] = int(cadence_seconds) + + if db_key: + database.update({"databaseKey": db_key, "stages": target_stages, + "lastAttempt": stamp}) + if row_count is not None: + database["rowCount"] = max(0, int(row_count)) + if refreshed_at: + database["recordRefreshed"] = str(refreshed_at) + if safe_receipt is not None: + database["reconciliationReceipt"] = safe_receipt + db_failure = _odoo_stage_failure(target_stages) + if db_failure: + database["failure"] = db_failure + else: + database.pop("failure", None) + if state == "ready" and stage == "freshness": + database["lastSuccess"] = stamp + databases[db_key] = database + connection["databases"] = databases + else: + stages[stage] = stage_row + connection["stages"] = stages + connection_failure = _odoo_stage_failure(stages) + if connection_failure: + connection["failure"] = connection_failure + else: + connection.pop("failure", None) + if state == "ready" and stage == "freshness": + connection["lastSuccess"] = stamp + connections[cid] = connection + document["connections"] = connections + return document + + rt.update(_ODOO_HEALTH_STATUS, _update, flush="sync") + return odoo_health_receipt(rt, cid) + + +def _lag_seconds(stamp, now=None): + if not stamp: + return None + try: + parsed = datetime.fromisoformat(str(stamp).replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + current = now or datetime.now(timezone.utc) + return max(0, int((current - parsed.astimezone(timezone.utc)).total_seconds())) + except (TypeError, ValueError): + return None + + +def _empty_odoo_stages(): + return {stage: {"state": "unknown", "at": None, "startedAt": None, + "finishedAt": None, "lastSuccess": None, "failure": None} + for stage in _ODOO_HEALTH_STAGES} + + +def odoo_health_receipt(rt, connection_id=None, *, now=None): + """Project the truthful Odoo receipt for one connection and each of its databases.""" + cid = _odoo_connection_slot(rt, connection_id) + try: + document = rt.get(_ODOO_HEALTH_STATUS, fresh=True) or {} + except TypeError: + document = rt.get(_ODOO_HEALTH_STATUS) or {} + except Exception: # noqa: BLE001 + document = {} + stored = dict(((document.get("connections") or {}).get(cid)) or {}) + stages = _empty_odoo_stages() + stages.update({key: dict(value) for key, value in (stored.get("stages") or {}).items() + if key in stages and isinstance(value, dict)}) + databases = {str(key): dict(value) for key, value in (stored.get("databases") or {}).items() + if isinstance(value, dict)} + + # Durable materialization receipts are the authority for database work and survive restarts. + for job in connector_sync.jobs(rt, "odoo", cid): + key = str(job.get("databaseKey") or "") + if not key: + continue + database = dict(databases.get(key) or {"databaseKey": key}) + db_stages = _empty_odoo_stages() + db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items() + if name in db_stages and isinstance(value, dict)}) + job_state = str(job.get("state") or "unknown") + material_state = {"configured": "queued", "catalogue-ready": "queued", + "queued": "queued", "syncing": "running", "ready": "ready", + "failed": "failed"}.get(job_state, "unknown") + job_at = None + if job.get("at") is not None: + try: + job_at = datetime.fromtimestamp(float(job["at"]), timezone.utc).isoformat() + except (TypeError, ValueError, OSError): + job_at = None + job_failure = (_odoo_failure(job.get("error"), "materialization") + if material_state == "failed" else None) + db_stages["materialization"] = { + **db_stages["materialization"], "state": material_state, + "at": job_at, "finishedAt": job_at if material_state in ("ready", "failed") else None, + "lastSuccess": (job_at if material_state == "ready" else + db_stages["materialization"].get("lastSuccess")), + "failure": job_failure, + } + progress = job.get("progress") if isinstance(job.get("progress"), dict) else {} + if progress.get("rows") is not None: + database["rowCount"] = max(0, int(progress["rows"])) + database.update({"databaseKey": key, "model": str(job.get("model") or ""), + "state": material_state, "stages": db_stages}) + persisted_failure = database.get("failure") or _odoo_stage_failure(db_stages) + if persisted_failure or job_failure: + database["failure"] = persisted_failure or job_failure + elif material_state == "ready" and not database.get("failure"): + database["lastSuccess"] = database.get("lastSuccess") or job_at + databases[key] = database + + # The resident mirror state is process-local by design; overlay it only on the connection + # that owns this tenant's mirror. A named sibling must never inherit default's fresh badge. + cadence = stored.get("cadenceSeconds") + try: + _main = running_main() # D-781: resolve, never import + resident = dict(_main.RESYNC_STATE) + resident_cid = str(resident.get("connectionId") or "") + if resident_cid == cid: + cadence = resident.get("resolvedSeconds") or cadence + mirror_failure = resident.get("lastError") + mirror_state = ("paused" if resident.get("manual") else + "failed" if resident.get("lastPassOk") is False else + "ready" if resident.get("lastPassOk") is True else "unknown") + stages["mirrorPull"] = {**stages["mirrorPull"], "state": mirror_state, + "at": resident.get("lastPassAt"), + "finishedAt": resident.get("lastPassAt"), + "lastSuccess": resident.get("lastSuccessAt"), + "failure": (_clean_odoo_failure(mirror_failure, "mirrorPull") + if mirror_failure else None)} + except Exception: # noqa: BLE001 + pass + + for database in databases.values(): + db_stages = _empty_odoo_stages() + db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items() + if name in db_stages and isinstance(value, dict)}) + # Connection source/mirror/reconciliation facts apply to each database, while its own + # materialization/freshness facts remain independent. + for stage in ("source", "mirrorPull", "reconciliation"): + if db_stages[stage].get("state") == "unknown": + db_stages[stage] = dict(stages[stage]) + database["stages"] = db_stages + success = database.get("lastSuccess") or db_stages["freshness"].get("lastSuccess") + database["lastSuccess"] = success + database["currentLagSeconds"] = _lag_seconds(success, now) + database.setdefault("rowCount", None) + database.setdefault("recordRefreshed", None) + stage_failure = _odoo_stage_failure(db_stages) + if stage_failure: + database["failure"] = stage_failure + elif not database.get("failure"): + database["failure"] = None + + successes = [row.get("lastSuccess") for row in databases.values() if row.get("lastSuccess")] + last_success = max(successes, default=stored.get("lastSuccess")) + row_counts = [row.get("rowCount") for row in databases.values() + if row.get("rowCount") is not None] + failure = (_odoo_stage_failure(stages) or stored.get("failure") or next( + (row.get("failure") for row in databases.values() if row.get("failure")), None)) + return {"connectionId": cid, "cadenceSeconds": cadence, + "lastSuccess": last_success, "currentLagSeconds": _lag_seconds(last_success, now), + "rowCount": sum(row_counts) if row_counts else None, + "failure": failure, "stages": stages, + "databases": [databases[key] for key in sorted(databases)]} + + +def _odoo_admin(session): + if not session.admin: + raise err(403, "not_admin", "only an administrator can inspect the Odoo schema") + + +def _sync_admin(session): + if not session.admin: + raise err(403, "not_admin", "only an administrator can retry a connector sync") + + +def _connection_jobs(rt, provider, connection_id): + """A status endpoint selects one connection; the jobs inventory may enumerate them all.""" + cid = str(connection_id or "").strip() or "default" + if provider == "odoo" and cid == "default": + cid = _odoo_connection_slot(rt, connection_id) + return connector_sync.jobs(rt, provider, cid) + + +def _sync_payload(connector_key, connection_id, row=None): + """One wire shape for every registered connector, including one not yet run.""" + meta = connector_sync.providers()[connector_key] + slot = str(connection_id or "").strip() or "default" + if row is None: + return _safe_connector_status(connector_key, {"provider": connector_key, "label": meta["label"], + "connectionId": slot, "recorded": False, "state": None, + "at": None, "stateAt": None, "from": None, + "detail": "no sync has been recorded", "errors": [], "errorsTotal": 0, + "databases": [], "retryable": False, "retryHint": meta["retryHint"]}) + answer = dict(row) + answer.update({"provider": connector_key, "connectionId": slot, + "label": meta["label"], "recorded": True, + "stateAt": row.get("at"), "retryHint": meta["retryHint"]}) + return _safe_connector_status(connector_key, answer) + + +def _odoo_result_database_failures(rt, connection_id, result): + """Map provider-reported model/table failures to their durable database keys.""" + import odoo_relational as rel + try: + choices = rel.grid_choices(rt, connection_id) + except Exception: # noqa: BLE001 + choices = [] + by_model = {str(row.get("model") or ""): str(row.get("key") or "") + for row in choices if row.get("model") and row.get("key")} + failures = {str(key): _clean_odoo_failure(value, "materialization") + for key, value in (result.get("databaseErrors") or {}).items()} + receipt_groups = [] + for name in ("databases", "canonicalDatabases"): + group = result.get(name) if isinstance(result, dict) else None + if isinstance(group, dict): + receipt_groups.append(group) + for group in receipt_groups: + for raw_key, receipt in group.items(): + if not isinstance(receipt, dict) or receipt.get("state") == "succeeded": + continue + key = str(raw_key or "") + if not key or key in failures: + continue + raw_failure = receipt.get("error") or receipt.get("sourceError") or { + "stage": "reconciliation", "code": "reconciliation_incomplete", + "type": "ReconciliationError", + "message": "The Odoo database reconciliation is incomplete.", + } + failures[key] = _clean_odoo_failure(raw_failure, "reconciliation") + unmatched = [] + discovered = result.get("discovered") if isinstance(result, dict) else {} + for raw in ((discovered or {}).get("errors") or ()): + model = str(raw).split(":", 1)[0].strip() + key = by_model.get(model) + if key: + failures[key] = _odoo_failure(raw, "materialization") + else: + unmatched.append(str(raw)) + limits = result.get("limits") if isinstance(result, dict) else None + if isinstance(limits, dict): + for raw_key, raw in limits.items(): + key = by_model.get(str(raw_key or ""), str(raw_key or "")) + if key: + failures[key] = _odoo_failure(raw, "materialization") + elif limits: + unmatched.append(str(limits)) + return failures, unmatched + + +def _odoo_sync_runner(session, connection_id): + """Run Odoo through C6 without moving provider imports into the data layer.""" + cid = _odoo_connection_slot(session.runtime, connection_id) + connector_sync.record(session.runtime, "odoo", cid, "syncing") + record_odoo_stage(session.runtime, cid, "source", "running") + record_odoo_stage(session.runtime, cid, "materialization", "queued") + import odoo_relational + def _materializing(): + connector_sync.record(session.runtime, "odoo", cid, "materializing") + record_odoo_stage(session.runtime, cid, "materialization", "running") + try: + result = odoo_relational.refresh(session.runtime, session.tenant, + username=session.uname or "automation", + connection_id=connection_id, + on_apply=_materializing) + except Exception as exc: + failure = _odoo_failure(exc, "source") + record_odoo_stage(session.runtime, cid, failure["stage"], "failed", failure=failure) + record_odoo_stage(session.runtime, cid, "freshness", "failed", + failure={**failure, "stage": "freshness"}) + connector_sync.record(session.runtime, "odoo", cid, "error", + errors=[f"{failure['code']}: {failure['message']}"]) + raise + tables = result.get("tables") if isinstance(result, dict) else {} + published = dict(tables or {}) + database_receipts = result.get("databases") if isinstance(result, dict) else {} + database_receipts = database_receipts if isinstance(database_receipts, dict) else {} + canonical_receipts = result.get("canonicalDatabases") if isinstance(result, dict) else {} + canonical_receipts = canonical_receipts if isinstance(canonical_receipts, dict) else {} + reconciliation_receipts = {**database_receipts, **canonical_receipts} + databases = sorted(str(key) for key in published) + sync_errors = [] + if isinstance(result, dict): + if result.get("failed"): + sync_errors.append(str(result["failed"])) + discovered = result.get("discovered") + if isinstance(discovered, dict): + sync_errors.extend(str(value) for value in (discovered.get("errors") or ())) + discovered_tables = discovered.get("tables") + if isinstance(discovered_tables, dict): + published.update(discovered_tables) + databases.extend(str(key) for key in discovered_tables) + if result.get("frozen"): + sync_errors.append(str(result.get("note") or "the connector is frozen")) + database_failures, unmatched_errors = _odoo_result_database_failures( + session.runtime, connection_id, result if isinstance(result, dict) else {}) + databases.extend(reconciliation_receipts) + databases.extend(database_failures) + sync_errors.extend(f"{key}: {failure['code']}" for key, failure in database_failures.items()) + sync_errors.extend(raw for raw in unmatched_errors if raw not in sync_errors) + state = ("partial" if published else "error") if sync_errors else "ready" + record_odoo_stage(session.runtime, cid, "source", "ready") + refreshed_at = datetime.now(timezone.utc).isoformat() + database_keys = list(dict.fromkeys([*reconciliation_receipts, *published])) + for key in database_keys: + if key in database_failures: + continue + counts = published.get(key) + receipt = reconciliation_receipts.get(key) + receipt = receipt if isinstance(receipt, dict) else None + nrows = (receipt.get("servingCount") if receipt is not None + else counts.get("rows") if isinstance(counts, dict) else None) + record_odoo_stage(session.runtime, cid, "source", "ready", database_key=key, + receipt=receipt) + record_odoo_stage(session.runtime, cid, "materialization", "ready", + database_key=key, row_count=nrows, receipt=receipt) + record_odoo_stage(session.runtime, cid, "reconciliation", "ready", + database_key=key, row_count=nrows, receipt=receipt) + record_odoo_stage(session.runtime, cid, "freshness", "ready", + database_key=key, row_count=nrows, refreshed_at=refreshed_at, + receipt=receipt) + for key, failure in database_failures.items(): + receipt = reconciliation_receipts.get(key) + receipt = receipt if isinstance(receipt, dict) else None + failed_stage = str(failure.get("stage") or "materialization") + failed_stage = failed_stage if failed_stage in _ODOO_HEALTH_STAGES else "materialization" + record_odoo_stage(session.runtime, cid, failed_stage, "failed", + database_key=key, failure=failure, receipt=receipt) + record_odoo_stage(session.runtime, cid, "freshness", "failed", database_key=key, + failure={**failure, "stage": "freshness"}, receipt=receipt) + if sync_errors: + failure = _odoo_failure("provider result contained errors", "materialization") + record_odoo_stage(session.runtime, cid, "materialization", "failed", failure=failure) + record_odoo_stage(session.runtime, cid, "freshness", "failed", + failure={**failure, "stage": "freshness"}) + else: + record_odoo_stage(session.runtime, cid, "materialization", "ready") + record_odoo_stage(session.runtime, cid, "reconciliation", "ready") + record_odoo_stage(session.runtime, cid, "freshness", "ready") + return connector_sync.record( + session.runtime, "odoo", cid, state, + detail=str(result.get("note") or "") if isinstance(result, dict) else "", + errors=sync_errors, databases=sorted(set(databases)), + ) + + +# Provider runners are an API-layer capability. The data-layer registration proves Amazon uses +# the same contract while this table honestly leaves it without an executable sync. +_SYNC_RUNNERS = {"odoo": _odoo_sync_runner} + + +ODOO_DISCONNECTED_CAUSE = ( + "Odoo is disconnected, so rows cannot load. " + "Reconnect Odoo in the Keychain and this resumes on its own.") + + +class _SupersededMaterialization(Exception): + pass + + +def _invalidate_materialization(rt, key): + """Fence a publisher in the same atomic document that holds the records.""" + import core.user_tables as ut + def apply(current): + doc = dict(current or {}) + if key in doc: + doc[key] = {**doc[key], "_connectorJob": None} + return doc + rt.update(ut.STORE_KEY, apply, flush="sync") + + +def _queue_odoo_database(rt, key, *, model="", retry=False, changed=False): + import odoo_relational as rel + cid = rel.connection_for_table_key(key, rt) + revision = connector_sync.configuration_revision( + "odoo", cid, key, model, rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}) + previous = (rt.get(connector_sync.JOBS_KEY, fresh=True) or {}).get( + connector_sync.job_key("odoo", cid, key)) or {} + if previous.get("state") == "syncing" and (changed or previous.get("revision") != revision): + _invalidate_materialization(rt, key) + job = connector_sync.enqueue( + rt, "odoo", cid, key, model=model, retry=retry, changed=changed, revision=revision) + # The runtime service is started by ASGI lifespan, never by a handler or a fixture import. + _materialization_wake.set() + return job + + +class _MaterializationRuntime: + """Delegate tenant reads; fence each row publication inside the store's atomic updater.""" + def __init__(self, rt, job): + self.rt, self.job = rt, job + + def __getattr__(self, name): + return getattr(self.rt, name) + + def valid(self): + import odoo_relational as rel + key = self.job["databaseKey"] + if not connector_sync.owns(self.rt, self.job): + raise _SupersededMaterialization() + cfg = self.rt.get(rel.CONFIG_KEY, fresh=True) or {} + if cfg.get("frozen") or (cfg.get("grids") or {}).get(key) is False: + raise _SupersededMaterialization() + # Save commits field choices before the queue update. The lease alone cannot + # detect that newer configuration if enqueue fails or has not run yet. + revision = self.job.get("revision") + if revision is not None and revision != connector_sync.configuration_revision( + self.job["provider"], self.job["connectionId"], key, self.job["model"], + self.rt.get(_ODOO_SCHEMA_ENABLEMENTS, fresh=True) or {}): + raise _SupersededMaterialization() + + def update(self, name, fn, flush="sync", **kwargs): + import core.user_tables as ut + key = self.job["databaseKey"] + marker = [self.job["id"], self.job["leaseToken"]] + def apply(current): + self.valid() + missing = name == ut.STORE_KEY and key not in current + if (name == ut.STORE_KEY and not missing + and (current.get(key) or {}).get("_connectorJob") != marker): + raise _SupersededMaterialization() + answer = fn(current) + if missing and key in answer: + # A reconciled receipt may precede its definition. Its first schema write + # installs the marker under the same lease and configuration fence. + answer[key]["_connectorJob"] = marker + # Lease expiry during a long merge cannot acknowledge a stale write either. + self.valid() + return answer + return self.rt.update(name, apply, flush=flush, **kwargs) + + +def _odoo_read_access_refused(exc): + """Match only XML-RPC faults with explicit access/read refusal markers.""" + if not isinstance(exc, xmlrpc.client.Fault): + return False + message = str(exc).casefold() + return "security restrictions" in message or "operation: read" in message + + +def _run_database_job(rt, candidate): + """Run one persisted receipt. Repeating a completed publication reconciles the same rows.""" + import core.user_tables as ut + import odoo_relational as rel + # A disconnected source is paused work, not a failed attempt. Preserve the receipt so + # reconnect can resume it; the publication fence still rechecks a mid-run disconnect. + if (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"): + connector_sync.stall(rt, candidate, ODOO_DISCONNECTED_CAUSE) + return + job = connector_sync.claim(rt, candidate) + if job is None: + return + fenced = _MaterializationRuntime(rt, job) + key = job["databaseKey"] + cid = job["connectionId"] + record_odoo_stage(rt, cid, "source", "running", database_key=key) + record_odoo_stage(rt, cid, "materialization", "running", database_key=key) + def finish(success=False, cause=""): + # The pre-claim check cannot cover a disconnect during source I/O or publication. + # Only the current lease may requeue; disable and a newer Save still win. + interrupted = bool((rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen")) + if success and not interrupted: + # A newer field Save can persist after rows commit but before this acknowledgement. + # The old rows remain valid history; they cannot establish the new settings ready. + fenced.valid() + # Only provider-authored sentences reach this seam, never raw exception messages. + cause = str(cause or "").replace("\u2014", ",").replace("\u2013", "-") + result = connector_sync.finish( + rt, job, success=success, interrupted=interrupted, cause=cause) + if interrupted: + connector_sync.stall(rt, job, ODOO_DISCONNECTED_CAUSE) + return result + + stopped = threading.Event() + def heartbeat(): + while not stopped.wait(connector_sync.LEASE_SECONDS / 3): + try: + connector_sync.finish(rt, job, renew=True) + except Exception: + return # An unrenewed lease cannot publish. + heartbeat_thread = None + try: + def install(current): + fenced.valid() + if key in (current or {}): + current[key]["_connectorJob"] = [job["id"], job["leaseToken"]] + return current + rt.update(ut.STORE_KEY, install, flush="sync") + heartbeat_thread = threading.Thread(target=heartbeat, daemon=True, + name="connector-lease") + heartbeat_thread.start() + cid = job["connectionId"] + source_id = None if cid == "default" else cid + if key not in (rt.get(ut.STORE_KEY, fresh=True) or {}): + schema = rel.publish_database_schema(fenced, key, connection_id=source_id, + username="automation") + if not job["model"]: + # Reconciliation cannot infer a missing definition's model. Now the provider + # has resolved it, queue its real field fingerprint before reading any rows. + fenced.valid() + _queue_odoo_database(rt, key, model=schema["identity"]["model"], changed=True) + return + if rel.base_table_key(key) in rel.ODOO_TABLE_MODELS: + result = rel.refresh( + fenced, getattr(rt, "tenant", None), username="automation", + connection_id=source_id, table_keys={key}) + else: + # A source-only model must not depend on an unrelated DuckDB mirror being warm. + _cid, namespace = rel.connection_namespace(rt, source_id) + result = {"discovered": rel.sync_discovered_models( + fenced, source_connection_id=source_id, namespace=namespace, + username="automation", table_keys={key}, + on_progress=lambda rows, total: connector_sync.progress( + fenced, job, rows=rows, total=total))} + discovered = result.get("discovered") or {} + published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})} + receipt = (result.get("databases") or {}).get(key) + receipt = receipt if isinstance(receipt, dict) else None + database_errors = result.get("databaseErrors") or {} + success = (key in published and not result.get("failed") and not result.get("frozen") + and not result.get("limits") and not discovered.get("errors") + and key not in database_errors + and (receipt is None or receipt.get("state") == "succeeded")) + counts = published.get(key) if isinstance(published.get(key), dict) else {} + nrows = (receipt.get("servingCount") if receipt is not None + else counts.get("rows") if isinstance(counts, dict) else None) + if success: + refreshed_at = datetime.now(timezone.utc).isoformat() + record_odoo_stage(rt, cid, "source", "ready", database_key=key, + receipt=receipt) + record_odoo_stage(rt, cid, "materialization", "ready", + database_key=key, row_count=nrows, receipt=receipt) + record_odoo_stage(rt, cid, "reconciliation", "ready", + database_key=key, row_count=nrows, receipt=receipt) + record_odoo_stage(rt, cid, "freshness", "ready", database_key=key, + row_count=nrows, refreshed_at=refreshed_at, receipt=receipt) + finish(success=True) + else: + raw_failure = ((database_errors.get(key) if isinstance(database_errors, dict) else None) + or (receipt or {}).get("error") or (receipt or {}).get("sourceError")) + failure = (_clean_odoo_failure(raw_failure, "reconciliation") if raw_failure + else _odoo_failure("provider result was incomplete", "materialization")) + failed_stage = str(failure.get("stage") or "materialization") + failed_stage = (failed_stage if failed_stage in _ODOO_HEALTH_STAGES + else "materialization") + record_odoo_stage(rt, cid, failed_stage, "failed", + database_key=key, failure=failure, receipt=receipt) + record_odoo_stage(rt, cid, "freshness", "failed", database_key=key, + failure={**failure, "stage": "freshness"}, receipt=receipt) + finish(cause=f"{failure['code']}: {failure['message']}") + except _SupersededMaterialization: + # Disable/new Save already invalidated the table marker; never restore it here. + finish() + except rel.Refused as exc: + failure = _odoo_failure(exc, "materialization") + record_odoo_stage(rt, cid, failure["stage"], "failed", + database_key=key, failure=failure) + record_odoo_stage(rt, cid, "freshness", "failed", database_key=key, + failure={**failure, "stage": "freshness"}) + finish(cause=f"{failure['code']}: {failure['message']}") + except Exception as exc: + # Raw provider exceptions may carry credentials or source values. Store only a named, + # data-free classification, including the fail-closed TLS case. + failure = _odoo_failure(exc, "source") + if _odoo_read_access_refused(exc): + failure = {"stage": "source", "code": "source_read_refused", + "type": type(exc).__name__, + "message": (f"Odoo refused read access to {job['model']}. Give the " + "connected account read access, or switch this database off.")} + record_odoo_stage(rt, cid, failure["stage"], "failed", + database_key=key, failure=failure) + record_odoo_stage(rt, cid, "freshness", "failed", database_key=key, + failure={**failure, "stage": "freshness"}) + finish(cause=f"{failure['code']}: {failure['message']}") + finally: + stopped.set() + if heartbeat_thread: + heartbeat_thread.join(timeout=1) + + +def _reconcile_database_jobs(rt): + """Repair enabled Odoo grids with no receipt using only tenant-local definitions.""" + import core.user_tables as ut + import odoo_relational as rel + grids = (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("grids") or {} + receipts = {job["databaseKey"] for job in connector_sync.jobs(rt, "odoo")} + missing = [key for key, enabled in grids.items() if enabled is True and key not in receipts] + if not missing: + return + definitions = rt.get_projection(ut.STORE_KEY, drop=("rows",)) or {} + for key in missing: + try: + model = str(((definitions.get(key) or {}).get("connector") or {}).get("model") or "") + # The queue's identity validator also decides whether an unknown model is legal. + _queue_odoo_database(rt, key, model=model) + except Exception: + continue # One unresolvable definition cannot starve other queued work. + + +def _resume_database_jobs(rt): + """One serial pass, also usable by isolated synthetic fixtures.""" + try: + _reconcile_database_jobs(rt) + except Exception: + pass # A reconciliation failure must still fall through to the receipt scan. + for candidate in connector_sync.jobs(rt): + if candidate.get("provider") == "odoo" and ( + candidate.get("state") == "queued" or + candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()): + try: + _run_database_job(rt, candidate) + except Exception: + # A failed claim/status write leaves durable work for a later scan. It must + # not starve sibling databases; never log raw store/provider exceptions. + continue + + +_materialization_wake = threading.Event() +_materialization_stop = threading.Event() +_materialization_thread = None +_materialization_lifecycle = threading.Lock() +# 3 is a starting point, not tuned: each worker holds an Odoo read and a growing store document in a small Space container. +MATERIALIZATION_WORKERS = 3 +_materialization_pool = None +_materialization_inflight = set() +_materialization_inflight_lock = threading.Lock() + + +def _release_database_job(key): + # Completion and cancellation both release the reservation, including failed claims. + with _materialization_inflight_lock: + _materialization_inflight.discard(key) + + +def _dispatch_database_jobs(rt): + """Submit a resident scan without waiting for source reads or row publication.""" + global _materialization_pool + try: + _reconcile_database_jobs(rt) + except Exception: + pass # Match the serial fixture pass even when reconciliation is unavailable. + for candidate in connector_sync.jobs(rt): + if candidate.get("provider") == "odoo" and ( + candidate.get("state") == "queued" or + candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()): + key = (rt.key, candidate["provider"], candidate["connectionId"], candidate["databaseKey"]) + with _materialization_inflight_lock: + if key in _materialization_inflight: + continue + _materialization_inflight.add(key) + try: + with _materialization_lifecycle: + if _materialization_stop.is_set(): + _release_database_job(key) + return + if _materialization_pool is None: + _materialization_pool = ThreadPoolExecutor( + max_workers=MATERIALIZATION_WORKERS, + thread_name_prefix="connector-materialization-job") + pool = _materialization_pool + future = pool.submit(_run_database_job, rt, candidate) + # Register outside both locks: an already finished future calls back inline. + future.add_done_callback(lambda _future, key=key: _release_database_job(key)) + except Exception: + _release_database_job(key) + # A concurrent shutdown or failed submit leaves the receipt for the next pass. + continue + + +@router.on_event("startup") +def _start_database_jobs(): + """Discover durable work at boot, including tenants that have no active browser session.""" + global _materialization_thread + def run(): + global _materialization_thread + from harness import runtime + while True: + with _materialization_lifecycle: + if _materialization_stop.is_set(): + # Retire under the startup lock, before this thread exits. A new lifespan + # either revives this worker or starts its replacement, never a dying one. + _materialization_thread = None + return + _materialization_wake.clear() + try: + tenants = runtime.known_tenants() + for tenant in tenants: + if _materialization_stop.is_set(): + break + try: + _dispatch_database_jobs(runtime.get_runtime(tenant)) + except Exception: + continue # Durable queued/leased work is retried by the next scan. + except Exception: + pass + _materialization_wake.wait(15) + with _materialization_lifecycle: + _materialization_stop.clear() + _materialization_wake.set() + if _materialization_thread and _materialization_thread.is_alive(): + return + _materialization_thread = threading.Thread(target=run, daemon=True, + name="connector-materialization") + _materialization_thread.start() + + +@router.on_event("shutdown") +def _stop_database_jobs(): + global _materialization_pool + with _materialization_lifecycle: + _materialization_stop.set() + _materialization_wake.set() + pool, _materialization_pool = _materialization_pool, None + if pool is not None: + # Running reads retain their reservation until done; lease expiry permits recovery. + pool.shutdown(wait=False, cancel_futures=True) + + +def _sync_refusals(): + """The provider refusal types whose OWN sentence is the answer, as a tuple for `except`. + + ⚠ Resolved lazily and defensively: `odoo_relational` is imported inside functions throughout + this module to avoid an import cycle, and a refusal handler that could itself raise on import + would turn every failed sync into a different failure. + """ + try: + import odoo_relational # noqa: PLC0415 + return (odoo_relational.Refused,) + except Exception: # noqa: BLE001 + return () + + +def _schema_catalog(session): + """Rows-free Odoo field catalogue; source fields remain discoverable when every value is empty.""" + return _schema_catalog_rt(session.runtime) + + +def _odoo_raw_models(rt, snapshot=None, connection_id=None): + """Raw fields from stamped discovery, or the legacy snapshot before discovery exists.""" + if snapshot is None: + import core.user_tables as user_tables + snapshot = user_tables.odoo_schema_snapshot(rt) + legacy = dict((snapshot or {}).get("models") or {}) if connection_id is None else {} + try: + import odoo_relational + discovered = odoo_relational.readable_model_catalog(rt, connection_id) + except Exception: # noqa: BLE001 + discovered = {} + if not discovered.get("capturedAt"): + return legacy + # A stamp means this tenant-bound account has spoken. Even a partial stamped set is safer + # than adding snapshot-only models that the current account never proved it may read. + return {str(item.get("model") or ""): dict(item.get("fields") or {}) + for item in (discovered.get("models") or ()) + if str(item.get("model") or "")} + + +def _schema_catalog_rt(rt, database_key=None, connection_id=None): + """The same catalogue, over a RUNTIME rather than a session. + + ⚠ SPLIT IN W49-T12 SO THE DETAIL ROUTE CAN REUSE IT RATHER THAN RE-DERIVE IT. The connector + modal's Databases half needs exactly this catalogue, and writing a second reader of + `odoo_schema_snapshot` here would be the two-answers-to-one-question shape this module has + already paid for twice ([[one-question-two-normalizers]]). `_schema_catalog` keeps its + signature so every existing caller is untouched. + """ + import core.user_tables as user_tables + enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} + snapshot = user_tables.odoo_schema_snapshot(rt) + raw_models = _odoo_raw_models(rt, snapshot, connection_id) + if raw_models: + rows = [] + for model, raw_fields in raw_models.items(): + for raw_name, meta in (raw_fields or {}).items(): + legacy_key = f"{model}.{raw_name}" + key = (_odoo_field_key(rt, database_key, model, raw_name) + if database_key else legacy_key) + rows.append({"key": key, "model": model, "rawName": raw_name, + "label": str(meta.get("string") or raw_name), + "type": str(meta.get("type") or "text"), "source": "Odoo", + "emptyDiscoverable": True, "defaultEnabled": False, + # Legacy choices seed default-connection databases until each has + # its own explicit override. Scoped writes never change this seed. + "enabled": enabled.get(key, enabled.get(legacy_key) + if connection_id is None else False) is True}) + return sorted(rows, key=lambda row: row["label"].casefold()) + # Never present compiled mirror declarations as an authoritative fields_get catalogue. + # Until a sync captures metadata, the schema is unavailable and enablement stays refused. + return [] + + +def _odoo_field_key(rt, database_key, model, raw_name, connection_id=...): + # An explicitly resolved None must not trigger another lookup. + if connection_id is ...: + import odoo_relational + connection_id = odoo_relational.connection_for_table_key(database_key, rt) + return connector_sync.field_key( + "odoo", connection_id, database_key, model, raw_name) + + +@router.get("/connectors/odoo/status") +def odoo_status(connection_id: str = None, session: Session = Depends(require_session)): + _odoo_admin(session) + import odoo_relational + resolved_connection = _odoo_connection_slot(session.runtime, connection_id) + cfg = odoo_relational.read_config(session.runtime) + stored = session.runtime.get(_ODOO_SYNC_STATUS) or {} + errors = stored.get("errors") if isinstance(stored.get("errors"), list) else [] + phase = str(stored.get("phase") or "unknown") + lifecycle = connector_sync.read(session.runtime, "odoo", resolved_connection) + durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id)) + receipt = odoo_health_receipt(session.runtime, resolved_connection) + if durable: + return {"schedule": cfg.get("syncEvery"), + "health": "paused" if cfg.get("frozen") else durable["state"], + "lastRun": durable["at"], "frozen": cfg.get("frozen") is True, + **durable, "stateAt": durable["at"], "connectionId": resolved_connection, + "receipt": receipt} + # The old keys stay the old default-connection answer. A caller that names a connection gets + # that slot's lifecycle instead of the tenant-wide legacy document that caused the C6 defect. + named = connection_id is not None + shown_errors = (list(lifecycle.get("errors") or ()) if named and lifecycle else + [] if named else errors) + shown_phase = ((lifecycle or {}).get("state") or "unknown") if named else phase + shown_last_run = ((lifecycle or {}).get("at") if named else stored.get("lastRun")) + state = ((lifecycle or {}).get("state") + or {"syncing": "syncing", "ok": "ready", "error": "error"}.get(shown_phase) + or "queued") + errors_total = ((lifecycle or {}).get("errorsTotal", 0) if named or lifecycle + else len(errors)) + return {"schedule": cfg.get("syncEvery"), + "health": "paused" if cfg.get("frozen") else + shown_phase if named else "error" if shown_errors else shown_phase, + "lastRun": shown_last_run, "errors": shown_errors[:20], + "frozen": cfg.get("frozen") is True, + "state": state, + "stateAt": (lifecycle or {}).get("at") or shown_last_run, + "databases": list((lifecycle or {}).get("databases") or ()), + "retryable": state in ("error", "partial"), + "errorsTotal": int(errors_total or 0), + "connectionId": (lifecycle or {}).get("connectionId") + or resolved_connection, + "receipt": receipt} + + +@router.get("/connectors/{connector_key}/sync") +def connector_sync_status(connector_key: str, connection_id: str = None, + session: Session = Depends(require_session)): + registered = connector_sync.providers() + if connector_key not in registered: + raise err(404, "sync_provider_unknown", + f"{connector_key} is not registered for connector sync") + row = (connector_sync.job_summary(_connection_jobs(session.runtime, connector_key, connection_id)) + or connector_sync.read(session.runtime, connector_key, connection_id)) + return _sync_payload(connector_key, connection_id, row) + + +@router.post("/connectors/{connector_key}/sync/now") +def connector_sync_now(connector_key: str, connection_id: str = None, + session: Session = Depends(require_session)): + _sync_admin(session) + if connector_key not in connector_sync.providers(): + raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") + try: + requested_connection = (_odoo_connection_slot(session.runtime, connection_id) + if connector_key == "odoo" else + str(connection_id or "default").strip() or "default") + key = (connector_key, session.runtime.key) + with _SYNC_REQUEST_CONNECTION_LOCK: + # Bind under one API-side lock so a duplicate request cannot race ahead of the + # accepted request and attach its different connection id to the queued intent. + receipt = connector_sync.request_sync(connector_key, session.runtime.key, + requested_by=session.uname) + if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS: + _SYNC_REQUEST_CONNECTIONS[key] = requested_connection + receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key] + return receipt + except ValueError: + raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None + + +@router.get("/connectors/{connector_key}/sync/now") +def connector_sync_now_status(connector_key: str, session: Session = Depends(require_session)): + _sync_admin(session) + if connector_key not in connector_sync.providers(): + raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") + receipt = connector_sync.sync_request(connector_key, session.runtime.key) + if receipt is not None: + receipt["connectionId"] = sync_request_connection(connector_key, session.runtime.key) + return receipt + return { + "provider": connector_key, "tenant": session.runtime.key, "state": "idle", + "requestedAt": None, "startedAt": None, "finishedAt": None, + "requestedBy": "", "detail": "", "accepted": False, + } + + +@router.post("/connectors/{connector_key}/sync/retry") +def connector_sync_retry(connector_key: str, connection_id: str = None, + session: Session = Depends(require_session), database_key: str = None): + _sync_admin(session) + if connection_id == "default": + connection_id = None + registered = connector_sync.providers() + if connector_key not in registered: + raise err(404, "sync_provider_unknown", + f"{connector_key} is not registered for connector sync") + if connector_key != "odoo": + raise err(501, "sync_runner_unavailable", registered[connector_key]["retryHint"]) + import odoo_relational as rel + if database_key: + key, _ns, choice = _odoo_database_target(session.runtime, database_key, connection_id) + choices = [choice] + else: + choices = rel.grid_choices(session.runtime, connection_id) + receipts = [] + for choice in choices: + if not choice.get("enabled"): + continue + key = choice["key"] + existing = connector_sync.jobs(session.runtime, connector_key, + rel.connection_for_table_key(key, session.runtime)) + previous = next((j for j in existing if j["databaseKey"] == key), {}) + if previous.get("state") == "failed" and not previous.get("retryable"): + if database_key: + raise err(409, "sync_attempts_exhausted", previous["error"]) + continue + receipts.append(_queue_odoo_database(session.runtime, key, model=choice.get("model", ""), + retry=True)) + if not receipts: + raise err(409, "sync_not_retryable", "No enabled database can be retried. Check its settings.") + return {"provider": connector_key, "jobs": receipts, **connector_sync.job_summary(receipts)} + + +@router.get("/connectors/{connector_key}/jobs") +def connector_database_jobs(connector_key: str, session: Session = Depends(require_session), + connection_id: str = None): + _sync_admin(session) + if connector_key not in connector_sync.providers(): + raise err(404, "sync_provider_unknown", "This connector has no database sync runner.") + payload = {"provider": connector_key, "jobs": connector_sync.jobs( + session.runtime, connector_key, connection_id)} + if connector_key == "odoo": + payload["receipt"] = odoo_health_receipt(session.runtime, connection_id) + return _safe_connector_status(connector_key, payload) + + +@router.get("/connectors/odoo/schema") +def odoo_schema(session: Session = Depends(require_session)): + _odoo_admin(session) + fields = _schema_catalog(session) + import core.user_tables as user_tables + snapshot = user_tables.odoo_schema_snapshot(session.runtime) + try: + import odoo_relational + discovered = odoo_relational.readable_model_catalog(session.runtime) + except Exception: # noqa: BLE001 + discovered = {} + has_discovery = bool(discovered.get("capturedAt")) + complete = (discovered.get("complete") is True if has_discovery + else snapshot.get("complete") is True) + return {"fields": fields, "default": "off", "adminOnly": True, + "complete": complete, + "capturedAt": (discovered.get("capturedAt") if has_discovery + else snapshot.get("capturedAt")), + "errors": list((discovered.get("errors") if has_discovery + else snapshot.get("errors")) or ()), + "source": "Odoo"} + + +@router.post("/connectors/odoo/census") +def odoo_population_census(session: Session = Depends(require_session), exact: bool = False): + """Count up to 101 ids per model by default; exact=true requests the full census.""" + _odoo_admin(session) + import odoo_relational + census = odoo_relational.refresh_population_census(session.runtime, force=True, exact=exact) + return {"capturedAt": str(census.get("capturedAt") or ""), + "mode": census["mode"], + "counts": census["counts"], + "complete": census.get("complete") is True, + "durationMs": int(census.get("durationMs") or 0), + "modelsCounted": int(census.get("modelsCounted") or 0), + "modelsWithRows": int(census.get("modelsWithRows") or 0), + "errors": [str(value).replace("\u2014", ",").replace("\u2013", "-") + for value in (census.get("errors") or ())][:50]} + + +def _materialisation_failure_reason(result): + """The materialiser's own reason, made safe for a sentence shown in the connector modal.""" + result = result if isinstance(result, dict) else {} + candidates = [] + if result.get("note"): + candidates.append(result.get("note")) + if result.get("failed"): + candidates.append(result.get("failed")) + errors = result.get("errors") + candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,)) + discovered = result.get("discovered") + if isinstance(discovered, dict): + errors = discovered.get("errors") + candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,)) + reason = str(next((value for value in candidates if str(value or "").strip()), "")).strip() + # Provider errors can carry typographic punctuation. Screen strings use plain punctuation. + # ⛔ Standing rule 2 names the EM DASH and the EN DASH, never the ASCII hyphen. A blanket + # strip of every Unicode dash category also breaks "read-only" and "e-commerce", so the + # two characters the rule names are rewritten and nothing else is touched. + reason = reason.replace("\u2014", ",").replace("\u2013", "-") + return reason[:1].upper() + reason[1:] if reason else "" + + +def _evict_odoo_product_pools(rt): + """Drop every raw-field-sensitive row pool after field enablement changes.""" + for cache_key in list(getattr(rt, "pool_cache", {}) or {}): + if isinstance(cache_key, tuple) and cache_key[:1] == ("product_pool",): + rt.pool_cache.pop(cache_key, None) + + +# W57-T31 / R10 / C9: switching off starts a seven-day lock to prevent repeated toggling. +# The config document is shared with read_config, which ignores unknown keys: disabledAt +# belongs beside grids, not in a second document that could drift from enablement. +_RELOCK_DAYS = 7 +_RELOCK_SECONDS = _RELOCK_DAYS * 86400 + + +def _relock_stamps(rt): + """Read disable epochs without letting an absent or malformed map break a request.""" + try: + import odoo_relational + stamps = (rt.get(odoo_relational.CONFIG_KEY) or {}).get("disabledAt", {}) + if not isinstance(stamps, dict): + return {} + return {str(key): int(value) for key, value in stamps.items() + if isinstance(value, (int, float)) and not isinstance(value, bool)} + except Exception: # noqa: BLE001 + return {} + + +def _relock_days_left(stamp, now=None): + """Round any remaining part of a lock day up to a whole day.""" + if isinstance(stamp, bool) or not isinstance(stamp, (int, float)): + return 0 + try: + from math import ceil + remaining = stamp + _RELOCK_SECONDS - (time.time() if now is None else now) + return max(0, ceil(remaining / 86400)) + except Exception: # noqa: BLE001 + return 0 + + +def _odoo_database_target(rt, table_key, connection_id=None): + """Resolve a database once; explicit connection/key disagreement fails closed.""" + import odoo_relational + raw = str(table_key or "").strip() + base, namespace = odoo_relational.split_table_key(raw) + requested_connection = connection_id if connection_id is not None else namespace + try: + _cid, ns = odoo_relational.connection_namespace(rt, requested_connection) + choices = odoo_relational.grid_choices(rt, requested_connection) + except odoo_relational.Refused as exc: + raise err(404, "unknown_odoo_connection", str(exc)) + key = odoo_relational.table_key_for(base or raw, ns) + if namespace is not None and namespace != ns: + raise err(409, "database_connection_mismatch", + "the database belongs to a different connection") + choice = next((row for row in choices if row["key"] == key), None) + if choice is None: + raise err(404, "unknown_odoo_database", + f"Odoo does not declare a database with key {raw!r}") + return key, ns, choice + + +def _enable_odoo_database(table_key, session, connection_id=None): + """Turn on one database and publish its schema; row materialization is separate work.""" + if not session.admin: + raise err(403, "not_admin", "only an administrator can enable an Odoo database") + + import odoo_relational # noqa: PLC0415 - provider owns both paths + key, namespace, choice = _odoo_database_target(session.runtime, table_key, connection_id) + + # R10 / C9: only a transition back on is locked; an already-enabled database may retry + # materialisation. A successful enable clears its stamp so the next switch-off starts fresh. + # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail. + if choice.get("enabled") is not True: + days = _relock_days_left(_relock_stamps(session.runtime).get(key)) + if days > 0: + label = str(choice.get("label") or key) + unit = "day" if days == 1 else "days" + raise err(409, "database_relocked", + f"{label} was switched off less than {_RELOCK_DAYS} days ago. " + f"It cannot be switched back on for another {days} {unit}. " + f"The {_RELOCK_DAYS} day lock is what stops a database being switched " + "on and off repeatedly.") + + # Publishing the definition makes Data and rename usable before any rows have loaded. + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur.setdefault("grids", {}).update({key: True}) + if isinstance(cur.get("disabledAt"), dict): + cur["disabledAt"].pop(key, None) + return cur + + failure = None + try: + result = odoo_relational.publish_database_schema( + session.runtime, key, connection_id=namespace, username=session.uname) + except odoo_relational.Refused as exc: + # The provider owns this sentence. In particular it names missing credentials and the + # MAX_TABLES refusal without this route guessing which one happened. + failure = (409, "odoo_database_refused", str(exc)) + except Exception: # noqa: BLE001 + failure = (500, "database_schema_failed", + "The setting was not changed and the database was not created because the " + "field catalogue could not be published. Refresh the catalogue and try again.") + + # A refused publication must leave both the flag and its relock stamp untouched. + if failure is None: + try: + session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync") + except Exception: # noqa: BLE001 + raise err(503, "store_unavailable", "the database setting was not saved. Try again.") + + facts = (_rename_rights(session.runtime, session.uname, bool(session.admin), [key]).get(key) + or {"exists": False, "canRename": False}) + exists = facts.get("exists") is True + can_rename = facts.get("canRename") is True + # ⛔⛔ THE ERROR ENVELOPE STAYS THE ONE SHAPE. `deps.err` is documented as X2's single + # non-2xx body, `{"error": {"code", "message"}}`, and an earlier draft of this door attached + # `key`, `enabled` and the fresh facts BESIDE that key so the client could read them off a + # failure. Two shapes for one thing is what this module's own comments call the defect it has + # paid for twice, and a caller that learns to read a sibling off an error will keep doing it. + # The client re-reads `/detail` on both paths; publication precedes the flag write and row + # materialisation remains separate work. + if failure is not None: + raise err(*failure) + model = str(choice.get("model") or "").strip() + if not exists: + reason = _materialisation_failure_reason(result) + message = "The setting was enabled, but the database was not created." + if reason: + message = f"{message} {reason}" + else: + message = f"{message} The field catalogue could not be published." + try: + _queue_odoo_database(session.runtime, key, model=model) + finally: + # Preserve the original refusal envelope even if the queue store is unavailable. + raise err(409, "database_not_created", message) + + # Every permission-visible field is available from schema alone. Existing per-database + # choices survive a retry; population counts are optional background observations. + try: + schema_keys = {_odoo_field_key(session.runtime, key, model, name): True + for name in result["sourceFields"]} + session.runtime.update( + _ODOO_SCHEMA_ENABLEMENTS, + lambda current: { + **schema_keys, + **(current if isinstance(current, dict) else {}), + }, + flush="sync") + except Exception: # noqa: BLE001 + try: + _queue_odoo_database(session.runtime, key, model=model) + finally: + raise err(503, "field_auto_enable_failed", + "The database was created, but its field choices could not be saved. " + "Try again.") + _evict_odoo_product_pools(session.runtime) + + # W56-T36: there is deliberately no navigation projection eviction here. Data's database + # list comes from + # `GET /nav` -> `core.user_tables.all_defs()` -> `TenantRuntime.get_projection(user_tables)`. + # Schema publication synchronously updates that same `user_tables` document; the HF + # backend replaces its whole-document cache as part of that write, while Postgres performs a + # fresh SELECT for every projection. `pool_cache` contains row pools for built-in databases, + # not the `ut_*` definition list, and compiled registry rows never contain tenant databases. + # The table now exists, so clients must discard navigation-derived database lists. This is a + # wire instruction rather than a guessed client row: `/nav` remains the permission-filtered + # authority for which databases the current reader may see. + try: + job = _queue_odoo_database(session.runtime, key, model=model) + except Exception: + raise err(503, "sync_queue_unavailable", + "The database settings were saved, but row sync could not be queued. Save again.") + return {"key": key, "enabled": True, "exists": exists, "canRename": can_rename, + "identity": connector_sync.database_identity( + "odoo", odoo_relational.connection_for_table_key(key, session.runtime), key, model), + "workspaceChanged": True, "state": job["state"], "job": job, + "schemaState": result["state"], + "schemaFields": result["fieldCount"], "populatedFieldsEnabled": 0, + "fieldCensusComplete": False, "fieldCensusErrors": [], + "autoEnableAlsoAffects": []} + + +@router.post("/connectors/odoo/databases/{table_key}/enable") +def odoo_database_enable(table_key: str, session: Session = Depends(require_session), + connection_id: str = None): + """Enable and create one declared Odoo database in this tenant's governed store.""" + return _enable_odoo_database(table_key, session, connection_id) + + +def _disable_odoo_database(table_key, session, connection_id=None): + """Turn off one declared Odoo database and persist its re-enable lock.""" + if not session.admin: + raise err(403, "not_admin", "only an administrator can disable an Odoo database") + + import odoo_relational + key, namespace, choice = _odoo_database_target(session.runtime, table_key, connection_id) + + # W57-T31 / R10 / C9: one config-only write starts the lock, with no sync or row deletion. + # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail. + # Repeating a disable must preserve the original deadline. + was_enabled = choice.get("enabled") is True + _invalidate_materialization(session.runtime, key) + connector_sync.cancel(session.runtime, "odoo", + odoo_relational.connection_for_table_key(key, session.runtime), key) + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + grids = cur.get("grids") + grids = grids if isinstance(grids, dict) else {} + grids[key] = False + cur["grids"] = grids + if was_enabled: + if not isinstance(cur.get("disabledAt"), dict): + cur["disabledAt"] = {} + cur["disabledAt"][key] = int(time.time()) + return cur + + try: + session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync") + except Exception: # noqa: BLE001 + raise err(503, "store_unavailable", "the database setting was not saved. Try again.") + + stamp = _relock_stamps(session.runtime).get(key) + return {"key": key, "enabled": False, "relockDays": _RELOCK_DAYS, + "identity": connector_sync.database_identity( + "odoo", odoo_relational.connection_for_table_key(key, session.runtime), + key, choice.get("model")), + "disabledAt": stamp, + "canEnableAt": stamp + _RELOCK_SECONDS if stamp is not None else None} + + +@router.post("/connectors/odoo/databases/{table_key}/disable") +def odoo_database_disable(table_key: str, session: Session = Depends(require_session), + connection_id: str = None): + """Disable one declared Odoo database in this tenant's governed store.""" + return _disable_odoo_database(table_key, session, connection_id) + + +@router.patch("/connectors/odoo/schema/{field_key}") +def odoo_schema_enable(field_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + _odoo_admin(session) + import core.user_tables as user_tables + import odoo_relational + request = body if isinstance(body, dict) else {} + key = str(field_key or "").strip() + scoped = connector_sync.parse_field_key(key) + database_key = request.get("databaseKey") + connection_id = request.get("connectionId") + model, raw_name = key.rsplit(".", 1) if "." in key else ("", "") + if scoped: + provider, connection_id, database_key, model, raw_name = scoped + if provider != "odoo": + raise err(404, "unknown_odoo_field", "that field belongs to another connector") + if connection_id == "default": + connection_id = None + if database_key is None: + candidates = [row for row in odoo_relational.grid_choices(session.runtime, connection_id) + if row.get("model") == model] + if len(candidates) > 1: + raise err(400, "database_required", "choose the database for this field") + if candidates: + database_key = candidates[0]["key"] + namespace = None + if database_key: + database_key, namespace, choice = _odoo_database_target( + session.runtime, database_key, connection_id) + if model != choice.get("model"): + raise err(404, "unknown_odoo_field", "that model does not belong to this database") + canonical = _odoo_field_key(session.runtime, database_key, model, raw_name) + if scoped and key != canonical: + raise err(409, "database_connection_mismatch", + "the field belongs to a different connection") + key = canonical + _snapshot = user_tables.odoo_schema_snapshot(session.runtime) + try: + import odoo_relational + _discovered_complete = odoo_relational.readable_model_catalog( + session.runtime, namespace).get("complete") is True + except Exception: # noqa: BLE001 + _discovered_complete = False + if (namespace is not None or _snapshot.get("complete") is not True) and not _discovered_complete: + raise err(409, "odoo_schema_unavailable", + "run a successful schema sync before changing raw-field enablement") + known = {field["key"] for field in _schema_catalog_rt( + session.runtime, database_key, namespace)} + if key not in known: + raise err(404, "unknown_odoo_field", "that field is not in the Odoo schema catalogue") + enabled = request.get("enabled") + if not isinstance(enabled, bool): + raise err(400, "bad_enabled", "enabled must be true or false") + previous_enabled = (session.runtime.get(_ODOO_SCHEMA_ENABLEMENTS) or {}).get(key) is True + session.runtime.update( + _ODOO_SCHEMA_ENABLEMENTS, + lambda current: {**(current if isinstance(current, dict) else {}), key: enabled}, + flush="sync") + _evict_odoo_product_pools(session.runtime) + job = None + if database_key and choice.get("enabled"): + try: + job = _queue_odoo_database(session.runtime, database_key, model=model, + changed=previous_enabled != enabled) + except Exception: + raise err(503, "sync_queue_unavailable", + "The field settings were saved, but row sync could not be queued. Save again.") + return {"key": key, "enabled": enabled, "source": "Odoo", + "permissionWall": "normal field permissions", "job": job} + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W49-T12 · RULING R26 · CONTRACT C4 — ONE CONNECTOR, IN FULL, FOR THE MODAL. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# R26: *"The connector popup is a centered MODAL with tabs Overview, Keychain, Databases, +# Relationships, opened from the card. The Keychain moves INTO it and Settings > Keychains becomes +# a link to the modal (one door). Field enablement moves into the Databases tab."* +# +# ⛔ ALL FOUR HALVES ARE COMPOSED NOW even though only Overview renders this wave, and that is +# C4's own instruction. The alternative — ship `overview` and add the rest as each tab lands — +# would make the DTO's shape a moving target across four tickets and give T13/T14/T15 nothing to +# build against. Every half is read from the source that already answers the question: +# +# overview · the directory row this module already composes, plus the Databases labels +# keychain · `routes_keychain.visible_entries`, i.e. the exact rows the Keychains room +# shows THIS reader, with a member's masked preview already blanked by R4 +# databases · `odoo_relational.grid_choices` (derived from `TABLES`, never a second list) +# plus the Odoo field catalogue this module already serves at +# `/connectors/odoo/schema` +# relationships · `relationship_graphs` UNCHANGED — the same DTO the directory already binds +# +# ⛔ AND NOTHING IS FABRICATED. A connector with no introspectable schema gets `databases: []`; a +# connector with no credential door gets an empty `entries` list and `canConnect: false`; a +# connection with no materialised tables gets the graph's own `unavailable` envelope. The rule this +# module was built on is that a plausible-looking placeholder is worse than an honest empty. + + +def _entry_type_for(connector_key): + """The keychain entry type behind one card, or `""` when the card has no stored credential. + + ⛔ DERIVED FROM `TOKEN_CONNECTORS`, NOT RE-TYPED. This module's header rule is that the row set + is composed from the registries that already exist; a second hand-written + `{"stripe": "stripe", ...}` map here would rot in exactly the way the header describes, one + connector at a time. Odoo is the one built-in whose credential lives in the keychain, so it is + named once, beside the row that names it. + """ + key = str(connector_key or "") + if key == "odoo": + return "odoo" + # ⛔⛔ FOUND ONLY ON THE MERGED TREE, AT THE WAVE-53 COMBINE, AND IT IS A 500 ON A LIVE ROUTE. + # `directory` answers 200 with a NAMED gap when any one contributor registry raises (W32-T14 / + # R6: "it used to 500 and show nothing"), and every contributor is called inside that guard. + # The `allowedScopes` pass added this wave runs AFTER the composition, OUTSIDE the guard, and + # this loop is a second unguarded read of `TOKEN_CONNECTORS` - so a broken token registry took + # the whole page back to a 500. Neither lane could see it: one owns this route, the other owns + # the gate that drives the broken registry, and only the merged tree holds both. + # ⚠ `""` is not a swallow, it is this function's own documented answer for a row with no + # keychain type, and `allowed_scopes("")` offers BOTH scopes - the permissive default the + # composer already states, chosen because the toggle must never hide a connection the reader + # actually holds. A token connector's rows are already absent from the set in this state. + try: + for token in TOKEN_CONNECTORS: + if str(token.get("key") or "") == key: + return str(token.get("entry") or "") + except Exception: # noqa: BLE001 - a registry, not a bug + return "" + return "" + + +#: ⭐⭐ W49-T13 — WHAT A CONNECTOR ASKS FOR WHEN YOU CONNECT IT, declared server-side. +#: +#: ⛔ THE CLIENT MUST NOT OWN THIS MAP. R26 moves the add-a-key form into the connector modal, and +#: the obvious client implementation is `type === "odoo" ? [url, db, user, api_key] : [api_key]` — +#: a second copy of a credential SHAPE that already lives in `harness/connectors/odoo.py` and in +#: `routes_keychain._env_odoo_fields`. The wave-9 law is about vocabularies and this is one: a +#: connector whose credential grows a field would take a client edit to become storable. +#: ⚠ `secret` decides the input type, so a field marked secret is typed into a password box and +#: never echoed back — the masked preview is the only form the stored value takes afterwards. +#: Sentence case with the initialism capitalised, which is what the rest of this app does. +_AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret", + "refresh_token": "Refresh token", "seller_id": "Seller ID", + "marketplace_id": "Marketplace ID"} +_CONNECT_FIELDS = { + "odoo": [{"key": "url", "label": "Server URL (https://...)", "secret": False}, + {"key": "db", "label": "Database", "secret": False}, + {"key": "user", "label": "API user", "secret": False}, + {"key": "api_key", "label": "API key", "secret": True}], + # ⭐ THE KEYS AND THEIR ORDER ARE DERIVED, NEVER TYPED. `amazon_client.CREDENTIAL_FIELDS` is + # the one roster, and lane G's `verify_amazon` asserts this sequence against it, so a field + # added there appears here rather than drifting. Labels are the one thing NOT derived: + # `"client_id".replace("_", " ").capitalize()` renders "Client id", and this app writes the + # initialism. The map carries a derived fallback so a new credential field still renders. + "amazon_sp": [ + {"key": key, + "label": _AMAZON_FIELD_LABELS.get(key, key.replace("_", " ").capitalize()), + "secret": key in ("client_secret", "refresh_token"), + **({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})} + for key in amazon_client.CREDENTIAL_FIELDS + ], +} +#: Everything else is one pasted token. The field name is `api_key` because that is the first of +#: `core.keychain._PREVIEW_FIELDS`, so the stored entry gets a preview rather than a blank cell — +#: the wave-31 defect that comment records, avoided rather than repeated. +_CONNECT_TOKEN_FIELD = [{"key": "api_key", "label": "API key", "secret": True}] + + +def _safe_connector_status(connector_key, payload): + """Apply the provider's disclosure guard at the status boundary, outside fallback catches.""" + if connector_key == "amazon_sp": + import connectors_amazon as cx + return cx.assert_safe_status(payload) + return payload + + +def _sync_stamp(value): + """Return a parseable public timestamp, never a creation-time freshness substitute.""" + text = str(value or "").strip() + if not text: + return None + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc).isoformat() + except (TypeError, ValueError, OverflowError): + return None + + +def _database_sync_health(*, enabled=True, state="never_run", last_success=None, + source_timestamp=None, materialized_timestamp=None, row_count=None, + error=None, now=None): + """Normalize one database's durable facts into S63-T34's shared public vocabulary.""" + clean_state = str(state or "never_run") + if clean_state not in DATABASE_SYNC_STATES: + clean_state = "failed" + error = {"code": "sync_state_unreadable", + "message": "This database's saved sync state could not be read."} + if not enabled: + clean_state = "disabled" + count = row_count if type(row_count) is int and row_count >= 0 else None + success = _sync_stamp(last_success) + source = _sync_stamp(source_timestamp) + materialized = _sync_stamp(materialized_timestamp) + safe_error = None + if error is not None: + raw = error if isinstance(error, dict) else {} + requested_code = str(raw.get("code") or "") + code = requested_code if requested_code in _DATABASE_SAFE_ERRORS else "connector_sync_failed" + safe_error = { + "code": code, + # Raw provider and exception text never crosses this boundary. + "message": _DATABASE_SAFE_ERRORS.get(code, "This database could not be synced."), + } + # A current failure may retain the prior success timestamp, but it can never look current. + lag = _lag_seconds(success, now) if success else None + return {"state": clean_state, "lastSuccess": success, + "sourceTimestamp": source, "materializedTimestamp": materialized, + "rowCount": count, "lagSeconds": lag, "error": safe_error} + + +def _odoo_database_sync_health(raw, enabled=True, *, now=None): + row = raw if isinstance(raw, dict) else {} + stages = row.get("stages") if isinstance(row.get("stages"), dict) else {} + failure = row.get("failure") if isinstance(row.get("failure"), dict) else None + raw_state = str(row.get("state") or "unknown") + stage_states = {str((value or {}).get("state") or "unknown") + for value in stages.values() if isinstance(value, dict)} + last_success = row.get("lastSuccess") + if failure or "failed" in stage_states or raw_state == "failed": + state = "partial" if last_success else "failed" + elif raw_state == "running" or "running" in stage_states: + state = "running" + elif raw_state == "queued" or "queued" in stage_states: + state = "queued" + elif last_success or raw_state == "ready": + state = "succeeded" + else: + state = "never_run" + source_stage = stages.get("source") if isinstance(stages.get("source"), dict) else {} + material_stage = (stages.get("materialization") + if isinstance(stages.get("materialization"), dict) else {}) + return _database_sync_health( + enabled=enabled, state=state, last_success=last_success, + source_timestamp=source_stage.get("lastSuccess"), + materialized_timestamp=material_stage.get("lastSuccess"), + row_count=row.get("rowCount"), error=failure, now=now) + + +def _amazon_database_sync_health(raw, enabled=True, *, now=None): + if not enabled: + return _database_sync_health(enabled=False) + if not isinstance(raw, dict): + return _database_sync_health( + state="failed", error={"code": "sync_status_unavailable", + "message": "Sync status is unavailable for this database."}) + success = raw.get("lastSuccess") + if raw.get("stuck") is True: + state = "partial" if success else "failed" + error = {"code": "sync_stalled", + "message": "This database sync is stalled."} + elif raw.get("inProgress") is True: + state, error = "running", None + elif success: + state, error = "succeeded", None + elif raw.get("deferred"): + state, error = "queued", None + else: + state, error = "never_run", None + return _database_sync_health( + state=state, last_success=success, materialized_timestamp=success, + row_count=raw.get("rowsLoaded"), error=error, now=now) + + +def _stored_database_sync_health(raw, enabled=True, *, now=None): + row = raw if isinstance(raw, dict) else {} + return _database_sync_health( + enabled=enabled, state=row.get("state") or "never_run", + last_success=row.get("lastSuccess"), source_timestamp=row.get("sourceTimestamp"), + materialized_timestamp=row.get("materializedTimestamp"), row_count=row.get("rowCount"), + error=row.get("error"), now=now) + + +def _detail_connect(entry_type, is_admin): + """C4's `keychain.connect` half: what the Connect form must ask for, and what it may store. + + ⭐ AN ADDITION TO C4, DECLARED. The contract pins `entries`, `canConnect` and `canDisconnect`; + it does not say where the form gets its fields, and the only two answers are "the server says" + and "the client re-derives". This is the first. + + `scope` carries the credential's own rule rather than a permission: a tenant-wide type has + exactly one legal scope, so the form states it instead of offering a picker whose save 400s + (`routes_keychain.add_key` refuses it, and R7's banned shape is a button that leads nowhere). + """ + if not entry_type: + return None + fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD) + tenant_wide = entry_type in TENANT_WIDE_TYPES + return {"type": entry_type, + "fields": [dict(field) for field in fields], + # A member may always store a personal one; business-wide is an administrator's to + # give (R4), and a tenant-wide type is business-wide by construction. + "scopes": (["business"] if tenant_wide + else ["business", "personal"] if is_admin + else ["personal"]), + "tenantWide": tenant_wide} + + +def _replacement_field_specs(entry_type): + """The write-only fields that may change on an existing credential. + + This is deliberately a projection of the Connect descriptor rather than another provider + vocabulary. URL, database, account and every future non-secret field stay server-held during + replacement. Optional rotation metadata is accepted separately from this descriptor. + """ + spec = _detail_connect(entry_type, True) or {} + return [dict(field) for field in (spec.get("fields") or []) + if field.get("secret") is True and str(field.get("key") or "").strip()] + + +def _oauth_slot_entries(rt, uname, row): + """A per-user OAuth connection, rendered as C4's `keychain` entry shape. + + ⛔⛔ THE KEYCHAIN TAB HAS A SECOND SOURCE, AND WITHOUT IT AN OAUTH CARD SAYS THE WRONG THING. + `_entry_type_for` covers `odoo` and `TOKEN_CONNECTORS` only, because an OAuth credential is + NOT a keychain entry: it lives in `core/keychain`'s separate `user_secrets` bucket, which + `list_entries` deliberately does not walk (*"a refresh token is identity, not shared + infrastructure"*). So an OAuth card asked for its entries got `[]` — i.e. *"nobody has + connected this"* about a provider this very reader may be signed in to. That is the same + confidently-wrong shape `accessSentence` was corrected for one tab over. + ⚠ AND IT IS NOT WIDENED INTO `_entry_type_for`. These are different stores with different + owners and different doors; teaching one function to answer for both would put a keychain + entry type on a credential that has none. + + ⛔ `testable` IS FALSE HERE, AND THAT IS THE HONEST VALUE RATHER THAN THE CAUTIOUS ONE. + `POST /admin/keychain/{id}/test` resolves its id through `visible_entries` -> `list_entries`, + which walks the `entries` bucket ONLY, so a Test control on a user secret would 404 every + time. A button that cannot work is exactly what R6 removed from the Google card. + + ⚠ NO SECRET IS READ. `oauth_connect.status` returns the account email and two flags; the + refresh token itself is never opened, and `masked` is empty because there is nothing about + this credential a preview could honestly show. + """ + slug = str(row.get("key") or "") + try: + state = (oauth_connect.status(rt, uname) or {}).get(slug) or {} + except Exception: # noqa: BLE001 + return [] + if not state.get("connected"): + return [] + email = str(state.get("email") or "") + label = str(row.get("label") or slug) + return [{"id": f"oauth:{slug}", + "label": f"{label} ({email})" if email else label, + "masked": "", + # A per-user slot is personal by construction, in the vocabulary `routes_keychain` + # already declares. Never a third word for the same idea. + "scope": "personal", + "testable": False, + "replaceable": False}] + + +def _detail_keychain(rt, uname, is_admin, row): + """C4's `keychain` half: the credential rows THIS reader may see, and what they may do. + + ⚠ `visible_entries` IS CALLED WITH THE READER'S OWN ADMIN FLAG, deliberately. R4 blanks the + masked preview on a business entry a member did not create, and passing `True` to get a fuller + answer would hand a member four characters of the workspace's Odoo key. The restricted default + is the correct one and it is passed through rather than routed around. + + ⚠ A KEYCHAIN THAT CANNOT BE READ COSTS THE ENTRY LIST AND NOTHING ELSE, and it is not silent: + `directory()` records that failure in `incomplete`, and the page this modal opens from renders + that banner ABOVE the card grid. The reader has already been told before they click. + """ + entry_type = _entry_type_for(row.get("key")) + manage = str(row.get("manage") or "") + # The door this reader was offered on the CARD, not a second opinion about it: `directory` + # already decided admin vs member vs nothing, and deciding again here is how two answers to + # one question get born ([[one-question-two-normalizers]]). + can_connect = manage in ("keychain", "keychain_personal") + can_replace = bool(is_admin and can_connect and _replacement_field_specs(entry_type)) + entries = [] + if entry_type: + try: + import routes_keychain as rk + for entry in rk.visible_entries(rt, uname, bool(is_admin)) or []: + if str(entry.get("type") or "") != entry_type: + continue + entries.append({ + "id": str(entry.get("id") or ""), + "label": str(entry.get("label") or entry.get("id") or ""), + "masked": str(entry.get("preview") or ""), + "scope": str(entry.get("scope") or rk.DEFAULT_SCOPE), + # `keychain.test_entry` answers for every stored entry: a registered prober for + # the type when the connector layer filled that slot, and a read-only shape or + # xmlrpc check otherwise. The flag says whether a Test control would have + # something to call, and it always does. + "testable": True, + # W55-T18 / C10: credential replacement is an administrator action. The + # client reads this wall instead of inferring it from scope or from whether + # a masked preview happened to be visible. The replacement endpoint changes + # this row's encrypted payload in place; no secret is added to this DTO. + "replaceable": can_replace, + **({AMAZON_ROTATION_DEADLINE_FIELD: + _stored_amazon_rotation_deadline(rt, entry.get("id"))} + if entry_type == "amazon_sp" else {}), + }) + except Exception: # noqa: BLE001 + entries = [] + # ⭐ THE SECOND SOURCE. An OAuth card has no keychain entry type at all, so `entries` above is + # empty for it by construction and this is the only reading that can fill the tab. + if not entry_type and manage == "oauth": + entries = _oauth_slot_entries(rt, uname, row) + # `startUrl` is the Connect door for this kind, and `disconnect` is a per-user route, so + # both answers come from the ROW rather than from a credential door this card has none of. + return {"entries": entries, + "canConnect": bool(row.get("startUrl")), + "canDisconnect": bool(entries), + "connect": None} + return {"entries": entries, + "canConnect": can_connect, + "canDisconnect": bool(can_connect and entries), + # What the Connect form asks for. `None` when this card has no credential door at all, + # which is what stops the client inventing one (Webhooks, the Scraper, a planned row). + "connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None} + + +def _odoo_catalog_by_model(rt, connection_id=None): + """`{odoo model: (fields, excluded)}` from the schema catalogue this module already serves. + + `excluded` is a DIFF, not a guess: every raw field the snapshot captured that the catalogue did + not offer. Today that diff is empty, which is the honest answer, and it fills itself the moment + the catalogue starts filtering instead of needing somebody to remember. + """ + raw_models = _odoo_raw_models(rt, connection_id=connection_id) + offered = {} + for row in _schema_catalog_rt(rt, connection_id=connection_id): + offered.setdefault(str(row.get("model") or ""), []).append(row) + out = {} + for model, raw_fields in raw_models.items(): + rows = offered.get(str(model), []) + offered_raw = {str(row.get("rawName") or "") for row in rows} + out[str(model)] = ( + [{"key": str(row.get("key") or ""), + "label": str(row.get("label") or ""), + "type": str(row.get("type") or "text"), + "source_field": str(row.get("rawName") or ""), + "enabled": bool(row.get("enabled"))} + for row in rows], + [{"source_field": str(name), + "reason": "This source field was captured from Odoo but the field catalogue did not " + "offer it."} + for name in sorted(raw_fields or {}) if str(name) not in offered_raw], + ) + return out + + +def _spec_field(item, name): + """One attribute off a C2 table spec, whether lane D ships dataclasses or dicts.""" + if isinstance(item, dict): + return item.get(name) + return getattr(item, name, None) + + +def _connector_spec_items(connector_key): + """C2's raw table specs for one connector, as a sequence of items, or `()`. + + ⭐⭐ W53-T30 — EXTRACTED FROM `_connector_spec_tables` (W53-T27) SO THERE IS STILL EXACTLY ONE + IMPORT PATH TO LANE D'S MODULE. Two consumers now read the spec: the Databases tab wants + `native` and `label`, the relationship graph wants `links`. A second `from core import + connector_spec` with its own lookup order is how the two would start disagreeing about which + attribute of D's module is the spec, and only one of them would be exercised by any given + tenant. The lookup order, the failure posture and the reasons for both are unchanged and are + documented on `_connector_spec_tables` below. + """ + try: + from core import connector_spec # noqa: PLC0415 - lane D's module (C2) + except Exception: # noqa: BLE001 + return () + key = str(connector_key or "") + # The directory uses the keychain type; the table registry uses the provider name. + if key == "amazon_sp": + key = "amazon" + found = None + for name in ("tables_for", "for_connector", "tables", "spec", "spec_for"): + fn = getattr(connector_spec, name, None) + if not callable(fn): + continue + try: + got = fn(key) + except Exception: # noqa: BLE001 + continue + if got: + found = got + break + if not found: + for name in ("SPECS", "TABLES", "CONNECTOR_TABLES"): + blob = getattr(connector_spec, name, None) + if isinstance(blob, dict): + found = blob.get(key) or list(blob.values()) + elif isinstance(blob, (list, tuple)): + found = list(blob) + if found: + break + return list(found.values()) if isinstance(found, dict) else list(found or ()) + + +def _connector_spec_links(connector_key): + """C2's DECLARED relations for one connector: `{table key: [(from, target, to, label)]}`. + + ⭐⭐ W53-T30 / C2 — THE GRAPH'S SECOND SOURCE OF EDGES, and the contract's own words for it: + each spec row carries `links`, a list of `(from_col, target_key, to_col, cardinality)`. Today + every edge on this graph is read off a stored FIELD bag, which can only ever describe a + relation somebody wrote a column for; a spec-declared relation between two tables that carry + no link column has nowhere else to come from. + + ⛔ AN EDGE FROM HERE IS HELD TO THE SAME TWO PROOFS AS ONE FROM A FIELD BAG, in the caller: + both endpoints must be permitted NODES on this graph and both named columns must be VISIBLE + to this reader. A spec is a declaration about the shipped schema, never a permission decision + about a person, and an edge drawn on the strength of the declaration alone would name a column + an administrator hid from the account looking at it. + + ⛔ AND IT IS NOT AN OUTAGE WHEN LANE D'S MODULE IS ABSENT: `{}` means "no spec-declared + relations", the field bags still answer, and the graph is exactly what it is today. + + ⚠ `cardinality` IS NOT A LABEL and is not read as one. The wire's `label` is what the reader + sees on the line, and "one to many" is a fact about the join rather than a name for it, so the + caller labels a spec edge with the SOURCE COLUMN's own label instead. + """ + out = {} + for item in _connector_spec_items(connector_key): + table_key = str(_spec_field(item, "key") or "") + links = _spec_field(item, "links") + if not table_key or not isinstance(links, (list, tuple)): + continue + edges = [] + for link in links: + if not isinstance(link, (list, tuple)) or len(link) < 3: + continue + from_col = str(link[0] or "").strip() + target_key = str(link[1] or "").strip() + to_col = str(link[2] or "").strip() + if from_col and target_key and to_col: + edges.append((from_col, target_key, to_col)) + if edges: + out[table_key] = edges + return out + + +def _connector_spec_tables(connector_key): + """C2's spec for one connector as `{table key: {"native", "label"}}`, or `{}`. + + ⛔⛔ `core.connector_spec` IS LANE D'S MODULE AND IT IS NOT IN THIS TREE YET. C2 splits the + work: lane D declares the spec (`key`, `native`, `source`, `label`, `pinned`, `identity`, + `links`) and this lane CONSUMES it through `GET /connectors/{key}/detail`. So the import is + defensive and the fallback is the in-tree source each value already has — Odoo's native model + from `odoo_relational.ODOO_TABLE_MODELS`, its default label from `grid_choices`, Amazon's + report type from `connectors_amazon.REPORT_TYPES` and its label from `LABELS_AMAZON`. When D's + module lands, the spec WINS with no edit here. + + ⚠ D'S EXACT API IS NOT FROZEN, so the lookups tried are written down rather than guessed at by + the next reader: the callables `tables_for(key)`, `for_connector(key)`, `tables(key)`, + `spec(key)`, `spec_for(key)`, then the module attributes `SPECS`, `TABLES`, + `CONNECTOR_TABLES` (a dict keyed by connector, a dict keyed by table, or a flat sequence). + Whatever answers first is normalised by TABLE KEY, and only the keys this connector already + declares are ever consulted — so a spec carrying other connectors' tables is harmless. + ⚠ W53-T30 — THAT LOOKUP NOW LIVES IN `_connector_spec_items` ABOVE, unchanged, because the + relationship graph reads the same spec for its `links`. One import path, two readings. + + ⛔ A SPEC THAT CANNOT BE READ IS NOT AN OUTAGE. Every failure returns `{}` and the row keeps + the in-tree value it has today: C2 is a better source for these two strings, never the only + one, and a Databases tab that 500s because a sibling lane's module changed shape would be a + disclosure screen taken down by a naming question. + """ + out = {} + for item in _connector_spec_items(connector_key): + table_key = str(_spec_field(item, "key") or "") + if not table_key: + continue + out[table_key] = {"native": str(_spec_field(item, "native") or ""), + "label": str(_spec_field(item, "label") or "")} + return out + + +def _database_name_overrides(rt): + """This tenant's per-database name overrides: `{table key: {"name": ...}}`, or `{}`. + + ⛔⛔ `nav_meta` IS THE STORE, AND IT IS READ THROUGH THE NAVIGATION RAIL'S OWN READER. R9, + verbatim: *"the rename database under the Navigation rail also works, it should then sync to + the navigation in the connector too."* That rename has written `nav_meta` since wave 19 + (`routes_nav.save_nav_meta`), and `routes_tables.ut_label` already resolves a served name as + nav_meta's name, then the definition's label, then the key. Calling `routes_nav._read_nav_meta` + rather than re-deriving the document shape here is what makes the rail and this tab provably + ONE fact instead of two answers to one question. + + ⛔ AND IT IS WHY NOTHING HERE WRITES A TABLE DEFINITION'S LABEL. `user_tables.set_label` is + rewritten by a connector re-sync (`odoo_relational.apply_plan` rebuilds definitions), so an + override stored there would be silently clobbered on the next sync, which is precisely the + R9 violation this ticket exists to prevent. + + ⚠ ONE READ FOR THE WHOLE LIST, never one per row: `store.get` deep copies the tenant document + on every call, and this list is eight rows on Odoo and seven on Amazon. + ⚠ A store blip is "no overrides", never a failed page. The default name is still true. + """ + try: + import routes_nav # noqa: PLC0415 - lazy, avoids a cycle + meta = routes_nav._read_nav_meta(rt) or {} # noqa: SLF001 - the ONE reader (R9) + except Exception: # noqa: BLE001 + return {} + return {str(k): v for k, v in meta.items() if isinstance(v, dict)} + + +def _rename_rights(rt, uname, is_admin, keys): + """`{table key: {exists, canRename}}` for R9's write wall and its prerequisite. + + ⛔ THE SAME PREDICATE `routes_nav.save_nav_meta` APPLIES TO A `ut_` KEY, not a second one: + `user_tables.may_open`, i.e. the table's creator, a tenant admin, or a share grant. Two ideas + of who may rename a database would show one person a control that the other door refuses. + + ⛔ A DATABASE THIS CONNECTION HAS NOT CREATED YET IS NOT RENAMEABLE, and that is not a + technicality: `nav_meta` is keyed by TABLE key, so a name stored for a table nobody can open + would sit in the tenant's document naming nothing, and the rail would never show it. + + ⭐ W56-T31: `enabled` USED TO MEAN "will be included in the next sync", while the screen + read as though the database already existed. `canRename` was telling the truth about a table + that did not exist. Returning `exists` from this same projection lets the client distinguish + that state from a reader who may not open an existing table. + + ⚠ IT IS SERVED PER ROW because the answers ARE per row. One `lend_defs` read answers the + whole list; that projection carries definitions without rows (~0.1% of the document's bytes). + """ + try: + import core.user_tables as user_tables # noqa: PLC0415 - lazy, as elsewhere + definitions = user_tables.lend_defs(rt) + facts = {} + for key in keys: + exists = bool(user_tables.get(key, st=definitions)) + facts[key] = { + "exists": exists, + "canRename": bool(str(key).startswith("ut_") and exists + and user_tables.may_open( + key, uname, bool(is_admin), st=definitions)), + } + return facts + except Exception: # noqa: BLE001 + # Fail closed. A wall that could not be read is not a wall that admitted you. + return {key: {"exists": False, "canRename": False} for key in keys} + + +def _named_databases(rt, is_admin, uname, rows): + """The LAST step of every databases list, Odoo's and Amazon's alike: the three names. + + One place, deliberately. Each half of `_detail_databases` builds its own `native` and its own + shipped default from its own source, and then BOTH come through here — so "what is this + database called for this tenant" is answered once, by `schema_overlay.merge_database_labels`, + and a third connector cannot arrive with a fourth opinion about a blank override. + """ + named = schema_overlay.merge_database_labels(rows, _database_name_overrides(rt)) + rights = _rename_rights(rt, uname, is_admin, [str(row.get("key") or "") for row in named]) + for row in named: + facts = rights.get(str(row.get("key") or "")) or {} + row["exists"] = facts.get("exists") is True + row["canRename"] = facts.get("canRename") is True + return named + + +def _clean_scraper_recipe(raw): + """Return T11's persisted recipe shape and explicit structural validation errors. + + This function is intentionally pure. Preview and creation both call it, so opening a saved + recipe cannot reinterpret it differently, and validation cannot accidentally become a paid + provider probe. + """ + value = raw if isinstance(raw, dict) else {} + kind = str(value.get("kind") or "").strip().lower() + errors = [] + if kind not in SCRAPER_RECIPE_KINDS: + errors.append("Choose Instagram, TikTok, or Google Maps.") + return {}, errors + + limits = value.get("limits") if isinstance(value.get("limits"), dict) else {} + schedule = value.get("schedule") if isinstance(value.get("schedule"), dict) else {} + schedule_mode = str(schedule.get("mode") or "manual").strip().lower() + if schedule_mode != "manual": + errors.append("This recipe must use the manual Run now schedule.") + + if kind == "google_maps": + seed = value.get("seed") if isinstance(value.get("seed"), dict) else {} + search = str(seed.get("search") or value.get("search") or "").strip() + geography = str(seed.get("geography") or value.get("geography") or "").strip() + raw_cap = limits.get("records", value.get("limit")) + if not search: + errors.append("Enter a category or search term.") + if not geography: + errors.append("Enter a geography.") + if raw_cap in (None, ""): + errors.append("Enter a result cap.") + cap = 0 + else: + try: + cap = int(raw_cap) + except (TypeError, ValueError): + cap = 0 + errors.append("The result cap must be a whole number.") + if raw_cap not in (None, "") and (cap < 1 or cap > 500): + errors.append("The result cap must be between 1 and 500.") + return ({"kind": kind, "source": "places", + "seed": {"search": search, "geography": geography}, + "limits": {"records": cap}, "schedule": {"mode": "manual"}}, errors) + + seed = str(value.get("seed") or "").strip() + raw_limit = limits.get("posts", value.get("limit")) + post_type = str(limits.get("postType") or value.get("postType") or "video").strip().lower() + if not seed: + errors.append(f"Enter an {kind.title()} profile seed.") + if raw_limit in (None, ""): + errors.append("Enter a last-N post limit.") + limit = 0 + else: + try: + limit = int(raw_limit) + except (TypeError, ValueError): + limit = 0 + errors.append("The post limit must be a whole number.") + if raw_limit not in (None, "") and (limit < 1 or limit > SCRAPER_RECIPE_MAX_ROWS): + errors.append(f"The post limit must be between 1 and {SCRAPER_RECIPE_MAX_ROWS}.") + allowed_types = ("video", "image", "carousel") if kind == "instagram" else ("video",) + if post_type not in allowed_types: + errors.append("Choose video, image, or carousel posts." + if kind == "instagram" else + "TikTok's native profile window supports video posts only.") + return ({"kind": kind, "source": "profile", "seed": seed, + "limits": {"posts": limit, "postType": post_type}, + "schedule": {"mode": "manual"}}, errors) + + +def _scraper_recipe_fields(kind): + """Stable user-table schema for one recipe, derived from the shipped normalisers.""" + if kind == "google_maps": + return [dict(field) for field in _MAPS_RECIPE_FIELDS] + import automation_engine as engine # noqa: PLC0415 - existing schema owner + source = engine.POST_FIELDS if kind == "instagram" else engine.TT_POST_FIELDS + # Links and rollups target the shared automation tables. A recipe database is independent, + # so copying those references would create a plausible but broken relationship graph. + return [dict(field) for field in source + if isinstance(field, dict) and field.get("type") not in ("link", "rollup")] + + +def _scraper_plan(recipe): + """Network-free setup preview: declared provider, modelled spend, cap, and credential state.""" + kind = str((recipe or {}).get("kind") or "") + limit_key = "records" if kind == "google_maps" else "posts" + limit = int(((recipe or {}).get("limits") or {}).get(limit_key) or 0) + if kind == "google_maps": + import providers # noqa: PLC0415 - registry owns cost/capability + apify = providers.PROVIDERS["apify"] + capability = apify.cap("google_maps_stores") + per_record = float(capability.cost_per_record or 0.0) if capability else 0.0 + amount = round(per_record * limit, 4) + return {"provider": apify.label, "credentialReady": apify.configured(), + "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit, + "canRun": True, + "note": "Modelled from the declared per-place rate; the provider receipt " + "reconciles the bounded result count."} + + import providers # noqa: PLC0415 - registry is the authority + bright = providers.PROVIDERS["brightdata"] + apify = providers.PROVIDERS["apify"] + if kind == "tiktok": + per_record = float(bright.cap("tt_post_metrics").cost_per_record or 0.0) + profile_cost = float(bright.cap("tt_profile").cost_per_record or 0.0) + estimate_usd = profile_cost + (per_record * limit) + provider_label = bright.label + credential_ready = bright.configured() + else: + # The established Instagram runner can use Bright Data and its Apify fallback. The cap + # includes both post metrics and view enrichment, so it never hides the worst declared + # cost from the person authorising Run now. + bd_cost = float(bright.cap("ig_post_metrics").cost_per_record or 0.0) + profile_cost = float(bright.cap("ig_profile").cost_per_record or 0.0) + view_cost = float(apify.cap("ig_post_views").cost_per_record or 0.0) + estimate_usd = profile_cost + ((bd_cost + view_cost) * limit) + provider_label = (bright.label if bright.configured() or not apify.configured() + else apify.label) + credential_ready = bright.configured() or apify.configured() + amount = round(estimate_usd, 4) + return {"provider": provider_label, "credentialReady": bool(credential_ready), + "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit, + "canRun": True, "note": "Modelled from declared per-record list rates."} + + +def _scraper_databases(rt, is_admin, uname): + """Saved recipe databases this reader may open, with no row payloads or secrets.""" + try: + import core.user_tables as user_tables + definitions = user_tables.all_defs(rt) + lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions}) + except Exception: # noqa: BLE001 + return [] + rows = [] + for key, definition in (definitions or {}).items(): + recipe = definition.get("scraperRecipe") if isinstance(definition, dict) else None + if not isinstance(recipe, dict): + continue + try: + if not user_tables.may_open(key, uname, bool(is_admin), st=lent): + continue + except Exception: # noqa: BLE001 + continue + cleaned, errors = _clean_scraper_recipe(recipe) + if errors: + continue + plan = _scraper_plan(cleaned) + # Recipe existence alone is not enablement. The creation door stamps both ownership + # markers; a legacy/malformed row without them is disclosed as disabled and cannot run. + enabled = (definition.get("connectorKey") == "scraper" + and definition.get("recordMode") == "automation") + rows.append({"key": str(key), "label": str(definition.get("label") or key), + "defaultLabel": str(definition.get("label") or key), + "native": cleaned["kind"], "enabled": enabled, "exists": True, + "canRename": True, "sourceRows": None, "populated": None, + "fields": [dict(field) for field in (definition.get("fields") or ()) + if isinstance(field, dict)], + "excluded": [], "sharesFieldsWith": [], "scraperRecipe": cleaned, + "syncHealth": _stored_database_sync_health( + definition.get(user_tables.CONNECTOR_HEALTH_FIELD), enabled), + "scraperRuns": [dict(item) for item in (definition.get("scraperRuns") or ()) + if isinstance(item, dict)], + "runPath": f"/connectors/scraper/databases/{key}/run", + "canRun": enabled and plan["canRun"] is True}) + return _named_databases(rt, is_admin, uname, rows) + + +def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None): + """C4's `databases` half: what this connector materialises, and the fields behind each one. + + ⛔ ODOO IS THE ONLY INTROSPECTABLE CONNECTOR TODAY and every other card answers `[]`. That is + the contract's own instruction, and the alternative would be a list of database names no + connector actually creates. + + ⛔ THE ROW SURVIVES FOR A NON-ADMIN; ONLY THE CATALOGUE HALVES EMPTY. Returning `[]` to a + member would be indistinguishable from *"this connector creates no databases"*, which is the + confidently-wrong shape this module's header was written about. A database's NAME and whether + it is switched on are facts about their own workspace; the raw Odoo field catalogue is + administrator-only (`_odoo_admin` gates the route that serves it) and stays that way. + + Fields with the same source model have independent enablement keys in each database. + `sharesFieldsWith` remains an empty list for existing clients of the older shared switches. + + ⭐⭐ W53-T27 / R9 / C2 — THREE NAMES PER ROW, AND THEY ARE THREE DIFFERENT FACTS. `native` is + the VENDOR's own identifier (`res.partner`, or Amazon's report type) and is never editable by + anybody; `defaultLabel` is OUR shipped default; `override` is this tenant's own rename, held + in the `nav_meta` bucket the navigation rail already writes. `label` stays the EFFECTIVE name + because every existing reader of it — this tab's heading, `_detail_overview`'s `creates` list + — already means "what this database is called today", and serving the un-overridden default + there would be the drift R9 exists to close. + + ⛔ ALL THREE SURVIVE FOR A NON-ADMIN, like `key` and `enabled` above them and for the same + reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE + is walled, per row, by `canRename`. + """ + if str(connector_key) == "scraper": + return _scraper_databases(rt, is_admin, uname) + if not connector_directory.is_introspectable(connector_key): + return [] + spec = _connector_spec_tables(connector_key) + if str(connector_key) == "amazon_sp": + return _named_databases(rt, is_admin, uname, _amazon_databases(rt, is_admin, spec)) + try: + import odoo_relational + namespace = (odoo_relational.connection_namespace(rt, connection_id)[1] + if connection_id else None) + choices = odoo_relational.grid_choices(rt, connection_id) + models = dict(getattr(odoo_relational, "ODOO_TABLE_MODELS", {}) or {}) + # This read never contacts Odoo. A 499-model exact pass measured 67.87 s after the + # catalogue was warm, so modal detail must never turn a missing cache into inline work. + census = odoo_relational.population_census(rt, namespace) + except Exception: # noqa: BLE001 + return [] + health = odoo_health_receipt(rt, connection_id) + health_by_key = {str(item.get("databaseKey") or ""): item + for item in (health.get("databases") or ())} + census_at = str(census.get("capturedAt") or "") + counts = census.get("counts") if isinstance(census.get("counts"), dict) else {} + catalog = {} + if is_admin: + try: + catalog = _odoo_catalog_by_model(rt, namespace) + except Exception: # noqa: BLE001 + catalog = {} + enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {} + # Connection resolution reaches keychain.list_entries and therefore the store (Postgres). + # It ran 12,824 times for fields and database identities, then 501 with a per-key memo. + # Resolve the shared default lazily once instead. This local belongs only to this detail + # build: request scope keeps the answer per runtime and tenant, never process-wide. + default_connection_id = ... + + def resolve_connection_id(key): + nonlocal default_connection_id + base, cid = odoo_relational.split_table_key(key) + if base is None: + return None + if cid: + return cid + if default_connection_id is ...: + default_connection_id = (odoo_relational.default_connection_id(rt) + or odoo_relational.DEFAULT_CONNECTION) + return default_connection_id + + out = [] + for choice in choices: + key = str(choice.get("key") or "") + resolved_connection_id = resolve_connection_id(key) + model = str(choice.get("model") or models.get(key) or "") + fields, excluded = catalog.get(model, ([], [])) + scoped_fields = [] + for field in fields: + field_key = _odoo_field_key(rt, key, model, field["source_field"], + connection_id=resolved_connection_id) + scoped_fields.append({**field, "key": field_key, + "enabled": enabled_fields.get(field_key, + field.get("enabled") if namespace is None else False) is True}) + declared = spec.get(choice.get("baseKey") or key) or {} + # Counts are model-level facts. Databases backed by the same model deliberately show the + # same sourceRows number; inventing a per-database domain here would make the number lie. + source_rows = counts.get(model) if census_at and model in counts else None + database_health = health_by_key.get(key) or { + # Source rows is a model census. It is not this governed database's materialized + # row count, especially when two curated databases share one source model. + "databaseKey": key, "state": "unknown", "rowCount": None, + "lastSuccess": None, "currentLagSeconds": None, "recordRefreshed": None, + "failure": None, "stages": _empty_odoo_stages()} + out.append({"key": key, + "identity": connector_sync.database_identity( + "odoo", resolved_connection_id, key, model), + "label": str(declared.get("label") or choice.get("label") or key), + "defaultLabel": str(declared.get("label") or choice.get("label") or key), + "native": str(declared.get("native") or model), + "enabled": bool(choice.get("enabled")), + "sourceRows": source_rows, + "populated": None if source_rows is None else source_rows > 0, + # Data-free client receipt: source/mirror/materialization/reconciliation/ + # freshness are distinct, and one database's failure cannot make a sibling + # look newly successful. + "sync": database_health, + "syncHealth": _odoo_database_sync_health( + database_health, bool(choice.get("enabled"))), + "fields": scoped_fields, + "excluded": [dict(item) for item in excluded], + "sharesFieldsWith": []}) + return _named_databases(rt, is_admin, uname, out) + + +def _amazon_report_name(table): + """The report type this Amazon table reads, whichever key declares it. + + ⛔ SETTLEMENTS DECLARE `report_discover`, NOT `report` (W60-T36), because Amazon issues them on + its own schedule and refuses a createReport for that type: it is a getReports DISCOVERY filter + and is deliberately absent from `_APPROVED_REPORT_TYPES`. Reading only `report` here printed an + empty Source name for settlements on the connector screen, which reads as "this table has no + report" when the truth is "this table is not requested, it is listed". + """ + import connectors_amazon as cx # noqa: PLC0415 - provider owns state + spec = cx.REPORT_TYPES.get(table) or {} + return str(spec.get("report") or spec.get("report_discover") or "") + + +def _amazon_databases(rt, is_admin, spec=None): + """⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM. + + Fills the shared `ConnectorDatabase` shape and adds an administrator-only Amazon `sync` + block. Members retain the shared shape without the field catalogue or sync bookkeeping. + + ⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five + of the seven tables have NO published schema, because Amazon publishes artifacts only for its + JSON analytics reports and every flat-file report we depend on has none. Their columns are + measured from the report document's own header at first sync, so before that they genuinely + have zero fields. A screen that showed an empty field list with no explanation would say + *"this database has no fields"* when the truth is *"nobody has run it yet"* — the exact + confusion `_detail_introspection`'s header was written about, one layer in. + + ⛔ `sharesFieldsWith` IS REAL HERE, NOT COPIED. `orders` and `order_items` are BOTH filled from + ONE report document (`GET_FLAT_FILE_ALL_ORDERS_DATA_*`, which is one row per order ITEM), so + they share a header and therefore a field list. That is the same class of fact the Odoo pairs + carry, measured the same way: derived from the report map, never hand-typed. + + ⚠ NO FIELD TOGGLES. `canToggleFields` is False for Amazon in `_detail_introspection`; there is + no `PATCH /connectors/amazon_sp/schema/{field}` route and this wave does not build one. Fields + are disclosed, not switched. Database enablement is a separate provider capability and route, + never the generic Odoo-table flag consumed by `DatabasesTab`. + """ + try: + from harness import amazon_schema + import connectors_amazon as cx + except Exception: # noqa: BLE001 + return [] + enablement = cx.read_database_enablement(rt) + enabled = enablement.get("databases") or {} + cat = amazon_schema.catalogue() + tables = cat.get("tables") or {} + states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None + + # Which OTHER declared tables are filled from the SAME report document. Derived from the + # report map so a table added to `REPORT_TYPES` gets this disclosure for free. + by_report = {} + for name in amazon_schema.PROVENANCE: + rep = _amazon_report_name(name) + if rep: + by_report.setdefault(rep, []).append("ut_amzn_" + name) + + out = [] + for name, provenance in amazon_schema.PROVENANCE.items(): + key = "ut_amzn_" + name + spec = tables.get(name) or {} + cols = spec.get("columns") or [] + rep = _amazon_report_name(name) + fields = [{"key": str(c.get("name") or ""), + "label": str(c.get("name") or "").replace("_", " ").capitalize(), + "type": "text" if str(c.get("type")) == "VARCHAR" else "number", + "source_field": str(c.get("name") or ""), + # ⚠ EVERY DISCLOSED FIELD IS ON. There is no per-field switch for Amazon, and + # reporting `enabled: False` would describe a control that does not exist. + "enabled": True} + for c in cols] if is_admin else [] + excluded = [] + if provenance == "measured_at_sync" and not cols: + excluded.append({ + "source_field": "", + "reason": (amazon_schema.PROVENANCE_COPY["measured_at_sync"] + + f" The report is {rep}." if rep + else amazon_schema.PROVENANCE_COPY["measured_at_sync"])}) + # ⭐⭐ W53-T27 / C2 — THE SAME THREE FACTS AS THE ODOO ROWS, one source swapped. Amazon + # has no per-table model name; the identifier AMAZON itself uses for what fills a table is + # the REPORT TYPE (`rep`, already computed above for `sharesFieldsWith`). That is exactly + # what `native` carries: the vendor's own name for the thing, never one we invented. It is + # empty for a table whose report type is unknown, and the tab says so rather than guessing. + declared = (spec or {}).get(key) or {} + raw_sync = (states or {}).get(name) if is_admin else None + is_enabled = enabled.get(key) is True + out.append({"key": key, + "label": str(declared.get("label") or LABELS_AMAZON.get(key) or key), + "defaultLabel": str(declared.get("label") + or LABELS_AMAZON.get(key) or key), + "native": str(declared.get("native") or rep), + # W55-T19 / R7 — AMAZON'S OWN tenant-local answer. This is deliberately not + # read from Odoo's grid config, and disabling it leaves every stored row and + # definition untouched: only this provider config document changes. + "enabled": is_enabled, + **({"sync": raw_sync} if is_admin else {}), + "syncHealth": _amazon_database_sync_health(raw_sync, is_enabled), + "fields": fields, + "excluded": excluded, + "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]}) + return out + + +#: What an empty `excluded` list means for AMAZON, which is not what it means for Odoo. +#: +#: ⛔ A SEPARATE SENTENCE ON PURPOSE. Odoo's basis describes a DIFF of a live snapshot against a +#: catalogue. Amazon has no snapshot and no diff: a table either has a published schema or it +#: measures its columns at first sync. Reusing Odoo's wording would describe a computation that +#: does not happen here, which is the same class of error as reusing its toggles. +_AMAZON_EXCLUDED_BASIS = ( + "Amazon publishes a schema for its JSON analytics reports only. A table with no published " + "schema reads its columns from the report's own header the first time it syncs, so it lists " + "no fields until then. Nothing is being held back from you.") + + +#: Table labels for the Amazon disclosure. ⛔ NO EM DASH OR EN DASH: these reach a screen. +LABELS_AMAZON = { + "ut_amzn_orders": "Amazon orders", + "ut_amzn_order_items": "Amazon order items", + "ut_amzn_listings": "Amazon listings", + "ut_amzn_settlements": "Amazon settlements", + "ut_amzn_inventory": "Amazon inventory", + "ut_amzn_pricing": "Amazon pricing and Buy Box", + "ut_amzn_brand_analytics": "Amazon brand analytics", +} + + +#: What an EMPTY `excluded` list actually means, said once, on the server that computes it. +#: +#: ⛔⛔ THE TRAP THIS CLOSES, MEASURED: `_odoo_catalog_by_model` builds `excluded` as a DIFF of the +#: snapshot's raw fields against the catalogue, and the catalogue offers every raw field it +#: captured — so the diff is EMPTY TODAY BY CONSTRUCTION. A tab that rendered "Excluded: none" +#: over that would be telling an administrator that this workspace holds back nothing from Odoo, +#: which is a claim about POLICY that the number cannot support. The empty list means the +#: catalogue dropped nothing it saw; it says nothing at all about what Odoo never reported. +#: +#: ⚠ IT IS SERVED RATHER THAN WRITTEN IN THE CLIENT because the sentence is a description of a +#: server computation. A client copy would go stale the day the catalogue starts filtering, and +#: nothing would go red. +_EXCLUDED_BASIS = ("A field is excluded when this workspace captured it from Odoo and the field " + "catalogue did not offer it. Anything Odoo never reported cannot appear in " + "this list at all.") + + +def _detail_introspection(rt, is_admin, row): + """R24's half of the Databases tab: may this connector be disclosed, and to THIS reader? + + ⭐ AN ADDITION TO C4, DECLARED, and the same kind of addition `_detail_connect` is. C4 pins + the `databases` list and says nothing about how a client tells "this connector has no schema" + apart from "this connector's schema could not be read". Those two are the same empty list and + they mean opposite things, so the difference is stated rather than inferred. + + ⛔ EVERY FLAG HERE IS A CAPABILITY THE SERVER WILL HONOUR, not a guess the client makes. The + field toggle is `PATCH /connectors/odoo/schema/{field_key}`, which `_odoo_admin` gates and + which raises 409 `odoo_schema_unavailable` while no complete snapshot exists — so + `canToggleFields` carries BOTH conditions and the client renders no control that could only + ever 409. A button that always fails is R7's banned shape. + + ⚠ `fieldsVisible` IS SEPARATE FROM `canToggleFields` ON PURPOSE. A member sees the database + names and their on/off state (their own workspace's facts) and no field catalogue at all + (`_odoo_admin` gates the route that serves it); saying which of the two is missing is what + stops "administrator only" being rendered as "this database has no fields". + """ + key = str(row.get("key") or "") + if key == "scraper": + # Recipe databases publish their saved, stable schema to every reader who may open the + # database. They have no generic schema-toggle route, so no write capability is implied. + return {"introspectable": True, "reason": "", "fieldsVisible": True, + "canToggleFields": False, "canToggleTables": False, + "catalogueComplete": True, "capturedAt": "", "excludedBasis": "", + "censusAt": "", "censusComplete": False, "censusDurationMs": 0, + "censusModelsCounted": 0, "censusModelsWithRows": 0} + if not connector_directory.is_introspectable(key): + reason = ("Nothing is connected behind this listing yet, so there is no schema to read." + if str(row.get("state") or "") == connector_directory.CATALOGUE_STATE + else "This connector does not publish a field catalogue, so the databases it " + "creates cannot be listed field by field.") + return {"introspectable": False, "reason": reason, "fieldsVisible": False, + "canToggleFields": False, "canToggleTables": False, + "catalogueComplete": False, "capturedAt": "", "excludedBasis": "", + "censusAt": "", "censusComplete": False, "censusDurationMs": 0, + "censusModelsCounted": 0, "censusModelsWithRows": 0} + if key == "amazon_sp": + # ⭐⭐ W52 · R6. Introspectable under the AMENDED test (a real schema artifact, not a + # `fields_get`), and deliberately with NO generic toggles: there is no Amazon field-toggle + # route, and `canToggleTables` belongs to Odoo's config surface. W55-T19 serves Amazon's + # database switches through the separate `enablement` capability instead, so this flag + # stays False and can never send Amazon through an Odoo endpoint. + # + # ⛔ `catalogueComplete` IS FALSE AND SAYING SO IS THE HONEST ANSWER. Only two of the seven + # tables have a published artifact; the other five measure their columns from the report's + # own header at first sync. Reporting `True` would claim a completeness that does not + # exist, and reporting `introspectable: False` would hide the two that ARE published. + try: + from harness import amazon_schema + cat = amazon_schema.catalogue() + tables = cat.get("tables") or {} + complete = bool(tables) and all((t.get("columns") or []) for t in tables.values()) + captured = str(cat.get("fetched_utc") or "") + except Exception: # noqa: BLE001 + complete, captured = False, "" + return {"introspectable": True, + "reason": "", + "fieldsVisible": bool(is_admin), + "canToggleFields": False, + "canToggleTables": False, + "catalogueComplete": complete, + # ⚠ THE CATALOGUE'S OWN FETCH DATE, NEVER TODAY'S. R6 is explicit that an + # unstamped pinned catalogue is a stale seed waiting to happen, and this screen is + # the only place the staleness becomes visible. + "capturedAt": captured, + "excludedBasis": _AMAZON_EXCLUDED_BASIS, + "censusAt": "", "censusComplete": False, "censusDurationMs": 0, + "censusModelsCounted": 0, "censusModelsWithRows": 0} + complete = False + captured = "" + census = {} + try: + import core.user_tables as user_tables + snapshot = user_tables.odoo_schema_snapshot(rt) + import odoo_relational + discovered = odoo_relational.readable_model_catalog(rt) + census = odoo_relational.population_census(rt) + has_discovery = bool(discovered.get("capturedAt")) + complete = (discovered.get("complete") is True if has_discovery + else snapshot.get("complete") is True) + captured = str((discovered.get("capturedAt") if has_discovery + else snapshot.get("capturedAt")) or "") + except Exception: # noqa: BLE001 + complete, captured = False, "" + return {"introspectable": True, + "reason": "", + "fieldsVisible": bool(is_admin), + # BOTH conditions, because the write door enforces both. + "canToggleFields": bool(is_admin) and complete, + # ⭐ R22: ANY table can be switched off, not only the ones we mark optional. The write + # is `PUT /admin/connectors/odoo/config`, which is an administrator's door and accepts + # every key `grid_choices` names, including all of them at once. + "canToggleTables": bool(is_admin), + "catalogueComplete": complete, + "capturedAt": captured, + "excludedBasis": _EXCLUDED_BASIS, + "censusAt": str(census.get("capturedAt") or ""), + "censusComplete": census.get("complete") is True, + "censusDurationMs": int(census.get("durationMs") or 0), + "censusModelsCounted": int(census.get("modelsCounted") or 0), + "censusModelsWithRows": int(census.get("modelsWithRows") or 0)} + + +def _detail_overview(row, databases, graph): + """C4's `overview` half: what the modal's first tab reads, top to bottom. + + `creates` is the ENABLED table LABELS this connector stands behind: the declared grids when the + connector declares any, and otherwise the tables its own permitted graph actually found. Both + are real readings of THIS workspace; neither is a provider template. + """ + creates = [str(entry.get("label") or "") for entry in (databases or []) + if entry.get("enabled") is True and str(entry.get("label") or "")] + if not databases and isinstance(graph, dict) and graph.get("status") == "ready": + creates = [str(table.get("label") or "") for table in (graph.get("tables") or ()) + if str(table.get("label") or "")] + return {"key": str(row.get("key") or ""), + "label": str(row.get("label") or ""), + "status": str(row.get("state") or ""), + # Empty when the server has no account to name. The client says so in a sentence + # rather than printing a blank row. + "account": str(row.get("connectedAs") or ""), + # The system the data comes FROM, in the words a customer reads on the card. Never a + # provider slug: an internal identifier on screen is exactly what `parseDirectory` + # drops a row to avoid. + "source": str(row.get("label") or ""), + "creates": creates} + + +def _detail_enablement(rt, is_admin, row): + """A provider-owned enablement capability, or ``None`` when this connector has no door. + + Amazon is intentionally the first entry and is named here rather than inferred from whether a + database list happened to be non-empty. Odoo keeps its model-catalogue and config routes; + every other connector keeps no enablement affordance until its provider declares one. + """ + if str(row.get("key") or "") != "amazon_sp": + return None + try: + import connectors_amazon as cx + return cx.enablement_capability(rt, bool(is_admin)) + except Exception: # noqa: BLE001 + # Fail closed. An unreadable provider capability is no write door. + return {"provider": "amazon_sp", "mode": "database", "canManage": False, + "updatePath": "", "preservesData": True, "available": False} + + +def _scraper_preview(body): + request = body if isinstance(body, dict) else {} + raw_recipe = request.get("recipe") if isinstance(request.get("recipe"), dict) else request + recipe, errors = _clean_scraper_recipe(raw_recipe) + if errors: + raise err(400, "invalid_scraper_recipe", " ".join(errors)) + plan = _scraper_plan(recipe) + missing = [] if plan["credentialReady"] else ["credential"] + return {"recipe": recipe, "inputs": recipe["seed"], "provider": plan["provider"], + "credentialReady": plan["credentialReady"], "estimatedUsd": plan["estimatedUsd"], + "costCapUsd": plan["costCapUsd"], "recordsCap": plan["recordsCap"], + "canRun": plan["canRun"], "missing": missing, "note": plan["note"], + # This is a contract field, not a measurement. A preview and a create never call a + # provider, so both truthfully report zero incurred spend. + "incurredUsd": 0.0} + + +def _create_scraper_database(rt, uname, body): + """Persist one recipe database. Contains no provider import or runner call by construction.""" + if not rt.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + request = body if isinstance(body, dict) else {} + label = str(request.get("label") or "").strip() + if not label: + raise err(400, "missing_database_name", "Enter a database name.") + preview = _scraper_preview(request) + recipe = preview["recipe"] + import core.user_tables as user_tables # noqa: PLC0415 - existing create door + key = user_tables.create(label, uname, fields=_scraper_recipe_fields(recipe["kind"]), + source=user_tables.AUTOMATION_SOURCE, st=rt) + if not key: + raise err(409, "database_not_created", + "The database name is unavailable or this workspace reached its database limit.") + + def _attach(cur): + current = dict(cur or {}) + definition = dict(current.get(key) or {}) + if not definition: + raise RuntimeError("the new database disappeared before its recipe was attached") + definition.update({"connectorKey": "scraper", "recordMode": "automation", + "scraperRecipe": recipe, "scraperRuns": [], + user_tables.CONNECTOR_HEALTH_FIELD: + user_tables.connector_health_receipt("never_run")}) + current[key] = definition + return current + + try: + rt.update(user_tables.STORE_KEY, _attach, flush="sync") + except Exception: # noqa: BLE001 + user_tables.delete(key, st=rt) + raise err(503, "recipe_not_saved", + "The recipe could not be saved, so the new database was removed.") + return {"key": key, "label": label, "recipe": recipe, "preview": preview, + "createdWithoutProviderCall": True, "incurredUsd": 0.0} + + +def _scraper_database_run(rt, uname, is_admin, database_key): + """The one explicit paid-work boundary for any saved scraper recipe.""" + if not rt.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + import core.user_tables as user_tables + key = str(database_key or "").strip() + definitions = user_tables.lend_defs(rt) + definition = user_tables.get(key, st=definitions) or {} + if not definition: + raise err(404, "unknown_scraper_database", "This saved scraper database was not found.") + raw_recipe = definition.get("scraperRecipe") + recipe, errors = _clean_scraper_recipe(raw_recipe) + if errors: + raw_kind = str((raw_recipe or {}).get("kind") or "") if isinstance(raw_recipe, dict) else "" + raw_limits = ((raw_recipe or {}).get("limits") or {}) if isinstance(raw_recipe, dict) else {} + raw_cap = raw_limits.get("records") if isinstance(raw_limits, dict) else None + try: + bounded = 1 <= int(raw_cap) <= 500 + except (TypeError, ValueError): + bounded = False + if raw_kind == "google_maps" and not bounded: + raise err(400, "maps_unbounded_request", + "Google Maps discovery requires an explicit result cap between 1 and 500.") + raise err(409, "invalid_saved_scraper_recipe", + "This saved scraper recipe is no longer valid. Review its settings before running it.") + # Run now both spends against a provider account and mutates this database. A viewer grant + # may expose the rows without granting either power, so the ordinary open wall is too broad. + # Reuse the database editor authority that every other mutation surface already honours. + if not user_tables.may_edit_database(key, uname, bool(is_admin), st=definitions): + raise err(403, "database_forbidden", "You cannot run this database recipe.") + plan = _scraper_plan(recipe) + previous_health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD) + previous_health = previous_health if isinstance(previous_health, dict) else {} + + def record_attempt(state, error=None): + try: + return user_tables.record_connector_health( + key, state, st=rt, + last_success=previous_health.get("lastSuccess"), + source_timestamp=previous_health.get("sourceTimestamp"), + materialized_timestamp=previous_health.get("materializedTimestamp"), + row_count=previous_health.get("rowCount"), error=error) + except Exception: # noqa: BLE001 + raise err(503, "sync_status_not_saved", + "The run did not start because its durable status could not be saved.") from None + + if not plan["credentialReady"]: + code = "maps_credential_missing" if recipe["kind"] == "google_maps" else \ + "scraper_credential_missing" + message = "Add the scraper credential in Keychain before choosing Run now." + record_attempt("failed", {"code": code, "message": message}) + raise err(409, code, "Add the scraper credential in Keychain before choosing Run now.") + + # Imports and provider calls occur only below the explicit Run-now endpoint. Preview, create, + # detail, and page load never reach either runner. + started_at = datetime.now(timezone.utc).isoformat() + record_attempt("running") + key_field = "shortcode" + if recipe["kind"] == "google_maps": + import providers + try: + result = providers.discover_google_maps_stores( + recipe["seed"]["search"], recipe["seed"]["geography"], + recipe["limits"]["records"], captured_at=started_at) + except providers.MapsProviderRefusal as exc: + record_attempt("failed", {"code": exc.code, "message": exc.public_message}) + status = 409 if exc.code == "maps_credential_missing" else \ + (400 if exc.code in ("maps_unbounded_request", "maps_invalid_search") else 502) + raise err(status, exc.code, exc.public_message) + incoming_key = "rows" + key_field = "place_id" + else: + limit = recipe["limits"]["posts"] + post_type = recipe["limits"]["postType"] + try: + if recipe["kind"] == "instagram": + import connectors_ig as runner + run_recipe = runner.pull_ig_recipe + else: + import connectors_tt as runner + run_recipe = runner.pull_tt_recipe + result = run_recipe(recipe["seed"], limit, post_type=post_type, + log=lambda *_args: None) + except Exception: # noqa: BLE001 + record_attempt("failed", { + "code": "scraper_run_failed", + "message": "The provider could not complete this run. No result was written.", + }) + raise err(502, "scraper_run_failed", + "The provider could not complete this run. No result was written.") + incoming_key = "posts" + result = result if isinstance(result, dict) else {} + incoming = [dict(row) for row in (result.get(incoming_key) or ()) if isinstance(row, dict)] + state = str(result.get("state") or "error") + if state not in ("ok", "partial"): + incoming = [] + finished_at = datetime.now(timezone.utc).isoformat() + source_stamps = [_sync_stamp(row.get("source_timestamp")) for row in incoming] + source_timestamp = max((stamp for stamp in source_stamps if stamp), default=None) + provider_counts = dict(result.get("counts") or {}) + provider_counts.setdefault("returned", len(incoming)) + provider_cost = dict(result.get("cost") or {}) + provider_cost.setdefault("currency", "USD") + provider_cost.setdefault("estimatedUsd", plan["estimatedUsd"]) + provider_cost.setdefault("costCapUsd", plan["costCapUsd"]) + receipt = {"id": finished_at, "state": state, + "provider": str(result.get("via") or plan["provider"]), + "rows": len(incoming), "startedAt": started_at, "finishedAt": finished_at, + "estimatedUsd": plan["estimatedUsd"], "costCapUsd": plan["costCapUsd"], + "cost": provider_cost, "providerCounts": provider_counts, + "attempts": [dict(item) for item in (result.get("attempts") or ()) + if isinstance(item, dict)], + # Provider notes can echo a submitted handle. Run history stores operational + # metadata only; the saved recipe already owns the input and no customer value is + # duplicated into an audit-like receipt. + "note": f"Provider returned {state}."} + counts = {"inserted": 0, "updated": 0, "unchanged": 0, "skipped": 0, + "duplicates": 0, "orphans": 0, "capped": 0} + + def _write(cur): + nonlocal counts + current = dict(cur or {}) + stored = dict(current.get(key) or {}) + if not stored: + raise RuntimeError("the database disappeared before the run could be written") + if incoming: + import automation_engine as engine + rows, counts = engine.upsert_rows(stored.get("rows") or {}, incoming, key_field) + stored["rows"] = rows + processed = sum(int(counts.get(name) or 0) + for name in ("inserted", "updated", "unchanged", "skipped", "capped")) + receipt["writeCounts"] = dict(counts) + receipt["countReconciled"] = ( + int(provider_counts.get("returned") or 0) == len(incoming) == processed) + history = [dict(item) for item in (stored.get("scraperRuns") or ()) + if isinstance(item, dict)] + stored["scraperRuns"] = [receipt, *history][:SCRAPER_RUN_HISTORY_LIMIT] + final_state = "succeeded" if state == "ok" else "partial" if state == "partial" else "failed" + final_error = None if final_state != "failed" else { + "code": "provider_result_failed", + "message": "The provider returned no usable result for this database.", + } + stored[user_tables.CONNECTOR_HEALTH_FIELD] = user_tables.connector_health_receipt( + final_state, + last_success=finished_at if final_state in ("succeeded", "partial") + else previous_health.get("lastSuccess"), + source_timestamp=source_timestamp, + materialized_timestamp=finished_at if final_state in ("succeeded", "partial") else None, + row_count=len(stored.get("rows") or {}), error=final_error) + current[key] = stored + return current + + try: + rt.update(user_tables.STORE_KEY, _write, flush="sync") + except Exception: # noqa: BLE001 + try: + record_attempt("failed", { + "code": "scraper_result_not_saved", + "message": "The provider answered, but the result could not be saved.", + }) + except Exception: # noqa: BLE001 + pass + raise err(503, "scraper_result_not_saved", + "The provider answered, but the result could not be saved. Reload before retrying.") + return {"key": key, "state": state, "counts": counts, "receipt": receipt} + + +def connector_detail(rt, user, uname, is_admin, connector_key): + """The whole C4 detail DTO for one connector, or `None` when this reader has no such card. + + Pure over the session for the same reason `directory` is: a gate can drive it without a + request, so the modal's payload is provable without a browser. + """ + budget = _ConnectorDetailBudget() + rows = directory(rt, uname, bool(is_admin)) + budget.check() + key = str(connector_key or "") + row = next((candidate for candidate in rows + if str(candidate.get("key") or "") == key), None) + if row is None: + return None + graph = relationship_graphs(rows, user, rt, uname, bool(is_admin)).get(key) or _graph_refusal( + key, "No actual relationship graph was returned for this connection. A provider template " + "is not shown.") + budget.check() + # ⚠ `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a + # question about a PERSON and a table, not about an administrator flag alone. + databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or "")) + budget.check() + overview = _detail_overview(row, databases, graph) + budget.check() + keychain = _detail_keychain(rt, uname, bool(is_admin), row) + budget.check() + enablement = _detail_enablement(rt, bool(is_admin), row) + budget.check() + introspection = _detail_introspection(rt, bool(is_admin), row) + budget.check() + return _safe_connector_status(key, {"overview": overview, + "keychain": keychain, + "databases": databases, + "syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"} + if is_admin and key in connector_sync.providers() else None), + "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs", + "retryPath": f"/connectors/{key}/sync/retry"} + if is_admin and key == "odoo" else None), + # W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it + # from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints. + "enablement": enablement, + # ⭐ W49-T14 / R24 — WHETHER THE DATABASES TAB MAY DISCLOSE ANYTHING AT ALL, and what + # THIS reader may do there. An addition to C4, declared on `_detail_introspection`: + # without it an empty `databases` list is "this connector has no schema" and "this + # connector's schema could not be read" at the same time. + "introspection": introspection, + # ⛔ THE `relationship_graphs` SHAPE, UNCHANGED. C4 says so in as many words, and the + # client already has a reader for it (`parseRelationshipGraph`) that refuses a graph + # bound to another connection. A second shape here would need a second reader. + "relationships": graph}) + + +def _set_amazon_database_enablement(database_key, body, session): + """The testable core of Amazon's tenant-scoped administrator enablement door.""" + if not session.admin: + raise err(403, "not_admin", + "only an administrator can change Amazon database enablement") + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + request = body if isinstance(body, dict) else {} + if set(request) != {"enabled"} or type(request.get("enabled")) is not bool: + raise err(400, "bad_enabled", "enabled must be true or false") + + import connectors_amazon as cx # noqa: PLC0415 - provider owns state + key = str(database_key or "").strip() + try: + state = cx.set_database_enabled(session.runtime, key, request["enabled"]) + except KeyError: + # One refusal for blank, malformed and cross-provider keys. The response does not expose + # the rest of the tenant's Amazon roster. + raise err(404, "unknown_amazon_database", + "Amazon does not declare a database with that key") + except cx.EnablementConfigError: + raise err(409, "amazon_enablement_unavailable", + "the stored Amazon enablement settings are invalid. Nothing was changed.") + except ValueError: + raise err(400, "bad_enabled", "enabled must be true or false") + except Exception: # noqa: BLE001 + raise err(503, "enablement_unconfirmed", + "the server could not confirm the Amazon database setting. Reload before " + "trying again.") + return {"provider": cx.ENTRY_TYPE, + "key": key, + "enabled": state["databases"].get(key) is True, + # This route changes only the provider config document. It never deletes a mirror or + # a materialised table, and the wire states that invariant explicitly. + "preservesData": True} + + +# Literal and provider-owned. It appears before the generic connector detail route so future +# dynamic route additions cannot turn Amazon enablement into an Odoo-model dispatch by accident. +@router.patch("/connectors/amazon_sp/databases/{database_key}/enablement") +def amazon_database_enablement(database_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Enable or disable one declared Amazon database for this tenant, without deleting data.""" + return _set_amazon_database_enablement(database_key, body, session) + + +@router.post("/connectors/scraper/databases/preview") +def scraper_database_preview(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Validate and price a recipe locally. This endpoint makes zero provider calls.""" + return _scraper_preview(body) + + +@router.post("/connectors/scraper/databases") +def scraper_database_create(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Create a saved recipe database for $0; paid work begins only at its Run-now door.""" + return _create_scraper_database(session.runtime, session.uname, body) + + +@router.post("/connectors/scraper/databases/{database_key}/run") +def scraper_database_run(database_key: str, session: Session = Depends(require_session)): + """Explicitly execute one bounded saved scraper recipe through its provider seam.""" + return _scraper_database_run(session.runtime, session.uname, bool(session.admin), database_key) + + +# ⚠ DECLARED AFTER THE LITERAL `/connectors/odoo/*` ROUTES. FastAPI matches in declaration order, +# and while none of those ends in `/detail` — so no shadowing is possible today — the ordering is +# the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot +# collide with this three-segment path at all. +@router.get("/connectors/{connector_key}/detail") +def connectors_detail(connector_key: str, session: Session = Depends(require_session)): + """C4 / R26: everything the connector modal shows, for THIS reader, in one read. + + Session-gated exactly like the directory it is opened from: a member may look at what the + workspace is connected to. What a member does NOT get is the tenant-wide credential door + (`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue. + """ + detail = connector_detail( + session.runtime, session.user, session.uname, + bool(session.admin), connector_key) + if detail is None: + raise err(404, "unknown_connector", + "this workspace has no connector with that key") + return detail + + +class _ReplacementConflict(Exception): + """The addressed keychain row changed between replacement preparation and its store update.""" + + +def _encrypted_replacement_payload(kc, label, entry_type, fields, username): + """Validate and encrypt through ``core.keychain.add_entry`` without writing a tenant row. + + ``add_entry`` is the keychain's one validation, preview and Fernet implementation. A tiny + memory-only runtime lets this route reuse it while discarding the temporary identity it mints; + the tenant store sees only the later, single-row payload swap. + """ + class _Stage: + def __init__(self): + self.bucket = {} + + def get(self, name): + return self.bucket if name == kc.KEY else {} + + def update(self, name, fn, flush="sync"): + if name != kc.KEY: + raise RuntimeError("the keychain encryptor wrote an unexpected bucket") + updated = fn(self.bucket) + self.bucket = updated if updated is not None else self.bucket + return self.bucket + + stage = _Stage() + temporary = kc.add_entry(stage, label, entry_type, fields, username) + row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {}) + if not row.get("enc"): + raise RuntimeError("the keychain encryptor returned no encrypted payload") + payload = {"enc": str(row["enc"]), "preview": str(row.get("preview") or ""), + "envelope_version": row["envelope_version"]} + if entry_type == "amazon_sp": + payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD] + return payload + + +def _replace_connector_key(connector_key, entry_id, body, session): + """Replace only one connector entry's declared secrets while preserving its identity. + + The existing row remains untouched through validation, decryption, merge and encryption. The + final store update compares the complete old row before swapping the prepared envelope; + a concurrent rename or credential change therefore conflicts instead of being overwritten. + Scope and owner live in a separate bucket and are never written by this operation. + """ + if not session.admin: + raise err(403, "not_admin", + "only an administrator can replace a workspace connection key") + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + + key = str(connector_key or "").strip() + connector = next((row for row in directory(session.runtime, session.uname, True) + if str(row.get("key") or "") == key), None) + entry_type = _entry_type_for(key) + if connector is None or not entry_type: + raise err(404, "unknown_connector", "this workspace has no connector with that key") + if str(connector.get("manage") or "") not in ("keychain", "keychain_personal"): + raise err(403, "replacement_unavailable", + "this connector has no replaceable keychain credential") + + import routes_keychain as rk # noqa: PLC0415 - circular at import + from core import keychain as kc # noqa: PLC0415 - keep boot lean + visible = rk.visible_entries(session.runtime, session.uname, True) + entry = next((row for row in visible + if str(row.get("id") or "") == str(entry_id) + and str(row.get("type") or "") == entry_type), None) + if entry is None: + # One 404 covers missing, cross-tenant, another user's personal entry and a real id bound + # to another connector. The mismatch itself is information this route does not disclose. + raise err(404, "no_entry", "no such key for this connector") + + secret_specs = _replacement_field_specs(entry_type) + secret_keys = [str(field["key"]) for field in secret_specs if field.get("secret") is True] + optional_keys = {AMAZON_ROTATION_DEADLINE_FIELD} if entry_type == "amazon_sp" else set() + request = body if isinstance(body, dict) else {} + supplied = request.get("fields") + if set(request) != {"fields"} or not isinstance(supplied, dict): + raise err(400, "bad_replacement", + "send only the connector's new secret fields") + if not set(secret_keys) <= set(supplied) <= set(secret_keys) | optional_keys: + raise err(400, "bad_replacement", + "send every new secret field, optional rotation metadata, and no account or server settings") + replacements = {} + for field in secret_keys: + value = supplied.get(field) + if not isinstance(value, str) or not value.strip(): + raise err(400, "bad_replacement", "a replacement secret cannot be blank") + replacements[field] = value.strip() + + # Snapshot before decryption so a concurrent replacement cannot become our baseline later. + from copy import deepcopy + raw_bucket = session.runtime.get(kc.KEY) or {} + original = deepcopy((raw_bucket.get("entries") or {}).get(str(entry_id))) + if not isinstance(original, dict): + raise err(409, "credential_changed", + "the stored credential changed while replacement was prepared. Reload it.") + try: + current_fields = kc.read_fields(session.runtime, entry_id) + except kc.KeychainEnvelopeVersionError: + raise err(409, "credential_envelope_unsupported", + "This credential uses an unsupported encryption envelope version. Nothing was changed.") + except kc.KeychainLocked as exc: + raise err(503, "keychain_locked", + f"the keychain is locked: {exc}. Nothing was changed.") + except Exception: # noqa: BLE001 + raise err(503, "store_unavailable", + "the current key could not be read. Nothing was changed.") + if not isinstance(current_fields, dict) or not current_fields: + raise err(409, "credential_unreadable", + "the stored credential is incomplete. Nothing was changed.") + + merged_fields = {**current_fields, **replacements} + if entry_type == "amazon_sp": + # The previous secret's deadline does not describe its replacement. Absence is unknown. + merged_fields[AMAZON_ROTATION_DEADLINE_FIELD] = supplied.get(AMAZON_ROTATION_DEADLINE_FIELD, "") + try: + encrypted = _encrypted_replacement_payload( + kc, entry.get("label"), entry_type, merged_fields, session.uname) + except kc.KeychainLocked as exc: + raise err(503, "keychain_locked", + f"the keychain is locked: {exc}. Nothing was changed.") + except ValueError as exc: + raise err(400, "bad_replacement", str(exc)) + except Exception: # noqa: BLE001 + raise err(503, "replacement_not_stored", + "the replacement could not be encrypted. The current key is unchanged.") + + def _swap(cur): + entries = cur.get("entries") if isinstance(cur, dict) else None + stored = entries.get(str(entry_id)) if isinstance(entries, dict) else None + if not isinstance(stored, dict) or stored != original: + raise _ReplacementConflict() + replacement = dict(stored) + replacement.update(encrypted) + replacement.pop("probe", None) + # Copy on write: a store failure after this callback must not mutate its cached old row. + return {**cur, "entries": {**entries, str(entry_id): replacement}} + + try: + session.runtime.update(kc.KEY, _swap, flush="sync") + except _ReplacementConflict: + raise err(409, "credential_changed", + "the stored credential changed while replacement was prepared. Reload it.") + except Exception: # noqa: BLE001 + # A remote store can fail after accepting a write. Claiming either outcome would be a lie; + # the safe next act is a fresh detail read, never an automatic retry or delete. + raise err(503, "replacement_unconfirmed", + "the server could not confirm whether the replacement landed. Reload this " + "connector before trying again.") + return {"ok": True} + + +@router.post("/connectors/{connector_key}/keychain/{entry_id}/replace") +def replace_connector_key(connector_key: str, entry_id: str, + body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Admin-only, write-only replacement for one visible connector credential.""" + return _replace_connector_key(connector_key, entry_id, body, session) + + +# ⚠ FIVE SEGMENTS, so it can collide with nothing already declared here: the literal Odoo routes +# are four (`/connectors/odoo/schema/{field_key}`) and `/detail` is three. +@router.patch("/connectors/{connector_key}/databases/{database_key}/name") +def connector_database_name(connector_key: str, database_key: str, + body: dict = Body(default=None), + session: Session = Depends(require_session), + connection_id: str = None): + """W53-T27 / R9 — set or CLEAR this tenant's own name for one of a connector's databases. + + ⛔⛔ IT WRITES THE BUCKET THE NAVIGATION RAIL WRITES, `nav_meta`, IN THE SAME SHAPE AND WITH + THE SAME PER-KEY MERGE. That is the whole ruling: *"the rename database under the Navigation + rail also works, it should then sync to the navigation in the connector too."* A rename made + here is the one the rail shows, and a rename made on the rail is the one this connector's + Databases tab shows, because there is one document and one reader of it + ([[one-question-two-normalizers]]). The key names, the length cap and the merge semantics are + IMPORTED from `routes_nav` rather than copied, so the two doors cannot drift apart. + + ⛔ SO WHY A SECOND DOOR AT ALL? `POST /nav/meta` cannot CLEAR a name: an empty or null `name` + 400s there ("a database needs a name"), so "clearing the override falls back to our shipped + default" has nowhere to happen today. This route is that door. `name: null` and `name: ""` + both clear; any other string sets. The day `/nav/meta` grows the same clearing capability this + becomes a thin alias of it and should be folded in. + + ⛔ AND IT NEVER TOUCHES A TABLE DEFINITION. `user_tables.set_label` would be rewritten by the + next connector re-sync (`odoo_relational.apply_plan` rebuilds definitions), so an override + stored there would vanish on a sync nobody connected to a rename. R9 forbids exactly that, and + the store choice is what makes it impossible rather than merely unlikely. + + THE WALLS, in order, and each one names what it refused: + · the database must be one THIS connector declares (404) - the door is scoped to the card + the reader opened it from, so it cannot be used to rename an unrelated table; + · the key must be a `ut_` key (400 `name_not_allowed`, the same refusal `/nav/meta` gives) - + a built-in module's label is a compiled registry literal; + · the table must EXIST in this workspace (409) - a name for a table nobody can open would + sit in the document naming nothing; + · `may_open` - the creator, an admin, or a share grant, the same predicate `/nav/meta` uses + and the same one `canRename` reports, so the control and the door always agree. + """ + import routes_nav # noqa: PLC0415 - the ONE nav owner + rt = session.runtime + key = str(database_key or "").strip() + if connector_key == "odoo": + key, connection_id, _choice = _odoo_database_target(rt, key, connection_id) + rows = _detail_databases(rt, bool(session.admin), str(connector_key or ""), + uname=session.uname, connection_id=connection_id) + row = next((candidate for candidate in rows + if str(candidate.get("key") or "") == key), None) + if row is None: + raise err(404, "unknown_database", + "this connection does not create a database with that name") + if not key.startswith("ut_"): + raise err(400, "name_not_allowed", + "only a database this connection created can be renamed. This one's name comes " + "from the module registry") + import core.user_tables as user_tables # noqa: PLC0415 - lazy, as elsewhere + definitions = user_tables.lend_defs(rt) + if not user_tables.get(key, st=definitions): + raise err(409, "database_not_created", + "this connection has not created that database in this workspace yet, so there " + "is nothing to rename") + if not user_tables.may_open(key, session.uname, session.admin, st=definitions): + raise err(403, "forbidden", "that database belongs to another user") + + body = body if isinstance(body, dict) else {} + if "name" not in body: + raise err(400, "bad_request", + "no name was sent. Send a name to set one, or null to go back to the default") + # The same normalisation `/nav/meta` applies: whitespace collapsed, capped at its own limit. + # A blank result CLEARS, which is the half that route refuses and this one exists for. + name = " ".join(str(body.get("name") or "").split())[:routes_nav._MAX_NAV_NAME] + + if not rt.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + # `routes_nav.save_nav_meta._up`, clause for clause: an entry left empty is REMOVED rather + # than stored as `{}`, and an icon set from the rail survives a rename made here. + data = data if isinstance(data, dict) else {} + entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} + if name: + entry["name"] = name + else: + entry.pop("name", None) + if entry: + data[key] = entry + else: + data.pop(key, None) + return data + + try: + rt.update(routes_nav._NAV_META_KEY, _up) # noqa: SLF001 - the nav's own key + except Exception: # noqa: BLE001 + raise err(503, "store_unavailable", + "the change was not saved: the store refused the write.") + + # Read BACK through the same resolver the tab reads, so the answer cannot be the echo of a + # write that landed somewhere else. + default_label = str(row.get("defaultLabel") or "") + override = schema_overlay.override_name(_database_name_overrides(rt).get(key)) + return {"key": key, + "identity": row.get("identity"), + # ⛔ SERVED, NEVER ACCEPTED. `native` is the vendor's identifier; this door reads + # `name` and nothing else off the body (C2: `native` is never editable). + "native": str(row.get("native") or ""), + "defaultLabel": default_label, + "override": override, + "label": schema_overlay.database_label(default_label, override)} + + +# ⛔⛔ W49-T16 — THE MOUNT. `dependents_router` DECLARES NO PREFIX OF ITS OWN, so including it in +# this router (which carries `prefix="/api/v1"`) yields exactly `/api/v1/dependents`. Declaring +# the prefix on both would serve `/api/v1/api/v1/dependents` — the mirror image of the wave-23 +# defect this file's own header records, where the MISSING prefix 404'd a shipped route with +# every gate green. `verify_shared_views.py` asserts the assembled path off this very object +# rather than trusting the arithmetic. +router.include_router(dependents_router) + +# ⛔⛔ W49-T19 — THE SECOND MOUNT, IDENTICAL IN SHAPE. `monid_router` is a bare `APIRouter()`, +# so this yields `/api/v1/monid/amazon-reviews/meta`, `/api/v1/monid/amazon-reviews/run` and +# `/api/v1/monid/spend`. +# ⚠ THE ASSEMBLED PATH IS NOT VISIBLE IN `router.routes`. FastAPI 0.139.2 leaves an +# `_IncludedRouter` placeholder there, so a walk of that list finds nothing and a gate written +# against it would pass on a router that serves the paths twice-prefixed. `app.openapi()["paths"]` +# is the reliable read, and it is what `verify_meta.py::section_monid_mount` uses. +router.include_router(monid_router)