diff --git "a/api/routes_connectors.py" "b/api/routes_connectors.py"
--- "a/api/routes_connectors.py"
+++ "b/api/routes_connectors.py"
@@ -1,4558 +1,4558 @@
-"""THE CONNECTORS DIRECTORY (wave 23, contract C11 / owner ruling R8).
-
-⭐ WHAT THE OWNER ASKED FOR, verbatim: *"Build a connector module where it stores everything… BUT
-if its not yet integrated make it faded grey or something so i can use it as a to-do list for
-integrations."* So this endpoint deliberately answers with things we have NOT built. A directory
-that listed only working connectors would be honest about each row and dishonest about the shape
-of the product — the owner wants to see the map, including the empty parts.
-
-⛔ THE ROW SET IS COMPOSED HERE, ONCE, FROM THE REGISTRIES THAT ALREADY EXIST — the OAuth
-provider registry (`oauth_connect.PROVIDERS`), the keychain's entry types, the automation source
-registry, and the env-configured built-ins. It is NOT a hand-written list of cards. That matters
-because the alternative rots in a specific way: a provider gets wired up somewhere and the
-directory keeps calling it "planned" for a wave or two, which is exactly the misinformation this
-surface exists to prevent. The only hand-written part is `PLANNED` — the things that have no
-registry entry anywhere yet, which is precisely what "not yet integrated" means.
-
-⚠ STATE IS PER-READER. `connected` is a fact about THIS user for a per-user credential (OAuth
-slots are per-user by R7) and about the tenant for a shared one (Odoo, the Scraper). A directory
-that reported another user's Gmail as "connected" would be both wrong and a small disclosure.
-"""
-from __future__ import annotations
-
-import os
-import ssl
-import time
-import threading
-import xmlrpc.client
-from concurrent.futures import ThreadPoolExecutor
-from datetime import datetime, timezone
-
-from fastapi import APIRouter, Body, Depends
-
-import oauth_connect
-from deps import Session, err, require_session, running_main
-#: ⭐⭐ W49-T18 / R27 / C4 — THE TAXONOMY LIVES IN THE DATA LAYER, NOT HERE. `core` is already on
-#: the path by the time this module loads (`deps` imports three of its modules), so this is the
-#: same import shape the rest of the API uses. It is in `platform/core/` rather than beside this
-#: file because the crosswalk is authored CONTENT about connectors, not routing: a second reader
-#: of it (a gate, a future module page) must not have to import FastAPI to ask what category a
-#: connector is in.
-from core import connector_directory
-from core import connector_sync
-from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD
-from harness import amazon_client
-#: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default,
-#: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already
-#: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module
-#: holds the tenant's store handle and does the reading; nothing about what a blank override MEANS
-#: is decided here.
-from core import schema_overlay
-#: ⛔ W32-T10 — IMPORTED, NEVER RE-DERIVED. "Does the environment serve THIS tenant's Odoo?" is
-#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
-#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
-#: second correct one here would leave the same two-copies shape that produced the defect.
-from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available
-#: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a
-#: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any
-#: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router`
-#: line for exactly this reason — see `routes_charts.py`'s header). This file is already
-#: included by `main.py`, so mounting the child here puts `/api/v1/dependents` on the app with
-#: no cross-fence edit. The `include_router` call is at the BOTTOM of this file; the import is
-#: here so the module's dependencies read in one place.
-from routes_dependents import router as dependents_router
-
-#: ⭐⭐ W49-T19 · R31 / C5 — THE MONID SLICE, MOUNTED THE SAME WAY AND FOR THE SAME REASON.
-#: `routes_monid.py` carries the Amazon reviews pull. It declares NO prefix of its own; the
-#: `include_router` call is at the BOTTOM of this file beside the dependents one, so there is a
-#: single place to read what this router actually serves.
-#: ⛔ NEVER FROM `main.py` (C5). The connectors directory owns its children.
-from routes_monid import router as monid_router
-
-# ⛔ THE PREFIX IS LOAD-BEARING AND IT WAS MISSING (found at the wave-23 close-out audit).
-# This router declared a bare `APIRouter()`, so its one path was served at `/connectors/directory`
-# while `ConnectorsPage.tsx` fetches `${API_V1}/connectors/directory` — the directory would have
-# 404'd in production with every gate green. Every other router in this package self-declares
-# `prefix="/api/v1"`; this was the only one that did not.
-#
-# ⚠ AND THE WIRING GATE COULD NOT SEE IT. `verify_wiring`'s W23-W4 row matches the SERVER side on
-# the substring `connectors/directory` and the CLIENT side on `${API_V1}/connectors/directory` —
-# both matched, both sides "wired", prefixes disagreeing. A cross-fence row that greps each side
-# independently proves the two files mention the same path SEGMENT, never that they agree on the
-# same URL. `verify_api` now asserts the prefix on every mounted router instead.
-router = APIRouter(prefix="/api/v1")
-
-CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
-SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps"))
-SCRAPER_RECIPE_MAX_ROWS = 30
-SCRAPER_RUN_HISTORY_LIMIT = 20
-DATABASE_SYNC_STATES = frozenset(
- ("never_run", "queued", "running", "succeeded", "partial", "failed", "disabled")
-)
-_DATABASE_SAFE_ERRORS = {
- "tls_certificate_invalid": ("Odoo TLS certificate verification failed because the certificate "
- "is expired or untrusted. No source data was accepted."),
- "source_read_refused": "Odoo refused the configured account's read access.",
- "source_auth_failed": "Odoo rejected the configured read-only credential.",
- "source_unavailable": "Odoo could not be reached for this read-only sync.",
- "source_read_failed": "Odoo source validation failed.",
- "mirror_pull_failed": "The Odoo mirror pull failed.",
- "materialization_failed": "The database could not be materialized.",
- "reconciliation_failed": "The database reconciliation failed.",
- "freshness_unconfirmed": "Freshness could not be confirmed for this database.",
- "sync_status_unavailable": "Sync status is unavailable for this database.",
- "sync_stalled": "This database sync is stalled.",
- "sync_state_unreadable": "This database's saved sync state could not be read.",
- "maps_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
- "scraper_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
- "maps_unbounded_request": "Google Maps discovery requires an explicit bounded result cap.",
- "maps_invalid_search": "Google Maps discovery needs a valid category or search.",
- "maps_vendor_error": "The Maps provider could not complete this run.",
- "maps_schema_drift": "The Maps provider returned an unreadable result shape.",
- "scraper_run_failed": "The provider could not complete this run. No result was written.",
- "provider_result_failed": "The provider returned no usable result for this database.",
- "scraper_result_not_saved": "The provider answered, but the result could not be saved.",
-}
-
-# T11 established the saved database contract; T12's provider normalises every Apify answer into
-# exactly this schema. Vendor field names never reach a saved View or downstream automation.
-_MAPS_RECIPE_FIELDS = [
- {"key": "place_id", "label": "Place id", "type": "text", "source": "automation"},
- {"key": "name", "label": "Name", "type": "text", "source": "automation"},
- {"key": "category", "label": "Category", "type": "text", "source": "automation"},
- {"key": "address", "label": "Address", "type": "text", "source": "automation"},
- {"key": "city", "label": "City", "type": "text", "source": "automation"},
- {"key": "region", "label": "Region", "type": "text", "source": "automation"},
- {"key": "postal_code", "label": "Postal code", "type": "text", "source": "automation"},
- {"key": "country", "label": "Country", "type": "text", "source": "automation"},
- {"key": "phone", "label": "Phone", "type": "phone", "source": "automation"},
- {"key": "website", "label": "Website", "type": "url", "source": "automation"},
- # Coordinates remain text because `number` is not a user-table field kind; the Map binding
- # parses the canonical scalar value. Declaring an unsupported kind would silently drop both
- # columns at database creation.
- {"key": "latitude", "label": "Latitude", "type": "text", "source": "automation"},
- {"key": "longitude", "label": "Longitude", "type": "text", "source": "automation"},
- {"key": "rating", "label": "Rating", "type": "rating", "source": "automation"},
- {"key": "reviews", "label": "Reviews", "type": "int", "source": "automation"},
- {"key": "source_timestamp", "label": "Source timestamp", "type": "date",
- "source": "automation"},
- {"key": "source_payload", "label": "Source data", "type": "json", "source": "automation"},
-]
-
-
-# Measured reads take 1.34 s cold, 0.01 s warm, and above 8 s under boot contention.
-# A thread pool deadline was tried and produced three consecutive 30 s 504s because
-# core/store.py uses reentrant locks whose ownership does not survive a thread
-# boundary while the request thread waits on the result. Keep the work on the
-# request thread. This budget is therefore cooperative: it bounds a read whose
-# phases are each finite and cannot interrupt a single call that is already wedged.
-class _ConnectorDetailBudget:
- def __init__(self):
- self._started = time.monotonic()
-
- def check(self):
- if time.monotonic() - self._started >= CONNECTOR_DETAIL_DEADLINE_SECONDS:
- raise err(504, "connector_detail_timeout",
- f"The connector store did not answer within {CONNECTOR_DETAIL_DEADLINE_SECONDS} "
- "seconds, so the read was stopped.")
-
-
-#: The to-do list (R8). Order is the owner's reading order, not alphabetical: the things most
-#: likely to come next first. `queue` mirrors the standing provider queue in wave22-split C5-A2
-#: so the two cannot drift into disagreeing about what is coming.
-PLANNED = [
- # ⭐⭐ WAVE 31 · R7 — SIX ROWS LEFT THIS LIST FOR `NANGO_PROVIDERS` BELOW, and that move is the
- # ticket. `slack`, `airtable`, `outlook`, `quickbooks`, `xero` and `hubspot` are providers
- # Nango can AUTHENTICATE today, so leaving them here as "not built yet" is the same
- # misinformation this module's header warns about — one wave later and about a different
- # mechanism. What they still cannot do is INGEST, and that is what their own cards now say.
- # ⚠ WAVE 32 · ITEM 4 — FIVE, NOW. The Airtable row is deleted from the directory entirely
- # (the reason is in the provider registry below). It is still NAMED in the sentence above
- # because that sentence is the record of where those six rows WENT, and a record that quietly
- # drops one of its own entries is worse than no record.
- # ⚠ W32-T14 — THIS SENTENCE STOPPED BEING TRUE IN THE SAME CHANGE THAT HID THE GOOGLE CARD.
- # It read *"the Google connection already exists"*, which was the one row on this page that
- # made a claim about ANOTHER row — so R6 hiding that other row turned a to-do entry into a
- # false statement. Exactly the rot this module's header names, arriving sideways.
- # ⛔ THE THREE `google_*` ROWS THEMSELVES STAY. Item 12 is about the connectors that offered a
- # button and answered `503 oauth_unavailable`; these are `planned` to-do entries with no door,
- # no state and no promise — which is precisely the map R8 asked this page to be.
- {"key": "google_sheets", "label": "Google Sheets", "kind": "oauth",
- "desc": "Read a sheet as a database."},
- {"key": "google_drive", "label": "Google Drive", "kind": "oauth",
- "desc": "Attach files from Drive to records."},
- {"key": "google_calendar", "label": "Google Calendar", "kind": "oauth",
- "desc": "Turn dated records into calendar entries."},
- # ⭐⭐ WAVE 30 · R4 — TWO ROWS WERE DELETED HERE, AND THEY WERE MISINFORMATION, not clutter.
- #
- # {"key": "tiktok", "label": "TikTok", ... "The second social source beside Instagram."}
- # {"key": "meta", "label": "WhatsApp and Instagram (Meta)", ... "Blocked on Meta business
- # verification; Instagram data arrives through the Scraper connector"}
- #
- # ⛔ THE TIKTOK ROW SAID "not built yet" ABOUT SOMETHING THAT SHIPPED IN WAVE 29 — which is
- # precisely the rot this module's header warns about: *"a provider gets wired up somewhere and
- # the directory keeps calling it planned for a wave or two, which is exactly the
- # misinformation this surface exists to prevent."* TikTok is a PLATFORM of the Scraper
- # connector now (R3/R4), and it appears in the Scraper card's own drill below.
- # ⚠ THE META ROW IS A DIFFERENT CASE and is deleted for a different reason: what it described
- # (Instagram arriving another way) is now visible ON the Scraper card, so keeping it would put
- # Instagram on this page twice, once as working and once as blocked. The Meta business
- # verification blocker itself is not lost — it is D-86's subject and lives in
- # `AutoWiki/research/ads-connectors-owner-actions.md`, where an owner action belongs.
-]
-
-#: ⭐ WAVE 30 · R4 — THE SCRAPER'S PLATFORMS, and the whole shape of the ruling in one list.
-#: The owner: *"the damn Tiktok and Instagram belongs in the same bucket … Only when I click
-#: 'Scraper' … would I see the option to choose either Instagram OR TikTok."* So the directory
-#: mirrors the automation menus exactly: ONE card for the capability, and the platforms are behind
-#: it rather than beside it.
-#: ⚠ These are not connectors and must never become rows: they have no state or manage door of
-#: their own. Instagram and TikTok share the Scraper credential; Maps uses the Apify credential
-#: through the same governed provider registry.
-SCRAPER_PLATFORMS = [
- {"key": "instagram", "label": "Instagram",
- "desc": "Profiles, posts and comments, by search or by handle."},
- {"key": "tiktok", "label": "TikTok",
- "desc": "The same three, on TikTok's own schema."},
- {"key": "google_maps", "label": "Google Maps",
- "desc": "Places by category or search and geography, through a bounded Apify run."},
-]
-
-#: Token connectors an admin can wire up TODAY by pasting a key — no OAuth client, no app review.
-TOKEN_CONNECTORS = [
- {"key": "stripe", "label": "Stripe", "entry": "stripe",
- "desc": "Payments and payouts against your customer records.",
- "hint": "Create a restricted key with read access in the Stripe dashboard.",
- "note": "Storing the key connects the account; reading its data lands in a later release."},
- {"key": "shopify", "label": "Shopify", "entry": "shopify",
- "desc": "Orders and products from a Shopify storefront.",
- "hint": "Create a custom app in your store admin and copy its Admin API token.",
- "note": "Storing the key connects the account; reading its data lands in a later release."},
- # ⭐⭐ WAVE 31 · R2/R8 — META ADS IS A TOKEN CONNECTOR, AND IT IS THE ONE NEW CARD THAT IS
- # ACTIONABLE TODAY. R8 put the schema work on a pasted token deliberately — *"so that you can
- # also explore the Schema, without any user logging into it with Facebook yet"* — and T41
- # MEASURED that the delegated-OAuth door is not available anyway: the Nango account holds one
- # integration and it is a GitHub sample, so there is no Facebook integration to request
- # `ads_read` from. This row is therefore the real door and the Nango one is honest about not
- # being ready. ⚠ Its `entry` must match `keychain.ENTRY_TYPES`, or storing 400s.
- {"key": "meta_ads", "label": "Meta Ads", "entry": "meta_ads",
- "desc": "Campaigns, ad sets, ads, creatives and daily performance from your Meta ad account.",
- "hint": "Paste a Marketing API access token with `ads_read`; the connector verifies it "
- "against your ad accounts before storing it.",
- "note": "The token connects the account and is verified live. The spawned databases "
- "(285 measured columns) land with the Meta grids."},
- # ⭐⭐ AMAZON SP-API. FIVE values rather than one token, because Login With Amazon
- # exchanges a refresh token for a short-lived bearer on every call, and the seller and
- # marketplace ids scope the reads. ⚠ `entry` must match `keychain.ENTRY_TYPES` or storing
- # 400s, and `refresh_token` had to join `_PREVIEW_FIELDS` in the same change or the row
- # renders a blank preview with nothing raising (the wave-31 defect, second edition).
- # ⛔ Read-only by construction: `connectors_amazon._request` refuses any method but GET
- # before it builds a request, so no route, payload or UI can reach a write.
- {"key": "amazon_sp", "label": "Amazon", "entry": "amazon_sp",
- "desc": "Orders, settlements and fees, FBA and AWD inventory, listings and Buy Box "
- "pricing from your Amazon selling account.",
- "hint": "Create a Production app client in the Solution Provider Portal, self authorize "
- "it, then paste client_id, client_secret, refresh_token, seller_id and "
- "marketplace_id.",
- "note": "The credential is verified live against your marketplace participations before "
- "it is stored. Bulk reads use the Reports API, so a backfill is one document "
- "rather than a page loop."},
-]
-
-#: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
-#: other potential connection as possible, such as slack etc."*
-#:
-#: ⛔ ADDING A PROVIDER HERE IS THE WHOLE FEATURE — one row, no code — which is the claim T43 has
-#: to make demonstrable rather than assert. The row carries only what a CARD needs; the auth flow
-#: is Nango's and is identical for every one of them.
-#:
-#: ⛔⛔ R7's SECOND SENTENCE IS THE HARD PART AND IT IS ENFORCED IN `_nango_rows` BELOW: *"Nango
-#: buys CONNECT, not INGEST — every lit provider states on its own card what it can do today, or
-#: we ship a directory of buttons that lead nowhere."* MEASURED for T42: the owner's Nango account
-#: holds **1 integration (`github-getting-started`) and 0 connections**, so there is no sync on it
-#: to inspect and NO provider here has a data layer. Every card says so, in its own words, and
-#: none of them offers a button.
-NANGO_PROVIDERS = [
- {"key": "slack", "label": "Slack", "provider": "slack",
- "desc": "Send messages into a channel when an automation reaches a step."},
- {"key": "hubspot", "label": "HubSpot", "provider": "hubspot",
- "desc": "Deals and contacts beside the Odoo customer book."},
- # ⭐⭐ WAVE 32 · OWNER ITEM 4 / R3 — THE AIRTABLE ROW IS DELETED. It read:
- #
- # {"key": "airtable", "label": "Airtable", "provider": "airtable",
- # "desc": "Import an existing base as a database."}
- #
- # The owner asked for it to go, and R3 says how: **de-brand, don't revoke.** So the ROW goes
- # and the transport does not — Nango can still authenticate `airtable` if a future ruling
- # wants it back, and nothing in `oauth_connect` changed.
- # ⚠ THE REASON IS NOT THAT IT WAS BROKEN. This connector is the one competitor whose model
- # this product is built against, and offering to import a base is a promise nobody here is
- # ready to keep. Every DESIGN-PARITY comment naming Airtable stays exactly where it is
- # (~30 of them across the grid and automation clients): those are how we know what good looks
- # like, and deleting them would cost the reasoning while saving nothing a user can see.
- {"key": "quickbooks", "label": "QuickBooks", "provider": "quickbooks",
- "desc": "Accounting actuals beside the Odoo sales data."},
- {"key": "xero", "label": "Xero", "provider": "xero",
- "desc": "Accounting actuals for tenants who do not use QuickBooks."},
- {"key": "outlook", "label": "Microsoft Outlook", "provider": "outlook",
- "desc": "The Outlook half of the email trigger."},
-]
-
-
-#: ⭐⭐ W31-T44 / OWNER ITEM 6b — WHICH `ut_*` PREFIXES BELONG TO A CONNECTOR (contract C2).
-#:
-#: B's `user_tables.register_connected_prefix` (W31-T25) replaced a hardcoded `ut_odoo_` literal
-#: with a registered SET, and this is where the connector layer declares its half. It lives beside
-#: the other connector registries deliberately: "which databases does this connector own" is the
-#: same kind of fact as "what is this connector called", and putting it anywhere else would make
-#: the answer to one question live in two files.
-#:
-#: ⛔ REGISTERING A PREFIX DOES NOT MAKE A TABLE CONNECTED, and B posted this as the trap: the
-#: third leg of `is_connected` still demands `source == AUTOMATION_SOURCE` **and**
-#: `recordMode == AUTOMATION_RECORD_MODE`. A person who names a database "Meta ads spend" gets a
-#: `ut_meta_…` key too, and exempting it from `MAX_ROWS` on the strength of its NAME is the silent
-#: hole the stamp closes. The spawn stamps; this only says which family may qualify.
-#: ⚠ `ut_odoo_` is SEEDED in `user_tables` rather than registered here, also deliberately — the
-#: convention leg has to answer in a worker process where nothing registered anything.
-#: ⛔⛔ `ut_amzn_` JOINED IN WAVE 52, AND OMITTING IT WOULD HAVE BROKEN STANDING RULE 1 SILENTLY.
-#: The rule is that a connector's records carry NO cap: `records_mutable(key)` is False for them,
-#: so `MAX_ROWS` does not apply. But that answer is reached through `is_connected`, whose third leg
-#: is this prefix set — so an Amazon table missing from here does not raise, does not warn, and
-#: simply falls through to the 60,000-row ceiling. A seller with more than 60,000 order lines would
-#: get a truncated grid and no error, which is precisely the silent cap the rule forbids. **A new
-#: connector joins this dict in the same change that spawns its tables, or its no-cap guarantee is
-#: decorative.**
-CONNECTED_PREFIXES = {
- "odoo": "ut_odoo_",
- "meta_ads": "ut_meta_",
- "amazon_sp": "ut_amzn_",
-}
-
-
-def _register_prefixes():
- """Declare `CONNECTED_PREFIXES` to the store layer. Called at import — this module is mounted
- by `main.py`, so any process that can serve the directory has also made the declaration.
-
- Never raises: a store layer that cannot take the declaration must not stop the API booting,
- and the consequence of a missed registration is a row CAP (R6's subject), not a wrong answer.
- """
- try:
- import core.user_tables as ut
- return ut.register_connected_prefix(*sorted(CONNECTED_PREFIXES.values()))
- except Exception: # noqa: BLE001
- return frozenset()
-
-
-_register_prefixes()
-
-
-def _register_probers():
- """Let each connector answer `keychain.test_entry` for its OWN type — the SLOT, filled from
- the layer that knows how to ask.
-
- ⛔ THIS DIRECTION IS THE POINT. `core.keychain` must not import a connector module: `core`
- never imports up (ARCHITECTURE.md / CLAUDE.md), and the tempting shortcut — a `sys.path`
- insert and an `import connectors_meta` from inside `core` — is an architecture inversion that
- no gate was watching for. Same idiom as `datastore.set_paused_probe`, one layer down, for the
- same reason.
-
- Import-time and fail-quiet: an unfillable slot costs a liveness CHECK, never the ability to
- store a credential and never the API's ability to boot.
- """
- ok = False
- for mod in ("connectors_meta", "connectors_amazon"):
- try:
- ok = __import__(mod).register() or ok
- except Exception: # noqa: BLE001
- continue
- return ok
-
-
-_register_probers()
-
-
-# W48C-T08 / C6 — the graph is a reader-scoped DTO over tables that exist in THIS workspace,
-# never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family;
-# `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter.
-def _graph_refusal(connection_key, reason, status="refused"):
- return {"connectionKey": str(connection_key or ""), "status": status,
- "reason": str(reason or ""), "tables": [], "relationships": []}
-
-
-def _table_connection_key(table_key, definition):
- """The owning connection declared by this actual table, or ``""`` when it is not one."""
- definition = definition if isinstance(definition, dict) else {}
- # Future connector materialisers may stamp their directory key explicitly. It is accepted
- # only as a key, never as table/field data, so no provider can manufacture a graph template.
- for name in ("connectionKey", "connectorKey", "connector_key", "connector"):
- candidate = definition.get(name)
- if isinstance(candidate, str) and candidate.strip():
- return candidate.strip()
- key = str(table_key or "")
- for connection_key, prefix in CONNECTED_PREFIXES.items():
- if key.startswith(prefix):
- return connection_key
- return ""
-
-
-def _module_connection_key(subject):
- """Which connected card a REGISTRY MODULE's `subject` belongs to, or `""`.
-
- ⭐⭐ W53-T30 — DERIVED FROM `CONNECTED_PREFIXES`, NEVER A SECOND HAND-TYPED PAIR LIST. A
- module's subject is namespaced by the system the records come from (`odoo:res.partner`,
- `odoo:product.product` in `core/registry.py`), and the connector families this file already
- knows carry that same namespace inside their table prefix (`odoo` inside `ut_odoo_`). So the
- two are matched rather than mapped: a connector added to `CONNECTED_PREFIXES` in the change
- that spawns its tables gets this for free, and a module whose namespace names no connected
- family here simply does not appear on anybody's graph.
-
- ⛔ THE SUBJECT, NOT THE `source` STRING. `source` is display copy ("Odoo") and a rename of it
- is a copy edit; `subject` is the identity `core.registry` already refuses to let two rows
- claim at once, which is what makes it safe to key a graph off.
- """
- namespace = str(subject or "").split(":", 1)[0].strip()
- if not namespace:
- return ""
- for connection_key, prefix in CONNECTED_PREFIXES.items():
- parts = str(prefix or "").strip("_").split("_")
- if len(parts) > 1 and namespace == "_".join(parts[1:]):
- return connection_key
- return ""
-
-
-def _module_graph_entries(rt, user, lent, connected):
- """The REGISTRY MODULES this reader may see on a connected card's graph, per connection.
-
- Returns `{connection key: {module key: {"dto", "visible"}}}` in exactly the shape the `ut_`
- loop below builds, so the edge pass treats both kinds of node identically.
-
- ⭐⭐ W53-T30 — THE WHOLE TICKET. `relationship_graphs` walks `user_tables.all_defs`, and
- `customer_data` / `product_data` are COMPILED REGISTRY ROWS rather than stored definitions, so
- that loop could never reach them however permissive it became. They were absent from the Odoo
- diagram for that reason, not for a permission one. ⛔ The retired `ut_odoo_customers` /
- `ut_odoo_products` keys are NOT the subject and must not be revived: W33-T44 / A2 retired
- them, `odoo_relational.RETIRED_KEYS` pops them out of every tenant's document on rebuild, and
- their subjects are claimed by these two registry rows, which is now the only claim on each.
-
- ⛔⛔ THREE PREDICATES, THE SAME THREE `routes_tables._module_link_targets` USES ON THESE EXACT
- TWO KEYS, and none of them is `is_admin`. A blanket administrator check here would silently
- drop two nodes from a member's diagram for no stated reason, while every other node on the
- same picture is walled per table:
- 1. `perms.tenant_has_module` — is this database part of THIS tenant's world at all? Every
- provisioned tenant but #0 carries a restricted `modules:` list, so without it another
- company's admin is shown a database their workspace does not have.
- 2. `perm_scope.nav_may_open` — ADMISSION, the predicate `deps.py`'s module gate itself asks.
- `may_read` alone would draw a node whose door then 403s.
- 3. `perm_scope.may_read` — the READ wall, the same one the `ut_` loop applies per table.
-
- ⛔ `user_tables.may_open` IS NOT IN THAT LIST AND ITS ABSENCE IS DELIBERATE: it answers about
- a STORED definition's creator and share grants, and a registry module has no such row, so it
- refuses every reader including the tenant's owner.
-
- ⭐ THE FIELDS ARE `automation_engine._module_table`'s, WHICH IS THE ONE COMPOSER OF A MODULE'S
- FIELD CONTRACT (`aios_grid` plus `odoo_relational.module_link_fields`). Restating either half
- here is how this diagram would start disagreeing with the engine that resolves the very links
- it draws. `want_rows=False`, so no pool is built and no store is written.
-
- ⚠ AND THE FIELD LIST IS THE VIEWER'S: `visible_fields` strips the hidden closure, so a column
- an administrator hid from this account is neither drawn on the card nor usable as an edge
- endpoint. Fail-closed everywhere: a wall that cannot be answered drops the node.
- """
- try:
- import automation_engine # noqa: PLC0415 - lazy, as elsewhere
- import core.perm_scope as perm_scope
- import core.perms as perms
- import core.registry as registry
- import core.user_tables as user_tables
- except Exception: # noqa: BLE001
- return {}
- out = {}
- memo = {}
- for module_key in getattr(user_tables, "LINKABLE_MODULE_KEYS", ()):
- key = str(module_key or "")
- row = registry.BY_KEY.get(key) or {}
- connection_key = _module_connection_key(row.get("subject"))
- if not key or connection_key not in connected:
- continue
- try:
- if not perms.tenant_has_module(rt, key):
- continue
- if not perm_scope.nav_may_open(user, key, st=lent):
- continue
- if not perm_scope.may_read(user, key, st=lent):
- continue
- # ⛔ `_module_table`, THE ENGINE'S OWN COMPOSER (`# noqa: SLF001` for the same reason
- # `_database_name_overrides` calls `routes_nav._read_nav_meta`: one reader for one
- # fact beats a correct-looking second copy).
- table, _limit = automation_engine._module_table(rt, key, memo) # noqa: SLF001
- declared = [dict(field) for field in ((table or {}).get("fields") or ())
- if isinstance(field, dict)]
- if not declared:
- continue
- visible = perm_scope.visible_fields(declared, user, key, st=lent)
- except Exception: # noqa: BLE001
- continue
- fields = []
- for field in visible:
- field_key = str(field.get("key") or "").strip()
- if field_key:
- fields.append({"key": field_key,
- "label": str(field.get("label") or field_key).strip() or field_key})
- if not fields:
- continue
- # ⚠ THE REGISTRY'S OWN LABEL, READ RATHER THAN RETYPED. "Odoo customers" and "Odoo
- # products" are `core/registry.py`'s `label` values (renamed there once already, W33-T42),
- # and a literal copy here is a second authority that goes stale the day the row is
- # renamed. There is no tenant override to merge in: `routes_nav` refuses a name for any
- # key that does not start with `ut_`, so a registry module cannot be renamed at all.
- label = str(row.get("label") or key).strip() or key
- out.setdefault(connection_key, {})[key] = {
- "dto": {"key": key, "label": label, "fields": fields},
- "visible": visible,
- }
- return out
-
-
-def relationship_graphs(rows, user, rt, uname, is_admin=False):
- """Actual, permitted tables and same-connection declared links for directory rows.
-
- There is deliberately no provider fallback. A connected credential with no materialised
- database gets an explicit unavailable envelope; a non-connected card gets an explicit
- refusal. The client can therefore distinguish "nothing exists for this selection" from a
- plausible-looking sample schema.
- """
- row_by_key = {str(row.get("key") or ""): row for row in (rows or [])
- if isinstance(row, dict) and str(row.get("key") or "")}
- graphs = {}
- connected = set()
- for connection_key, row in row_by_key.items():
- if row.get("state") == "connected":
- connected.add(connection_key)
- graphs[connection_key] = _graph_refusal(
- connection_key,
- "No permitted tables are materialised for this connection. A provider template is not shown.",
- status="unavailable")
- else:
- graphs[connection_key] = _graph_refusal(
- connection_key,
- "Connect this provider before viewing its actual relationship graph.")
- if not connected:
- return graphs
-
- import core.perm_scope as perm_scope
- import core.user_tables as user_tables
- try:
- definitions = user_tables.all_defs(st=rt)
- lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
- except Exception: # noqa: BLE001
- for connection_key in connected:
- graphs[connection_key] = _graph_refusal(
- connection_key,
- "The workspace tables could not be read, so this relationship graph is unavailable.",
- status="unavailable")
- return graphs
-
- tables_by_connection = {connection_key: {} for connection_key in connected}
- for table_key, definition in (definitions or {}).items():
- connection_key = _table_connection_key(table_key, definition)
- if connection_key not in connected:
- continue
- try:
- # Prefixes are not enough: a table must carry the connector materialisation stamps
- # before it can describe a connector graph.
- if user_tables.is_managed_system_table(table_key, st=lent, definition=definition):
- continue
- if not user_tables.is_connected(table_key, st=lent):
- continue
- if not user_tables.may_open(table_key, uname, is_admin, st=lent):
- continue
- if not perm_scope.may_read(user, table_key, st=lent):
- continue
- visible = perm_scope.visible_fields(
- [dict(field) for field in (definition.get("fields") or ())
- if isinstance(field, dict)],
- user, table_key, st=lent)
- except Exception: # noqa: BLE001
- # A table whose permission proof cannot be completed does not become a partial leak.
- continue
- fields = []
- for field in visible:
- key = str(field.get("key") or "").strip()
- if key:
- fields.append({"key": key,
- "label": str(field.get("label") or key).strip() or key})
- tables_by_connection[connection_key][str(table_key)] = {
- "dto": {"key": str(table_key),
- "label": str(definition.get("label") or table_key).strip() or str(table_key),
- "fields": fields},
- "visible": visible,
- }
-
- # ⭐⭐ W53-T30 — AND THE REGISTRY MODULES THIS CONNECTION STANDS BEHIND. They are not stored
- # definitions, so the loop above cannot reach them; they are permitted by their own three
- # predicates and then join the SAME edge pass, which is what makes an already-declared link
- # between a `ut_` table and a module resolvable at both ends for the first time.
- for connection_key, entries in _module_graph_entries(rt, user, lent, connected).items():
- tables_by_connection.setdefault(connection_key, {}).update(entries)
-
- for connection_key, entries in tables_by_connection.items():
- if not entries:
- continue
- table_keys = set(entries)
- visible_keys = {
- table_key: {str(field.get("key") or "").strip()
- for field in entry["visible"] if isinstance(field, dict)}
- for table_key, entry in entries.items()
- }
- relationships = []
- seen = set()
- for source_table, entry in entries.items():
- for field in entry["visible"]:
- if not isinstance(field, dict) or field.get("type") != "link":
- continue
- link = field.get("link")
- source_field = str(field.get("key") or "").strip()
- target_table = str((link or {}).get("table") or "").strip() if isinstance(link, dict) else ""
- target_field = str((link or {}).get("on") or (link or {}).get("inverse") or "").strip() if isinstance(link, dict) else ""
- # An outside, unavailable, or field-hidden endpoint is not a relationship on the
- # selected connection. Omit it rather than exposing the target through an edge.
- if (not source_field or target_table not in table_keys or not target_field
- or target_field not in visible_keys.get(target_table, set())):
- continue
- signature = (source_table, source_field, target_table, target_field)
- if signature in seen:
- continue
- seen.add(signature)
- relationships.append({
- "from": {"table": source_table, "field": source_field},
- "to": {"table": target_table, "field": target_field},
- "label": str(field.get("label") or source_field).strip() or source_field,
- })
- # ⭐ W53-T30 / C2 — THE SPEC'S OWN DECLARED RELATIONS, held to the same two proofs as the
- # field bags above: both endpoints must be nodes on THIS graph and both named columns must
- # be visible to THIS reader. `{}` until lane D's `core.connector_spec` lands, and the
- # `seen` signature is shared so a relation declared in both places is drawn once.
- label_of = {(table_key, str(field.get("key") or "")): str(field.get("label") or "")
- for table_key, entry in entries.items()
- for field in entry["dto"]["fields"]}
- for source_table, declared in _connector_spec_links(connection_key).items():
- if source_table not in table_keys:
- continue
- for source_field, target_table, target_field in declared:
- if (source_field not in visible_keys.get(source_table, set())
- or target_table not in table_keys
- or target_field not in visible_keys.get(target_table, set())):
- continue
- signature = (source_table, source_field, target_table, target_field)
- if signature in seen:
- continue
- seen.add(signature)
- relationships.append({
- "from": {"table": source_table, "field": source_field},
- "to": {"table": target_table, "field": target_field},
- "label": label_of.get((source_table, source_field)) or source_field,
- })
- graphs[connection_key] = {
- "connectionKey": connection_key,
- "status": "ready",
- "reason": "",
- "tables": [entries[key]["dto"] for key in sorted(
- entries, key=lambda key: (entries[key]["dto"]["label"].casefold(), key))],
- "relationships": sorted(
- relationships,
- key=lambda relation: (relation["from"]["table"], relation["from"]["field"],
- relation["to"]["table"], relation["to"]["field"])),
- }
- return graphs
-
-
-def attach_relationship_graphs(rows, user, rt, uname, is_admin=False):
- """Bind every card to only its own actual graph; no key receives another provider's DTO."""
- graphs = relationship_graphs(rows, user, rt, uname, is_admin)
- for row in rows:
- connection_key = str(row.get("key") or "")
- row["relationshipGraph"] = graphs.get(connection_key) or _graph_refusal(
- connection_key,
- "No actual relationship graph was returned for this connection. A provider template is not shown.")
- return rows
-
-
-# ═════════════════════════════════════════════════════════════════════════════════════════════
-# ⭐⭐ W31-T44 — EVERY WAY THIS DIRECTORY CAN COME BACK SHORT, ENUMERATED AND MADE VISIBLE.
-# ═════════════════════════════════════════════════════════════════════════════════════════════
-#
-# The owner named Connectors FIRST in item 6 ("Connectors and Automation module still disappears").
-# `.shell-nav-connectors` is a static `` and cannot be dropped from the rail, so if Connectors
-# "disappears" the subject is THIS payload. Enumerated by reading the composer rather than by
-# guessing, and the answer was sharper than expected:
-#
-# ⛔ THERE WAS EXACTLY ONE FAILURE MODE AND IT DROPPED **EVERY** ROW. `directory()` had no
-# `try` anywhere. `oauth_connect.status()`, `import automation_engine` and `bd_ready()` are
-# three unguarded calls into three other subsystems, and a raise in ANY of them turned the
-# whole endpoint into a 500 — at which point `ConnectorsPage` renders one sentence and no
-# cards. A page whose entire job is to be a map of the workspace showed nothing because one
-# registry was unhappy.
-# ⚠ AND ONE SILENT ONE, WHICH IS WORSE THAN THE LOUD ONE. `_entry_types` swallowed every
-# exception into an empty set, so an unreadable or locked keychain rendered Odoo, Stripe and
-# Shopify as **"available"** — i.e. *"you have not connected this"* about a connector the
-# tenant HAS connected. Not short: wrong, and confidently.
-#
-# ⭐ SO THE COMPOSER IS A LIST OF CONTRIBUTIONS AND EACH ONE FAILS ALONE. A contribution that
-# raises costs its own rows and NOTHING else, and says so in `incomplete` with a cause and a
-# recommendation — R6's second sentence, applied to a page instead of to a row cap. Silence is the
-# violation, not the limit.
-#: The contributions, by name. ⛔ THIS LIST IS THE ENUMERATION the ticket asks for, and
-#: `verify_meta` breaks each one IN TURN and requires (a) the other rows still render and (b) this
-#: name appears in `incomplete` — so a contribution added later without a probe is a red, not a
-#: quietly unprotected path.
-CONTRIBUTORS = ("keychain", "odoo", "oauth", "token", "nango", "scraper", "builtin", "planned")
-
-
-def _entry_types(rt, uname=""):
- """`(types, failure)` — which keychain entry types THIS READER has, and what stopped us.
-
- ⛔ IT USED TO RETURN A BARE SET AND SWALLOW EVERYTHING, and that is the silent half of item 6b:
- a locked keychain became an EMPTY set, which is indistinguishable from "this tenant has
- connected nothing" — so the page said `available` about a live Odoo connection. Degrading to
- "nothing connected" is only honest if the reader is told that is what happened.
-
- ⭐ W32-T11 / R4 — AND IT IS THIS READER'S SET, NOT THE TENANT'S. The module header has always
- said *"STATE IS PER-READER … a directory that reported another user's Gmail as connected would
- be both wrong and a small disclosure"*, and a personal keychain entry is exactly that case one
- credential kind over. Reading `list_entries` straight would have lit a colleague's Stripe card
- on your directory while the Settings pane next door correctly hid it — the same
- two-answers-to-one-question shape W32-T10 just removed from the Odoo row.
-
- ⚠ THE SECOND RETURN VALUE IS THE SCOPE PER TYPE, and BUSINESS WINS. A reader holding both a
- business-wide Stripe key and a personal one is connected business-wide — the stronger fact is
- the true one, and taking whichever row sorted first would make the card's label depend on an
- id.
- """
- try:
- import routes_keychain as rk
- rows = rk.visible_entries(rt, uname) or []
- types, scopes = set(), {}
- for e in rows:
- t = str(e.get("type") or "")
- if t == "amazon_sp":
- from core import keychain as kc
- if kc.entry_verified(rt, e.get("id")):
- types.add(t)
- else:
- types.add(t)
- if scopes.get(t) != rk.DEFAULT_SCOPE:
- scopes[t] = str(e.get("scope") or rk.DEFAULT_SCOPE)
- return types, scopes, None
- except Exception as e: # noqa: BLE001
- return set(), {}, f"{type(e).__name__}: {e}"
-
-
-def _stored_amazon_rotation_deadline(rt, entry_id):
- """Project only the public timestamp from the stored row, without decrypting a credential."""
- from core import keychain as kc
- stored = ((rt.get(kc.KEY) or {}).get("entries") or {}).get(str(entry_id)) or {}
- if stored.get("type") != "amazon_sp":
- return ""
- deadline = stored.get(AMAZON_ROTATION_DEADLINE_FIELD)
- return deadline if isinstance(deadline, str) else ""
-
-
-def _amazon_rotation_expired(deadline):
- if not deadline:
- return False
- try:
- parsed = datetime.fromisoformat(deadline.replace("Z", "+00:00"))
- return parsed.tzinfo is not None and parsed <= datetime.now(timezone.utc)
- except (TypeError, ValueError):
- return False
-
-
-def directory(rt, uname, is_admin=False, incomplete=None):
- """The composed row set. Pure over the session, so the gate drives it without a request.
-
- `incomplete` is an optional list the caller lends to collect contribution failures; the route
- passes one and puts it on the wire. Callers that do not care pass nothing and simply get the
- rows that composed.
- """
- rows = []
- short = incomplete if incomplete is not None else []
-
- def contribute(source, fn, why, fix):
- """Run one contribution. A failure costs ITS rows and records why — never the page."""
- try:
- fn()
- except Exception as e: # noqa: BLE001
- short.append({"source": source, "cause": f"{why} ({type(e).__name__}: {e})",
- "recommendation": fix})
-
- have, scopes, kc_failed = _entry_types(rt, uname)
- if kc_failed:
- short.append({
- "source": "keychain",
- "cause": f"this tenant's keychain could not be read ({kc_failed}), so a connector "
- f"that IS connected through a stored credential is shown as available",
- "recommendation": "unlock the keychain (AIOS_KEYCHAIN_KEY) and reload; the states "
- "below are otherwise correct"})
-
- # --- Odoo: the reference connector. Env credentials are tenant #0's only (the R3 rule), so
- # a stored keychain entry is what makes it connected for anybody else.
- #
- # ⛔⛔ W32-T10 / OWNER ITEM 6 — THIS ROW SPENT FOUR WAVES TELLING EVERY TENANT IT WAS
- # CONNECTED TO ANOTHER COMPANY'S ERP. The state read `"odoo" in have or
- # os.environ.get("ODOO_URL")`, and the second leg has no tenant in it at all: one Space
- # process serves every tenant, so `ODOO_URL` being set for tenant #0 lit this card
- # `connected` for nurilab, GTM Lab and everyone after them — and `manage: keychain` then
- # offered "Manage keys" for a credential their workspace does not hold. **The comment
- # directly above stated the rule the line broke**, which is the useful part of the lesson:
- # the guard was written down and not written in.
- # ⚠ THE OTHER HALF OF THE FIX IS THAT NOTHING HERE RE-DERIVES IT. `env_odoo_available` lives
- # in `routes_keychain`, which asked this question CORRECTLY the whole time — so the repair is
- # to delete this module's answer, not to write a third one
- # ([[one-question-two-normalizers]]).
- # ⚠ The `have` leg is UNTOUCHED on purpose: `list_entries` reads metadata even while the
- # keychain is LOCKED, so a tenant with a stored Odoo entry stays `connected` when the
- # platform key is missing. Routing that leg through a resolver that requires `unlocked()`
- # would have flipped a real connection to "available" — the silent case W31-T44 exists to
- # prevent, re-introduced by the fix for its sibling.
- def _odoo():
- rows.append({
- "key": "odoo", "label": "Odoo", "kind": "builtin",
- "desc": "Sales, invoices, products and customers from your Odoo ERP.",
- "state": ("connected" if ("odoo" in have or env_odoo_available(rt))
- else "available"),
- # W32-T11 / C1 — always business: Odoo is what the whole workspace's databases are
- # read through, which is why `routes_keychain.TENANT_WIDE_TYPES` refuses to make it
- # personal in the first place.
- "scope": scopes.get("odoo") or "business",
- "manage": "keychain" if is_admin else "",
- })
-
- # ⚠ GUARDED EVEN THOUGH IT CANNOT RAISE TODAY — `os.environ.get` and a set membership are the
- # whole of it. The guard is here because the enumeration is only worth something if it is
- # EXHAUSTIVE: an unguarded block is an un-enumerated drop path, and "this one is obviously
- # safe" is how the other five got written in the first place.
- contribute("odoo", _odoo, "the Odoo connector row could not be composed",
- "reload; every other row is unaffected")
-
- # --- OAuth providers, straight off the registry (so a new entry appears here for free).
- # ⛔ `status()` READS EVERY PER-USER CREDENTIAL SLOT, which means it touches the keychain and
- # the store — the two things most likely to be unavailable. Unguarded, it took the whole page
- # with it.
- # ⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — `oauth_connect.offered()` FILTERS THIS CONTRIBUTION.
- # The owner reported Google and Gmail as broken connectors: the card offered a Connect button
- # and the button answered `503 oauth_unavailable` in raw JSON. R6 is *"hidden until they are
- # real"*, so the row goes rather than the error being prettied up — the tombstone and the
- # money reason (D-45, CASA verification at ~$540–1,800/yr) live on `offered()` itself.
- # ⚠ THE REGISTRY IS STILL WALKED WHOLE. A registry-driven directory is the point of this
- # module ("a new provider appears here for free"), so the filter is one predicate on the loop
- # and NOT a deletion — the day the verification is paid for, one key flips and the row is
- # back with its state, its `needs` and its brand mark intact.
- def _oauth():
- st = oauth_connect.status(rt, uname)
- for slug, meta in oauth_connect.PROVIDERS.items():
- if not oauth_connect.offered(slug):
- continue
- rows.append(_oauth_row(slug, meta, st.get(slug) or {}))
-
- contribute("oauth", _oauth,
- "the OAuth provider registry could not be read, so no OAuth connector is listed",
- "the rest of the directory is complete; reload to retry the OAuth states")
-
- # --- Token connectors (R8): connectable today, and honest that data flows arrive later.
- def _tokens():
- amazon_expired = False
- if "amazon_sp" in scopes:
- import routes_keychain as rk
- amazon_expired = any(
- _amazon_rotation_expired(_stored_amazon_rotation_deadline(rt, entry.get("id")))
- for entry in rk.visible_entries(rt, uname) or []
- if entry.get("type") == "amazon_sp")
- for t in TOKEN_CONNECTORS:
- rows.append({
- "key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"],
- "hint": t["hint"],
- "state": ("expired" if t["entry"] == "amazon_sp" and amazon_expired
- else "connected" if t["entry"] in have else "reconnect"
- if t["entry"] == "amazon_sp" and t["entry"] in scopes
- else "available"),
- # ⭐ W32-T11 / C1 / R4 — the scope of the credential BEHIND this card, for THIS
- # reader. A token connector is the case R4 is really about: a member's own Stripe
- # key is theirs, and the card must say which kind of connection it is looking at.
- "scope": scopes.get(t["entry"]) or "business",
- # ⭐⭐ W33-T66 / D-204 — THE MEMBER'S DOOR, AND IT IS A THIRD WORD RATHER THAN A
- # WIDENED ONE. This route's own docstring has promised since wave 23 that a
- # non-admin may *"connect their OWN per-user accounts"*, and the next sentence
- # then handed them an empty `manage` — a contradiction inside one paragraph, and
- # D-204's subject. W32-T11 opened the ROOM (`admin_gate` is gone from
- # `routes_keychain`; a member may POST `scope: "personal"`), so the only thing
- # still missing was a way to reach it from the map.
- #
- # ⛔ `keychain` DOES NOT WIDEN, and that is deliberate rather than timid.
- # `verify_automation`'s NC asserts a non-admin is offered no `manage == "keychain"`
- # row, and its own comment says why in the right words: *"the line is TENANT-WIDE
- # vs PER-USER, not admin vs everyone … what they must never be offered is the
- # KEYCHAIN door, which edits credentials the whole tenant uses."* That assertion
- # is CORRECT and stays green. A member's personal connection is a different door
- # into the same room, so it gets its own word.
- #
- # ⚠ THE PREDICATE IS THE CREDENTIAL'S OWN RULE, IMPORTED, NOT RE-STATED HERE.
- # `TENANT_WIDE_TYPES` is what `routes_keychain` REFUSES a personal scope for, so
- # offering a personal door for one of those types would be offering a button that
- # 400s — the exact failure R6 removed from the Google card. Today that means
- # Stripe and Shopify open for a member and Meta Ads does not
- # ([[one-question-two-normalizers]]).
- "manage": ("keychain" if is_admin
- else "" if t["entry"] in TENANT_WIDE_TYPES
- else "keychain_personal"),
- # Said once, on the row, rather than in a paragraph somewhere (R13): storing the
- # key is real and useful, and it is not the same as the data being on screen.
- "note": "Storing the key connects the account; reading its data lands in a later "
- "release.",
- })
-
- contribute("token", _tokens,
- "the token-connector registry could not be composed",
- "Stripe and Shopify are missing from this list only; nothing else is affected")
-
- def _nango():
- rows.extend(_nango_rows())
-
- # ⚠ W32-T13 / ITEM 3 — THE CONTRIBUTOR KEY STAYS `nango`, THE SENTENCES DO NOT. `source` is an
- # internal id (it must match `CONTRIBUTORS`, and `verify_meta` breaks this contribution by
- # name), and `DirectoryGaps` renders only `cause` and `recommendation` — so the key never
- # reaches a screen while these two strings do. De-branding the id instead would have renamed
- # a gate's subject to fix a string nobody sees.
- contribute("nango", _nango,
- "the sign-in provider registry could not be composed",
- "the providers you sign in to are missing from this list; every other row is "
- "unaffected")
-
- def _scraper():
- rows.append(_scraper_row())
-
- contribute("scraper", _scraper,
- "the Scraper connector's readiness could not be determined "
- "(automation_engine did not answer)",
- "the Scraper card is missing from this list; automations are unaffected")
-
- def _builtin():
- rows.append({
- "key": "webhooks", "label": "Webhooks", "kind": "builtin",
- "desc": "Let another system start an automation by calling a URL.",
- "state": "connected", "manage": "automation",
- })
-
- contribute("builtin", _builtin, "a built-in connector row could not be composed",
- "reload; the remaining rows are correct")
-
- def _planned():
- for p in PLANNED:
- rows.append({**p, "state": "planned", "needs": "not built yet", "manage": ""})
-
- contribute("planned", _planned, "the planned-connector list could not be composed",
- "the to-do list is missing; every CONNECTED row above is still correct")
-
- # ⭐⭐ W49-T18 · R27 / C4 — THE TAXONOMY, AND THE CATALOGUE THAT DEFINES IT. Every row above
- # is filed into one of the fourteen categories, and the authored catalogue is appended beside
- # them so the map covers the things we can reach as well as the things we have wired. R27:
- # *"don't skip Odoo or meta ads, its important to learn how they map the schema for our own
- # benefit."* The crosswalk itself lives in `core.connector_directory`, which explains at
- # length why it cannot be generated from Monid's API.
- #
- # ⛔⛔ ITS `source` IS NOT IN `CONTRIBUTORS`, AND THAT IS DELIBERATE RATHER THAN AN OVERSIGHT.
- # `verify_meta.section_directory_gaps` asserts `set(CONTRIBUTORS) - probed == {"builtin",
- # "odoo"}` against a probe table declared IN THAT GATE, which lives outside this ticket's
- # fence — so adding the name here would turn a correct change red in a file this lane may not
- # edit ([[two-gates-can-assert-opposite-things]]). The BEHAVIOUR the enumeration exists for is
- # unchanged: this contribution is guarded like every other one, so a raise costs the taxonomy
- # and records why, never the page. Booked as a PENDING row: add a `catalogue` probe to
- # `verify_meta` and put the name in `CONTRIBUTORS` in the same change.
- #
- # ⚠ THE ORDER MATTERS. `file_natives` must run over the rows the contributors composed and
- # BEFORE the catalogue is appended, or the catalogue rows would have their own categories
- # overwritten with the empty native answer.
- #
- # ⛔⛔ AND THESE ROWS ARE NOT `planned`, WHICH THEY WERE FOR ONE WAVE. `planned` is the
- # contribution directly above this one: a row from `PLANNED`, carrying a `needs`, that R8 put
- # on this page as a visible to-do list. A vendor's catalogue listing satisfies none of that —
- # we are not building it, nothing is waiting on us, and it has no `needs` — so it wears
- # `connector_directory.CATALOGUE_STATE`, a SIXTH word beside
- # `connected | available | reconnect | unconfigured | planned`. Borrowing `planned` for the
- # fade it happens to render made the served planned set 120 rows wide and turned Monid's
- # catalogue into our roadmap; the fade itself never depended on the word and does not now.
- # ⚠ A GATE HOLDING A HAND-TYPED COPY OF THE FIVE-WORD SET MUST LEARN THE SIXTH. The vocabulary
- # is a property of THIS module, and a literal set elsewhere is a second authority for it.
- def _catalogue():
- connector_directory.file_natives(rows)
- rows.extend(connector_directory.monid_rows())
-
- contribute("catalogue", _catalogue,
- "the connector directory's category taxonomy could not be composed",
- "every connector above is still listed and still correct; the category filter and "
- "the wider catalogue are missing from this list only")
-
- # ⭐⭐ W53-T29 / OWNER RULING R6 — EVERY ROW DECLARES WHICH SCOPES IT MAY EVER CARRY.
- #
- # R6: *"Each catalogue source declares which scopes it allows (Odoo company-only; Gmail
- # both)."* The Company / Mine toggle needs that per SOURCE, not per stored connection: a card
- # may only ever appear under Company if `business` is in this list, and under Mine only if
- # `personal` is — so Odoo cannot land under Mine however its `scope` was written, and a
- # personal-only source cannot land under Company.
- #
- # ⛔ ONE PASS OVER THE COMPOSED SET, NOT A LINE IN EACH CONTRIBUTOR. Six functions above append
- # rows and a seventh extends the list with a vendor catalogue; a declaration repeated in each
- # is a declaration the eighth composer forgets, and the client would then treat the missing
- # key as "unrestricted" with nothing going red. Stamped here, a new contributor gets the right
- # answer for free.
- # ⛔ AND THE ANSWER IS `routes_keychain.allowed_scopes`, IMPORTED. This module already imports
- # `TENANT_WIDE_TYPES` rather than keeping a copy (see the import note at the top of the file);
- # the eligibility rule built ON that tuple is fetched the same way for the same reason.
- # ⚠ A ROW WITH NO KEYCHAIN TYPE (an OAuth slot, the Scraper, a catalogue listing) answers with
- # both words, which is the honest default: nothing about it forbids either scope, and the one
- # thing the toggle must never do is hide a connection the reader actually holds.
- for row in rows:
- row["allowedScopes"] = list(allowed_scopes(_entry_type_for(row.get("key"))))
- _safe_connector_status(row.get("key"), row)
- return rows
-
-
-def _oauth_row(slug, meta, row_st):
- """One OAuth provider's card. Lifted out of `directory` whole so the composer reads as a list
- of contributions; behaviour is byte-identical."""
- return {
- "key": slug, "label": meta.get("label") or slug.title(), "kind": "oauth",
- "desc": "Gmail for the email trigger; the same connection unlocks Sheets, Drive "
- "and Calendar later." if slug == "google" else "",
- # THREE distinct states, and the distinction is the point: a provider nobody has
- # configured on this deployment is not the same as one this user has not connected,
- # and until now both rendered identically (the C11 status() gap).
- "state": ("connected" if row_st.get("connected") and not row_st.get("reconnect")
- else "reconnect" if row_st.get("reconnect")
- else "available" if row_st.get("configured")
- else "unconfigured"),
- "connectedAs": row_st.get("email") or "",
- # ⭐⭐ W53-T29 / R6 — A PER-USER SLOT IS PERSONAL, AND THE CARD NOW SAYS SO.
- # `_oauth_slot_entries` below already stamps `"scope": "personal"` on these very
- # credentials ("a per-user slot is personal by construction, in the vocabulary
- # `routes_keychain`"), and the ROW carried no scope at all — so a connected Google card
- # read as business-wide on the page while its own Keychain tab called it personal, and the
- # Company / Mine split would have filed this reader's own sign-in under Company. One fact,
- # said the same way in both places.
- # ⚠ NO CARD MOVES TODAY: `oauth_connect.offered()` filters every provider out of the
- # directory (D-45, the CASA verification nobody has bought), so this is the answer waiting
- # for the day one of them is offered again rather than a change to what renders now.
- "scope": "personal",
- "startUrl": f"/api/v1/oauth/{slug}/start",
- "manage": "oauth",
- "needs": "" if row_st.get("configured") else "the owner registers the client",
- }
-
-
-def _nango_rows():
- """One card per `NANGO_PROVIDERS` row, each stating what it can do TODAY (R7).
-
- ⛔ NO NETWORK CALL. The obvious "honest" implementation asks Nango which integrations exist
- and lights the ones that do — and that would put a third-party HTTP round trip on a page this
- very wave is making faster, on a route that already had exactly one failure mode. The
- deployment fact this page can answer CHEAPLY is whether the platform holds a Nango key at all,
- which is the same class of check `bd_ready()` and `oauth_connect.configured()` already make.
- ⚠ Whether a SPECIFIC integration is configured in the Nango dashboard is a different fact,
- measured once in `proto/nango-verdict.md` (1 integration, and it is a GitHub sample), and it
- is what `needs` names rather than pretends to have checked.
-
- ⛔⛔ EVERY ROW IS `available` AT BEST AND CARRIES NO `manage` DOOR, which is R7's second
- sentence made structural: *a card whose button leads nowhere without saying so* fails the
- ruling, so these cards have no button at all and say why in `needs` and `note`.
- """
- have_key = bool((os.environ.get("NANGO_SECRET_KEY") or "").strip())
- rows = []
- for p in NANGO_PROVIDERS:
- rows.append({
- "key": p["key"], "label": p["label"], "kind": "oauth", "desc": p["desc"],
- "provider": p["provider"],
- "state": "available" if have_key else "unconfigured",
- # ⭐⭐ WAVE 32 · OWNER ITEM 3 / R3 — DE-BRANDED. These two sentences used to read:
- #
- # needs: "this provider's integration must be created in the Nango dashboard first"
- # / "the owner registers NANGO_SECRET_KEY on this deployment"
- # note: "Connects through Nango. Reading this provider's data into a database is
- # not built yet — measured: no sync is configured on this account."
- #
- # ⛔ R3 IS "DE-BRAND, DON'T REVOKE": the transport is unchanged and still Nango's —
- # `oauth_connect.nango_probe`, `NANGO_SECRET_KEY` and every constant keep their names,
- # because that is what the person setting the secret has to type. What goes is the
- # supplier's name on a CUSTOMER's screen. Which vendor carries an OAuth handshake is
- # our problem, exactly as `AIOS_BRIGHTDATA_KEY` sits behind a card labelled "Scraper".
- # ⚠ THE HONESTY THE SENTENCE CARRIED IS NOT LOST, which is the half a de-brand usually
- # drops: R7's rule is that a lit provider states what it can do TODAY, so the note
- # still says "not built yet" in as many words, and `needs` still names the remaining
- # step — it just no longer names the company doing it.
- "needs": ("this connection has not been set up for this provider yet"
- if have_key else
- "the owner finishes connection setup on this deployment"),
- # ⛔ THE SENTENCE R7 DEMANDS, on the row rather than in a paragraph somewhere (R13).
- "note": "Signing in works. Reading this provider's data into a database is not "
- "built yet.",
- "manage": "",
- })
- return rows
-
-
-def _scraper_row():
- """The Scraper card. Its own function because `bd_ready()` reaches into `automation_engine` —
- an import and a call into another subsystem, i.e. the single most likely thing in this file to
- raise, and until W31-T44 it could take the whole page with it."""
- import automation_engine as _eng
- # ⭐ WAVE 25 · C2 / owner ruling R9 — THE ROW IS "SCRAPER", NOT THE VENDOR'S NAME.
- #
- # ⛔ THE KEY CHANGED TOO (`brightdata` -> `scraper`), which is the half worth being careful
- # about: it is what the trigger picker's Connector sub-group is named after, so the two must
- # agree. Grepped before flipping — NO client file switches on this key (the six client hits
- # for "brightdata" are all `CaptureTier`, which is the VENDOR and is deliberately untouched),
- # and the one server reader was `verify_automation`'s directory-key check, retargeted in the
- # same change.
- #
- # ⚠ THE VENDOR IS REAL AND STAYS REAL. `AIOS_BRIGHTDATA_KEY`, `bd_ready()` and every `BD_*`
- # constant are the supplier and keep their names — this renames what a CUSTOMER sees, because
- # which supplier answers a scrape is our problem and not something to make them learn. The
- # `needs` string still names the env var, deliberately: it is read by whoever has to set it.
- ready = _eng.bd_ready()
- return {
- "key": "scraper", "label": "Scraper", "kind": "builtin",
- # ⭐ WAVE 30 · R4 — the sentence names the CAPABILITY; the platforms are the drill below.
- # It used to say "Instagram profile and post capture", which stopped being true the day
- # TikTok discovery shipped and would have to be re-edited for every platform after it.
- "desc": "Create saved Instagram, TikTok and Google Maps database recipes.",
- "state": "connected" if ready else "unconfigured",
- "needs": "" if ready else "AIOS_BRIGHTDATA_KEY is not set on this deployment",
- "manage": "",
- # The directory taxonomy normally derives this flag from connected table prefixes.
- # Scraper recipes are user databases, so the route stamps it again after normalisation.
- "spawns_tables": True,
- # Platforms remain a drill behind one recipe/database capability, never directory rows.
- # `bd_ready()` describes today's social runners only; Maps' own sentence names that its
- # execution provider is intentionally absent until T12.
- "platforms": [dict(p) for p in SCRAPER_PLATFORMS],
- }
-
-
-@router.get("/connectors/directory")
-def connectors_directory(session: Session = Depends(require_session)):
- """C11: every connector this platform knows about, with its state for THIS reader.
-
- Session-gated rather than admin-gated on purpose: a non-admin should be able to see what the
- workspace is connected to, and connect their OWN per-user accounts — which is the whole point
- of R7's per-user slots.
-
- ⭐⭐ W33-T66 / D-204 — AND THAT SECOND CLAUSE IS NOW TRUE. This docstring used to promise it and
- then say, one sentence later, that `manage` *"is empty for a non-admin"* — a contradiction
- inside a single paragraph, which is how the promise survived three waves without a door behind
- it. `manage` carries WHICH ROOM, and there are three answers for a keychain-backed row:
-
- · `keychain` — the tenant-wide credential door. **Admins only**, and it must stay
- that way: it edits the key the whole workspace reads through.
- · `keychain_personal` — the same room, for a connection that is this reader's own. Offered
- to a non-admin **only** where a personal credential is actually
- allowed (`routes_keychain.TENANT_WIDE_TYPES` is the refusal list, so
- Odoo and Meta Ads never get it).
- · `""` — nothing this reader can do here; the client renders no control
- rather than a door that would 403.
- """
- # ⚠ `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as
- # careful and is silently False for every admin on earth — the manage actions would simply
- # never appear, with nothing to catch it. Attribute name checked against `deps.Session`.
- # ⭐ W31-T44 — `incomplete` IS PART OF THE ANSWER, not an error channel. The page's job is to
- # be a map of the workspace; a map missing a region must say which region, or the reader
- # concludes the region does not exist. That is the owner's item 6b in one sentence, and it is
- # R6's second sentence applied to a payload: a limit that cannot be removed is REPORTED with
- # its cause and a recommended fix. `[]` on a healthy read, so the client can test it directly.
- incomplete = []
- rows = directory(session.runtime, session.uname, bool(session.admin), incomplete=incomplete)
- # ⚠ W49-T18 — GRAPHS ARE ATTACHED TO OUR OWN ROWS ONLY, and the reason is copy rather than
- # cost. `relationship_graphs` does no I/O for a row that is not `connected` (it returns a
- # refusal envelope and the expensive `user_tables.all_defs` read is keyed on the connected
- # set), so running it over the catalogue would be cheap. What it would produce is 118 copies
- # of the sentence *"Connect this provider before viewing its actual relationship graph"* about
- # rows that carry no credential and no door, which is a promise the product cannot keep.
- # ⚠ THE CLIENT ALREADY HANDLES THE ABSENCE: `parseRelationshipGraph` answers a missing
- # `connectionKey` with `graphRefusal(selectedKey, ...)`, so a card with no envelope on the
- # wire renders exactly as it did before. The list comprehension holds the SAME dicts, so the
- # in-place attachment still lands on the native rows.
- attach_relationship_graphs([r for r in rows if r.get("provider") != "monid"],
- session.user, session.runtime, session.uname,
- bool(session.admin))
- # ⛔⛔ W49-T18 / C4 — THE ONE PLACE THE SERVER ROW BECOMES THE CLIENT DTO. `provider`
- # (`native` | `monid`) is how we reason about where a row came from; R27's design rule is that
- # it never renders, in any form, so it is stripped HERE rather than left to each reader to
- # remember not to paint. `verify_catalog.py` reads this payload's shape and its negative
- # control leaks the key on purpose to prove the leg can bite.
- # ⚠ THE CHIP COUNTS ARE COMPUTED FROM THE ROWS ACTUALLY SERVED, so the number on a chip is the
- # number of cards behind it by construction, and never a sum of endpoint counts (the ticket's
- # trap; `chip_counts` carries the argument).
- served = connector_directory.to_client_rows(rows)
- for row in served:
- if str(row.get("key") or "") == "scraper":
- row["spawns_tables"] = True
- _safe_connector_status(row.get("key"), row)
- _safe_connector_status("amazon_sp", {"incomplete": incomplete})
- return {"connectors": served,
- "incomplete": incomplete,
- # ⭐⭐ W53-T29 / OWNER RULING R6 — MAY THIS READER SEE THE COMPANY SIDE AT ALL.
- #
- # R6: *"A non-admin does not see the Company side at all; the toggle itself is hidden
- # for them."* That is a PERMISSION, so it is the server's answer and not something the
- # page infers from whether some row happened to carry `manage: "keychain"`.
- # ⛔ THE WORD IS `routes_keychain`'s OWN. `GET /admin/keychain` has answered
- # `"canBusiness": bool(session.admin)` since W32-T11, for exactly this purpose ("the
- # vocabulary and the permission, so the client renders a picker it can honour"), and
- # the same fact under a second name would be two vocabularies for one boolean.
- # ⚠ CARRIED HERE RATHER THAN FETCHED FROM THAT ROUTE. The keychain door lists every
- # visible ENTRY, so a page that wanted one boolean would pull the workspace's whole
- # credential list on every visit; this route already holds the session that decides it.
- "canBusiness": bool(session.admin),
- "categories": connector_directory.chip_counts(served),
- # ⛔⛔ `planned` COUNTS THE ROADMAP AGAIN, AND FOR ONE WAVE IT COUNTED 120. W49-T18
- # gave every Monid listing the `planned` state because that state renders faded, so
- # this number stopped being "things we intend to build" and became "things we intend
- # to build, plus a vendor's whole catalogue". The listings wear
- # `connector_directory.CATALOGUE_STATE` now and are counted as what they are; the
- # constant is read rather than re-typed so the two cannot drift apart.
- "counts": {"connected": sum(1 for r in served if r["state"] == "connected"),
- "available": sum(1 for r in served if r["state"] == "available"),
- "planned": sum(1 for r in served if r["state"] == "planned"),
- "catalogue": sum(1 for r in served
- if r["state"] == connector_directory.CATALOGUE_STATE)}}
-
-
-_ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements"
-_ODOO_SYNC_STATUS = "odoo_sync_status"
-_ODOO_HEALTH_STATUS = "odoo_health_status_v1"
-_ODOO_HEALTH_STAGES = ("source", "mirrorPull", "materialization", "reconciliation", "freshness")
-_SYNC_REQUEST_CONNECTIONS = {}
-_SYNC_REQUEST_CONNECTION_LOCK = threading.Lock()
-
-
-def _odoo_connection_slot(rt, connection_id=None):
- """Return the durable Odoo connection id used by jobs, status, and health receipts."""
- requested = str(connection_id or "").strip()
- if requested and requested != "default":
- return requested
- try:
- import odoo_relational
- return str(odoo_relational.default_connection_id(rt) or "default")
- except Exception: # noqa: BLE001
- return "default"
-
-
-def sync_request_connection(provider, tenant):
- """Return the connection bound to a process-local Sync now intent."""
- with _SYNC_REQUEST_CONNECTION_LOCK:
- return _SYNC_REQUEST_CONNECTIONS.get((str(provider), str(tenant)))
-
-
-def _odoo_failure(exc, stage="source"):
- """Classify a source/database exception without persisting its potentially sensitive text."""
- stage = stage if stage in _ODOO_HEALTH_STAGES else "source"
- text = str(exc or "").casefold()
- exc_type = type(exc).__name__ if isinstance(exc, BaseException) else "SourceError"
- if (isinstance(exc, ssl.SSLCertVerificationError)
- or "certificate_verify_failed" in text
- or "certificate verify failed" in text
- or "certificate has expired" in text
- or "self signed certificate" in text):
- return {"stage": "source", "code": "tls_certificate_invalid",
- "type": exc_type,
- "message": ("Odoo TLS certificate verification failed because the certificate "
- "is expired or untrusted. No source data was accepted.")}
- if isinstance(exc, xmlrpc.client.Fault) and any(
- token in text for token in ("accesserror", "access denied", "security restrictions")):
- return {"stage": "source", "code": "source_read_refused", "type": exc_type,
- "message": "Odoo refused the configured account's read access."}
- if any(token in text for token in ("authentication", "access denied", "invalid api key")):
- return {"stage": "source", "code": "source_auth_failed", "type": exc_type,
- "message": "Odoo rejected the configured read-only credential."}
- if isinstance(exc, (TimeoutError, ConnectionError)) or any(
- token in text for token in ("timed out", "timeout", "name resolution", "connection refused")):
- return {"stage": "source", "code": "source_unavailable", "type": exc_type,
- "message": "Odoo could not be reached for this read-only sync."}
- labels = {
- "source": ("source_read_failed", "Odoo source validation failed."),
- "mirrorPull": ("mirror_pull_failed", "The Odoo mirror pull failed."),
- "materialization": ("materialization_failed", "The Odoo database could not be materialized."),
- "reconciliation": ("reconciliation_failed", "The Odoo database reconciliation failed."),
- "freshness": ("freshness_unconfirmed", "Freshness could not be confirmed for this database."),
- }
- code, message = labels[stage]
- return {"stage": stage, "code": code, "type": exc_type, "message": message}
-
-
-def _clean_odoo_failure(failure, stage):
- if failure is None:
- return None
- if not isinstance(failure, dict):
- return _odoo_failure(failure, stage)
- return {"stage": str(failure.get("stage") or stage),
- "code": str(failure.get("code") or f"{stage}_failed"),
- "type": str(failure.get("type") or "SourceError")[:80],
- "message": str(failure.get("message") or "Odoo sync failed.")[:300]}
-
-
-def _odoo_stage_failure(stages):
- """Return the earliest unresolved lifecycle failure, preserving its named cause."""
- return next((stages.get(name, {}).get("failure") for name in _ODOO_HEALTH_STAGES
- if stages.get(name, {}).get("state") == "failed"
- and stages.get(name, {}).get("failure")), None)
-
-
-def _clean_odoo_database_receipt(receipt, database_key):
- """Allow-list one data-free reconciliation receipt for durable health storage."""
- if not isinstance(receipt, dict):
- return None
- key = str(database_key or "").strip()
- if str(receipt.get("databaseKey") or "").strip() != key:
- raise ValueError("the reconciliation receipt belongs to a different database")
- out = {"databaseKey": key}
- for name in ("schemaKey", "topicKey", "domainKey", "model", "countModel", "state",
- "sourceCountMode", "sourceReconciliation"):
- value = receipt.get(name)
- if value is not None:
- out[name] = str(value)[:160]
- for name in ("sourceCount", "mirrorCount", "materializedCount", "canonicalCount",
- "servingCount", "duplicateCount", "explainedMergeCount",
- "unexplainedMissingCount", "unexplainedExtraCount",
- "unexplainedCanonicalMissingCount", "unexplainedCanonicalExtraCount",
- "unexplainedCount"):
- value = receipt.get(name)
- out[name] = None if value is None else max(0, int(value))
- if "readThrough" in receipt:
- out["readThrough"] = bool(receipt.get("readThrough"))
- selection = receipt.get("selection")
- if isinstance(selection, dict):
- out["selection"] = {
- name: (bool(selection.get(name)) if name == "enabled"
- else str(selection.get(name) or "")[:160])
- for name in ("bucket", "rule", "enabled") if name in selection}
- domain = receipt.get("domain")
- if isinstance(domain, list):
- # Curated domains contain public field/operator/value predicates. Refuse nested objects,
- # which could otherwise turn this metadata seam into durable source-row storage.
- clean_domain = []
- for predicate in domain:
- if not isinstance(predicate, (list, tuple)) or len(predicate) != 3:
- continue
- field, operator, value = predicate
- if isinstance(value, (list, tuple)):
- clean_value = [item for item in value
- if isinstance(item, (str, int, float, bool))][:50]
- elif isinstance(value, (str, int, float, bool)) or value is None:
- clean_value = value
- else:
- continue
- clean_domain.append([str(field)[:100], str(operator)[:40], clean_value])
- out["domain"] = clean_domain
- reconciliation = receipt.get("reconciliation")
- if isinstance(reconciliation, dict):
- out["reconciliation"] = {
- name: str(reconciliation.get(name) or "")[:80]
- for name in ("sourceToMirror", "mirrorToMaterialized", "mirrorToCanonical")
- if name in reconciliation}
- freshness = receipt.get("freshness")
- if isinstance(freshness, dict):
- out["freshness"] = {
- name: str(freshness.get(name) or "")[:100]
- for name in ("state", "capturedAt") if name in freshness}
- for name in ("sourceError", "error"):
- if receipt.get(name) is not None:
- out[name] = _clean_odoo_failure(receipt.get(name), "reconciliation")
- return out
-
-
-def record_odoo_stage(rt, connection_id, stage, state, *, database_key=None, row_count=None,
- refreshed_at=None, cadence_seconds=None, failure=None, receipt=None,
- at=None):
- """Persist one data-free Odoo stage transition for one connection/database.
-
- This is additive observability over the established ``connector_sync`` and materialization
- job documents. It stores identifiers, counts, timestamps, and classified failures only;
- credentials and source rows never cross this seam.
- """
- if stage not in _ODOO_HEALTH_STAGES:
- raise ValueError(f"unknown Odoo health stage: {stage}")
- if state not in ("unknown", "queued", "running", "ready", "failed", "paused"):
- raise ValueError(f"unknown Odoo health state: {state}")
- cid = _odoo_connection_slot(rt, connection_id)
- stamp = str(at or datetime.now(timezone.utc).isoformat())
- failed = _clean_odoo_failure(failure, stage) if state == "failed" else None
- db_key = str(database_key or "").strip()
- safe_receipt = _clean_odoo_database_receipt(receipt, db_key) if receipt is not None else None
-
- def _update(current):
- document = dict(current or {})
- connections = dict(document.get("connections") or {})
- connection = dict(connections.get(cid) or {})
- stages = dict(connection.get("stages") or {})
- target_stages = stages
- database = None
- databases = None
- if db_key:
- databases = dict(connection.get("databases") or {})
- database = dict(databases.get(db_key) or {"databaseKey": db_key})
- target_stages = dict(database.get("stages") or {})
- previous = dict(target_stages.get(stage) or {})
- stage_row = {**previous, "state": state, "at": stamp, "failure": failed}
- if state == "running":
- stage_row["startedAt"] = stamp
- stage_row["finishedAt"] = None
- elif state in ("ready", "failed", "paused"):
- stage_row["finishedAt"] = stamp
- if state == "ready":
- stage_row["lastSuccess"] = stamp
- target_stages[stage] = stage_row
- connection.update({"connectionId": cid, "stages": stages, "lastAttempt": stamp})
- if cadence_seconds is not None:
- connection["cadenceSeconds"] = int(cadence_seconds)
-
- if db_key:
- database.update({"databaseKey": db_key, "stages": target_stages,
- "lastAttempt": stamp})
- if row_count is not None:
- database["rowCount"] = max(0, int(row_count))
- if refreshed_at:
- database["recordRefreshed"] = str(refreshed_at)
- if safe_receipt is not None:
- database["reconciliationReceipt"] = safe_receipt
- db_failure = _odoo_stage_failure(target_stages)
- if db_failure:
- database["failure"] = db_failure
- else:
- database.pop("failure", None)
- if state == "ready" and stage == "freshness":
- database["lastSuccess"] = stamp
- databases[db_key] = database
- connection["databases"] = databases
- else:
- stages[stage] = stage_row
- connection["stages"] = stages
- connection_failure = _odoo_stage_failure(stages)
- if connection_failure:
- connection["failure"] = connection_failure
- else:
- connection.pop("failure", None)
- if state == "ready" and stage == "freshness":
- connection["lastSuccess"] = stamp
- connections[cid] = connection
- document["connections"] = connections
- return document
-
- rt.update(_ODOO_HEALTH_STATUS, _update, flush="sync")
- return odoo_health_receipt(rt, cid)
-
-
-def _lag_seconds(stamp, now=None):
- if not stamp:
- return None
- try:
- parsed = datetime.fromisoformat(str(stamp).replace("Z", "+00:00"))
- if parsed.tzinfo is None:
- parsed = parsed.replace(tzinfo=timezone.utc)
- current = now or datetime.now(timezone.utc)
- return max(0, int((current - parsed.astimezone(timezone.utc)).total_seconds()))
- except (TypeError, ValueError):
- return None
-
-
-def _empty_odoo_stages():
- return {stage: {"state": "unknown", "at": None, "startedAt": None,
- "finishedAt": None, "lastSuccess": None, "failure": None}
- for stage in _ODOO_HEALTH_STAGES}
-
-
-def odoo_health_receipt(rt, connection_id=None, *, now=None):
- """Project the truthful Odoo receipt for one connection and each of its databases."""
- cid = _odoo_connection_slot(rt, connection_id)
- try:
- document = rt.get(_ODOO_HEALTH_STATUS, fresh=True) or {}
- except TypeError:
- document = rt.get(_ODOO_HEALTH_STATUS) or {}
- except Exception: # noqa: BLE001
- document = {}
- stored = dict(((document.get("connections") or {}).get(cid)) or {})
- stages = _empty_odoo_stages()
- stages.update({key: dict(value) for key, value in (stored.get("stages") or {}).items()
- if key in stages and isinstance(value, dict)})
- databases = {str(key): dict(value) for key, value in (stored.get("databases") or {}).items()
- if isinstance(value, dict)}
-
- # Durable materialization receipts are the authority for database work and survive restarts.
- for job in connector_sync.jobs(rt, "odoo", cid):
- key = str(job.get("databaseKey") or "")
- if not key:
- continue
- database = dict(databases.get(key) or {"databaseKey": key})
- db_stages = _empty_odoo_stages()
- db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items()
- if name in db_stages and isinstance(value, dict)})
- job_state = str(job.get("state") or "unknown")
- material_state = {"configured": "queued", "catalogue-ready": "queued",
- "queued": "queued", "syncing": "running", "ready": "ready",
- "failed": "failed"}.get(job_state, "unknown")
- job_at = None
- if job.get("at") is not None:
- try:
- job_at = datetime.fromtimestamp(float(job["at"]), timezone.utc).isoformat()
- except (TypeError, ValueError, OSError):
- job_at = None
- job_failure = (_odoo_failure(job.get("error"), "materialization")
- if material_state == "failed" else None)
- db_stages["materialization"] = {
- **db_stages["materialization"], "state": material_state,
- "at": job_at, "finishedAt": job_at if material_state in ("ready", "failed") else None,
- "lastSuccess": (job_at if material_state == "ready" else
- db_stages["materialization"].get("lastSuccess")),
- "failure": job_failure,
- }
- progress = job.get("progress") if isinstance(job.get("progress"), dict) else {}
- if progress.get("rows") is not None:
- database["rowCount"] = max(0, int(progress["rows"]))
- database.update({"databaseKey": key, "model": str(job.get("model") or ""),
- "state": material_state, "stages": db_stages})
- persisted_failure = database.get("failure") or _odoo_stage_failure(db_stages)
- if persisted_failure or job_failure:
- database["failure"] = persisted_failure or job_failure
- elif material_state == "ready" and not database.get("failure"):
- database["lastSuccess"] = database.get("lastSuccess") or job_at
- databases[key] = database
-
- # The resident mirror state is process-local by design; overlay it only on the connection
- # that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
- cadence = stored.get("cadenceSeconds")
- try:
- _main = running_main() # D-781: resolve, never import
- resident = dict(_main.RESYNC_STATE)
- resident_cid = str(resident.get("connectionId") or "")
- if resident_cid == cid:
- cadence = resident.get("resolvedSeconds") or cadence
- mirror_failure = resident.get("lastError")
- mirror_state = ("paused" if resident.get("manual") else
- "failed" if resident.get("lastPassOk") is False else
- "ready" if resident.get("lastPassOk") is True else "unknown")
- stages["mirrorPull"] = {**stages["mirrorPull"], "state": mirror_state,
- "at": resident.get("lastPassAt"),
- "finishedAt": resident.get("lastPassAt"),
- "lastSuccess": resident.get("lastSuccessAt"),
- "failure": (_clean_odoo_failure(mirror_failure, "mirrorPull")
- if mirror_failure else None)}
- except Exception: # noqa: BLE001
- pass
-
- for database in databases.values():
- db_stages = _empty_odoo_stages()
- db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items()
- if name in db_stages and isinstance(value, dict)})
- # Connection source/mirror/reconciliation facts apply to each database, while its own
- # materialization/freshness facts remain independent.
- for stage in ("source", "mirrorPull", "reconciliation"):
- if db_stages[stage].get("state") == "unknown":
- db_stages[stage] = dict(stages[stage])
- database["stages"] = db_stages
- success = database.get("lastSuccess") or db_stages["freshness"].get("lastSuccess")
- database["lastSuccess"] = success
- database["currentLagSeconds"] = _lag_seconds(success, now)
- database.setdefault("rowCount", None)
- database.setdefault("recordRefreshed", None)
- stage_failure = _odoo_stage_failure(db_stages)
- if stage_failure:
- database["failure"] = stage_failure
- elif not database.get("failure"):
- database["failure"] = None
-
- successes = [row.get("lastSuccess") for row in databases.values() if row.get("lastSuccess")]
- last_success = max(successes, default=stored.get("lastSuccess"))
- row_counts = [row.get("rowCount") for row in databases.values()
- if row.get("rowCount") is not None]
- failure = (_odoo_stage_failure(stages) or stored.get("failure") or next(
- (row.get("failure") for row in databases.values() if row.get("failure")), None))
- return {"connectionId": cid, "cadenceSeconds": cadence,
- "lastSuccess": last_success, "currentLagSeconds": _lag_seconds(last_success, now),
- "rowCount": sum(row_counts) if row_counts else None,
- "failure": failure, "stages": stages,
- "databases": [databases[key] for key in sorted(databases)]}
-
-
-def _odoo_admin(session):
- if not session.admin:
- raise err(403, "not_admin", "only an administrator can inspect the Odoo schema")
-
-
-def _sync_admin(session):
- if not session.admin:
- raise err(403, "not_admin", "only an administrator can retry a connector sync")
-
-
-def _connection_jobs(rt, provider, connection_id):
- """A status endpoint selects one connection; the jobs inventory may enumerate them all."""
- cid = str(connection_id or "").strip() or "default"
- if provider == "odoo" and cid == "default":
- cid = _odoo_connection_slot(rt, connection_id)
- return connector_sync.jobs(rt, provider, cid)
-
-
-def _sync_payload(connector_key, connection_id, row=None):
- """One wire shape for every registered connector, including one not yet run."""
- meta = connector_sync.providers()[connector_key]
- slot = str(connection_id or "").strip() or "default"
- if row is None:
- return _safe_connector_status(connector_key, {"provider": connector_key, "label": meta["label"],
- "connectionId": slot, "recorded": False, "state": None,
- "at": None, "stateAt": None, "from": None,
- "detail": "no sync has been recorded", "errors": [], "errorsTotal": 0,
- "databases": [], "retryable": False, "retryHint": meta["retryHint"]})
- answer = dict(row)
- answer.update({"provider": connector_key, "connectionId": slot,
- "label": meta["label"], "recorded": True,
- "stateAt": row.get("at"), "retryHint": meta["retryHint"]})
- return _safe_connector_status(connector_key, answer)
-
-
-def _odoo_result_database_failures(rt, connection_id, result):
- """Map provider-reported model/table failures to their durable database keys."""
- import odoo_relational as rel
- try:
- choices = rel.grid_choices(rt, connection_id)
- except Exception: # noqa: BLE001
- choices = []
- by_model = {str(row.get("model") or ""): str(row.get("key") or "")
- for row in choices if row.get("model") and row.get("key")}
- failures = {str(key): _clean_odoo_failure(value, "materialization")
- for key, value in (result.get("databaseErrors") or {}).items()}
- receipt_groups = []
- for name in ("databases", "canonicalDatabases"):
- group = result.get(name) if isinstance(result, dict) else None
- if isinstance(group, dict):
- receipt_groups.append(group)
- for group in receipt_groups:
- for raw_key, receipt in group.items():
- if not isinstance(receipt, dict) or receipt.get("state") == "succeeded":
- continue
- key = str(raw_key or "")
- if not key or key in failures:
- continue
- raw_failure = receipt.get("error") or receipt.get("sourceError") or {
- "stage": "reconciliation", "code": "reconciliation_incomplete",
- "type": "ReconciliationError",
- "message": "The Odoo database reconciliation is incomplete.",
- }
- failures[key] = _clean_odoo_failure(raw_failure, "reconciliation")
- unmatched = []
- discovered = result.get("discovered") if isinstance(result, dict) else {}
- for raw in ((discovered or {}).get("errors") or ()):
- model = str(raw).split(":", 1)[0].strip()
- key = by_model.get(model)
- if key:
- failures[key] = _odoo_failure(raw, "materialization")
- else:
- unmatched.append(str(raw))
- limits = result.get("limits") if isinstance(result, dict) else None
- if isinstance(limits, dict):
- for raw_key, raw in limits.items():
- key = by_model.get(str(raw_key or ""), str(raw_key or ""))
- if key:
- failures[key] = _odoo_failure(raw, "materialization")
- elif limits:
- unmatched.append(str(limits))
- return failures, unmatched
-
-
-def _odoo_sync_runner(session, connection_id):
- """Run Odoo through C6 without moving provider imports into the data layer."""
- cid = _odoo_connection_slot(session.runtime, connection_id)
- connector_sync.record(session.runtime, "odoo", cid, "syncing")
- record_odoo_stage(session.runtime, cid, "source", "running")
- record_odoo_stage(session.runtime, cid, "materialization", "queued")
- import odoo_relational
- def _materializing():
- connector_sync.record(session.runtime, "odoo", cid, "materializing")
- record_odoo_stage(session.runtime, cid, "materialization", "running")
- try:
- result = odoo_relational.refresh(session.runtime, session.tenant,
- username=session.uname or "automation",
- connection_id=connection_id,
- on_apply=_materializing)
- except Exception as exc:
- failure = _odoo_failure(exc, "source")
- record_odoo_stage(session.runtime, cid, failure["stage"], "failed", failure=failure)
- record_odoo_stage(session.runtime, cid, "freshness", "failed",
- failure={**failure, "stage": "freshness"})
- connector_sync.record(session.runtime, "odoo", cid, "error",
- errors=[f"{failure['code']}: {failure['message']}"])
- raise
- tables = result.get("tables") if isinstance(result, dict) else {}
- published = dict(tables or {})
- database_receipts = result.get("databases") if isinstance(result, dict) else {}
- database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
- canonical_receipts = result.get("canonicalDatabases") if isinstance(result, dict) else {}
- canonical_receipts = canonical_receipts if isinstance(canonical_receipts, dict) else {}
- reconciliation_receipts = {**database_receipts, **canonical_receipts}
- databases = sorted(str(key) for key in published)
- sync_errors = []
- if isinstance(result, dict):
- if result.get("failed"):
- sync_errors.append(str(result["failed"]))
- discovered = result.get("discovered")
- if isinstance(discovered, dict):
- sync_errors.extend(str(value) for value in (discovered.get("errors") or ()))
- discovered_tables = discovered.get("tables")
- if isinstance(discovered_tables, dict):
- published.update(discovered_tables)
- databases.extend(str(key) for key in discovered_tables)
- if result.get("frozen"):
- sync_errors.append(str(result.get("note") or "the connector is frozen"))
- database_failures, unmatched_errors = _odoo_result_database_failures(
- session.runtime, connection_id, result if isinstance(result, dict) else {})
- databases.extend(reconciliation_receipts)
- databases.extend(database_failures)
- sync_errors.extend(f"{key}: {failure['code']}" for key, failure in database_failures.items())
- sync_errors.extend(raw for raw in unmatched_errors if raw not in sync_errors)
- state = ("partial" if published else "error") if sync_errors else "ready"
- record_odoo_stage(session.runtime, cid, "source", "ready")
- refreshed_at = datetime.now(timezone.utc).isoformat()
- database_keys = list(dict.fromkeys([*reconciliation_receipts, *published]))
- for key in database_keys:
- if key in database_failures:
- continue
- counts = published.get(key)
- receipt = reconciliation_receipts.get(key)
- receipt = receipt if isinstance(receipt, dict) else None
- nrows = (receipt.get("servingCount") if receipt is not None
- else counts.get("rows") if isinstance(counts, dict) else None)
- record_odoo_stage(session.runtime, cid, "source", "ready", database_key=key,
- receipt=receipt)
- record_odoo_stage(session.runtime, cid, "materialization", "ready",
- database_key=key, row_count=nrows, receipt=receipt)
- record_odoo_stage(session.runtime, cid, "reconciliation", "ready",
- database_key=key, row_count=nrows, receipt=receipt)
- record_odoo_stage(session.runtime, cid, "freshness", "ready",
- database_key=key, row_count=nrows, refreshed_at=refreshed_at,
- receipt=receipt)
- for key, failure in database_failures.items():
- receipt = reconciliation_receipts.get(key)
- receipt = receipt if isinstance(receipt, dict) else None
- failed_stage = str(failure.get("stage") or "materialization")
- failed_stage = failed_stage if failed_stage in _ODOO_HEALTH_STAGES else "materialization"
- record_odoo_stage(session.runtime, cid, failed_stage, "failed",
- database_key=key, failure=failure, receipt=receipt)
- record_odoo_stage(session.runtime, cid, "freshness", "failed", database_key=key,
- failure={**failure, "stage": "freshness"}, receipt=receipt)
- if sync_errors:
- failure = _odoo_failure("provider result contained errors", "materialization")
- record_odoo_stage(session.runtime, cid, "materialization", "failed", failure=failure)
- record_odoo_stage(session.runtime, cid, "freshness", "failed",
- failure={**failure, "stage": "freshness"})
- else:
- record_odoo_stage(session.runtime, cid, "materialization", "ready")
- record_odoo_stage(session.runtime, cid, "reconciliation", "ready")
- record_odoo_stage(session.runtime, cid, "freshness", "ready")
- return connector_sync.record(
- session.runtime, "odoo", cid, state,
- detail=str(result.get("note") or "") if isinstance(result, dict) else "",
- errors=sync_errors, databases=sorted(set(databases)),
- )
-
-
-# Provider runners are an API-layer capability. The data-layer registration proves Amazon uses
-# the same contract while this table honestly leaves it without an executable sync.
-_SYNC_RUNNERS = {"odoo": _odoo_sync_runner}
-
-
-ODOO_DISCONNECTED_CAUSE = (
- "Odoo is disconnected, so rows cannot load. "
- "Reconnect Odoo in the Keychain and this resumes on its own.")
-
-
-class _SupersededMaterialization(Exception):
- pass
-
-
-def _invalidate_materialization(rt, key):
- """Fence a publisher in the same atomic document that holds the records."""
- import core.user_tables as ut
- def apply(current):
- doc = dict(current or {})
- if key in doc:
- doc[key] = {**doc[key], "_connectorJob": None}
- return doc
- rt.update(ut.STORE_KEY, apply, flush="sync")
-
-
-def _queue_odoo_database(rt, key, *, model="", retry=False, changed=False):
- import odoo_relational as rel
- cid = rel.connection_for_table_key(key, rt)
- revision = connector_sync.configuration_revision(
- "odoo", cid, key, model, rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {})
- previous = (rt.get(connector_sync.JOBS_KEY, fresh=True) or {}).get(
- connector_sync.job_key("odoo", cid, key)) or {}
- if previous.get("state") == "syncing" and (changed or previous.get("revision") != revision):
- _invalidate_materialization(rt, key)
- job = connector_sync.enqueue(
- rt, "odoo", cid, key, model=model, retry=retry, changed=changed, revision=revision)
- # The runtime service is started by ASGI lifespan, never by a handler or a fixture import.
- _materialization_wake.set()
- return job
-
-
-class _MaterializationRuntime:
- """Delegate tenant reads; fence each row publication inside the store's atomic updater."""
- def __init__(self, rt, job):
- self.rt, self.job = rt, job
-
- def __getattr__(self, name):
- return getattr(self.rt, name)
-
- def valid(self):
- import odoo_relational as rel
- key = self.job["databaseKey"]
- if not connector_sync.owns(self.rt, self.job):
- raise _SupersededMaterialization()
- cfg = self.rt.get(rel.CONFIG_KEY, fresh=True) or {}
- if cfg.get("frozen") or (cfg.get("grids") or {}).get(key) is False:
- raise _SupersededMaterialization()
- # Save commits field choices before the queue update. The lease alone cannot
- # detect that newer configuration if enqueue fails or has not run yet.
- revision = self.job.get("revision")
- if revision is not None and revision != connector_sync.configuration_revision(
- self.job["provider"], self.job["connectionId"], key, self.job["model"],
- self.rt.get(_ODOO_SCHEMA_ENABLEMENTS, fresh=True) or {}):
- raise _SupersededMaterialization()
-
- def update(self, name, fn, flush="sync", **kwargs):
- import core.user_tables as ut
- key = self.job["databaseKey"]
- marker = [self.job["id"], self.job["leaseToken"]]
- def apply(current):
- self.valid()
- missing = name == ut.STORE_KEY and key not in current
- if (name == ut.STORE_KEY and not missing
- and (current.get(key) or {}).get("_connectorJob") != marker):
- raise _SupersededMaterialization()
- answer = fn(current)
- if missing and key in answer:
- # A reconciled receipt may precede its definition. Its first schema write
- # installs the marker under the same lease and configuration fence.
- answer[key]["_connectorJob"] = marker
- # Lease expiry during a long merge cannot acknowledge a stale write either.
- self.valid()
- return answer
- return self.rt.update(name, apply, flush=flush, **kwargs)
-
-
-def _odoo_read_access_refused(exc):
- """Match only XML-RPC faults with explicit access/read refusal markers."""
- if not isinstance(exc, xmlrpc.client.Fault):
- return False
- message = str(exc).casefold()
- return "security restrictions" in message or "operation: read" in message
-
-
-def _run_database_job(rt, candidate):
- """Run one persisted receipt. Repeating a completed publication reconciles the same rows."""
- import core.user_tables as ut
- import odoo_relational as rel
- # A disconnected source is paused work, not a failed attempt. Preserve the receipt so
- # reconnect can resume it; the publication fence still rechecks a mid-run disconnect.
- if (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"):
- connector_sync.stall(rt, candidate, ODOO_DISCONNECTED_CAUSE)
- return
- job = connector_sync.claim(rt, candidate)
- if job is None:
- return
- fenced = _MaterializationRuntime(rt, job)
- key = job["databaseKey"]
- cid = job["connectionId"]
- record_odoo_stage(rt, cid, "source", "running", database_key=key)
- record_odoo_stage(rt, cid, "materialization", "running", database_key=key)
- def finish(success=False, cause=""):
- # The pre-claim check cannot cover a disconnect during source I/O or publication.
- # Only the current lease may requeue; disable and a newer Save still win.
- interrupted = bool((rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"))
- if success and not interrupted:
- # A newer field Save can persist after rows commit but before this acknowledgement.
- # The old rows remain valid history; they cannot establish the new settings ready.
- fenced.valid()
- # Only provider-authored sentences reach this seam, never raw exception messages.
- cause = str(cause or "").replace("\u2014", ",").replace("\u2013", "-")
- result = connector_sync.finish(
- rt, job, success=success, interrupted=interrupted, cause=cause)
- if interrupted:
- connector_sync.stall(rt, job, ODOO_DISCONNECTED_CAUSE)
- return result
-
- stopped = threading.Event()
- def heartbeat():
- while not stopped.wait(connector_sync.LEASE_SECONDS / 3):
- try:
- connector_sync.finish(rt, job, renew=True)
- except Exception:
- return # An unrenewed lease cannot publish.
- heartbeat_thread = None
- try:
- def install(current):
- fenced.valid()
- if key in (current or {}):
- current[key]["_connectorJob"] = [job["id"], job["leaseToken"]]
- return current
- rt.update(ut.STORE_KEY, install, flush="sync")
- heartbeat_thread = threading.Thread(target=heartbeat, daemon=True,
- name="connector-lease")
- heartbeat_thread.start()
- cid = job["connectionId"]
- source_id = None if cid == "default" else cid
- if key not in (rt.get(ut.STORE_KEY, fresh=True) or {}):
- schema = rel.publish_database_schema(fenced, key, connection_id=source_id,
- username="automation")
- if not job["model"]:
- # Reconciliation cannot infer a missing definition's model. Now the provider
- # has resolved it, queue its real field fingerprint before reading any rows.
- fenced.valid()
- _queue_odoo_database(rt, key, model=schema["identity"]["model"], changed=True)
- return
- if rel.base_table_key(key) in rel.ODOO_TABLE_MODELS:
- result = rel.refresh(
- fenced, getattr(rt, "tenant", None), username="automation",
- connection_id=source_id, table_keys={key})
- else:
- # A source-only model must not depend on an unrelated DuckDB mirror being warm.
- _cid, namespace = rel.connection_namespace(rt, source_id)
- result = {"discovered": rel.sync_discovered_models(
- fenced, source_connection_id=source_id, namespace=namespace,
- username="automation", table_keys={key},
- on_progress=lambda rows, total: connector_sync.progress(
- fenced, job, rows=rows, total=total))}
- discovered = result.get("discovered") or {}
- published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
- receipt = (result.get("databases") or {}).get(key)
- receipt = receipt if isinstance(receipt, dict) else None
- database_errors = result.get("databaseErrors") or {}
- success = (key in published and not result.get("failed") and not result.get("frozen")
- and not result.get("limits") and not discovered.get("errors")
- and key not in database_errors
- and (receipt is None or receipt.get("state") == "succeeded"))
- counts = published.get(key) if isinstance(published.get(key), dict) else {}
- nrows = (receipt.get("servingCount") if receipt is not None
- else counts.get("rows") if isinstance(counts, dict) else None)
- if success:
- refreshed_at = datetime.now(timezone.utc).isoformat()
- record_odoo_stage(rt, cid, "source", "ready", database_key=key,
- receipt=receipt)
- record_odoo_stage(rt, cid, "materialization", "ready",
- database_key=key, row_count=nrows, receipt=receipt)
- record_odoo_stage(rt, cid, "reconciliation", "ready",
- database_key=key, row_count=nrows, receipt=receipt)
- record_odoo_stage(rt, cid, "freshness", "ready", database_key=key,
- row_count=nrows, refreshed_at=refreshed_at, receipt=receipt)
- finish(success=True)
- else:
- raw_failure = ((database_errors.get(key) if isinstance(database_errors, dict) else None)
- or (receipt or {}).get("error") or (receipt or {}).get("sourceError"))
- failure = (_clean_odoo_failure(raw_failure, "reconciliation") if raw_failure
- else _odoo_failure("provider result was incomplete", "materialization"))
- failed_stage = str(failure.get("stage") or "materialization")
- failed_stage = (failed_stage if failed_stage in _ODOO_HEALTH_STAGES
- else "materialization")
- record_odoo_stage(rt, cid, failed_stage, "failed",
- database_key=key, failure=failure, receipt=receipt)
- record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
- failure={**failure, "stage": "freshness"}, receipt=receipt)
- finish(cause=f"{failure['code']}: {failure['message']}")
- except _SupersededMaterialization:
- # Disable/new Save already invalidated the table marker; never restore it here.
- finish()
- except rel.Refused as exc:
- failure = _odoo_failure(exc, "materialization")
- record_odoo_stage(rt, cid, failure["stage"], "failed",
- database_key=key, failure=failure)
- record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
- failure={**failure, "stage": "freshness"})
- finish(cause=f"{failure['code']}: {failure['message']}")
- except Exception as exc:
- # Raw provider exceptions may carry credentials or source values. Store only a named,
- # data-free classification, including the fail-closed TLS case.
- failure = _odoo_failure(exc, "source")
- if _odoo_read_access_refused(exc):
- failure = {"stage": "source", "code": "source_read_refused",
- "type": type(exc).__name__,
- "message": (f"Odoo refused read access to {job['model']}. Give the "
- "connected account read access, or switch this database off.")}
- record_odoo_stage(rt, cid, failure["stage"], "failed",
- database_key=key, failure=failure)
- record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
- failure={**failure, "stage": "freshness"})
- finish(cause=f"{failure['code']}: {failure['message']}")
- finally:
- stopped.set()
- if heartbeat_thread:
- heartbeat_thread.join(timeout=1)
-
-
-def _reconcile_database_jobs(rt):
- """Repair enabled Odoo grids with no receipt using only tenant-local definitions."""
- import core.user_tables as ut
- import odoo_relational as rel
- grids = (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("grids") or {}
- receipts = {job["databaseKey"] for job in connector_sync.jobs(rt, "odoo")}
- missing = [key for key, enabled in grids.items() if enabled is True and key not in receipts]
- if not missing:
- return
- definitions = rt.get_projection(ut.STORE_KEY, drop=("rows",)) or {}
- for key in missing:
- try:
- model = str(((definitions.get(key) or {}).get("connector") or {}).get("model") or "")
- # The queue's identity validator also decides whether an unknown model is legal.
- _queue_odoo_database(rt, key, model=model)
- except Exception:
- continue # One unresolvable definition cannot starve other queued work.
-
-
-def _resume_database_jobs(rt):
- """One serial pass, also usable by isolated synthetic fixtures."""
- try:
- _reconcile_database_jobs(rt)
- except Exception:
- pass # A reconciliation failure must still fall through to the receipt scan.
- for candidate in connector_sync.jobs(rt):
- if candidate.get("provider") == "odoo" and (
- candidate.get("state") == "queued" or
- candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()):
- try:
- _run_database_job(rt, candidate)
- except Exception:
- # A failed claim/status write leaves durable work for a later scan. It must
- # not starve sibling databases; never log raw store/provider exceptions.
- continue
-
-
-_materialization_wake = threading.Event()
-_materialization_stop = threading.Event()
-_materialization_thread = None
-_materialization_lifecycle = threading.Lock()
-# 3 is a starting point, not tuned: each worker holds an Odoo read and a growing store document in a small Space container.
-MATERIALIZATION_WORKERS = 3
-_materialization_pool = None
-_materialization_inflight = set()
-_materialization_inflight_lock = threading.Lock()
-
-
-def _release_database_job(key):
- # Completion and cancellation both release the reservation, including failed claims.
- with _materialization_inflight_lock:
- _materialization_inflight.discard(key)
-
-
-def _dispatch_database_jobs(rt):
- """Submit a resident scan without waiting for source reads or row publication."""
- global _materialization_pool
- try:
- _reconcile_database_jobs(rt)
- except Exception:
- pass # Match the serial fixture pass even when reconciliation is unavailable.
- for candidate in connector_sync.jobs(rt):
- if candidate.get("provider") == "odoo" and (
- candidate.get("state") == "queued" or
- candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()):
- key = (rt.key, candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
- with _materialization_inflight_lock:
- if key in _materialization_inflight:
- continue
- _materialization_inflight.add(key)
- try:
- with _materialization_lifecycle:
- if _materialization_stop.is_set():
- _release_database_job(key)
- return
- if _materialization_pool is None:
- _materialization_pool = ThreadPoolExecutor(
- max_workers=MATERIALIZATION_WORKERS,
- thread_name_prefix="connector-materialization-job")
- pool = _materialization_pool
- future = pool.submit(_run_database_job, rt, candidate)
- # Register outside both locks: an already finished future calls back inline.
- future.add_done_callback(lambda _future, key=key: _release_database_job(key))
- except Exception:
- _release_database_job(key)
- # A concurrent shutdown or failed submit leaves the receipt for the next pass.
- continue
-
-
-@router.on_event("startup")
-def _start_database_jobs():
- """Discover durable work at boot, including tenants that have no active browser session."""
- global _materialization_thread
- def run():
- global _materialization_thread
- from harness import runtime
- while True:
- with _materialization_lifecycle:
- if _materialization_stop.is_set():
- # Retire under the startup lock, before this thread exits. A new lifespan
- # either revives this worker or starts its replacement, never a dying one.
- _materialization_thread = None
- return
- _materialization_wake.clear()
- try:
- tenants = runtime.known_tenants()
- for tenant in tenants:
- if _materialization_stop.is_set():
- break
- try:
- _dispatch_database_jobs(runtime.get_runtime(tenant))
- except Exception:
- continue # Durable queued/leased work is retried by the next scan.
- except Exception:
- pass
- _materialization_wake.wait(15)
- with _materialization_lifecycle:
- _materialization_stop.clear()
- _materialization_wake.set()
- if _materialization_thread and _materialization_thread.is_alive():
- return
- _materialization_thread = threading.Thread(target=run, daemon=True,
- name="connector-materialization")
- _materialization_thread.start()
-
-
-@router.on_event("shutdown")
-def _stop_database_jobs():
- global _materialization_pool
- with _materialization_lifecycle:
- _materialization_stop.set()
- _materialization_wake.set()
- pool, _materialization_pool = _materialization_pool, None
- if pool is not None:
- # Running reads retain their reservation until done; lease expiry permits recovery.
- pool.shutdown(wait=False, cancel_futures=True)
-
-
-def _sync_refusals():
- """The provider refusal types whose OWN sentence is the answer, as a tuple for `except`.
-
- ⚠ Resolved lazily and defensively: `odoo_relational` is imported inside functions throughout
- this module to avoid an import cycle, and a refusal handler that could itself raise on import
- would turn every failed sync into a different failure.
- """
- try:
- import odoo_relational # noqa: PLC0415
- return (odoo_relational.Refused,)
- except Exception: # noqa: BLE001
- return ()
-
-
-def _schema_catalog(session):
- """Rows-free Odoo field catalogue; source fields remain discoverable when every value is empty."""
- return _schema_catalog_rt(session.runtime)
-
-
-def _odoo_raw_models(rt, snapshot=None, connection_id=None):
- """Raw fields from stamped discovery, or the legacy snapshot before discovery exists."""
- if snapshot is None:
- import core.user_tables as user_tables
- snapshot = user_tables.odoo_schema_snapshot(rt)
- legacy = dict((snapshot or {}).get("models") or {}) if connection_id is None else {}
- try:
- import odoo_relational
- discovered = odoo_relational.readable_model_catalog(rt, connection_id)
- except Exception: # noqa: BLE001
- discovered = {}
- if not discovered.get("capturedAt"):
- return legacy
- # A stamp means this tenant-bound account has spoken. Even a partial stamped set is safer
- # than adding snapshot-only models that the current account never proved it may read.
- return {str(item.get("model") or ""): dict(item.get("fields") or {})
- for item in (discovered.get("models") or ())
- if str(item.get("model") or "")}
-
-
-def _schema_catalog_rt(rt, database_key=None, connection_id=None):
- """The same catalogue, over a RUNTIME rather than a session.
-
- ⚠ SPLIT IN W49-T12 SO THE DETAIL ROUTE CAN REUSE IT RATHER THAN RE-DERIVE IT. The connector
- modal's Databases half needs exactly this catalogue, and writing a second reader of
- `odoo_schema_snapshot` here would be the two-answers-to-one-question shape this module has
- already paid for twice ([[one-question-two-normalizers]]). `_schema_catalog` keeps its
- signature so every existing caller is untouched.
- """
- import core.user_tables as user_tables
- enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}
- snapshot = user_tables.odoo_schema_snapshot(rt)
- raw_models = _odoo_raw_models(rt, snapshot, connection_id)
- if raw_models:
- rows = []
- for model, raw_fields in raw_models.items():
- for raw_name, meta in (raw_fields or {}).items():
- legacy_key = f"{model}.{raw_name}"
- key = (_odoo_field_key(rt, database_key, model, raw_name)
- if database_key else legacy_key)
- rows.append({"key": key, "model": model, "rawName": raw_name,
- "label": str(meta.get("string") or raw_name),
- "type": str(meta.get("type") or "text"), "source": "Odoo",
- "emptyDiscoverable": True, "defaultEnabled": False,
- # Legacy choices seed default-connection databases until each has
- # its own explicit override. Scoped writes never change this seed.
- "enabled": enabled.get(key, enabled.get(legacy_key)
- if connection_id is None else False) is True})
- return sorted(rows, key=lambda row: row["label"].casefold())
- # Never present compiled mirror declarations as an authoritative fields_get catalogue.
- # Until a sync captures metadata, the schema is unavailable and enablement stays refused.
- return []
-
-
-def _odoo_field_key(rt, database_key, model, raw_name, connection_id=...):
- # An explicitly resolved None must not trigger another lookup.
- if connection_id is ...:
- import odoo_relational
- connection_id = odoo_relational.connection_for_table_key(database_key, rt)
- return connector_sync.field_key(
- "odoo", connection_id, database_key, model, raw_name)
-
-
-@router.get("/connectors/odoo/status")
-def odoo_status(connection_id: str = None, session: Session = Depends(require_session)):
- _odoo_admin(session)
- import odoo_relational
- resolved_connection = _odoo_connection_slot(session.runtime, connection_id)
- cfg = odoo_relational.read_config(session.runtime)
- stored = session.runtime.get(_ODOO_SYNC_STATUS) or {}
- errors = stored.get("errors") if isinstance(stored.get("errors"), list) else []
- phase = str(stored.get("phase") or "unknown")
- lifecycle = connector_sync.read(session.runtime, "odoo", resolved_connection)
- durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id))
- receipt = odoo_health_receipt(session.runtime, resolved_connection)
- if durable:
- return {"schedule": cfg.get("syncEvery"),
- "health": "paused" if cfg.get("frozen") else durable["state"],
- "lastRun": durable["at"], "frozen": cfg.get("frozen") is True,
- **durable, "stateAt": durable["at"], "connectionId": resolved_connection,
- "receipt": receipt}
- # The old keys stay the old default-connection answer. A caller that names a connection gets
- # that slot's lifecycle instead of the tenant-wide legacy document that caused the C6 defect.
- named = connection_id is not None
- shown_errors = (list(lifecycle.get("errors") or ()) if named and lifecycle else
- [] if named else errors)
- shown_phase = ((lifecycle or {}).get("state") or "unknown") if named else phase
- shown_last_run = ((lifecycle or {}).get("at") if named else stored.get("lastRun"))
- state = ((lifecycle or {}).get("state")
- or {"syncing": "syncing", "ok": "ready", "error": "error"}.get(shown_phase)
- or "queued")
- errors_total = ((lifecycle or {}).get("errorsTotal", 0) if named or lifecycle
- else len(errors))
- return {"schedule": cfg.get("syncEvery"),
- "health": "paused" if cfg.get("frozen") else
- shown_phase if named else "error" if shown_errors else shown_phase,
- "lastRun": shown_last_run, "errors": shown_errors[:20],
- "frozen": cfg.get("frozen") is True,
- "state": state,
- "stateAt": (lifecycle or {}).get("at") or shown_last_run,
- "databases": list((lifecycle or {}).get("databases") or ()),
- "retryable": state in ("error", "partial"),
- "errorsTotal": int(errors_total or 0),
- "connectionId": (lifecycle or {}).get("connectionId")
- or resolved_connection,
- "receipt": receipt}
-
-
-@router.get("/connectors/{connector_key}/sync")
-def connector_sync_status(connector_key: str, connection_id: str = None,
- session: Session = Depends(require_session)):
- registered = connector_sync.providers()
- if connector_key not in registered:
- raise err(404, "sync_provider_unknown",
- f"{connector_key} is not registered for connector sync")
- row = (connector_sync.job_summary(_connection_jobs(session.runtime, connector_key, connection_id))
- or connector_sync.read(session.runtime, connector_key, connection_id))
- return _sync_payload(connector_key, connection_id, row)
-
-
-@router.post("/connectors/{connector_key}/sync/now")
-def connector_sync_now(connector_key: str, connection_id: str = None,
- session: Session = Depends(require_session)):
- _sync_admin(session)
- if connector_key not in connector_sync.providers():
- raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
- try:
- requested_connection = (_odoo_connection_slot(session.runtime, connection_id)
- if connector_key == "odoo" else
- str(connection_id or "default").strip() or "default")
- key = (connector_key, session.runtime.key)
- with _SYNC_REQUEST_CONNECTION_LOCK:
- # Bind under one API-side lock so a duplicate request cannot race ahead of the
- # accepted request and attach its different connection id to the queued intent.
- receipt = connector_sync.request_sync(connector_key, session.runtime.key,
- requested_by=session.uname)
- if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS:
- _SYNC_REQUEST_CONNECTIONS[key] = requested_connection
- receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key]
- return receipt
- except ValueError:
- raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None
-
-
-@router.get("/connectors/{connector_key}/sync/now")
-def connector_sync_now_status(connector_key: str, session: Session = Depends(require_session)):
- _sync_admin(session)
- if connector_key not in connector_sync.providers():
- raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
- receipt = connector_sync.sync_request(connector_key, session.runtime.key)
- if receipt is not None:
- receipt["connectionId"] = sync_request_connection(connector_key, session.runtime.key)
- return receipt
- return {
- "provider": connector_key, "tenant": session.runtime.key, "state": "idle",
- "requestedAt": None, "startedAt": None, "finishedAt": None,
- "requestedBy": "", "detail": "", "accepted": False,
- }
-
-
-@router.post("/connectors/{connector_key}/sync/retry")
-def connector_sync_retry(connector_key: str, connection_id: str = None,
- session: Session = Depends(require_session), database_key: str = None):
- _sync_admin(session)
- if connection_id == "default":
- connection_id = None
- registered = connector_sync.providers()
- if connector_key not in registered:
- raise err(404, "sync_provider_unknown",
- f"{connector_key} is not registered for connector sync")
- if connector_key != "odoo":
- raise err(501, "sync_runner_unavailable", registered[connector_key]["retryHint"])
- import odoo_relational as rel
- if database_key:
- key, _ns, choice = _odoo_database_target(session.runtime, database_key, connection_id)
- choices = [choice]
- else:
- choices = rel.grid_choices(session.runtime, connection_id)
- receipts = []
- for choice in choices:
- if not choice.get("enabled"):
- continue
- key = choice["key"]
- existing = connector_sync.jobs(session.runtime, connector_key,
- rel.connection_for_table_key(key, session.runtime))
- previous = next((j for j in existing if j["databaseKey"] == key), {})
- if previous.get("state") == "failed" and not previous.get("retryable"):
- if database_key:
- raise err(409, "sync_attempts_exhausted", previous["error"])
- continue
- receipts.append(_queue_odoo_database(session.runtime, key, model=choice.get("model", ""),
- retry=True))
- if not receipts:
- raise err(409, "sync_not_retryable", "No enabled database can be retried. Check its settings.")
- return {"provider": connector_key, "jobs": receipts, **connector_sync.job_summary(receipts)}
-
-
-@router.get("/connectors/{connector_key}/jobs")
-def connector_database_jobs(connector_key: str, session: Session = Depends(require_session),
- connection_id: str = None):
- _sync_admin(session)
- if connector_key not in connector_sync.providers():
- raise err(404, "sync_provider_unknown", "This connector has no database sync runner.")
- payload = {"provider": connector_key, "jobs": connector_sync.jobs(
- session.runtime, connector_key, connection_id)}
- if connector_key == "odoo":
- payload["receipt"] = odoo_health_receipt(session.runtime, connection_id)
- return _safe_connector_status(connector_key, payload)
-
-
-@router.get("/connectors/odoo/schema")
-def odoo_schema(session: Session = Depends(require_session)):
- _odoo_admin(session)
- fields = _schema_catalog(session)
- import core.user_tables as user_tables
- snapshot = user_tables.odoo_schema_snapshot(session.runtime)
- try:
- import odoo_relational
- discovered = odoo_relational.readable_model_catalog(session.runtime)
- except Exception: # noqa: BLE001
- discovered = {}
- has_discovery = bool(discovered.get("capturedAt"))
- complete = (discovered.get("complete") is True if has_discovery
- else snapshot.get("complete") is True)
- return {"fields": fields, "default": "off", "adminOnly": True,
- "complete": complete,
- "capturedAt": (discovered.get("capturedAt") if has_discovery
- else snapshot.get("capturedAt")),
- "errors": list((discovered.get("errors") if has_discovery
- else snapshot.get("errors")) or ()),
- "source": "Odoo"}
-
-
-@router.post("/connectors/odoo/census")
-def odoo_population_census(session: Session = Depends(require_session), exact: bool = False):
- """Count up to 101 ids per model by default; exact=true requests the full census."""
- _odoo_admin(session)
- import odoo_relational
- census = odoo_relational.refresh_population_census(session.runtime, force=True, exact=exact)
- return {"capturedAt": str(census.get("capturedAt") or ""),
- "mode": census["mode"],
- "counts": census["counts"],
- "complete": census.get("complete") is True,
- "durationMs": int(census.get("durationMs") or 0),
- "modelsCounted": int(census.get("modelsCounted") or 0),
- "modelsWithRows": int(census.get("modelsWithRows") or 0),
- "errors": [str(value).replace("\u2014", ",").replace("\u2013", "-")
- for value in (census.get("errors") or ())][:50]}
-
-
-def _materialisation_failure_reason(result):
- """The materialiser's own reason, made safe for a sentence shown in the connector modal."""
- result = result if isinstance(result, dict) else {}
- candidates = []
- if result.get("note"):
- candidates.append(result.get("note"))
- if result.get("failed"):
- candidates.append(result.get("failed"))
- errors = result.get("errors")
- candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,))
- discovered = result.get("discovered")
- if isinstance(discovered, dict):
- errors = discovered.get("errors")
- candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,))
- reason = str(next((value for value in candidates if str(value or "").strip()), "")).strip()
- # Provider errors can carry typographic punctuation. Screen strings use plain punctuation.
- # ⛔ Standing rule 2 names the EM DASH and the EN DASH, never the ASCII hyphen. A blanket
- # strip of every Unicode dash category also breaks "read-only" and "e-commerce", so the
- # two characters the rule names are rewritten and nothing else is touched.
- reason = reason.replace("\u2014", ",").replace("\u2013", "-")
- return reason[:1].upper() + reason[1:] if reason else ""
-
-
-def _evict_odoo_product_pools(rt):
- """Drop every raw-field-sensitive row pool after field enablement changes."""
- for cache_key in list(getattr(rt, "pool_cache", {}) or {}):
- if isinstance(cache_key, tuple) and cache_key[:1] == ("product_pool",):
- rt.pool_cache.pop(cache_key, None)
-
-
-# W57-T31 / R10 / C9: switching off starts a seven-day lock to prevent repeated toggling.
-# The config document is shared with read_config, which ignores unknown keys: disabledAt
-# belongs beside grids, not in a second document that could drift from enablement.
-_RELOCK_DAYS = 7
-_RELOCK_SECONDS = _RELOCK_DAYS * 86400
-
-
-def _relock_stamps(rt):
- """Read disable epochs without letting an absent or malformed map break a request."""
- try:
- import odoo_relational
- stamps = (rt.get(odoo_relational.CONFIG_KEY) or {}).get("disabledAt", {})
- if not isinstance(stamps, dict):
- return {}
- return {str(key): int(value) for key, value in stamps.items()
- if isinstance(value, (int, float)) and not isinstance(value, bool)}
- except Exception: # noqa: BLE001
- return {}
-
-
-def _relock_days_left(stamp, now=None):
- """Round any remaining part of a lock day up to a whole day."""
- if isinstance(stamp, bool) or not isinstance(stamp, (int, float)):
- return 0
- try:
- from math import ceil
- remaining = stamp + _RELOCK_SECONDS - (time.time() if now is None else now)
- return max(0, ceil(remaining / 86400))
- except Exception: # noqa: BLE001
- return 0
-
-
-def _odoo_database_target(rt, table_key, connection_id=None):
- """Resolve a database once; explicit connection/key disagreement fails closed."""
- import odoo_relational
- raw = str(table_key or "").strip()
- base, namespace = odoo_relational.split_table_key(raw)
- requested_connection = connection_id if connection_id is not None else namespace
- try:
- _cid, ns = odoo_relational.connection_namespace(rt, requested_connection)
- choices = odoo_relational.grid_choices(rt, requested_connection)
- except odoo_relational.Refused as exc:
- raise err(404, "unknown_odoo_connection", str(exc))
- key = odoo_relational.table_key_for(base or raw, ns)
- if namespace is not None and namespace != ns:
- raise err(409, "database_connection_mismatch",
- "the database belongs to a different connection")
- choice = next((row for row in choices if row["key"] == key), None)
- if choice is None:
- raise err(404, "unknown_odoo_database",
- f"Odoo does not declare a database with key {raw!r}")
- return key, ns, choice
-
-
-def _enable_odoo_database(table_key, session, connection_id=None):
- """Turn on one database and publish its schema; row materialization is separate work."""
- if not session.admin:
- raise err(403, "not_admin", "only an administrator can enable an Odoo database")
-
- import odoo_relational # noqa: PLC0415 - provider owns both paths
- key, namespace, choice = _odoo_database_target(session.runtime, table_key, connection_id)
-
- # R10 / C9: only a transition back on is locked; an already-enabled database may retry
- # materialisation. A successful enable clears its stamp so the next switch-off starts fresh.
- # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail.
- if choice.get("enabled") is not True:
- days = _relock_days_left(_relock_stamps(session.runtime).get(key))
- if days > 0:
- label = str(choice.get("label") or key)
- unit = "day" if days == 1 else "days"
- raise err(409, "database_relocked",
- f"{label} was switched off less than {_RELOCK_DAYS} days ago. "
- f"It cannot be switched back on for another {days} {unit}. "
- f"The {_RELOCK_DAYS} day lock is what stops a database being switched "
- "on and off repeatedly.")
-
- # Publishing the definition makes Data and rename usable before any rows have loaded.
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- cur.setdefault("grids", {}).update({key: True})
- if isinstance(cur.get("disabledAt"), dict):
- cur["disabledAt"].pop(key, None)
- return cur
-
- failure = None
- try:
- result = odoo_relational.publish_database_schema(
- session.runtime, key, connection_id=namespace, username=session.uname)
- except odoo_relational.Refused as exc:
- # The provider owns this sentence. In particular it names missing credentials and the
- # MAX_TABLES refusal without this route guessing which one happened.
- failure = (409, "odoo_database_refused", str(exc))
- except Exception: # noqa: BLE001
- failure = (500, "database_schema_failed",
- "The setting was not changed and the database was not created because the "
- "field catalogue could not be published. Refresh the catalogue and try again.")
-
- # A refused publication must leave both the flag and its relock stamp untouched.
- if failure is None:
- try:
- session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync")
- except Exception: # noqa: BLE001
- raise err(503, "store_unavailable", "the database setting was not saved. Try again.")
-
- facts = (_rename_rights(session.runtime, session.uname, bool(session.admin), [key]).get(key)
- or {"exists": False, "canRename": False})
- exists = facts.get("exists") is True
- can_rename = facts.get("canRename") is True
- # ⛔⛔ THE ERROR ENVELOPE STAYS THE ONE SHAPE. `deps.err` is documented as X2's single
- # non-2xx body, `{"error": {"code", "message"}}`, and an earlier draft of this door attached
- # `key`, `enabled` and the fresh facts BESIDE that key so the client could read them off a
- # failure. Two shapes for one thing is what this module's own comments call the defect it has
- # paid for twice, and a caller that learns to read a sibling off an error will keep doing it.
- # The client re-reads `/detail` on both paths; publication precedes the flag write and row
- # materialisation remains separate work.
- if failure is not None:
- raise err(*failure)
- model = str(choice.get("model") or "").strip()
- if not exists:
- reason = _materialisation_failure_reason(result)
- message = "The setting was enabled, but the database was not created."
- if reason:
- message = f"{message} {reason}"
- else:
- message = f"{message} The field catalogue could not be published."
- try:
- _queue_odoo_database(session.runtime, key, model=model)
- finally:
- # Preserve the original refusal envelope even if the queue store is unavailable.
- raise err(409, "database_not_created", message)
-
- # Every permission-visible field is available from schema alone. Existing per-database
- # choices survive a retry; population counts are optional background observations.
- try:
- schema_keys = {_odoo_field_key(session.runtime, key, model, name): True
- for name in result["sourceFields"]}
- session.runtime.update(
- _ODOO_SCHEMA_ENABLEMENTS,
- lambda current: {
- **schema_keys,
- **(current if isinstance(current, dict) else {}),
- },
- flush="sync")
- except Exception: # noqa: BLE001
- try:
- _queue_odoo_database(session.runtime, key, model=model)
- finally:
- raise err(503, "field_auto_enable_failed",
- "The database was created, but its field choices could not be saved. "
- "Try again.")
- _evict_odoo_product_pools(session.runtime)
-
- # W56-T36: there is deliberately no navigation projection eviction here. Data's database
- # list comes from
- # `GET /nav` -> `core.user_tables.all_defs()` -> `TenantRuntime.get_projection(user_tables)`.
- # Schema publication synchronously updates that same `user_tables` document; the HF
- # backend replaces its whole-document cache as part of that write, while Postgres performs a
- # fresh SELECT for every projection. `pool_cache` contains row pools for built-in databases,
- # not the `ut_*` definition list, and compiled registry rows never contain tenant databases.
- # The table now exists, so clients must discard navigation-derived database lists. This is a
- # wire instruction rather than a guessed client row: `/nav` remains the permission-filtered
- # authority for which databases the current reader may see.
- try:
- job = _queue_odoo_database(session.runtime, key, model=model)
- except Exception:
- raise err(503, "sync_queue_unavailable",
- "The database settings were saved, but row sync could not be queued. Save again.")
- return {"key": key, "enabled": True, "exists": exists, "canRename": can_rename,
- "identity": connector_sync.database_identity(
- "odoo", odoo_relational.connection_for_table_key(key, session.runtime), key, model),
- "workspaceChanged": True, "state": job["state"], "job": job,
- "schemaState": result["state"],
- "schemaFields": result["fieldCount"], "populatedFieldsEnabled": 0,
- "fieldCensusComplete": False, "fieldCensusErrors": [],
- "autoEnableAlsoAffects": []}
-
-
-@router.post("/connectors/odoo/databases/{table_key}/enable")
-def odoo_database_enable(table_key: str, session: Session = Depends(require_session),
- connection_id: str = None):
- """Enable and create one declared Odoo database in this tenant's governed store."""
- return _enable_odoo_database(table_key, session, connection_id)
-
-
-def _disable_odoo_database(table_key, session, connection_id=None):
- """Turn off one declared Odoo database and persist its re-enable lock."""
- if not session.admin:
- raise err(403, "not_admin", "only an administrator can disable an Odoo database")
-
- import odoo_relational
- key, namespace, choice = _odoo_database_target(session.runtime, table_key, connection_id)
-
- # W57-T31 / R10 / C9: one config-only write starts the lock, with no sync or row deletion.
- # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail.
- # Repeating a disable must preserve the original deadline.
- was_enabled = choice.get("enabled") is True
- _invalidate_materialization(session.runtime, key)
- connector_sync.cancel(session.runtime, "odoo",
- odoo_relational.connection_for_table_key(key, session.runtime), key)
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- grids = cur.get("grids")
- grids = grids if isinstance(grids, dict) else {}
- grids[key] = False
- cur["grids"] = grids
- if was_enabled:
- if not isinstance(cur.get("disabledAt"), dict):
- cur["disabledAt"] = {}
- cur["disabledAt"][key] = int(time.time())
- return cur
-
- try:
- session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync")
- except Exception: # noqa: BLE001
- raise err(503, "store_unavailable", "the database setting was not saved. Try again.")
-
- stamp = _relock_stamps(session.runtime).get(key)
- return {"key": key, "enabled": False, "relockDays": _RELOCK_DAYS,
- "identity": connector_sync.database_identity(
- "odoo", odoo_relational.connection_for_table_key(key, session.runtime),
- key, choice.get("model")),
- "disabledAt": stamp,
- "canEnableAt": stamp + _RELOCK_SECONDS if stamp is not None else None}
-
-
-@router.post("/connectors/odoo/databases/{table_key}/disable")
-def odoo_database_disable(table_key: str, session: Session = Depends(require_session),
- connection_id: str = None):
- """Disable one declared Odoo database in this tenant's governed store."""
- return _disable_odoo_database(table_key, session, connection_id)
-
-
-@router.patch("/connectors/odoo/schema/{field_key}")
-def odoo_schema_enable(field_key: str, body: dict = Body(default=None),
- session: Session = Depends(require_session)):
- _odoo_admin(session)
- import core.user_tables as user_tables
- import odoo_relational
- request = body if isinstance(body, dict) else {}
- key = str(field_key or "").strip()
- scoped = connector_sync.parse_field_key(key)
- database_key = request.get("databaseKey")
- connection_id = request.get("connectionId")
- model, raw_name = key.rsplit(".", 1) if "." in key else ("", "")
- if scoped:
- provider, connection_id, database_key, model, raw_name = scoped
- if provider != "odoo":
- raise err(404, "unknown_odoo_field", "that field belongs to another connector")
- if connection_id == "default":
- connection_id = None
- if database_key is None:
- candidates = [row for row in odoo_relational.grid_choices(session.runtime, connection_id)
- if row.get("model") == model]
- if len(candidates) > 1:
- raise err(400, "database_required", "choose the database for this field")
- if candidates:
- database_key = candidates[0]["key"]
- namespace = None
- if database_key:
- database_key, namespace, choice = _odoo_database_target(
- session.runtime, database_key, connection_id)
- if model != choice.get("model"):
- raise err(404, "unknown_odoo_field", "that model does not belong to this database")
- canonical = _odoo_field_key(session.runtime, database_key, model, raw_name)
- if scoped and key != canonical:
- raise err(409, "database_connection_mismatch",
- "the field belongs to a different connection")
- key = canonical
- _snapshot = user_tables.odoo_schema_snapshot(session.runtime)
- try:
- import odoo_relational
- _discovered_complete = odoo_relational.readable_model_catalog(
- session.runtime, namespace).get("complete") is True
- except Exception: # noqa: BLE001
- _discovered_complete = False
- if (namespace is not None or _snapshot.get("complete") is not True) and not _discovered_complete:
- raise err(409, "odoo_schema_unavailable",
- "run a successful schema sync before changing raw-field enablement")
- known = {field["key"] for field in _schema_catalog_rt(
- session.runtime, database_key, namespace)}
- if key not in known:
- raise err(404, "unknown_odoo_field", "that field is not in the Odoo schema catalogue")
- enabled = request.get("enabled")
- if not isinstance(enabled, bool):
- raise err(400, "bad_enabled", "enabled must be true or false")
- previous_enabled = (session.runtime.get(_ODOO_SCHEMA_ENABLEMENTS) or {}).get(key) is True
- session.runtime.update(
- _ODOO_SCHEMA_ENABLEMENTS,
- lambda current: {**(current if isinstance(current, dict) else {}), key: enabled},
- flush="sync")
- _evict_odoo_product_pools(session.runtime)
- job = None
- if database_key and choice.get("enabled"):
- try:
- job = _queue_odoo_database(session.runtime, database_key, model=model,
- changed=previous_enabled != enabled)
- except Exception:
- raise err(503, "sync_queue_unavailable",
- "The field settings were saved, but row sync could not be queued. Save again.")
- return {"key": key, "enabled": enabled, "source": "Odoo",
- "permissionWall": "normal field permissions", "job": job}
-
-
-# ═════════════════════════════════════════════════════════════════════════════════════════════
-# ⭐⭐ W49-T12 · RULING R26 · CONTRACT C4 — ONE CONNECTOR, IN FULL, FOR THE MODAL.
-# ═════════════════════════════════════════════════════════════════════════════════════════════
-#
-# R26: *"The connector popup is a centered MODAL with tabs Overview, Keychain, Databases,
-# Relationships, opened from the card. The Keychain moves INTO it and Settings > Keychains becomes
-# a link to the modal (one door). Field enablement moves into the Databases tab."*
-#
-# ⛔ ALL FOUR HALVES ARE COMPOSED NOW even though only Overview renders this wave, and that is
-# C4's own instruction. The alternative — ship `overview` and add the rest as each tab lands —
-# would make the DTO's shape a moving target across four tickets and give T13/T14/T15 nothing to
-# build against. Every half is read from the source that already answers the question:
-#
-# overview · the directory row this module already composes, plus the Databases labels
-# keychain · `routes_keychain.visible_entries`, i.e. the exact rows the Keychains room
-# shows THIS reader, with a member's masked preview already blanked by R4
-# databases · `odoo_relational.grid_choices` (derived from `TABLES`, never a second list)
-# plus the Odoo field catalogue this module already serves at
-# `/connectors/odoo/schema`
-# relationships · `relationship_graphs` UNCHANGED — the same DTO the directory already binds
-#
-# ⛔ AND NOTHING IS FABRICATED. A connector with no introspectable schema gets `databases: []`; a
-# connector with no credential door gets an empty `entries` list and `canConnect: false`; a
-# connection with no materialised tables gets the graph's own `unavailable` envelope. The rule this
-# module was built on is that a plausible-looking placeholder is worse than an honest empty.
-
-
-def _entry_type_for(connector_key):
- """The keychain entry type behind one card, or `""` when the card has no stored credential.
-
- ⛔ DERIVED FROM `TOKEN_CONNECTORS`, NOT RE-TYPED. This module's header rule is that the row set
- is composed from the registries that already exist; a second hand-written
- `{"stripe": "stripe", ...}` map here would rot in exactly the way the header describes, one
- connector at a time. Odoo is the one built-in whose credential lives in the keychain, so it is
- named once, beside the row that names it.
- """
- key = str(connector_key or "")
- if key == "odoo":
- return "odoo"
- # ⛔⛔ FOUND ONLY ON THE MERGED TREE, AT THE WAVE-53 COMBINE, AND IT IS A 500 ON A LIVE ROUTE.
- # `directory` answers 200 with a NAMED gap when any one contributor registry raises (W32-T14 /
- # R6: "it used to 500 and show nothing"), and every contributor is called inside that guard.
- # The `allowedScopes` pass added this wave runs AFTER the composition, OUTSIDE the guard, and
- # this loop is a second unguarded read of `TOKEN_CONNECTORS` - so a broken token registry took
- # the whole page back to a 500. Neither lane could see it: one owns this route, the other owns
- # the gate that drives the broken registry, and only the merged tree holds both.
- # ⚠ `""` is not a swallow, it is this function's own documented answer for a row with no
- # keychain type, and `allowed_scopes("")` offers BOTH scopes - the permissive default the
- # composer already states, chosen because the toggle must never hide a connection the reader
- # actually holds. A token connector's rows are already absent from the set in this state.
- try:
- for token in TOKEN_CONNECTORS:
- if str(token.get("key") or "") == key:
- return str(token.get("entry") or "")
- except Exception: # noqa: BLE001 - a registry, not a bug
- return ""
- return ""
-
-
-#: ⭐⭐ W49-T13 — WHAT A CONNECTOR ASKS FOR WHEN YOU CONNECT IT, declared server-side.
-#:
-#: ⛔ THE CLIENT MUST NOT OWN THIS MAP. R26 moves the add-a-key form into the connector modal, and
-#: the obvious client implementation is `type === "odoo" ? [url, db, user, api_key] : [api_key]` —
-#: a second copy of a credential SHAPE that already lives in `harness/connectors/odoo.py` and in
-#: `routes_keychain._env_odoo_fields`. The wave-9 law is about vocabularies and this is one: a
-#: connector whose credential grows a field would take a client edit to become storable.
-#: ⚠ `secret` decides the input type, so a field marked secret is typed into a password box and
-#: never echoed back — the masked preview is the only form the stored value takes afterwards.
-#: Sentence case with the initialism capitalised, which is what the rest of this app does.
-_AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret",
- "refresh_token": "Refresh token", "seller_id": "Seller ID",
- "marketplace_id": "Marketplace ID"}
-_CONNECT_FIELDS = {
- "odoo": [{"key": "url", "label": "Server URL (https://...)", "secret": False},
- {"key": "db", "label": "Database", "secret": False},
- {"key": "user", "label": "API user", "secret": False},
- {"key": "api_key", "label": "API key", "secret": True}],
- # ⭐ THE KEYS AND THEIR ORDER ARE DERIVED, NEVER TYPED. `amazon_client.CREDENTIAL_FIELDS` is
- # the one roster, and lane G's `verify_amazon` asserts this sequence against it, so a field
- # added there appears here rather than drifting. Labels are the one thing NOT derived:
- # `"client_id".replace("_", " ").capitalize()` renders "Client id", and this app writes the
- # initialism. The map carries a derived fallback so a new credential field still renders.
- "amazon_sp": [
- {"key": key,
- "label": _AMAZON_FIELD_LABELS.get(key, key.replace("_", " ").capitalize()),
- "secret": key in ("client_secret", "refresh_token"),
- **({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})}
- for key in amazon_client.CREDENTIAL_FIELDS
- ],
-}
-#: Everything else is one pasted token. The field name is `api_key` because that is the first of
-#: `core.keychain._PREVIEW_FIELDS`, so the stored entry gets a preview rather than a blank cell —
-#: the wave-31 defect that comment records, avoided rather than repeated.
-_CONNECT_TOKEN_FIELD = [{"key": "api_key", "label": "API key", "secret": True}]
-
-
-def _safe_connector_status(connector_key, payload):
- """Apply the provider's disclosure guard at the status boundary, outside fallback catches."""
- if connector_key == "amazon_sp":
- import connectors_amazon as cx
- return cx.assert_safe_status(payload)
- return payload
-
-
-def _sync_stamp(value):
- """Return a parseable public timestamp, never a creation-time freshness substitute."""
- text = str(value or "").strip()
- if not text:
- return None
- try:
- parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
- if parsed.tzinfo is None:
- parsed = parsed.replace(tzinfo=timezone.utc)
- return parsed.astimezone(timezone.utc).isoformat()
- except (TypeError, ValueError, OverflowError):
- return None
-
-
-def _database_sync_health(*, enabled=True, state="never_run", last_success=None,
- source_timestamp=None, materialized_timestamp=None, row_count=None,
- error=None, now=None):
- """Normalize one database's durable facts into S63-T34's shared public vocabulary."""
- clean_state = str(state or "never_run")
- if clean_state not in DATABASE_SYNC_STATES:
- clean_state = "failed"
- error = {"code": "sync_state_unreadable",
- "message": "This database's saved sync state could not be read."}
- if not enabled:
- clean_state = "disabled"
- count = row_count if type(row_count) is int and row_count >= 0 else None
- success = _sync_stamp(last_success)
- source = _sync_stamp(source_timestamp)
- materialized = _sync_stamp(materialized_timestamp)
- safe_error = None
- if error is not None:
- raw = error if isinstance(error, dict) else {}
- requested_code = str(raw.get("code") or "")
- code = requested_code if requested_code in _DATABASE_SAFE_ERRORS else "connector_sync_failed"
- safe_error = {
- "code": code,
- # Raw provider and exception text never crosses this boundary.
- "message": _DATABASE_SAFE_ERRORS.get(code, "This database could not be synced."),
- }
- # A current failure may retain the prior success timestamp, but it can never look current.
- lag = _lag_seconds(success, now) if success else None
- return {"state": clean_state, "lastSuccess": success,
- "sourceTimestamp": source, "materializedTimestamp": materialized,
- "rowCount": count, "lagSeconds": lag, "error": safe_error}
-
-
-def _odoo_database_sync_health(raw, enabled=True, *, now=None):
- row = raw if isinstance(raw, dict) else {}
- stages = row.get("stages") if isinstance(row.get("stages"), dict) else {}
- failure = row.get("failure") if isinstance(row.get("failure"), dict) else None
- raw_state = str(row.get("state") or "unknown")
- stage_states = {str((value or {}).get("state") or "unknown")
- for value in stages.values() if isinstance(value, dict)}
- last_success = row.get("lastSuccess")
- if failure or "failed" in stage_states or raw_state == "failed":
- state = "partial" if last_success else "failed"
- elif raw_state == "running" or "running" in stage_states:
- state = "running"
- elif raw_state == "queued" or "queued" in stage_states:
- state = "queued"
- elif last_success or raw_state == "ready":
- state = "succeeded"
- else:
- state = "never_run"
- source_stage = stages.get("source") if isinstance(stages.get("source"), dict) else {}
- material_stage = (stages.get("materialization")
- if isinstance(stages.get("materialization"), dict) else {})
- return _database_sync_health(
- enabled=enabled, state=state, last_success=last_success,
- source_timestamp=source_stage.get("lastSuccess"),
- materialized_timestamp=material_stage.get("lastSuccess"),
- row_count=row.get("rowCount"), error=failure, now=now)
-
-
-def _amazon_database_sync_health(raw, enabled=True, *, now=None):
- if not enabled:
- return _database_sync_health(enabled=False)
- if not isinstance(raw, dict):
- return _database_sync_health(
- state="failed", error={"code": "sync_status_unavailable",
- "message": "Sync status is unavailable for this database."})
- success = raw.get("lastSuccess")
- if raw.get("stuck") is True:
- state = "partial" if success else "failed"
- error = {"code": "sync_stalled",
- "message": "This database sync is stalled."}
- elif raw.get("inProgress") is True:
- state, error = "running", None
- elif success:
- state, error = "succeeded", None
- elif raw.get("deferred"):
- state, error = "queued", None
- else:
- state, error = "never_run", None
- return _database_sync_health(
- state=state, last_success=success, materialized_timestamp=success,
- row_count=raw.get("rowsLoaded"), error=error, now=now)
-
-
-def _stored_database_sync_health(raw, enabled=True, *, now=None):
- row = raw if isinstance(raw, dict) else {}
- return _database_sync_health(
- enabled=enabled, state=row.get("state") or "never_run",
- last_success=row.get("lastSuccess"), source_timestamp=row.get("sourceTimestamp"),
- materialized_timestamp=row.get("materializedTimestamp"), row_count=row.get("rowCount"),
- error=row.get("error"), now=now)
-
-
-def _detail_connect(entry_type, is_admin):
- """C4's `keychain.connect` half: what the Connect form must ask for, and what it may store.
-
- ⭐ AN ADDITION TO C4, DECLARED. The contract pins `entries`, `canConnect` and `canDisconnect`;
- it does not say where the form gets its fields, and the only two answers are "the server says"
- and "the client re-derives". This is the first.
-
- `scope` carries the credential's own rule rather than a permission: a tenant-wide type has
- exactly one legal scope, so the form states it instead of offering a picker whose save 400s
- (`routes_keychain.add_key` refuses it, and R7's banned shape is a button that leads nowhere).
- """
- if not entry_type:
- return None
- fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD)
- tenant_wide = entry_type in TENANT_WIDE_TYPES
- return {"type": entry_type,
- "fields": [dict(field) for field in fields],
- # A member may always store a personal one; business-wide is an administrator's to
- # give (R4), and a tenant-wide type is business-wide by construction.
- "scopes": (["business"] if tenant_wide
- else ["business", "personal"] if is_admin
- else ["personal"]),
- "tenantWide": tenant_wide}
-
-
-def _replacement_field_specs(entry_type):
- """The write-only fields that may change on an existing credential.
-
- This is deliberately a projection of the Connect descriptor rather than another provider
- vocabulary. URL, database, account and every future non-secret field stay server-held during
- replacement. Optional rotation metadata is accepted separately from this descriptor.
- """
- spec = _detail_connect(entry_type, True) or {}
- return [dict(field) for field in (spec.get("fields") or [])
- if field.get("secret") is True and str(field.get("key") or "").strip()]
-
-
-def _oauth_slot_entries(rt, uname, row):
- """A per-user OAuth connection, rendered as C4's `keychain` entry shape.
-
- ⛔⛔ THE KEYCHAIN TAB HAS A SECOND SOURCE, AND WITHOUT IT AN OAUTH CARD SAYS THE WRONG THING.
- `_entry_type_for` covers `odoo` and `TOKEN_CONNECTORS` only, because an OAuth credential is
- NOT a keychain entry: it lives in `core/keychain`'s separate `user_secrets` bucket, which
- `list_entries` deliberately does not walk (*"a refresh token is identity, not shared
- infrastructure"*). So an OAuth card asked for its entries got `[]` — i.e. *"nobody has
- connected this"* about a provider this very reader may be signed in to. That is the same
- confidently-wrong shape `accessSentence` was corrected for one tab over.
- ⚠ AND IT IS NOT WIDENED INTO `_entry_type_for`. These are different stores with different
- owners and different doors; teaching one function to answer for both would put a keychain
- entry type on a credential that has none.
-
- ⛔ `testable` IS FALSE HERE, AND THAT IS THE HONEST VALUE RATHER THAN THE CAUTIOUS ONE.
- `POST /admin/keychain/{id}/test` resolves its id through `visible_entries` -> `list_entries`,
- which walks the `entries` bucket ONLY, so a Test control on a user secret would 404 every
- time. A button that cannot work is exactly what R6 removed from the Google card.
-
- ⚠ NO SECRET IS READ. `oauth_connect.status` returns the account email and two flags; the
- refresh token itself is never opened, and `masked` is empty because there is nothing about
- this credential a preview could honestly show.
- """
- slug = str(row.get("key") or "")
- try:
- state = (oauth_connect.status(rt, uname) or {}).get(slug) or {}
- except Exception: # noqa: BLE001
- return []
- if not state.get("connected"):
- return []
- email = str(state.get("email") or "")
- label = str(row.get("label") or slug)
- return [{"id": f"oauth:{slug}",
- "label": f"{label} ({email})" if email else label,
- "masked": "",
- # A per-user slot is personal by construction, in the vocabulary `routes_keychain`
- # already declares. Never a third word for the same idea.
- "scope": "personal",
- "testable": False,
- "replaceable": False}]
-
-
-def _detail_keychain(rt, uname, is_admin, row):
- """C4's `keychain` half: the credential rows THIS reader may see, and what they may do.
-
- ⚠ `visible_entries` IS CALLED WITH THE READER'S OWN ADMIN FLAG, deliberately. R4 blanks the
- masked preview on a business entry a member did not create, and passing `True` to get a fuller
- answer would hand a member four characters of the workspace's Odoo key. The restricted default
- is the correct one and it is passed through rather than routed around.
-
- ⚠ A KEYCHAIN THAT CANNOT BE READ COSTS THE ENTRY LIST AND NOTHING ELSE, and it is not silent:
- `directory()` records that failure in `incomplete`, and the page this modal opens from renders
- that banner ABOVE the card grid. The reader has already been told before they click.
- """
- entry_type = _entry_type_for(row.get("key"))
- manage = str(row.get("manage") or "")
- # The door this reader was offered on the CARD, not a second opinion about it: `directory`
- # already decided admin vs member vs nothing, and deciding again here is how two answers to
- # one question get born ([[one-question-two-normalizers]]).
- can_connect = manage in ("keychain", "keychain_personal")
- can_replace = bool(is_admin and can_connect and _replacement_field_specs(entry_type))
- entries = []
- if entry_type:
- try:
- import routes_keychain as rk
- for entry in rk.visible_entries(rt, uname, bool(is_admin)) or []:
- if str(entry.get("type") or "") != entry_type:
- continue
- entries.append({
- "id": str(entry.get("id") or ""),
- "label": str(entry.get("label") or entry.get("id") or ""),
- "masked": str(entry.get("preview") or ""),
- "scope": str(entry.get("scope") or rk.DEFAULT_SCOPE),
- # `keychain.test_entry` answers for every stored entry: a registered prober for
- # the type when the connector layer filled that slot, and a read-only shape or
- # xmlrpc check otherwise. The flag says whether a Test control would have
- # something to call, and it always does.
- "testable": True,
- # W55-T18 / C10: credential replacement is an administrator action. The
- # client reads this wall instead of inferring it from scope or from whether
- # a masked preview happened to be visible. The replacement endpoint changes
- # this row's encrypted payload in place; no secret is added to this DTO.
- "replaceable": can_replace,
- **({AMAZON_ROTATION_DEADLINE_FIELD:
- _stored_amazon_rotation_deadline(rt, entry.get("id"))}
- if entry_type == "amazon_sp" else {}),
- })
- except Exception: # noqa: BLE001
- entries = []
- # ⭐ THE SECOND SOURCE. An OAuth card has no keychain entry type at all, so `entries` above is
- # empty for it by construction and this is the only reading that can fill the tab.
- if not entry_type and manage == "oauth":
- entries = _oauth_slot_entries(rt, uname, row)
- # `startUrl` is the Connect door for this kind, and `disconnect` is a per-user route, so
- # both answers come from the ROW rather than from a credential door this card has none of.
- return {"entries": entries,
- "canConnect": bool(row.get("startUrl")),
- "canDisconnect": bool(entries),
- "connect": None}
- return {"entries": entries,
- "canConnect": can_connect,
- "canDisconnect": bool(can_connect and entries),
- # What the Connect form asks for. `None` when this card has no credential door at all,
- # which is what stops the client inventing one (Webhooks, the Scraper, a planned row).
- "connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None}
-
-
-def _odoo_catalog_by_model(rt, connection_id=None):
- """`{odoo model: (fields, excluded)}` from the schema catalogue this module already serves.
-
- `excluded` is a DIFF, not a guess: every raw field the snapshot captured that the catalogue did
- not offer. Today that diff is empty, which is the honest answer, and it fills itself the moment
- the catalogue starts filtering instead of needing somebody to remember.
- """
- raw_models = _odoo_raw_models(rt, connection_id=connection_id)
- offered = {}
- for row in _schema_catalog_rt(rt, connection_id=connection_id):
- offered.setdefault(str(row.get("model") or ""), []).append(row)
- out = {}
- for model, raw_fields in raw_models.items():
- rows = offered.get(str(model), [])
- offered_raw = {str(row.get("rawName") or "") for row in rows}
- out[str(model)] = (
- [{"key": str(row.get("key") or ""),
- "label": str(row.get("label") or ""),
- "type": str(row.get("type") or "text"),
- "source_field": str(row.get("rawName") or ""),
- "enabled": bool(row.get("enabled"))}
- for row in rows],
- [{"source_field": str(name),
- "reason": "This source field was captured from Odoo but the field catalogue did not "
- "offer it."}
- for name in sorted(raw_fields or {}) if str(name) not in offered_raw],
- )
- return out
-
-
-def _spec_field(item, name):
- """One attribute off a C2 table spec, whether lane D ships dataclasses or dicts."""
- if isinstance(item, dict):
- return item.get(name)
- return getattr(item, name, None)
-
-
-def _connector_spec_items(connector_key):
- """C2's raw table specs for one connector, as a sequence of items, or `()`.
-
- ⭐⭐ W53-T30 — EXTRACTED FROM `_connector_spec_tables` (W53-T27) SO THERE IS STILL EXACTLY ONE
- IMPORT PATH TO LANE D'S MODULE. Two consumers now read the spec: the Databases tab wants
- `native` and `label`, the relationship graph wants `links`. A second `from core import
- connector_spec` with its own lookup order is how the two would start disagreeing about which
- attribute of D's module is the spec, and only one of them would be exercised by any given
- tenant. The lookup order, the failure posture and the reasons for both are unchanged and are
- documented on `_connector_spec_tables` below.
- """
- try:
- from core import connector_spec # noqa: PLC0415 - lane D's module (C2)
- except Exception: # noqa: BLE001
- return ()
- key = str(connector_key or "")
- # The directory uses the keychain type; the table registry uses the provider name.
- if key == "amazon_sp":
- key = "amazon"
- found = None
- for name in ("tables_for", "for_connector", "tables", "spec", "spec_for"):
- fn = getattr(connector_spec, name, None)
- if not callable(fn):
- continue
- try:
- got = fn(key)
- except Exception: # noqa: BLE001
- continue
- if got:
- found = got
- break
- if not found:
- for name in ("SPECS", "TABLES", "CONNECTOR_TABLES"):
- blob = getattr(connector_spec, name, None)
- if isinstance(blob, dict):
- found = blob.get(key) or list(blob.values())
- elif isinstance(blob, (list, tuple)):
- found = list(blob)
- if found:
- break
- return list(found.values()) if isinstance(found, dict) else list(found or ())
-
-
-def _connector_spec_links(connector_key):
- """C2's DECLARED relations for one connector: `{table key: [(from, target, to, label)]}`.
-
- ⭐⭐ W53-T30 / C2 — THE GRAPH'S SECOND SOURCE OF EDGES, and the contract's own words for it:
- each spec row carries `links`, a list of `(from_col, target_key, to_col, cardinality)`. Today
- every edge on this graph is read off a stored FIELD bag, which can only ever describe a
- relation somebody wrote a column for; a spec-declared relation between two tables that carry
- no link column has nowhere else to come from.
-
- ⛔ AN EDGE FROM HERE IS HELD TO THE SAME TWO PROOFS AS ONE FROM A FIELD BAG, in the caller:
- both endpoints must be permitted NODES on this graph and both named columns must be VISIBLE
- to this reader. A spec is a declaration about the shipped schema, never a permission decision
- about a person, and an edge drawn on the strength of the declaration alone would name a column
- an administrator hid from the account looking at it.
-
- ⛔ AND IT IS NOT AN OUTAGE WHEN LANE D'S MODULE IS ABSENT: `{}` means "no spec-declared
- relations", the field bags still answer, and the graph is exactly what it is today.
-
- ⚠ `cardinality` IS NOT A LABEL and is not read as one. The wire's `label` is what the reader
- sees on the line, and "one to many" is a fact about the join rather than a name for it, so the
- caller labels a spec edge with the SOURCE COLUMN's own label instead.
- """
- out = {}
- for item in _connector_spec_items(connector_key):
- table_key = str(_spec_field(item, "key") or "")
- links = _spec_field(item, "links")
- if not table_key or not isinstance(links, (list, tuple)):
- continue
- edges = []
- for link in links:
- if not isinstance(link, (list, tuple)) or len(link) < 3:
- continue
- from_col = str(link[0] or "").strip()
- target_key = str(link[1] or "").strip()
- to_col = str(link[2] or "").strip()
- if from_col and target_key and to_col:
- edges.append((from_col, target_key, to_col))
- if edges:
- out[table_key] = edges
- return out
-
-
-def _connector_spec_tables(connector_key):
- """C2's spec for one connector as `{table key: {"native", "label"}}`, or `{}`.
-
- ⛔⛔ `core.connector_spec` IS LANE D'S MODULE AND IT IS NOT IN THIS TREE YET. C2 splits the
- work: lane D declares the spec (`key`, `native`, `source`, `label`, `pinned`, `identity`,
- `links`) and this lane CONSUMES it through `GET /connectors/{key}/detail`. So the import is
- defensive and the fallback is the in-tree source each value already has — Odoo's native model
- from `odoo_relational.ODOO_TABLE_MODELS`, its default label from `grid_choices`, Amazon's
- report type from `connectors_amazon.REPORT_TYPES` and its label from `LABELS_AMAZON`. When D's
- module lands, the spec WINS with no edit here.
-
- ⚠ D'S EXACT API IS NOT FROZEN, so the lookups tried are written down rather than guessed at by
- the next reader: the callables `tables_for(key)`, `for_connector(key)`, `tables(key)`,
- `spec(key)`, `spec_for(key)`, then the module attributes `SPECS`, `TABLES`,
- `CONNECTOR_TABLES` (a dict keyed by connector, a dict keyed by table, or a flat sequence).
- Whatever answers first is normalised by TABLE KEY, and only the keys this connector already
- declares are ever consulted — so a spec carrying other connectors' tables is harmless.
- ⚠ W53-T30 — THAT LOOKUP NOW LIVES IN `_connector_spec_items` ABOVE, unchanged, because the
- relationship graph reads the same spec for its `links`. One import path, two readings.
-
- ⛔ A SPEC THAT CANNOT BE READ IS NOT AN OUTAGE. Every failure returns `{}` and the row keeps
- the in-tree value it has today: C2 is a better source for these two strings, never the only
- one, and a Databases tab that 500s because a sibling lane's module changed shape would be a
- disclosure screen taken down by a naming question.
- """
- out = {}
- for item in _connector_spec_items(connector_key):
- table_key = str(_spec_field(item, "key") or "")
- if not table_key:
- continue
- out[table_key] = {"native": str(_spec_field(item, "native") or ""),
- "label": str(_spec_field(item, "label") or "")}
- return out
-
-
-def _database_name_overrides(rt):
- """This tenant's per-database name overrides: `{table key: {"name": ...}}`, or `{}`.
-
- ⛔⛔ `nav_meta` IS THE STORE, AND IT IS READ THROUGH THE NAVIGATION RAIL'S OWN READER. R9,
- verbatim: *"the rename database under the Navigation rail also works, it should then sync to
- the navigation in the connector too."* That rename has written `nav_meta` since wave 19
- (`routes_nav.save_nav_meta`), and `routes_tables.ut_label` already resolves a served name as
- nav_meta's name, then the definition's label, then the key. Calling `routes_nav._read_nav_meta`
- rather than re-deriving the document shape here is what makes the rail and this tab provably
- ONE fact instead of two answers to one question.
-
- ⛔ AND IT IS WHY NOTHING HERE WRITES A TABLE DEFINITION'S LABEL. `user_tables.set_label` is
- rewritten by a connector re-sync (`odoo_relational.apply_plan` rebuilds definitions), so an
- override stored there would be silently clobbered on the next sync, which is precisely the
- R9 violation this ticket exists to prevent.
-
- ⚠ ONE READ FOR THE WHOLE LIST, never one per row: `store.get` deep copies the tenant document
- on every call, and this list is eight rows on Odoo and seven on Amazon.
- ⚠ A store blip is "no overrides", never a failed page. The default name is still true.
- """
- try:
- import routes_nav # noqa: PLC0415 - lazy, avoids a cycle
- meta = routes_nav._read_nav_meta(rt) or {} # noqa: SLF001 - the ONE reader (R9)
- except Exception: # noqa: BLE001
- return {}
- return {str(k): v for k, v in meta.items() if isinstance(v, dict)}
-
-
-def _rename_rights(rt, uname, is_admin, keys):
- """`{table key: {exists, canRename}}` for R9's write wall and its prerequisite.
-
- ⛔ THE SAME PREDICATE `routes_nav.save_nav_meta` APPLIES TO A `ut_` KEY, not a second one:
- `user_tables.may_open`, i.e. the table's creator, a tenant admin, or a share grant. Two ideas
- of who may rename a database would show one person a control that the other door refuses.
-
- ⛔ A DATABASE THIS CONNECTION HAS NOT CREATED YET IS NOT RENAMEABLE, and that is not a
- technicality: `nav_meta` is keyed by TABLE key, so a name stored for a table nobody can open
- would sit in the tenant's document naming nothing, and the rail would never show it.
-
- ⭐ W56-T31: `enabled` USED TO MEAN "will be included in the next sync", while the screen
- read as though the database already existed. `canRename` was telling the truth about a table
- that did not exist. Returning `exists` from this same projection lets the client distinguish
- that state from a reader who may not open an existing table.
-
- ⚠ IT IS SERVED PER ROW because the answers ARE per row. One `lend_defs` read answers the
- whole list; that projection carries definitions without rows (~0.1% of the document's bytes).
- """
- try:
- import core.user_tables as user_tables # noqa: PLC0415 - lazy, as elsewhere
- definitions = user_tables.lend_defs(rt)
- facts = {}
- for key in keys:
- exists = bool(user_tables.get(key, st=definitions))
- facts[key] = {
- "exists": exists,
- "canRename": bool(str(key).startswith("ut_") and exists
- and user_tables.may_open(
- key, uname, bool(is_admin), st=definitions)),
- }
- return facts
- except Exception: # noqa: BLE001
- # Fail closed. A wall that could not be read is not a wall that admitted you.
- return {key: {"exists": False, "canRename": False} for key in keys}
-
-
-def _named_databases(rt, is_admin, uname, rows):
- """The LAST step of every databases list, Odoo's and Amazon's alike: the three names.
-
- One place, deliberately. Each half of `_detail_databases` builds its own `native` and its own
- shipped default from its own source, and then BOTH come through here — so "what is this
- database called for this tenant" is answered once, by `schema_overlay.merge_database_labels`,
- and a third connector cannot arrive with a fourth opinion about a blank override.
- """
- named = schema_overlay.merge_database_labels(rows, _database_name_overrides(rt))
- rights = _rename_rights(rt, uname, is_admin, [str(row.get("key") or "") for row in named])
- for row in named:
- facts = rights.get(str(row.get("key") or "")) or {}
- row["exists"] = facts.get("exists") is True
- row["canRename"] = facts.get("canRename") is True
- return named
-
-
-def _clean_scraper_recipe(raw):
- """Return T11's persisted recipe shape and explicit structural validation errors.
-
- This function is intentionally pure. Preview and creation both call it, so opening a saved
- recipe cannot reinterpret it differently, and validation cannot accidentally become a paid
- provider probe.
- """
- value = raw if isinstance(raw, dict) else {}
- kind = str(value.get("kind") or "").strip().lower()
- errors = []
- if kind not in SCRAPER_RECIPE_KINDS:
- errors.append("Choose Instagram, TikTok, or Google Maps.")
- return {}, errors
-
- limits = value.get("limits") if isinstance(value.get("limits"), dict) else {}
- schedule = value.get("schedule") if isinstance(value.get("schedule"), dict) else {}
- schedule_mode = str(schedule.get("mode") or "manual").strip().lower()
- if schedule_mode != "manual":
- errors.append("This recipe must use the manual Run now schedule.")
-
- if kind == "google_maps":
- seed = value.get("seed") if isinstance(value.get("seed"), dict) else {}
- search = str(seed.get("search") or value.get("search") or "").strip()
- geography = str(seed.get("geography") or value.get("geography") or "").strip()
- raw_cap = limits.get("records", value.get("limit"))
- if not search:
- errors.append("Enter a category or search term.")
- if not geography:
- errors.append("Enter a geography.")
- if raw_cap in (None, ""):
- errors.append("Enter a result cap.")
- cap = 0
- else:
- try:
- cap = int(raw_cap)
- except (TypeError, ValueError):
- cap = 0
- errors.append("The result cap must be a whole number.")
- if raw_cap not in (None, "") and (cap < 1 or cap > 500):
- errors.append("The result cap must be between 1 and 500.")
- return ({"kind": kind, "source": "places",
- "seed": {"search": search, "geography": geography},
- "limits": {"records": cap}, "schedule": {"mode": "manual"}}, errors)
-
- seed = str(value.get("seed") or "").strip()
- raw_limit = limits.get("posts", value.get("limit"))
- post_type = str(limits.get("postType") or value.get("postType") or "video").strip().lower()
- if not seed:
- errors.append(f"Enter an {kind.title()} profile seed.")
- if raw_limit in (None, ""):
- errors.append("Enter a last-N post limit.")
- limit = 0
- else:
- try:
- limit = int(raw_limit)
- except (TypeError, ValueError):
- limit = 0
- errors.append("The post limit must be a whole number.")
- if raw_limit not in (None, "") and (limit < 1 or limit > SCRAPER_RECIPE_MAX_ROWS):
- errors.append(f"The post limit must be between 1 and {SCRAPER_RECIPE_MAX_ROWS}.")
- allowed_types = ("video", "image", "carousel") if kind == "instagram" else ("video",)
- if post_type not in allowed_types:
- errors.append("Choose video, image, or carousel posts."
- if kind == "instagram" else
- "TikTok's native profile window supports video posts only.")
- return ({"kind": kind, "source": "profile", "seed": seed,
- "limits": {"posts": limit, "postType": post_type},
- "schedule": {"mode": "manual"}}, errors)
-
-
-def _scraper_recipe_fields(kind):
- """Stable user-table schema for one recipe, derived from the shipped normalisers."""
- if kind == "google_maps":
- return [dict(field) for field in _MAPS_RECIPE_FIELDS]
- import automation_engine as engine # noqa: PLC0415 - existing schema owner
- source = engine.POST_FIELDS if kind == "instagram" else engine.TT_POST_FIELDS
- # Links and rollups target the shared automation tables. A recipe database is independent,
- # so copying those references would create a plausible but broken relationship graph.
- return [dict(field) for field in source
- if isinstance(field, dict) and field.get("type") not in ("link", "rollup")]
-
-
-def _scraper_plan(recipe):
- """Network-free setup preview: declared provider, modelled spend, cap, and credential state."""
- kind = str((recipe or {}).get("kind") or "")
- limit_key = "records" if kind == "google_maps" else "posts"
- limit = int(((recipe or {}).get("limits") or {}).get(limit_key) or 0)
- if kind == "google_maps":
- import providers # noqa: PLC0415 - registry owns cost/capability
- apify = providers.PROVIDERS["apify"]
- capability = apify.cap("google_maps_stores")
- per_record = float(capability.cost_per_record or 0.0) if capability else 0.0
- amount = round(per_record * limit, 4)
- return {"provider": apify.label, "credentialReady": apify.configured(),
- "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit,
- "canRun": True,
- "note": "Modelled from the declared per-place rate; the provider receipt "
- "reconciles the bounded result count."}
-
- import providers # noqa: PLC0415 - registry is the authority
- bright = providers.PROVIDERS["brightdata"]
- apify = providers.PROVIDERS["apify"]
- if kind == "tiktok":
- per_record = float(bright.cap("tt_post_metrics").cost_per_record or 0.0)
- profile_cost = float(bright.cap("tt_profile").cost_per_record or 0.0)
- estimate_usd = profile_cost + (per_record * limit)
- provider_label = bright.label
- credential_ready = bright.configured()
- else:
- # The established Instagram runner can use Bright Data and its Apify fallback. The cap
- # includes both post metrics and view enrichment, so it never hides the worst declared
- # cost from the person authorising Run now.
- bd_cost = float(bright.cap("ig_post_metrics").cost_per_record or 0.0)
- profile_cost = float(bright.cap("ig_profile").cost_per_record or 0.0)
- view_cost = float(apify.cap("ig_post_views").cost_per_record or 0.0)
- estimate_usd = profile_cost + ((bd_cost + view_cost) * limit)
- provider_label = (bright.label if bright.configured() or not apify.configured()
- else apify.label)
- credential_ready = bright.configured() or apify.configured()
- amount = round(estimate_usd, 4)
- return {"provider": provider_label, "credentialReady": bool(credential_ready),
- "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit,
- "canRun": True, "note": "Modelled from declared per-record list rates."}
-
-
-def _scraper_databases(rt, is_admin, uname):
- """Saved recipe databases this reader may open, with no row payloads or secrets."""
- try:
- import core.user_tables as user_tables
- definitions = user_tables.all_defs(rt)
- lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
- except Exception: # noqa: BLE001
- return []
- rows = []
- for key, definition in (definitions or {}).items():
- recipe = definition.get("scraperRecipe") if isinstance(definition, dict) else None
- if not isinstance(recipe, dict):
- continue
- try:
- if not user_tables.may_open(key, uname, bool(is_admin), st=lent):
- continue
- except Exception: # noqa: BLE001
- continue
- cleaned, errors = _clean_scraper_recipe(recipe)
- if errors:
- continue
- plan = _scraper_plan(cleaned)
- # Recipe existence alone is not enablement. The creation door stamps both ownership
- # markers; a legacy/malformed row without them is disclosed as disabled and cannot run.
- enabled = (definition.get("connectorKey") == "scraper"
- and definition.get("recordMode") == "automation")
- rows.append({"key": str(key), "label": str(definition.get("label") or key),
- "defaultLabel": str(definition.get("label") or key),
- "native": cleaned["kind"], "enabled": enabled, "exists": True,
- "canRename": True, "sourceRows": None, "populated": None,
- "fields": [dict(field) for field in (definition.get("fields") or ())
- if isinstance(field, dict)],
- "excluded": [], "sharesFieldsWith": [], "scraperRecipe": cleaned,
- "syncHealth": _stored_database_sync_health(
- definition.get(user_tables.CONNECTOR_HEALTH_FIELD), enabled),
- "scraperRuns": [dict(item) for item in (definition.get("scraperRuns") or ())
- if isinstance(item, dict)],
- "runPath": f"/connectors/scraper/databases/{key}/run",
- "canRun": enabled and plan["canRun"] is True})
- return _named_databases(rt, is_admin, uname, rows)
-
-
-def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None):
- """C4's `databases` half: what this connector materialises, and the fields behind each one.
-
- ⛔ ODOO IS THE ONLY INTROSPECTABLE CONNECTOR TODAY and every other card answers `[]`. That is
- the contract's own instruction, and the alternative would be a list of database names no
- connector actually creates.
-
- ⛔ THE ROW SURVIVES FOR A NON-ADMIN; ONLY THE CATALOGUE HALVES EMPTY. Returning `[]` to a
- member would be indistinguishable from *"this connector creates no databases"*, which is the
- confidently-wrong shape this module's header was written about. A database's NAME and whether
- it is switched on are facts about their own workspace; the raw Odoo field catalogue is
- administrator-only (`_odoo_admin` gates the route that serves it) and stays that way.
-
- Fields with the same source model have independent enablement keys in each database.
- `sharesFieldsWith` remains an empty list for existing clients of the older shared switches.
-
- ⭐⭐ W53-T27 / R9 / C2 — THREE NAMES PER ROW, AND THEY ARE THREE DIFFERENT FACTS. `native` is
- the VENDOR's own identifier (`res.partner`, or Amazon's report type) and is never editable by
- anybody; `defaultLabel` is OUR shipped default; `override` is this tenant's own rename, held
- in the `nav_meta` bucket the navigation rail already writes. `label` stays the EFFECTIVE name
- because every existing reader of it — this tab's heading, `_detail_overview`'s `creates` list
- — already means "what this database is called today", and serving the un-overridden default
- there would be the drift R9 exists to close.
-
- ⛔ ALL THREE SURVIVE FOR A NON-ADMIN, like `key` and `enabled` above them and for the same
- reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
- is walled, per row, by `canRename`.
- """
- if str(connector_key) == "scraper":
- return _scraper_databases(rt, is_admin, uname)
- if not connector_directory.is_introspectable(connector_key):
- return []
- spec = _connector_spec_tables(connector_key)
- if str(connector_key) == "amazon_sp":
- return _named_databases(rt, is_admin, uname, _amazon_databases(rt, is_admin, spec))
- try:
- import odoo_relational
- namespace = (odoo_relational.connection_namespace(rt, connection_id)[1]
- if connection_id else None)
- choices = odoo_relational.grid_choices(rt, connection_id)
- models = dict(getattr(odoo_relational, "ODOO_TABLE_MODELS", {}) or {})
- # This read never contacts Odoo. A 499-model exact pass measured 67.87 s after the
- # catalogue was warm, so modal detail must never turn a missing cache into inline work.
- census = odoo_relational.population_census(rt, namespace)
- except Exception: # noqa: BLE001
- return []
- health = odoo_health_receipt(rt, connection_id)
- health_by_key = {str(item.get("databaseKey") or ""): item
- for item in (health.get("databases") or ())}
- census_at = str(census.get("capturedAt") or "")
- counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
- catalog = {}
- if is_admin:
- try:
- catalog = _odoo_catalog_by_model(rt, namespace)
- except Exception: # noqa: BLE001
- catalog = {}
- enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {}
- # Connection resolution reaches keychain.list_entries and therefore the store (Postgres).
- # It ran 12,824 times for fields and database identities, then 501 with a per-key memo.
- # Resolve the shared default lazily once instead. This local belongs only to this detail
- # build: request scope keeps the answer per runtime and tenant, never process-wide.
- default_connection_id = ...
-
- def resolve_connection_id(key):
- nonlocal default_connection_id
- base, cid = odoo_relational.split_table_key(key)
- if base is None:
- return None
- if cid:
- return cid
- if default_connection_id is ...:
- default_connection_id = (odoo_relational.default_connection_id(rt)
- or odoo_relational.DEFAULT_CONNECTION)
- return default_connection_id
-
- out = []
- for choice in choices:
- key = str(choice.get("key") or "")
- resolved_connection_id = resolve_connection_id(key)
- model = str(choice.get("model") or models.get(key) or "")
- fields, excluded = catalog.get(model, ([], []))
- scoped_fields = []
- for field in fields:
- field_key = _odoo_field_key(rt, key, model, field["source_field"],
- connection_id=resolved_connection_id)
- scoped_fields.append({**field, "key": field_key,
- "enabled": enabled_fields.get(field_key,
- field.get("enabled") if namespace is None else False) is True})
- declared = spec.get(choice.get("baseKey") or key) or {}
- # Counts are model-level facts. Databases backed by the same model deliberately show the
- # same sourceRows number; inventing a per-database domain here would make the number lie.
- source_rows = counts.get(model) if census_at and model in counts else None
- database_health = health_by_key.get(key) or {
- # Source rows is a model census. It is not this governed database's materialized
- # row count, especially when two curated databases share one source model.
- "databaseKey": key, "state": "unknown", "rowCount": None,
- "lastSuccess": None, "currentLagSeconds": None, "recordRefreshed": None,
- "failure": None, "stages": _empty_odoo_stages()}
- out.append({"key": key,
- "identity": connector_sync.database_identity(
- "odoo", resolved_connection_id, key, model),
- "label": str(declared.get("label") or choice.get("label") or key),
- "defaultLabel": str(declared.get("label") or choice.get("label") or key),
- "native": str(declared.get("native") or model),
- "enabled": bool(choice.get("enabled")),
- "sourceRows": source_rows,
- "populated": None if source_rows is None else source_rows > 0,
- # Data-free client receipt: source/mirror/materialization/reconciliation/
- # freshness are distinct, and one database's failure cannot make a sibling
- # look newly successful.
- "sync": database_health,
- "syncHealth": _odoo_database_sync_health(
- database_health, bool(choice.get("enabled"))),
- "fields": scoped_fields,
- "excluded": [dict(item) for item in excluded],
- "sharesFieldsWith": []})
- return _named_databases(rt, is_admin, uname, out)
-
-
-def _amazon_report_name(table):
- """The report type this Amazon table reads, whichever key declares it.
-
- ⛔ SETTLEMENTS DECLARE `report_discover`, NOT `report` (W60-T36), because Amazon issues them on
- its own schedule and refuses a createReport for that type: it is a getReports DISCOVERY filter
- and is deliberately absent from `_APPROVED_REPORT_TYPES`. Reading only `report` here printed an
- empty Source name for settlements on the connector screen, which reads as "this table has no
- report" when the truth is "this table is not requested, it is listed".
- """
- import connectors_amazon as cx # noqa: PLC0415 - provider owns state
- spec = cx.REPORT_TYPES.get(table) or {}
- return str(spec.get("report") or spec.get("report_discover") or "")
-
-
-def _amazon_databases(rt, is_admin, spec=None):
- """⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM.
-
- Fills the shared `ConnectorDatabase` shape and adds an administrator-only Amazon `sync`
- block. Members retain the shared shape without the field catalogue or sync bookkeeping.
-
- ⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five
- of the seven tables have NO published schema, because Amazon publishes artifacts only for its
- JSON analytics reports and every flat-file report we depend on has none. Their columns are
- measured from the report document's own header at first sync, so before that they genuinely
- have zero fields. A screen that showed an empty field list with no explanation would say
- *"this database has no fields"* when the truth is *"nobody has run it yet"* — the exact
- confusion `_detail_introspection`'s header was written about, one layer in.
-
- ⛔ `sharesFieldsWith` IS REAL HERE, NOT COPIED. `orders` and `order_items` are BOTH filled from
- ONE report document (`GET_FLAT_FILE_ALL_ORDERS_DATA_*`, which is one row per order ITEM), so
- they share a header and therefore a field list. That is the same class of fact the Odoo pairs
- carry, measured the same way: derived from the report map, never hand-typed.
-
- ⚠ NO FIELD TOGGLES. `canToggleFields` is False for Amazon in `_detail_introspection`; there is
- no `PATCH /connectors/amazon_sp/schema/{field}` route and this wave does not build one. Fields
- are disclosed, not switched. Database enablement is a separate provider capability and route,
- never the generic Odoo-table flag consumed by `DatabasesTab`.
- """
- try:
- from harness import amazon_schema
- import connectors_amazon as cx
- except Exception: # noqa: BLE001
- return []
- enablement = cx.read_database_enablement(rt)
- enabled = enablement.get("databases") or {}
- cat = amazon_schema.catalogue()
- tables = cat.get("tables") or {}
- states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
-
- # Which OTHER declared tables are filled from the SAME report document. Derived from the
- # report map so a table added to `REPORT_TYPES` gets this disclosure for free.
- by_report = {}
- for name in amazon_schema.PROVENANCE:
- rep = _amazon_report_name(name)
- if rep:
- by_report.setdefault(rep, []).append("ut_amzn_" + name)
-
- out = []
- for name, provenance in amazon_schema.PROVENANCE.items():
- key = "ut_amzn_" + name
- spec = tables.get(name) or {}
- cols = spec.get("columns") or []
- rep = _amazon_report_name(name)
- fields = [{"key": str(c.get("name") or ""),
- "label": str(c.get("name") or "").replace("_", " ").capitalize(),
- "type": "text" if str(c.get("type")) == "VARCHAR" else "number",
- "source_field": str(c.get("name") or ""),
- # ⚠ EVERY DISCLOSED FIELD IS ON. There is no per-field switch for Amazon, and
- # reporting `enabled: False` would describe a control that does not exist.
- "enabled": True}
- for c in cols] if is_admin else []
- excluded = []
- if provenance == "measured_at_sync" and not cols:
- excluded.append({
- "source_field": "",
- "reason": (amazon_schema.PROVENANCE_COPY["measured_at_sync"]
- + f" The report is {rep}." if rep
- else amazon_schema.PROVENANCE_COPY["measured_at_sync"])})
- # ⭐⭐ W53-T27 / C2 — THE SAME THREE FACTS AS THE ODOO ROWS, one source swapped. Amazon
- # has no per-table model name; the identifier AMAZON itself uses for what fills a table is
- # the REPORT TYPE (`rep`, already computed above for `sharesFieldsWith`). That is exactly
- # what `native` carries: the vendor's own name for the thing, never one we invented. It is
- # empty for a table whose report type is unknown, and the tab says so rather than guessing.
- declared = (spec or {}).get(key) or {}
- raw_sync = (states or {}).get(name) if is_admin else None
- is_enabled = enabled.get(key) is True
- out.append({"key": key,
- "label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
- "defaultLabel": str(declared.get("label")
- or LABELS_AMAZON.get(key) or key),
- "native": str(declared.get("native") or rep),
- # W55-T19 / R7 — AMAZON'S OWN tenant-local answer. This is deliberately not
- # read from Odoo's grid config, and disabling it leaves every stored row and
- # definition untouched: only this provider config document changes.
- "enabled": is_enabled,
- **({"sync": raw_sync} if is_admin else {}),
- "syncHealth": _amazon_database_sync_health(raw_sync, is_enabled),
- "fields": fields,
- "excluded": excluded,
- "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
- return out
-
-
-#: What an empty `excluded` list means for AMAZON, which is not what it means for Odoo.
-#:
-#: ⛔ A SEPARATE SENTENCE ON PURPOSE. Odoo's basis describes a DIFF of a live snapshot against a
-#: catalogue. Amazon has no snapshot and no diff: a table either has a published schema or it
-#: measures its columns at first sync. Reusing Odoo's wording would describe a computation that
-#: does not happen here, which is the same class of error as reusing its toggles.
-_AMAZON_EXCLUDED_BASIS = (
- "Amazon publishes a schema for its JSON analytics reports only. A table with no published "
- "schema reads its columns from the report's own header the first time it syncs, so it lists "
- "no fields until then. Nothing is being held back from you.")
-
-
-#: Table labels for the Amazon disclosure. ⛔ NO EM DASH OR EN DASH: these reach a screen.
-LABELS_AMAZON = {
- "ut_amzn_orders": "Amazon orders",
- "ut_amzn_order_items": "Amazon order items",
- "ut_amzn_listings": "Amazon listings",
- "ut_amzn_settlements": "Amazon settlements",
- "ut_amzn_inventory": "Amazon inventory",
- "ut_amzn_pricing": "Amazon pricing and Buy Box",
- "ut_amzn_brand_analytics": "Amazon brand analytics",
-}
-
-
-#: What an EMPTY `excluded` list actually means, said once, on the server that computes it.
-#:
-#: ⛔⛔ THE TRAP THIS CLOSES, MEASURED: `_odoo_catalog_by_model` builds `excluded` as a DIFF of the
-#: snapshot's raw fields against the catalogue, and the catalogue offers every raw field it
-#: captured — so the diff is EMPTY TODAY BY CONSTRUCTION. A tab that rendered "Excluded: none"
-#: over that would be telling an administrator that this workspace holds back nothing from Odoo,
-#: which is a claim about POLICY that the number cannot support. The empty list means the
-#: catalogue dropped nothing it saw; it says nothing at all about what Odoo never reported.
-#:
-#: ⚠ IT IS SERVED RATHER THAN WRITTEN IN THE CLIENT because the sentence is a description of a
-#: server computation. A client copy would go stale the day the catalogue starts filtering, and
-#: nothing would go red.
-_EXCLUDED_BASIS = ("A field is excluded when this workspace captured it from Odoo and the field "
- "catalogue did not offer it. Anything Odoo never reported cannot appear in "
- "this list at all.")
-
-
-def _detail_introspection(rt, is_admin, row):
- """R24's half of the Databases tab: may this connector be disclosed, and to THIS reader?
-
- ⭐ AN ADDITION TO C4, DECLARED, and the same kind of addition `_detail_connect` is. C4 pins
- the `databases` list and says nothing about how a client tells "this connector has no schema"
- apart from "this connector's schema could not be read". Those two are the same empty list and
- they mean opposite things, so the difference is stated rather than inferred.
-
- ⛔ EVERY FLAG HERE IS A CAPABILITY THE SERVER WILL HONOUR, not a guess the client makes. The
- field toggle is `PATCH /connectors/odoo/schema/{field_key}`, which `_odoo_admin` gates and
- which raises 409 `odoo_schema_unavailable` while no complete snapshot exists — so
- `canToggleFields` carries BOTH conditions and the client renders no control that could only
- ever 409. A button that always fails is R7's banned shape.
-
- ⚠ `fieldsVisible` IS SEPARATE FROM `canToggleFields` ON PURPOSE. A member sees the database
- names and their on/off state (their own workspace's facts) and no field catalogue at all
- (`_odoo_admin` gates the route that serves it); saying which of the two is missing is what
- stops "administrator only" being rendered as "this database has no fields".
- """
- key = str(row.get("key") or "")
- if key == "scraper":
- # Recipe databases publish their saved, stable schema to every reader who may open the
- # database. They have no generic schema-toggle route, so no write capability is implied.
- return {"introspectable": True, "reason": "", "fieldsVisible": True,
- "canToggleFields": False, "canToggleTables": False,
- "catalogueComplete": True, "capturedAt": "", "excludedBasis": "",
- "censusAt": "", "censusComplete": False, "censusDurationMs": 0,
- "censusModelsCounted": 0, "censusModelsWithRows": 0}
- if not connector_directory.is_introspectable(key):
- reason = ("Nothing is connected behind this listing yet, so there is no schema to read."
- if str(row.get("state") or "") == connector_directory.CATALOGUE_STATE
- else "This connector does not publish a field catalogue, so the databases it "
- "creates cannot be listed field by field.")
- return {"introspectable": False, "reason": reason, "fieldsVisible": False,
- "canToggleFields": False, "canToggleTables": False,
- "catalogueComplete": False, "capturedAt": "", "excludedBasis": "",
- "censusAt": "", "censusComplete": False, "censusDurationMs": 0,
- "censusModelsCounted": 0, "censusModelsWithRows": 0}
- if key == "amazon_sp":
- # ⭐⭐ W52 · R6. Introspectable under the AMENDED test (a real schema artifact, not a
- # `fields_get`), and deliberately with NO generic toggles: there is no Amazon field-toggle
- # route, and `canToggleTables` belongs to Odoo's config surface. W55-T19 serves Amazon's
- # database switches through the separate `enablement` capability instead, so this flag
- # stays False and can never send Amazon through an Odoo endpoint.
- #
- # ⛔ `catalogueComplete` IS FALSE AND SAYING SO IS THE HONEST ANSWER. Only two of the seven
- # tables have a published artifact; the other five measure their columns from the report's
- # own header at first sync. Reporting `True` would claim a completeness that does not
- # exist, and reporting `introspectable: False` would hide the two that ARE published.
- try:
- from harness import amazon_schema
- cat = amazon_schema.catalogue()
- tables = cat.get("tables") or {}
- complete = bool(tables) and all((t.get("columns") or []) for t in tables.values())
- captured = str(cat.get("fetched_utc") or "")
- except Exception: # noqa: BLE001
- complete, captured = False, ""
- return {"introspectable": True,
- "reason": "",
- "fieldsVisible": bool(is_admin),
- "canToggleFields": False,
- "canToggleTables": False,
- "catalogueComplete": complete,
- # ⚠ THE CATALOGUE'S OWN FETCH DATE, NEVER TODAY'S. R6 is explicit that an
- # unstamped pinned catalogue is a stale seed waiting to happen, and this screen is
- # the only place the staleness becomes visible.
- "capturedAt": captured,
- "excludedBasis": _AMAZON_EXCLUDED_BASIS,
- "censusAt": "", "censusComplete": False, "censusDurationMs": 0,
- "censusModelsCounted": 0, "censusModelsWithRows": 0}
- complete = False
- captured = ""
- census = {}
- try:
- import core.user_tables as user_tables
- snapshot = user_tables.odoo_schema_snapshot(rt)
- import odoo_relational
- discovered = odoo_relational.readable_model_catalog(rt)
- census = odoo_relational.population_census(rt)
- has_discovery = bool(discovered.get("capturedAt"))
- complete = (discovered.get("complete") is True if has_discovery
- else snapshot.get("complete") is True)
- captured = str((discovered.get("capturedAt") if has_discovery
- else snapshot.get("capturedAt")) or "")
- except Exception: # noqa: BLE001
- complete, captured = False, ""
- return {"introspectable": True,
- "reason": "",
- "fieldsVisible": bool(is_admin),
- # BOTH conditions, because the write door enforces both.
- "canToggleFields": bool(is_admin) and complete,
- # ⭐ R22: ANY table can be switched off, not only the ones we mark optional. The write
- # is `PUT /admin/connectors/odoo/config`, which is an administrator's door and accepts
- # every key `grid_choices` names, including all of them at once.
- "canToggleTables": bool(is_admin),
- "catalogueComplete": complete,
- "capturedAt": captured,
- "excludedBasis": _EXCLUDED_BASIS,
- "censusAt": str(census.get("capturedAt") or ""),
- "censusComplete": census.get("complete") is True,
- "censusDurationMs": int(census.get("durationMs") or 0),
- "censusModelsCounted": int(census.get("modelsCounted") or 0),
- "censusModelsWithRows": int(census.get("modelsWithRows") or 0)}
-
-
-def _detail_overview(row, databases, graph):
- """C4's `overview` half: what the modal's first tab reads, top to bottom.
-
- `creates` is the ENABLED table LABELS this connector stands behind: the declared grids when the
- connector declares any, and otherwise the tables its own permitted graph actually found. Both
- are real readings of THIS workspace; neither is a provider template.
- """
- creates = [str(entry.get("label") or "") for entry in (databases or [])
- if entry.get("enabled") is True and str(entry.get("label") or "")]
- if not databases and isinstance(graph, dict) and graph.get("status") == "ready":
- creates = [str(table.get("label") or "") for table in (graph.get("tables") or ())
- if str(table.get("label") or "")]
- return {"key": str(row.get("key") or ""),
- "label": str(row.get("label") or ""),
- "status": str(row.get("state") or ""),
- # Empty when the server has no account to name. The client says so in a sentence
- # rather than printing a blank row.
- "account": str(row.get("connectedAs") or ""),
- # The system the data comes FROM, in the words a customer reads on the card. Never a
- # provider slug: an internal identifier on screen is exactly what `parseDirectory`
- # drops a row to avoid.
- "source": str(row.get("label") or ""),
- "creates": creates}
-
-
-def _detail_enablement(rt, is_admin, row):
- """A provider-owned enablement capability, or ``None`` when this connector has no door.
-
- Amazon is intentionally the first entry and is named here rather than inferred from whether a
- database list happened to be non-empty. Odoo keeps its model-catalogue and config routes;
- every other connector keeps no enablement affordance until its provider declares one.
- """
- if str(row.get("key") or "") != "amazon_sp":
- return None
- try:
- import connectors_amazon as cx
- return cx.enablement_capability(rt, bool(is_admin))
- except Exception: # noqa: BLE001
- # Fail closed. An unreadable provider capability is no write door.
- return {"provider": "amazon_sp", "mode": "database", "canManage": False,
- "updatePath": "", "preservesData": True, "available": False}
-
-
-def _scraper_preview(body):
- request = body if isinstance(body, dict) else {}
- raw_recipe = request.get("recipe") if isinstance(request.get("recipe"), dict) else request
- recipe, errors = _clean_scraper_recipe(raw_recipe)
- if errors:
- raise err(400, "invalid_scraper_recipe", " ".join(errors))
- plan = _scraper_plan(recipe)
- missing = [] if plan["credentialReady"] else ["credential"]
- return {"recipe": recipe, "inputs": recipe["seed"], "provider": plan["provider"],
- "credentialReady": plan["credentialReady"], "estimatedUsd": plan["estimatedUsd"],
- "costCapUsd": plan["costCapUsd"], "recordsCap": plan["recordsCap"],
- "canRun": plan["canRun"], "missing": missing, "note": plan["note"],
- # This is a contract field, not a measurement. A preview and a create never call a
- # provider, so both truthfully report zero incurred spend.
- "incurredUsd": 0.0}
-
-
-def _create_scraper_database(rt, uname, body):
- """Persist one recipe database. Contains no provider import or runner call by construction."""
- if not rt.available():
- raise err(503, "store_unavailable", "the tenant store is unavailable")
- request = body if isinstance(body, dict) else {}
- label = str(request.get("label") or "").strip()
- if not label:
- raise err(400, "missing_database_name", "Enter a database name.")
- preview = _scraper_preview(request)
- recipe = preview["recipe"]
- import core.user_tables as user_tables # noqa: PLC0415 - existing create door
- key = user_tables.create(label, uname, fields=_scraper_recipe_fields(recipe["kind"]),
- source=user_tables.AUTOMATION_SOURCE, st=rt)
- if not key:
- raise err(409, "database_not_created",
- "The database name is unavailable or this workspace reached its database limit.")
-
- def _attach(cur):
- current = dict(cur or {})
- definition = dict(current.get(key) or {})
- if not definition:
- raise RuntimeError("the new database disappeared before its recipe was attached")
- definition.update({"connectorKey": "scraper", "recordMode": "automation",
- "scraperRecipe": recipe, "scraperRuns": [],
- user_tables.CONNECTOR_HEALTH_FIELD:
- user_tables.connector_health_receipt("never_run")})
- current[key] = definition
- return current
-
- try:
- rt.update(user_tables.STORE_KEY, _attach, flush="sync")
- except Exception: # noqa: BLE001
- user_tables.delete(key, st=rt)
- raise err(503, "recipe_not_saved",
- "The recipe could not be saved, so the new database was removed.")
- return {"key": key, "label": label, "recipe": recipe, "preview": preview,
- "createdWithoutProviderCall": True, "incurredUsd": 0.0}
-
-
-def _scraper_database_run(rt, uname, is_admin, database_key):
- """The one explicit paid-work boundary for any saved scraper recipe."""
- if not rt.available():
- raise err(503, "store_unavailable", "the tenant store is unavailable")
- import core.user_tables as user_tables
- key = str(database_key or "").strip()
- definitions = user_tables.lend_defs(rt)
- definition = user_tables.get(key, st=definitions) or {}
- if not definition:
- raise err(404, "unknown_scraper_database", "This saved scraper database was not found.")
- raw_recipe = definition.get("scraperRecipe")
- recipe, errors = _clean_scraper_recipe(raw_recipe)
- if errors:
- raw_kind = str((raw_recipe or {}).get("kind") or "") if isinstance(raw_recipe, dict) else ""
- raw_limits = ((raw_recipe or {}).get("limits") or {}) if isinstance(raw_recipe, dict) else {}
- raw_cap = raw_limits.get("records") if isinstance(raw_limits, dict) else None
- try:
- bounded = 1 <= int(raw_cap) <= 500
- except (TypeError, ValueError):
- bounded = False
- if raw_kind == "google_maps" and not bounded:
- raise err(400, "maps_unbounded_request",
- "Google Maps discovery requires an explicit result cap between 1 and 500.")
- raise err(409, "invalid_saved_scraper_recipe",
- "This saved scraper recipe is no longer valid. Review its settings before running it.")
- # Run now both spends against a provider account and mutates this database. A viewer grant
- # may expose the rows without granting either power, so the ordinary open wall is too broad.
- # Reuse the database editor authority that every other mutation surface already honours.
- if not user_tables.may_edit_database(key, uname, bool(is_admin), st=definitions):
- raise err(403, "database_forbidden", "You cannot run this database recipe.")
- plan = _scraper_plan(recipe)
- previous_health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD)
- previous_health = previous_health if isinstance(previous_health, dict) else {}
-
- def record_attempt(state, error=None):
- try:
- return user_tables.record_connector_health(
- key, state, st=rt,
- last_success=previous_health.get("lastSuccess"),
- source_timestamp=previous_health.get("sourceTimestamp"),
- materialized_timestamp=previous_health.get("materializedTimestamp"),
- row_count=previous_health.get("rowCount"), error=error)
- except Exception: # noqa: BLE001
- raise err(503, "sync_status_not_saved",
- "The run did not start because its durable status could not be saved.") from None
-
- if not plan["credentialReady"]:
- code = "maps_credential_missing" if recipe["kind"] == "google_maps" else \
- "scraper_credential_missing"
- message = "Add the scraper credential in Keychain before choosing Run now."
- record_attempt("failed", {"code": code, "message": message})
- raise err(409, code, "Add the scraper credential in Keychain before choosing Run now.")
-
- # Imports and provider calls occur only below the explicit Run-now endpoint. Preview, create,
- # detail, and page load never reach either runner.
- started_at = datetime.now(timezone.utc).isoformat()
- record_attempt("running")
- key_field = "shortcode"
- if recipe["kind"] == "google_maps":
- import providers
- try:
- result = providers.discover_google_maps_stores(
- recipe["seed"]["search"], recipe["seed"]["geography"],
- recipe["limits"]["records"], captured_at=started_at)
- except providers.MapsProviderRefusal as exc:
- record_attempt("failed", {"code": exc.code, "message": exc.public_message})
- status = 409 if exc.code == "maps_credential_missing" else \
- (400 if exc.code in ("maps_unbounded_request", "maps_invalid_search") else 502)
- raise err(status, exc.code, exc.public_message)
- incoming_key = "rows"
- key_field = "place_id"
- else:
- limit = recipe["limits"]["posts"]
- post_type = recipe["limits"]["postType"]
- try:
- if recipe["kind"] == "instagram":
- import connectors_ig as runner
- run_recipe = runner.pull_ig_recipe
- else:
- import connectors_tt as runner
- run_recipe = runner.pull_tt_recipe
- result = run_recipe(recipe["seed"], limit, post_type=post_type,
- log=lambda *_args: None)
- except Exception: # noqa: BLE001
- record_attempt("failed", {
- "code": "scraper_run_failed",
- "message": "The provider could not complete this run. No result was written.",
- })
- raise err(502, "scraper_run_failed",
- "The provider could not complete this run. No result was written.")
- incoming_key = "posts"
- result = result if isinstance(result, dict) else {}
- incoming = [dict(row) for row in (result.get(incoming_key) or ()) if isinstance(row, dict)]
- state = str(result.get("state") or "error")
- if state not in ("ok", "partial"):
- incoming = []
- finished_at = datetime.now(timezone.utc).isoformat()
- source_stamps = [_sync_stamp(row.get("source_timestamp")) for row in incoming]
- source_timestamp = max((stamp for stamp in source_stamps if stamp), default=None)
- provider_counts = dict(result.get("counts") or {})
- provider_counts.setdefault("returned", len(incoming))
- provider_cost = dict(result.get("cost") or {})
- provider_cost.setdefault("currency", "USD")
- provider_cost.setdefault("estimatedUsd", plan["estimatedUsd"])
- provider_cost.setdefault("costCapUsd", plan["costCapUsd"])
- receipt = {"id": finished_at, "state": state,
- "provider": str(result.get("via") or plan["provider"]),
- "rows": len(incoming), "startedAt": started_at, "finishedAt": finished_at,
- "estimatedUsd": plan["estimatedUsd"], "costCapUsd": plan["costCapUsd"],
- "cost": provider_cost, "providerCounts": provider_counts,
- "attempts": [dict(item) for item in (result.get("attempts") or ())
- if isinstance(item, dict)],
- # Provider notes can echo a submitted handle. Run history stores operational
- # metadata only; the saved recipe already owns the input and no customer value is
- # duplicated into an audit-like receipt.
- "note": f"Provider returned {state}."}
- counts = {"inserted": 0, "updated": 0, "unchanged": 0, "skipped": 0,
- "duplicates": 0, "orphans": 0, "capped": 0}
-
- def _write(cur):
- nonlocal counts
- current = dict(cur or {})
- stored = dict(current.get(key) or {})
- if not stored:
- raise RuntimeError("the database disappeared before the run could be written")
- if incoming:
- import automation_engine as engine
- rows, counts = engine.upsert_rows(stored.get("rows") or {}, incoming, key_field)
- stored["rows"] = rows
- processed = sum(int(counts.get(name) or 0)
- for name in ("inserted", "updated", "unchanged", "skipped", "capped"))
- receipt["writeCounts"] = dict(counts)
- receipt["countReconciled"] = (
- int(provider_counts.get("returned") or 0) == len(incoming) == processed)
- history = [dict(item) for item in (stored.get("scraperRuns") or ())
- if isinstance(item, dict)]
- stored["scraperRuns"] = [receipt, *history][:SCRAPER_RUN_HISTORY_LIMIT]
- final_state = "succeeded" if state == "ok" else "partial" if state == "partial" else "failed"
- final_error = None if final_state != "failed" else {
- "code": "provider_result_failed",
- "message": "The provider returned no usable result for this database.",
- }
- stored[user_tables.CONNECTOR_HEALTH_FIELD] = user_tables.connector_health_receipt(
- final_state,
- last_success=finished_at if final_state in ("succeeded", "partial")
- else previous_health.get("lastSuccess"),
- source_timestamp=source_timestamp,
- materialized_timestamp=finished_at if final_state in ("succeeded", "partial") else None,
- row_count=len(stored.get("rows") or {}), error=final_error)
- current[key] = stored
- return current
-
- try:
- rt.update(user_tables.STORE_KEY, _write, flush="sync")
- except Exception: # noqa: BLE001
- try:
- record_attempt("failed", {
- "code": "scraper_result_not_saved",
- "message": "The provider answered, but the result could not be saved.",
- })
- except Exception: # noqa: BLE001
- pass
- raise err(503, "scraper_result_not_saved",
- "The provider answered, but the result could not be saved. Reload before retrying.")
- return {"key": key, "state": state, "counts": counts, "receipt": receipt}
-
-
-def connector_detail(rt, user, uname, is_admin, connector_key):
- """The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
-
- Pure over the session for the same reason `directory` is: a gate can drive it without a
- request, so the modal's payload is provable without a browser.
- """
- budget = _ConnectorDetailBudget()
- rows = directory(rt, uname, bool(is_admin))
- budget.check()
- key = str(connector_key or "")
- row = next((candidate for candidate in rows
- if str(candidate.get("key") or "") == key), None)
- if row is None:
- return None
- graph = relationship_graphs(rows, user, rt, uname, bool(is_admin)).get(key) or _graph_refusal(
- key, "No actual relationship graph was returned for this connection. A provider template "
- "is not shown.")
- budget.check()
- # ⚠ `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a
- # question about a PERSON and a table, not about an administrator flag alone.
- databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or ""))
- budget.check()
- overview = _detail_overview(row, databases, graph)
- budget.check()
- keychain = _detail_keychain(rt, uname, bool(is_admin), row)
- budget.check()
- enablement = _detail_enablement(rt, bool(is_admin), row)
- budget.check()
- introspection = _detail_introspection(rt, bool(is_admin), row)
- budget.check()
- return _safe_connector_status(key, {"overview": overview,
- "keychain": keychain,
- "databases": databases,
- "syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"}
- if is_admin and key in connector_sync.providers() else None),
- "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
- "retryPath": f"/connectors/{key}/sync/retry"}
- if is_admin and key == "odoo" else None),
- # W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it
- # from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints.
- "enablement": enablement,
- # ⭐ W49-T14 / R24 — WHETHER THE DATABASES TAB MAY DISCLOSE ANYTHING AT ALL, and what
- # THIS reader may do there. An addition to C4, declared on `_detail_introspection`:
- # without it an empty `databases` list is "this connector has no schema" and "this
- # connector's schema could not be read" at the same time.
- "introspection": introspection,
- # ⛔ THE `relationship_graphs` SHAPE, UNCHANGED. C4 says so in as many words, and the
- # client already has a reader for it (`parseRelationshipGraph`) that refuses a graph
- # bound to another connection. A second shape here would need a second reader.
- "relationships": graph})
-
-
-def _set_amazon_database_enablement(database_key, body, session):
- """The testable core of Amazon's tenant-scoped administrator enablement door."""
- if not session.admin:
- raise err(403, "not_admin",
- "only an administrator can change Amazon database enablement")
- if not session.runtime.available():
- raise err(503, "store_unavailable", "the tenant store is unavailable")
- request = body if isinstance(body, dict) else {}
- if set(request) != {"enabled"} or type(request.get("enabled")) is not bool:
- raise err(400, "bad_enabled", "enabled must be true or false")
-
- import connectors_amazon as cx # noqa: PLC0415 - provider owns state
- key = str(database_key or "").strip()
- try:
- state = cx.set_database_enabled(session.runtime, key, request["enabled"])
- except KeyError:
- # One refusal for blank, malformed and cross-provider keys. The response does not expose
- # the rest of the tenant's Amazon roster.
- raise err(404, "unknown_amazon_database",
- "Amazon does not declare a database with that key")
- except cx.EnablementConfigError:
- raise err(409, "amazon_enablement_unavailable",
- "the stored Amazon enablement settings are invalid. Nothing was changed.")
- except ValueError:
- raise err(400, "bad_enabled", "enabled must be true or false")
- except Exception: # noqa: BLE001
- raise err(503, "enablement_unconfirmed",
- "the server could not confirm the Amazon database setting. Reload before "
- "trying again.")
- return {"provider": cx.ENTRY_TYPE,
- "key": key,
- "enabled": state["databases"].get(key) is True,
- # This route changes only the provider config document. It never deletes a mirror or
- # a materialised table, and the wire states that invariant explicitly.
- "preservesData": True}
-
-
-# Literal and provider-owned. It appears before the generic connector detail route so future
-# dynamic route additions cannot turn Amazon enablement into an Odoo-model dispatch by accident.
-@router.patch("/connectors/amazon_sp/databases/{database_key}/enablement")
-def amazon_database_enablement(database_key: str, body: dict = Body(default=None),
- session: Session = Depends(require_session)):
- """Enable or disable one declared Amazon database for this tenant, without deleting data."""
- return _set_amazon_database_enablement(database_key, body, session)
-
-
-@router.post("/connectors/scraper/databases/preview")
-def scraper_database_preview(body: dict = Body(default=None),
- session: Session = Depends(require_session)):
- """Validate and price a recipe locally. This endpoint makes zero provider calls."""
- return _scraper_preview(body)
-
-
-@router.post("/connectors/scraper/databases")
-def scraper_database_create(body: dict = Body(default=None),
- session: Session = Depends(require_session)):
- """Create a saved recipe database for $0; paid work begins only at its Run-now door."""
- return _create_scraper_database(session.runtime, session.uname, body)
-
-
-@router.post("/connectors/scraper/databases/{database_key}/run")
-def scraper_database_run(database_key: str, session: Session = Depends(require_session)):
- """Explicitly execute one bounded saved scraper recipe through its provider seam."""
- return _scraper_database_run(session.runtime, session.uname, bool(session.admin), database_key)
-
-
-# ⚠ DECLARED AFTER THE LITERAL `/connectors/odoo/*` ROUTES. FastAPI matches in declaration order,
-# and while none of those ends in `/detail` — so no shadowing is possible today — the ordering is
-# the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
-# collide with this three-segment path at all.
-@router.get("/connectors/{connector_key}/detail")
-def connectors_detail(connector_key: str, session: Session = Depends(require_session)):
- """C4 / R26: everything the connector modal shows, for THIS reader, in one read.
-
- Session-gated exactly like the directory it is opened from: a member may look at what the
- workspace is connected to. What a member does NOT get is the tenant-wide credential door
- (`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue.
- """
- detail = connector_detail(
- session.runtime, session.user, session.uname,
- bool(session.admin), connector_key)
- if detail is None:
- raise err(404, "unknown_connector",
- "this workspace has no connector with that key")
- return detail
-
-
-class _ReplacementConflict(Exception):
- """The addressed keychain row changed between replacement preparation and its store update."""
-
-
-def _encrypted_replacement_payload(kc, label, entry_type, fields, username):
- """Validate and encrypt through ``core.keychain.add_entry`` without writing a tenant row.
-
- ``add_entry`` is the keychain's one validation, preview and Fernet implementation. A tiny
- memory-only runtime lets this route reuse it while discarding the temporary identity it mints;
- the tenant store sees only the later, single-row payload swap.
- """
- class _Stage:
- def __init__(self):
- self.bucket = {}
-
- def get(self, name):
- return self.bucket if name == kc.KEY else {}
-
- def update(self, name, fn, flush="sync"):
- if name != kc.KEY:
- raise RuntimeError("the keychain encryptor wrote an unexpected bucket")
- updated = fn(self.bucket)
- self.bucket = updated if updated is not None else self.bucket
- return self.bucket
-
- stage = _Stage()
- temporary = kc.add_entry(stage, label, entry_type, fields, username)
- row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {})
- if not row.get("enc"):
- raise RuntimeError("the keychain encryptor returned no encrypted payload")
- payload = {"enc": str(row["enc"]), "preview": str(row.get("preview") or ""),
- "envelope_version": row["envelope_version"]}
- if entry_type == "amazon_sp":
- payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD]
- return payload
-
-
-def _replace_connector_key(connector_key, entry_id, body, session):
- """Replace only one connector entry's declared secrets while preserving its identity.
-
- The existing row remains untouched through validation, decryption, merge and encryption. The
- final store update compares the complete old row before swapping the prepared envelope;
- a concurrent rename or credential change therefore conflicts instead of being overwritten.
- Scope and owner live in a separate bucket and are never written by this operation.
- """
- if not session.admin:
- raise err(403, "not_admin",
- "only an administrator can replace a workspace connection key")
- if not session.runtime.available():
- raise err(503, "store_unavailable", "the tenant store is unavailable")
-
- key = str(connector_key or "").strip()
- connector = next((row for row in directory(session.runtime, session.uname, True)
- if str(row.get("key") or "") == key), None)
- entry_type = _entry_type_for(key)
- if connector is None or not entry_type:
- raise err(404, "unknown_connector", "this workspace has no connector with that key")
- if str(connector.get("manage") or "") not in ("keychain", "keychain_personal"):
- raise err(403, "replacement_unavailable",
- "this connector has no replaceable keychain credential")
-
- import routes_keychain as rk # noqa: PLC0415 - circular at import
- from core import keychain as kc # noqa: PLC0415 - keep boot lean
- visible = rk.visible_entries(session.runtime, session.uname, True)
- entry = next((row for row in visible
- if str(row.get("id") or "") == str(entry_id)
- and str(row.get("type") or "") == entry_type), None)
- if entry is None:
- # One 404 covers missing, cross-tenant, another user's personal entry and a real id bound
- # to another connector. The mismatch itself is information this route does not disclose.
- raise err(404, "no_entry", "no such key for this connector")
-
- secret_specs = _replacement_field_specs(entry_type)
- secret_keys = [str(field["key"]) for field in secret_specs if field.get("secret") is True]
- optional_keys = {AMAZON_ROTATION_DEADLINE_FIELD} if entry_type == "amazon_sp" else set()
- request = body if isinstance(body, dict) else {}
- supplied = request.get("fields")
- if set(request) != {"fields"} or not isinstance(supplied, dict):
- raise err(400, "bad_replacement",
- "send only the connector's new secret fields")
- if not set(secret_keys) <= set(supplied) <= set(secret_keys) | optional_keys:
- raise err(400, "bad_replacement",
- "send every new secret field, optional rotation metadata, and no account or server settings")
- replacements = {}
- for field in secret_keys:
- value = supplied.get(field)
- if not isinstance(value, str) or not value.strip():
- raise err(400, "bad_replacement", "a replacement secret cannot be blank")
- replacements[field] = value.strip()
-
- # Snapshot before decryption so a concurrent replacement cannot become our baseline later.
- from copy import deepcopy
- raw_bucket = session.runtime.get(kc.KEY) or {}
- original = deepcopy((raw_bucket.get("entries") or {}).get(str(entry_id)))
- if not isinstance(original, dict):
- raise err(409, "credential_changed",
- "the stored credential changed while replacement was prepared. Reload it.")
- try:
- current_fields = kc.read_fields(session.runtime, entry_id)
- except kc.KeychainEnvelopeVersionError:
- raise err(409, "credential_envelope_unsupported",
- "This credential uses an unsupported encryption envelope version. Nothing was changed.")
- except kc.KeychainLocked as exc:
- raise err(503, "keychain_locked",
- f"the keychain is locked: {exc}. Nothing was changed.")
- except Exception: # noqa: BLE001
- raise err(503, "store_unavailable",
- "the current key could not be read. Nothing was changed.")
- if not isinstance(current_fields, dict) or not current_fields:
- raise err(409, "credential_unreadable",
- "the stored credential is incomplete. Nothing was changed.")
-
- merged_fields = {**current_fields, **replacements}
- if entry_type == "amazon_sp":
- # The previous secret's deadline does not describe its replacement. Absence is unknown.
- merged_fields[AMAZON_ROTATION_DEADLINE_FIELD] = supplied.get(AMAZON_ROTATION_DEADLINE_FIELD, "")
- try:
- encrypted = _encrypted_replacement_payload(
- kc, entry.get("label"), entry_type, merged_fields, session.uname)
- except kc.KeychainLocked as exc:
- raise err(503, "keychain_locked",
- f"the keychain is locked: {exc}. Nothing was changed.")
- except ValueError as exc:
- raise err(400, "bad_replacement", str(exc))
- except Exception: # noqa: BLE001
- raise err(503, "replacement_not_stored",
- "the replacement could not be encrypted. The current key is unchanged.")
-
- def _swap(cur):
- entries = cur.get("entries") if isinstance(cur, dict) else None
- stored = entries.get(str(entry_id)) if isinstance(entries, dict) else None
- if not isinstance(stored, dict) or stored != original:
- raise _ReplacementConflict()
- replacement = dict(stored)
- replacement.update(encrypted)
- replacement.pop("probe", None)
- # Copy on write: a store failure after this callback must not mutate its cached old row.
- return {**cur, "entries": {**entries, str(entry_id): replacement}}
-
- try:
- session.runtime.update(kc.KEY, _swap, flush="sync")
- except _ReplacementConflict:
- raise err(409, "credential_changed",
- "the stored credential changed while replacement was prepared. Reload it.")
- except Exception: # noqa: BLE001
- # A remote store can fail after accepting a write. Claiming either outcome would be a lie;
- # the safe next act is a fresh detail read, never an automatic retry or delete.
- raise err(503, "replacement_unconfirmed",
- "the server could not confirm whether the replacement landed. Reload this "
- "connector before trying again.")
- return {"ok": True}
-
-
-@router.post("/connectors/{connector_key}/keychain/{entry_id}/replace")
-def replace_connector_key(connector_key: str, entry_id: str,
- body: dict = Body(default=None),
- session: Session = Depends(require_session)):
- """Admin-only, write-only replacement for one visible connector credential."""
- return _replace_connector_key(connector_key, entry_id, body, session)
-
-
-# ⚠ FIVE SEGMENTS, so it can collide with nothing already declared here: the literal Odoo routes
-# are four (`/connectors/odoo/schema/{field_key}`) and `/detail` is three.
-@router.patch("/connectors/{connector_key}/databases/{database_key}/name")
-def connector_database_name(connector_key: str, database_key: str,
- body: dict = Body(default=None),
- session: Session = Depends(require_session),
- connection_id: str = None):
- """W53-T27 / R9 — set or CLEAR this tenant's own name for one of a connector's databases.
-
- ⛔⛔ IT WRITES THE BUCKET THE NAVIGATION RAIL WRITES, `nav_meta`, IN THE SAME SHAPE AND WITH
- THE SAME PER-KEY MERGE. That is the whole ruling: *"the rename database under the Navigation
- rail also works, it should then sync to the navigation in the connector too."* A rename made
- here is the one the rail shows, and a rename made on the rail is the one this connector's
- Databases tab shows, because there is one document and one reader of it
- ([[one-question-two-normalizers]]). The key names, the length cap and the merge semantics are
- IMPORTED from `routes_nav` rather than copied, so the two doors cannot drift apart.
-
- ⛔ SO WHY A SECOND DOOR AT ALL? `POST /nav/meta` cannot CLEAR a name: an empty or null `name`
- 400s there ("a database needs a name"), so "clearing the override falls back to our shipped
- default" has nowhere to happen today. This route is that door. `name: null` and `name: ""`
- both clear; any other string sets. The day `/nav/meta` grows the same clearing capability this
- becomes a thin alias of it and should be folded in.
-
- ⛔ AND IT NEVER TOUCHES A TABLE DEFINITION. `user_tables.set_label` would be rewritten by the
- next connector re-sync (`odoo_relational.apply_plan` rebuilds definitions), so an override
- stored there would vanish on a sync nobody connected to a rename. R9 forbids exactly that, and
- the store choice is what makes it impossible rather than merely unlikely.
-
- THE WALLS, in order, and each one names what it refused:
- · the database must be one THIS connector declares (404) - the door is scoped to the card
- the reader opened it from, so it cannot be used to rename an unrelated table;
- · the key must be a `ut_` key (400 `name_not_allowed`, the same refusal `/nav/meta` gives) -
- a built-in module's label is a compiled registry literal;
- · the table must EXIST in this workspace (409) - a name for a table nobody can open would
- sit in the document naming nothing;
- · `may_open` - the creator, an admin, or a share grant, the same predicate `/nav/meta` uses
- and the same one `canRename` reports, so the control and the door always agree.
- """
- import routes_nav # noqa: PLC0415 - the ONE nav owner
- rt = session.runtime
- key = str(database_key or "").strip()
- if connector_key == "odoo":
- key, connection_id, _choice = _odoo_database_target(rt, key, connection_id)
- rows = _detail_databases(rt, bool(session.admin), str(connector_key or ""),
- uname=session.uname, connection_id=connection_id)
- row = next((candidate for candidate in rows
- if str(candidate.get("key") or "") == key), None)
- if row is None:
- raise err(404, "unknown_database",
- "this connection does not create a database with that name")
- if not key.startswith("ut_"):
- raise err(400, "name_not_allowed",
- "only a database this connection created can be renamed. This one's name comes "
- "from the module registry")
- import core.user_tables as user_tables # noqa: PLC0415 - lazy, as elsewhere
- definitions = user_tables.lend_defs(rt)
- if not user_tables.get(key, st=definitions):
- raise err(409, "database_not_created",
- "this connection has not created that database in this workspace yet, so there "
- "is nothing to rename")
- if not user_tables.may_open(key, session.uname, session.admin, st=definitions):
- raise err(403, "forbidden", "that database belongs to another user")
-
- body = body if isinstance(body, dict) else {}
- if "name" not in body:
- raise err(400, "bad_request",
- "no name was sent. Send a name to set one, or null to go back to the default")
- # The same normalisation `/nav/meta` applies: whitespace collapsed, capped at its own limit.
- # A blank result CLEARS, which is the half that route refuses and this one exists for.
- name = " ".join(str(body.get("name") or "").split())[:routes_nav._MAX_NAV_NAME]
-
- if not rt.available():
- raise err(503, "store_unavailable",
- "the tenant store is unavailable. Nothing was saved.")
-
- def _up(data):
- # `routes_nav.save_nav_meta._up`, clause for clause: an entry left empty is REMOVED rather
- # than stored as `{}`, and an icon set from the rail survives a rename made here.
- data = data if isinstance(data, dict) else {}
- entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {}
- if name:
- entry["name"] = name
- else:
- entry.pop("name", None)
- if entry:
- data[key] = entry
- else:
- data.pop(key, None)
- return data
-
- try:
- rt.update(routes_nav._NAV_META_KEY, _up) # noqa: SLF001 - the nav's own key
- except Exception: # noqa: BLE001
- raise err(503, "store_unavailable",
- "the change was not saved: the store refused the write.")
-
- # Read BACK through the same resolver the tab reads, so the answer cannot be the echo of a
- # write that landed somewhere else.
- default_label = str(row.get("defaultLabel") or "")
- override = schema_overlay.override_name(_database_name_overrides(rt).get(key))
- return {"key": key,
- "identity": row.get("identity"),
- # ⛔ SERVED, NEVER ACCEPTED. `native` is the vendor's identifier; this door reads
- # `name` and nothing else off the body (C2: `native` is never editable).
- "native": str(row.get("native") or ""),
- "defaultLabel": default_label,
- "override": override,
- "label": schema_overlay.database_label(default_label, override)}
-
-
-# ⛔⛔ W49-T16 — THE MOUNT. `dependents_router` DECLARES NO PREFIX OF ITS OWN, so including it in
-# this router (which carries `prefix="/api/v1"`) yields exactly `/api/v1/dependents`. Declaring
-# the prefix on both would serve `/api/v1/api/v1/dependents` — the mirror image of the wave-23
-# defect this file's own header records, where the MISSING prefix 404'd a shipped route with
-# every gate green. `verify_shared_views.py` asserts the assembled path off this very object
-# rather than trusting the arithmetic.
-router.include_router(dependents_router)
-
-# ⛔⛔ W49-T19 — THE SECOND MOUNT, IDENTICAL IN SHAPE. `monid_router` is a bare `APIRouter()`,
-# so this yields `/api/v1/monid/amazon-reviews/meta`, `/api/v1/monid/amazon-reviews/run` and
-# `/api/v1/monid/spend`.
-# ⚠ THE ASSEMBLED PATH IS NOT VISIBLE IN `router.routes`. FastAPI 0.139.2 leaves an
-# `_IncludedRouter` placeholder there, so a walk of that list finds nothing and a gate written
-# against it would pass on a router that serves the paths twice-prefixed. `app.openapi()["paths"]`
-# is the reliable read, and it is what `verify_meta.py::section_monid_mount` uses.
-router.include_router(monid_router)
+"""THE CONNECTORS DIRECTORY (wave 23, contract C11 / owner ruling R8).
+
+⭐ WHAT THE OWNER ASKED FOR, verbatim: *"Build a connector module where it stores everything… BUT
+if its not yet integrated make it faded grey or something so i can use it as a to-do list for
+integrations."* So this endpoint deliberately answers with things we have NOT built. A directory
+that listed only working connectors would be honest about each row and dishonest about the shape
+of the product — the owner wants to see the map, including the empty parts.
+
+⛔ THE ROW SET IS COMPOSED HERE, ONCE, FROM THE REGISTRIES THAT ALREADY EXIST — the OAuth
+provider registry (`oauth_connect.PROVIDERS`), the keychain's entry types, the automation source
+registry, and the env-configured built-ins. It is NOT a hand-written list of cards. That matters
+because the alternative rots in a specific way: a provider gets wired up somewhere and the
+directory keeps calling it "planned" for a wave or two, which is exactly the misinformation this
+surface exists to prevent. The only hand-written part is `PLANNED` — the things that have no
+registry entry anywhere yet, which is precisely what "not yet integrated" means.
+
+⚠ STATE IS PER-READER. `connected` is a fact about THIS user for a per-user credential (OAuth
+slots are per-user by R7) and about the tenant for a shared one (Odoo, the Scraper). A directory
+that reported another user's Gmail as "connected" would be both wrong and a small disclosure.
+"""
+from __future__ import annotations
+
+import os
+import ssl
+import time
+import threading
+import xmlrpc.client
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, Body, Depends
+
+import oauth_connect
+from deps import Session, err, require_session, running_main
+#: ⭐⭐ W49-T18 / R27 / C4 — THE TAXONOMY LIVES IN THE DATA LAYER, NOT HERE. `core` is already on
+#: the path by the time this module loads (`deps` imports three of its modules), so this is the
+#: same import shape the rest of the API uses. It is in `platform/core/` rather than beside this
+#: file because the crosswalk is authored CONTENT about connectors, not routing: a second reader
+#: of it (a gate, a future module page) must not have to import FastAPI to ask what category a
+#: connector is in.
+from core import connector_directory
+from core import connector_sync
+from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD
+from harness import amazon_client
+#: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default,
+#: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already
+#: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module
+#: holds the tenant's store handle and does the reading; nothing about what a blank override MEANS
+#: is decided here.
+from core import schema_overlay
+#: ⛔ W32-T10 — IMPORTED, NEVER RE-DERIVED. "Does the environment serve THIS tenant's Odoo?" is
+#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
+#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
+#: second correct one here would leave the same two-copies shape that produced the defect.
+from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available
+#: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a
+#: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any
+#: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router`
+#: line for exactly this reason — see `routes_charts.py`'s header). This file is already
+#: included by `main.py`, so mounting the child here puts `/api/v1/dependents` on the app with
+#: no cross-fence edit. The `include_router` call is at the BOTTOM of this file; the import is
+#: here so the module's dependencies read in one place.
+from routes_dependents import router as dependents_router
+
+#: ⭐⭐ W49-T19 · R31 / C5 — THE MONID SLICE, MOUNTED THE SAME WAY AND FOR THE SAME REASON.
+#: `routes_monid.py` carries the Amazon reviews pull. It declares NO prefix of its own; the
+#: `include_router` call is at the BOTTOM of this file beside the dependents one, so there is a
+#: single place to read what this router actually serves.
+#: ⛔ NEVER FROM `main.py` (C5). The connectors directory owns its children.
+from routes_monid import router as monid_router
+
+# ⛔ THE PREFIX IS LOAD-BEARING AND IT WAS MISSING (found at the wave-23 close-out audit).
+# This router declared a bare `APIRouter()`, so its one path was served at `/connectors/directory`
+# while `ConnectorsPage.tsx` fetches `${API_V1}/connectors/directory` — the directory would have
+# 404'd in production with every gate green. Every other router in this package self-declares
+# `prefix="/api/v1"`; this was the only one that did not.
+#
+# ⚠ AND THE WIRING GATE COULD NOT SEE IT. `verify_wiring`'s W23-W4 row matches the SERVER side on
+# the substring `connectors/directory` and the CLIENT side on `${API_V1}/connectors/directory` —
+# both matched, both sides "wired", prefixes disagreeing. A cross-fence row that greps each side
+# independently proves the two files mention the same path SEGMENT, never that they agree on the
+# same URL. `verify_api` now asserts the prefix on every mounted router instead.
+router = APIRouter(prefix="/api/v1")
+
+CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
+SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps"))
+SCRAPER_RECIPE_MAX_ROWS = 30
+SCRAPER_RUN_HISTORY_LIMIT = 20
+DATABASE_SYNC_STATES = frozenset(
+ ("never_run", "queued", "running", "succeeded", "partial", "failed", "disabled")
+)
+_DATABASE_SAFE_ERRORS = {
+ "tls_certificate_invalid": ("Odoo TLS certificate verification failed because the certificate "
+ "is expired or untrusted. No source data was accepted."),
+ "source_read_refused": "Odoo refused the configured account's read access.",
+ "source_auth_failed": "Odoo rejected the configured read-only credential.",
+ "source_unavailable": "Odoo could not be reached for this read-only sync.",
+ "source_read_failed": "Odoo source validation failed.",
+ "mirror_pull_failed": "The Odoo mirror pull failed.",
+ "materialization_failed": "The database could not be materialized.",
+ "reconciliation_failed": "The database reconciliation failed.",
+ "freshness_unconfirmed": "Freshness could not be confirmed for this database.",
+ "sync_status_unavailable": "Sync status is unavailable for this database.",
+ "sync_stalled": "This database sync is stalled.",
+ "sync_state_unreadable": "This database's saved sync state could not be read.",
+ "maps_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
+ "scraper_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
+ "maps_unbounded_request": "Google Maps discovery requires an explicit bounded result cap.",
+ "maps_invalid_search": "Google Maps discovery needs a valid category or search.",
+ "maps_vendor_error": "The Maps provider could not complete this run.",
+ "maps_schema_drift": "The Maps provider returned an unreadable result shape.",
+ "scraper_run_failed": "The provider could not complete this run. No result was written.",
+ "provider_result_failed": "The provider returned no usable result for this database.",
+ "scraper_result_not_saved": "The provider answered, but the result could not be saved.",
+}
+
+# T11 established the saved database contract; T12's provider normalises every Apify answer into
+# exactly this schema. Vendor field names never reach a saved View or downstream automation.
+_MAPS_RECIPE_FIELDS = [
+ {"key": "place_id", "label": "Place id", "type": "text", "source": "automation"},
+ {"key": "name", "label": "Name", "type": "text", "source": "automation"},
+ {"key": "category", "label": "Category", "type": "text", "source": "automation"},
+ {"key": "address", "label": "Address", "type": "text", "source": "automation"},
+ {"key": "city", "label": "City", "type": "text", "source": "automation"},
+ {"key": "region", "label": "Region", "type": "text", "source": "automation"},
+ {"key": "postal_code", "label": "Postal code", "type": "text", "source": "automation"},
+ {"key": "country", "label": "Country", "type": "text", "source": "automation"},
+ {"key": "phone", "label": "Phone", "type": "phone", "source": "automation"},
+ {"key": "website", "label": "Website", "type": "url", "source": "automation"},
+ # Coordinates remain text because `number` is not a user-table field kind; the Map binding
+ # parses the canonical scalar value. Declaring an unsupported kind would silently drop both
+ # columns at database creation.
+ {"key": "latitude", "label": "Latitude", "type": "text", "source": "automation"},
+ {"key": "longitude", "label": "Longitude", "type": "text", "source": "automation"},
+ {"key": "rating", "label": "Rating", "type": "rating", "source": "automation"},
+ {"key": "reviews", "label": "Reviews", "type": "int", "source": "automation"},
+ {"key": "source_timestamp", "label": "Source timestamp", "type": "date",
+ "source": "automation"},
+ {"key": "source_payload", "label": "Source data", "type": "json", "source": "automation"},
+]
+
+
+# Measured reads take 1.34 s cold, 0.01 s warm, and above 8 s under boot contention.
+# A thread pool deadline was tried and produced three consecutive 30 s 504s because
+# core/store.py uses reentrant locks whose ownership does not survive a thread
+# boundary while the request thread waits on the result. Keep the work on the
+# request thread. This budget is therefore cooperative: it bounds a read whose
+# phases are each finite and cannot interrupt a single call that is already wedged.
+class _ConnectorDetailBudget:
+ def __init__(self):
+ self._started = time.monotonic()
+
+ def check(self):
+ if time.monotonic() - self._started >= CONNECTOR_DETAIL_DEADLINE_SECONDS:
+ raise err(504, "connector_detail_timeout",
+ f"The connector store did not answer within {CONNECTOR_DETAIL_DEADLINE_SECONDS} "
+ "seconds, so the read was stopped.")
+
+
+#: The to-do list (R8). Order is the owner's reading order, not alphabetical: the things most
+#: likely to come next first. `queue` mirrors the standing provider queue in wave22-split C5-A2
+#: so the two cannot drift into disagreeing about what is coming.
+PLANNED = [
+ # ⭐⭐ WAVE 31 · R7 — SIX ROWS LEFT THIS LIST FOR `NANGO_PROVIDERS` BELOW, and that move is the
+ # ticket. `slack`, `airtable`, `outlook`, `quickbooks`, `xero` and `hubspot` are providers
+ # Nango can AUTHENTICATE today, so leaving them here as "not built yet" is the same
+ # misinformation this module's header warns about — one wave later and about a different
+ # mechanism. What they still cannot do is INGEST, and that is what their own cards now say.
+ # ⚠ WAVE 32 · ITEM 4 — FIVE, NOW. The Airtable row is deleted from the directory entirely
+ # (the reason is in the provider registry below). It is still NAMED in the sentence above
+ # because that sentence is the record of where those six rows WENT, and a record that quietly
+ # drops one of its own entries is worse than no record.
+ # ⚠ W32-T14 — THIS SENTENCE STOPPED BEING TRUE IN THE SAME CHANGE THAT HID THE GOOGLE CARD.
+ # It read *"the Google connection already exists"*, which was the one row on this page that
+ # made a claim about ANOTHER row — so R6 hiding that other row turned a to-do entry into a
+ # false statement. Exactly the rot this module's header names, arriving sideways.
+ # ⛔ THE THREE `google_*` ROWS THEMSELVES STAY. Item 12 is about the connectors that offered a
+ # button and answered `503 oauth_unavailable`; these are `planned` to-do entries with no door,
+ # no state and no promise — which is precisely the map R8 asked this page to be.
+ {"key": "google_sheets", "label": "Google Sheets", "kind": "oauth",
+ "desc": "Read a sheet as a database."},
+ {"key": "google_drive", "label": "Google Drive", "kind": "oauth",
+ "desc": "Attach files from Drive to records."},
+ {"key": "google_calendar", "label": "Google Calendar", "kind": "oauth",
+ "desc": "Turn dated records into calendar entries."},
+ # ⭐⭐ WAVE 30 · R4 — TWO ROWS WERE DELETED HERE, AND THEY WERE MISINFORMATION, not clutter.
+ #
+ # {"key": "tiktok", "label": "TikTok", ... "The second social source beside Instagram."}
+ # {"key": "meta", "label": "WhatsApp and Instagram (Meta)", ... "Blocked on Meta business
+ # verification; Instagram data arrives through the Scraper connector"}
+ #
+ # ⛔ THE TIKTOK ROW SAID "not built yet" ABOUT SOMETHING THAT SHIPPED IN WAVE 29 — which is
+ # precisely the rot this module's header warns about: *"a provider gets wired up somewhere and
+ # the directory keeps calling it planned for a wave or two, which is exactly the
+ # misinformation this surface exists to prevent."* TikTok is a PLATFORM of the Scraper
+ # connector now (R3/R4), and it appears in the Scraper card's own drill below.
+ # ⚠ THE META ROW IS A DIFFERENT CASE and is deleted for a different reason: what it described
+ # (Instagram arriving another way) is now visible ON the Scraper card, so keeping it would put
+ # Instagram on this page twice, once as working and once as blocked. The Meta business
+ # verification blocker itself is not lost — it is D-86's subject and lives in
+ # `AutoWiki/research/ads-connectors-owner-actions.md`, where an owner action belongs.
+]
+
+#: ⭐ WAVE 30 · R4 — THE SCRAPER'S PLATFORMS, and the whole shape of the ruling in one list.
+#: The owner: *"the damn Tiktok and Instagram belongs in the same bucket … Only when I click
+#: 'Scraper' … would I see the option to choose either Instagram OR TikTok."* So the directory
+#: mirrors the automation menus exactly: ONE card for the capability, and the platforms are behind
+#: it rather than beside it.
+#: ⚠ These are not connectors and must never become rows: they have no state or manage door of
+#: their own. Instagram and TikTok share the Scraper credential; Maps uses the Apify credential
+#: through the same governed provider registry.
+SCRAPER_PLATFORMS = [
+ {"key": "instagram", "label": "Instagram",
+ "desc": "Profiles, posts and comments, by search or by handle."},
+ {"key": "tiktok", "label": "TikTok",
+ "desc": "The same three, on TikTok's own schema."},
+ {"key": "google_maps", "label": "Google Maps",
+ "desc": "Places by category or search and geography, through a bounded Apify run."},
+]
+
+#: Token connectors an admin can wire up TODAY by pasting a key — no OAuth client, no app review.
+TOKEN_CONNECTORS = [
+ {"key": "stripe", "label": "Stripe", "entry": "stripe",
+ "desc": "Payments and payouts against your customer records.",
+ "hint": "Create a restricted key with read access in the Stripe dashboard.",
+ "note": "Storing the key connects the account; reading its data lands in a later release."},
+ {"key": "shopify", "label": "Shopify", "entry": "shopify",
+ "desc": "Orders and products from a Shopify storefront.",
+ "hint": "Create a custom app in your store admin and copy its Admin API token.",
+ "note": "Storing the key connects the account; reading its data lands in a later release."},
+ # ⭐⭐ WAVE 31 · R2/R8 — META ADS IS A TOKEN CONNECTOR, AND IT IS THE ONE NEW CARD THAT IS
+ # ACTIONABLE TODAY. R8 put the schema work on a pasted token deliberately — *"so that you can
+ # also explore the Schema, without any user logging into it with Facebook yet"* — and T41
+ # MEASURED that the delegated-OAuth door is not available anyway: the Nango account holds one
+ # integration and it is a GitHub sample, so there is no Facebook integration to request
+ # `ads_read` from. This row is therefore the real door and the Nango one is honest about not
+ # being ready. ⚠ Its `entry` must match `keychain.ENTRY_TYPES`, or storing 400s.
+ {"key": "meta_ads", "label": "Meta Ads", "entry": "meta_ads",
+ "desc": "Campaigns, ad sets, ads, creatives and daily performance from your Meta ad account.",
+ "hint": "Paste a Marketing API access token with `ads_read`; the connector verifies it "
+ "against your ad accounts before storing it.",
+ "note": "The token connects the account and is verified live. The spawned databases "
+ "(285 measured columns) land with the Meta grids."},
+ # ⭐⭐ AMAZON SP-API. FIVE values rather than one token, because Login With Amazon
+ # exchanges a refresh token for a short-lived bearer on every call, and the seller and
+ # marketplace ids scope the reads. ⚠ `entry` must match `keychain.ENTRY_TYPES` or storing
+ # 400s, and `refresh_token` had to join `_PREVIEW_FIELDS` in the same change or the row
+ # renders a blank preview with nothing raising (the wave-31 defect, second edition).
+ # ⛔ Read-only by construction: `connectors_amazon._request` refuses any method but GET
+ # before it builds a request, so no route, payload or UI can reach a write.
+ {"key": "amazon_sp", "label": "Amazon", "entry": "amazon_sp",
+ "desc": "Orders, settlements and fees, FBA and AWD inventory, listings and Buy Box "
+ "pricing from your Amazon selling account.",
+ "hint": "Create a Production app client in the Solution Provider Portal, self authorize "
+ "it, then paste client_id, client_secret, refresh_token, seller_id and "
+ "marketplace_id.",
+ "note": "The credential is verified live against your marketplace participations before "
+ "it is stored. Bulk reads use the Reports API, so a backfill is one document "
+ "rather than a page loop."},
+]
+
+#: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
+#: other potential connection as possible, such as slack etc."*
+#:
+#: ⛔ ADDING A PROVIDER HERE IS THE WHOLE FEATURE — one row, no code — which is the claim T43 has
+#: to make demonstrable rather than assert. The row carries only what a CARD needs; the auth flow
+#: is Nango's and is identical for every one of them.
+#:
+#: ⛔⛔ R7's SECOND SENTENCE IS THE HARD PART AND IT IS ENFORCED IN `_nango_rows` BELOW: *"Nango
+#: buys CONNECT, not INGEST — every lit provider states on its own card what it can do today, or
+#: we ship a directory of buttons that lead nowhere."* MEASURED for T42: the owner's Nango account
+#: holds **1 integration (`github-getting-started`) and 0 connections**, so there is no sync on it
+#: to inspect and NO provider here has a data layer. Every card says so, in its own words, and
+#: none of them offers a button.
+NANGO_PROVIDERS = [
+ {"key": "slack", "label": "Slack", "provider": "slack",
+ "desc": "Send messages into a channel when an automation reaches a step."},
+ {"key": "hubspot", "label": "HubSpot", "provider": "hubspot",
+ "desc": "Deals and contacts beside the Odoo customer book."},
+ # ⭐⭐ WAVE 32 · OWNER ITEM 4 / R3 — THE AIRTABLE ROW IS DELETED. It read:
+ #
+ # {"key": "airtable", "label": "Airtable", "provider": "airtable",
+ # "desc": "Import an existing base as a database."}
+ #
+ # The owner asked for it to go, and R3 says how: **de-brand, don't revoke.** So the ROW goes
+ # and the transport does not — Nango can still authenticate `airtable` if a future ruling
+ # wants it back, and nothing in `oauth_connect` changed.
+ # ⚠ THE REASON IS NOT THAT IT WAS BROKEN. This connector is the one competitor whose model
+ # this product is built against, and offering to import a base is a promise nobody here is
+ # ready to keep. Every DESIGN-PARITY comment naming Airtable stays exactly where it is
+ # (~30 of them across the grid and automation clients): those are how we know what good looks
+ # like, and deleting them would cost the reasoning while saving nothing a user can see.
+ {"key": "quickbooks", "label": "QuickBooks", "provider": "quickbooks",
+ "desc": "Accounting actuals beside the Odoo sales data."},
+ {"key": "xero", "label": "Xero", "provider": "xero",
+ "desc": "Accounting actuals for tenants who do not use QuickBooks."},
+ {"key": "outlook", "label": "Microsoft Outlook", "provider": "outlook",
+ "desc": "The Outlook half of the email trigger."},
+]
+
+
+#: ⭐⭐ W31-T44 / OWNER ITEM 6b — WHICH `ut_*` PREFIXES BELONG TO A CONNECTOR (contract C2).
+#:
+#: B's `user_tables.register_connected_prefix` (W31-T25) replaced a hardcoded `ut_odoo_` literal
+#: with a registered SET, and this is where the connector layer declares its half. It lives beside
+#: the other connector registries deliberately: "which databases does this connector own" is the
+#: same kind of fact as "what is this connector called", and putting it anywhere else would make
+#: the answer to one question live in two files.
+#:
+#: ⛔ REGISTERING A PREFIX DOES NOT MAKE A TABLE CONNECTED, and B posted this as the trap: the
+#: third leg of `is_connected` still demands `source == AUTOMATION_SOURCE` **and**
+#: `recordMode == AUTOMATION_RECORD_MODE`. A person who names a database "Meta ads spend" gets a
+#: `ut_meta_…` key too, and exempting it from `MAX_ROWS` on the strength of its NAME is the silent
+#: hole the stamp closes. The spawn stamps; this only says which family may qualify.
+#: ⚠ `ut_odoo_` is SEEDED in `user_tables` rather than registered here, also deliberately — the
+#: convention leg has to answer in a worker process where nothing registered anything.
+#: ⛔⛔ `ut_amzn_` JOINED IN WAVE 52, AND OMITTING IT WOULD HAVE BROKEN STANDING RULE 1 SILENTLY.
+#: The rule is that a connector's records carry NO cap: `records_mutable(key)` is False for them,
+#: so `MAX_ROWS` does not apply. But that answer is reached through `is_connected`, whose third leg
+#: is this prefix set — so an Amazon table missing from here does not raise, does not warn, and
+#: simply falls through to the 60,000-row ceiling. A seller with more than 60,000 order lines would
+#: get a truncated grid and no error, which is precisely the silent cap the rule forbids. **A new
+#: connector joins this dict in the same change that spawns its tables, or its no-cap guarantee is
+#: decorative.**
+CONNECTED_PREFIXES = {
+ "odoo": "ut_odoo_",
+ "meta_ads": "ut_meta_",
+ "amazon_sp": "ut_amzn_",
+}
+
+
+def _register_prefixes():
+ """Declare `CONNECTED_PREFIXES` to the store layer. Called at import — this module is mounted
+ by `main.py`, so any process that can serve the directory has also made the declaration.
+
+ Never raises: a store layer that cannot take the declaration must not stop the API booting,
+ and the consequence of a missed registration is a row CAP (R6's subject), not a wrong answer.
+ """
+ try:
+ import core.user_tables as ut
+ return ut.register_connected_prefix(*sorted(CONNECTED_PREFIXES.values()))
+ except Exception: # noqa: BLE001
+ return frozenset()
+
+
+_register_prefixes()
+
+
+def _register_probers():
+ """Let each connector answer `keychain.test_entry` for its OWN type — the SLOT, filled from
+ the layer that knows how to ask.
+
+ ⛔ THIS DIRECTION IS THE POINT. `core.keychain` must not import a connector module: `core`
+ never imports up (ARCHITECTURE.md / CLAUDE.md), and the tempting shortcut — a `sys.path`
+ insert and an `import connectors_meta` from inside `core` — is an architecture inversion that
+ no gate was watching for. Same idiom as `datastore.set_paused_probe`, one layer down, for the
+ same reason.
+
+ Import-time and fail-quiet: an unfillable slot costs a liveness CHECK, never the ability to
+ store a credential and never the API's ability to boot.
+ """
+ ok = False
+ for mod in ("connectors_meta", "connectors_amazon"):
+ try:
+ ok = __import__(mod).register() or ok
+ except Exception: # noqa: BLE001
+ continue
+ return ok
+
+
+_register_probers()
+
+
+# W48C-T08 / C6 — the graph is a reader-scoped DTO over tables that exist in THIS workspace,
+# never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family;
+# `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter.
+def _graph_refusal(connection_key, reason, status="refused"):
+ return {"connectionKey": str(connection_key or ""), "status": status,
+ "reason": str(reason or ""), "tables": [], "relationships": []}
+
+
+def _table_connection_key(table_key, definition):
+ """The owning connection declared by this actual table, or ``""`` when it is not one."""
+ definition = definition if isinstance(definition, dict) else {}
+ # Future connector materialisers may stamp their directory key explicitly. It is accepted
+ # only as a key, never as table/field data, so no provider can manufacture a graph template.
+ for name in ("connectionKey", "connectorKey", "connector_key", "connector"):
+ candidate = definition.get(name)
+ if isinstance(candidate, str) and candidate.strip():
+ return candidate.strip()
+ key = str(table_key or "")
+ for connection_key, prefix in CONNECTED_PREFIXES.items():
+ if key.startswith(prefix):
+ return connection_key
+ return ""
+
+
+def _module_connection_key(subject):
+ """Which connected card a REGISTRY MODULE's `subject` belongs to, or `""`.
+
+ ⭐⭐ W53-T30 — DERIVED FROM `CONNECTED_PREFIXES`, NEVER A SECOND HAND-TYPED PAIR LIST. A
+ module's subject is namespaced by the system the records come from (`odoo:res.partner`,
+ `odoo:product.product` in `core/registry.py`), and the connector families this file already
+ knows carry that same namespace inside their table prefix (`odoo` inside `ut_odoo_`). So the
+ two are matched rather than mapped: a connector added to `CONNECTED_PREFIXES` in the change
+ that spawns its tables gets this for free, and a module whose namespace names no connected
+ family here simply does not appear on anybody's graph.
+
+ ⛔ THE SUBJECT, NOT THE `source` STRING. `source` is display copy ("Odoo") and a rename of it
+ is a copy edit; `subject` is the identity `core.registry` already refuses to let two rows
+ claim at once, which is what makes it safe to key a graph off.
+ """
+ namespace = str(subject or "").split(":", 1)[0].strip()
+ if not namespace:
+ return ""
+ for connection_key, prefix in CONNECTED_PREFIXES.items():
+ parts = str(prefix or "").strip("_").split("_")
+ if len(parts) > 1 and namespace == "_".join(parts[1:]):
+ return connection_key
+ return ""
+
+
+def _module_graph_entries(rt, user, lent, connected):
+ """The REGISTRY MODULES this reader may see on a connected card's graph, per connection.
+
+ Returns `{connection key: {module key: {"dto", "visible"}}}` in exactly the shape the `ut_`
+ loop below builds, so the edge pass treats both kinds of node identically.
+
+ ⭐⭐ W53-T30 — THE WHOLE TICKET. `relationship_graphs` walks `user_tables.all_defs`, and
+ `customer_data` / `product_data` are COMPILED REGISTRY ROWS rather than stored definitions, so
+ that loop could never reach them however permissive it became. They were absent from the Odoo
+ diagram for that reason, not for a permission one. ⛔ The retired `ut_odoo_customers` /
+ `ut_odoo_products` keys are NOT the subject and must not be revived: W33-T44 / A2 retired
+ them, `odoo_relational.RETIRED_KEYS` pops them out of every tenant's document on rebuild, and
+ their subjects are claimed by these two registry rows, which is now the only claim on each.
+
+ ⛔⛔ THREE PREDICATES, THE SAME THREE `routes_tables._module_link_targets` USES ON THESE EXACT
+ TWO KEYS, and none of them is `is_admin`. A blanket administrator check here would silently
+ drop two nodes from a member's diagram for no stated reason, while every other node on the
+ same picture is walled per table:
+ 1. `perms.tenant_has_module` — is this database part of THIS tenant's world at all? Every
+ provisioned tenant but #0 carries a restricted `modules:` list, so without it another
+ company's admin is shown a database their workspace does not have.
+ 2. `perm_scope.nav_may_open` — ADMISSION, the predicate `deps.py`'s module gate itself asks.
+ `may_read` alone would draw a node whose door then 403s.
+ 3. `perm_scope.may_read` — the READ wall, the same one the `ut_` loop applies per table.
+
+ ⛔ `user_tables.may_open` IS NOT IN THAT LIST AND ITS ABSENCE IS DELIBERATE: it answers about
+ a STORED definition's creator and share grants, and a registry module has no such row, so it
+ refuses every reader including the tenant's owner.
+
+ ⭐ THE FIELDS ARE `automation_engine._module_table`'s, WHICH IS THE ONE COMPOSER OF A MODULE'S
+ FIELD CONTRACT (`aios_grid` plus `odoo_relational.module_link_fields`). Restating either half
+ here is how this diagram would start disagreeing with the engine that resolves the very links
+ it draws. `want_rows=False`, so no pool is built and no store is written.
+
+ ⚠ AND THE FIELD LIST IS THE VIEWER'S: `visible_fields` strips the hidden closure, so a column
+ an administrator hid from this account is neither drawn on the card nor usable as an edge
+ endpoint. Fail-closed everywhere: a wall that cannot be answered drops the node.
+ """
+ try:
+ import automation_engine # noqa: PLC0415 - lazy, as elsewhere
+ import core.perm_scope as perm_scope
+ import core.perms as perms
+ import core.registry as registry
+ import core.user_tables as user_tables
+ except Exception: # noqa: BLE001
+ return {}
+ out = {}
+ memo = {}
+ for module_key in getattr(user_tables, "LINKABLE_MODULE_KEYS", ()):
+ key = str(module_key or "")
+ row = registry.BY_KEY.get(key) or {}
+ connection_key = _module_connection_key(row.get("subject"))
+ if not key or connection_key not in connected:
+ continue
+ try:
+ if not perms.tenant_has_module(rt, key):
+ continue
+ if not perm_scope.nav_may_open(user, key, st=lent):
+ continue
+ if not perm_scope.may_read(user, key, st=lent):
+ continue
+ # ⛔ `_module_table`, THE ENGINE'S OWN COMPOSER (`# noqa: SLF001` for the same reason
+ # `_database_name_overrides` calls `routes_nav._read_nav_meta`: one reader for one
+ # fact beats a correct-looking second copy).
+ table, _limit = automation_engine._module_table(rt, key, memo) # noqa: SLF001
+ declared = [dict(field) for field in ((table or {}).get("fields") or ())
+ if isinstance(field, dict)]
+ if not declared:
+ continue
+ visible = perm_scope.visible_fields(declared, user, key, st=lent)
+ except Exception: # noqa: BLE001
+ continue
+ fields = []
+ for field in visible:
+ field_key = str(field.get("key") or "").strip()
+ if field_key:
+ fields.append({"key": field_key,
+ "label": str(field.get("label") or field_key).strip() or field_key})
+ if not fields:
+ continue
+ # ⚠ THE REGISTRY'S OWN LABEL, READ RATHER THAN RETYPED. "Odoo customers" and "Odoo
+ # products" are `core/registry.py`'s `label` values (renamed there once already, W33-T42),
+ # and a literal copy here is a second authority that goes stale the day the row is
+ # renamed. There is no tenant override to merge in: `routes_nav` refuses a name for any
+ # key that does not start with `ut_`, so a registry module cannot be renamed at all.
+ label = str(row.get("label") or key).strip() or key
+ out.setdefault(connection_key, {})[key] = {
+ "dto": {"key": key, "label": label, "fields": fields},
+ "visible": visible,
+ }
+ return out
+
+
+def relationship_graphs(rows, user, rt, uname, is_admin=False):
+ """Actual, permitted tables and same-connection declared links for directory rows.
+
+ There is deliberately no provider fallback. A connected credential with no materialised
+ database gets an explicit unavailable envelope; a non-connected card gets an explicit
+ refusal. The client can therefore distinguish "nothing exists for this selection" from a
+ plausible-looking sample schema.
+ """
+ row_by_key = {str(row.get("key") or ""): row for row in (rows or [])
+ if isinstance(row, dict) and str(row.get("key") or "")}
+ graphs = {}
+ connected = set()
+ for connection_key, row in row_by_key.items():
+ if row.get("state") == "connected":
+ connected.add(connection_key)
+ graphs[connection_key] = _graph_refusal(
+ connection_key,
+ "No permitted tables are materialised for this connection. A provider template is not shown.",
+ status="unavailable")
+ else:
+ graphs[connection_key] = _graph_refusal(
+ connection_key,
+ "Connect this provider before viewing its actual relationship graph.")
+ if not connected:
+ return graphs
+
+ import core.perm_scope as perm_scope
+ import core.user_tables as user_tables
+ try:
+ definitions = user_tables.all_defs(st=rt)
+ lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
+ except Exception: # noqa: BLE001
+ for connection_key in connected:
+ graphs[connection_key] = _graph_refusal(
+ connection_key,
+ "The workspace tables could not be read, so this relationship graph is unavailable.",
+ status="unavailable")
+ return graphs
+
+ tables_by_connection = {connection_key: {} for connection_key in connected}
+ for table_key, definition in (definitions or {}).items():
+ connection_key = _table_connection_key(table_key, definition)
+ if connection_key not in connected:
+ continue
+ try:
+ # Prefixes are not enough: a table must carry the connector materialisation stamps
+ # before it can describe a connector graph.
+ if user_tables.is_managed_system_table(table_key, st=lent, definition=definition):
+ continue
+ if not user_tables.is_connected(table_key, st=lent):
+ continue
+ if not user_tables.may_open(table_key, uname, is_admin, st=lent):
+ continue
+ if not perm_scope.may_read(user, table_key, st=lent):
+ continue
+ visible = perm_scope.visible_fields(
+ [dict(field) for field in (definition.get("fields") or ())
+ if isinstance(field, dict)],
+ user, table_key, st=lent)
+ except Exception: # noqa: BLE001
+ # A table whose permission proof cannot be completed does not become a partial leak.
+ continue
+ fields = []
+ for field in visible:
+ key = str(field.get("key") or "").strip()
+ if key:
+ fields.append({"key": key,
+ "label": str(field.get("label") or key).strip() or key})
+ tables_by_connection[connection_key][str(table_key)] = {
+ "dto": {"key": str(table_key),
+ "label": str(definition.get("label") or table_key).strip() or str(table_key),
+ "fields": fields},
+ "visible": visible,
+ }
+
+ # ⭐⭐ W53-T30 — AND THE REGISTRY MODULES THIS CONNECTION STANDS BEHIND. They are not stored
+ # definitions, so the loop above cannot reach them; they are permitted by their own three
+ # predicates and then join the SAME edge pass, which is what makes an already-declared link
+ # between a `ut_` table and a module resolvable at both ends for the first time.
+ for connection_key, entries in _module_graph_entries(rt, user, lent, connected).items():
+ tables_by_connection.setdefault(connection_key, {}).update(entries)
+
+ for connection_key, entries in tables_by_connection.items():
+ if not entries:
+ continue
+ table_keys = set(entries)
+ visible_keys = {
+ table_key: {str(field.get("key") or "").strip()
+ for field in entry["visible"] if isinstance(field, dict)}
+ for table_key, entry in entries.items()
+ }
+ relationships = []
+ seen = set()
+ for source_table, entry in entries.items():
+ for field in entry["visible"]:
+ if not isinstance(field, dict) or field.get("type") != "link":
+ continue
+ link = field.get("link")
+ source_field = str(field.get("key") or "").strip()
+ target_table = str((link or {}).get("table") or "").strip() if isinstance(link, dict) else ""
+ target_field = str((link or {}).get("on") or (link or {}).get("inverse") or "").strip() if isinstance(link, dict) else ""
+ # An outside, unavailable, or field-hidden endpoint is not a relationship on the
+ # selected connection. Omit it rather than exposing the target through an edge.
+ if (not source_field or target_table not in table_keys or not target_field
+ or target_field not in visible_keys.get(target_table, set())):
+ continue
+ signature = (source_table, source_field, target_table, target_field)
+ if signature in seen:
+ continue
+ seen.add(signature)
+ relationships.append({
+ "from": {"table": source_table, "field": source_field},
+ "to": {"table": target_table, "field": target_field},
+ "label": str(field.get("label") or source_field).strip() or source_field,
+ })
+ # ⭐ W53-T30 / C2 — THE SPEC'S OWN DECLARED RELATIONS, held to the same two proofs as the
+ # field bags above: both endpoints must be nodes on THIS graph and both named columns must
+ # be visible to THIS reader. `{}` until lane D's `core.connector_spec` lands, and the
+ # `seen` signature is shared so a relation declared in both places is drawn once.
+ label_of = {(table_key, str(field.get("key") or "")): str(field.get("label") or "")
+ for table_key, entry in entries.items()
+ for field in entry["dto"]["fields"]}
+ for source_table, declared in _connector_spec_links(connection_key).items():
+ if source_table not in table_keys:
+ continue
+ for source_field, target_table, target_field in declared:
+ if (source_field not in visible_keys.get(source_table, set())
+ or target_table not in table_keys
+ or target_field not in visible_keys.get(target_table, set())):
+ continue
+ signature = (source_table, source_field, target_table, target_field)
+ if signature in seen:
+ continue
+ seen.add(signature)
+ relationships.append({
+ "from": {"table": source_table, "field": source_field},
+ "to": {"table": target_table, "field": target_field},
+ "label": label_of.get((source_table, source_field)) or source_field,
+ })
+ graphs[connection_key] = {
+ "connectionKey": connection_key,
+ "status": "ready",
+ "reason": "",
+ "tables": [entries[key]["dto"] for key in sorted(
+ entries, key=lambda key: (entries[key]["dto"]["label"].casefold(), key))],
+ "relationships": sorted(
+ relationships,
+ key=lambda relation: (relation["from"]["table"], relation["from"]["field"],
+ relation["to"]["table"], relation["to"]["field"])),
+ }
+ return graphs
+
+
+def attach_relationship_graphs(rows, user, rt, uname, is_admin=False):
+ """Bind every card to only its own actual graph; no key receives another provider's DTO."""
+ graphs = relationship_graphs(rows, user, rt, uname, is_admin)
+ for row in rows:
+ connection_key = str(row.get("key") or "")
+ row["relationshipGraph"] = graphs.get(connection_key) or _graph_refusal(
+ connection_key,
+ "No actual relationship graph was returned for this connection. A provider template is not shown.")
+ return rows
+
+
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+# ⭐⭐ W31-T44 — EVERY WAY THIS DIRECTORY CAN COME BACK SHORT, ENUMERATED AND MADE VISIBLE.
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+#
+# The owner named Connectors FIRST in item 6 ("Connectors and Automation module still disappears").
+# `.shell-nav-connectors` is a static `` and cannot be dropped from the rail, so if Connectors
+# "disappears" the subject is THIS payload. Enumerated by reading the composer rather than by
+# guessing, and the answer was sharper than expected:
+#
+# ⛔ THERE WAS EXACTLY ONE FAILURE MODE AND IT DROPPED **EVERY** ROW. `directory()` had no
+# `try` anywhere. `oauth_connect.status()`, `import automation_engine` and `bd_ready()` are
+# three unguarded calls into three other subsystems, and a raise in ANY of them turned the
+# whole endpoint into a 500 — at which point `ConnectorsPage` renders one sentence and no
+# cards. A page whose entire job is to be a map of the workspace showed nothing because one
+# registry was unhappy.
+# ⚠ AND ONE SILENT ONE, WHICH IS WORSE THAN THE LOUD ONE. `_entry_types` swallowed every
+# exception into an empty set, so an unreadable or locked keychain rendered Odoo, Stripe and
+# Shopify as **"available"** — i.e. *"you have not connected this"* about a connector the
+# tenant HAS connected. Not short: wrong, and confidently.
+#
+# ⭐ SO THE COMPOSER IS A LIST OF CONTRIBUTIONS AND EACH ONE FAILS ALONE. A contribution that
+# raises costs its own rows and NOTHING else, and says so in `incomplete` with a cause and a
+# recommendation — R6's second sentence, applied to a page instead of to a row cap. Silence is the
+# violation, not the limit.
+#: The contributions, by name. ⛔ THIS LIST IS THE ENUMERATION the ticket asks for, and
+#: `verify_meta` breaks each one IN TURN and requires (a) the other rows still render and (b) this
+#: name appears in `incomplete` — so a contribution added later without a probe is a red, not a
+#: quietly unprotected path.
+CONTRIBUTORS = ("keychain", "odoo", "oauth", "token", "nango", "scraper", "builtin", "planned")
+
+
+def _entry_types(rt, uname=""):
+ """`(types, failure)` — which keychain entry types THIS READER has, and what stopped us.
+
+ ⛔ IT USED TO RETURN A BARE SET AND SWALLOW EVERYTHING, and that is the silent half of item 6b:
+ a locked keychain became an EMPTY set, which is indistinguishable from "this tenant has
+ connected nothing" — so the page said `available` about a live Odoo connection. Degrading to
+ "nothing connected" is only honest if the reader is told that is what happened.
+
+ ⭐ W32-T11 / R4 — AND IT IS THIS READER'S SET, NOT THE TENANT'S. The module header has always
+ said *"STATE IS PER-READER … a directory that reported another user's Gmail as connected would
+ be both wrong and a small disclosure"*, and a personal keychain entry is exactly that case one
+ credential kind over. Reading `list_entries` straight would have lit a colleague's Stripe card
+ on your directory while the Settings pane next door correctly hid it — the same
+ two-answers-to-one-question shape W32-T10 just removed from the Odoo row.
+
+ ⚠ THE SECOND RETURN VALUE IS THE SCOPE PER TYPE, and BUSINESS WINS. A reader holding both a
+ business-wide Stripe key and a personal one is connected business-wide — the stronger fact is
+ the true one, and taking whichever row sorted first would make the card's label depend on an
+ id.
+ """
+ try:
+ import routes_keychain as rk
+ rows = rk.visible_entries(rt, uname) or []
+ types, scopes = set(), {}
+ for e in rows:
+ t = str(e.get("type") or "")
+ if t == "amazon_sp":
+ from core import keychain as kc
+ if kc.entry_verified(rt, e.get("id")):
+ types.add(t)
+ else:
+ types.add(t)
+ if scopes.get(t) != rk.DEFAULT_SCOPE:
+ scopes[t] = str(e.get("scope") or rk.DEFAULT_SCOPE)
+ return types, scopes, None
+ except Exception as e: # noqa: BLE001
+ return set(), {}, f"{type(e).__name__}: {e}"
+
+
+def _stored_amazon_rotation_deadline(rt, entry_id):
+ """Project only the public timestamp from the stored row, without decrypting a credential."""
+ from core import keychain as kc
+ stored = ((rt.get(kc.KEY) or {}).get("entries") or {}).get(str(entry_id)) or {}
+ if stored.get("type") != "amazon_sp":
+ return ""
+ deadline = stored.get(AMAZON_ROTATION_DEADLINE_FIELD)
+ return deadline if isinstance(deadline, str) else ""
+
+
+def _amazon_rotation_expired(deadline):
+ if not deadline:
+ return False
+ try:
+ parsed = datetime.fromisoformat(deadline.replace("Z", "+00:00"))
+ return parsed.tzinfo is not None and parsed <= datetime.now(timezone.utc)
+ except (TypeError, ValueError):
+ return False
+
+
+def directory(rt, uname, is_admin=False, incomplete=None):
+ """The composed row set. Pure over the session, so the gate drives it without a request.
+
+ `incomplete` is an optional list the caller lends to collect contribution failures; the route
+ passes one and puts it on the wire. Callers that do not care pass nothing and simply get the
+ rows that composed.
+ """
+ rows = []
+ short = incomplete if incomplete is not None else []
+
+ def contribute(source, fn, why, fix):
+ """Run one contribution. A failure costs ITS rows and records why — never the page."""
+ try:
+ fn()
+ except Exception as e: # noqa: BLE001
+ short.append({"source": source, "cause": f"{why} ({type(e).__name__}: {e})",
+ "recommendation": fix})
+
+ have, scopes, kc_failed = _entry_types(rt, uname)
+ if kc_failed:
+ short.append({
+ "source": "keychain",
+ "cause": f"this tenant's keychain could not be read ({kc_failed}), so a connector "
+ f"that IS connected through a stored credential is shown as available",
+ "recommendation": "unlock the keychain (AIOS_KEYCHAIN_KEY) and reload; the states "
+ "below are otherwise correct"})
+
+ # --- Odoo: the reference connector. Env credentials are tenant #0's only (the R3 rule), so
+ # a stored keychain entry is what makes it connected for anybody else.
+ #
+ # ⛔⛔ W32-T10 / OWNER ITEM 6 — THIS ROW SPENT FOUR WAVES TELLING EVERY TENANT IT WAS
+ # CONNECTED TO ANOTHER COMPANY'S ERP. The state read `"odoo" in have or
+ # os.environ.get("ODOO_URL")`, and the second leg has no tenant in it at all: one Space
+ # process serves every tenant, so `ODOO_URL` being set for tenant #0 lit this card
+ # `connected` for nurilab, GTM Lab and everyone after them — and `manage: keychain` then
+ # offered "Manage keys" for a credential their workspace does not hold. **The comment
+ # directly above stated the rule the line broke**, which is the useful part of the lesson:
+ # the guard was written down and not written in.
+ # ⚠ THE OTHER HALF OF THE FIX IS THAT NOTHING HERE RE-DERIVES IT. `env_odoo_available` lives
+ # in `routes_keychain`, which asked this question CORRECTLY the whole time — so the repair is
+ # to delete this module's answer, not to write a third one
+ # ([[one-question-two-normalizers]]).
+ # ⚠ The `have` leg is UNTOUCHED on purpose: `list_entries` reads metadata even while the
+ # keychain is LOCKED, so a tenant with a stored Odoo entry stays `connected` when the
+ # platform key is missing. Routing that leg through a resolver that requires `unlocked()`
+ # would have flipped a real connection to "available" — the silent case W31-T44 exists to
+ # prevent, re-introduced by the fix for its sibling.
+ def _odoo():
+ rows.append({
+ "key": "odoo", "label": "Odoo", "kind": "builtin",
+ "desc": "Sales, invoices, products and customers from your Odoo ERP.",
+ "state": ("connected" if ("odoo" in have or env_odoo_available(rt))
+ else "available"),
+ # W32-T11 / C1 — always business: Odoo is what the whole workspace's databases are
+ # read through, which is why `routes_keychain.TENANT_WIDE_TYPES` refuses to make it
+ # personal in the first place.
+ "scope": scopes.get("odoo") or "business",
+ "manage": "keychain" if is_admin else "",
+ })
+
+ # ⚠ GUARDED EVEN THOUGH IT CANNOT RAISE TODAY — `os.environ.get` and a set membership are the
+ # whole of it. The guard is here because the enumeration is only worth something if it is
+ # EXHAUSTIVE: an unguarded block is an un-enumerated drop path, and "this one is obviously
+ # safe" is how the other five got written in the first place.
+ contribute("odoo", _odoo, "the Odoo connector row could not be composed",
+ "reload; every other row is unaffected")
+
+ # --- OAuth providers, straight off the registry (so a new entry appears here for free).
+ # ⛔ `status()` READS EVERY PER-USER CREDENTIAL SLOT, which means it touches the keychain and
+ # the store — the two things most likely to be unavailable. Unguarded, it took the whole page
+ # with it.
+ # ⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — `oauth_connect.offered()` FILTERS THIS CONTRIBUTION.
+ # The owner reported Google and Gmail as broken connectors: the card offered a Connect button
+ # and the button answered `503 oauth_unavailable` in raw JSON. R6 is *"hidden until they are
+ # real"*, so the row goes rather than the error being prettied up — the tombstone and the
+ # money reason (D-45, CASA verification at ~$540–1,800/yr) live on `offered()` itself.
+ # ⚠ THE REGISTRY IS STILL WALKED WHOLE. A registry-driven directory is the point of this
+ # module ("a new provider appears here for free"), so the filter is one predicate on the loop
+ # and NOT a deletion — the day the verification is paid for, one key flips and the row is
+ # back with its state, its `needs` and its brand mark intact.
+ def _oauth():
+ st = oauth_connect.status(rt, uname)
+ for slug, meta in oauth_connect.PROVIDERS.items():
+ if not oauth_connect.offered(slug):
+ continue
+ rows.append(_oauth_row(slug, meta, st.get(slug) or {}))
+
+ contribute("oauth", _oauth,
+ "the OAuth provider registry could not be read, so no OAuth connector is listed",
+ "the rest of the directory is complete; reload to retry the OAuth states")
+
+ # --- Token connectors (R8): connectable today, and honest that data flows arrive later.
+ def _tokens():
+ amazon_expired = False
+ if "amazon_sp" in scopes:
+ import routes_keychain as rk
+ amazon_expired = any(
+ _amazon_rotation_expired(_stored_amazon_rotation_deadline(rt, entry.get("id")))
+ for entry in rk.visible_entries(rt, uname) or []
+ if entry.get("type") == "amazon_sp")
+ for t in TOKEN_CONNECTORS:
+ rows.append({
+ "key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"],
+ "hint": t["hint"],
+ "state": ("expired" if t["entry"] == "amazon_sp" and amazon_expired
+ else "connected" if t["entry"] in have else "reconnect"
+ if t["entry"] == "amazon_sp" and t["entry"] in scopes
+ else "available"),
+ # ⭐ W32-T11 / C1 / R4 — the scope of the credential BEHIND this card, for THIS
+ # reader. A token connector is the case R4 is really about: a member's own Stripe
+ # key is theirs, and the card must say which kind of connection it is looking at.
+ "scope": scopes.get(t["entry"]) or "business",
+ # ⭐⭐ W33-T66 / D-204 — THE MEMBER'S DOOR, AND IT IS A THIRD WORD RATHER THAN A
+ # WIDENED ONE. This route's own docstring has promised since wave 23 that a
+ # non-admin may *"connect their OWN per-user accounts"*, and the next sentence
+ # then handed them an empty `manage` — a contradiction inside one paragraph, and
+ # D-204's subject. W32-T11 opened the ROOM (`admin_gate` is gone from
+ # `routes_keychain`; a member may POST `scope: "personal"`), so the only thing
+ # still missing was a way to reach it from the map.
+ #
+ # ⛔ `keychain` DOES NOT WIDEN, and that is deliberate rather than timid.
+ # `verify_automation`'s NC asserts a non-admin is offered no `manage == "keychain"`
+ # row, and its own comment says why in the right words: *"the line is TENANT-WIDE
+ # vs PER-USER, not admin vs everyone … what they must never be offered is the
+ # KEYCHAIN door, which edits credentials the whole tenant uses."* That assertion
+ # is CORRECT and stays green. A member's personal connection is a different door
+ # into the same room, so it gets its own word.
+ #
+ # ⚠ THE PREDICATE IS THE CREDENTIAL'S OWN RULE, IMPORTED, NOT RE-STATED HERE.
+ # `TENANT_WIDE_TYPES` is what `routes_keychain` REFUSES a personal scope for, so
+ # offering a personal door for one of those types would be offering a button that
+ # 400s — the exact failure R6 removed from the Google card. Today that means
+ # Stripe and Shopify open for a member and Meta Ads does not
+ # ([[one-question-two-normalizers]]).
+ "manage": ("keychain" if is_admin
+ else "" if t["entry"] in TENANT_WIDE_TYPES
+ else "keychain_personal"),
+ # Said once, on the row, rather than in a paragraph somewhere (R13): storing the
+ # key is real and useful, and it is not the same as the data being on screen.
+ "note": "Storing the key connects the account; reading its data lands in a later "
+ "release.",
+ })
+
+ contribute("token", _tokens,
+ "the token-connector registry could not be composed",
+ "Stripe and Shopify are missing from this list only; nothing else is affected")
+
+ def _nango():
+ rows.extend(_nango_rows())
+
+ # ⚠ W32-T13 / ITEM 3 — THE CONTRIBUTOR KEY STAYS `nango`, THE SENTENCES DO NOT. `source` is an
+ # internal id (it must match `CONTRIBUTORS`, and `verify_meta` breaks this contribution by
+ # name), and `DirectoryGaps` renders only `cause` and `recommendation` — so the key never
+ # reaches a screen while these two strings do. De-branding the id instead would have renamed
+ # a gate's subject to fix a string nobody sees.
+ contribute("nango", _nango,
+ "the sign-in provider registry could not be composed",
+ "the providers you sign in to are missing from this list; every other row is "
+ "unaffected")
+
+ def _scraper():
+ rows.append(_scraper_row())
+
+ contribute("scraper", _scraper,
+ "the Scraper connector's readiness could not be determined "
+ "(automation_engine did not answer)",
+ "the Scraper card is missing from this list; automations are unaffected")
+
+ def _builtin():
+ rows.append({
+ "key": "webhooks", "label": "Webhooks", "kind": "builtin",
+ "desc": "Let another system start an automation by calling a URL.",
+ "state": "connected", "manage": "automation",
+ })
+
+ contribute("builtin", _builtin, "a built-in connector row could not be composed",
+ "reload; the remaining rows are correct")
+
+ def _planned():
+ for p in PLANNED:
+ rows.append({**p, "state": "planned", "needs": "not built yet", "manage": ""})
+
+ contribute("planned", _planned, "the planned-connector list could not be composed",
+ "the to-do list is missing; every CONNECTED row above is still correct")
+
+ # ⭐⭐ W49-T18 · R27 / C4 — THE TAXONOMY, AND THE CATALOGUE THAT DEFINES IT. Every row above
+ # is filed into one of the fourteen categories, and the authored catalogue is appended beside
+ # them so the map covers the things we can reach as well as the things we have wired. R27:
+ # *"don't skip Odoo or meta ads, its important to learn how they map the schema for our own
+ # benefit."* The crosswalk itself lives in `core.connector_directory`, which explains at
+ # length why it cannot be generated from Monid's API.
+ #
+ # ⛔⛔ ITS `source` IS NOT IN `CONTRIBUTORS`, AND THAT IS DELIBERATE RATHER THAN AN OVERSIGHT.
+ # `verify_meta.section_directory_gaps` asserts `set(CONTRIBUTORS) - probed == {"builtin",
+ # "odoo"}` against a probe table declared IN THAT GATE, which lives outside this ticket's
+ # fence — so adding the name here would turn a correct change red in a file this lane may not
+ # edit ([[two-gates-can-assert-opposite-things]]). The BEHAVIOUR the enumeration exists for is
+ # unchanged: this contribution is guarded like every other one, so a raise costs the taxonomy
+ # and records why, never the page. Booked as a PENDING row: add a `catalogue` probe to
+ # `verify_meta` and put the name in `CONTRIBUTORS` in the same change.
+ #
+ # ⚠ THE ORDER MATTERS. `file_natives` must run over the rows the contributors composed and
+ # BEFORE the catalogue is appended, or the catalogue rows would have their own categories
+ # overwritten with the empty native answer.
+ #
+ # ⛔⛔ AND THESE ROWS ARE NOT `planned`, WHICH THEY WERE FOR ONE WAVE. `planned` is the
+ # contribution directly above this one: a row from `PLANNED`, carrying a `needs`, that R8 put
+ # on this page as a visible to-do list. A vendor's catalogue listing satisfies none of that —
+ # we are not building it, nothing is waiting on us, and it has no `needs` — so it wears
+ # `connector_directory.CATALOGUE_STATE`, a SIXTH word beside
+ # `connected | available | reconnect | unconfigured | planned`. Borrowing `planned` for the
+ # fade it happens to render made the served planned set 120 rows wide and turned Monid's
+ # catalogue into our roadmap; the fade itself never depended on the word and does not now.
+ # ⚠ A GATE HOLDING A HAND-TYPED COPY OF THE FIVE-WORD SET MUST LEARN THE SIXTH. The vocabulary
+ # is a property of THIS module, and a literal set elsewhere is a second authority for it.
+ def _catalogue():
+ connector_directory.file_natives(rows)
+ rows.extend(connector_directory.monid_rows())
+
+ contribute("catalogue", _catalogue,
+ "the connector directory's category taxonomy could not be composed",
+ "every connector above is still listed and still correct; the category filter and "
+ "the wider catalogue are missing from this list only")
+
+ # ⭐⭐ W53-T29 / OWNER RULING R6 — EVERY ROW DECLARES WHICH SCOPES IT MAY EVER CARRY.
+ #
+ # R6: *"Each catalogue source declares which scopes it allows (Odoo company-only; Gmail
+ # both)."* The Company / Mine toggle needs that per SOURCE, not per stored connection: a card
+ # may only ever appear under Company if `business` is in this list, and under Mine only if
+ # `personal` is — so Odoo cannot land under Mine however its `scope` was written, and a
+ # personal-only source cannot land under Company.
+ #
+ # ⛔ ONE PASS OVER THE COMPOSED SET, NOT A LINE IN EACH CONTRIBUTOR. Six functions above append
+ # rows and a seventh extends the list with a vendor catalogue; a declaration repeated in each
+ # is a declaration the eighth composer forgets, and the client would then treat the missing
+ # key as "unrestricted" with nothing going red. Stamped here, a new contributor gets the right
+ # answer for free.
+ # ⛔ AND THE ANSWER IS `routes_keychain.allowed_scopes`, IMPORTED. This module already imports
+ # `TENANT_WIDE_TYPES` rather than keeping a copy (see the import note at the top of the file);
+ # the eligibility rule built ON that tuple is fetched the same way for the same reason.
+ # ⚠ A ROW WITH NO KEYCHAIN TYPE (an OAuth slot, the Scraper, a catalogue listing) answers with
+ # both words, which is the honest default: nothing about it forbids either scope, and the one
+ # thing the toggle must never do is hide a connection the reader actually holds.
+ for row in rows:
+ row["allowedScopes"] = list(allowed_scopes(_entry_type_for(row.get("key"))))
+ _safe_connector_status(row.get("key"), row)
+ return rows
+
+
+def _oauth_row(slug, meta, row_st):
+ """One OAuth provider's card. Lifted out of `directory` whole so the composer reads as a list
+ of contributions; behaviour is byte-identical."""
+ return {
+ "key": slug, "label": meta.get("label") or slug.title(), "kind": "oauth",
+ "desc": "Gmail for the email trigger; the same connection unlocks Sheets, Drive "
+ "and Calendar later." if slug == "google" else "",
+ # THREE distinct states, and the distinction is the point: a provider nobody has
+ # configured on this deployment is not the same as one this user has not connected,
+ # and until now both rendered identically (the C11 status() gap).
+ "state": ("connected" if row_st.get("connected") and not row_st.get("reconnect")
+ else "reconnect" if row_st.get("reconnect")
+ else "available" if row_st.get("configured")
+ else "unconfigured"),
+ "connectedAs": row_st.get("email") or "",
+ # ⭐⭐ W53-T29 / R6 — A PER-USER SLOT IS PERSONAL, AND THE CARD NOW SAYS SO.
+ # `_oauth_slot_entries` below already stamps `"scope": "personal"` on these very
+ # credentials ("a per-user slot is personal by construction, in the vocabulary
+ # `routes_keychain`"), and the ROW carried no scope at all — so a connected Google card
+ # read as business-wide on the page while its own Keychain tab called it personal, and the
+ # Company / Mine split would have filed this reader's own sign-in under Company. One fact,
+ # said the same way in both places.
+ # ⚠ NO CARD MOVES TODAY: `oauth_connect.offered()` filters every provider out of the
+ # directory (D-45, the CASA verification nobody has bought), so this is the answer waiting
+ # for the day one of them is offered again rather than a change to what renders now.
+ "scope": "personal",
+ "startUrl": f"/api/v1/oauth/{slug}/start",
+ "manage": "oauth",
+ "needs": "" if row_st.get("configured") else "the owner registers the client",
+ }
+
+
+def _nango_rows():
+ """One card per `NANGO_PROVIDERS` row, each stating what it can do TODAY (R7).
+
+ ⛔ NO NETWORK CALL. The obvious "honest" implementation asks Nango which integrations exist
+ and lights the ones that do — and that would put a third-party HTTP round trip on a page this
+ very wave is making faster, on a route that already had exactly one failure mode. The
+ deployment fact this page can answer CHEAPLY is whether the platform holds a Nango key at all,
+ which is the same class of check `bd_ready()` and `oauth_connect.configured()` already make.
+ ⚠ Whether a SPECIFIC integration is configured in the Nango dashboard is a different fact,
+ measured once in `proto/nango-verdict.md` (1 integration, and it is a GitHub sample), and it
+ is what `needs` names rather than pretends to have checked.
+
+ ⛔⛔ EVERY ROW IS `available` AT BEST AND CARRIES NO `manage` DOOR, which is R7's second
+ sentence made structural: *a card whose button leads nowhere without saying so* fails the
+ ruling, so these cards have no button at all and say why in `needs` and `note`.
+ """
+ have_key = bool((os.environ.get("NANGO_SECRET_KEY") or "").strip())
+ rows = []
+ for p in NANGO_PROVIDERS:
+ rows.append({
+ "key": p["key"], "label": p["label"], "kind": "oauth", "desc": p["desc"],
+ "provider": p["provider"],
+ "state": "available" if have_key else "unconfigured",
+ # ⭐⭐ WAVE 32 · OWNER ITEM 3 / R3 — DE-BRANDED. These two sentences used to read:
+ #
+ # needs: "this provider's integration must be created in the Nango dashboard first"
+ # / "the owner registers NANGO_SECRET_KEY on this deployment"
+ # note: "Connects through Nango. Reading this provider's data into a database is
+ # not built yet — measured: no sync is configured on this account."
+ #
+ # ⛔ R3 IS "DE-BRAND, DON'T REVOKE": the transport is unchanged and still Nango's —
+ # `oauth_connect.nango_probe`, `NANGO_SECRET_KEY` and every constant keep their names,
+ # because that is what the person setting the secret has to type. What goes is the
+ # supplier's name on a CUSTOMER's screen. Which vendor carries an OAuth handshake is
+ # our problem, exactly as `AIOS_BRIGHTDATA_KEY` sits behind a card labelled "Scraper".
+ # ⚠ THE HONESTY THE SENTENCE CARRIED IS NOT LOST, which is the half a de-brand usually
+ # drops: R7's rule is that a lit provider states what it can do TODAY, so the note
+ # still says "not built yet" in as many words, and `needs` still names the remaining
+ # step — it just no longer names the company doing it.
+ "needs": ("this connection has not been set up for this provider yet"
+ if have_key else
+ "the owner finishes connection setup on this deployment"),
+ # ⛔ THE SENTENCE R7 DEMANDS, on the row rather than in a paragraph somewhere (R13).
+ "note": "Signing in works. Reading this provider's data into a database is not "
+ "built yet.",
+ "manage": "",
+ })
+ return rows
+
+
+def _scraper_row():
+ """The Scraper card. Its own function because `bd_ready()` reaches into `automation_engine` —
+ an import and a call into another subsystem, i.e. the single most likely thing in this file to
+ raise, and until W31-T44 it could take the whole page with it."""
+ import automation_engine as _eng
+ # ⭐ WAVE 25 · C2 / owner ruling R9 — THE ROW IS "SCRAPER", NOT THE VENDOR'S NAME.
+ #
+ # ⛔ THE KEY CHANGED TOO (`brightdata` -> `scraper`), which is the half worth being careful
+ # about: it is what the trigger picker's Connector sub-group is named after, so the two must
+ # agree. Grepped before flipping — NO client file switches on this key (the six client hits
+ # for "brightdata" are all `CaptureTier`, which is the VENDOR and is deliberately untouched),
+ # and the one server reader was `verify_automation`'s directory-key check, retargeted in the
+ # same change.
+ #
+ # ⚠ THE VENDOR IS REAL AND STAYS REAL. `AIOS_BRIGHTDATA_KEY`, `bd_ready()` and every `BD_*`
+ # constant are the supplier and keep their names — this renames what a CUSTOMER sees, because
+ # which supplier answers a scrape is our problem and not something to make them learn. The
+ # `needs` string still names the env var, deliberately: it is read by whoever has to set it.
+ ready = _eng.bd_ready()
+ return {
+ "key": "scraper", "label": "Scraper", "kind": "builtin",
+ # ⭐ WAVE 30 · R4 — the sentence names the CAPABILITY; the platforms are the drill below.
+ # It used to say "Instagram profile and post capture", which stopped being true the day
+ # TikTok discovery shipped and would have to be re-edited for every platform after it.
+ "desc": "Create saved Instagram, TikTok and Google Maps database recipes.",
+ "state": "connected" if ready else "unconfigured",
+ "needs": "" if ready else "AIOS_BRIGHTDATA_KEY is not set on this deployment",
+ "manage": "",
+ # The directory taxonomy normally derives this flag from connected table prefixes.
+ # Scraper recipes are user databases, so the route stamps it again after normalisation.
+ "spawns_tables": True,
+ # Platforms remain a drill behind one recipe/database capability, never directory rows.
+ # `bd_ready()` describes today's social runners only; Maps' own sentence names that its
+ # execution provider is intentionally absent until T12.
+ "platforms": [dict(p) for p in SCRAPER_PLATFORMS],
+ }
+
+
+@router.get("/connectors/directory")
+def connectors_directory(session: Session = Depends(require_session)):
+ """C11: every connector this platform knows about, with its state for THIS reader.
+
+ Session-gated rather than admin-gated on purpose: a non-admin should be able to see what the
+ workspace is connected to, and connect their OWN per-user accounts — which is the whole point
+ of R7's per-user slots.
+
+ ⭐⭐ W33-T66 / D-204 — AND THAT SECOND CLAUSE IS NOW TRUE. This docstring used to promise it and
+ then say, one sentence later, that `manage` *"is empty for a non-admin"* — a contradiction
+ inside a single paragraph, which is how the promise survived three waves without a door behind
+ it. `manage` carries WHICH ROOM, and there are three answers for a keychain-backed row:
+
+ · `keychain` — the tenant-wide credential door. **Admins only**, and it must stay
+ that way: it edits the key the whole workspace reads through.
+ · `keychain_personal` — the same room, for a connection that is this reader's own. Offered
+ to a non-admin **only** where a personal credential is actually
+ allowed (`routes_keychain.TENANT_WIDE_TYPES` is the refusal list, so
+ Odoo and Meta Ads never get it).
+ · `""` — nothing this reader can do here; the client renders no control
+ rather than a door that would 403.
+ """
+ # ⚠ `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as
+ # careful and is silently False for every admin on earth — the manage actions would simply
+ # never appear, with nothing to catch it. Attribute name checked against `deps.Session`.
+ # ⭐ W31-T44 — `incomplete` IS PART OF THE ANSWER, not an error channel. The page's job is to
+ # be a map of the workspace; a map missing a region must say which region, or the reader
+ # concludes the region does not exist. That is the owner's item 6b in one sentence, and it is
+ # R6's second sentence applied to a payload: a limit that cannot be removed is REPORTED with
+ # its cause and a recommended fix. `[]` on a healthy read, so the client can test it directly.
+ incomplete = []
+ rows = directory(session.runtime, session.uname, bool(session.admin), incomplete=incomplete)
+ # ⚠ W49-T18 — GRAPHS ARE ATTACHED TO OUR OWN ROWS ONLY, and the reason is copy rather than
+ # cost. `relationship_graphs` does no I/O for a row that is not `connected` (it returns a
+ # refusal envelope and the expensive `user_tables.all_defs` read is keyed on the connected
+ # set), so running it over the catalogue would be cheap. What it would produce is 118 copies
+ # of the sentence *"Connect this provider before viewing its actual relationship graph"* about
+ # rows that carry no credential and no door, which is a promise the product cannot keep.
+ # ⚠ THE CLIENT ALREADY HANDLES THE ABSENCE: `parseRelationshipGraph` answers a missing
+ # `connectionKey` with `graphRefusal(selectedKey, ...)`, so a card with no envelope on the
+ # wire renders exactly as it did before. The list comprehension holds the SAME dicts, so the
+ # in-place attachment still lands on the native rows.
+ attach_relationship_graphs([r for r in rows if r.get("provider") != "monid"],
+ session.user, session.runtime, session.uname,
+ bool(session.admin))
+ # ⛔⛔ W49-T18 / C4 — THE ONE PLACE THE SERVER ROW BECOMES THE CLIENT DTO. `provider`
+ # (`native` | `monid`) is how we reason about where a row came from; R27's design rule is that
+ # it never renders, in any form, so it is stripped HERE rather than left to each reader to
+ # remember not to paint. `verify_catalog.py` reads this payload's shape and its negative
+ # control leaks the key on purpose to prove the leg can bite.
+ # ⚠ THE CHIP COUNTS ARE COMPUTED FROM THE ROWS ACTUALLY SERVED, so the number on a chip is the
+ # number of cards behind it by construction, and never a sum of endpoint counts (the ticket's
+ # trap; `chip_counts` carries the argument).
+ served = connector_directory.to_client_rows(rows)
+ for row in served:
+ if str(row.get("key") or "") == "scraper":
+ row["spawns_tables"] = True
+ _safe_connector_status(row.get("key"), row)
+ _safe_connector_status("amazon_sp", {"incomplete": incomplete})
+ return {"connectors": served,
+ "incomplete": incomplete,
+ # ⭐⭐ W53-T29 / OWNER RULING R6 — MAY THIS READER SEE THE COMPANY SIDE AT ALL.
+ #
+ # R6: *"A non-admin does not see the Company side at all; the toggle itself is hidden
+ # for them."* That is a PERMISSION, so it is the server's answer and not something the
+ # page infers from whether some row happened to carry `manage: "keychain"`.
+ # ⛔ THE WORD IS `routes_keychain`'s OWN. `GET /admin/keychain` has answered
+ # `"canBusiness": bool(session.admin)` since W32-T11, for exactly this purpose ("the
+ # vocabulary and the permission, so the client renders a picker it can honour"), and
+ # the same fact under a second name would be two vocabularies for one boolean.
+ # ⚠ CARRIED HERE RATHER THAN FETCHED FROM THAT ROUTE. The keychain door lists every
+ # visible ENTRY, so a page that wanted one boolean would pull the workspace's whole
+ # credential list on every visit; this route already holds the session that decides it.
+ "canBusiness": bool(session.admin),
+ "categories": connector_directory.chip_counts(served),
+ # ⛔⛔ `planned` COUNTS THE ROADMAP AGAIN, AND FOR ONE WAVE IT COUNTED 120. W49-T18
+ # gave every Monid listing the `planned` state because that state renders faded, so
+ # this number stopped being "things we intend to build" and became "things we intend
+ # to build, plus a vendor's whole catalogue". The listings wear
+ # `connector_directory.CATALOGUE_STATE` now and are counted as what they are; the
+ # constant is read rather than re-typed so the two cannot drift apart.
+ "counts": {"connected": sum(1 for r in served if r["state"] == "connected"),
+ "available": sum(1 for r in served if r["state"] == "available"),
+ "planned": sum(1 for r in served if r["state"] == "planned"),
+ "catalogue": sum(1 for r in served
+ if r["state"] == connector_directory.CATALOGUE_STATE)}}
+
+
+_ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements"
+_ODOO_SYNC_STATUS = "odoo_sync_status"
+_ODOO_HEALTH_STATUS = "odoo_health_status_v1"
+_ODOO_HEALTH_STAGES = ("source", "mirrorPull", "materialization", "reconciliation", "freshness")
+_SYNC_REQUEST_CONNECTIONS = {}
+_SYNC_REQUEST_CONNECTION_LOCK = threading.Lock()
+
+
+def _odoo_connection_slot(rt, connection_id=None):
+ """Return the durable Odoo connection id used by jobs, status, and health receipts."""
+ requested = str(connection_id or "").strip()
+ if requested and requested != "default":
+ return requested
+ try:
+ import odoo_relational
+ return str(odoo_relational.default_connection_id(rt) or "default")
+ except Exception: # noqa: BLE001
+ return "default"
+
+
+def sync_request_connection(provider, tenant):
+ """Return the connection bound to a process-local Sync now intent."""
+ with _SYNC_REQUEST_CONNECTION_LOCK:
+ return _SYNC_REQUEST_CONNECTIONS.get((str(provider), str(tenant)))
+
+
+def _odoo_failure(exc, stage="source"):
+ """Classify a source/database exception without persisting its potentially sensitive text."""
+ stage = stage if stage in _ODOO_HEALTH_STAGES else "source"
+ text = str(exc or "").casefold()
+ exc_type = type(exc).__name__ if isinstance(exc, BaseException) else "SourceError"
+ if (isinstance(exc, ssl.SSLCertVerificationError)
+ or "certificate_verify_failed" in text
+ or "certificate verify failed" in text
+ or "certificate has expired" in text
+ or "self signed certificate" in text):
+ return {"stage": "source", "code": "tls_certificate_invalid",
+ "type": exc_type,
+ "message": ("Odoo TLS certificate verification failed because the certificate "
+ "is expired or untrusted. No source data was accepted.")}
+ if isinstance(exc, xmlrpc.client.Fault) and any(
+ token in text for token in ("accesserror", "access denied", "security restrictions")):
+ return {"stage": "source", "code": "source_read_refused", "type": exc_type,
+ "message": "Odoo refused the configured account's read access."}
+ if any(token in text for token in ("authentication", "access denied", "invalid api key")):
+ return {"stage": "source", "code": "source_auth_failed", "type": exc_type,
+ "message": "Odoo rejected the configured read-only credential."}
+ if isinstance(exc, (TimeoutError, ConnectionError)) or any(
+ token in text for token in ("timed out", "timeout", "name resolution", "connection refused")):
+ return {"stage": "source", "code": "source_unavailable", "type": exc_type,
+ "message": "Odoo could not be reached for this read-only sync."}
+ labels = {
+ "source": ("source_read_failed", "Odoo source validation failed."),
+ "mirrorPull": ("mirror_pull_failed", "The Odoo mirror pull failed."),
+ "materialization": ("materialization_failed", "The Odoo database could not be materialized."),
+ "reconciliation": ("reconciliation_failed", "The Odoo database reconciliation failed."),
+ "freshness": ("freshness_unconfirmed", "Freshness could not be confirmed for this database."),
+ }
+ code, message = labels[stage]
+ return {"stage": stage, "code": code, "type": exc_type, "message": message}
+
+
+def _clean_odoo_failure(failure, stage):
+ if failure is None:
+ return None
+ if not isinstance(failure, dict):
+ return _odoo_failure(failure, stage)
+ return {"stage": str(failure.get("stage") or stage),
+ "code": str(failure.get("code") or f"{stage}_failed"),
+ "type": str(failure.get("type") or "SourceError")[:80],
+ "message": str(failure.get("message") or "Odoo sync failed.")[:300]}
+
+
+def _odoo_stage_failure(stages):
+ """Return the earliest unresolved lifecycle failure, preserving its named cause."""
+ return next((stages.get(name, {}).get("failure") for name in _ODOO_HEALTH_STAGES
+ if stages.get(name, {}).get("state") == "failed"
+ and stages.get(name, {}).get("failure")), None)
+
+
+def _clean_odoo_database_receipt(receipt, database_key):
+ """Allow-list one data-free reconciliation receipt for durable health storage."""
+ if not isinstance(receipt, dict):
+ return None
+ key = str(database_key or "").strip()
+ if str(receipt.get("databaseKey") or "").strip() != key:
+ raise ValueError("the reconciliation receipt belongs to a different database")
+ out = {"databaseKey": key}
+ for name in ("schemaKey", "topicKey", "domainKey", "model", "countModel", "state",
+ "sourceCountMode", "sourceReconciliation"):
+ value = receipt.get(name)
+ if value is not None:
+ out[name] = str(value)[:160]
+ for name in ("sourceCount", "mirrorCount", "materializedCount", "canonicalCount",
+ "servingCount", "duplicateCount", "explainedMergeCount",
+ "unexplainedMissingCount", "unexplainedExtraCount",
+ "unexplainedCanonicalMissingCount", "unexplainedCanonicalExtraCount",
+ "unexplainedCount"):
+ value = receipt.get(name)
+ out[name] = None if value is None else max(0, int(value))
+ if "readThrough" in receipt:
+ out["readThrough"] = bool(receipt.get("readThrough"))
+ selection = receipt.get("selection")
+ if isinstance(selection, dict):
+ out["selection"] = {
+ name: (bool(selection.get(name)) if name == "enabled"
+ else str(selection.get(name) or "")[:160])
+ for name in ("bucket", "rule", "enabled") if name in selection}
+ domain = receipt.get("domain")
+ if isinstance(domain, list):
+ # Curated domains contain public field/operator/value predicates. Refuse nested objects,
+ # which could otherwise turn this metadata seam into durable source-row storage.
+ clean_domain = []
+ for predicate in domain:
+ if not isinstance(predicate, (list, tuple)) or len(predicate) != 3:
+ continue
+ field, operator, value = predicate
+ if isinstance(value, (list, tuple)):
+ clean_value = [item for item in value
+ if isinstance(item, (str, int, float, bool))][:50]
+ elif isinstance(value, (str, int, float, bool)) or value is None:
+ clean_value = value
+ else:
+ continue
+ clean_domain.append([str(field)[:100], str(operator)[:40], clean_value])
+ out["domain"] = clean_domain
+ reconciliation = receipt.get("reconciliation")
+ if isinstance(reconciliation, dict):
+ out["reconciliation"] = {
+ name: str(reconciliation.get(name) or "")[:80]
+ for name in ("sourceToMirror", "mirrorToMaterialized", "mirrorToCanonical")
+ if name in reconciliation}
+ freshness = receipt.get("freshness")
+ if isinstance(freshness, dict):
+ out["freshness"] = {
+ name: str(freshness.get(name) or "")[:100]
+ for name in ("state", "capturedAt") if name in freshness}
+ for name in ("sourceError", "error"):
+ if receipt.get(name) is not None:
+ out[name] = _clean_odoo_failure(receipt.get(name), "reconciliation")
+ return out
+
+
+def record_odoo_stage(rt, connection_id, stage, state, *, database_key=None, row_count=None,
+ refreshed_at=None, cadence_seconds=None, failure=None, receipt=None,
+ at=None):
+ """Persist one data-free Odoo stage transition for one connection/database.
+
+ This is additive observability over the established ``connector_sync`` and materialization
+ job documents. It stores identifiers, counts, timestamps, and classified failures only;
+ credentials and source rows never cross this seam.
+ """
+ if stage not in _ODOO_HEALTH_STAGES:
+ raise ValueError(f"unknown Odoo health stage: {stage}")
+ if state not in ("unknown", "queued", "running", "ready", "failed", "paused"):
+ raise ValueError(f"unknown Odoo health state: {state}")
+ cid = _odoo_connection_slot(rt, connection_id)
+ stamp = str(at or datetime.now(timezone.utc).isoformat())
+ failed = _clean_odoo_failure(failure, stage) if state == "failed" else None
+ db_key = str(database_key or "").strip()
+ safe_receipt = _clean_odoo_database_receipt(receipt, db_key) if receipt is not None else None
+
+ def _update(current):
+ document = dict(current or {})
+ connections = dict(document.get("connections") or {})
+ connection = dict(connections.get(cid) or {})
+ stages = dict(connection.get("stages") or {})
+ target_stages = stages
+ database = None
+ databases = None
+ if db_key:
+ databases = dict(connection.get("databases") or {})
+ database = dict(databases.get(db_key) or {"databaseKey": db_key})
+ target_stages = dict(database.get("stages") or {})
+ previous = dict(target_stages.get(stage) or {})
+ stage_row = {**previous, "state": state, "at": stamp, "failure": failed}
+ if state == "running":
+ stage_row["startedAt"] = stamp
+ stage_row["finishedAt"] = None
+ elif state in ("ready", "failed", "paused"):
+ stage_row["finishedAt"] = stamp
+ if state == "ready":
+ stage_row["lastSuccess"] = stamp
+ target_stages[stage] = stage_row
+ connection.update({"connectionId": cid, "stages": stages, "lastAttempt": stamp})
+ if cadence_seconds is not None:
+ connection["cadenceSeconds"] = int(cadence_seconds)
+
+ if db_key:
+ database.update({"databaseKey": db_key, "stages": target_stages,
+ "lastAttempt": stamp})
+ if row_count is not None:
+ database["rowCount"] = max(0, int(row_count))
+ if refreshed_at:
+ database["recordRefreshed"] = str(refreshed_at)
+ if safe_receipt is not None:
+ database["reconciliationReceipt"] = safe_receipt
+ db_failure = _odoo_stage_failure(target_stages)
+ if db_failure:
+ database["failure"] = db_failure
+ else:
+ database.pop("failure", None)
+ if state == "ready" and stage == "freshness":
+ database["lastSuccess"] = stamp
+ databases[db_key] = database
+ connection["databases"] = databases
+ else:
+ stages[stage] = stage_row
+ connection["stages"] = stages
+ connection_failure = _odoo_stage_failure(stages)
+ if connection_failure:
+ connection["failure"] = connection_failure
+ else:
+ connection.pop("failure", None)
+ if state == "ready" and stage == "freshness":
+ connection["lastSuccess"] = stamp
+ connections[cid] = connection
+ document["connections"] = connections
+ return document
+
+ rt.update(_ODOO_HEALTH_STATUS, _update, flush="sync")
+ return odoo_health_receipt(rt, cid)
+
+
+def _lag_seconds(stamp, now=None):
+ if not stamp:
+ return None
+ try:
+ parsed = datetime.fromisoformat(str(stamp).replace("Z", "+00:00"))
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=timezone.utc)
+ current = now or datetime.now(timezone.utc)
+ return max(0, int((current - parsed.astimezone(timezone.utc)).total_seconds()))
+ except (TypeError, ValueError):
+ return None
+
+
+def _empty_odoo_stages():
+ return {stage: {"state": "unknown", "at": None, "startedAt": None,
+ "finishedAt": None, "lastSuccess": None, "failure": None}
+ for stage in _ODOO_HEALTH_STAGES}
+
+
+def odoo_health_receipt(rt, connection_id=None, *, now=None):
+ """Project the truthful Odoo receipt for one connection and each of its databases."""
+ cid = _odoo_connection_slot(rt, connection_id)
+ try:
+ document = rt.get(_ODOO_HEALTH_STATUS, fresh=True) or {}
+ except TypeError:
+ document = rt.get(_ODOO_HEALTH_STATUS) or {}
+ except Exception: # noqa: BLE001
+ document = {}
+ stored = dict(((document.get("connections") or {}).get(cid)) or {})
+ stages = _empty_odoo_stages()
+ stages.update({key: dict(value) for key, value in (stored.get("stages") or {}).items()
+ if key in stages and isinstance(value, dict)})
+ databases = {str(key): dict(value) for key, value in (stored.get("databases") or {}).items()
+ if isinstance(value, dict)}
+
+ # Durable materialization receipts are the authority for database work and survive restarts.
+ for job in connector_sync.jobs(rt, "odoo", cid):
+ key = str(job.get("databaseKey") or "")
+ if not key:
+ continue
+ database = dict(databases.get(key) or {"databaseKey": key})
+ db_stages = _empty_odoo_stages()
+ db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items()
+ if name in db_stages and isinstance(value, dict)})
+ job_state = str(job.get("state") or "unknown")
+ material_state = {"configured": "queued", "catalogue-ready": "queued",
+ "queued": "queued", "syncing": "running", "ready": "ready",
+ "failed": "failed"}.get(job_state, "unknown")
+ job_at = None
+ if job.get("at") is not None:
+ try:
+ job_at = datetime.fromtimestamp(float(job["at"]), timezone.utc).isoformat()
+ except (TypeError, ValueError, OSError):
+ job_at = None
+ job_failure = (_odoo_failure(job.get("error"), "materialization")
+ if material_state == "failed" else None)
+ db_stages["materialization"] = {
+ **db_stages["materialization"], "state": material_state,
+ "at": job_at, "finishedAt": job_at if material_state in ("ready", "failed") else None,
+ "lastSuccess": (job_at if material_state == "ready" else
+ db_stages["materialization"].get("lastSuccess")),
+ "failure": job_failure,
+ }
+ progress = job.get("progress") if isinstance(job.get("progress"), dict) else {}
+ if progress.get("rows") is not None:
+ database["rowCount"] = max(0, int(progress["rows"]))
+ database.update({"databaseKey": key, "model": str(job.get("model") or ""),
+ "state": material_state, "stages": db_stages})
+ persisted_failure = database.get("failure") or _odoo_stage_failure(db_stages)
+ if persisted_failure or job_failure:
+ database["failure"] = persisted_failure or job_failure
+ elif material_state == "ready" and not database.get("failure"):
+ database["lastSuccess"] = database.get("lastSuccess") or job_at
+ databases[key] = database
+
+ # The resident mirror state is process-local by design; overlay it only on the connection
+ # that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
+ cadence = stored.get("cadenceSeconds")
+ try:
+ _main = running_main() # D-781: resolve, never import
+ resident = dict(_main.RESYNC_STATE)
+ resident_cid = str(resident.get("connectionId") or "")
+ if resident_cid == cid:
+ cadence = resident.get("resolvedSeconds") or cadence
+ mirror_failure = resident.get("lastError")
+ mirror_state = ("paused" if resident.get("manual") else
+ "failed" if resident.get("lastPassOk") is False else
+ "ready" if resident.get("lastPassOk") is True else "unknown")
+ stages["mirrorPull"] = {**stages["mirrorPull"], "state": mirror_state,
+ "at": resident.get("lastPassAt"),
+ "finishedAt": resident.get("lastPassAt"),
+ "lastSuccess": resident.get("lastSuccessAt"),
+ "failure": (_clean_odoo_failure(mirror_failure, "mirrorPull")
+ if mirror_failure else None)}
+ except Exception: # noqa: BLE001
+ pass
+
+ for database in databases.values():
+ db_stages = _empty_odoo_stages()
+ db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items()
+ if name in db_stages and isinstance(value, dict)})
+ # Connection source/mirror/reconciliation facts apply to each database, while its own
+ # materialization/freshness facts remain independent.
+ for stage in ("source", "mirrorPull", "reconciliation"):
+ if db_stages[stage].get("state") == "unknown":
+ db_stages[stage] = dict(stages[stage])
+ database["stages"] = db_stages
+ success = database.get("lastSuccess") or db_stages["freshness"].get("lastSuccess")
+ database["lastSuccess"] = success
+ database["currentLagSeconds"] = _lag_seconds(success, now)
+ database.setdefault("rowCount", None)
+ database.setdefault("recordRefreshed", None)
+ stage_failure = _odoo_stage_failure(db_stages)
+ if stage_failure:
+ database["failure"] = stage_failure
+ elif not database.get("failure"):
+ database["failure"] = None
+
+ successes = [row.get("lastSuccess") for row in databases.values() if row.get("lastSuccess")]
+ last_success = max(successes, default=stored.get("lastSuccess"))
+ row_counts = [row.get("rowCount") for row in databases.values()
+ if row.get("rowCount") is not None]
+ failure = (_odoo_stage_failure(stages) or stored.get("failure") or next(
+ (row.get("failure") for row in databases.values() if row.get("failure")), None))
+ return {"connectionId": cid, "cadenceSeconds": cadence,
+ "lastSuccess": last_success, "currentLagSeconds": _lag_seconds(last_success, now),
+ "rowCount": sum(row_counts) if row_counts else None,
+ "failure": failure, "stages": stages,
+ "databases": [databases[key] for key in sorted(databases)]}
+
+
+def _odoo_admin(session):
+ if not session.admin:
+ raise err(403, "not_admin", "only an administrator can inspect the Odoo schema")
+
+
+def _sync_admin(session):
+ if not session.admin:
+ raise err(403, "not_admin", "only an administrator can retry a connector sync")
+
+
+def _connection_jobs(rt, provider, connection_id):
+ """A status endpoint selects one connection; the jobs inventory may enumerate them all."""
+ cid = str(connection_id or "").strip() or "default"
+ if provider == "odoo" and cid == "default":
+ cid = _odoo_connection_slot(rt, connection_id)
+ return connector_sync.jobs(rt, provider, cid)
+
+
+def _sync_payload(connector_key, connection_id, row=None):
+ """One wire shape for every registered connector, including one not yet run."""
+ meta = connector_sync.providers()[connector_key]
+ slot = str(connection_id or "").strip() or "default"
+ if row is None:
+ return _safe_connector_status(connector_key, {"provider": connector_key, "label": meta["label"],
+ "connectionId": slot, "recorded": False, "state": None,
+ "at": None, "stateAt": None, "from": None,
+ "detail": "no sync has been recorded", "errors": [], "errorsTotal": 0,
+ "databases": [], "retryable": False, "retryHint": meta["retryHint"]})
+ answer = dict(row)
+ answer.update({"provider": connector_key, "connectionId": slot,
+ "label": meta["label"], "recorded": True,
+ "stateAt": row.get("at"), "retryHint": meta["retryHint"]})
+ return _safe_connector_status(connector_key, answer)
+
+
+def _odoo_result_database_failures(rt, connection_id, result):
+ """Map provider-reported model/table failures to their durable database keys."""
+ import odoo_relational as rel
+ try:
+ choices = rel.grid_choices(rt, connection_id)
+ except Exception: # noqa: BLE001
+ choices = []
+ by_model = {str(row.get("model") or ""): str(row.get("key") or "")
+ for row in choices if row.get("model") and row.get("key")}
+ failures = {str(key): _clean_odoo_failure(value, "materialization")
+ for key, value in (result.get("databaseErrors") or {}).items()}
+ receipt_groups = []
+ for name in ("databases", "canonicalDatabases"):
+ group = result.get(name) if isinstance(result, dict) else None
+ if isinstance(group, dict):
+ receipt_groups.append(group)
+ for group in receipt_groups:
+ for raw_key, receipt in group.items():
+ if not isinstance(receipt, dict) or receipt.get("state") == "succeeded":
+ continue
+ key = str(raw_key or "")
+ if not key or key in failures:
+ continue
+ raw_failure = receipt.get("error") or receipt.get("sourceError") or {
+ "stage": "reconciliation", "code": "reconciliation_incomplete",
+ "type": "ReconciliationError",
+ "message": "The Odoo database reconciliation is incomplete.",
+ }
+ failures[key] = _clean_odoo_failure(raw_failure, "reconciliation")
+ unmatched = []
+ discovered = result.get("discovered") if isinstance(result, dict) else {}
+ for raw in ((discovered or {}).get("errors") or ()):
+ model = str(raw).split(":", 1)[0].strip()
+ key = by_model.get(model)
+ if key:
+ failures[key] = _odoo_failure(raw, "materialization")
+ else:
+ unmatched.append(str(raw))
+ limits = result.get("limits") if isinstance(result, dict) else None
+ if isinstance(limits, dict):
+ for raw_key, raw in limits.items():
+ key = by_model.get(str(raw_key or ""), str(raw_key or ""))
+ if key:
+ failures[key] = _odoo_failure(raw, "materialization")
+ elif limits:
+ unmatched.append(str(limits))
+ return failures, unmatched
+
+
+def _odoo_sync_runner(session, connection_id):
+ """Run Odoo through C6 without moving provider imports into the data layer."""
+ cid = _odoo_connection_slot(session.runtime, connection_id)
+ connector_sync.record(session.runtime, "odoo", cid, "syncing")
+ record_odoo_stage(session.runtime, cid, "source", "running")
+ record_odoo_stage(session.runtime, cid, "materialization", "queued")
+ import odoo_relational
+ def _materializing():
+ connector_sync.record(session.runtime, "odoo", cid, "materializing")
+ record_odoo_stage(session.runtime, cid, "materialization", "running")
+ try:
+ result = odoo_relational.refresh(session.runtime, session.tenant,
+ username=session.uname or "automation",
+ connection_id=connection_id,
+ on_apply=_materializing)
+ except Exception as exc:
+ failure = _odoo_failure(exc, "source")
+ record_odoo_stage(session.runtime, cid, failure["stage"], "failed", failure=failure)
+ record_odoo_stage(session.runtime, cid, "freshness", "failed",
+ failure={**failure, "stage": "freshness"})
+ connector_sync.record(session.runtime, "odoo", cid, "error",
+ errors=[f"{failure['code']}: {failure['message']}"])
+ raise
+ tables = result.get("tables") if isinstance(result, dict) else {}
+ published = dict(tables or {})
+ database_receipts = result.get("databases") if isinstance(result, dict) else {}
+ database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
+ canonical_receipts = result.get("canonicalDatabases") if isinstance(result, dict) else {}
+ canonical_receipts = canonical_receipts if isinstance(canonical_receipts, dict) else {}
+ reconciliation_receipts = {**database_receipts, **canonical_receipts}
+ databases = sorted(str(key) for key in published)
+ sync_errors = []
+ if isinstance(result, dict):
+ if result.get("failed"):
+ sync_errors.append(str(result["failed"]))
+ discovered = result.get("discovered")
+ if isinstance(discovered, dict):
+ sync_errors.extend(str(value) for value in (discovered.get("errors") or ()))
+ discovered_tables = discovered.get("tables")
+ if isinstance(discovered_tables, dict):
+ published.update(discovered_tables)
+ databases.extend(str(key) for key in discovered_tables)
+ if result.get("frozen"):
+ sync_errors.append(str(result.get("note") or "the connector is frozen"))
+ database_failures, unmatched_errors = _odoo_result_database_failures(
+ session.runtime, connection_id, result if isinstance(result, dict) else {})
+ databases.extend(reconciliation_receipts)
+ databases.extend(database_failures)
+ sync_errors.extend(f"{key}: {failure['code']}" for key, failure in database_failures.items())
+ sync_errors.extend(raw for raw in unmatched_errors if raw not in sync_errors)
+ state = ("partial" if published else "error") if sync_errors else "ready"
+ record_odoo_stage(session.runtime, cid, "source", "ready")
+ refreshed_at = datetime.now(timezone.utc).isoformat()
+ database_keys = list(dict.fromkeys([*reconciliation_receipts, *published]))
+ for key in database_keys:
+ if key in database_failures:
+ continue
+ counts = published.get(key)
+ receipt = reconciliation_receipts.get(key)
+ receipt = receipt if isinstance(receipt, dict) else None
+ nrows = (receipt.get("servingCount") if receipt is not None
+ else counts.get("rows") if isinstance(counts, dict) else None)
+ record_odoo_stage(session.runtime, cid, "source", "ready", database_key=key,
+ receipt=receipt)
+ record_odoo_stage(session.runtime, cid, "materialization", "ready",
+ database_key=key, row_count=nrows, receipt=receipt)
+ record_odoo_stage(session.runtime, cid, "reconciliation", "ready",
+ database_key=key, row_count=nrows, receipt=receipt)
+ record_odoo_stage(session.runtime, cid, "freshness", "ready",
+ database_key=key, row_count=nrows, refreshed_at=refreshed_at,
+ receipt=receipt)
+ for key, failure in database_failures.items():
+ receipt = reconciliation_receipts.get(key)
+ receipt = receipt if isinstance(receipt, dict) else None
+ failed_stage = str(failure.get("stage") or "materialization")
+ failed_stage = failed_stage if failed_stage in _ODOO_HEALTH_STAGES else "materialization"
+ record_odoo_stage(session.runtime, cid, failed_stage, "failed",
+ database_key=key, failure=failure, receipt=receipt)
+ record_odoo_stage(session.runtime, cid, "freshness", "failed", database_key=key,
+ failure={**failure, "stage": "freshness"}, receipt=receipt)
+ if sync_errors:
+ failure = _odoo_failure("provider result contained errors", "materialization")
+ record_odoo_stage(session.runtime, cid, "materialization", "failed", failure=failure)
+ record_odoo_stage(session.runtime, cid, "freshness", "failed",
+ failure={**failure, "stage": "freshness"})
+ else:
+ record_odoo_stage(session.runtime, cid, "materialization", "ready")
+ record_odoo_stage(session.runtime, cid, "reconciliation", "ready")
+ record_odoo_stage(session.runtime, cid, "freshness", "ready")
+ return connector_sync.record(
+ session.runtime, "odoo", cid, state,
+ detail=str(result.get("note") or "") if isinstance(result, dict) else "",
+ errors=sync_errors, databases=sorted(set(databases)),
+ )
+
+
+# Provider runners are an API-layer capability. The data-layer registration proves Amazon uses
+# the same contract while this table honestly leaves it without an executable sync.
+_SYNC_RUNNERS = {"odoo": _odoo_sync_runner}
+
+
+ODOO_DISCONNECTED_CAUSE = (
+ "Odoo is disconnected, so rows cannot load. "
+ "Reconnect Odoo in the Keychain and this resumes on its own.")
+
+
+class _SupersededMaterialization(Exception):
+ pass
+
+
+def _invalidate_materialization(rt, key):
+ """Fence a publisher in the same atomic document that holds the records."""
+ import core.user_tables as ut
+ def apply(current):
+ doc = dict(current or {})
+ if key in doc:
+ doc[key] = {**doc[key], "_connectorJob": None}
+ return doc
+ rt.update(ut.STORE_KEY, apply, flush="sync")
+
+
+def _queue_odoo_database(rt, key, *, model="", retry=False, changed=False):
+ import odoo_relational as rel
+ cid = rel.connection_for_table_key(key, rt)
+ revision = connector_sync.configuration_revision(
+ "odoo", cid, key, model, rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {})
+ previous = (rt.get(connector_sync.JOBS_KEY, fresh=True) or {}).get(
+ connector_sync.job_key("odoo", cid, key)) or {}
+ if previous.get("state") == "syncing" and (changed or previous.get("revision") != revision):
+ _invalidate_materialization(rt, key)
+ job = connector_sync.enqueue(
+ rt, "odoo", cid, key, model=model, retry=retry, changed=changed, revision=revision)
+ # The runtime service is started by ASGI lifespan, never by a handler or a fixture import.
+ _materialization_wake.set()
+ return job
+
+
+class _MaterializationRuntime:
+ """Delegate tenant reads; fence each row publication inside the store's atomic updater."""
+ def __init__(self, rt, job):
+ self.rt, self.job = rt, job
+
+ def __getattr__(self, name):
+ return getattr(self.rt, name)
+
+ def valid(self):
+ import odoo_relational as rel
+ key = self.job["databaseKey"]
+ if not connector_sync.owns(self.rt, self.job):
+ raise _SupersededMaterialization()
+ cfg = self.rt.get(rel.CONFIG_KEY, fresh=True) or {}
+ if cfg.get("frozen") or (cfg.get("grids") or {}).get(key) is False:
+ raise _SupersededMaterialization()
+ # Save commits field choices before the queue update. The lease alone cannot
+ # detect that newer configuration if enqueue fails or has not run yet.
+ revision = self.job.get("revision")
+ if revision is not None and revision != connector_sync.configuration_revision(
+ self.job["provider"], self.job["connectionId"], key, self.job["model"],
+ self.rt.get(_ODOO_SCHEMA_ENABLEMENTS, fresh=True) or {}):
+ raise _SupersededMaterialization()
+
+ def update(self, name, fn, flush="sync", **kwargs):
+ import core.user_tables as ut
+ key = self.job["databaseKey"]
+ marker = [self.job["id"], self.job["leaseToken"]]
+ def apply(current):
+ self.valid()
+ missing = name == ut.STORE_KEY and key not in current
+ if (name == ut.STORE_KEY and not missing
+ and (current.get(key) or {}).get("_connectorJob") != marker):
+ raise _SupersededMaterialization()
+ answer = fn(current)
+ if missing and key in answer:
+ # A reconciled receipt may precede its definition. Its first schema write
+ # installs the marker under the same lease and configuration fence.
+ answer[key]["_connectorJob"] = marker
+ # Lease expiry during a long merge cannot acknowledge a stale write either.
+ self.valid()
+ return answer
+ return self.rt.update(name, apply, flush=flush, **kwargs)
+
+
+def _odoo_read_access_refused(exc):
+ """Match only XML-RPC faults with explicit access/read refusal markers."""
+ if not isinstance(exc, xmlrpc.client.Fault):
+ return False
+ message = str(exc).casefold()
+ return "security restrictions" in message or "operation: read" in message
+
+
+def _run_database_job(rt, candidate):
+ """Run one persisted receipt. Repeating a completed publication reconciles the same rows."""
+ import core.user_tables as ut
+ import odoo_relational as rel
+ # A disconnected source is paused work, not a failed attempt. Preserve the receipt so
+ # reconnect can resume it; the publication fence still rechecks a mid-run disconnect.
+ if (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"):
+ connector_sync.stall(rt, candidate, ODOO_DISCONNECTED_CAUSE)
+ return
+ job = connector_sync.claim(rt, candidate)
+ if job is None:
+ return
+ fenced = _MaterializationRuntime(rt, job)
+ key = job["databaseKey"]
+ cid = job["connectionId"]
+ record_odoo_stage(rt, cid, "source", "running", database_key=key)
+ record_odoo_stage(rt, cid, "materialization", "running", database_key=key)
+ def finish(success=False, cause=""):
+ # The pre-claim check cannot cover a disconnect during source I/O or publication.
+ # Only the current lease may requeue; disable and a newer Save still win.
+ interrupted = bool((rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"))
+ if success and not interrupted:
+ # A newer field Save can persist after rows commit but before this acknowledgement.
+ # The old rows remain valid history; they cannot establish the new settings ready.
+ fenced.valid()
+ # Only provider-authored sentences reach this seam, never raw exception messages.
+ cause = str(cause or "").replace("\u2014", ",").replace("\u2013", "-")
+ result = connector_sync.finish(
+ rt, job, success=success, interrupted=interrupted, cause=cause)
+ if interrupted:
+ connector_sync.stall(rt, job, ODOO_DISCONNECTED_CAUSE)
+ return result
+
+ stopped = threading.Event()
+ def heartbeat():
+ while not stopped.wait(connector_sync.LEASE_SECONDS / 3):
+ try:
+ connector_sync.finish(rt, job, renew=True)
+ except Exception:
+ return # An unrenewed lease cannot publish.
+ heartbeat_thread = None
+ try:
+ def install(current):
+ fenced.valid()
+ if key in (current or {}):
+ current[key]["_connectorJob"] = [job["id"], job["leaseToken"]]
+ return current
+ rt.update(ut.STORE_KEY, install, flush="sync")
+ heartbeat_thread = threading.Thread(target=heartbeat, daemon=True,
+ name="connector-lease")
+ heartbeat_thread.start()
+ cid = job["connectionId"]
+ source_id = None if cid == "default" else cid
+ if key not in (rt.get(ut.STORE_KEY, fresh=True) or {}):
+ schema = rel.publish_database_schema(fenced, key, connection_id=source_id,
+ username="automation")
+ if not job["model"]:
+ # Reconciliation cannot infer a missing definition's model. Now the provider
+ # has resolved it, queue its real field fingerprint before reading any rows.
+ fenced.valid()
+ _queue_odoo_database(rt, key, model=schema["identity"]["model"], changed=True)
+ return
+ if rel.base_table_key(key) in rel.ODOO_TABLE_MODELS:
+ result = rel.refresh(
+ fenced, getattr(rt, "tenant", None), username="automation",
+ connection_id=source_id, table_keys={key})
+ else:
+ # A source-only model must not depend on an unrelated DuckDB mirror being warm.
+ _cid, namespace = rel.connection_namespace(rt, source_id)
+ result = {"discovered": rel.sync_discovered_models(
+ fenced, source_connection_id=source_id, namespace=namespace,
+ username="automation", table_keys={key},
+ on_progress=lambda rows, total: connector_sync.progress(
+ fenced, job, rows=rows, total=total))}
+ discovered = result.get("discovered") or {}
+ published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
+ receipt = (result.get("databases") or {}).get(key)
+ receipt = receipt if isinstance(receipt, dict) else None
+ database_errors = result.get("databaseErrors") or {}
+ success = (key in published and not result.get("failed") and not result.get("frozen")
+ and not result.get("limits") and not discovered.get("errors")
+ and key not in database_errors
+ and (receipt is None or receipt.get("state") == "succeeded"))
+ counts = published.get(key) if isinstance(published.get(key), dict) else {}
+ nrows = (receipt.get("servingCount") if receipt is not None
+ else counts.get("rows") if isinstance(counts, dict) else None)
+ if success:
+ refreshed_at = datetime.now(timezone.utc).isoformat()
+ record_odoo_stage(rt, cid, "source", "ready", database_key=key,
+ receipt=receipt)
+ record_odoo_stage(rt, cid, "materialization", "ready",
+ database_key=key, row_count=nrows, receipt=receipt)
+ record_odoo_stage(rt, cid, "reconciliation", "ready",
+ database_key=key, row_count=nrows, receipt=receipt)
+ record_odoo_stage(rt, cid, "freshness", "ready", database_key=key,
+ row_count=nrows, refreshed_at=refreshed_at, receipt=receipt)
+ finish(success=True)
+ else:
+ raw_failure = ((database_errors.get(key) if isinstance(database_errors, dict) else None)
+ or (receipt or {}).get("error") or (receipt or {}).get("sourceError"))
+ failure = (_clean_odoo_failure(raw_failure, "reconciliation") if raw_failure
+ else _odoo_failure("provider result was incomplete", "materialization"))
+ failed_stage = str(failure.get("stage") or "materialization")
+ failed_stage = (failed_stage if failed_stage in _ODOO_HEALTH_STAGES
+ else "materialization")
+ record_odoo_stage(rt, cid, failed_stage, "failed",
+ database_key=key, failure=failure, receipt=receipt)
+ record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
+ failure={**failure, "stage": "freshness"}, receipt=receipt)
+ finish(cause=f"{failure['code']}: {failure['message']}")
+ except _SupersededMaterialization:
+ # Disable/new Save already invalidated the table marker; never restore it here.
+ finish()
+ except rel.Refused as exc:
+ failure = _odoo_failure(exc, "materialization")
+ record_odoo_stage(rt, cid, failure["stage"], "failed",
+ database_key=key, failure=failure)
+ record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
+ failure={**failure, "stage": "freshness"})
+ finish(cause=f"{failure['code']}: {failure['message']}")
+ except Exception as exc:
+ # Raw provider exceptions may carry credentials or source values. Store only a named,
+ # data-free classification, including the fail-closed TLS case.
+ failure = _odoo_failure(exc, "source")
+ if _odoo_read_access_refused(exc):
+ failure = {"stage": "source", "code": "source_read_refused",
+ "type": type(exc).__name__,
+ "message": (f"Odoo refused read access to {job['model']}. Give the "
+ "connected account read access, or switch this database off.")}
+ record_odoo_stage(rt, cid, failure["stage"], "failed",
+ database_key=key, failure=failure)
+ record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
+ failure={**failure, "stage": "freshness"})
+ finish(cause=f"{failure['code']}: {failure['message']}")
+ finally:
+ stopped.set()
+ if heartbeat_thread:
+ heartbeat_thread.join(timeout=1)
+
+
+def _reconcile_database_jobs(rt):
+ """Repair enabled Odoo grids with no receipt using only tenant-local definitions."""
+ import core.user_tables as ut
+ import odoo_relational as rel
+ grids = (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("grids") or {}
+ receipts = {job["databaseKey"] for job in connector_sync.jobs(rt, "odoo")}
+ missing = [key for key, enabled in grids.items() if enabled is True and key not in receipts]
+ if not missing:
+ return
+ definitions = rt.get_projection(ut.STORE_KEY, drop=("rows",)) or {}
+ for key in missing:
+ try:
+ model = str(((definitions.get(key) or {}).get("connector") or {}).get("model") or "")
+ # The queue's identity validator also decides whether an unknown model is legal.
+ _queue_odoo_database(rt, key, model=model)
+ except Exception:
+ continue # One unresolvable definition cannot starve other queued work.
+
+
+def _resume_database_jobs(rt):
+ """One serial pass, also usable by isolated synthetic fixtures."""
+ try:
+ _reconcile_database_jobs(rt)
+ except Exception:
+ pass # A reconciliation failure must still fall through to the receipt scan.
+ for candidate in connector_sync.jobs(rt):
+ if candidate.get("provider") == "odoo" and (
+ candidate.get("state") == "queued" or
+ candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()):
+ try:
+ _run_database_job(rt, candidate)
+ except Exception:
+ # A failed claim/status write leaves durable work for a later scan. It must
+ # not starve sibling databases; never log raw store/provider exceptions.
+ continue
+
+
+_materialization_wake = threading.Event()
+_materialization_stop = threading.Event()
+_materialization_thread = None
+_materialization_lifecycle = threading.Lock()
+# 3 is a starting point, not tuned: each worker holds an Odoo read and a growing store document in a small Space container.
+MATERIALIZATION_WORKERS = 3
+_materialization_pool = None
+_materialization_inflight = set()
+_materialization_inflight_lock = threading.Lock()
+
+
+def _release_database_job(key):
+ # Completion and cancellation both release the reservation, including failed claims.
+ with _materialization_inflight_lock:
+ _materialization_inflight.discard(key)
+
+
+def _dispatch_database_jobs(rt):
+ """Submit a resident scan without waiting for source reads or row publication."""
+ global _materialization_pool
+ try:
+ _reconcile_database_jobs(rt)
+ except Exception:
+ pass # Match the serial fixture pass even when reconciliation is unavailable.
+ for candidate in connector_sync.jobs(rt):
+ if candidate.get("provider") == "odoo" and (
+ candidate.get("state") == "queued" or
+ candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()):
+ key = (rt.key, candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
+ with _materialization_inflight_lock:
+ if key in _materialization_inflight:
+ continue
+ _materialization_inflight.add(key)
+ try:
+ with _materialization_lifecycle:
+ if _materialization_stop.is_set():
+ _release_database_job(key)
+ return
+ if _materialization_pool is None:
+ _materialization_pool = ThreadPoolExecutor(
+ max_workers=MATERIALIZATION_WORKERS,
+ thread_name_prefix="connector-materialization-job")
+ pool = _materialization_pool
+ future = pool.submit(_run_database_job, rt, candidate)
+ # Register outside both locks: an already finished future calls back inline.
+ future.add_done_callback(lambda _future, key=key: _release_database_job(key))
+ except Exception:
+ _release_database_job(key)
+ # A concurrent shutdown or failed submit leaves the receipt for the next pass.
+ continue
+
+
+@router.on_event("startup")
+def _start_database_jobs():
+ """Discover durable work at boot, including tenants that have no active browser session."""
+ global _materialization_thread
+ def run():
+ global _materialization_thread
+ from harness import runtime
+ while True:
+ with _materialization_lifecycle:
+ if _materialization_stop.is_set():
+ # Retire under the startup lock, before this thread exits. A new lifespan
+ # either revives this worker or starts its replacement, never a dying one.
+ _materialization_thread = None
+ return
+ _materialization_wake.clear()
+ try:
+ tenants = runtime.known_tenants()
+ for tenant in tenants:
+ if _materialization_stop.is_set():
+ break
+ try:
+ _dispatch_database_jobs(runtime.get_runtime(tenant))
+ except Exception:
+ continue # Durable queued/leased work is retried by the next scan.
+ except Exception:
+ pass
+ _materialization_wake.wait(15)
+ with _materialization_lifecycle:
+ _materialization_stop.clear()
+ _materialization_wake.set()
+ if _materialization_thread and _materialization_thread.is_alive():
+ return
+ _materialization_thread = threading.Thread(target=run, daemon=True,
+ name="connector-materialization")
+ _materialization_thread.start()
+
+
+@router.on_event("shutdown")
+def _stop_database_jobs():
+ global _materialization_pool
+ with _materialization_lifecycle:
+ _materialization_stop.set()
+ _materialization_wake.set()
+ pool, _materialization_pool = _materialization_pool, None
+ if pool is not None:
+ # Running reads retain their reservation until done; lease expiry permits recovery.
+ pool.shutdown(wait=False, cancel_futures=True)
+
+
+def _sync_refusals():
+ """The provider refusal types whose OWN sentence is the answer, as a tuple for `except`.
+
+ ⚠ Resolved lazily and defensively: `odoo_relational` is imported inside functions throughout
+ this module to avoid an import cycle, and a refusal handler that could itself raise on import
+ would turn every failed sync into a different failure.
+ """
+ try:
+ import odoo_relational # noqa: PLC0415
+ return (odoo_relational.Refused,)
+ except Exception: # noqa: BLE001
+ return ()
+
+
+def _schema_catalog(session):
+ """Rows-free Odoo field catalogue; source fields remain discoverable when every value is empty."""
+ return _schema_catalog_rt(session.runtime)
+
+
+def _odoo_raw_models(rt, snapshot=None, connection_id=None):
+ """Raw fields from stamped discovery, or the legacy snapshot before discovery exists."""
+ if snapshot is None:
+ import core.user_tables as user_tables
+ snapshot = user_tables.odoo_schema_snapshot(rt)
+ legacy = dict((snapshot or {}).get("models") or {}) if connection_id is None else {}
+ try:
+ import odoo_relational
+ discovered = odoo_relational.readable_model_catalog(rt, connection_id)
+ except Exception: # noqa: BLE001
+ discovered = {}
+ if not discovered.get("capturedAt"):
+ return legacy
+ # A stamp means this tenant-bound account has spoken. Even a partial stamped set is safer
+ # than adding snapshot-only models that the current account never proved it may read.
+ return {str(item.get("model") or ""): dict(item.get("fields") or {})
+ for item in (discovered.get("models") or ())
+ if str(item.get("model") or "")}
+
+
+def _schema_catalog_rt(rt, database_key=None, connection_id=None):
+ """The same catalogue, over a RUNTIME rather than a session.
+
+ ⚠ SPLIT IN W49-T12 SO THE DETAIL ROUTE CAN REUSE IT RATHER THAN RE-DERIVE IT. The connector
+ modal's Databases half needs exactly this catalogue, and writing a second reader of
+ `odoo_schema_snapshot` here would be the two-answers-to-one-question shape this module has
+ already paid for twice ([[one-question-two-normalizers]]). `_schema_catalog` keeps its
+ signature so every existing caller is untouched.
+ """
+ import core.user_tables as user_tables
+ enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}
+ snapshot = user_tables.odoo_schema_snapshot(rt)
+ raw_models = _odoo_raw_models(rt, snapshot, connection_id)
+ if raw_models:
+ rows = []
+ for model, raw_fields in raw_models.items():
+ for raw_name, meta in (raw_fields or {}).items():
+ legacy_key = f"{model}.{raw_name}"
+ key = (_odoo_field_key(rt, database_key, model, raw_name)
+ if database_key else legacy_key)
+ rows.append({"key": key, "model": model, "rawName": raw_name,
+ "label": str(meta.get("string") or raw_name),
+ "type": str(meta.get("type") or "text"), "source": "Odoo",
+ "emptyDiscoverable": True, "defaultEnabled": False,
+ # Legacy choices seed default-connection databases until each has
+ # its own explicit override. Scoped writes never change this seed.
+ "enabled": enabled.get(key, enabled.get(legacy_key)
+ if connection_id is None else False) is True})
+ return sorted(rows, key=lambda row: row["label"].casefold())
+ # Never present compiled mirror declarations as an authoritative fields_get catalogue.
+ # Until a sync captures metadata, the schema is unavailable and enablement stays refused.
+ return []
+
+
+def _odoo_field_key(rt, database_key, model, raw_name, connection_id=...):
+ # An explicitly resolved None must not trigger another lookup.
+ if connection_id is ...:
+ import odoo_relational
+ connection_id = odoo_relational.connection_for_table_key(database_key, rt)
+ return connector_sync.field_key(
+ "odoo", connection_id, database_key, model, raw_name)
+
+
+@router.get("/connectors/odoo/status")
+def odoo_status(connection_id: str = None, session: Session = Depends(require_session)):
+ _odoo_admin(session)
+ import odoo_relational
+ resolved_connection = _odoo_connection_slot(session.runtime, connection_id)
+ cfg = odoo_relational.read_config(session.runtime)
+ stored = session.runtime.get(_ODOO_SYNC_STATUS) or {}
+ errors = stored.get("errors") if isinstance(stored.get("errors"), list) else []
+ phase = str(stored.get("phase") or "unknown")
+ lifecycle = connector_sync.read(session.runtime, "odoo", resolved_connection)
+ durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id))
+ receipt = odoo_health_receipt(session.runtime, resolved_connection)
+ if durable:
+ return {"schedule": cfg.get("syncEvery"),
+ "health": "paused" if cfg.get("frozen") else durable["state"],
+ "lastRun": durable["at"], "frozen": cfg.get("frozen") is True,
+ **durable, "stateAt": durable["at"], "connectionId": resolved_connection,
+ "receipt": receipt}
+ # The old keys stay the old default-connection answer. A caller that names a connection gets
+ # that slot's lifecycle instead of the tenant-wide legacy document that caused the C6 defect.
+ named = connection_id is not None
+ shown_errors = (list(lifecycle.get("errors") or ()) if named and lifecycle else
+ [] if named else errors)
+ shown_phase = ((lifecycle or {}).get("state") or "unknown") if named else phase
+ shown_last_run = ((lifecycle or {}).get("at") if named else stored.get("lastRun"))
+ state = ((lifecycle or {}).get("state")
+ or {"syncing": "syncing", "ok": "ready", "error": "error"}.get(shown_phase)
+ or "queued")
+ errors_total = ((lifecycle or {}).get("errorsTotal", 0) if named or lifecycle
+ else len(errors))
+ return {"schedule": cfg.get("syncEvery"),
+ "health": "paused" if cfg.get("frozen") else
+ shown_phase if named else "error" if shown_errors else shown_phase,
+ "lastRun": shown_last_run, "errors": shown_errors[:20],
+ "frozen": cfg.get("frozen") is True,
+ "state": state,
+ "stateAt": (lifecycle or {}).get("at") or shown_last_run,
+ "databases": list((lifecycle or {}).get("databases") or ()),
+ "retryable": state in ("error", "partial"),
+ "errorsTotal": int(errors_total or 0),
+ "connectionId": (lifecycle or {}).get("connectionId")
+ or resolved_connection,
+ "receipt": receipt}
+
+
+@router.get("/connectors/{connector_key}/sync")
+def connector_sync_status(connector_key: str, connection_id: str = None,
+ session: Session = Depends(require_session)):
+ registered = connector_sync.providers()
+ if connector_key not in registered:
+ raise err(404, "sync_provider_unknown",
+ f"{connector_key} is not registered for connector sync")
+ row = (connector_sync.job_summary(_connection_jobs(session.runtime, connector_key, connection_id))
+ or connector_sync.read(session.runtime, connector_key, connection_id))
+ return _sync_payload(connector_key, connection_id, row)
+
+
+@router.post("/connectors/{connector_key}/sync/now")
+def connector_sync_now(connector_key: str, connection_id: str = None,
+ session: Session = Depends(require_session)):
+ _sync_admin(session)
+ if connector_key not in connector_sync.providers():
+ raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
+ try:
+ requested_connection = (_odoo_connection_slot(session.runtime, connection_id)
+ if connector_key == "odoo" else
+ str(connection_id or "default").strip() or "default")
+ key = (connector_key, session.runtime.key)
+ with _SYNC_REQUEST_CONNECTION_LOCK:
+ # Bind under one API-side lock so a duplicate request cannot race ahead of the
+ # accepted request and attach its different connection id to the queued intent.
+ receipt = connector_sync.request_sync(connector_key, session.runtime.key,
+ requested_by=session.uname)
+ if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS:
+ _SYNC_REQUEST_CONNECTIONS[key] = requested_connection
+ receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key]
+ return receipt
+ except ValueError:
+ raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None
+
+
+@router.get("/connectors/{connector_key}/sync/now")
+def connector_sync_now_status(connector_key: str, session: Session = Depends(require_session)):
+ _sync_admin(session)
+ if connector_key not in connector_sync.providers():
+ raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
+ receipt = connector_sync.sync_request(connector_key, session.runtime.key)
+ if receipt is not None:
+ receipt["connectionId"] = sync_request_connection(connector_key, session.runtime.key)
+ return receipt
+ return {
+ "provider": connector_key, "tenant": session.runtime.key, "state": "idle",
+ "requestedAt": None, "startedAt": None, "finishedAt": None,
+ "requestedBy": "", "detail": "", "accepted": False,
+ }
+
+
+@router.post("/connectors/{connector_key}/sync/retry")
+def connector_sync_retry(connector_key: str, connection_id: str = None,
+ session: Session = Depends(require_session), database_key: str = None):
+ _sync_admin(session)
+ if connection_id == "default":
+ connection_id = None
+ registered = connector_sync.providers()
+ if connector_key not in registered:
+ raise err(404, "sync_provider_unknown",
+ f"{connector_key} is not registered for connector sync")
+ if connector_key != "odoo":
+ raise err(501, "sync_runner_unavailable", registered[connector_key]["retryHint"])
+ import odoo_relational as rel
+ if database_key:
+ key, _ns, choice = _odoo_database_target(session.runtime, database_key, connection_id)
+ choices = [choice]
+ else:
+ choices = rel.grid_choices(session.runtime, connection_id)
+ receipts = []
+ for choice in choices:
+ if not choice.get("enabled"):
+ continue
+ key = choice["key"]
+ existing = connector_sync.jobs(session.runtime, connector_key,
+ rel.connection_for_table_key(key, session.runtime))
+ previous = next((j for j in existing if j["databaseKey"] == key), {})
+ if previous.get("state") == "failed" and not previous.get("retryable"):
+ if database_key:
+ raise err(409, "sync_attempts_exhausted", previous["error"])
+ continue
+ receipts.append(_queue_odoo_database(session.runtime, key, model=choice.get("model", ""),
+ retry=True))
+ if not receipts:
+ raise err(409, "sync_not_retryable", "No enabled database can be retried. Check its settings.")
+ return {"provider": connector_key, "jobs": receipts, **connector_sync.job_summary(receipts)}
+
+
+@router.get("/connectors/{connector_key}/jobs")
+def connector_database_jobs(connector_key: str, session: Session = Depends(require_session),
+ connection_id: str = None):
+ _sync_admin(session)
+ if connector_key not in connector_sync.providers():
+ raise err(404, "sync_provider_unknown", "This connector has no database sync runner.")
+ payload = {"provider": connector_key, "jobs": connector_sync.jobs(
+ session.runtime, connector_key, connection_id)}
+ if connector_key == "odoo":
+ payload["receipt"] = odoo_health_receipt(session.runtime, connection_id)
+ return _safe_connector_status(connector_key, payload)
+
+
+@router.get("/connectors/odoo/schema")
+def odoo_schema(session: Session = Depends(require_session)):
+ _odoo_admin(session)
+ fields = _schema_catalog(session)
+ import core.user_tables as user_tables
+ snapshot = user_tables.odoo_schema_snapshot(session.runtime)
+ try:
+ import odoo_relational
+ discovered = odoo_relational.readable_model_catalog(session.runtime)
+ except Exception: # noqa: BLE001
+ discovered = {}
+ has_discovery = bool(discovered.get("capturedAt"))
+ complete = (discovered.get("complete") is True if has_discovery
+ else snapshot.get("complete") is True)
+ return {"fields": fields, "default": "off", "adminOnly": True,
+ "complete": complete,
+ "capturedAt": (discovered.get("capturedAt") if has_discovery
+ else snapshot.get("capturedAt")),
+ "errors": list((discovered.get("errors") if has_discovery
+ else snapshot.get("errors")) or ()),
+ "source": "Odoo"}
+
+
+@router.post("/connectors/odoo/census")
+def odoo_population_census(session: Session = Depends(require_session), exact: bool = False):
+ """Count up to 101 ids per model by default; exact=true requests the full census."""
+ _odoo_admin(session)
+ import odoo_relational
+ census = odoo_relational.refresh_population_census(session.runtime, force=True, exact=exact)
+ return {"capturedAt": str(census.get("capturedAt") or ""),
+ "mode": census["mode"],
+ "counts": census["counts"],
+ "complete": census.get("complete") is True,
+ "durationMs": int(census.get("durationMs") or 0),
+ "modelsCounted": int(census.get("modelsCounted") or 0),
+ "modelsWithRows": int(census.get("modelsWithRows") or 0),
+ "errors": [str(value).replace("\u2014", ",").replace("\u2013", "-")
+ for value in (census.get("errors") or ())][:50]}
+
+
+def _materialisation_failure_reason(result):
+ """The materialiser's own reason, made safe for a sentence shown in the connector modal."""
+ result = result if isinstance(result, dict) else {}
+ candidates = []
+ if result.get("note"):
+ candidates.append(result.get("note"))
+ if result.get("failed"):
+ candidates.append(result.get("failed"))
+ errors = result.get("errors")
+ candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,))
+ discovered = result.get("discovered")
+ if isinstance(discovered, dict):
+ errors = discovered.get("errors")
+ candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,))
+ reason = str(next((value for value in candidates if str(value or "").strip()), "")).strip()
+ # Provider errors can carry typographic punctuation. Screen strings use plain punctuation.
+ # ⛔ Standing rule 2 names the EM DASH and the EN DASH, never the ASCII hyphen. A blanket
+ # strip of every Unicode dash category also breaks "read-only" and "e-commerce", so the
+ # two characters the rule names are rewritten and nothing else is touched.
+ reason = reason.replace("\u2014", ",").replace("\u2013", "-")
+ return reason[:1].upper() + reason[1:] if reason else ""
+
+
+def _evict_odoo_product_pools(rt):
+ """Drop every raw-field-sensitive row pool after field enablement changes."""
+ for cache_key in list(getattr(rt, "pool_cache", {}) or {}):
+ if isinstance(cache_key, tuple) and cache_key[:1] == ("product_pool",):
+ rt.pool_cache.pop(cache_key, None)
+
+
+# W57-T31 / R10 / C9: switching off starts a seven-day lock to prevent repeated toggling.
+# The config document is shared with read_config, which ignores unknown keys: disabledAt
+# belongs beside grids, not in a second document that could drift from enablement.
+_RELOCK_DAYS = 7
+_RELOCK_SECONDS = _RELOCK_DAYS * 86400
+
+
+def _relock_stamps(rt):
+ """Read disable epochs without letting an absent or malformed map break a request."""
+ try:
+ import odoo_relational
+ stamps = (rt.get(odoo_relational.CONFIG_KEY) or {}).get("disabledAt", {})
+ if not isinstance(stamps, dict):
+ return {}
+ return {str(key): int(value) for key, value in stamps.items()
+ if isinstance(value, (int, float)) and not isinstance(value, bool)}
+ except Exception: # noqa: BLE001
+ return {}
+
+
+def _relock_days_left(stamp, now=None):
+ """Round any remaining part of a lock day up to a whole day."""
+ if isinstance(stamp, bool) or not isinstance(stamp, (int, float)):
+ return 0
+ try:
+ from math import ceil
+ remaining = stamp + _RELOCK_SECONDS - (time.time() if now is None else now)
+ return max(0, ceil(remaining / 86400))
+ except Exception: # noqa: BLE001
+ return 0
+
+
+def _odoo_database_target(rt, table_key, connection_id=None):
+ """Resolve a database once; explicit connection/key disagreement fails closed."""
+ import odoo_relational
+ raw = str(table_key or "").strip()
+ base, namespace = odoo_relational.split_table_key(raw)
+ requested_connection = connection_id if connection_id is not None else namespace
+ try:
+ _cid, ns = odoo_relational.connection_namespace(rt, requested_connection)
+ choices = odoo_relational.grid_choices(rt, requested_connection)
+ except odoo_relational.Refused as exc:
+ raise err(404, "unknown_odoo_connection", str(exc))
+ key = odoo_relational.table_key_for(base or raw, ns)
+ if namespace is not None and namespace != ns:
+ raise err(409, "database_connection_mismatch",
+ "the database belongs to a different connection")
+ choice = next((row for row in choices if row["key"] == key), None)
+ if choice is None:
+ raise err(404, "unknown_odoo_database",
+ f"Odoo does not declare a database with key {raw!r}")
+ return key, ns, choice
+
+
+def _enable_odoo_database(table_key, session, connection_id=None):
+ """Turn on one database and publish its schema; row materialization is separate work."""
+ if not session.admin:
+ raise err(403, "not_admin", "only an administrator can enable an Odoo database")
+
+ import odoo_relational # noqa: PLC0415 - provider owns both paths
+ key, namespace, choice = _odoo_database_target(session.runtime, table_key, connection_id)
+
+ # R10 / C9: only a transition back on is locked; an already-enabled database may retry
+ # materialisation. A successful enable clears its stamp so the next switch-off starts fresh.
+ # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail.
+ if choice.get("enabled") is not True:
+ days = _relock_days_left(_relock_stamps(session.runtime).get(key))
+ if days > 0:
+ label = str(choice.get("label") or key)
+ unit = "day" if days == 1 else "days"
+ raise err(409, "database_relocked",
+ f"{label} was switched off less than {_RELOCK_DAYS} days ago. "
+ f"It cannot be switched back on for another {days} {unit}. "
+ f"The {_RELOCK_DAYS} day lock is what stops a database being switched "
+ "on and off repeatedly.")
+
+ # Publishing the definition makes Data and rename usable before any rows have loaded.
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ cur.setdefault("grids", {}).update({key: True})
+ if isinstance(cur.get("disabledAt"), dict):
+ cur["disabledAt"].pop(key, None)
+ return cur
+
+ failure = None
+ try:
+ result = odoo_relational.publish_database_schema(
+ session.runtime, key, connection_id=namespace, username=session.uname)
+ except odoo_relational.Refused as exc:
+ # The provider owns this sentence. In particular it names missing credentials and the
+ # MAX_TABLES refusal without this route guessing which one happened.
+ failure = (409, "odoo_database_refused", str(exc))
+ except Exception: # noqa: BLE001
+ failure = (500, "database_schema_failed",
+ "The setting was not changed and the database was not created because the "
+ "field catalogue could not be published. Refresh the catalogue and try again.")
+
+ # A refused publication must leave both the flag and its relock stamp untouched.
+ if failure is None:
+ try:
+ session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync")
+ except Exception: # noqa: BLE001
+ raise err(503, "store_unavailable", "the database setting was not saved. Try again.")
+
+ facts = (_rename_rights(session.runtime, session.uname, bool(session.admin), [key]).get(key)
+ or {"exists": False, "canRename": False})
+ exists = facts.get("exists") is True
+ can_rename = facts.get("canRename") is True
+ # ⛔⛔ THE ERROR ENVELOPE STAYS THE ONE SHAPE. `deps.err` is documented as X2's single
+ # non-2xx body, `{"error": {"code", "message"}}`, and an earlier draft of this door attached
+ # `key`, `enabled` and the fresh facts BESIDE that key so the client could read them off a
+ # failure. Two shapes for one thing is what this module's own comments call the defect it has
+ # paid for twice, and a caller that learns to read a sibling off an error will keep doing it.
+ # The client re-reads `/detail` on both paths; publication precedes the flag write and row
+ # materialisation remains separate work.
+ if failure is not None:
+ raise err(*failure)
+ model = str(choice.get("model") or "").strip()
+ if not exists:
+ reason = _materialisation_failure_reason(result)
+ message = "The setting was enabled, but the database was not created."
+ if reason:
+ message = f"{message} {reason}"
+ else:
+ message = f"{message} The field catalogue could not be published."
+ try:
+ _queue_odoo_database(session.runtime, key, model=model)
+ finally:
+ # Preserve the original refusal envelope even if the queue store is unavailable.
+ raise err(409, "database_not_created", message)
+
+ # Every permission-visible field is available from schema alone. Existing per-database
+ # choices survive a retry; population counts are optional background observations.
+ try:
+ schema_keys = {_odoo_field_key(session.runtime, key, model, name): True
+ for name in result["sourceFields"]}
+ session.runtime.update(
+ _ODOO_SCHEMA_ENABLEMENTS,
+ lambda current: {
+ **schema_keys,
+ **(current if isinstance(current, dict) else {}),
+ },
+ flush="sync")
+ except Exception: # noqa: BLE001
+ try:
+ _queue_odoo_database(session.runtime, key, model=model)
+ finally:
+ raise err(503, "field_auto_enable_failed",
+ "The database was created, but its field choices could not be saved. "
+ "Try again.")
+ _evict_odoo_product_pools(session.runtime)
+
+ # W56-T36: there is deliberately no navigation projection eviction here. Data's database
+ # list comes from
+ # `GET /nav` -> `core.user_tables.all_defs()` -> `TenantRuntime.get_projection(user_tables)`.
+ # Schema publication synchronously updates that same `user_tables` document; the HF
+ # backend replaces its whole-document cache as part of that write, while Postgres performs a
+ # fresh SELECT for every projection. `pool_cache` contains row pools for built-in databases,
+ # not the `ut_*` definition list, and compiled registry rows never contain tenant databases.
+ # The table now exists, so clients must discard navigation-derived database lists. This is a
+ # wire instruction rather than a guessed client row: `/nav` remains the permission-filtered
+ # authority for which databases the current reader may see.
+ try:
+ job = _queue_odoo_database(session.runtime, key, model=model)
+ except Exception:
+ raise err(503, "sync_queue_unavailable",
+ "The database settings were saved, but row sync could not be queued. Save again.")
+ return {"key": key, "enabled": True, "exists": exists, "canRename": can_rename,
+ "identity": connector_sync.database_identity(
+ "odoo", odoo_relational.connection_for_table_key(key, session.runtime), key, model),
+ "workspaceChanged": True, "state": job["state"], "job": job,
+ "schemaState": result["state"],
+ "schemaFields": result["fieldCount"], "populatedFieldsEnabled": 0,
+ "fieldCensusComplete": False, "fieldCensusErrors": [],
+ "autoEnableAlsoAffects": []}
+
+
+@router.post("/connectors/odoo/databases/{table_key}/enable")
+def odoo_database_enable(table_key: str, session: Session = Depends(require_session),
+ connection_id: str = None):
+ """Enable and create one declared Odoo database in this tenant's governed store."""
+ return _enable_odoo_database(table_key, session, connection_id)
+
+
+def _disable_odoo_database(table_key, session, connection_id=None):
+ """Turn off one declared Odoo database and persist its re-enable lock."""
+ if not session.admin:
+ raise err(403, "not_admin", "only an administrator can disable an Odoo database")
+
+ import odoo_relational
+ key, namespace, choice = _odoo_database_target(session.runtime, table_key, connection_id)
+
+ # W57-T31 / R10 / C9: one config-only write starts the lock, with no sync or row deletion.
+ # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail.
+ # Repeating a disable must preserve the original deadline.
+ was_enabled = choice.get("enabled") is True
+ _invalidate_materialization(session.runtime, key)
+ connector_sync.cancel(session.runtime, "odoo",
+ odoo_relational.connection_for_table_key(key, session.runtime), key)
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ grids = cur.get("grids")
+ grids = grids if isinstance(grids, dict) else {}
+ grids[key] = False
+ cur["grids"] = grids
+ if was_enabled:
+ if not isinstance(cur.get("disabledAt"), dict):
+ cur["disabledAt"] = {}
+ cur["disabledAt"][key] = int(time.time())
+ return cur
+
+ try:
+ session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync")
+ except Exception: # noqa: BLE001
+ raise err(503, "store_unavailable", "the database setting was not saved. Try again.")
+
+ stamp = _relock_stamps(session.runtime).get(key)
+ return {"key": key, "enabled": False, "relockDays": _RELOCK_DAYS,
+ "identity": connector_sync.database_identity(
+ "odoo", odoo_relational.connection_for_table_key(key, session.runtime),
+ key, choice.get("model")),
+ "disabledAt": stamp,
+ "canEnableAt": stamp + _RELOCK_SECONDS if stamp is not None else None}
+
+
+@router.post("/connectors/odoo/databases/{table_key}/disable")
+def odoo_database_disable(table_key: str, session: Session = Depends(require_session),
+ connection_id: str = None):
+ """Disable one declared Odoo database in this tenant's governed store."""
+ return _disable_odoo_database(table_key, session, connection_id)
+
+
+@router.patch("/connectors/odoo/schema/{field_key}")
+def odoo_schema_enable(field_key: str, body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ _odoo_admin(session)
+ import core.user_tables as user_tables
+ import odoo_relational
+ request = body if isinstance(body, dict) else {}
+ key = str(field_key or "").strip()
+ scoped = connector_sync.parse_field_key(key)
+ database_key = request.get("databaseKey")
+ connection_id = request.get("connectionId")
+ model, raw_name = key.rsplit(".", 1) if "." in key else ("", "")
+ if scoped:
+ provider, connection_id, database_key, model, raw_name = scoped
+ if provider != "odoo":
+ raise err(404, "unknown_odoo_field", "that field belongs to another connector")
+ if connection_id == "default":
+ connection_id = None
+ if database_key is None:
+ candidates = [row for row in odoo_relational.grid_choices(session.runtime, connection_id)
+ if row.get("model") == model]
+ if len(candidates) > 1:
+ raise err(400, "database_required", "choose the database for this field")
+ if candidates:
+ database_key = candidates[0]["key"]
+ namespace = None
+ if database_key:
+ database_key, namespace, choice = _odoo_database_target(
+ session.runtime, database_key, connection_id)
+ if model != choice.get("model"):
+ raise err(404, "unknown_odoo_field", "that model does not belong to this database")
+ canonical = _odoo_field_key(session.runtime, database_key, model, raw_name)
+ if scoped and key != canonical:
+ raise err(409, "database_connection_mismatch",
+ "the field belongs to a different connection")
+ key = canonical
+ _snapshot = user_tables.odoo_schema_snapshot(session.runtime)
+ try:
+ import odoo_relational
+ _discovered_complete = odoo_relational.readable_model_catalog(
+ session.runtime, namespace).get("complete") is True
+ except Exception: # noqa: BLE001
+ _discovered_complete = False
+ if (namespace is not None or _snapshot.get("complete") is not True) and not _discovered_complete:
+ raise err(409, "odoo_schema_unavailable",
+ "run a successful schema sync before changing raw-field enablement")
+ known = {field["key"] for field in _schema_catalog_rt(
+ session.runtime, database_key, namespace)}
+ if key not in known:
+ raise err(404, "unknown_odoo_field", "that field is not in the Odoo schema catalogue")
+ enabled = request.get("enabled")
+ if not isinstance(enabled, bool):
+ raise err(400, "bad_enabled", "enabled must be true or false")
+ previous_enabled = (session.runtime.get(_ODOO_SCHEMA_ENABLEMENTS) or {}).get(key) is True
+ session.runtime.update(
+ _ODOO_SCHEMA_ENABLEMENTS,
+ lambda current: {**(current if isinstance(current, dict) else {}), key: enabled},
+ flush="sync")
+ _evict_odoo_product_pools(session.runtime)
+ job = None
+ if database_key and choice.get("enabled"):
+ try:
+ job = _queue_odoo_database(session.runtime, database_key, model=model,
+ changed=previous_enabled != enabled)
+ except Exception:
+ raise err(503, "sync_queue_unavailable",
+ "The field settings were saved, but row sync could not be queued. Save again.")
+ return {"key": key, "enabled": enabled, "source": "Odoo",
+ "permissionWall": "normal field permissions", "job": job}
+
+
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+# ⭐⭐ W49-T12 · RULING R26 · CONTRACT C4 — ONE CONNECTOR, IN FULL, FOR THE MODAL.
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+#
+# R26: *"The connector popup is a centered MODAL with tabs Overview, Keychain, Databases,
+# Relationships, opened from the card. The Keychain moves INTO it and Settings > Keychains becomes
+# a link to the modal (one door). Field enablement moves into the Databases tab."*
+#
+# ⛔ ALL FOUR HALVES ARE COMPOSED NOW even though only Overview renders this wave, and that is
+# C4's own instruction. The alternative — ship `overview` and add the rest as each tab lands —
+# would make the DTO's shape a moving target across four tickets and give T13/T14/T15 nothing to
+# build against. Every half is read from the source that already answers the question:
+#
+# overview · the directory row this module already composes, plus the Databases labels
+# keychain · `routes_keychain.visible_entries`, i.e. the exact rows the Keychains room
+# shows THIS reader, with a member's masked preview already blanked by R4
+# databases · `odoo_relational.grid_choices` (derived from `TABLES`, never a second list)
+# plus the Odoo field catalogue this module already serves at
+# `/connectors/odoo/schema`
+# relationships · `relationship_graphs` UNCHANGED — the same DTO the directory already binds
+#
+# ⛔ AND NOTHING IS FABRICATED. A connector with no introspectable schema gets `databases: []`; a
+# connector with no credential door gets an empty `entries` list and `canConnect: false`; a
+# connection with no materialised tables gets the graph's own `unavailable` envelope. The rule this
+# module was built on is that a plausible-looking placeholder is worse than an honest empty.
+
+
+def _entry_type_for(connector_key):
+ """The keychain entry type behind one card, or `""` when the card has no stored credential.
+
+ ⛔ DERIVED FROM `TOKEN_CONNECTORS`, NOT RE-TYPED. This module's header rule is that the row set
+ is composed from the registries that already exist; a second hand-written
+ `{"stripe": "stripe", ...}` map here would rot in exactly the way the header describes, one
+ connector at a time. Odoo is the one built-in whose credential lives in the keychain, so it is
+ named once, beside the row that names it.
+ """
+ key = str(connector_key or "")
+ if key == "odoo":
+ return "odoo"
+ # ⛔⛔ FOUND ONLY ON THE MERGED TREE, AT THE WAVE-53 COMBINE, AND IT IS A 500 ON A LIVE ROUTE.
+ # `directory` answers 200 with a NAMED gap when any one contributor registry raises (W32-T14 /
+ # R6: "it used to 500 and show nothing"), and every contributor is called inside that guard.
+ # The `allowedScopes` pass added this wave runs AFTER the composition, OUTSIDE the guard, and
+ # this loop is a second unguarded read of `TOKEN_CONNECTORS` - so a broken token registry took
+ # the whole page back to a 500. Neither lane could see it: one owns this route, the other owns
+ # the gate that drives the broken registry, and only the merged tree holds both.
+ # ⚠ `""` is not a swallow, it is this function's own documented answer for a row with no
+ # keychain type, and `allowed_scopes("")` offers BOTH scopes - the permissive default the
+ # composer already states, chosen because the toggle must never hide a connection the reader
+ # actually holds. A token connector's rows are already absent from the set in this state.
+ try:
+ for token in TOKEN_CONNECTORS:
+ if str(token.get("key") or "") == key:
+ return str(token.get("entry") or "")
+ except Exception: # noqa: BLE001 - a registry, not a bug
+ return ""
+ return ""
+
+
+#: ⭐⭐ W49-T13 — WHAT A CONNECTOR ASKS FOR WHEN YOU CONNECT IT, declared server-side.
+#:
+#: ⛔ THE CLIENT MUST NOT OWN THIS MAP. R26 moves the add-a-key form into the connector modal, and
+#: the obvious client implementation is `type === "odoo" ? [url, db, user, api_key] : [api_key]` —
+#: a second copy of a credential SHAPE that already lives in `harness/connectors/odoo.py` and in
+#: `routes_keychain._env_odoo_fields`. The wave-9 law is about vocabularies and this is one: a
+#: connector whose credential grows a field would take a client edit to become storable.
+#: ⚠ `secret` decides the input type, so a field marked secret is typed into a password box and
+#: never echoed back — the masked preview is the only form the stored value takes afterwards.
+#: Sentence case with the initialism capitalised, which is what the rest of this app does.
+_AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret",
+ "refresh_token": "Refresh token", "seller_id": "Seller ID",
+ "marketplace_id": "Marketplace ID"}
+_CONNECT_FIELDS = {
+ "odoo": [{"key": "url", "label": "Server URL (https://...)", "secret": False},
+ {"key": "db", "label": "Database", "secret": False},
+ {"key": "user", "label": "API user", "secret": False},
+ {"key": "api_key", "label": "API key", "secret": True}],
+ # ⭐ THE KEYS AND THEIR ORDER ARE DERIVED, NEVER TYPED. `amazon_client.CREDENTIAL_FIELDS` is
+ # the one roster, and lane G's `verify_amazon` asserts this sequence against it, so a field
+ # added there appears here rather than drifting. Labels are the one thing NOT derived:
+ # `"client_id".replace("_", " ").capitalize()` renders "Client id", and this app writes the
+ # initialism. The map carries a derived fallback so a new credential field still renders.
+ "amazon_sp": [
+ {"key": key,
+ "label": _AMAZON_FIELD_LABELS.get(key, key.replace("_", " ").capitalize()),
+ "secret": key in ("client_secret", "refresh_token"),
+ **({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})}
+ for key in amazon_client.CREDENTIAL_FIELDS
+ ],
+}
+#: Everything else is one pasted token. The field name is `api_key` because that is the first of
+#: `core.keychain._PREVIEW_FIELDS`, so the stored entry gets a preview rather than a blank cell —
+#: the wave-31 defect that comment records, avoided rather than repeated.
+_CONNECT_TOKEN_FIELD = [{"key": "api_key", "label": "API key", "secret": True}]
+
+
+def _safe_connector_status(connector_key, payload):
+ """Apply the provider's disclosure guard at the status boundary, outside fallback catches."""
+ if connector_key == "amazon_sp":
+ import connectors_amazon as cx
+ return cx.assert_safe_status(payload)
+ return payload
+
+
+def _sync_stamp(value):
+ """Return a parseable public timestamp, never a creation-time freshness substitute."""
+ text = str(value or "").strip()
+ if not text:
+ return None
+ try:
+ parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=timezone.utc)
+ return parsed.astimezone(timezone.utc).isoformat()
+ except (TypeError, ValueError, OverflowError):
+ return None
+
+
+def _database_sync_health(*, enabled=True, state="never_run", last_success=None,
+ source_timestamp=None, materialized_timestamp=None, row_count=None,
+ error=None, now=None):
+ """Normalize one database's durable facts into S63-T34's shared public vocabulary."""
+ clean_state = str(state or "never_run")
+ if clean_state not in DATABASE_SYNC_STATES:
+ clean_state = "failed"
+ error = {"code": "sync_state_unreadable",
+ "message": "This database's saved sync state could not be read."}
+ if not enabled:
+ clean_state = "disabled"
+ count = row_count if type(row_count) is int and row_count >= 0 else None
+ success = _sync_stamp(last_success)
+ source = _sync_stamp(source_timestamp)
+ materialized = _sync_stamp(materialized_timestamp)
+ safe_error = None
+ if error is not None:
+ raw = error if isinstance(error, dict) else {}
+ requested_code = str(raw.get("code") or "")
+ code = requested_code if requested_code in _DATABASE_SAFE_ERRORS else "connector_sync_failed"
+ safe_error = {
+ "code": code,
+ # Raw provider and exception text never crosses this boundary.
+ "message": _DATABASE_SAFE_ERRORS.get(code, "This database could not be synced."),
+ }
+ # A current failure may retain the prior success timestamp, but it can never look current.
+ lag = _lag_seconds(success, now) if success else None
+ return {"state": clean_state, "lastSuccess": success,
+ "sourceTimestamp": source, "materializedTimestamp": materialized,
+ "rowCount": count, "lagSeconds": lag, "error": safe_error}
+
+
+def _odoo_database_sync_health(raw, enabled=True, *, now=None):
+ row = raw if isinstance(raw, dict) else {}
+ stages = row.get("stages") if isinstance(row.get("stages"), dict) else {}
+ failure = row.get("failure") if isinstance(row.get("failure"), dict) else None
+ raw_state = str(row.get("state") or "unknown")
+ stage_states = {str((value or {}).get("state") or "unknown")
+ for value in stages.values() if isinstance(value, dict)}
+ last_success = row.get("lastSuccess")
+ if failure or "failed" in stage_states or raw_state == "failed":
+ state = "partial" if last_success else "failed"
+ elif raw_state == "running" or "running" in stage_states:
+ state = "running"
+ elif raw_state == "queued" or "queued" in stage_states:
+ state = "queued"
+ elif last_success or raw_state == "ready":
+ state = "succeeded"
+ else:
+ state = "never_run"
+ source_stage = stages.get("source") if isinstance(stages.get("source"), dict) else {}
+ material_stage = (stages.get("materialization")
+ if isinstance(stages.get("materialization"), dict) else {})
+ return _database_sync_health(
+ enabled=enabled, state=state, last_success=last_success,
+ source_timestamp=source_stage.get("lastSuccess"),
+ materialized_timestamp=material_stage.get("lastSuccess"),
+ row_count=row.get("rowCount"), error=failure, now=now)
+
+
+def _amazon_database_sync_health(raw, enabled=True, *, now=None):
+ if not enabled:
+ return _database_sync_health(enabled=False)
+ if not isinstance(raw, dict):
+ return _database_sync_health(
+ state="failed", error={"code": "sync_status_unavailable",
+ "message": "Sync status is unavailable for this database."})
+ success = raw.get("lastSuccess")
+ if raw.get("stuck") is True:
+ state = "partial" if success else "failed"
+ error = {"code": "sync_stalled",
+ "message": "This database sync is stalled."}
+ elif raw.get("inProgress") is True:
+ state, error = "running", None
+ elif success:
+ state, error = "succeeded", None
+ elif raw.get("deferred"):
+ state, error = "queued", None
+ else:
+ state, error = "never_run", None
+ return _database_sync_health(
+ state=state, last_success=success, materialized_timestamp=success,
+ row_count=raw.get("rowsLoaded"), error=error, now=now)
+
+
+def _stored_database_sync_health(raw, enabled=True, *, now=None):
+ row = raw if isinstance(raw, dict) else {}
+ return _database_sync_health(
+ enabled=enabled, state=row.get("state") or "never_run",
+ last_success=row.get("lastSuccess"), source_timestamp=row.get("sourceTimestamp"),
+ materialized_timestamp=row.get("materializedTimestamp"), row_count=row.get("rowCount"),
+ error=row.get("error"), now=now)
+
+
+def _detail_connect(entry_type, is_admin):
+ """C4's `keychain.connect` half: what the Connect form must ask for, and what it may store.
+
+ ⭐ AN ADDITION TO C4, DECLARED. The contract pins `entries`, `canConnect` and `canDisconnect`;
+ it does not say where the form gets its fields, and the only two answers are "the server says"
+ and "the client re-derives". This is the first.
+
+ `scope` carries the credential's own rule rather than a permission: a tenant-wide type has
+ exactly one legal scope, so the form states it instead of offering a picker whose save 400s
+ (`routes_keychain.add_key` refuses it, and R7's banned shape is a button that leads nowhere).
+ """
+ if not entry_type:
+ return None
+ fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD)
+ tenant_wide = entry_type in TENANT_WIDE_TYPES
+ return {"type": entry_type,
+ "fields": [dict(field) for field in fields],
+ # A member may always store a personal one; business-wide is an administrator's to
+ # give (R4), and a tenant-wide type is business-wide by construction.
+ "scopes": (["business"] if tenant_wide
+ else ["business", "personal"] if is_admin
+ else ["personal"]),
+ "tenantWide": tenant_wide}
+
+
+def _replacement_field_specs(entry_type):
+ """The write-only fields that may change on an existing credential.
+
+ This is deliberately a projection of the Connect descriptor rather than another provider
+ vocabulary. URL, database, account and every future non-secret field stay server-held during
+ replacement. Optional rotation metadata is accepted separately from this descriptor.
+ """
+ spec = _detail_connect(entry_type, True) or {}
+ return [dict(field) for field in (spec.get("fields") or [])
+ if field.get("secret") is True and str(field.get("key") or "").strip()]
+
+
+def _oauth_slot_entries(rt, uname, row):
+ """A per-user OAuth connection, rendered as C4's `keychain` entry shape.
+
+ ⛔⛔ THE KEYCHAIN TAB HAS A SECOND SOURCE, AND WITHOUT IT AN OAUTH CARD SAYS THE WRONG THING.
+ `_entry_type_for` covers `odoo` and `TOKEN_CONNECTORS` only, because an OAuth credential is
+ NOT a keychain entry: it lives in `core/keychain`'s separate `user_secrets` bucket, which
+ `list_entries` deliberately does not walk (*"a refresh token is identity, not shared
+ infrastructure"*). So an OAuth card asked for its entries got `[]` — i.e. *"nobody has
+ connected this"* about a provider this very reader may be signed in to. That is the same
+ confidently-wrong shape `accessSentence` was corrected for one tab over.
+ ⚠ AND IT IS NOT WIDENED INTO `_entry_type_for`. These are different stores with different
+ owners and different doors; teaching one function to answer for both would put a keychain
+ entry type on a credential that has none.
+
+ ⛔ `testable` IS FALSE HERE, AND THAT IS THE HONEST VALUE RATHER THAN THE CAUTIOUS ONE.
+ `POST /admin/keychain/{id}/test` resolves its id through `visible_entries` -> `list_entries`,
+ which walks the `entries` bucket ONLY, so a Test control on a user secret would 404 every
+ time. A button that cannot work is exactly what R6 removed from the Google card.
+
+ ⚠ NO SECRET IS READ. `oauth_connect.status` returns the account email and two flags; the
+ refresh token itself is never opened, and `masked` is empty because there is nothing about
+ this credential a preview could honestly show.
+ """
+ slug = str(row.get("key") or "")
+ try:
+ state = (oauth_connect.status(rt, uname) or {}).get(slug) or {}
+ except Exception: # noqa: BLE001
+ return []
+ if not state.get("connected"):
+ return []
+ email = str(state.get("email") or "")
+ label = str(row.get("label") or slug)
+ return [{"id": f"oauth:{slug}",
+ "label": f"{label} ({email})" if email else label,
+ "masked": "",
+ # A per-user slot is personal by construction, in the vocabulary `routes_keychain`
+ # already declares. Never a third word for the same idea.
+ "scope": "personal",
+ "testable": False,
+ "replaceable": False}]
+
+
+def _detail_keychain(rt, uname, is_admin, row):
+ """C4's `keychain` half: the credential rows THIS reader may see, and what they may do.
+
+ ⚠ `visible_entries` IS CALLED WITH THE READER'S OWN ADMIN FLAG, deliberately. R4 blanks the
+ masked preview on a business entry a member did not create, and passing `True` to get a fuller
+ answer would hand a member four characters of the workspace's Odoo key. The restricted default
+ is the correct one and it is passed through rather than routed around.
+
+ ⚠ A KEYCHAIN THAT CANNOT BE READ COSTS THE ENTRY LIST AND NOTHING ELSE, and it is not silent:
+ `directory()` records that failure in `incomplete`, and the page this modal opens from renders
+ that banner ABOVE the card grid. The reader has already been told before they click.
+ """
+ entry_type = _entry_type_for(row.get("key"))
+ manage = str(row.get("manage") or "")
+ # The door this reader was offered on the CARD, not a second opinion about it: `directory`
+ # already decided admin vs member vs nothing, and deciding again here is how two answers to
+ # one question get born ([[one-question-two-normalizers]]).
+ can_connect = manage in ("keychain", "keychain_personal")
+ can_replace = bool(is_admin and can_connect and _replacement_field_specs(entry_type))
+ entries = []
+ if entry_type:
+ try:
+ import routes_keychain as rk
+ for entry in rk.visible_entries(rt, uname, bool(is_admin)) or []:
+ if str(entry.get("type") or "") != entry_type:
+ continue
+ entries.append({
+ "id": str(entry.get("id") or ""),
+ "label": str(entry.get("label") or entry.get("id") or ""),
+ "masked": str(entry.get("preview") or ""),
+ "scope": str(entry.get("scope") or rk.DEFAULT_SCOPE),
+ # `keychain.test_entry` answers for every stored entry: a registered prober for
+ # the type when the connector layer filled that slot, and a read-only shape or
+ # xmlrpc check otherwise. The flag says whether a Test control would have
+ # something to call, and it always does.
+ "testable": True,
+ # W55-T18 / C10: credential replacement is an administrator action. The
+ # client reads this wall instead of inferring it from scope or from whether
+ # a masked preview happened to be visible. The replacement endpoint changes
+ # this row's encrypted payload in place; no secret is added to this DTO.
+ "replaceable": can_replace,
+ **({AMAZON_ROTATION_DEADLINE_FIELD:
+ _stored_amazon_rotation_deadline(rt, entry.get("id"))}
+ if entry_type == "amazon_sp" else {}),
+ })
+ except Exception: # noqa: BLE001
+ entries = []
+ # ⭐ THE SECOND SOURCE. An OAuth card has no keychain entry type at all, so `entries` above is
+ # empty for it by construction and this is the only reading that can fill the tab.
+ if not entry_type and manage == "oauth":
+ entries = _oauth_slot_entries(rt, uname, row)
+ # `startUrl` is the Connect door for this kind, and `disconnect` is a per-user route, so
+ # both answers come from the ROW rather than from a credential door this card has none of.
+ return {"entries": entries,
+ "canConnect": bool(row.get("startUrl")),
+ "canDisconnect": bool(entries),
+ "connect": None}
+ return {"entries": entries,
+ "canConnect": can_connect,
+ "canDisconnect": bool(can_connect and entries),
+ # What the Connect form asks for. `None` when this card has no credential door at all,
+ # which is what stops the client inventing one (Webhooks, the Scraper, a planned row).
+ "connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None}
+
+
+def _odoo_catalog_by_model(rt, connection_id=None):
+ """`{odoo model: (fields, excluded)}` from the schema catalogue this module already serves.
+
+ `excluded` is a DIFF, not a guess: every raw field the snapshot captured that the catalogue did
+ not offer. Today that diff is empty, which is the honest answer, and it fills itself the moment
+ the catalogue starts filtering instead of needing somebody to remember.
+ """
+ raw_models = _odoo_raw_models(rt, connection_id=connection_id)
+ offered = {}
+ for row in _schema_catalog_rt(rt, connection_id=connection_id):
+ offered.setdefault(str(row.get("model") or ""), []).append(row)
+ out = {}
+ for model, raw_fields in raw_models.items():
+ rows = offered.get(str(model), [])
+ offered_raw = {str(row.get("rawName") or "") for row in rows}
+ out[str(model)] = (
+ [{"key": str(row.get("key") or ""),
+ "label": str(row.get("label") or ""),
+ "type": str(row.get("type") or "text"),
+ "source_field": str(row.get("rawName") or ""),
+ "enabled": bool(row.get("enabled"))}
+ for row in rows],
+ [{"source_field": str(name),
+ "reason": "This source field was captured from Odoo but the field catalogue did not "
+ "offer it."}
+ for name in sorted(raw_fields or {}) if str(name) not in offered_raw],
+ )
+ return out
+
+
+def _spec_field(item, name):
+ """One attribute off a C2 table spec, whether lane D ships dataclasses or dicts."""
+ if isinstance(item, dict):
+ return item.get(name)
+ return getattr(item, name, None)
+
+
+def _connector_spec_items(connector_key):
+ """C2's raw table specs for one connector, as a sequence of items, or `()`.
+
+ ⭐⭐ W53-T30 — EXTRACTED FROM `_connector_spec_tables` (W53-T27) SO THERE IS STILL EXACTLY ONE
+ IMPORT PATH TO LANE D'S MODULE. Two consumers now read the spec: the Databases tab wants
+ `native` and `label`, the relationship graph wants `links`. A second `from core import
+ connector_spec` with its own lookup order is how the two would start disagreeing about which
+ attribute of D's module is the spec, and only one of them would be exercised by any given
+ tenant. The lookup order, the failure posture and the reasons for both are unchanged and are
+ documented on `_connector_spec_tables` below.
+ """
+ try:
+ from core import connector_spec # noqa: PLC0415 - lane D's module (C2)
+ except Exception: # noqa: BLE001
+ return ()
+ key = str(connector_key or "")
+ # The directory uses the keychain type; the table registry uses the provider name.
+ if key == "amazon_sp":
+ key = "amazon"
+ found = None
+ for name in ("tables_for", "for_connector", "tables", "spec", "spec_for"):
+ fn = getattr(connector_spec, name, None)
+ if not callable(fn):
+ continue
+ try:
+ got = fn(key)
+ except Exception: # noqa: BLE001
+ continue
+ if got:
+ found = got
+ break
+ if not found:
+ for name in ("SPECS", "TABLES", "CONNECTOR_TABLES"):
+ blob = getattr(connector_spec, name, None)
+ if isinstance(blob, dict):
+ found = blob.get(key) or list(blob.values())
+ elif isinstance(blob, (list, tuple)):
+ found = list(blob)
+ if found:
+ break
+ return list(found.values()) if isinstance(found, dict) else list(found or ())
+
+
+def _connector_spec_links(connector_key):
+ """C2's DECLARED relations for one connector: `{table key: [(from, target, to, label)]}`.
+
+ ⭐⭐ W53-T30 / C2 — THE GRAPH'S SECOND SOURCE OF EDGES, and the contract's own words for it:
+ each spec row carries `links`, a list of `(from_col, target_key, to_col, cardinality)`. Today
+ every edge on this graph is read off a stored FIELD bag, which can only ever describe a
+ relation somebody wrote a column for; a spec-declared relation between two tables that carry
+ no link column has nowhere else to come from.
+
+ ⛔ AN EDGE FROM HERE IS HELD TO THE SAME TWO PROOFS AS ONE FROM A FIELD BAG, in the caller:
+ both endpoints must be permitted NODES on this graph and both named columns must be VISIBLE
+ to this reader. A spec is a declaration about the shipped schema, never a permission decision
+ about a person, and an edge drawn on the strength of the declaration alone would name a column
+ an administrator hid from the account looking at it.
+
+ ⛔ AND IT IS NOT AN OUTAGE WHEN LANE D'S MODULE IS ABSENT: `{}` means "no spec-declared
+ relations", the field bags still answer, and the graph is exactly what it is today.
+
+ ⚠ `cardinality` IS NOT A LABEL and is not read as one. The wire's `label` is what the reader
+ sees on the line, and "one to many" is a fact about the join rather than a name for it, so the
+ caller labels a spec edge with the SOURCE COLUMN's own label instead.
+ """
+ out = {}
+ for item in _connector_spec_items(connector_key):
+ table_key = str(_spec_field(item, "key") or "")
+ links = _spec_field(item, "links")
+ if not table_key or not isinstance(links, (list, tuple)):
+ continue
+ edges = []
+ for link in links:
+ if not isinstance(link, (list, tuple)) or len(link) < 3:
+ continue
+ from_col = str(link[0] or "").strip()
+ target_key = str(link[1] or "").strip()
+ to_col = str(link[2] or "").strip()
+ if from_col and target_key and to_col:
+ edges.append((from_col, target_key, to_col))
+ if edges:
+ out[table_key] = edges
+ return out
+
+
+def _connector_spec_tables(connector_key):
+ """C2's spec for one connector as `{table key: {"native", "label"}}`, or `{}`.
+
+ ⛔⛔ `core.connector_spec` IS LANE D'S MODULE AND IT IS NOT IN THIS TREE YET. C2 splits the
+ work: lane D declares the spec (`key`, `native`, `source`, `label`, `pinned`, `identity`,
+ `links`) and this lane CONSUMES it through `GET /connectors/{key}/detail`. So the import is
+ defensive and the fallback is the in-tree source each value already has — Odoo's native model
+ from `odoo_relational.ODOO_TABLE_MODELS`, its default label from `grid_choices`, Amazon's
+ report type from `connectors_amazon.REPORT_TYPES` and its label from `LABELS_AMAZON`. When D's
+ module lands, the spec WINS with no edit here.
+
+ ⚠ D'S EXACT API IS NOT FROZEN, so the lookups tried are written down rather than guessed at by
+ the next reader: the callables `tables_for(key)`, `for_connector(key)`, `tables(key)`,
+ `spec(key)`, `spec_for(key)`, then the module attributes `SPECS`, `TABLES`,
+ `CONNECTOR_TABLES` (a dict keyed by connector, a dict keyed by table, or a flat sequence).
+ Whatever answers first is normalised by TABLE KEY, and only the keys this connector already
+ declares are ever consulted — so a spec carrying other connectors' tables is harmless.
+ ⚠ W53-T30 — THAT LOOKUP NOW LIVES IN `_connector_spec_items` ABOVE, unchanged, because the
+ relationship graph reads the same spec for its `links`. One import path, two readings.
+
+ ⛔ A SPEC THAT CANNOT BE READ IS NOT AN OUTAGE. Every failure returns `{}` and the row keeps
+ the in-tree value it has today: C2 is a better source for these two strings, never the only
+ one, and a Databases tab that 500s because a sibling lane's module changed shape would be a
+ disclosure screen taken down by a naming question.
+ """
+ out = {}
+ for item in _connector_spec_items(connector_key):
+ table_key = str(_spec_field(item, "key") or "")
+ if not table_key:
+ continue
+ out[table_key] = {"native": str(_spec_field(item, "native") or ""),
+ "label": str(_spec_field(item, "label") or "")}
+ return out
+
+
+def _database_name_overrides(rt):
+ """This tenant's per-database name overrides: `{table key: {"name": ...}}`, or `{}`.
+
+ ⛔⛔ `nav_meta` IS THE STORE, AND IT IS READ THROUGH THE NAVIGATION RAIL'S OWN READER. R9,
+ verbatim: *"the rename database under the Navigation rail also works, it should then sync to
+ the navigation in the connector too."* That rename has written `nav_meta` since wave 19
+ (`routes_nav.save_nav_meta`), and `routes_tables.ut_label` already resolves a served name as
+ nav_meta's name, then the definition's label, then the key. Calling `routes_nav._read_nav_meta`
+ rather than re-deriving the document shape here is what makes the rail and this tab provably
+ ONE fact instead of two answers to one question.
+
+ ⛔ AND IT IS WHY NOTHING HERE WRITES A TABLE DEFINITION'S LABEL. `user_tables.set_label` is
+ rewritten by a connector re-sync (`odoo_relational.apply_plan` rebuilds definitions), so an
+ override stored there would be silently clobbered on the next sync, which is precisely the
+ R9 violation this ticket exists to prevent.
+
+ ⚠ ONE READ FOR THE WHOLE LIST, never one per row: `store.get` deep copies the tenant document
+ on every call, and this list is eight rows on Odoo and seven on Amazon.
+ ⚠ A store blip is "no overrides", never a failed page. The default name is still true.
+ """
+ try:
+ import routes_nav # noqa: PLC0415 - lazy, avoids a cycle
+ meta = routes_nav._read_nav_meta(rt) or {} # noqa: SLF001 - the ONE reader (R9)
+ except Exception: # noqa: BLE001
+ return {}
+ return {str(k): v for k, v in meta.items() if isinstance(v, dict)}
+
+
+def _rename_rights(rt, uname, is_admin, keys):
+ """`{table key: {exists, canRename}}` for R9's write wall and its prerequisite.
+
+ ⛔ THE SAME PREDICATE `routes_nav.save_nav_meta` APPLIES TO A `ut_` KEY, not a second one:
+ `user_tables.may_open`, i.e. the table's creator, a tenant admin, or a share grant. Two ideas
+ of who may rename a database would show one person a control that the other door refuses.
+
+ ⛔ A DATABASE THIS CONNECTION HAS NOT CREATED YET IS NOT RENAMEABLE, and that is not a
+ technicality: `nav_meta` is keyed by TABLE key, so a name stored for a table nobody can open
+ would sit in the tenant's document naming nothing, and the rail would never show it.
+
+ ⭐ W56-T31: `enabled` USED TO MEAN "will be included in the next sync", while the screen
+ read as though the database already existed. `canRename` was telling the truth about a table
+ that did not exist. Returning `exists` from this same projection lets the client distinguish
+ that state from a reader who may not open an existing table.
+
+ ⚠ IT IS SERVED PER ROW because the answers ARE per row. One `lend_defs` read answers the
+ whole list; that projection carries definitions without rows (~0.1% of the document's bytes).
+ """
+ try:
+ import core.user_tables as user_tables # noqa: PLC0415 - lazy, as elsewhere
+ definitions = user_tables.lend_defs(rt)
+ facts = {}
+ for key in keys:
+ exists = bool(user_tables.get(key, st=definitions))
+ facts[key] = {
+ "exists": exists,
+ "canRename": bool(str(key).startswith("ut_") and exists
+ and user_tables.may_open(
+ key, uname, bool(is_admin), st=definitions)),
+ }
+ return facts
+ except Exception: # noqa: BLE001
+ # Fail closed. A wall that could not be read is not a wall that admitted you.
+ return {key: {"exists": False, "canRename": False} for key in keys}
+
+
+def _named_databases(rt, is_admin, uname, rows):
+ """The LAST step of every databases list, Odoo's and Amazon's alike: the three names.
+
+ One place, deliberately. Each half of `_detail_databases` builds its own `native` and its own
+ shipped default from its own source, and then BOTH come through here — so "what is this
+ database called for this tenant" is answered once, by `schema_overlay.merge_database_labels`,
+ and a third connector cannot arrive with a fourth opinion about a blank override.
+ """
+ named = schema_overlay.merge_database_labels(rows, _database_name_overrides(rt))
+ rights = _rename_rights(rt, uname, is_admin, [str(row.get("key") or "") for row in named])
+ for row in named:
+ facts = rights.get(str(row.get("key") or "")) or {}
+ row["exists"] = facts.get("exists") is True
+ row["canRename"] = facts.get("canRename") is True
+ return named
+
+
+def _clean_scraper_recipe(raw):
+ """Return T11's persisted recipe shape and explicit structural validation errors.
+
+ This function is intentionally pure. Preview and creation both call it, so opening a saved
+ recipe cannot reinterpret it differently, and validation cannot accidentally become a paid
+ provider probe.
+ """
+ value = raw if isinstance(raw, dict) else {}
+ kind = str(value.get("kind") or "").strip().lower()
+ errors = []
+ if kind not in SCRAPER_RECIPE_KINDS:
+ errors.append("Choose Instagram, TikTok, or Google Maps.")
+ return {}, errors
+
+ limits = value.get("limits") if isinstance(value.get("limits"), dict) else {}
+ schedule = value.get("schedule") if isinstance(value.get("schedule"), dict) else {}
+ schedule_mode = str(schedule.get("mode") or "manual").strip().lower()
+ if schedule_mode != "manual":
+ errors.append("This recipe must use the manual Run now schedule.")
+
+ if kind == "google_maps":
+ seed = value.get("seed") if isinstance(value.get("seed"), dict) else {}
+ search = str(seed.get("search") or value.get("search") or "").strip()
+ geography = str(seed.get("geography") or value.get("geography") or "").strip()
+ raw_cap = limits.get("records", value.get("limit"))
+ if not search:
+ errors.append("Enter a category or search term.")
+ if not geography:
+ errors.append("Enter a geography.")
+ if raw_cap in (None, ""):
+ errors.append("Enter a result cap.")
+ cap = 0
+ else:
+ try:
+ cap = int(raw_cap)
+ except (TypeError, ValueError):
+ cap = 0
+ errors.append("The result cap must be a whole number.")
+ if raw_cap not in (None, "") and (cap < 1 or cap > 500):
+ errors.append("The result cap must be between 1 and 500.")
+ return ({"kind": kind, "source": "places",
+ "seed": {"search": search, "geography": geography},
+ "limits": {"records": cap}, "schedule": {"mode": "manual"}}, errors)
+
+ seed = str(value.get("seed") or "").strip()
+ raw_limit = limits.get("posts", value.get("limit"))
+ post_type = str(limits.get("postType") or value.get("postType") or "video").strip().lower()
+ if not seed:
+ errors.append(f"Enter an {kind.title()} profile seed.")
+ if raw_limit in (None, ""):
+ errors.append("Enter a last-N post limit.")
+ limit = 0
+ else:
+ try:
+ limit = int(raw_limit)
+ except (TypeError, ValueError):
+ limit = 0
+ errors.append("The post limit must be a whole number.")
+ if raw_limit not in (None, "") and (limit < 1 or limit > SCRAPER_RECIPE_MAX_ROWS):
+ errors.append(f"The post limit must be between 1 and {SCRAPER_RECIPE_MAX_ROWS}.")
+ allowed_types = ("video", "image", "carousel") if kind == "instagram" else ("video",)
+ if post_type not in allowed_types:
+ errors.append("Choose video, image, or carousel posts."
+ if kind == "instagram" else
+ "TikTok's native profile window supports video posts only.")
+ return ({"kind": kind, "source": "profile", "seed": seed,
+ "limits": {"posts": limit, "postType": post_type},
+ "schedule": {"mode": "manual"}}, errors)
+
+
+def _scraper_recipe_fields(kind):
+ """Stable user-table schema for one recipe, derived from the shipped normalisers."""
+ if kind == "google_maps":
+ return [dict(field) for field in _MAPS_RECIPE_FIELDS]
+ import automation_engine as engine # noqa: PLC0415 - existing schema owner
+ source = engine.POST_FIELDS if kind == "instagram" else engine.TT_POST_FIELDS
+ # Links and rollups target the shared automation tables. A recipe database is independent,
+ # so copying those references would create a plausible but broken relationship graph.
+ return [dict(field) for field in source
+ if isinstance(field, dict) and field.get("type") not in ("link", "rollup")]
+
+
+def _scraper_plan(recipe):
+ """Network-free setup preview: declared provider, modelled spend, cap, and credential state."""
+ kind = str((recipe or {}).get("kind") or "")
+ limit_key = "records" if kind == "google_maps" else "posts"
+ limit = int(((recipe or {}).get("limits") or {}).get(limit_key) or 0)
+ if kind == "google_maps":
+ import providers # noqa: PLC0415 - registry owns cost/capability
+ apify = providers.PROVIDERS["apify"]
+ capability = apify.cap("google_maps_stores")
+ per_record = float(capability.cost_per_record or 0.0) if capability else 0.0
+ amount = round(per_record * limit, 4)
+ return {"provider": apify.label, "credentialReady": apify.configured(),
+ "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit,
+ "canRun": True,
+ "note": "Modelled from the declared per-place rate; the provider receipt "
+ "reconciles the bounded result count."}
+
+ import providers # noqa: PLC0415 - registry is the authority
+ bright = providers.PROVIDERS["brightdata"]
+ apify = providers.PROVIDERS["apify"]
+ if kind == "tiktok":
+ per_record = float(bright.cap("tt_post_metrics").cost_per_record or 0.0)
+ profile_cost = float(bright.cap("tt_profile").cost_per_record or 0.0)
+ estimate_usd = profile_cost + (per_record * limit)
+ provider_label = bright.label
+ credential_ready = bright.configured()
+ else:
+ # The established Instagram runner can use Bright Data and its Apify fallback. The cap
+ # includes both post metrics and view enrichment, so it never hides the worst declared
+ # cost from the person authorising Run now.
+ bd_cost = float(bright.cap("ig_post_metrics").cost_per_record or 0.0)
+ profile_cost = float(bright.cap("ig_profile").cost_per_record or 0.0)
+ view_cost = float(apify.cap("ig_post_views").cost_per_record or 0.0)
+ estimate_usd = profile_cost + ((bd_cost + view_cost) * limit)
+ provider_label = (bright.label if bright.configured() or not apify.configured()
+ else apify.label)
+ credential_ready = bright.configured() or apify.configured()
+ amount = round(estimate_usd, 4)
+ return {"provider": provider_label, "credentialReady": bool(credential_ready),
+ "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit,
+ "canRun": True, "note": "Modelled from declared per-record list rates."}
+
+
+def _scraper_databases(rt, is_admin, uname):
+ """Saved recipe databases this reader may open, with no row payloads or secrets."""
+ try:
+ import core.user_tables as user_tables
+ definitions = user_tables.all_defs(rt)
+ lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
+ except Exception: # noqa: BLE001
+ return []
+ rows = []
+ for key, definition in (definitions or {}).items():
+ recipe = definition.get("scraperRecipe") if isinstance(definition, dict) else None
+ if not isinstance(recipe, dict):
+ continue
+ try:
+ if not user_tables.may_open(key, uname, bool(is_admin), st=lent):
+ continue
+ except Exception: # noqa: BLE001
+ continue
+ cleaned, errors = _clean_scraper_recipe(recipe)
+ if errors:
+ continue
+ plan = _scraper_plan(cleaned)
+ # Recipe existence alone is not enablement. The creation door stamps both ownership
+ # markers; a legacy/malformed row without them is disclosed as disabled and cannot run.
+ enabled = (definition.get("connectorKey") == "scraper"
+ and definition.get("recordMode") == "automation")
+ rows.append({"key": str(key), "label": str(definition.get("label") or key),
+ "defaultLabel": str(definition.get("label") or key),
+ "native": cleaned["kind"], "enabled": enabled, "exists": True,
+ "canRename": True, "sourceRows": None, "populated": None,
+ "fields": [dict(field) for field in (definition.get("fields") or ())
+ if isinstance(field, dict)],
+ "excluded": [], "sharesFieldsWith": [], "scraperRecipe": cleaned,
+ "syncHealth": _stored_database_sync_health(
+ definition.get(user_tables.CONNECTOR_HEALTH_FIELD), enabled),
+ "scraperRuns": [dict(item) for item in (definition.get("scraperRuns") or ())
+ if isinstance(item, dict)],
+ "runPath": f"/connectors/scraper/databases/{key}/run",
+ "canRun": enabled and plan["canRun"] is True})
+ return _named_databases(rt, is_admin, uname, rows)
+
+
+def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None):
+ """C4's `databases` half: what this connector materialises, and the fields behind each one.
+
+ ⛔ ODOO IS THE ONLY INTROSPECTABLE CONNECTOR TODAY and every other card answers `[]`. That is
+ the contract's own instruction, and the alternative would be a list of database names no
+ connector actually creates.
+
+ ⛔ THE ROW SURVIVES FOR A NON-ADMIN; ONLY THE CATALOGUE HALVES EMPTY. Returning `[]` to a
+ member would be indistinguishable from *"this connector creates no databases"*, which is the
+ confidently-wrong shape this module's header was written about. A database's NAME and whether
+ it is switched on are facts about their own workspace; the raw Odoo field catalogue is
+ administrator-only (`_odoo_admin` gates the route that serves it) and stays that way.
+
+ Fields with the same source model have independent enablement keys in each database.
+ `sharesFieldsWith` remains an empty list for existing clients of the older shared switches.
+
+ ⭐⭐ W53-T27 / R9 / C2 — THREE NAMES PER ROW, AND THEY ARE THREE DIFFERENT FACTS. `native` is
+ the VENDOR's own identifier (`res.partner`, or Amazon's report type) and is never editable by
+ anybody; `defaultLabel` is OUR shipped default; `override` is this tenant's own rename, held
+ in the `nav_meta` bucket the navigation rail already writes. `label` stays the EFFECTIVE name
+ because every existing reader of it — this tab's heading, `_detail_overview`'s `creates` list
+ — already means "what this database is called today", and serving the un-overridden default
+ there would be the drift R9 exists to close.
+
+ ⛔ ALL THREE SURVIVE FOR A NON-ADMIN, like `key` and `enabled` above them and for the same
+ reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
+ is walled, per row, by `canRename`.
+ """
+ if str(connector_key) == "scraper":
+ return _scraper_databases(rt, is_admin, uname)
+ if not connector_directory.is_introspectable(connector_key):
+ return []
+ spec = _connector_spec_tables(connector_key)
+ if str(connector_key) == "amazon_sp":
+ return _named_databases(rt, is_admin, uname, _amazon_databases(rt, is_admin, spec))
+ try:
+ import odoo_relational
+ namespace = (odoo_relational.connection_namespace(rt, connection_id)[1]
+ if connection_id else None)
+ choices = odoo_relational.grid_choices(rt, connection_id)
+ models = dict(getattr(odoo_relational, "ODOO_TABLE_MODELS", {}) or {})
+ # This read never contacts Odoo. A 499-model exact pass measured 67.87 s after the
+ # catalogue was warm, so modal detail must never turn a missing cache into inline work.
+ census = odoo_relational.population_census(rt, namespace)
+ except Exception: # noqa: BLE001
+ return []
+ health = odoo_health_receipt(rt, connection_id)
+ health_by_key = {str(item.get("databaseKey") or ""): item
+ for item in (health.get("databases") or ())}
+ census_at = str(census.get("capturedAt") or "")
+ counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
+ catalog = {}
+ if is_admin:
+ try:
+ catalog = _odoo_catalog_by_model(rt, namespace)
+ except Exception: # noqa: BLE001
+ catalog = {}
+ enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {}
+ # Connection resolution reaches keychain.list_entries and therefore the store (Postgres).
+ # It ran 12,824 times for fields and database identities, then 501 with a per-key memo.
+ # Resolve the shared default lazily once instead. This local belongs only to this detail
+ # build: request scope keeps the answer per runtime and tenant, never process-wide.
+ default_connection_id = ...
+
+ def resolve_connection_id(key):
+ nonlocal default_connection_id
+ base, cid = odoo_relational.split_table_key(key)
+ if base is None:
+ return None
+ if cid:
+ return cid
+ if default_connection_id is ...:
+ default_connection_id = (odoo_relational.default_connection_id(rt)
+ or odoo_relational.DEFAULT_CONNECTION)
+ return default_connection_id
+
+ out = []
+ for choice in choices:
+ key = str(choice.get("key") or "")
+ resolved_connection_id = resolve_connection_id(key)
+ model = str(choice.get("model") or models.get(key) or "")
+ fields, excluded = catalog.get(model, ([], []))
+ scoped_fields = []
+ for field in fields:
+ field_key = _odoo_field_key(rt, key, model, field["source_field"],
+ connection_id=resolved_connection_id)
+ scoped_fields.append({**field, "key": field_key,
+ "enabled": enabled_fields.get(field_key,
+ field.get("enabled") if namespace is None else False) is True})
+ declared = spec.get(choice.get("baseKey") or key) or {}
+ # Counts are model-level facts. Databases backed by the same model deliberately show the
+ # same sourceRows number; inventing a per-database domain here would make the number lie.
+ source_rows = counts.get(model) if census_at and model in counts else None
+ database_health = health_by_key.get(key) or {
+ # Source rows is a model census. It is not this governed database's materialized
+ # row count, especially when two curated databases share one source model.
+ "databaseKey": key, "state": "unknown", "rowCount": None,
+ "lastSuccess": None, "currentLagSeconds": None, "recordRefreshed": None,
+ "failure": None, "stages": _empty_odoo_stages()}
+ out.append({"key": key,
+ "identity": connector_sync.database_identity(
+ "odoo", resolved_connection_id, key, model),
+ "label": str(declared.get("label") or choice.get("label") or key),
+ "defaultLabel": str(declared.get("label") or choice.get("label") or key),
+ "native": str(declared.get("native") or model),
+ "enabled": bool(choice.get("enabled")),
+ "sourceRows": source_rows,
+ "populated": None if source_rows is None else source_rows > 0,
+ # Data-free client receipt: source/mirror/materialization/reconciliation/
+ # freshness are distinct, and one database's failure cannot make a sibling
+ # look newly successful.
+ "sync": database_health,
+ "syncHealth": _odoo_database_sync_health(
+ database_health, bool(choice.get("enabled"))),
+ "fields": scoped_fields,
+ "excluded": [dict(item) for item in excluded],
+ "sharesFieldsWith": []})
+ return _named_databases(rt, is_admin, uname, out)
+
+
+def _amazon_report_name(table):
+ """The report type this Amazon table reads, whichever key declares it.
+
+ ⛔ SETTLEMENTS DECLARE `report_discover`, NOT `report` (W60-T36), because Amazon issues them on
+ its own schedule and refuses a createReport for that type: it is a getReports DISCOVERY filter
+ and is deliberately absent from `_APPROVED_REPORT_TYPES`. Reading only `report` here printed an
+ empty Source name for settlements on the connector screen, which reads as "this table has no
+ report" when the truth is "this table is not requested, it is listed".
+ """
+ import connectors_amazon as cx # noqa: PLC0415 - provider owns state
+ spec = cx.REPORT_TYPES.get(table) or {}
+ return str(spec.get("report") or spec.get("report_discover") or "")
+
+
+def _amazon_databases(rt, is_admin, spec=None):
+ """⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM.
+
+ Fills the shared `ConnectorDatabase` shape and adds an administrator-only Amazon `sync`
+ block. Members retain the shared shape without the field catalogue or sync bookkeeping.
+
+ ⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five
+ of the seven tables have NO published schema, because Amazon publishes artifacts only for its
+ JSON analytics reports and every flat-file report we depend on has none. Their columns are
+ measured from the report document's own header at first sync, so before that they genuinely
+ have zero fields. A screen that showed an empty field list with no explanation would say
+ *"this database has no fields"* when the truth is *"nobody has run it yet"* — the exact
+ confusion `_detail_introspection`'s header was written about, one layer in.
+
+ ⛔ `sharesFieldsWith` IS REAL HERE, NOT COPIED. `orders` and `order_items` are BOTH filled from
+ ONE report document (`GET_FLAT_FILE_ALL_ORDERS_DATA_*`, which is one row per order ITEM), so
+ they share a header and therefore a field list. That is the same class of fact the Odoo pairs
+ carry, measured the same way: derived from the report map, never hand-typed.
+
+ ⚠ NO FIELD TOGGLES. `canToggleFields` is False for Amazon in `_detail_introspection`; there is
+ no `PATCH /connectors/amazon_sp/schema/{field}` route and this wave does not build one. Fields
+ are disclosed, not switched. Database enablement is a separate provider capability and route,
+ never the generic Odoo-table flag consumed by `DatabasesTab`.
+ """
+ try:
+ from harness import amazon_schema
+ import connectors_amazon as cx
+ except Exception: # noqa: BLE001
+ return []
+ enablement = cx.read_database_enablement(rt)
+ enabled = enablement.get("databases") or {}
+ cat = amazon_schema.catalogue()
+ tables = cat.get("tables") or {}
+ states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
+
+ # Which OTHER declared tables are filled from the SAME report document. Derived from the
+ # report map so a table added to `REPORT_TYPES` gets this disclosure for free.
+ by_report = {}
+ for name in amazon_schema.PROVENANCE:
+ rep = _amazon_report_name(name)
+ if rep:
+ by_report.setdefault(rep, []).append("ut_amzn_" + name)
+
+ out = []
+ for name, provenance in amazon_schema.PROVENANCE.items():
+ key = "ut_amzn_" + name
+ spec = tables.get(name) or {}
+ cols = spec.get("columns") or []
+ rep = _amazon_report_name(name)
+ fields = [{"key": str(c.get("name") or ""),
+ "label": str(c.get("name") or "").replace("_", " ").capitalize(),
+ "type": "text" if str(c.get("type")) == "VARCHAR" else "number",
+ "source_field": str(c.get("name") or ""),
+ # ⚠ EVERY DISCLOSED FIELD IS ON. There is no per-field switch for Amazon, and
+ # reporting `enabled: False` would describe a control that does not exist.
+ "enabled": True}
+ for c in cols] if is_admin else []
+ excluded = []
+ if provenance == "measured_at_sync" and not cols:
+ excluded.append({
+ "source_field": "",
+ "reason": (amazon_schema.PROVENANCE_COPY["measured_at_sync"]
+ + f" The report is {rep}." if rep
+ else amazon_schema.PROVENANCE_COPY["measured_at_sync"])})
+ # ⭐⭐ W53-T27 / C2 — THE SAME THREE FACTS AS THE ODOO ROWS, one source swapped. Amazon
+ # has no per-table model name; the identifier AMAZON itself uses for what fills a table is
+ # the REPORT TYPE (`rep`, already computed above for `sharesFieldsWith`). That is exactly
+ # what `native` carries: the vendor's own name for the thing, never one we invented. It is
+ # empty for a table whose report type is unknown, and the tab says so rather than guessing.
+ declared = (spec or {}).get(key) or {}
+ raw_sync = (states or {}).get(name) if is_admin else None
+ is_enabled = enabled.get(key) is True
+ out.append({"key": key,
+ "label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
+ "defaultLabel": str(declared.get("label")
+ or LABELS_AMAZON.get(key) or key),
+ "native": str(declared.get("native") or rep),
+ # W55-T19 / R7 — AMAZON'S OWN tenant-local answer. This is deliberately not
+ # read from Odoo's grid config, and disabling it leaves every stored row and
+ # definition untouched: only this provider config document changes.
+ "enabled": is_enabled,
+ **({"sync": raw_sync} if is_admin else {}),
+ "syncHealth": _amazon_database_sync_health(raw_sync, is_enabled),
+ "fields": fields,
+ "excluded": excluded,
+ "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
+ return out
+
+
+#: What an empty `excluded` list means for AMAZON, which is not what it means for Odoo.
+#:
+#: ⛔ A SEPARATE SENTENCE ON PURPOSE. Odoo's basis describes a DIFF of a live snapshot against a
+#: catalogue. Amazon has no snapshot and no diff: a table either has a published schema or it
+#: measures its columns at first sync. Reusing Odoo's wording would describe a computation that
+#: does not happen here, which is the same class of error as reusing its toggles.
+_AMAZON_EXCLUDED_BASIS = (
+ "Amazon publishes a schema for its JSON analytics reports only. A table with no published "
+ "schema reads its columns from the report's own header the first time it syncs, so it lists "
+ "no fields until then. Nothing is being held back from you.")
+
+
+#: Table labels for the Amazon disclosure. ⛔ NO EM DASH OR EN DASH: these reach a screen.
+LABELS_AMAZON = {
+ "ut_amzn_orders": "Amazon orders",
+ "ut_amzn_order_items": "Amazon order items",
+ "ut_amzn_listings": "Amazon listings",
+ "ut_amzn_settlements": "Amazon settlements",
+ "ut_amzn_inventory": "Amazon inventory",
+ "ut_amzn_pricing": "Amazon pricing and Buy Box",
+ "ut_amzn_brand_analytics": "Amazon brand analytics",
+}
+
+
+#: What an EMPTY `excluded` list actually means, said once, on the server that computes it.
+#:
+#: ⛔⛔ THE TRAP THIS CLOSES, MEASURED: `_odoo_catalog_by_model` builds `excluded` as a DIFF of the
+#: snapshot's raw fields against the catalogue, and the catalogue offers every raw field it
+#: captured — so the diff is EMPTY TODAY BY CONSTRUCTION. A tab that rendered "Excluded: none"
+#: over that would be telling an administrator that this workspace holds back nothing from Odoo,
+#: which is a claim about POLICY that the number cannot support. The empty list means the
+#: catalogue dropped nothing it saw; it says nothing at all about what Odoo never reported.
+#:
+#: ⚠ IT IS SERVED RATHER THAN WRITTEN IN THE CLIENT because the sentence is a description of a
+#: server computation. A client copy would go stale the day the catalogue starts filtering, and
+#: nothing would go red.
+_EXCLUDED_BASIS = ("A field is excluded when this workspace captured it from Odoo and the field "
+ "catalogue did not offer it. Anything Odoo never reported cannot appear in "
+ "this list at all.")
+
+
+def _detail_introspection(rt, is_admin, row):
+ """R24's half of the Databases tab: may this connector be disclosed, and to THIS reader?
+
+ ⭐ AN ADDITION TO C4, DECLARED, and the same kind of addition `_detail_connect` is. C4 pins
+ the `databases` list and says nothing about how a client tells "this connector has no schema"
+ apart from "this connector's schema could not be read". Those two are the same empty list and
+ they mean opposite things, so the difference is stated rather than inferred.
+
+ ⛔ EVERY FLAG HERE IS A CAPABILITY THE SERVER WILL HONOUR, not a guess the client makes. The
+ field toggle is `PATCH /connectors/odoo/schema/{field_key}`, which `_odoo_admin` gates and
+ which raises 409 `odoo_schema_unavailable` while no complete snapshot exists — so
+ `canToggleFields` carries BOTH conditions and the client renders no control that could only
+ ever 409. A button that always fails is R7's banned shape.
+
+ ⚠ `fieldsVisible` IS SEPARATE FROM `canToggleFields` ON PURPOSE. A member sees the database
+ names and their on/off state (their own workspace's facts) and no field catalogue at all
+ (`_odoo_admin` gates the route that serves it); saying which of the two is missing is what
+ stops "administrator only" being rendered as "this database has no fields".
+ """
+ key = str(row.get("key") or "")
+ if key == "scraper":
+ # Recipe databases publish their saved, stable schema to every reader who may open the
+ # database. They have no generic schema-toggle route, so no write capability is implied.
+ return {"introspectable": True, "reason": "", "fieldsVisible": True,
+ "canToggleFields": False, "canToggleTables": False,
+ "catalogueComplete": True, "capturedAt": "", "excludedBasis": "",
+ "censusAt": "", "censusComplete": False, "censusDurationMs": 0,
+ "censusModelsCounted": 0, "censusModelsWithRows": 0}
+ if not connector_directory.is_introspectable(key):
+ reason = ("Nothing is connected behind this listing yet, so there is no schema to read."
+ if str(row.get("state") or "") == connector_directory.CATALOGUE_STATE
+ else "This connector does not publish a field catalogue, so the databases it "
+ "creates cannot be listed field by field.")
+ return {"introspectable": False, "reason": reason, "fieldsVisible": False,
+ "canToggleFields": False, "canToggleTables": False,
+ "catalogueComplete": False, "capturedAt": "", "excludedBasis": "",
+ "censusAt": "", "censusComplete": False, "censusDurationMs": 0,
+ "censusModelsCounted": 0, "censusModelsWithRows": 0}
+ if key == "amazon_sp":
+ # ⭐⭐ W52 · R6. Introspectable under the AMENDED test (a real schema artifact, not a
+ # `fields_get`), and deliberately with NO generic toggles: there is no Amazon field-toggle
+ # route, and `canToggleTables` belongs to Odoo's config surface. W55-T19 serves Amazon's
+ # database switches through the separate `enablement` capability instead, so this flag
+ # stays False and can never send Amazon through an Odoo endpoint.
+ #
+ # ⛔ `catalogueComplete` IS FALSE AND SAYING SO IS THE HONEST ANSWER. Only two of the seven
+ # tables have a published artifact; the other five measure their columns from the report's
+ # own header at first sync. Reporting `True` would claim a completeness that does not
+ # exist, and reporting `introspectable: False` would hide the two that ARE published.
+ try:
+ from harness import amazon_schema
+ cat = amazon_schema.catalogue()
+ tables = cat.get("tables") or {}
+ complete = bool(tables) and all((t.get("columns") or []) for t in tables.values())
+ captured = str(cat.get("fetched_utc") or "")
+ except Exception: # noqa: BLE001
+ complete, captured = False, ""
+ return {"introspectable": True,
+ "reason": "",
+ "fieldsVisible": bool(is_admin),
+ "canToggleFields": False,
+ "canToggleTables": False,
+ "catalogueComplete": complete,
+ # ⚠ THE CATALOGUE'S OWN FETCH DATE, NEVER TODAY'S. R6 is explicit that an
+ # unstamped pinned catalogue is a stale seed waiting to happen, and this screen is
+ # the only place the staleness becomes visible.
+ "capturedAt": captured,
+ "excludedBasis": _AMAZON_EXCLUDED_BASIS,
+ "censusAt": "", "censusComplete": False, "censusDurationMs": 0,
+ "censusModelsCounted": 0, "censusModelsWithRows": 0}
+ complete = False
+ captured = ""
+ census = {}
+ try:
+ import core.user_tables as user_tables
+ snapshot = user_tables.odoo_schema_snapshot(rt)
+ import odoo_relational
+ discovered = odoo_relational.readable_model_catalog(rt)
+ census = odoo_relational.population_census(rt)
+ has_discovery = bool(discovered.get("capturedAt"))
+ complete = (discovered.get("complete") is True if has_discovery
+ else snapshot.get("complete") is True)
+ captured = str((discovered.get("capturedAt") if has_discovery
+ else snapshot.get("capturedAt")) or "")
+ except Exception: # noqa: BLE001
+ complete, captured = False, ""
+ return {"introspectable": True,
+ "reason": "",
+ "fieldsVisible": bool(is_admin),
+ # BOTH conditions, because the write door enforces both.
+ "canToggleFields": bool(is_admin) and complete,
+ # ⭐ R22: ANY table can be switched off, not only the ones we mark optional. The write
+ # is `PUT /admin/connectors/odoo/config`, which is an administrator's door and accepts
+ # every key `grid_choices` names, including all of them at once.
+ "canToggleTables": bool(is_admin),
+ "catalogueComplete": complete,
+ "capturedAt": captured,
+ "excludedBasis": _EXCLUDED_BASIS,
+ "censusAt": str(census.get("capturedAt") or ""),
+ "censusComplete": census.get("complete") is True,
+ "censusDurationMs": int(census.get("durationMs") or 0),
+ "censusModelsCounted": int(census.get("modelsCounted") or 0),
+ "censusModelsWithRows": int(census.get("modelsWithRows") or 0)}
+
+
+def _detail_overview(row, databases, graph):
+ """C4's `overview` half: what the modal's first tab reads, top to bottom.
+
+ `creates` is the ENABLED table LABELS this connector stands behind: the declared grids when the
+ connector declares any, and otherwise the tables its own permitted graph actually found. Both
+ are real readings of THIS workspace; neither is a provider template.
+ """
+ creates = [str(entry.get("label") or "") for entry in (databases or [])
+ if entry.get("enabled") is True and str(entry.get("label") or "")]
+ if not databases and isinstance(graph, dict) and graph.get("status") == "ready":
+ creates = [str(table.get("label") or "") for table in (graph.get("tables") or ())
+ if str(table.get("label") or "")]
+ return {"key": str(row.get("key") or ""),
+ "label": str(row.get("label") or ""),
+ "status": str(row.get("state") or ""),
+ # Empty when the server has no account to name. The client says so in a sentence
+ # rather than printing a blank row.
+ "account": str(row.get("connectedAs") or ""),
+ # The system the data comes FROM, in the words a customer reads on the card. Never a
+ # provider slug: an internal identifier on screen is exactly what `parseDirectory`
+ # drops a row to avoid.
+ "source": str(row.get("label") or ""),
+ "creates": creates}
+
+
+def _detail_enablement(rt, is_admin, row):
+ """A provider-owned enablement capability, or ``None`` when this connector has no door.
+
+ Amazon is intentionally the first entry and is named here rather than inferred from whether a
+ database list happened to be non-empty. Odoo keeps its model-catalogue and config routes;
+ every other connector keeps no enablement affordance until its provider declares one.
+ """
+ if str(row.get("key") or "") != "amazon_sp":
+ return None
+ try:
+ import connectors_amazon as cx
+ return cx.enablement_capability(rt, bool(is_admin))
+ except Exception: # noqa: BLE001
+ # Fail closed. An unreadable provider capability is no write door.
+ return {"provider": "amazon_sp", "mode": "database", "canManage": False,
+ "updatePath": "", "preservesData": True, "available": False}
+
+
+def _scraper_preview(body):
+ request = body if isinstance(body, dict) else {}
+ raw_recipe = request.get("recipe") if isinstance(request.get("recipe"), dict) else request
+ recipe, errors = _clean_scraper_recipe(raw_recipe)
+ if errors:
+ raise err(400, "invalid_scraper_recipe", " ".join(errors))
+ plan = _scraper_plan(recipe)
+ missing = [] if plan["credentialReady"] else ["credential"]
+ return {"recipe": recipe, "inputs": recipe["seed"], "provider": plan["provider"],
+ "credentialReady": plan["credentialReady"], "estimatedUsd": plan["estimatedUsd"],
+ "costCapUsd": plan["costCapUsd"], "recordsCap": plan["recordsCap"],
+ "canRun": plan["canRun"], "missing": missing, "note": plan["note"],
+ # This is a contract field, not a measurement. A preview and a create never call a
+ # provider, so both truthfully report zero incurred spend.
+ "incurredUsd": 0.0}
+
+
+def _create_scraper_database(rt, uname, body):
+ """Persist one recipe database. Contains no provider import or runner call by construction."""
+ if not rt.available():
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
+ request = body if isinstance(body, dict) else {}
+ label = str(request.get("label") or "").strip()
+ if not label:
+ raise err(400, "missing_database_name", "Enter a database name.")
+ preview = _scraper_preview(request)
+ recipe = preview["recipe"]
+ import core.user_tables as user_tables # noqa: PLC0415 - existing create door
+ key = user_tables.create(label, uname, fields=_scraper_recipe_fields(recipe["kind"]),
+ source=user_tables.AUTOMATION_SOURCE, st=rt)
+ if not key:
+ raise err(409, "database_not_created",
+ "The database name is unavailable or this workspace reached its database limit.")
+
+ def _attach(cur):
+ current = dict(cur or {})
+ definition = dict(current.get(key) or {})
+ if not definition:
+ raise RuntimeError("the new database disappeared before its recipe was attached")
+ definition.update({"connectorKey": "scraper", "recordMode": "automation",
+ "scraperRecipe": recipe, "scraperRuns": [],
+ user_tables.CONNECTOR_HEALTH_FIELD:
+ user_tables.connector_health_receipt("never_run")})
+ current[key] = definition
+ return current
+
+ try:
+ rt.update(user_tables.STORE_KEY, _attach, flush="sync")
+ except Exception: # noqa: BLE001
+ user_tables.delete(key, st=rt)
+ raise err(503, "recipe_not_saved",
+ "The recipe could not be saved, so the new database was removed.")
+ return {"key": key, "label": label, "recipe": recipe, "preview": preview,
+ "createdWithoutProviderCall": True, "incurredUsd": 0.0}
+
+
+def _scraper_database_run(rt, uname, is_admin, database_key):
+ """The one explicit paid-work boundary for any saved scraper recipe."""
+ if not rt.available():
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
+ import core.user_tables as user_tables
+ key = str(database_key or "").strip()
+ definitions = user_tables.lend_defs(rt)
+ definition = user_tables.get(key, st=definitions) or {}
+ if not definition:
+ raise err(404, "unknown_scraper_database", "This saved scraper database was not found.")
+ raw_recipe = definition.get("scraperRecipe")
+ recipe, errors = _clean_scraper_recipe(raw_recipe)
+ if errors:
+ raw_kind = str((raw_recipe or {}).get("kind") or "") if isinstance(raw_recipe, dict) else ""
+ raw_limits = ((raw_recipe or {}).get("limits") or {}) if isinstance(raw_recipe, dict) else {}
+ raw_cap = raw_limits.get("records") if isinstance(raw_limits, dict) else None
+ try:
+ bounded = 1 <= int(raw_cap) <= 500
+ except (TypeError, ValueError):
+ bounded = False
+ if raw_kind == "google_maps" and not bounded:
+ raise err(400, "maps_unbounded_request",
+ "Google Maps discovery requires an explicit result cap between 1 and 500.")
+ raise err(409, "invalid_saved_scraper_recipe",
+ "This saved scraper recipe is no longer valid. Review its settings before running it.")
+ # Run now both spends against a provider account and mutates this database. A viewer grant
+ # may expose the rows without granting either power, so the ordinary open wall is too broad.
+ # Reuse the database editor authority that every other mutation surface already honours.
+ if not user_tables.may_edit_database(key, uname, bool(is_admin), st=definitions):
+ raise err(403, "database_forbidden", "You cannot run this database recipe.")
+ plan = _scraper_plan(recipe)
+ previous_health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD)
+ previous_health = previous_health if isinstance(previous_health, dict) else {}
+
+ def record_attempt(state, error=None):
+ try:
+ return user_tables.record_connector_health(
+ key, state, st=rt,
+ last_success=previous_health.get("lastSuccess"),
+ source_timestamp=previous_health.get("sourceTimestamp"),
+ materialized_timestamp=previous_health.get("materializedTimestamp"),
+ row_count=previous_health.get("rowCount"), error=error)
+ except Exception: # noqa: BLE001
+ raise err(503, "sync_status_not_saved",
+ "The run did not start because its durable status could not be saved.") from None
+
+ if not plan["credentialReady"]:
+ code = "maps_credential_missing" if recipe["kind"] == "google_maps" else \
+ "scraper_credential_missing"
+ message = "Add the scraper credential in Keychain before choosing Run now."
+ record_attempt("failed", {"code": code, "message": message})
+ raise err(409, code, "Add the scraper credential in Keychain before choosing Run now.")
+
+ # Imports and provider calls occur only below the explicit Run-now endpoint. Preview, create,
+ # detail, and page load never reach either runner.
+ started_at = datetime.now(timezone.utc).isoformat()
+ record_attempt("running")
+ key_field = "shortcode"
+ if recipe["kind"] == "google_maps":
+ import providers
+ try:
+ result = providers.discover_google_maps_stores(
+ recipe["seed"]["search"], recipe["seed"]["geography"],
+ recipe["limits"]["records"], captured_at=started_at)
+ except providers.MapsProviderRefusal as exc:
+ record_attempt("failed", {"code": exc.code, "message": exc.public_message})
+ status = 409 if exc.code == "maps_credential_missing" else \
+ (400 if exc.code in ("maps_unbounded_request", "maps_invalid_search") else 502)
+ raise err(status, exc.code, exc.public_message)
+ incoming_key = "rows"
+ key_field = "place_id"
+ else:
+ limit = recipe["limits"]["posts"]
+ post_type = recipe["limits"]["postType"]
+ try:
+ if recipe["kind"] == "instagram":
+ import connectors_ig as runner
+ run_recipe = runner.pull_ig_recipe
+ else:
+ import connectors_tt as runner
+ run_recipe = runner.pull_tt_recipe
+ result = run_recipe(recipe["seed"], limit, post_type=post_type,
+ log=lambda *_args: None)
+ except Exception: # noqa: BLE001
+ record_attempt("failed", {
+ "code": "scraper_run_failed",
+ "message": "The provider could not complete this run. No result was written.",
+ })
+ raise err(502, "scraper_run_failed",
+ "The provider could not complete this run. No result was written.")
+ incoming_key = "posts"
+ result = result if isinstance(result, dict) else {}
+ incoming = [dict(row) for row in (result.get(incoming_key) or ()) if isinstance(row, dict)]
+ state = str(result.get("state") or "error")
+ if state not in ("ok", "partial"):
+ incoming = []
+ finished_at = datetime.now(timezone.utc).isoformat()
+ source_stamps = [_sync_stamp(row.get("source_timestamp")) for row in incoming]
+ source_timestamp = max((stamp for stamp in source_stamps if stamp), default=None)
+ provider_counts = dict(result.get("counts") or {})
+ provider_counts.setdefault("returned", len(incoming))
+ provider_cost = dict(result.get("cost") or {})
+ provider_cost.setdefault("currency", "USD")
+ provider_cost.setdefault("estimatedUsd", plan["estimatedUsd"])
+ provider_cost.setdefault("costCapUsd", plan["costCapUsd"])
+ receipt = {"id": finished_at, "state": state,
+ "provider": str(result.get("via") or plan["provider"]),
+ "rows": len(incoming), "startedAt": started_at, "finishedAt": finished_at,
+ "estimatedUsd": plan["estimatedUsd"], "costCapUsd": plan["costCapUsd"],
+ "cost": provider_cost, "providerCounts": provider_counts,
+ "attempts": [dict(item) for item in (result.get("attempts") or ())
+ if isinstance(item, dict)],
+ # Provider notes can echo a submitted handle. Run history stores operational
+ # metadata only; the saved recipe already owns the input and no customer value is
+ # duplicated into an audit-like receipt.
+ "note": f"Provider returned {state}."}
+ counts = {"inserted": 0, "updated": 0, "unchanged": 0, "skipped": 0,
+ "duplicates": 0, "orphans": 0, "capped": 0}
+
+ def _write(cur):
+ nonlocal counts
+ current = dict(cur or {})
+ stored = dict(current.get(key) or {})
+ if not stored:
+ raise RuntimeError("the database disappeared before the run could be written")
+ if incoming:
+ import automation_engine as engine
+ rows, counts = engine.upsert_rows(stored.get("rows") or {}, incoming, key_field)
+ stored["rows"] = rows
+ processed = sum(int(counts.get(name) or 0)
+ for name in ("inserted", "updated", "unchanged", "skipped", "capped"))
+ receipt["writeCounts"] = dict(counts)
+ receipt["countReconciled"] = (
+ int(provider_counts.get("returned") or 0) == len(incoming) == processed)
+ history = [dict(item) for item in (stored.get("scraperRuns") or ())
+ if isinstance(item, dict)]
+ stored["scraperRuns"] = [receipt, *history][:SCRAPER_RUN_HISTORY_LIMIT]
+ final_state = "succeeded" if state == "ok" else "partial" if state == "partial" else "failed"
+ final_error = None if final_state != "failed" else {
+ "code": "provider_result_failed",
+ "message": "The provider returned no usable result for this database.",
+ }
+ stored[user_tables.CONNECTOR_HEALTH_FIELD] = user_tables.connector_health_receipt(
+ final_state,
+ last_success=finished_at if final_state in ("succeeded", "partial")
+ else previous_health.get("lastSuccess"),
+ source_timestamp=source_timestamp,
+ materialized_timestamp=finished_at if final_state in ("succeeded", "partial") else None,
+ row_count=len(stored.get("rows") or {}), error=final_error)
+ current[key] = stored
+ return current
+
+ try:
+ rt.update(user_tables.STORE_KEY, _write, flush="sync")
+ except Exception: # noqa: BLE001
+ try:
+ record_attempt("failed", {
+ "code": "scraper_result_not_saved",
+ "message": "The provider answered, but the result could not be saved.",
+ })
+ except Exception: # noqa: BLE001
+ pass
+ raise err(503, "scraper_result_not_saved",
+ "The provider answered, but the result could not be saved. Reload before retrying.")
+ return {"key": key, "state": state, "counts": counts, "receipt": receipt}
+
+
+def connector_detail(rt, user, uname, is_admin, connector_key):
+ """The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
+
+ Pure over the session for the same reason `directory` is: a gate can drive it without a
+ request, so the modal's payload is provable without a browser.
+ """
+ budget = _ConnectorDetailBudget()
+ rows = directory(rt, uname, bool(is_admin))
+ budget.check()
+ key = str(connector_key or "")
+ row = next((candidate for candidate in rows
+ if str(candidate.get("key") or "") == key), None)
+ if row is None:
+ return None
+ graph = relationship_graphs(rows, user, rt, uname, bool(is_admin)).get(key) or _graph_refusal(
+ key, "No actual relationship graph was returned for this connection. A provider template "
+ "is not shown.")
+ budget.check()
+ # ⚠ `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a
+ # question about a PERSON and a table, not about an administrator flag alone.
+ databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or ""))
+ budget.check()
+ overview = _detail_overview(row, databases, graph)
+ budget.check()
+ keychain = _detail_keychain(rt, uname, bool(is_admin), row)
+ budget.check()
+ enablement = _detail_enablement(rt, bool(is_admin), row)
+ budget.check()
+ introspection = _detail_introspection(rt, bool(is_admin), row)
+ budget.check()
+ return _safe_connector_status(key, {"overview": overview,
+ "keychain": keychain,
+ "databases": databases,
+ "syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"}
+ if is_admin and key in connector_sync.providers() else None),
+ "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
+ "retryPath": f"/connectors/{key}/sync/retry"}
+ if is_admin and key == "odoo" else None),
+ # W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it
+ # from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints.
+ "enablement": enablement,
+ # ⭐ W49-T14 / R24 — WHETHER THE DATABASES TAB MAY DISCLOSE ANYTHING AT ALL, and what
+ # THIS reader may do there. An addition to C4, declared on `_detail_introspection`:
+ # without it an empty `databases` list is "this connector has no schema" and "this
+ # connector's schema could not be read" at the same time.
+ "introspection": introspection,
+ # ⛔ THE `relationship_graphs` SHAPE, UNCHANGED. C4 says so in as many words, and the
+ # client already has a reader for it (`parseRelationshipGraph`) that refuses a graph
+ # bound to another connection. A second shape here would need a second reader.
+ "relationships": graph})
+
+
+def _set_amazon_database_enablement(database_key, body, session):
+ """The testable core of Amazon's tenant-scoped administrator enablement door."""
+ if not session.admin:
+ raise err(403, "not_admin",
+ "only an administrator can change Amazon database enablement")
+ if not session.runtime.available():
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
+ request = body if isinstance(body, dict) else {}
+ if set(request) != {"enabled"} or type(request.get("enabled")) is not bool:
+ raise err(400, "bad_enabled", "enabled must be true or false")
+
+ import connectors_amazon as cx # noqa: PLC0415 - provider owns state
+ key = str(database_key or "").strip()
+ try:
+ state = cx.set_database_enabled(session.runtime, key, request["enabled"])
+ except KeyError:
+ # One refusal for blank, malformed and cross-provider keys. The response does not expose
+ # the rest of the tenant's Amazon roster.
+ raise err(404, "unknown_amazon_database",
+ "Amazon does not declare a database with that key")
+ except cx.EnablementConfigError:
+ raise err(409, "amazon_enablement_unavailable",
+ "the stored Amazon enablement settings are invalid. Nothing was changed.")
+ except ValueError:
+ raise err(400, "bad_enabled", "enabled must be true or false")
+ except Exception: # noqa: BLE001
+ raise err(503, "enablement_unconfirmed",
+ "the server could not confirm the Amazon database setting. Reload before "
+ "trying again.")
+ return {"provider": cx.ENTRY_TYPE,
+ "key": key,
+ "enabled": state["databases"].get(key) is True,
+ # This route changes only the provider config document. It never deletes a mirror or
+ # a materialised table, and the wire states that invariant explicitly.
+ "preservesData": True}
+
+
+# Literal and provider-owned. It appears before the generic connector detail route so future
+# dynamic route additions cannot turn Amazon enablement into an Odoo-model dispatch by accident.
+@router.patch("/connectors/amazon_sp/databases/{database_key}/enablement")
+def amazon_database_enablement(database_key: str, body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Enable or disable one declared Amazon database for this tenant, without deleting data."""
+ return _set_amazon_database_enablement(database_key, body, session)
+
+
+@router.post("/connectors/scraper/databases/preview")
+def scraper_database_preview(body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Validate and price a recipe locally. This endpoint makes zero provider calls."""
+ return _scraper_preview(body)
+
+
+@router.post("/connectors/scraper/databases")
+def scraper_database_create(body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Create a saved recipe database for $0; paid work begins only at its Run-now door."""
+ return _create_scraper_database(session.runtime, session.uname, body)
+
+
+@router.post("/connectors/scraper/databases/{database_key}/run")
+def scraper_database_run(database_key: str, session: Session = Depends(require_session)):
+ """Explicitly execute one bounded saved scraper recipe through its provider seam."""
+ return _scraper_database_run(session.runtime, session.uname, bool(session.admin), database_key)
+
+
+# ⚠ DECLARED AFTER THE LITERAL `/connectors/odoo/*` ROUTES. FastAPI matches in declaration order,
+# and while none of those ends in `/detail` — so no shadowing is possible today — the ordering is
+# the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
+# collide with this three-segment path at all.
+@router.get("/connectors/{connector_key}/detail")
+def connectors_detail(connector_key: str, session: Session = Depends(require_session)):
+ """C4 / R26: everything the connector modal shows, for THIS reader, in one read.
+
+ Session-gated exactly like the directory it is opened from: a member may look at what the
+ workspace is connected to. What a member does NOT get is the tenant-wide credential door
+ (`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue.
+ """
+ detail = connector_detail(
+ session.runtime, session.user, session.uname,
+ bool(session.admin), connector_key)
+ if detail is None:
+ raise err(404, "unknown_connector",
+ "this workspace has no connector with that key")
+ return detail
+
+
+class _ReplacementConflict(Exception):
+ """The addressed keychain row changed between replacement preparation and its store update."""
+
+
+def _encrypted_replacement_payload(kc, label, entry_type, fields, username):
+ """Validate and encrypt through ``core.keychain.add_entry`` without writing a tenant row.
+
+ ``add_entry`` is the keychain's one validation, preview and Fernet implementation. A tiny
+ memory-only runtime lets this route reuse it while discarding the temporary identity it mints;
+ the tenant store sees only the later, single-row payload swap.
+ """
+ class _Stage:
+ def __init__(self):
+ self.bucket = {}
+
+ def get(self, name):
+ return self.bucket if name == kc.KEY else {}
+
+ def update(self, name, fn, flush="sync"):
+ if name != kc.KEY:
+ raise RuntimeError("the keychain encryptor wrote an unexpected bucket")
+ updated = fn(self.bucket)
+ self.bucket = updated if updated is not None else self.bucket
+ return self.bucket
+
+ stage = _Stage()
+ temporary = kc.add_entry(stage, label, entry_type, fields, username)
+ row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {})
+ if not row.get("enc"):
+ raise RuntimeError("the keychain encryptor returned no encrypted payload")
+ payload = {"enc": str(row["enc"]), "preview": str(row.get("preview") or ""),
+ "envelope_version": row["envelope_version"]}
+ if entry_type == "amazon_sp":
+ payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD]
+ return payload
+
+
+def _replace_connector_key(connector_key, entry_id, body, session):
+ """Replace only one connector entry's declared secrets while preserving its identity.
+
+ The existing row remains untouched through validation, decryption, merge and encryption. The
+ final store update compares the complete old row before swapping the prepared envelope;
+ a concurrent rename or credential change therefore conflicts instead of being overwritten.
+ Scope and owner live in a separate bucket and are never written by this operation.
+ """
+ if not session.admin:
+ raise err(403, "not_admin",
+ "only an administrator can replace a workspace connection key")
+ if not session.runtime.available():
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
+
+ key = str(connector_key or "").strip()
+ connector = next((row for row in directory(session.runtime, session.uname, True)
+ if str(row.get("key") or "") == key), None)
+ entry_type = _entry_type_for(key)
+ if connector is None or not entry_type:
+ raise err(404, "unknown_connector", "this workspace has no connector with that key")
+ if str(connector.get("manage") or "") not in ("keychain", "keychain_personal"):
+ raise err(403, "replacement_unavailable",
+ "this connector has no replaceable keychain credential")
+
+ import routes_keychain as rk # noqa: PLC0415 - circular at import
+ from core import keychain as kc # noqa: PLC0415 - keep boot lean
+ visible = rk.visible_entries(session.runtime, session.uname, True)
+ entry = next((row for row in visible
+ if str(row.get("id") or "") == str(entry_id)
+ and str(row.get("type") or "") == entry_type), None)
+ if entry is None:
+ # One 404 covers missing, cross-tenant, another user's personal entry and a real id bound
+ # to another connector. The mismatch itself is information this route does not disclose.
+ raise err(404, "no_entry", "no such key for this connector")
+
+ secret_specs = _replacement_field_specs(entry_type)
+ secret_keys = [str(field["key"]) for field in secret_specs if field.get("secret") is True]
+ optional_keys = {AMAZON_ROTATION_DEADLINE_FIELD} if entry_type == "amazon_sp" else set()
+ request = body if isinstance(body, dict) else {}
+ supplied = request.get("fields")
+ if set(request) != {"fields"} or not isinstance(supplied, dict):
+ raise err(400, "bad_replacement",
+ "send only the connector's new secret fields")
+ if not set(secret_keys) <= set(supplied) <= set(secret_keys) | optional_keys:
+ raise err(400, "bad_replacement",
+ "send every new secret field, optional rotation metadata, and no account or server settings")
+ replacements = {}
+ for field in secret_keys:
+ value = supplied.get(field)
+ if not isinstance(value, str) or not value.strip():
+ raise err(400, "bad_replacement", "a replacement secret cannot be blank")
+ replacements[field] = value.strip()
+
+ # Snapshot before decryption so a concurrent replacement cannot become our baseline later.
+ from copy import deepcopy
+ raw_bucket = session.runtime.get(kc.KEY) or {}
+ original = deepcopy((raw_bucket.get("entries") or {}).get(str(entry_id)))
+ if not isinstance(original, dict):
+ raise err(409, "credential_changed",
+ "the stored credential changed while replacement was prepared. Reload it.")
+ try:
+ current_fields = kc.read_fields(session.runtime, entry_id)
+ except kc.KeychainEnvelopeVersionError:
+ raise err(409, "credential_envelope_unsupported",
+ "This credential uses an unsupported encryption envelope version. Nothing was changed.")
+ except kc.KeychainLocked as exc:
+ raise err(503, "keychain_locked",
+ f"the keychain is locked: {exc}. Nothing was changed.")
+ except Exception: # noqa: BLE001
+ raise err(503, "store_unavailable",
+ "the current key could not be read. Nothing was changed.")
+ if not isinstance(current_fields, dict) or not current_fields:
+ raise err(409, "credential_unreadable",
+ "the stored credential is incomplete. Nothing was changed.")
+
+ merged_fields = {**current_fields, **replacements}
+ if entry_type == "amazon_sp":
+ # The previous secret's deadline does not describe its replacement. Absence is unknown.
+ merged_fields[AMAZON_ROTATION_DEADLINE_FIELD] = supplied.get(AMAZON_ROTATION_DEADLINE_FIELD, "")
+ try:
+ encrypted = _encrypted_replacement_payload(
+ kc, entry.get("label"), entry_type, merged_fields, session.uname)
+ except kc.KeychainLocked as exc:
+ raise err(503, "keychain_locked",
+ f"the keychain is locked: {exc}. Nothing was changed.")
+ except ValueError as exc:
+ raise err(400, "bad_replacement", str(exc))
+ except Exception: # noqa: BLE001
+ raise err(503, "replacement_not_stored",
+ "the replacement could not be encrypted. The current key is unchanged.")
+
+ def _swap(cur):
+ entries = cur.get("entries") if isinstance(cur, dict) else None
+ stored = entries.get(str(entry_id)) if isinstance(entries, dict) else None
+ if not isinstance(stored, dict) or stored != original:
+ raise _ReplacementConflict()
+ replacement = dict(stored)
+ replacement.update(encrypted)
+ replacement.pop("probe", None)
+ # Copy on write: a store failure after this callback must not mutate its cached old row.
+ return {**cur, "entries": {**entries, str(entry_id): replacement}}
+
+ try:
+ session.runtime.update(kc.KEY, _swap, flush="sync")
+ except _ReplacementConflict:
+ raise err(409, "credential_changed",
+ "the stored credential changed while replacement was prepared. Reload it.")
+ except Exception: # noqa: BLE001
+ # A remote store can fail after accepting a write. Claiming either outcome would be a lie;
+ # the safe next act is a fresh detail read, never an automatic retry or delete.
+ raise err(503, "replacement_unconfirmed",
+ "the server could not confirm whether the replacement landed. Reload this "
+ "connector before trying again.")
+ return {"ok": True}
+
+
+@router.post("/connectors/{connector_key}/keychain/{entry_id}/replace")
+def replace_connector_key(connector_key: str, entry_id: str,
+ body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Admin-only, write-only replacement for one visible connector credential."""
+ return _replace_connector_key(connector_key, entry_id, body, session)
+
+
+# ⚠ FIVE SEGMENTS, so it can collide with nothing already declared here: the literal Odoo routes
+# are four (`/connectors/odoo/schema/{field_key}`) and `/detail` is three.
+@router.patch("/connectors/{connector_key}/databases/{database_key}/name")
+def connector_database_name(connector_key: str, database_key: str,
+ body: dict = Body(default=None),
+ session: Session = Depends(require_session),
+ connection_id: str = None):
+ """W53-T27 / R9 — set or CLEAR this tenant's own name for one of a connector's databases.
+
+ ⛔⛔ IT WRITES THE BUCKET THE NAVIGATION RAIL WRITES, `nav_meta`, IN THE SAME SHAPE AND WITH
+ THE SAME PER-KEY MERGE. That is the whole ruling: *"the rename database under the Navigation
+ rail also works, it should then sync to the navigation in the connector too."* A rename made
+ here is the one the rail shows, and a rename made on the rail is the one this connector's
+ Databases tab shows, because there is one document and one reader of it
+ ([[one-question-two-normalizers]]). The key names, the length cap and the merge semantics are
+ IMPORTED from `routes_nav` rather than copied, so the two doors cannot drift apart.
+
+ ⛔ SO WHY A SECOND DOOR AT ALL? `POST /nav/meta` cannot CLEAR a name: an empty or null `name`
+ 400s there ("a database needs a name"), so "clearing the override falls back to our shipped
+ default" has nowhere to happen today. This route is that door. `name: null` and `name: ""`
+ both clear; any other string sets. The day `/nav/meta` grows the same clearing capability this
+ becomes a thin alias of it and should be folded in.
+
+ ⛔ AND IT NEVER TOUCHES A TABLE DEFINITION. `user_tables.set_label` would be rewritten by the
+ next connector re-sync (`odoo_relational.apply_plan` rebuilds definitions), so an override
+ stored there would vanish on a sync nobody connected to a rename. R9 forbids exactly that, and
+ the store choice is what makes it impossible rather than merely unlikely.
+
+ THE WALLS, in order, and each one names what it refused:
+ · the database must be one THIS connector declares (404) - the door is scoped to the card
+ the reader opened it from, so it cannot be used to rename an unrelated table;
+ · the key must be a `ut_` key (400 `name_not_allowed`, the same refusal `/nav/meta` gives) -
+ a built-in module's label is a compiled registry literal;
+ · the table must EXIST in this workspace (409) - a name for a table nobody can open would
+ sit in the document naming nothing;
+ · `may_open` - the creator, an admin, or a share grant, the same predicate `/nav/meta` uses
+ and the same one `canRename` reports, so the control and the door always agree.
+ """
+ import routes_nav # noqa: PLC0415 - the ONE nav owner
+ rt = session.runtime
+ key = str(database_key or "").strip()
+ if connector_key == "odoo":
+ key, connection_id, _choice = _odoo_database_target(rt, key, connection_id)
+ rows = _detail_databases(rt, bool(session.admin), str(connector_key or ""),
+ uname=session.uname, connection_id=connection_id)
+ row = next((candidate for candidate in rows
+ if str(candidate.get("key") or "") == key), None)
+ if row is None:
+ raise err(404, "unknown_database",
+ "this connection does not create a database with that name")
+ if not key.startswith("ut_"):
+ raise err(400, "name_not_allowed",
+ "only a database this connection created can be renamed. This one's name comes "
+ "from the module registry")
+ import core.user_tables as user_tables # noqa: PLC0415 - lazy, as elsewhere
+ definitions = user_tables.lend_defs(rt)
+ if not user_tables.get(key, st=definitions):
+ raise err(409, "database_not_created",
+ "this connection has not created that database in this workspace yet, so there "
+ "is nothing to rename")
+ if not user_tables.may_open(key, session.uname, session.admin, st=definitions):
+ raise err(403, "forbidden", "that database belongs to another user")
+
+ body = body if isinstance(body, dict) else {}
+ if "name" not in body:
+ raise err(400, "bad_request",
+ "no name was sent. Send a name to set one, or null to go back to the default")
+ # The same normalisation `/nav/meta` applies: whitespace collapsed, capped at its own limit.
+ # A blank result CLEARS, which is the half that route refuses and this one exists for.
+ name = " ".join(str(body.get("name") or "").split())[:routes_nav._MAX_NAV_NAME]
+
+ if not rt.available():
+ raise err(503, "store_unavailable",
+ "the tenant store is unavailable. Nothing was saved.")
+
+ def _up(data):
+ # `routes_nav.save_nav_meta._up`, clause for clause: an entry left empty is REMOVED rather
+ # than stored as `{}`, and an icon set from the rail survives a rename made here.
+ data = data if isinstance(data, dict) else {}
+ entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {}
+ if name:
+ entry["name"] = name
+ else:
+ entry.pop("name", None)
+ if entry:
+ data[key] = entry
+ else:
+ data.pop(key, None)
+ return data
+
+ try:
+ rt.update(routes_nav._NAV_META_KEY, _up) # noqa: SLF001 - the nav's own key
+ except Exception: # noqa: BLE001
+ raise err(503, "store_unavailable",
+ "the change was not saved: the store refused the write.")
+
+ # Read BACK through the same resolver the tab reads, so the answer cannot be the echo of a
+ # write that landed somewhere else.
+ default_label = str(row.get("defaultLabel") or "")
+ override = schema_overlay.override_name(_database_name_overrides(rt).get(key))
+ return {"key": key,
+ "identity": row.get("identity"),
+ # ⛔ SERVED, NEVER ACCEPTED. `native` is the vendor's identifier; this door reads
+ # `name` and nothing else off the body (C2: `native` is never editable).
+ "native": str(row.get("native") or ""),
+ "defaultLabel": default_label,
+ "override": override,
+ "label": schema_overlay.database_label(default_label, override)}
+
+
+# ⛔⛔ W49-T16 — THE MOUNT. `dependents_router` DECLARES NO PREFIX OF ITS OWN, so including it in
+# this router (which carries `prefix="/api/v1"`) yields exactly `/api/v1/dependents`. Declaring
+# the prefix on both would serve `/api/v1/api/v1/dependents` — the mirror image of the wave-23
+# defect this file's own header records, where the MISSING prefix 404'd a shipped route with
+# every gate green. `verify_shared_views.py` asserts the assembled path off this very object
+# rather than trusting the arithmetic.
+router.include_router(dependents_router)
+
+# ⛔⛔ W49-T19 — THE SECOND MOUNT, IDENTICAL IN SHAPE. `monid_router` is a bare `APIRouter()`,
+# so this yields `/api/v1/monid/amazon-reviews/meta`, `/api/v1/monid/amazon-reviews/run` and
+# `/api/v1/monid/spend`.
+# ⚠ THE ASSEMBLED PATH IS NOT VISIBLE IN `router.routes`. FastAPI 0.139.2 leaves an
+# `_IncludedRouter` placeholder there, so a walk of that list finds nothing and a gate written
+# against it would pass on a router that serves the paths twice-prefixed. `app.openapi()["paths"]`
+# is the reliable read, and it is what `verify_meta.py::section_monid_mount` uses.
+router.include_router(monid_router)