"""harness/runtime.py — X7: per-REQUEST tenant resolution for the shared stateless API (EXIT-4a). THE COST ARGUMENT, in one rule: *no per-tenant state may be resident in the app process.* Today each tenant carries its own interpreter, its own never-evicted pool, its own 9 prewarmed bundles and its own DuckDB — EXIT-0 measured a **~0.6 GB commit floor per tenant**, duplicated exactly (tenant B came within 2% of tenant A; nothing is shared). The target is ~30–80 MB. That is only reachable if the API resolves the tenant from the REQUEST and holds nothing tenant-shaped between requests except a bounded cache it is willing to throw away. WHAT THIS REPLACES. `ui/cache.py`'s `st.cache_resource` singleton — one process, one tenant, caches that are never evicted by design. That is correct for Streamlit and fatal for a shared process. The Streamlit side keeps its singleton untouched this wave (two processes, two caches, accepted for the strangler period; `ui/cache.py` dies with app.py at EXIT-6). THREE RULES THIS FILE OBEYS, each of which is a way the old shape leaked: 1. **No module-level per-tenant globals.** Every tenant-shaped value hangs off a TenantRuntime reached through `get_runtime(slug)`; nothing is stashed at import time. A module global is exactly what makes "which tenant is this process serving?" unanswerable. 2. **LRU-BOUNDED.** `_MAX_RUNTIMES` runtimes live at once; the least recently used is evicted. An unbounded registry cache is a memory leak with a tenant-count-shaped growth curve — the thing this whole program exists to remove. 3. **The store handle is tenant-bound**, so a write cannot land in another tenant's namespace even if a caller passes the wrong key downstream (EXIT-4b proof #3). Request cursors bind to a tenant handle. The legacy writer keeps its explicit single-store pin. """ from __future__ import annotations import os import threading from collections import OrderedDict from dataclasses import dataclass, field from pathlib import Path import core.odoo as _odoo import core.store as store from harness import datastore as _ds from harness import tenants as _tenants #: How many tenant runtimes stay resident. Small on purpose: a runtime is cheap to rebuild (a #: Tenant literal + a couple of handles) and expensive to leak. Raise only with a measurement. _MAX_RUNTIMES = int(os.environ.get("AIOS_MAX_RUNTIMES") or 32) #: slug -> builder. The literal registry `harness/tenants.py` describes; per its own docstring #: this becomes "a tenants store (DB / HF dataset)" — X4's control DB is where it goes, and #: `register()` below is the seam that makes the swap a one-liner rather than a refactor. _BUILDERS = {"royal-imports": _tenants.royal_imports} # X8's isolation fixture, admitted ONLY on request. A stub tenant that is always present is a # slug an attacker can mint a session against in production; gated, it exists exactly where it is # needed (the TestClient isolation proofs) and nowhere else. Read at import so the flag cannot be # flipped by a request. if os.environ.get("AIOS_ENABLE_QA_TENANT") == "1": _BUILDERS["qa-b"] = _tenants.qa_tenant_b _CACHE: "OrderedDict[str, TenantRuntime]" = OrderedDict() _LOCK = threading.RLock() _DELETION_OWNER_INITIALIZERS = {} def _initialize_core_deletion_owners(rt, registrar): from core import sheets initializer = getattr(sheets, 'register_amazon_deletion_owner', None) if callable(initializer): initializer(rt, registrar) def register_deletion_owner_initializer(name, initializer): """Mount an injected service owner on resident runtimes and every later rebuild.""" if not isinstance(name, str) or not name.strip() or not callable(initializer): raise ValueError('A deletion owner initializer needs a name and callback.') from harness.amazon_store import register_deletion_owner with _LOCK: _DELETION_OWNER_INITIALIZERS[name] = initializer for rt in _CACHE.values(): initializer(rt, register_deletion_owner) def _initialize_deletion_owners(rt): from harness.amazon_store import register_deletion_owner _initialize_core_deletion_owners(rt, register_deletion_owner) for initializer in tuple(_DELETION_OWNER_INITIALIZERS.values()): initializer(rt, register_deletion_owner) #: The store bucket Settings > Connectors writes pause flags into. ONE literal, shared with the #: API layer (`routes_keychain` aliases this) — the flag is written by one module and read by #: three, and two spellings of the key would be a pause that silently freezes nothing. CONNECTOR_FLAGS_KEY = "connectors" #: The flag key tenant #0's compiled ENV Odoo source is stored under. It is not a keychain entry #: id, so it needs a name of its own; `routes_keychain`'s connector row uses the same literal. ENV_ODOO_FLAG_KEY = "odoo-env" class DatastoreMismatch(RuntimeError): """A mirror handle or cursor does not belong to the requested tenant. ⛔ IT SUBCLASSES `RuntimeError` DELIBERATELY, AND THE REASON IS A LIVE TRAP RATHER THAN TIDINESS. `harness.datastore.ro_con()` already raises a bare `RuntimeError` for *"the tenant data store is completing its FIRST sync"*, and the API doors translate that into `503 store_not_ready` — i.e. "wait a few minutes and retry". A cross-tenant mismatch NEVER fixes itself by waiting: it is a pinned deployment serving the wrong file, and rendering it as a retry banner would convert the loudest refusal in the system into a spinner. So the type does two jobs at once. Being a `RuntimeError` keeps every existing `except RuntimeError` working unchanged — including `verify_api::_guard_ok`, which is what made the guard's own gate green before it had any production caller. Being a SUBCLASS lets each door catch this FIRST and answer with its own status and sentence. ⚠ Order matters at every catch site: a bare `except RuntimeError` placed above this one swallows it silently, which is the exact shape of [[defects-that-mask-each-other]]. """ def mirror_cursor(rt, *, require_ready=True): """Return this runtime's cursor, preserving readiness unless its caller gates a specific source.""" if isinstance(rt, TenantRuntime): handle = rt.mirror_handle() cur = handle.ro_con() if require_ready else handle.ro_cursor() try: rt._assert_mirror_cursor(handle, cur) except BaseException: cur.close() raise return cur # Legacy single-store adapters retain their existing assertion and readiness contract. rt.assert_datastore_matches() return _ds.ro_con() if require_ready else _ds.ro_cursor() @dataclass class TenantRuntime: """Everything a request needs to serve ONE tenant, and nothing about any other. `store_namespace` is the prefix every product-data key is written under. Tenant #0 keeps the EMPTY prefix so its existing keys (`users`, `customer_lists`, `customer_docs`, …) are found exactly where they already live — a namespacing change that renamed tenant #0's keys would be a silent data loss dressed as a refactor. Every OTHER tenant is prefixed, which is what makes proof #3 hold. """ key: str tenant: object # harness.base.Tenant store_namespace: str = "" pool_cache: dict = field(default_factory=dict) # Names/edges only, bounded and revision-keyed by routes_grid; no row pools live here. related_cache: dict = field(default_factory=dict) #: EXIT-5 — the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps #: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on #: (stamp, scope, pool identity, question) so nothing user-shaped lives in them. measure_memo: dict = field(default_factory=dict) mset_memo: dict = field(default_factory=dict) #: Wave 2026-08-02 (C-TS): the time-series channel's own memo — separate from #: measure_memo so column churn and series churn cannot evict each other's answers. series_memo: dict = field(default_factory=dict) _lock: threading.RLock = field(default_factory=threading.RLock, repr=False) @property def name(self): return getattr(self.tenant, "name", self.key) @property def datastore_path(self): """The DuckDB file this tenant's analytical reads belong to (file-per-tenant IS the isolation model — DuckDB has no RLS, so the OS boundary is the only one that holds).""" return _ds.path_for(self.key) # ---------------------------------------------------------------- the store handle #: Wave 18 (C1-TENANT, R2): a tenant that OWNS its dataset repo carries its bound Store #: instance here (namespace stays EMPTY — a whole repo needs no prefix). None = the shared #: default repo through the MODULE functions, which is what keeps the gates' fake-store #: installation effective (the fake patches module attributes, and royal-imports + the #: prefix tenants resolve through exactly those). store_handle: object = None def _store(self): return self.store_handle if self.store_handle is not None else store def _document_store(self): backend = self._store() backend = store._d() if backend is store else backend slug = getattr(backend, 'tenant_slug', None) if slug is not None and slug != self.key: raise DatastoreMismatch('The store handle belongs to another tenant.') return backend def store_key(self, name): """Namespace a product-data key for this tenant. The ONE place the prefix is applied.""" return f"{self.store_namespace}{name}" if self.store_namespace else str(name) def get(self, name, fresh=False): # A fresh read of these keys raises on a store error: a deletion obligation is never read as {}. if fresh and str(name).startswith(("sheet_scope_cache_", "sheet_amazon_deletions", "amazon_connection_revocations")): backend = self._store() reader = getattr(backend, '_read_strict', None) if callable(reader): return reader(self.store_key(name)) return self._store().get(self.store_key(name), fresh=fresh) def get_projection(self, name, drop=()): """A tenant-scoped PROJECTED read — the bucket without the keys named in `drop`. ⭐⭐ W32-T01 (D-185). `get()` deep-copies the whole document; on tenant #0's `user_tables` that is 28.6 MB / ~703 ms warm, and 99.9% of it is `rows` that no nav render, permission check or workspace envelope reads. ⛔ **`_store()` returns the MODULE for every tenant with `store_handle is None`** — which includes tenant #0 — so this delegates to a MODULE-LEVEL `get_projection`, not to a method. Both backends implement it and `verify_store_pg.IFACE` pins that they must; see `core.store.get_projection`'s note for why the method alone would have been unreachable from exactly the caller it was built for. """ return self._store().get_projection(self.store_key(name), drop=drop) def get_projections(self, names, drop=(), timeout_s=None): """Batch projected reads when the active bound store has a native batch operation. ``None`` means the backend has no batch primitive; callers retain their ordinary per-key path. That keeps this an optional optimization rather than widening the store contract. """ reader = getattr(self._store(), "get_projections", None) if not callable(reader): return None keyed = [(str(name), self.store_key(name)) for name in (names or ())] answer = reader([store_key for _name, store_key in keyed], drop=drop, timeout_s=timeout_s) return {name: answer.get(store_key, {}) for name, store_key in keyed} def get_workbook_projection(self, name, *, metadata_only=False, tab_ids=None, cells=None, expected_rev=None): """Read one tenant's selected workbook cells without caching permissions or source answers.""" reader = getattr(self._document_store(), 'get_workbook_projection', None) if callable(reader): return reader(self.store_key(name), metadata_only=metadata_only, tab_ids=tab_ids, cells=cells, expected_rev=expected_rev) metadata_only, tab_ids, cells = store._workbook_selection( metadata_only=metadata_only, tab_ids=tab_ids, cells=cells) return store._project_workbook(self.get(name), metadata_only=metadata_only, tab_ids=tab_ids, cells=cells, expected_rev=expected_rev) def list_keys(self, prefix=''): names = self._document_store().list_keys(self.store_key(prefix)) return [name[len(self.store_namespace):] for name in names if name.startswith(self.store_namespace)] def list_paths(self, prefix=''): paths = self._document_store().list_paths(self.store_key(prefix)) return [path[len(self.store_namespace):] for path in paths if path.startswith(self.store_namespace)] def delete_document(self, name): return self._document_store().delete_document(self.store_key(name)) def amazon_backup_policy(self, *, trigger_at, checked_at): """Delegate backup evidence only to this runtime's tenant-bound backend.""" backend = self._document_store() reader = getattr(backend, 'amazon_backup_policy', None) if callable(reader): policy = reader(trigger_at=trigger_at, checked_at=checked_at) if not isinstance(policy, dict) or policy.get('tenant') != self.key: raise DatastoreMismatch('Backup evidence belongs to another tenant.') return policy return {'tenant': self.key, 'fixture': False, 'state': 'UNPROVEN', 'policyId': None, 'source': None, 'method': None, 'expiresAt': None, 'verifiedExpiredAt': None, 'cause': 'tenant_backup_policy_unavailable'} def get_many(self, names): """Batch ordinary reads when the active bound store offers that optimization.""" reader = getattr(self._store(), "get_many", None) if not callable(reader): return None keyed = [(str(name), self.store_key(name)) for name in (names or ())] answer = reader([store_key for _name, store_key in keyed]) return {name: answer.get(store_key, {}) for name, store_key in keyed} def get_entry(self, name, entry): """One top-level entry of `name` when the bound store offers it, else None (HOTFIX 2026-09-28). None also means absent; either way the caller falls back to `get`.""" reader = getattr(self._store(), "get_entry", None) if not callable(reader): return None return reader(self.store_key(name), entry) def revision(self, name): """Return the tiny durable change token for this tenant's addressed bucket.""" return self._store().revision(self.store_key(name)) def put(self, name, data): return self._store().put(self.store_key(name), data) def transaction(self, connection): """Enlist this tenant's writes in an existing receipt transaction.""" return self._document_store().transaction(connection) def update(self, name, fn, flush="sync", expect_rev=None, audit_row=None): """Read-modify-write this tenant's bucket, optionally rev-guarded and audited (C3 / R9). ⭐⭐ W50-T28 CLAUSE 1 — THE TWO KEYWORDS EXIST SO `control.audit` CAN BE REACHED AT ALL. `core/store_pg.py::update` grew `expect_rev` and `audit_row` in W50-T25 and every API caller passes `st=session.runtime`, so until this method NAMED them the capability was written, gated and unreachable — the [[tenant-runtime-update-is-a-keyword-bottleneck]] shape. `expect_rev` refuses a write whose caller was looking at an older revision; `audit_row` is the LIST the caller's own closure fills, carried into the same transaction as the workspace write so the trail commits with the change or not at all. ⛔⛔ THE FORWARD IS CONDITIONAL, AND THAT IS THE ENTIRE CARE IN THIS METHOD. `_store()` returns the `core.store` MODULE for every tenant with `store_handle is None` — which includes tenant #0 — and that module's `update(name, fn, flush='sync')` names neither keyword. An unconditional forward is a `TypeError` on the hot path for exactly the tenant that matters most. ⛔⛔ AND A SIGNATURE IS A CAPABILITY DECLARATION HERE, NOT A CONVENIENCE. `view_audit.home()` and `view_audit.guards()` answer by inspecting THIS method's parameters, so merely naming the keywords tells the whole repo that every tenant can carry a row into its transaction and enforce a revision inside its own lock. Where the inner handle cannot, both promises must be kept HERE or they are broken silently: * `expect_rev` -> the ADVISORY check (read `revision()`, compare, refuse). Identical to the fallback `table_store._audited_update` used to take on its own, which this signature change is what disables. It has a real TOCTOU window and still refuses every stale write it can see. * `audit_row` -> appended to the `__audit__` DOCUMENT ring inside the caller's closure, which is where `view_audit.record` would have put it had `home()` still said `DOCUMENT`. Without this the row is not relocated, it is LOST. ⚠ WHAT THIS DOES NOT CLOSE, and the handoff says so: tenant #0 resolves through the `core.store` MODULE function, which drops both keywords before `_d()` ever reaches `PgStore.update`. That tenant therefore takes the compensating path above and its trail stays in the ring. The second bottleneck is `core/store.py::update`, in nobody's fence. """ st = self._store() slug = getattr(st, 'tenant_slug', None) if slug is not None and slug != self.key: raise DatastoreMismatch('The store handle belongs to another tenant.') key = self.store_key(name) if expect_rev is None and audit_row is None: return st.update(key, fn, flush=flush) # Imported HERE, not at module scope: `api/main.py` imports half the platform before it # has read an environment variable, and `core.view_audit` reaches `core.store_pg`. from core import view_audit if view_audit.guards(st) and view_audit.home(st) == view_audit.CONTROL_AUDIT: return st.update(key, fn, flush=flush, expect_rev=expect_rev, audit_row=audit_row) if expect_rev is not None: self._refuse_stale_revision(name, expect_rev) if audit_row is not None: fn = self._appending_audit_rows(fn, audit_row) return st.update(key, fn, flush=flush) def _refuse_stale_revision(self, name, expect_rev): """Raise `ViewRevisionStale` when this bucket has moved past `expect_rev`. The ADVISORY half of the guard, for a handle that cannot enforce one inside its own locking read. `RevisionUnverifiable` rather than a silent pass when the revision cannot be read at all: "I could not tell" and "it matches" are different answers, and only one of them may let a write land. """ from core import view_audit if not callable(getattr(self._store(), "revision", None)): raise view_audit.RevisionUnverifiable( "this deployment cannot tell which revision of the view list you are looking " "at, so the change was not saved. Reload and try again.") try: current = int((self.revision(name) or {}).get("rev") or 0) except Exception as exc: # noqa: BLE001 raise view_audit.RevisionUnverifiable( "the view list revision could not be read, so the change was not saved. " "Reload and try again.") from exc if current != int(expect_rev): raise view_audit.ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev)) @staticmethod def _appending_audit_rows(fn, rows): """Wrap `fn` so the rows it collected land in the document ring, in its own transaction. ⚠ `rows` IS EMPTY WHEN THIS RUNS AND FULL AFTERWARDS — the caller's closure appends to it while it executes, which is the only moment a row's `before` and `after` are both knowable. So the append happens after `fn` returns and reads the list then, never now. """ def _apply(data): from core import view_audit result = fn(data) target = result if isinstance(result, dict) else data for entry in rows: view_audit.append_to_document(target, entry) return result return _apply def exists(self, name): return self._store().exists(self.store_key(name)) def available(self): return self._store().available() # ---------------------------------------------------------------- BYTES (wave 25, C's ask) # ⭐ WHY THESE EXIST: `grid_events._blobs()` had no tenant-scoped way to reach bytes, so an R2 # tenant's UPLOADED FILES landed in the shared repo while its buckets went to its own — the # residency fix with a hole in it. SESSION C wrote `_blobs` as a CAPABILITY TEST # (`hasattr(st, "upload_bytes")`) precisely so the fallback disappears the moment these land, # with no edit on that side. # # ⛔ THEY GO THROUGH `store_key`, AND THAT IS THE WHOLE POINT. A byte path is a different # address space from a bucket name, and the temptation is to pass it straight through — but # the isolation requirement is identical: a tenant on the SHARED repo (namespace `t//`) # writing `uploads/photo.png` would otherwise land on the exact path another tenant writes. # An own-repo tenant has an EMPTY namespace, so the path is untouched and the repo boundary # does the work, which is the same asymmetry every method above already has. Reusing # `store_key` keeps "the ONE place the prefix is applied" literally true rather than nearly. def upload_bytes(self, path_in_repo, data, message=None): return self._store().upload_bytes(self.store_key(path_in_repo), data, message=message) def download_bytes(self, path_in_repo): return self._store().download_bytes(self.store_key(path_in_repo)) def delete_path(self, path_in_repo): return self._store().delete_path(self.store_key(path_in_repo)) # ---------------------------------------------------------------- the Odoo source (R3) def odoo_source(self, connection_id=None): """THIS tenant's Odoo connector, or None — the keychain CUTOVER's resolution seam (2026-08-04; `keychain.odoo_creds` was previously called by nothing but its test). Resolution, fail-closed: 1. the keychain `odoo` connection NAMED by `connection_id` (W54-T45 / R4), or the tenant's DESIGNATED DEFAULT connection when no id is named → a connector bound to THOSE creds; 2. tenant #0 with an empty/locked keychain and NO id named → its compiled env connector (R3's "live with env fallback", which keeps Royal working unchanged); 3. anyone else → None. NEVER the environment: env is tenant #0's connection, and handing it to another tenant is the leak this method exists to prevent. ⛔⛔ W54-T45 — THE ENV FALLBACK NARROWS HERE AND MUST NEVER WIDEN. A caller that NAMES a connection and gets nothing back is answered None, full stop: not the default connection, not a sibling, and above all not tenant #0's `.env`. Under R4 an agency holds one Odoo connection per client, and under R7 that isolation is strict — a named-but-unresolvable connection quietly falling through to the environment would serve tenant #0's own Odoo to a question asked about somebody else's books, with nothing red anywhere. Leg (2) is reachable only on the UNNAMED path, which is exactly the pre-wave behaviour. Rebuilt per call on purpose — `core.odoo` caches the CLIENT per (slug, creds fingerprint), so this stays cheap while a rotated key reconnects on the next call.""" import core.keychain as keychain named = connection_id is not None try: # ⚠ THE UNNAMED PATH IS SPELLED AS THE PRE-WAVE CALL, VERBATIM, and the branch is not # decoration: `verify_api`'s R11 leg pins the literal `keychain.odoo_creds(self)` in # this method's source as its proof that the resolver is KEYCHAIN-FIRST. Writing only # the two-argument form would have turned that gate red for a reason that has nothing # to do with the rule it guards, in a file this lane may not edit. The two arms are # the same question; the left one is the one every existing caller asks. creds = (keychain.odoo_creds(self) if not named else keychain.odoo_creds(self, connection_id)) except Exception: # locked or unreadable keychain — fall through, fail closed creds = None if creds: from harness.connectors.odoo import OdooConnector cfg = getattr(self.tenant, "config", {}) or {} odoo_cfg = self.get(_ds._ODOO_CONFIG_KEY) or {} source = OdooConnector(tenant_slug=self.key, creds=creds, scope=team_scope(cfg.get("business_units"), cfg.get("team_pricelists"))) return _business_source(source, odoo_cfg.get('businessTz')) # ⛔ THE NAMED LEG STOPS HERE. See the docstring: an explicit connection that did not # resolve is None, never the environment and never another connection. if named: return None if self.key == "royal-imports": source = (getattr(self.tenant, "sources", {}) or {}).get("odoo") return _business_source(source, (self.get(_ds._ODOO_CONFIG_KEY) or {}).get('businessTz')) return None # ------------------------------------------------------- the connector PAUSE (D-10, wave 24) def odoo_flag_key(self, connection_id=None): """WHICH connector flag governs this tenant's Odoo — the key `connectors[].paused` is stored under, or None when nothing would serve. THE SAME RESOLUTION AS `odoo_source()` ABOVE, expressed as an identity instead of a client: the keychain `odoo` connection named (or the designated default when none is), else tenant #0's compiled env source, else nothing. It lives here so there is ONE resolver — `routes_keychain._resolved_odoo_key` was a second copy of these three rules in the API layer, and a pause flag written against one resolution and read against the other freezes nothing while reporting success. ⛔⛔ W54-T45 — AND THE INLINE WALK IS GONE, WHICH IS WHAT MADE THE SENTENCE ABOVE TRUE. This method claimed to be the one resolver while carrying its own copy of "the first `odoo` entry", so `keychain` and `runtime` each decided which connection serves. Both now ask `keychain.resolve_connection_id`, which decrypts nothing — a pause flag has to resolve on a LOCKED keychain, which is why this is an identity call and not a credential one. ⛔ A NAMED CONNECTION THAT DOES NOT RESOLVE IS None, exactly as in `odoo_source`: pausing by a connection id must never silently address tenant #0's environment source instead. """ import core.keychain as keychain named = connection_id is not None try: resolved = keychain.resolve_connection_id(self, "odoo", connection_id) except Exception: resolved = None if resolved and keychain.unlocked(): return resolved if named: return None if self.key == "royal-imports" and os.environ.get("ODOO_URL"): return ENV_ODOO_FLAG_KEY return None def odoo_paused(self, connection_id=None): """Is THIS tenant's RESOLVED Odoo source paused? ⚠ RESOLVED, not "any entry" — pausing an entry that is not the one serving freezes nothing, because it serves nothing. Never raises: an unreachable flags bucket answers False, because a store hiccup must not freeze a live surface. That policy is inherited from the API-layer function this one replaced, deliberately, so there is one answer to "what happens when we cannot tell" rather than two that can disagree. ⭐ W54-T45 — `connection_id` passes straight through to `odoo_flag_key`, so "is the connection serving THIS client paused" is answerable per connection under R4. No id keeps meaning the tenant's designated default, which is every existing caller. """ try: flag_key = self.odoo_flag_key(connection_id) if not flag_key: return False flags = self.get(CONNECTOR_FLAGS_KEY) or {} return bool((flags.get(flag_key) or {}).get("paused")) except Exception: return False def mirror_handle(self): """Resolve only this tenant's configured mirror, even when its file is absent.""" return _ds.tenant_handle(self.key, self.datastore_path) def assert_datastore_matches(self, cur=None): """Refuse a handle or borrowed cursor belonging to another tenant.""" handle = self.mirror_handle() self._assert_mirror_cursor(handle, cur) def _assert_mirror_cursor(self, handle, cur): """Validate a borrowed cursor against the handle acquired for this operation.""" if handle.tenant_key != self.key: raise DatastoreMismatch("The mirror handle belongs to another tenant.") if cur is None: return bound = getattr(cur, "_mirror_handle", None) if bound is not None: if bound.tenant_key != self.key or bound.path != handle.path: raise DatastoreMismatch("The mirror cursor belongs to another tenant.") return marks = _ds.mirror_file_marks(cur) expected = _ds.serving_file(handle.path)[0] if _ds.versioned_mode() else handle.path if marks is None or marks[0] != str(expected.resolve()): raise DatastoreMismatch("The borrowed mirror cursor belongs to another tenant.") @staticmethod def _store_owner(path): """Return the forward owner and whether the registry and path mapping prove isolation. ⛔ FORWARD-MAPPED, never parsed out of the filename — `datastore.path_for` substitutes `_` for every non-alphanumeric character and truncates at 60, so `a.b` and `a_b` produce the same file and the name is genuinely not invertible. `tenant_for_store_path` above says the same thing and is not reused here for one reason: it swallows every failure into `""`, and this caller must be able to tell "nobody owns it" from "I could not find out". ⚠ AND `known_tenants()` IS NOT REUSED EITHER, FOR THE SAME REASON ONE LAYER DOWN — it cannot raise: `_tenant_records()` returns `{}` on any failure and `known_tenants` wraps that in another `except: pass`. Calling it here would have made `registry_ok` a constant `True` and the fail-closed branch above unreachable, i.e. a guard whose most important clause is dead code that reads as if it fires ([[gate-can-report-green-on-nothing]]). The control-plane bucket is therefore read HERE, where its failure is still visible. """ keys, registry_ok = set(_BUILDERS), True try: registries = [store.get(TENANTS_KEY)] # HF display failures cannot prove uniqueness. Confirm through its strict reader. if store.backend() != "pg": registries.append(store._read_strict(TENANTS_KEY)) for recs in registries: recs = {} if recs is None else recs if not isinstance(recs, dict) or any(not isinstance(v, dict) for v in recs.values()): raise ValueError("The tenant ownership registry is malformed.") keys |= {str(k).strip().lower() for k in recs} except Exception: # noqa: BLE001 registry_ok = False owners = [] for key in sorted(keys): try: if _ds.path_for(key).resolve() == path: owners.append(key) except Exception: # noqa: BLE001 registry_ok = False continue return (owners[0] if len(owners) == 1 else "ambiguous" if owners else ""), registry_ok and len(owners) <= 1 def register(key, builder): """Add a tenant to the registry (a stub connector is a legitimate builder — see X8's qa-b). Invalidates any cached runtime for the slug so a re-registration cannot be shadowed.""" _BUILDERS[str(key)] = builder invalidate(key) def worker_tenant(): """The one tenant a per-tenant sync worker serves, or "" in every other process.""" if (os.environ.get("AIOS_PROCESS_ROLE") != "worker" or os.environ.get("AIOS_WORKER_SCOPE") != "tenant"): return "" return str(os.environ.get("AIOS_SYNC_WRITER_TENANT") or "").strip().lower() def known_tenants(): keys = set(_BUILDERS) try: keys |= set(_tenant_records()) except Exception: pass # A per-tenant sync worker holds one tenant's file, so it never sees another tenant. only = worker_tenant() if only: keys &= {only} return sorted(keys) #: Wave 18 (C1-TENANT) — the CONTROL-PLANE bucket: tenants provisioned at runtime rather than #: compiled in. Lives in the DEFAULT store (tenant #0's repo) deliberately: it answers "which #: tenants exist", which is a platform fact, not any one tenant's data. TENANTS_KEY = "tenants" def _tenant_records(): """{slug: record} from the control-plane bucket. {} on any failure — an unreachable bucket must degrade to "only the compiled-in tenants exist", never to an exception at login.""" try: recs = store.get(TENANTS_KEY) or {} return {str(k).strip().lower(): v for k, v in recs.items() if isinstance(v, dict)} except Exception: return {} def _builder_from_record(key, rec): """A Tenant for a bucket-provisioned client: no sources yet (connectors arrive via the Keychain flow), config carries the platform fields the request path reads.""" def _build(): from harness.base import Tenant return Tenant(key=key, name=str(rec.get("name") or key), config={"modules": rec.get("modules", []), "store_repo": rec.get("store_repo") or None}) return _build def get_runtime(tenant_key): """The runtime for `tenant_key`, built on first use and LRU-cached. Raises KeyError for an unknown slug — the caller turns that into a 401 (fail-closed: an unknown tenant is not a 404, because confirming which slugs exist answers a question the request was not entitled to ask). """ key = str(tenant_key or "").strip().lower() if not key: raise KeyError("no tenant") only = worker_tenant() if only and key != only: # Fail closed: this process's DuckDB file belongs to `only`. raise KeyError(key) with _LOCK: rt = _CACHE.get(key) if rt is not None: _CACHE.move_to_end(key) # LRU: touch on read return rt builder = _BUILDERS.get(key) # Wave 18 (C1-TENANT): a slug the compiled registry does not know may be a PROVISIONED # tenant — consult the control-plane bucket before failing closed. A suspended record is # the same KeyError as an unknown one: "which tenants exist" is not this caller's question. if builder is None: rec = _tenant_records().get(key) if not rec or rec.get("status", "active") != "active": raise KeyError(key) builder = _builder_from_record(key, rec) # Built OUTSIDE the lock: a builder may do real work (a connector handshake), and holding the # registry lock through it would serialise every other tenant's requests behind it. tenant = builder() with _LOCK: existing = _CACHE.get(key) if existing is not None: # another thread won the race — keep ONE runtime _CACHE.move_to_end(key) return existing # R2: a tenant with its OWN repo gets a bound Store instance and an EMPTY prefix (the # repo boundary is the namespace); everyone else keeps the t// prefix in the # shared repo. Tenant #0 keeps both defaults — empty prefix, shared repo. # # ⭐ WAVE 20 (R1 / D-4): under Postgres the address is a SCHEMA, so `store.handle()` is # asked for one by SLUG and the namespace prefix goes to EMPTY for EVERY tenant — # `t_.store_kv` already isolates, and prefixing keys inside an isolated schema # would namespace them twice. That is not a cosmetic difference: a `t/nurilab/`-prefixed # key written into `t_nurilab` is a key the migration did not copy and no reader looks # for, i.e. a silent empty workspace. # # ⚠ NOTHING ELSE IN THIS FILE CHANGES, and that is the design working. Every product-data # read/write already goes through `rt.get/put/update/exists` (the `st=` handle threaded # through the modules), so binding the handle correctly here IS the cutover for the # tenant-aware paths — the module-level `core.store` functions cover the rest via `_d()`. repo = (getattr(tenant, "config", {}) or {}).get("store_repo") if store.backend() == "pg": rt = TenantRuntime(key=key, tenant=tenant, store_namespace="", store_handle=store.handle(slug=key)) else: rt = TenantRuntime( key=key, tenant=tenant, store_namespace=("" if (key == "royal-imports" or repo) else f"t/{key}/"), store_handle=(store.for_repo(repo) if repo else None)) _initialize_deletion_owners(rt) _CACHE[key] = rt while len(_CACHE) > _MAX_RUNTIMES: _CACHE.popitem(last=False) # evict least recently used return rt def invalidate(tenant_key=None): """Drop one tenant's runtime, or all of them. Explicit, per X7 — a cache with no documented way to clear it becomes a restart.""" with _LOCK: if tenant_key is None: _CACHE.clear() else: _CACHE.pop(str(tenant_key or "").strip().lower(), None) def resident(): """The tenant slugs currently holding a runtime — LRU order, oldest first. For the EXIT-4 measurement and for asserting the bound actually binds.""" with _LOCK: return list(_CACHE) # ───────────────────────────────────────────────────────────────────────────────────────────── # ⭐ DEBT D-10 (wave 24) — TEACHING THE MEASURE MIRROR ABOUT THE CONNECTOR PAUSE. # # `harness.datastore` is the store every measure column and condition is answered from, and it # pulled from Odoo straight through a pause: 12 sync passes at boot and one every 1800 s. It # cannot ask the question itself — the flag lives in a tenant's store bucket, which is this # module's to read, and this module already imports datastore, so an import back would be a # cycle. So datastore exposes a probe slot and this module fills it, once, at import. def tenant_for_store_path(path=None): """Which tenant owns the DuckDB file currently open (or `path`)? `""` when nothing matches. ⛔ RESOLVED BY FORWARD-MAPPING every known tenant, never by parsing the filename, and that is a correctness point rather than a style one: `datastore.path_for` substitutes `_` for every non-alphanumeric character and truncates at 60, so the name is genuinely NOT invertible — `a.b` and `a_b` produce the same file. Guessing the slug back out of it would eventually name the wrong tenant, and this answer decides whether another tenant's connector freezes ours. """ want = Path(path or _ds.DB_PATH).resolve() # Tenant #0 first: it is the only tenant the unattended sync actually runs for today (see # D-29 — the resync loop prewarms `royal-imports` hardcoded), so this is the hot answer. for key in ["royal-imports"] + [k for k in known_tenants() if k != "royal-imports"]: try: if _ds.path_for(key).resolve() == want: return key except Exception: continue return "" def store_source_paused(): """The probe `harness.datastore` calls before reaching Odoo. Never raises; False when the tenant cannot be resolved, matching the fail-towards-live policy `odoo_paused` documents.""" try: key = tenant_for_store_path() return bool(key) and get_runtime(key).odoo_paused() except Exception: return False _ds.set_paused_probe(store_source_paused) def _business_source(source, zone): """Adapt the connector's explicit binding without mutating its shared tenant source.""" if source is None: return None import copy result = copy.copy(source) result._bound = lambda: _odoo.tenant_scope( result._slug, result._creds, **result._scope, business_tz=zone) return result def team_scope(business_units, team_pricelists=None): """A tenant config's `business_units` ({label: team id}) and `team_pricelists` ({label: lists}) as `core.odoo.tenant_scope` kwargs. A pricelist row whose label names no team is dropped.""" bus = dict(business_units or {}) lists = {bus[label]: tuple(names) for label, names in (team_pricelists or {}).items() if label in bus} return {"team_ids": list(bus.values()) or None, "team_names": {v: k for k, v in bus.items()} or None, "team_pricelists": lists or None} def mirror_team_names(slug): """{team_id: Odoo team name} from a RESIDENT tenant's mirror, read only; {} when none is. Never builds a runtime: a team lookup must not reach the store or the network.""" with _LOCK: rt = _CACHE.get(str(slug or "").strip().lower()) if rt is None: return {} cur = mirror_cursor(rt) try: try: rows = cur.execute("SELECT team_id, max(team_name) FROM sale_order " "WHERE team_id IS NOT NULL GROUP BY team_id").fetchall() except Exception: # a mirror without the name column still lists its team ids rows = [(t, None) for (t,) in cur.execute( "SELECT DISTINCT team_id FROM sale_order WHERE team_id IS NOT NULL").fetchall()] finally: cur.close() return {int(t): (str(n) if n else str(t)) for t, n in rows} # core may not import harness: push the default tenant's configured teams and the mirror reader # down at process start, the same way `set_paused_probe` above reaches the datastore. _odoo.set_default_scope(**team_scope(_tenants.TENANT0_BUSINESS_UNITS, _tenants.TENANT0_TEAM_PRICELISTS)) _odoo.set_team_reader(mirror_team_names)