Download platform/harness/runtime.py from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 44.2 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/platform/harness/runtime.py
- Command line
-
hf download hf://spaces/fsanyoto/loopable@9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/platform/harness/runtime.py
-
curl -L -o runtime.py https://huggingface.co/spaces/fsanyoto/loopable/resolve/9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/platform/harness/runtime.py
44.2 kB
| """harness/runtime.py β X7: per-REQUEST tenant resolution for the shared stateless API (EXIT-4a). | |
| THE COST ARGUMENT, in one rule: *no per-tenant state may be resident in the app process.* Today | |
| each tenant carries its own interpreter, its own never-evicted pool, its own 9 prewarmed bundles | |
| and its own DuckDB β EXIT-0 measured a **~0.6 GB commit floor per tenant**, duplicated exactly | |
| (tenant B came within 2% of tenant A; nothing is shared). The target is ~30β80 MB. That is only | |
| reachable if the API resolves the tenant from the REQUEST and holds nothing tenant-shaped between | |
| requests except a bounded cache it is willing to throw away. | |
| WHAT THIS REPLACES. `ui/cache.py`'s `st.cache_resource` singleton β one process, one tenant, | |
| caches that are never evicted by design. That is correct for Streamlit and fatal for a shared | |
| process. The Streamlit side keeps its singleton untouched this wave (two processes, two caches, | |
| accepted for the strangler period; `ui/cache.py` dies with app.py at EXIT-6). | |
| THREE RULES THIS FILE OBEYS, each of which is a way the old shape leaked: | |
| 1. **No module-level per-tenant globals.** Every tenant-shaped value hangs off a TenantRuntime | |
| reached through `get_runtime(slug)`; nothing is stashed at import time. A module global is | |
| exactly what makes "which tenant is this process serving?" unanswerable. | |
| 2. **LRU-BOUNDED.** `_MAX_RUNTIMES` runtimes live at once; the least recently used is evicted. | |
| An unbounded registry cache is a memory leak with a tenant-count-shaped growth curve β the | |
| thing this whole program exists to remove. | |
| 3. **The store handle is tenant-bound**, so a write cannot land in another tenant's namespace | |
| even if a caller passes the wrong key downstream (EXIT-4b proof #3). | |
| Request cursors bind to a tenant handle. The legacy writer keeps its explicit single-store pin. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import threading | |
| from collections import OrderedDict | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| import core.odoo as _odoo | |
| import core.store as store | |
| from harness import datastore as _ds | |
| from harness import tenants as _tenants | |
| #: How many tenant runtimes stay resident. Small on purpose: a runtime is cheap to rebuild (a | |
| #: Tenant literal + a couple of handles) and expensive to leak. Raise only with a measurement. | |
| _MAX_RUNTIMES = int(os.environ.get("AIOS_MAX_RUNTIMES") or 32) | |
| #: slug -> builder. The literal registry `harness/tenants.py` describes; per its own docstring | |
| #: this becomes "a tenants store (DB / HF dataset)" β X4's control DB is where it goes, and | |
| #: `register()` below is the seam that makes the swap a one-liner rather than a refactor. | |
| _BUILDERS = {"royal-imports": _tenants.royal_imports} | |
| # X8's isolation fixture, admitted ONLY on request. A stub tenant that is always present is a | |
| # slug an attacker can mint a session against in production; gated, it exists exactly where it is | |
| # needed (the TestClient isolation proofs) and nowhere else. Read at import so the flag cannot be | |
| # flipped by a request. | |
| if os.environ.get("AIOS_ENABLE_QA_TENANT") == "1": | |
| _BUILDERS["qa-b"] = _tenants.qa_tenant_b | |
| _CACHE: "OrderedDict[str, TenantRuntime]" = OrderedDict() | |
| _LOCK = threading.RLock() | |
| _DELETION_OWNER_INITIALIZERS = {} | |
| def _initialize_core_deletion_owners(rt, registrar): | |
| from core import sheets | |
| initializer = getattr(sheets, 'register_amazon_deletion_owner', None) | |
| if callable(initializer): | |
| initializer(rt, registrar) | |
| def register_deletion_owner_initializer(name, initializer): | |
| """Mount an injected service owner on resident runtimes and every later rebuild.""" | |
| if not isinstance(name, str) or not name.strip() or not callable(initializer): | |
| raise ValueError('A deletion owner initializer needs a name and callback.') | |
| from harness.amazon_store import register_deletion_owner | |
| with _LOCK: | |
| _DELETION_OWNER_INITIALIZERS[name] = initializer | |
| for rt in _CACHE.values(): | |
| initializer(rt, register_deletion_owner) | |
| def _initialize_deletion_owners(rt): | |
| from harness.amazon_store import register_deletion_owner | |
| _initialize_core_deletion_owners(rt, register_deletion_owner) | |
| for initializer in tuple(_DELETION_OWNER_INITIALIZERS.values()): | |
| initializer(rt, register_deletion_owner) | |
| #: The store bucket Settings > Connectors writes pause flags into. ONE literal, shared with the | |
| #: API layer (`routes_keychain` aliases this) β the flag is written by one module and read by | |
| #: three, and two spellings of the key would be a pause that silently freezes nothing. | |
| CONNECTOR_FLAGS_KEY = "connectors" | |
| #: The flag key tenant #0's compiled ENV Odoo source is stored under. It is not a keychain entry | |
| #: id, so it needs a name of its own; `routes_keychain`'s connector row uses the same literal. | |
| ENV_ODOO_FLAG_KEY = "odoo-env" | |
| class DatastoreMismatch(RuntimeError): | |
| """A mirror handle or cursor does not belong to the requested tenant. | |
| β IT SUBCLASSES `RuntimeError` DELIBERATELY, AND THE REASON IS A LIVE TRAP RATHER THAN | |
| TIDINESS. `harness.datastore.ro_con()` already raises a bare `RuntimeError` for *"the tenant | |
| data store is completing its FIRST sync"*, and the API doors translate that into | |
| `503 store_not_ready` β i.e. "wait a few minutes and retry". A cross-tenant mismatch NEVER | |
| fixes itself by waiting: it is a pinned deployment serving the wrong file, and rendering it as | |
| a retry banner would convert the loudest refusal in the system into a spinner. | |
| So the type does two jobs at once. Being a `RuntimeError` keeps every existing | |
| `except RuntimeError` working unchanged β including `verify_api::_guard_ok`, which is what | |
| made the guard's own gate green before it had any production caller. Being a SUBCLASS lets | |
| each door catch this FIRST and answer with its own status and sentence. β Order matters at | |
| every catch site: a bare `except RuntimeError` placed above this one swallows it silently, | |
| which is the exact shape of [[defects-that-mask-each-other]]. | |
| """ | |
| def mirror_cursor(rt, *, require_ready=True): | |
| """Return this runtime's cursor, preserving readiness unless its caller gates a specific source.""" | |
| if isinstance(rt, TenantRuntime): | |
| handle = rt.mirror_handle() | |
| cur = handle.ro_con() if require_ready else handle.ro_cursor() | |
| try: | |
| rt._assert_mirror_cursor(handle, cur) | |
| except BaseException: | |
| cur.close() | |
| raise | |
| return cur | |
| # Legacy single-store adapters retain their existing assertion and readiness contract. | |
| rt.assert_datastore_matches() | |
| return _ds.ro_con() if require_ready else _ds.ro_cursor() | |
| class TenantRuntime: | |
| """Everything a request needs to serve ONE tenant, and nothing about any other. | |
| `store_namespace` is the prefix every product-data key is written under. Tenant #0 keeps the | |
| EMPTY prefix so its existing keys (`users`, `customer_lists`, `customer_docs`, β¦) are found | |
| exactly where they already live β a namespacing change that renamed tenant #0's keys would be | |
| a silent data loss dressed as a refactor. Every OTHER tenant is prefixed, which is what makes | |
| proof #3 hold. | |
| """ | |
| key: str | |
| tenant: object # harness.base.Tenant | |
| store_namespace: str = "" | |
| pool_cache: dict = field(default_factory=dict) | |
| # Names/edges only, bounded and revision-keyed by routes_grid; no row pools live here. | |
| related_cache: dict = field(default_factory=dict) | |
| #: EXIT-5 β the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps | |
| #: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on | |
| #: (stamp, scope, pool identity, question) so nothing user-shaped lives in them. | |
| measure_memo: dict = field(default_factory=dict) | |
| mset_memo: dict = field(default_factory=dict) | |
| #: Wave 2026-08-02 (C-TS): the time-series channel's own memo β separate from | |
| #: measure_memo so column churn and series churn cannot evict each other's answers. | |
| series_memo: dict = field(default_factory=dict) | |
| _lock: threading.RLock = field(default_factory=threading.RLock, repr=False) | |
| def name(self): | |
| return getattr(self.tenant, "name", self.key) | |
| def datastore_path(self): | |
| """The DuckDB file this tenant's analytical reads belong to (file-per-tenant IS the | |
| isolation model β DuckDB has no RLS, so the OS boundary is the only one that holds).""" | |
| return _ds.path_for(self.key) | |
| # ---------------------------------------------------------------- the store handle | |
| #: Wave 18 (C1-TENANT, R2): a tenant that OWNS its dataset repo carries its bound Store | |
| #: instance here (namespace stays EMPTY β a whole repo needs no prefix). None = the shared | |
| #: default repo through the MODULE functions, which is what keeps the gates' fake-store | |
| #: installation effective (the fake patches module attributes, and royal-imports + the | |
| #: prefix tenants resolve through exactly those). | |
| store_handle: object = None | |
| def _store(self): | |
| return self.store_handle if self.store_handle is not None else store | |
| def _document_store(self): | |
| backend = self._store() | |
| backend = store._d() if backend is store else backend | |
| slug = getattr(backend, 'tenant_slug', None) | |
| if slug is not None and slug != self.key: | |
| raise DatastoreMismatch('The store handle belongs to another tenant.') | |
| return backend | |
| def store_key(self, name): | |
| """Namespace a product-data key for this tenant. The ONE place the prefix is applied.""" | |
| return f"{self.store_namespace}{name}" if self.store_namespace else str(name) | |
| def get(self, name, fresh=False): | |
| # A fresh read of these keys raises on a store error: a deletion obligation is never read as {}. | |
| if fresh and str(name).startswith(("sheet_scope_cache_", "sheet_amazon_deletions", | |
| "amazon_connection_revocations")): | |
| backend = self._store() | |
| reader = getattr(backend, '_read_strict', None) | |
| if callable(reader): | |
| return reader(self.store_key(name)) | |
| return self._store().get(self.store_key(name), fresh=fresh) | |
| def get_projection(self, name, drop=()): | |
| """A tenant-scoped PROJECTED read β the bucket without the keys named in `drop`. | |
| ββ W32-T01 (D-185). `get()` deep-copies the whole document; on tenant #0's `user_tables` | |
| that is 28.6 MB / ~703 ms warm, and 99.9% of it is `rows` that no nav render, permission | |
| check or workspace envelope reads. | |
| β **`_store()` returns the MODULE for every tenant with `store_handle is None`** β which | |
| includes tenant #0 β so this delegates to a MODULE-LEVEL `get_projection`, not to a method. | |
| Both backends implement it and `verify_store_pg.IFACE` pins that they must; see | |
| `core.store.get_projection`'s note for why the method alone would have been unreachable | |
| from exactly the caller it was built for. | |
| """ | |
| return self._store().get_projection(self.store_key(name), drop=drop) | |
| def get_projections(self, names, drop=(), timeout_s=None): | |
| """Batch projected reads when the active bound store has a native batch operation. | |
| ``None`` means the backend has no batch primitive; callers retain their ordinary per-key | |
| path. That keeps this an optional optimization rather than widening the store contract. | |
| """ | |
| reader = getattr(self._store(), "get_projections", None) | |
| if not callable(reader): | |
| return None | |
| keyed = [(str(name), self.store_key(name)) for name in (names or ())] | |
| answer = reader([store_key for _name, store_key in keyed], drop=drop, | |
| timeout_s=timeout_s) | |
| return {name: answer.get(store_key, {}) for name, store_key in keyed} | |
| def get_workbook_projection(self, name, *, metadata_only=False, tab_ids=None, cells=None, | |
| expected_rev=None): | |
| """Read one tenant's selected workbook cells without caching permissions or source answers.""" | |
| reader = getattr(self._document_store(), 'get_workbook_projection', None) | |
| if callable(reader): | |
| return reader(self.store_key(name), metadata_only=metadata_only, tab_ids=tab_ids, | |
| cells=cells, expected_rev=expected_rev) | |
| metadata_only, tab_ids, cells = store._workbook_selection( | |
| metadata_only=metadata_only, tab_ids=tab_ids, cells=cells) | |
| return store._project_workbook(self.get(name), metadata_only=metadata_only, | |
| tab_ids=tab_ids, cells=cells, expected_rev=expected_rev) | |
| def list_keys(self, prefix=''): | |
| names = self._document_store().list_keys(self.store_key(prefix)) | |
| return [name[len(self.store_namespace):] for name in names | |
| if name.startswith(self.store_namespace)] | |
| def list_paths(self, prefix=''): | |
| paths = self._document_store().list_paths(self.store_key(prefix)) | |
| return [path[len(self.store_namespace):] for path in paths | |
| if path.startswith(self.store_namespace)] | |
| def delete_document(self, name): | |
| return self._document_store().delete_document(self.store_key(name)) | |
| def amazon_backup_policy(self, *, trigger_at, checked_at): | |
| """Delegate backup evidence only to this runtime's tenant-bound backend.""" | |
| backend = self._document_store() | |
| reader = getattr(backend, 'amazon_backup_policy', None) | |
| if callable(reader): | |
| policy = reader(trigger_at=trigger_at, checked_at=checked_at) | |
| if not isinstance(policy, dict) or policy.get('tenant') != self.key: | |
| raise DatastoreMismatch('Backup evidence belongs to another tenant.') | |
| return policy | |
| return {'tenant': self.key, 'fixture': False, 'state': 'UNPROVEN', | |
| 'policyId': None, 'source': None, 'method': None, | |
| 'expiresAt': None, 'verifiedExpiredAt': None, | |
| 'cause': 'tenant_backup_policy_unavailable'} | |
| def get_many(self, names): | |
| """Batch ordinary reads when the active bound store offers that optimization.""" | |
| reader = getattr(self._store(), "get_many", None) | |
| if not callable(reader): | |
| return None | |
| keyed = [(str(name), self.store_key(name)) for name in (names or ())] | |
| answer = reader([store_key for _name, store_key in keyed]) | |
| return {name: answer.get(store_key, {}) for name, store_key in keyed} | |
| def get_entry(self, name, entry): | |
| """One top-level entry of `name` when the bound store offers it, else None (HOTFIX | |
| 2026-09-28). None also means absent; either way the caller falls back to `get`.""" | |
| reader = getattr(self._store(), "get_entry", None) | |
| if not callable(reader): | |
| return None | |
| return reader(self.store_key(name), entry) | |
| def revision(self, name): | |
| """Return the tiny durable change token for this tenant's addressed bucket.""" | |
| return self._store().revision(self.store_key(name)) | |
| def put(self, name, data): | |
| return self._store().put(self.store_key(name), data) | |
| def transaction(self, connection): | |
| """Enlist this tenant's writes in an existing receipt transaction.""" | |
| return self._document_store().transaction(connection) | |
| def update(self, name, fn, flush="sync", expect_rev=None, audit_row=None): | |
| """Read-modify-write this tenant's bucket, optionally rev-guarded and audited (C3 / R9). | |
| ββ W50-T28 CLAUSE 1 β THE TWO KEYWORDS EXIST SO `control.audit` CAN BE REACHED AT ALL. | |
| `core/store_pg.py::update` grew `expect_rev` and `audit_row` in W50-T25 and every API | |
| caller passes `st=session.runtime`, so until this method NAMED them the capability was | |
| written, gated and unreachable β the [[tenant-runtime-update-is-a-keyword-bottleneck]] | |
| shape. `expect_rev` refuses a write whose caller was looking at an older revision; | |
| `audit_row` is the LIST the caller's own closure fills, carried into the same transaction | |
| as the workspace write so the trail commits with the change or not at all. | |
| ββ THE FORWARD IS CONDITIONAL, AND THAT IS THE ENTIRE CARE IN THIS METHOD. `_store()` | |
| returns the `core.store` MODULE for every tenant with `store_handle is None` β which | |
| includes tenant #0 β and that module's `update(name, fn, flush='sync')` names neither | |
| keyword. An unconditional forward is a `TypeError` on the hot path for exactly the | |
| tenant that matters most. | |
| ββ AND A SIGNATURE IS A CAPABILITY DECLARATION HERE, NOT A CONVENIENCE. | |
| `view_audit.home()` and `view_audit.guards()` answer by inspecting THIS method's | |
| parameters, so merely naming the keywords tells the whole repo that every tenant can | |
| carry a row into its transaction and enforce a revision inside its own lock. Where the | |
| inner handle cannot, both promises must be kept HERE or they are broken silently: | |
| * `expect_rev` -> the ADVISORY check (read `revision()`, compare, refuse). Identical to | |
| the fallback `table_store._audited_update` used to take on its own, which this | |
| signature change is what disables. It has a real TOCTOU window and still refuses | |
| every stale write it can see. | |
| * `audit_row` -> appended to the `__audit__` DOCUMENT ring inside the caller's closure, | |
| which is where `view_audit.record` would have put it had `home()` still said | |
| `DOCUMENT`. Without this the row is not relocated, it is LOST. | |
| β WHAT THIS DOES NOT CLOSE, and the handoff says so: tenant #0 resolves through the | |
| `core.store` MODULE function, which drops both keywords before `_d()` ever reaches | |
| `PgStore.update`. That tenant therefore takes the compensating path above and its trail | |
| stays in the ring. The second bottleneck is `core/store.py::update`, in nobody's fence. | |
| """ | |
| st = self._store() | |
| slug = getattr(st, 'tenant_slug', None) | |
| if slug is not None and slug != self.key: | |
| raise DatastoreMismatch('The store handle belongs to another tenant.') | |
| key = self.store_key(name) | |
| if expect_rev is None and audit_row is None: | |
| return st.update(key, fn, flush=flush) | |
| # Imported HERE, not at module scope: `api/main.py` imports half the platform before it | |
| # has read an environment variable, and `core.view_audit` reaches `core.store_pg`. | |
| from core import view_audit | |
| if view_audit.guards(st) and view_audit.home(st) == view_audit.CONTROL_AUDIT: | |
| return st.update(key, fn, flush=flush, | |
| expect_rev=expect_rev, audit_row=audit_row) | |
| if expect_rev is not None: | |
| self._refuse_stale_revision(name, expect_rev) | |
| if audit_row is not None: | |
| fn = self._appending_audit_rows(fn, audit_row) | |
| return st.update(key, fn, flush=flush) | |
| def _refuse_stale_revision(self, name, expect_rev): | |
| """Raise `ViewRevisionStale` when this bucket has moved past `expect_rev`. | |
| The ADVISORY half of the guard, for a handle that cannot enforce one inside its own | |
| locking read. `RevisionUnverifiable` rather than a silent pass when the revision cannot | |
| be read at all: "I could not tell" and "it matches" are different answers, and only one | |
| of them may let a write land. | |
| """ | |
| from core import view_audit | |
| if not callable(getattr(self._store(), "revision", None)): | |
| raise view_audit.RevisionUnverifiable( | |
| "this deployment cannot tell which revision of the view list you are looking " | |
| "at, so the change was not saved. Reload and try again.") | |
| try: | |
| current = int((self.revision(name) or {}).get("rev") or 0) | |
| except Exception as exc: # noqa: BLE001 | |
| raise view_audit.RevisionUnverifiable( | |
| "the view list revision could not be read, so the change was not saved. " | |
| "Reload and try again.") from exc | |
| if current != int(expect_rev): | |
| raise view_audit.ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev)) | |
| def _appending_audit_rows(fn, rows): | |
| """Wrap `fn` so the rows it collected land in the document ring, in its own transaction. | |
| β `rows` IS EMPTY WHEN THIS RUNS AND FULL AFTERWARDS β the caller's closure appends to it | |
| while it executes, which is the only moment a row's `before` and `after` are both | |
| knowable. So the append happens after `fn` returns and reads the list then, never now. | |
| """ | |
| def _apply(data): | |
| from core import view_audit | |
| result = fn(data) | |
| target = result if isinstance(result, dict) else data | |
| for entry in rows: | |
| view_audit.append_to_document(target, entry) | |
| return result | |
| return _apply | |
| def exists(self, name): | |
| return self._store().exists(self.store_key(name)) | |
| def available(self): | |
| return self._store().available() | |
| # ---------------------------------------------------------------- BYTES (wave 25, C's ask) | |
| # β WHY THESE EXIST: `grid_events._blobs()` had no tenant-scoped way to reach bytes, so an R2 | |
| # tenant's UPLOADED FILES landed in the shared repo while its buckets went to its own β the | |
| # residency fix with a hole in it. SESSION C wrote `_blobs` as a CAPABILITY TEST | |
| # (`hasattr(st, "upload_bytes")`) precisely so the fallback disappears the moment these land, | |
| # with no edit on that side. | |
| # | |
| # β THEY GO THROUGH `store_key`, AND THAT IS THE WHOLE POINT. A byte path is a different | |
| # address space from a bucket name, and the temptation is to pass it straight through β but | |
| # the isolation requirement is identical: a tenant on the SHARED repo (namespace `t/<slug>/`) | |
| # writing `uploads/photo.png` would otherwise land on the exact path another tenant writes. | |
| # An own-repo tenant has an EMPTY namespace, so the path is untouched and the repo boundary | |
| # does the work, which is the same asymmetry every method above already has. Reusing | |
| # `store_key` keeps "the ONE place the prefix is applied" literally true rather than nearly. | |
| def upload_bytes(self, path_in_repo, data, message=None): | |
| return self._store().upload_bytes(self.store_key(path_in_repo), data, message=message) | |
| def download_bytes(self, path_in_repo): | |
| return self._store().download_bytes(self.store_key(path_in_repo)) | |
| def delete_path(self, path_in_repo): | |
| return self._store().delete_path(self.store_key(path_in_repo)) | |
| # ---------------------------------------------------------------- the Odoo source (R3) | |
| def odoo_source(self, connection_id=None): | |
| """THIS tenant's Odoo connector, or None β the keychain CUTOVER's resolution seam | |
| (2026-08-04; `keychain.odoo_creds` was previously called by nothing but its test). | |
| Resolution, fail-closed: | |
| 1. the keychain `odoo` connection NAMED by `connection_id` (W54-T45 / R4), or the | |
| tenant's DESIGNATED DEFAULT connection when no id is named β a connector bound to | |
| THOSE creds; | |
| 2. tenant #0 with an empty/locked keychain and NO id named β its compiled env | |
| connector (R3's "live with env fallback", which keeps Royal working unchanged); | |
| 3. anyone else β None. NEVER the environment: env is tenant #0's connection, and | |
| handing it to another tenant is the leak this method exists to prevent. | |
| ββ W54-T45 β THE ENV FALLBACK NARROWS HERE AND MUST NEVER WIDEN. A caller that NAMES a | |
| connection and gets nothing back is answered None, full stop: not the default connection, | |
| not a sibling, and above all not tenant #0's `.env`. Under R4 an agency holds one Odoo | |
| connection per client, and under R7 that isolation is strict β a named-but-unresolvable | |
| connection quietly falling through to the environment would serve tenant #0's own Odoo to | |
| a question asked about somebody else's books, with nothing red anywhere. Leg (2) is | |
| reachable only on the UNNAMED path, which is exactly the pre-wave behaviour. | |
| Rebuilt per call on purpose β `core.odoo` caches the CLIENT per (slug, creds | |
| fingerprint), so this stays cheap while a rotated key reconnects on the next call.""" | |
| import core.keychain as keychain | |
| named = connection_id is not None | |
| try: | |
| # β THE UNNAMED PATH IS SPELLED AS THE PRE-WAVE CALL, VERBATIM, and the branch is not | |
| # decoration: `verify_api`'s R11 leg pins the literal `keychain.odoo_creds(self)` in | |
| # this method's source as its proof that the resolver is KEYCHAIN-FIRST. Writing only | |
| # the two-argument form would have turned that gate red for a reason that has nothing | |
| # to do with the rule it guards, in a file this lane may not edit. The two arms are | |
| # the same question; the left one is the one every existing caller asks. | |
| creds = (keychain.odoo_creds(self) if not named | |
| else keychain.odoo_creds(self, connection_id)) | |
| except Exception: # locked or unreadable keychain β fall through, fail closed | |
| creds = None | |
| if creds: | |
| from harness.connectors.odoo import OdooConnector | |
| cfg = getattr(self.tenant, "config", {}) or {} | |
| odoo_cfg = self.get(_ds._ODOO_CONFIG_KEY) or {} | |
| source = OdooConnector(tenant_slug=self.key, creds=creds, | |
| scope=team_scope(cfg.get("business_units"), cfg.get("team_pricelists"))) | |
| return _business_source(source, odoo_cfg.get('businessTz')) | |
| # β THE NAMED LEG STOPS HERE. See the docstring: an explicit connection that did not | |
| # resolve is None, never the environment and never another connection. | |
| if named: | |
| return None | |
| if self.key == "royal-imports": | |
| source = (getattr(self.tenant, "sources", {}) or {}).get("odoo") | |
| return _business_source(source, (self.get(_ds._ODOO_CONFIG_KEY) or {}).get('businessTz')) | |
| return None | |
| # ------------------------------------------------------- the connector PAUSE (D-10, wave 24) | |
| def odoo_flag_key(self, connection_id=None): | |
| """WHICH connector flag governs this tenant's Odoo β the key `connectors[<k>].paused` | |
| is stored under, or None when nothing would serve. | |
| THE SAME RESOLUTION AS `odoo_source()` ABOVE, expressed as an identity instead of a | |
| client: the keychain `odoo` connection named (or the designated default when none is), | |
| else tenant #0's compiled env source, else nothing. It lives here so there is ONE | |
| resolver β `routes_keychain._resolved_odoo_key` was a second copy of these three rules in | |
| the API layer, and a pause flag written against one resolution and read against the other | |
| freezes nothing while reporting success. | |
| ββ W54-T45 β AND THE INLINE WALK IS GONE, WHICH IS WHAT MADE THE SENTENCE ABOVE TRUE. | |
| This method claimed to be the one resolver while carrying its own copy of "the first | |
| `odoo` entry", so `keychain` and `runtime` each decided which connection serves. Both now | |
| ask `keychain.resolve_connection_id`, which decrypts nothing β a pause flag has to resolve | |
| on a LOCKED keychain, which is why this is an identity call and not a credential one. | |
| β A NAMED CONNECTION THAT DOES NOT RESOLVE IS None, exactly as in `odoo_source`: pausing | |
| by a connection id must never silently address tenant #0's environment source instead. | |
| """ | |
| import core.keychain as keychain | |
| named = connection_id is not None | |
| try: | |
| resolved = keychain.resolve_connection_id(self, "odoo", connection_id) | |
| except Exception: | |
| resolved = None | |
| if resolved and keychain.unlocked(): | |
| return resolved | |
| if named: | |
| return None | |
| if self.key == "royal-imports" and os.environ.get("ODOO_URL"): | |
| return ENV_ODOO_FLAG_KEY | |
| return None | |
| def odoo_paused(self, connection_id=None): | |
| """Is THIS tenant's RESOLVED Odoo source paused? | |
| β RESOLVED, not "any entry" β pausing an entry that is not the one serving freezes | |
| nothing, because it serves nothing. Never raises: an unreachable flags bucket answers | |
| False, because a store hiccup must not freeze a live surface. That policy is inherited | |
| from the API-layer function this one replaced, deliberately, so there is one answer to | |
| "what happens when we cannot tell" rather than two that can disagree. | |
| β W54-T45 β `connection_id` passes straight through to `odoo_flag_key`, so "is the | |
| connection serving THIS client paused" is answerable per connection under R4. No id keeps | |
| meaning the tenant's designated default, which is every existing caller. | |
| """ | |
| try: | |
| flag_key = self.odoo_flag_key(connection_id) | |
| if not flag_key: | |
| return False | |
| flags = self.get(CONNECTOR_FLAGS_KEY) or {} | |
| return bool((flags.get(flag_key) or {}).get("paused")) | |
| except Exception: | |
| return False | |
| def mirror_handle(self): | |
| """Resolve only this tenant's configured mirror, even when its file is absent.""" | |
| return _ds.tenant_handle(self.key, self.datastore_path) | |
| def assert_datastore_matches(self, cur=None): | |
| """Refuse a handle or borrowed cursor belonging to another tenant.""" | |
| handle = self.mirror_handle() | |
| self._assert_mirror_cursor(handle, cur) | |
| def _assert_mirror_cursor(self, handle, cur): | |
| """Validate a borrowed cursor against the handle acquired for this operation.""" | |
| if handle.tenant_key != self.key: | |
| raise DatastoreMismatch("The mirror handle belongs to another tenant.") | |
| if cur is None: | |
| return | |
| bound = getattr(cur, "_mirror_handle", None) | |
| if bound is not None: | |
| if bound.tenant_key != self.key or bound.path != handle.path: | |
| raise DatastoreMismatch("The mirror cursor belongs to another tenant.") | |
| return | |
| marks = _ds.mirror_file_marks(cur) | |
| expected = _ds.serving_file(handle.path)[0] if _ds.versioned_mode() else handle.path | |
| if marks is None or marks[0] != str(expected.resolve()): | |
| raise DatastoreMismatch("The borrowed mirror cursor belongs to another tenant.") | |
| def _store_owner(path): | |
| """Return the forward owner and whether the registry and path mapping prove isolation. | |
| β FORWARD-MAPPED, never parsed out of the filename β `datastore.path_for` substitutes `_` | |
| for every non-alphanumeric character and truncates at 60, so `a.b` and `a_b` produce the | |
| same file and the name is genuinely not invertible. `tenant_for_store_path` above says the | |
| same thing and is not reused here for one reason: it swallows every failure into `""`, and | |
| this caller must be able to tell "nobody owns it" from "I could not find out". | |
| β AND `known_tenants()` IS NOT REUSED EITHER, FOR THE SAME REASON ONE LAYER DOWN β it | |
| cannot raise: `_tenant_records()` returns `{}` on any failure and `known_tenants` wraps | |
| that in another `except: pass`. Calling it here would have made `registry_ok` a constant | |
| `True` and the fail-closed branch above unreachable, i.e. a guard whose most important | |
| clause is dead code that reads as if it fires ([[gate-can-report-green-on-nothing]]). The | |
| control-plane bucket is therefore read HERE, where its failure is still visible. | |
| """ | |
| keys, registry_ok = set(_BUILDERS), True | |
| try: | |
| registries = [store.get(TENANTS_KEY)] | |
| # HF display failures cannot prove uniqueness. Confirm through its strict reader. | |
| if store.backend() != "pg": | |
| registries.append(store._read_strict(TENANTS_KEY)) | |
| for recs in registries: | |
| recs = {} if recs is None else recs | |
| if not isinstance(recs, dict) or any(not isinstance(v, dict) for v in recs.values()): | |
| raise ValueError("The tenant ownership registry is malformed.") | |
| keys |= {str(k).strip().lower() for k in recs} | |
| except Exception: # noqa: BLE001 | |
| registry_ok = False | |
| owners = [] | |
| for key in sorted(keys): | |
| try: | |
| if _ds.path_for(key).resolve() == path: | |
| owners.append(key) | |
| except Exception: # noqa: BLE001 | |
| registry_ok = False | |
| continue | |
| return (owners[0] if len(owners) == 1 else "ambiguous" if owners else ""), registry_ok and len(owners) <= 1 | |
| def register(key, builder): | |
| """Add a tenant to the registry (a stub connector is a legitimate builder β see X8's qa-b). | |
| Invalidates any cached runtime for the slug so a re-registration cannot be shadowed.""" | |
| _BUILDERS[str(key)] = builder | |
| invalidate(key) | |
| def worker_tenant(): | |
| """The one tenant a per-tenant sync worker serves, or "" in every other process.""" | |
| if (os.environ.get("AIOS_PROCESS_ROLE") != "worker" | |
| or os.environ.get("AIOS_WORKER_SCOPE") != "tenant"): | |
| return "" | |
| return str(os.environ.get("AIOS_SYNC_WRITER_TENANT") or "").strip().lower() | |
| def known_tenants(): | |
| keys = set(_BUILDERS) | |
| try: | |
| keys |= set(_tenant_records()) | |
| except Exception: | |
| pass | |
| # A per-tenant sync worker holds one tenant's file, so it never sees another tenant. | |
| only = worker_tenant() | |
| if only: | |
| keys &= {only} | |
| return sorted(keys) | |
| #: Wave 18 (C1-TENANT) β the CONTROL-PLANE bucket: tenants provisioned at runtime rather than | |
| #: compiled in. Lives in the DEFAULT store (tenant #0's repo) deliberately: it answers "which | |
| #: tenants exist", which is a platform fact, not any one tenant's data. | |
| TENANTS_KEY = "tenants" | |
| def _tenant_records(): | |
| """{slug: record} from the control-plane bucket. {} on any failure β an unreachable bucket | |
| must degrade to "only the compiled-in tenants exist", never to an exception at login.""" | |
| try: | |
| recs = store.get(TENANTS_KEY) or {} | |
| return {str(k).strip().lower(): v for k, v in recs.items() if isinstance(v, dict)} | |
| except Exception: | |
| return {} | |
| def _builder_from_record(key, rec): | |
| """A Tenant for a bucket-provisioned client: no sources yet (connectors arrive via the | |
| Keychain flow), config carries the platform fields the request path reads.""" | |
| def _build(): | |
| from harness.base import Tenant | |
| return Tenant(key=key, name=str(rec.get("name") or key), | |
| config={"modules": rec.get("modules", []), | |
| "store_repo": rec.get("store_repo") or None}) | |
| return _build | |
| def get_runtime(tenant_key): | |
| """The runtime for `tenant_key`, built on first use and LRU-cached. | |
| Raises KeyError for an unknown slug β the caller turns that into a 401 (fail-closed: an | |
| unknown tenant is not a 404, because confirming which slugs exist answers a question the | |
| request was not entitled to ask). | |
| """ | |
| key = str(tenant_key or "").strip().lower() | |
| if not key: | |
| raise KeyError("no tenant") | |
| only = worker_tenant() | |
| if only and key != only: | |
| # Fail closed: this process's DuckDB file belongs to `only`. | |
| raise KeyError(key) | |
| with _LOCK: | |
| rt = _CACHE.get(key) | |
| if rt is not None: | |
| _CACHE.move_to_end(key) # LRU: touch on read | |
| return rt | |
| builder = _BUILDERS.get(key) | |
| # Wave 18 (C1-TENANT): a slug the compiled registry does not know may be a PROVISIONED | |
| # tenant β consult the control-plane bucket before failing closed. A suspended record is | |
| # the same KeyError as an unknown one: "which tenants exist" is not this caller's question. | |
| if builder is None: | |
| rec = _tenant_records().get(key) | |
| if not rec or rec.get("status", "active") != "active": | |
| raise KeyError(key) | |
| builder = _builder_from_record(key, rec) | |
| # Built OUTSIDE the lock: a builder may do real work (a connector handshake), and holding the | |
| # registry lock through it would serialise every other tenant's requests behind it. | |
| tenant = builder() | |
| with _LOCK: | |
| existing = _CACHE.get(key) | |
| if existing is not None: # another thread won the race β keep ONE runtime | |
| _CACHE.move_to_end(key) | |
| return existing | |
| # R2: a tenant with its OWN repo gets a bound Store instance and an EMPTY prefix (the | |
| # repo boundary is the namespace); everyone else keeps the t/<slug>/ prefix in the | |
| # shared repo. Tenant #0 keeps both defaults β empty prefix, shared repo. | |
| # | |
| # β WAVE 20 (R1 / D-4): under Postgres the address is a SCHEMA, so `store.handle()` is | |
| # asked for one by SLUG and the namespace prefix goes to EMPTY for EVERY tenant β | |
| # `t_<slug>.store_kv` already isolates, and prefixing keys inside an isolated schema | |
| # would namespace them twice. That is not a cosmetic difference: a `t/nurilab/`-prefixed | |
| # key written into `t_nurilab` is a key the migration did not copy and no reader looks | |
| # for, i.e. a silent empty workspace. | |
| # | |
| # β NOTHING ELSE IN THIS FILE CHANGES, and that is the design working. Every product-data | |
| # read/write already goes through `rt.get/put/update/exists` (the `st=` handle threaded | |
| # through the modules), so binding the handle correctly here IS the cutover for the | |
| # tenant-aware paths β the module-level `core.store` functions cover the rest via `_d()`. | |
| repo = (getattr(tenant, "config", {}) or {}).get("store_repo") | |
| if store.backend() == "pg": | |
| rt = TenantRuntime(key=key, tenant=tenant, store_namespace="", | |
| store_handle=store.handle(slug=key)) | |
| else: | |
| rt = TenantRuntime( | |
| key=key, tenant=tenant, | |
| store_namespace=("" if (key == "royal-imports" or repo) else f"t/{key}/"), | |
| store_handle=(store.for_repo(repo) if repo else None)) | |
| _initialize_deletion_owners(rt) | |
| _CACHE[key] = rt | |
| while len(_CACHE) > _MAX_RUNTIMES: | |
| _CACHE.popitem(last=False) # evict least recently used | |
| return rt | |
| def invalidate(tenant_key=None): | |
| """Drop one tenant's runtime, or all of them. Explicit, per X7 β a cache with no documented | |
| way to clear it becomes a restart.""" | |
| with _LOCK: | |
| if tenant_key is None: | |
| _CACHE.clear() | |
| else: | |
| _CACHE.pop(str(tenant_key or "").strip().lower(), None) | |
| def resident(): | |
| """The tenant slugs currently holding a runtime β LRU order, oldest first. For the EXIT-4 | |
| measurement and for asserting the bound actually binds.""" | |
| with _LOCK: | |
| return list(_CACHE) | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # β DEBT D-10 (wave 24) β TEACHING THE MEASURE MIRROR ABOUT THE CONNECTOR PAUSE. | |
| # | |
| # `harness.datastore` is the store every measure column and condition is answered from, and it | |
| # pulled from Odoo straight through a pause: 12 sync passes at boot and one every 1800 s. It | |
| # cannot ask the question itself β the flag lives in a tenant's store bucket, which is this | |
| # module's to read, and this module already imports datastore, so an import back would be a | |
| # cycle. So datastore exposes a probe slot and this module fills it, once, at import. | |
| def tenant_for_store_path(path=None): | |
| """Which tenant owns the DuckDB file currently open (or `path`)? `""` when nothing matches. | |
| β RESOLVED BY FORWARD-MAPPING every known tenant, never by parsing the filename, and that is | |
| a correctness point rather than a style one: `datastore.path_for` substitutes `_` for every | |
| non-alphanumeric character and truncates at 60, so the name is genuinely NOT invertible β | |
| `a.b` and `a_b` produce the same file. Guessing the slug back out of it would eventually name | |
| the wrong tenant, and this answer decides whether another tenant's connector freezes ours. | |
| """ | |
| want = Path(path or _ds.DB_PATH).resolve() | |
| # Tenant #0 first: it is the only tenant the unattended sync actually runs for today (see | |
| # D-29 β the resync loop prewarms `royal-imports` hardcoded), so this is the hot answer. | |
| for key in ["royal-imports"] + [k for k in known_tenants() if k != "royal-imports"]: | |
| try: | |
| if _ds.path_for(key).resolve() == want: | |
| return key | |
| except Exception: | |
| continue | |
| return "" | |
| def store_source_paused(): | |
| """The probe `harness.datastore` calls before reaching Odoo. Never raises; False when the | |
| tenant cannot be resolved, matching the fail-towards-live policy `odoo_paused` documents.""" | |
| try: | |
| key = tenant_for_store_path() | |
| return bool(key) and get_runtime(key).odoo_paused() | |
| except Exception: | |
| return False | |
| _ds.set_paused_probe(store_source_paused) | |
| def _business_source(source, zone): | |
| """Adapt the connector's explicit binding without mutating its shared tenant source.""" | |
| if source is None: | |
| return None | |
| import copy | |
| result = copy.copy(source) | |
| result._bound = lambda: _odoo.tenant_scope( | |
| result._slug, result._creds, **result._scope, business_tz=zone) | |
| return result | |
| def team_scope(business_units, team_pricelists=None): | |
| """A tenant config's `business_units` ({label: team id}) and `team_pricelists` ({label: lists}) | |
| as `core.odoo.tenant_scope` kwargs. A pricelist row whose label names no team is dropped.""" | |
| bus = dict(business_units or {}) | |
| lists = {bus[label]: tuple(names) for label, names in (team_pricelists or {}).items() | |
| if label in bus} | |
| return {"team_ids": list(bus.values()) or None, | |
| "team_names": {v: k for k, v in bus.items()} or None, | |
| "team_pricelists": lists or None} | |
| def mirror_team_names(slug): | |
| """{team_id: Odoo team name} from a RESIDENT tenant's mirror, read only; {} when none is. | |
| Never builds a runtime: a team lookup must not reach the store or the network.""" | |
| with _LOCK: | |
| rt = _CACHE.get(str(slug or "").strip().lower()) | |
| if rt is None: | |
| return {} | |
| cur = mirror_cursor(rt) | |
| try: | |
| try: | |
| rows = cur.execute("SELECT team_id, max(team_name) FROM sale_order " | |
| "WHERE team_id IS NOT NULL GROUP BY team_id").fetchall() | |
| except Exception: # a mirror without the name column still lists its team ids | |
| rows = [(t, None) for (t,) in cur.execute( | |
| "SELECT DISTINCT team_id FROM sale_order WHERE team_id IS NOT NULL").fetchall()] | |
| finally: | |
| cur.close() | |
| return {int(t): (str(n) if n else str(t)) for t, n in rows} | |
| # core may not import harness: push the default tenant's configured teams and the mirror reader | |
| # down at process start, the same way `set_paused_probe` above reaches the datastore. | |
| _odoo.set_default_scope(**team_scope(_tenants.TENANT0_BUSINESS_UNITS, | |
| _tenants.TENANT0_TEAM_PRICELISTS)) | |
| _odoo.set_team_reader(mirror_team_names) | |