Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
File size: 44,245 Bytes
149aa87 f462120 149aa87 8133df9 149aa87 dec006b 149aa87 f462120 149aa87 f462120 149aa87 f462120 149aa87 c1fc133 149aa87 dec006b 149aa87 8873406 dec006b 149aa87 c1fc133 dec006b c1fc133 2eae9f0 bece3e9 149aa87 8873406 f530297 8873406 f530297 149aa87 82cdb59 149aa87 82cdb59 149aa87 82cdb59 149aa87 82cdb59 149aa87 82cdb59 149aa87 8133df9 82cdb59 149aa87 8133df9 149aa87 82cdb59 149aa87 82cdb59 149aa87 82cdb59 149aa87 82cdb59 149aa87 82cdb59 72b6f27 149aa87 82cdb59 149aa87 82cdb59 149aa87 82cdb59 149aa87 f462120 149aa87 f462120 149aa87 f462120 149aa87 f462120 149aa87 f462120 149aa87 f462120 149aa87 f462120 149aa87 f462120 149aa87 f462120 149aa87 62c5213 149aa87 62c5213 149aa87 62c5213 149aa87 dec006b 149aa87 8133df9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 | """harness/runtime.py β X7: per-REQUEST tenant resolution for the shared stateless API (EXIT-4a).
THE COST ARGUMENT, in one rule: *no per-tenant state may be resident in the app process.* Today
each tenant carries its own interpreter, its own never-evicted pool, its own 9 prewarmed bundles
and its own DuckDB β EXIT-0 measured a **~0.6 GB commit floor per tenant**, duplicated exactly
(tenant B came within 2% of tenant A; nothing is shared). The target is ~30β80 MB. That is only
reachable if the API resolves the tenant from the REQUEST and holds nothing tenant-shaped between
requests except a bounded cache it is willing to throw away.
WHAT THIS REPLACES. `ui/cache.py`'s `st.cache_resource` singleton β one process, one tenant,
caches that are never evicted by design. That is correct for Streamlit and fatal for a shared
process. The Streamlit side keeps its singleton untouched this wave (two processes, two caches,
accepted for the strangler period; `ui/cache.py` dies with app.py at EXIT-6).
THREE RULES THIS FILE OBEYS, each of which is a way the old shape leaked:
1. **No module-level per-tenant globals.** Every tenant-shaped value hangs off a TenantRuntime
reached through `get_runtime(slug)`; nothing is stashed at import time. A module global is
exactly what makes "which tenant is this process serving?" unanswerable.
2. **LRU-BOUNDED.** `_MAX_RUNTIMES` runtimes live at once; the least recently used is evicted.
An unbounded registry cache is a memory leak with a tenant-count-shaped growth curve β the
thing this whole program exists to remove.
3. **The store handle is tenant-bound**, so a write cannot land in another tenant's namespace
even if a caller passes the wrong key downstream (EXIT-4b proof #3).
Request cursors bind to a tenant handle. The legacy writer keeps its explicit single-store pin.
"""
from __future__ import annotations
import os
import threading
from collections import OrderedDict
from dataclasses import dataclass, field
from pathlib import Path
import core.odoo as _odoo
import core.store as store
from harness import datastore as _ds
from harness import tenants as _tenants
#: How many tenant runtimes stay resident. Small on purpose: a runtime is cheap to rebuild (a
#: Tenant literal + a couple of handles) and expensive to leak. Raise only with a measurement.
_MAX_RUNTIMES = int(os.environ.get("AIOS_MAX_RUNTIMES") or 32)
#: slug -> builder. The literal registry `harness/tenants.py` describes; per its own docstring
#: this becomes "a tenants store (DB / HF dataset)" β X4's control DB is where it goes, and
#: `register()` below is the seam that makes the swap a one-liner rather than a refactor.
_BUILDERS = {"royal-imports": _tenants.royal_imports}
# X8's isolation fixture, admitted ONLY on request. A stub tenant that is always present is a
# slug an attacker can mint a session against in production; gated, it exists exactly where it is
# needed (the TestClient isolation proofs) and nowhere else. Read at import so the flag cannot be
# flipped by a request.
if os.environ.get("AIOS_ENABLE_QA_TENANT") == "1":
_BUILDERS["qa-b"] = _tenants.qa_tenant_b
_CACHE: "OrderedDict[str, TenantRuntime]" = OrderedDict()
_LOCK = threading.RLock()
_DELETION_OWNER_INITIALIZERS = {}
def _initialize_core_deletion_owners(rt, registrar):
from core import sheets
initializer = getattr(sheets, 'register_amazon_deletion_owner', None)
if callable(initializer):
initializer(rt, registrar)
def register_deletion_owner_initializer(name, initializer):
"""Mount an injected service owner on resident runtimes and every later rebuild."""
if not isinstance(name, str) or not name.strip() or not callable(initializer):
raise ValueError('A deletion owner initializer needs a name and callback.')
from harness.amazon_store import register_deletion_owner
with _LOCK:
_DELETION_OWNER_INITIALIZERS[name] = initializer
for rt in _CACHE.values():
initializer(rt, register_deletion_owner)
def _initialize_deletion_owners(rt):
from harness.amazon_store import register_deletion_owner
_initialize_core_deletion_owners(rt, register_deletion_owner)
for initializer in tuple(_DELETION_OWNER_INITIALIZERS.values()):
initializer(rt, register_deletion_owner)
#: The store bucket Settings > Connectors writes pause flags into. ONE literal, shared with the
#: API layer (`routes_keychain` aliases this) β the flag is written by one module and read by
#: three, and two spellings of the key would be a pause that silently freezes nothing.
CONNECTOR_FLAGS_KEY = "connectors"
#: The flag key tenant #0's compiled ENV Odoo source is stored under. It is not a keychain entry
#: id, so it needs a name of its own; `routes_keychain`'s connector row uses the same literal.
ENV_ODOO_FLAG_KEY = "odoo-env"
class DatastoreMismatch(RuntimeError):
"""A mirror handle or cursor does not belong to the requested tenant.
β IT SUBCLASSES `RuntimeError` DELIBERATELY, AND THE REASON IS A LIVE TRAP RATHER THAN
TIDINESS. `harness.datastore.ro_con()` already raises a bare `RuntimeError` for *"the tenant
data store is completing its FIRST sync"*, and the API doors translate that into
`503 store_not_ready` β i.e. "wait a few minutes and retry". A cross-tenant mismatch NEVER
fixes itself by waiting: it is a pinned deployment serving the wrong file, and rendering it as
a retry banner would convert the loudest refusal in the system into a spinner.
So the type does two jobs at once. Being a `RuntimeError` keeps every existing
`except RuntimeError` working unchanged β including `verify_api::_guard_ok`, which is what
made the guard's own gate green before it had any production caller. Being a SUBCLASS lets
each door catch this FIRST and answer with its own status and sentence. β Order matters at
every catch site: a bare `except RuntimeError` placed above this one swallows it silently,
which is the exact shape of [[defects-that-mask-each-other]].
"""
def mirror_cursor(rt, *, require_ready=True):
"""Return this runtime's cursor, preserving readiness unless its caller gates a specific source."""
if isinstance(rt, TenantRuntime):
handle = rt.mirror_handle()
cur = handle.ro_con() if require_ready else handle.ro_cursor()
try:
rt._assert_mirror_cursor(handle, cur)
except BaseException:
cur.close()
raise
return cur
# Legacy single-store adapters retain their existing assertion and readiness contract.
rt.assert_datastore_matches()
return _ds.ro_con() if require_ready else _ds.ro_cursor()
@dataclass
class TenantRuntime:
"""Everything a request needs to serve ONE tenant, and nothing about any other.
`store_namespace` is the prefix every product-data key is written under. Tenant #0 keeps the
EMPTY prefix so its existing keys (`users`, `customer_lists`, `customer_docs`, β¦) are found
exactly where they already live β a namespacing change that renamed tenant #0's keys would be
a silent data loss dressed as a refactor. Every OTHER tenant is prefixed, which is what makes
proof #3 hold.
"""
key: str
tenant: object # harness.base.Tenant
store_namespace: str = ""
pool_cache: dict = field(default_factory=dict)
# Names/edges only, bounded and revision-keyed by routes_grid; no row pools live here.
related_cache: dict = field(default_factory=dict)
#: EXIT-5 β the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps
#: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on
#: (stamp, scope, pool identity, question) so nothing user-shaped lives in them.
measure_memo: dict = field(default_factory=dict)
mset_memo: dict = field(default_factory=dict)
#: Wave 2026-08-02 (C-TS): the time-series channel's own memo β separate from
#: measure_memo so column churn and series churn cannot evict each other's answers.
series_memo: dict = field(default_factory=dict)
_lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
@property
def name(self):
return getattr(self.tenant, "name", self.key)
@property
def datastore_path(self):
"""The DuckDB file this tenant's analytical reads belong to (file-per-tenant IS the
isolation model β DuckDB has no RLS, so the OS boundary is the only one that holds)."""
return _ds.path_for(self.key)
# ---------------------------------------------------------------- the store handle
#: Wave 18 (C1-TENANT, R2): a tenant that OWNS its dataset repo carries its bound Store
#: instance here (namespace stays EMPTY β a whole repo needs no prefix). None = the shared
#: default repo through the MODULE functions, which is what keeps the gates' fake-store
#: installation effective (the fake patches module attributes, and royal-imports + the
#: prefix tenants resolve through exactly those).
store_handle: object = None
def _store(self):
return self.store_handle if self.store_handle is not None else store
def _document_store(self):
backend = self._store()
backend = store._d() if backend is store else backend
slug = getattr(backend, 'tenant_slug', None)
if slug is not None and slug != self.key:
raise DatastoreMismatch('The store handle belongs to another tenant.')
return backend
def store_key(self, name):
"""Namespace a product-data key for this tenant. The ONE place the prefix is applied."""
return f"{self.store_namespace}{name}" if self.store_namespace else str(name)
def get(self, name, fresh=False):
# A fresh read of these keys raises on a store error: a deletion obligation is never read as {}.
if fresh and str(name).startswith(("sheet_scope_cache_", "sheet_amazon_deletions",
"amazon_connection_revocations")):
backend = self._store()
reader = getattr(backend, '_read_strict', None)
if callable(reader):
return reader(self.store_key(name))
return self._store().get(self.store_key(name), fresh=fresh)
def get_projection(self, name, drop=()):
"""A tenant-scoped PROJECTED read β the bucket without the keys named in `drop`.
ββ W32-T01 (D-185). `get()` deep-copies the whole document; on tenant #0's `user_tables`
that is 28.6 MB / ~703 ms warm, and 99.9% of it is `rows` that no nav render, permission
check or workspace envelope reads.
β **`_store()` returns the MODULE for every tenant with `store_handle is None`** β which
includes tenant #0 β so this delegates to a MODULE-LEVEL `get_projection`, not to a method.
Both backends implement it and `verify_store_pg.IFACE` pins that they must; see
`core.store.get_projection`'s note for why the method alone would have been unreachable
from exactly the caller it was built for.
"""
return self._store().get_projection(self.store_key(name), drop=drop)
def get_projections(self, names, drop=(), timeout_s=None):
"""Batch projected reads when the active bound store has a native batch operation.
``None`` means the backend has no batch primitive; callers retain their ordinary per-key
path. That keeps this an optional optimization rather than widening the store contract.
"""
reader = getattr(self._store(), "get_projections", None)
if not callable(reader):
return None
keyed = [(str(name), self.store_key(name)) for name in (names or ())]
answer = reader([store_key for _name, store_key in keyed], drop=drop,
timeout_s=timeout_s)
return {name: answer.get(store_key, {}) for name, store_key in keyed}
def get_workbook_projection(self, name, *, metadata_only=False, tab_ids=None, cells=None,
expected_rev=None):
"""Read one tenant's selected workbook cells without caching permissions or source answers."""
reader = getattr(self._document_store(), 'get_workbook_projection', None)
if callable(reader):
return reader(self.store_key(name), metadata_only=metadata_only, tab_ids=tab_ids,
cells=cells, expected_rev=expected_rev)
metadata_only, tab_ids, cells = store._workbook_selection(
metadata_only=metadata_only, tab_ids=tab_ids, cells=cells)
return store._project_workbook(self.get(name), metadata_only=metadata_only,
tab_ids=tab_ids, cells=cells, expected_rev=expected_rev)
def list_keys(self, prefix=''):
names = self._document_store().list_keys(self.store_key(prefix))
return [name[len(self.store_namespace):] for name in names
if name.startswith(self.store_namespace)]
def list_paths(self, prefix=''):
paths = self._document_store().list_paths(self.store_key(prefix))
return [path[len(self.store_namespace):] for path in paths
if path.startswith(self.store_namespace)]
def delete_document(self, name):
return self._document_store().delete_document(self.store_key(name))
def amazon_backup_policy(self, *, trigger_at, checked_at):
"""Delegate backup evidence only to this runtime's tenant-bound backend."""
backend = self._document_store()
reader = getattr(backend, 'amazon_backup_policy', None)
if callable(reader):
policy = reader(trigger_at=trigger_at, checked_at=checked_at)
if not isinstance(policy, dict) or policy.get('tenant') != self.key:
raise DatastoreMismatch('Backup evidence belongs to another tenant.')
return policy
return {'tenant': self.key, 'fixture': False, 'state': 'UNPROVEN',
'policyId': None, 'source': None, 'method': None,
'expiresAt': None, 'verifiedExpiredAt': None,
'cause': 'tenant_backup_policy_unavailable'}
def get_many(self, names):
"""Batch ordinary reads when the active bound store offers that optimization."""
reader = getattr(self._store(), "get_many", None)
if not callable(reader):
return None
keyed = [(str(name), self.store_key(name)) for name in (names or ())]
answer = reader([store_key for _name, store_key in keyed])
return {name: answer.get(store_key, {}) for name, store_key in keyed}
def get_entry(self, name, entry):
"""One top-level entry of `name` when the bound store offers it, else None (HOTFIX
2026-09-28). None also means absent; either way the caller falls back to `get`."""
reader = getattr(self._store(), "get_entry", None)
if not callable(reader):
return None
return reader(self.store_key(name), entry)
def revision(self, name):
"""Return the tiny durable change token for this tenant's addressed bucket."""
return self._store().revision(self.store_key(name))
def put(self, name, data):
return self._store().put(self.store_key(name), data)
def transaction(self, connection):
"""Enlist this tenant's writes in an existing receipt transaction."""
return self._document_store().transaction(connection)
def update(self, name, fn, flush="sync", expect_rev=None, audit_row=None):
"""Read-modify-write this tenant's bucket, optionally rev-guarded and audited (C3 / R9).
ββ W50-T28 CLAUSE 1 β THE TWO KEYWORDS EXIST SO `control.audit` CAN BE REACHED AT ALL.
`core/store_pg.py::update` grew `expect_rev` and `audit_row` in W50-T25 and every API
caller passes `st=session.runtime`, so until this method NAMED them the capability was
written, gated and unreachable β the [[tenant-runtime-update-is-a-keyword-bottleneck]]
shape. `expect_rev` refuses a write whose caller was looking at an older revision;
`audit_row` is the LIST the caller's own closure fills, carried into the same transaction
as the workspace write so the trail commits with the change or not at all.
ββ THE FORWARD IS CONDITIONAL, AND THAT IS THE ENTIRE CARE IN THIS METHOD. `_store()`
returns the `core.store` MODULE for every tenant with `store_handle is None` β which
includes tenant #0 β and that module's `update(name, fn, flush='sync')` names neither
keyword. An unconditional forward is a `TypeError` on the hot path for exactly the
tenant that matters most.
ββ AND A SIGNATURE IS A CAPABILITY DECLARATION HERE, NOT A CONVENIENCE.
`view_audit.home()` and `view_audit.guards()` answer by inspecting THIS method's
parameters, so merely naming the keywords tells the whole repo that every tenant can
carry a row into its transaction and enforce a revision inside its own lock. Where the
inner handle cannot, both promises must be kept HERE or they are broken silently:
* `expect_rev` -> the ADVISORY check (read `revision()`, compare, refuse). Identical to
the fallback `table_store._audited_update` used to take on its own, which this
signature change is what disables. It has a real TOCTOU window and still refuses
every stale write it can see.
* `audit_row` -> appended to the `__audit__` DOCUMENT ring inside the caller's closure,
which is where `view_audit.record` would have put it had `home()` still said
`DOCUMENT`. Without this the row is not relocated, it is LOST.
β WHAT THIS DOES NOT CLOSE, and the handoff says so: tenant #0 resolves through the
`core.store` MODULE function, which drops both keywords before `_d()` ever reaches
`PgStore.update`. That tenant therefore takes the compensating path above and its trail
stays in the ring. The second bottleneck is `core/store.py::update`, in nobody's fence.
"""
st = self._store()
slug = getattr(st, 'tenant_slug', None)
if slug is not None and slug != self.key:
raise DatastoreMismatch('The store handle belongs to another tenant.')
key = self.store_key(name)
if expect_rev is None and audit_row is None:
return st.update(key, fn, flush=flush)
# Imported HERE, not at module scope: `api/main.py` imports half the platform before it
# has read an environment variable, and `core.view_audit` reaches `core.store_pg`.
from core import view_audit
if view_audit.guards(st) and view_audit.home(st) == view_audit.CONTROL_AUDIT:
return st.update(key, fn, flush=flush,
expect_rev=expect_rev, audit_row=audit_row)
if expect_rev is not None:
self._refuse_stale_revision(name, expect_rev)
if audit_row is not None:
fn = self._appending_audit_rows(fn, audit_row)
return st.update(key, fn, flush=flush)
def _refuse_stale_revision(self, name, expect_rev):
"""Raise `ViewRevisionStale` when this bucket has moved past `expect_rev`.
The ADVISORY half of the guard, for a handle that cannot enforce one inside its own
locking read. `RevisionUnverifiable` rather than a silent pass when the revision cannot
be read at all: "I could not tell" and "it matches" are different answers, and only one
of them may let a write land.
"""
from core import view_audit
if not callable(getattr(self._store(), "revision", None)):
raise view_audit.RevisionUnverifiable(
"this deployment cannot tell which revision of the view list you are looking "
"at, so the change was not saved. Reload and try again.")
try:
current = int((self.revision(name) or {}).get("rev") or 0)
except Exception as exc: # noqa: BLE001
raise view_audit.RevisionUnverifiable(
"the view list revision could not be read, so the change was not saved. "
"Reload and try again.") from exc
if current != int(expect_rev):
raise view_audit.ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev))
@staticmethod
def _appending_audit_rows(fn, rows):
"""Wrap `fn` so the rows it collected land in the document ring, in its own transaction.
β `rows` IS EMPTY WHEN THIS RUNS AND FULL AFTERWARDS β the caller's closure appends to it
while it executes, which is the only moment a row's `before` and `after` are both
knowable. So the append happens after `fn` returns and reads the list then, never now.
"""
def _apply(data):
from core import view_audit
result = fn(data)
target = result if isinstance(result, dict) else data
for entry in rows:
view_audit.append_to_document(target, entry)
return result
return _apply
def exists(self, name):
return self._store().exists(self.store_key(name))
def available(self):
return self._store().available()
# ---------------------------------------------------------------- BYTES (wave 25, C's ask)
# β WHY THESE EXIST: `grid_events._blobs()` had no tenant-scoped way to reach bytes, so an R2
# tenant's UPLOADED FILES landed in the shared repo while its buckets went to its own β the
# residency fix with a hole in it. SESSION C wrote `_blobs` as a CAPABILITY TEST
# (`hasattr(st, "upload_bytes")`) precisely so the fallback disappears the moment these land,
# with no edit on that side.
#
# β THEY GO THROUGH `store_key`, AND THAT IS THE WHOLE POINT. A byte path is a different
# address space from a bucket name, and the temptation is to pass it straight through β but
# the isolation requirement is identical: a tenant on the SHARED repo (namespace `t/<slug>/`)
# writing `uploads/photo.png` would otherwise land on the exact path another tenant writes.
# An own-repo tenant has an EMPTY namespace, so the path is untouched and the repo boundary
# does the work, which is the same asymmetry every method above already has. Reusing
# `store_key` keeps "the ONE place the prefix is applied" literally true rather than nearly.
def upload_bytes(self, path_in_repo, data, message=None):
return self._store().upload_bytes(self.store_key(path_in_repo), data, message=message)
def download_bytes(self, path_in_repo):
return self._store().download_bytes(self.store_key(path_in_repo))
def delete_path(self, path_in_repo):
return self._store().delete_path(self.store_key(path_in_repo))
# ---------------------------------------------------------------- the Odoo source (R3)
def odoo_source(self, connection_id=None):
"""THIS tenant's Odoo connector, or None β the keychain CUTOVER's resolution seam
(2026-08-04; `keychain.odoo_creds` was previously called by nothing but its test).
Resolution, fail-closed:
1. the keychain `odoo` connection NAMED by `connection_id` (W54-T45 / R4), or the
tenant's DESIGNATED DEFAULT connection when no id is named β a connector bound to
THOSE creds;
2. tenant #0 with an empty/locked keychain and NO id named β its compiled env
connector (R3's "live with env fallback", which keeps Royal working unchanged);
3. anyone else β None. NEVER the environment: env is tenant #0's connection, and
handing it to another tenant is the leak this method exists to prevent.
ββ W54-T45 β THE ENV FALLBACK NARROWS HERE AND MUST NEVER WIDEN. A caller that NAMES a
connection and gets nothing back is answered None, full stop: not the default connection,
not a sibling, and above all not tenant #0's `.env`. Under R4 an agency holds one Odoo
connection per client, and under R7 that isolation is strict β a named-but-unresolvable
connection quietly falling through to the environment would serve tenant #0's own Odoo to
a question asked about somebody else's books, with nothing red anywhere. Leg (2) is
reachable only on the UNNAMED path, which is exactly the pre-wave behaviour.
Rebuilt per call on purpose β `core.odoo` caches the CLIENT per (slug, creds
fingerprint), so this stays cheap while a rotated key reconnects on the next call."""
import core.keychain as keychain
named = connection_id is not None
try:
# β THE UNNAMED PATH IS SPELLED AS THE PRE-WAVE CALL, VERBATIM, and the branch is not
# decoration: `verify_api`'s R11 leg pins the literal `keychain.odoo_creds(self)` in
# this method's source as its proof that the resolver is KEYCHAIN-FIRST. Writing only
# the two-argument form would have turned that gate red for a reason that has nothing
# to do with the rule it guards, in a file this lane may not edit. The two arms are
# the same question; the left one is the one every existing caller asks.
creds = (keychain.odoo_creds(self) if not named
else keychain.odoo_creds(self, connection_id))
except Exception: # locked or unreadable keychain β fall through, fail closed
creds = None
if creds:
from harness.connectors.odoo import OdooConnector
cfg = getattr(self.tenant, "config", {}) or {}
odoo_cfg = self.get(_ds._ODOO_CONFIG_KEY) or {}
source = OdooConnector(tenant_slug=self.key, creds=creds,
scope=team_scope(cfg.get("business_units"), cfg.get("team_pricelists")))
return _business_source(source, odoo_cfg.get('businessTz'))
# β THE NAMED LEG STOPS HERE. See the docstring: an explicit connection that did not
# resolve is None, never the environment and never another connection.
if named:
return None
if self.key == "royal-imports":
source = (getattr(self.tenant, "sources", {}) or {}).get("odoo")
return _business_source(source, (self.get(_ds._ODOO_CONFIG_KEY) or {}).get('businessTz'))
return None
# ------------------------------------------------------- the connector PAUSE (D-10, wave 24)
def odoo_flag_key(self, connection_id=None):
"""WHICH connector flag governs this tenant's Odoo β the key `connectors[<k>].paused`
is stored under, or None when nothing would serve.
THE SAME RESOLUTION AS `odoo_source()` ABOVE, expressed as an identity instead of a
client: the keychain `odoo` connection named (or the designated default when none is),
else tenant #0's compiled env source, else nothing. It lives here so there is ONE
resolver β `routes_keychain._resolved_odoo_key` was a second copy of these three rules in
the API layer, and a pause flag written against one resolution and read against the other
freezes nothing while reporting success.
ββ W54-T45 β AND THE INLINE WALK IS GONE, WHICH IS WHAT MADE THE SENTENCE ABOVE TRUE.
This method claimed to be the one resolver while carrying its own copy of "the first
`odoo` entry", so `keychain` and `runtime` each decided which connection serves. Both now
ask `keychain.resolve_connection_id`, which decrypts nothing β a pause flag has to resolve
on a LOCKED keychain, which is why this is an identity call and not a credential one.
β A NAMED CONNECTION THAT DOES NOT RESOLVE IS None, exactly as in `odoo_source`: pausing
by a connection id must never silently address tenant #0's environment source instead.
"""
import core.keychain as keychain
named = connection_id is not None
try:
resolved = keychain.resolve_connection_id(self, "odoo", connection_id)
except Exception:
resolved = None
if resolved and keychain.unlocked():
return resolved
if named:
return None
if self.key == "royal-imports" and os.environ.get("ODOO_URL"):
return ENV_ODOO_FLAG_KEY
return None
def odoo_paused(self, connection_id=None):
"""Is THIS tenant's RESOLVED Odoo source paused?
β RESOLVED, not "any entry" β pausing an entry that is not the one serving freezes
nothing, because it serves nothing. Never raises: an unreachable flags bucket answers
False, because a store hiccup must not freeze a live surface. That policy is inherited
from the API-layer function this one replaced, deliberately, so there is one answer to
"what happens when we cannot tell" rather than two that can disagree.
β W54-T45 β `connection_id` passes straight through to `odoo_flag_key`, so "is the
connection serving THIS client paused" is answerable per connection under R4. No id keeps
meaning the tenant's designated default, which is every existing caller.
"""
try:
flag_key = self.odoo_flag_key(connection_id)
if not flag_key:
return False
flags = self.get(CONNECTOR_FLAGS_KEY) or {}
return bool((flags.get(flag_key) or {}).get("paused"))
except Exception:
return False
def mirror_handle(self):
"""Resolve only this tenant's configured mirror, even when its file is absent."""
return _ds.tenant_handle(self.key, self.datastore_path)
def assert_datastore_matches(self, cur=None):
"""Refuse a handle or borrowed cursor belonging to another tenant."""
handle = self.mirror_handle()
self._assert_mirror_cursor(handle, cur)
def _assert_mirror_cursor(self, handle, cur):
"""Validate a borrowed cursor against the handle acquired for this operation."""
if handle.tenant_key != self.key:
raise DatastoreMismatch("The mirror handle belongs to another tenant.")
if cur is None:
return
bound = getattr(cur, "_mirror_handle", None)
if bound is not None:
if bound.tenant_key != self.key or bound.path != handle.path:
raise DatastoreMismatch("The mirror cursor belongs to another tenant.")
return
marks = _ds.mirror_file_marks(cur)
expected = _ds.serving_file(handle.path)[0] if _ds.versioned_mode() else handle.path
if marks is None or marks[0] != str(expected.resolve()):
raise DatastoreMismatch("The borrowed mirror cursor belongs to another tenant.")
@staticmethod
def _store_owner(path):
"""Return the forward owner and whether the registry and path mapping prove isolation.
β FORWARD-MAPPED, never parsed out of the filename β `datastore.path_for` substitutes `_`
for every non-alphanumeric character and truncates at 60, so `a.b` and `a_b` produce the
same file and the name is genuinely not invertible. `tenant_for_store_path` above says the
same thing and is not reused here for one reason: it swallows every failure into `""`, and
this caller must be able to tell "nobody owns it" from "I could not find out".
β AND `known_tenants()` IS NOT REUSED EITHER, FOR THE SAME REASON ONE LAYER DOWN β it
cannot raise: `_tenant_records()` returns `{}` on any failure and `known_tenants` wraps
that in another `except: pass`. Calling it here would have made `registry_ok` a constant
`True` and the fail-closed branch above unreachable, i.e. a guard whose most important
clause is dead code that reads as if it fires ([[gate-can-report-green-on-nothing]]). The
control-plane bucket is therefore read HERE, where its failure is still visible.
"""
keys, registry_ok = set(_BUILDERS), True
try:
registries = [store.get(TENANTS_KEY)]
# HF display failures cannot prove uniqueness. Confirm through its strict reader.
if store.backend() != "pg":
registries.append(store._read_strict(TENANTS_KEY))
for recs in registries:
recs = {} if recs is None else recs
if not isinstance(recs, dict) or any(not isinstance(v, dict) for v in recs.values()):
raise ValueError("The tenant ownership registry is malformed.")
keys |= {str(k).strip().lower() for k in recs}
except Exception: # noqa: BLE001
registry_ok = False
owners = []
for key in sorted(keys):
try:
if _ds.path_for(key).resolve() == path:
owners.append(key)
except Exception: # noqa: BLE001
registry_ok = False
continue
return (owners[0] if len(owners) == 1 else "ambiguous" if owners else ""), registry_ok and len(owners) <= 1
def register(key, builder):
"""Add a tenant to the registry (a stub connector is a legitimate builder β see X8's qa-b).
Invalidates any cached runtime for the slug so a re-registration cannot be shadowed."""
_BUILDERS[str(key)] = builder
invalidate(key)
def worker_tenant():
"""The one tenant a per-tenant sync worker serves, or "" in every other process."""
if (os.environ.get("AIOS_PROCESS_ROLE") != "worker"
or os.environ.get("AIOS_WORKER_SCOPE") != "tenant"):
return ""
return str(os.environ.get("AIOS_SYNC_WRITER_TENANT") or "").strip().lower()
def known_tenants():
keys = set(_BUILDERS)
try:
keys |= set(_tenant_records())
except Exception:
pass
# A per-tenant sync worker holds one tenant's file, so it never sees another tenant.
only = worker_tenant()
if only:
keys &= {only}
return sorted(keys)
#: Wave 18 (C1-TENANT) β the CONTROL-PLANE bucket: tenants provisioned at runtime rather than
#: compiled in. Lives in the DEFAULT store (tenant #0's repo) deliberately: it answers "which
#: tenants exist", which is a platform fact, not any one tenant's data.
TENANTS_KEY = "tenants"
def _tenant_records():
"""{slug: record} from the control-plane bucket. {} on any failure β an unreachable bucket
must degrade to "only the compiled-in tenants exist", never to an exception at login."""
try:
recs = store.get(TENANTS_KEY) or {}
return {str(k).strip().lower(): v for k, v in recs.items() if isinstance(v, dict)}
except Exception:
return {}
def _builder_from_record(key, rec):
"""A Tenant for a bucket-provisioned client: no sources yet (connectors arrive via the
Keychain flow), config carries the platform fields the request path reads."""
def _build():
from harness.base import Tenant
return Tenant(key=key, name=str(rec.get("name") or key),
config={"modules": rec.get("modules", []),
"store_repo": rec.get("store_repo") or None})
return _build
def get_runtime(tenant_key):
"""The runtime for `tenant_key`, built on first use and LRU-cached.
Raises KeyError for an unknown slug β the caller turns that into a 401 (fail-closed: an
unknown tenant is not a 404, because confirming which slugs exist answers a question the
request was not entitled to ask).
"""
key = str(tenant_key or "").strip().lower()
if not key:
raise KeyError("no tenant")
only = worker_tenant()
if only and key != only:
# Fail closed: this process's DuckDB file belongs to `only`.
raise KeyError(key)
with _LOCK:
rt = _CACHE.get(key)
if rt is not None:
_CACHE.move_to_end(key) # LRU: touch on read
return rt
builder = _BUILDERS.get(key)
# Wave 18 (C1-TENANT): a slug the compiled registry does not know may be a PROVISIONED
# tenant β consult the control-plane bucket before failing closed. A suspended record is
# the same KeyError as an unknown one: "which tenants exist" is not this caller's question.
if builder is None:
rec = _tenant_records().get(key)
if not rec or rec.get("status", "active") != "active":
raise KeyError(key)
builder = _builder_from_record(key, rec)
# Built OUTSIDE the lock: a builder may do real work (a connector handshake), and holding the
# registry lock through it would serialise every other tenant's requests behind it.
tenant = builder()
with _LOCK:
existing = _CACHE.get(key)
if existing is not None: # another thread won the race β keep ONE runtime
_CACHE.move_to_end(key)
return existing
# R2: a tenant with its OWN repo gets a bound Store instance and an EMPTY prefix (the
# repo boundary is the namespace); everyone else keeps the t/<slug>/ prefix in the
# shared repo. Tenant #0 keeps both defaults β empty prefix, shared repo.
#
# β WAVE 20 (R1 / D-4): under Postgres the address is a SCHEMA, so `store.handle()` is
# asked for one by SLUG and the namespace prefix goes to EMPTY for EVERY tenant β
# `t_<slug>.store_kv` already isolates, and prefixing keys inside an isolated schema
# would namespace them twice. That is not a cosmetic difference: a `t/nurilab/`-prefixed
# key written into `t_nurilab` is a key the migration did not copy and no reader looks
# for, i.e. a silent empty workspace.
#
# β NOTHING ELSE IN THIS FILE CHANGES, and that is the design working. Every product-data
# read/write already goes through `rt.get/put/update/exists` (the `st=` handle threaded
# through the modules), so binding the handle correctly here IS the cutover for the
# tenant-aware paths β the module-level `core.store` functions cover the rest via `_d()`.
repo = (getattr(tenant, "config", {}) or {}).get("store_repo")
if store.backend() == "pg":
rt = TenantRuntime(key=key, tenant=tenant, store_namespace="",
store_handle=store.handle(slug=key))
else:
rt = TenantRuntime(
key=key, tenant=tenant,
store_namespace=("" if (key == "royal-imports" or repo) else f"t/{key}/"),
store_handle=(store.for_repo(repo) if repo else None))
_initialize_deletion_owners(rt)
_CACHE[key] = rt
while len(_CACHE) > _MAX_RUNTIMES:
_CACHE.popitem(last=False) # evict least recently used
return rt
def invalidate(tenant_key=None):
"""Drop one tenant's runtime, or all of them. Explicit, per X7 β a cache with no documented
way to clear it becomes a restart."""
with _LOCK:
if tenant_key is None:
_CACHE.clear()
else:
_CACHE.pop(str(tenant_key or "").strip().lower(), None)
def resident():
"""The tenant slugs currently holding a runtime β LRU order, oldest first. For the EXIT-4
measurement and for asserting the bound actually binds."""
with _LOCK:
return list(_CACHE)
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# β DEBT D-10 (wave 24) β TEACHING THE MEASURE MIRROR ABOUT THE CONNECTOR PAUSE.
#
# `harness.datastore` is the store every measure column and condition is answered from, and it
# pulled from Odoo straight through a pause: 12 sync passes at boot and one every 1800 s. It
# cannot ask the question itself β the flag lives in a tenant's store bucket, which is this
# module's to read, and this module already imports datastore, so an import back would be a
# cycle. So datastore exposes a probe slot and this module fills it, once, at import.
def tenant_for_store_path(path=None):
"""Which tenant owns the DuckDB file currently open (or `path`)? `""` when nothing matches.
β RESOLVED BY FORWARD-MAPPING every known tenant, never by parsing the filename, and that is
a correctness point rather than a style one: `datastore.path_for` substitutes `_` for every
non-alphanumeric character and truncates at 60, so the name is genuinely NOT invertible β
`a.b` and `a_b` produce the same file. Guessing the slug back out of it would eventually name
the wrong tenant, and this answer decides whether another tenant's connector freezes ours.
"""
want = Path(path or _ds.DB_PATH).resolve()
# Tenant #0 first: it is the only tenant the unattended sync actually runs for today (see
# D-29 β the resync loop prewarms `royal-imports` hardcoded), so this is the hot answer.
for key in ["royal-imports"] + [k for k in known_tenants() if k != "royal-imports"]:
try:
if _ds.path_for(key).resolve() == want:
return key
except Exception:
continue
return ""
def store_source_paused():
"""The probe `harness.datastore` calls before reaching Odoo. Never raises; False when the
tenant cannot be resolved, matching the fail-towards-live policy `odoo_paused` documents."""
try:
key = tenant_for_store_path()
return bool(key) and get_runtime(key).odoo_paused()
except Exception:
return False
_ds.set_paused_probe(store_source_paused)
def _business_source(source, zone):
"""Adapt the connector's explicit binding without mutating its shared tenant source."""
if source is None:
return None
import copy
result = copy.copy(source)
result._bound = lambda: _odoo.tenant_scope(
result._slug, result._creds, **result._scope, business_tz=zone)
return result
def team_scope(business_units, team_pricelists=None):
"""A tenant config's `business_units` ({label: team id}) and `team_pricelists` ({label: lists})
as `core.odoo.tenant_scope` kwargs. A pricelist row whose label names no team is dropped."""
bus = dict(business_units or {})
lists = {bus[label]: tuple(names) for label, names in (team_pricelists or {}).items()
if label in bus}
return {"team_ids": list(bus.values()) or None,
"team_names": {v: k for k, v in bus.items()} or None,
"team_pricelists": lists or None}
def mirror_team_names(slug):
"""{team_id: Odoo team name} from a RESIDENT tenant's mirror, read only; {} when none is.
Never builds a runtime: a team lookup must not reach the store or the network."""
with _LOCK:
rt = _CACHE.get(str(slug or "").strip().lower())
if rt is None:
return {}
cur = mirror_cursor(rt)
try:
try:
rows = cur.execute("SELECT team_id, max(team_name) FROM sale_order "
"WHERE team_id IS NOT NULL GROUP BY team_id").fetchall()
except Exception: # a mirror without the name column still lists its team ids
rows = [(t, None) for (t,) in cur.execute(
"SELECT DISTINCT team_id FROM sale_order WHERE team_id IS NOT NULL").fetchall()]
finally:
cur.close()
return {int(t): (str(n) if n else str(t)) for t, n in rows}
# core may not import harness: push the default tenant's configured teams and the mirror reader
# down at process start, the same way `set_paused_probe` above reaches the datastore.
_odoo.set_default_scope(**team_scope(_tenants.TENANT0_BUSINESS_UNITS,
_tenants.TENANT0_TEAM_PRICELISTS))
_odoo.set_team_reader(mirror_team_names)
|