fsanyoto commited on
Commit
b170807
·
verified ·
1 Parent(s): 317b59d

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "1bc5ab5e283c (1bc5ab5e283c)",
3
  "releases": [
4
  {
5
  "version": "v83",
 
1
  {
2
+ "current": "5a3e53380f59 (5a3e53380f59)",
3
  "releases": [
4
  {
5
  "version": "v83",
VERSION CHANGED
@@ -1 +1 @@
1
- 1bc5ab5e283c (1bc5ab5e283c)
 
1
+ 5a3e53380f59 (5a3e53380f59)
api/automation_engine.py CHANGED
@@ -16723,8 +16723,11 @@ def start_scheduler(log=print):
16723
  return False
16724
  if _SCHEDULER[0] is not None:
16725
  return False
16726
- th = threading.Thread(target=scheduler_loop, kwargs={"log": log},
16727
- daemon=True, name="automation-scheduler")
 
 
 
16728
  _SCHEDULER[0] = th
16729
  th.start()
16730
  return True
 
16723
  return False
16724
  if _SCHEDULER[0] is not None:
16725
  return False
16726
+ import process_role
16727
+ th = process_role.thread(target=scheduler_loop, kwargs={"log": log},
16728
+ daemon=True, name="automation-scheduler", kind="heavy")
16729
+ if not process_role.admitted(th):
16730
+ return False
16731
  _SCHEDULER[0] = th
16732
  th.start()
16733
  return True
api/main.py CHANGED
@@ -36,6 +36,7 @@ for _p in (str(_RI), str(_HERE.parent)):
36
 
37
  from dotenv import load_dotenv # noqa: E402
38
  load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
 
39
 
40
  from fastapi import Body, Depends, FastAPI, Request # noqa: E402
41
  from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
@@ -923,7 +924,12 @@ def _pull_amazon(why, vendor=True):
923
 
924
  def _restore_amazon_mirror(why):
925
  """Restore every known tenant's Amazon manifest without moving the resident file."""
 
 
926
  try:
 
 
 
927
  from harness import runtime as _runtime
928
  results = [_restore_amazon_mirror_once(why, key) for key in _runtime.known_tenants()]
929
  errors = RESYNC_STATE.get("amazonRestoreErrors", {})
@@ -931,6 +937,8 @@ def _restore_amazon_mirror(why):
931
  RESYNC_STATE["amazonRestoreError"] = next(iter(errors.values()))
932
  else:
933
  RESYNC_STATE.pop("amazonRestoreError", None)
 
 
934
  return all(results) and not errors
935
  except Exception as exc: # noqa: BLE001
936
  RESYNC_STATE["amazonRestoreError"] = {
@@ -977,7 +985,9 @@ def _restore_amazon_mirror_once(why, tenant_key):
977
  if result.get("state") == "refused":
978
  raise RuntimeError("; ".join(result.get("problems") or ["Amazon mirror restore refused."]))
979
  # Publish measured definitions before any Odoo read can block the resident writer.
980
- if (result.get("state") != "absent" and
 
 
981
  rt.datastore_path.resolve() == _ds.DB_PATH.resolve()):
982
  state = _rebuild_amazon_relational(why, rt)
983
  if state.get("changed") and not state.get("applied") and not state.get("known"):
@@ -1293,14 +1303,12 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
1293
  # ⛔ HOTFIX 2026-09-29: NOT every 30 seconds. A pass that landed nothing can only defer
1294
  # again until Amazon's next report floor opens, and each of those passes republished
1295
  # the mirror's state to the store. Wait for the floor, then continue.
1296
- import threading as _threading_desk
1297
  delay = min(max(floor_wait + 5.0, 60.0), 86400.0)
1298
- timer = _threading_desk.Timer(delay, connector_sync.request_sync,
1299
- args=("amazon_sp", tenant),
1300
- kwargs={"requested_by": "first load"})
1301
- timer.daemon = True
1302
- timer.start()
1303
- print(f"[aios-api] amazon first load waits {int(delay)}s for the next report floor")
1304
  elif continuing and ok:
1305
  # Finish first: the desk refuses a second in-flight intent. The resident loop claims
1306
  # the continuation as a DESK-ONLY wake; the Odoo pass keeps its own cadence.
@@ -2776,14 +2784,17 @@ def _boot_app_migrations(why):
2776
  def _prewarm():
2777
  if _freeze_holds("prewarm"):
2778
  return
 
2779
  import time as _t
2780
  t0 = _t.time()
2781
  # The Odoo seed/sync can run for minutes and is separate from the finite cache warm below.
2782
  # Keep it opt-in: Live Pg serves its seeded analytical mirror at boot and lets explicit
2783
  # workers perform real sync work, so a routine restart can still scale Neon back to zero.
2784
  if os.environ.get("AIOS_BOOT_SYNC") == "1" and os.environ.get("AIOS_STORE_RESYNC") != "1":
2785
- import threading as _th
2786
- _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start()
 
 
2787
  try:
2788
  from harness import runtime as _runtime
2789
  rt = _runtime.get_runtime("royal-imports")
@@ -2948,25 +2959,43 @@ def _await_mirror_restore():
2948
  return
2949
  thread = globals().get("_SEED_THREAD")
2950
  if thread is None:
2951
- import threading
2952
- thread = threading.Thread(target=_boot_mirrors, daemon=True, name="mirror-seed")
 
2953
  globals()["_SEED_THREAD"] = thread
2954
  thread.start()
2955
 
2956
 
 
 
 
 
 
 
 
 
 
2957
  def _restore_sync_request_desks():
2958
  from core import connector_sync
2959
  from harness import runtime
 
 
 
2960
  for tenant_key in runtime.known_tenants():
2961
  try:
2962
- connector_sync.restore_sync_requests(runtime.get_runtime(tenant_key))
2963
  except Exception as exc:
2964
  print(f"[aios-api] sync request restore refused for {tenant_key} ({type(exc).__name__})")
2965
 
2966
 
2967
  def _boot_mirrors():
 
2968
  _restore_sync_request_desks()
2969
  _seed_mirror_if_absent()
 
 
 
 
2970
  _restore_amazon_mirror("boot")
2971
  _restore_declared_mirrors("boot")
2972
 
@@ -2997,9 +3026,9 @@ def _restore_declared_mirrors(why):
2997
  try:
2998
  from harness import datastore as _ds_boot
2999
  if not data_binding.gate_process() and (data_binding.is_space() or not _ds_boot.DB_PATH.exists()):
3000
- import threading as _threading_seed
3001
- _SEED_THREAD = _threading_seed.Thread(target=_boot_mirrors, daemon=True,
3002
- name="mirror-seed")
3003
  _SEED_THREAD.start()
3004
  except Exception as e: # noqa: BLE001
3005
  print(f"[aios-api] mirror seed not scheduled: {e}")
@@ -3012,25 +3041,26 @@ if (not data_binding.gate_process()
3012
  and os.environ.get("AIOS_ENABLE_QA_TENANT") == "1"
3013
  and os.environ.get("STORE_BACKEND") == "pg"
3014
  and not _freeze_holds("QA schema provisioning")):
3015
- import threading as _threading_qa
3016
- _threading_qa.Thread(target=_provision_qa_sandbox, daemon=True,
3017
- name="qa-sandbox-schema").start()
3018
 
3019
  if (not data_binding.gate_process()
3020
  and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
3021
  and (data_binding.is_space() or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
3022
  and not _freeze_holds("app migrations")):
3023
- import threading as _threading_apps
3024
- _threading_apps.Thread(target=_boot_app_migrations, args=("boot",), daemon=True,
3025
- name="app-migrations").start()
3026
 
3027
  if os.environ.get("AIOS_BOOT_SYNC") == "1" and freeze_notice.enabled():
3028
  _freeze_holds("boot store sync")
3029
 
3030
  if (not data_binding.gate_process()
3031
  and os.environ.get("AIOS_PREWARM") == "1" and not _freeze_holds("prewarm")):
3032
- import threading as _threading
3033
- _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
 
3034
 
3035
  # A boot prewarm is finite; the Odoo/store resync loop is resident work, so it stays opt-in.
3036
  #
@@ -3065,7 +3095,7 @@ if (not data_binding.gate_process()
3065
  # ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
3066
  if (not data_binding.gate_process()
3067
  and os.environ.get("AIOS_STORE_RESYNC") == "1" and not _freeze_holds("store resync")):
3068
- import threading as _threading_resync
3069
- _threading_resync.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
3070
  import sheet_warm as _sheet_warm
3071
  _sheet_warm.start()
 
36
 
37
  from dotenv import load_dotenv # noqa: E402
38
  load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
39
+ import process_role # noqa: E402 (before any DuckDB connect: the web role sets its memory cap)
40
 
41
  from fastapi import Body, Depends, FastAPI, Request # noqa: E402
42
  from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
 
924
 
925
  def _restore_amazon_mirror(why):
926
  """Restore every known tenant's Amazon manifest without moving the resident file."""
927
+ # One good boot restore per process: the boot seed thread and the resync loop both ask for it.
928
+ boot_done = globals().setdefault("_AMAZON_BOOT_RESTORE", {})
929
  try:
930
+ if why == "boot" and boot_done.get("result") is True:
931
+ print("[aios-api] amazon mirror boot restore already done in this process")
932
+ return True
933
  from harness import runtime as _runtime
934
  results = [_restore_amazon_mirror_once(why, key) for key in _runtime.known_tenants()]
935
  errors = RESYNC_STATE.get("amazonRestoreErrors", {})
 
937
  RESYNC_STATE["amazonRestoreError"] = next(iter(errors.values()))
938
  else:
939
  RESYNC_STATE.pop("amazonRestoreError", None)
940
+ if why == "boot":
941
+ boot_done["result"] = all(results) and not errors
942
  return all(results) and not errors
943
  except Exception as exc: # noqa: BLE001
944
  RESYNC_STATE["amazonRestoreError"] = {
 
985
  if result.get("state") == "refused":
986
  raise RuntimeError("; ".join(result.get("problems") or ["Amazon mirror restore refused."]))
987
  # Publish measured definitions before any Odoo read can block the resident writer.
988
+ import process_role as _process_role
989
+ # The relational rebuild writes the store, so only the sync worker runs it.
990
+ if (result.get("state") != "absent" and _process_role.allows("heavy") and
991
  rt.datastore_path.resolve() == _ds.DB_PATH.resolve()):
992
  state = _rebuild_amazon_relational(why, rt)
993
  if state.get("changed") and not state.get("applied") and not state.get("known"):
 
1303
  # ⛔ HOTFIX 2026-09-29: NOT every 30 seconds. A pass that landed nothing can only defer
1304
  # again until Amazon's next report floor opens, and each of those passes republished
1305
  # the mirror's state to the store. Wait for the floor, then continue.
 
1306
  delay = min(max(floor_wait + 5.0, 60.0), 86400.0)
1307
+ import process_role as _process_role
1308
+ if _process_role.start("amazon-first-load-floor", "heavy", connector_sync.request_sync,
1309
+ args=("amazon_sp", tenant), kwargs={"requested_by": "first load"},
1310
+ delay=delay) is not None:
1311
+ print(f"[aios-api] amazon first load waits {int(delay)}s for the next report floor")
 
1312
  elif continuing and ok:
1313
  # Finish first: the desk refuses a second in-flight intent. The resident loop claims
1314
  # the continuation as a DESK-ONLY wake; the Odoo pass keeps its own cadence.
 
2784
  def _prewarm():
2785
  if _freeze_holds("prewarm"):
2786
  return
2787
+ import process_role as _process_role
2788
  import time as _t
2789
  t0 = _t.time()
2790
  # The Odoo seed/sync can run for minutes and is separate from the finite cache warm below.
2791
  # Keep it opt-in: Live Pg serves its seeded analytical mirror at boot and lets explicit
2792
  # workers perform real sync work, so a routine restart can still scale Neon back to zero.
2793
  if os.environ.get("AIOS_BOOT_SYNC") == "1" and os.environ.get("AIOS_STORE_RESYNC") != "1":
2794
+ _process_role.start("store-seed-sync", "heavy", _seed_and_sync_store)
2795
+ if _process_role.role() == "worker":
2796
+ # The warm below fills per-process memos, which the sync worker never serves.
2797
+ return
2798
  try:
2799
  from harness import runtime as _runtime
2800
  rt = _runtime.get_runtime("royal-imports")
 
2959
  return
2960
  thread = globals().get("_SEED_THREAD")
2961
  if thread is None:
2962
+ import process_role as _process_role
2963
+ # Light: the web process restores its resident mirrors so it has data to read.
2964
+ thread = _process_role.thread(target=_boot_mirrors, daemon=True, name="mirror-seed", kind="light")
2965
  globals()["_SEED_THREAD"] = thread
2966
  thread.start()
2967
 
2968
 
2969
+ @app.on_event("startup")
2970
+ def _start_process_role():
2971
+ """Web role: spawn and supervise the sync worker, forward Sync now, install its snapshots."""
2972
+ if data_binding.gate_process():
2973
+ return
2974
+ import process_role as _process_role
2975
+ _process_role.start_web_services()
2976
+
2977
+
2978
  def _restore_sync_request_desks():
2979
  from core import connector_sync
2980
  from harness import runtime
2981
+ import process_role as _process_role
2982
+ # The web role never claims, so it must not requeue the worker's in-flight claims.
2983
+ requeue = _process_role.role() != "web"
2984
  for tenant_key in runtime.known_tenants():
2985
  try:
2986
+ connector_sync.restore_sync_requests(runtime.get_runtime(tenant_key), requeue=requeue)
2987
  except Exception as exc:
2988
  print(f"[aios-api] sync request restore refused for {tenant_key} ({type(exc).__name__})")
2989
 
2990
 
2991
  def _boot_mirrors():
2992
+ import process_role as _role_boot
2993
  _restore_sync_request_desks()
2994
  _seed_mirror_if_absent()
2995
+ if _role_boot.role() == "web":
2996
+ # Every connector but Odoo reaches the web process through the worker's hand-off.
2997
+ print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
2998
+ return
2999
  _restore_amazon_mirror("boot")
3000
  _restore_declared_mirrors("boot")
3001
 
 
3026
  try:
3027
  from harness import datastore as _ds_boot
3028
  if not data_binding.gate_process() and (data_binding.is_space() or not _ds_boot.DB_PATH.exists()):
3029
+ import process_role as _role_seed
3030
+ _SEED_THREAD = _role_seed.thread(target=_boot_mirrors, daemon=True, kind="light",
3031
+ name="mirror-seed")
3032
  _SEED_THREAD.start()
3033
  except Exception as e: # noqa: BLE001
3034
  print(f"[aios-api] mirror seed not scheduled: {e}")
 
3041
  and os.environ.get("AIOS_ENABLE_QA_TENANT") == "1"
3042
  and os.environ.get("STORE_BACKEND") == "pg"
3043
  and not _freeze_holds("QA schema provisioning")):
3044
+ import process_role as _role_qa
3045
+ _role_qa.thread(target=_provision_qa_sandbox, daemon=True, kind="heavy",
3046
+ name="qa-sandbox-schema").start()
3047
 
3048
  if (not data_binding.gate_process()
3049
  and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
3050
  and (data_binding.is_space() or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
3051
  and not _freeze_holds("app migrations")):
3052
+ import process_role as _role_apps
3053
+ _role_apps.thread(target=_boot_app_migrations, args=("boot",), daemon=True, kind="heavy",
3054
+ name="app-migrations").start()
3055
 
3056
  if os.environ.get("AIOS_BOOT_SYNC") == "1" and freeze_notice.enabled():
3057
  _freeze_holds("boot store sync")
3058
 
3059
  if (not data_binding.gate_process()
3060
  and os.environ.get("AIOS_PREWARM") == "1" and not _freeze_holds("prewarm")):
3061
+ import process_role as _role_prewarm
3062
+ # Light: the cache warm fills this process's memos. Its store sync half is heavy inside.
3063
+ _role_prewarm.thread(target=_prewarm, daemon=True, kind="light", name="prewarm").start()
3064
 
3065
  # A boot prewarm is finite; the Odoo/store resync loop is resident work, so it stays opt-in.
3066
  #
 
3095
  # ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
3096
  if (not data_binding.gate_process()
3097
  and os.environ.get("AIOS_STORE_RESYNC") == "1" and not _freeze_holds("store resync")):
3098
+ import process_role as _role_resync
3099
+ _role_resync.thread(target=_store_resync_loop, daemon=True, kind="heavy", name="store-resync").start()
3100
  import sheet_warm as _sheet_warm
3101
  _sheet_warm.start()
api/process_role.py ADDED
@@ -0,0 +1,463 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """One process rule for every database: heavy background work runs only in the sync worker.
2
+
3
+ Roles: `web` serves reads and installs finished mirror snapshots, `worker` runs every heavy job,
4
+ `all` does both in one process (local runs and gates). Kinds: `heavy` moves source data or writes
5
+ the store in the background; `light` serves this process's own reads.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import atexit
10
+ import json
11
+ import os
12
+ import shutil
13
+ import subprocess
14
+ import sys
15
+ import threading
16
+ import time
17
+ import uuid
18
+ from pathlib import Path
19
+
20
+ ROLES = ("web", "worker", "all")
21
+ KINDS = ("heavy", "light")
22
+ API_DIR = Path(__file__).resolve().parent
23
+ POLL_SECONDS = 15
24
+ HEARTBEAT_SECONDS = 30
25
+ HEARTBEAT_FILE = "worker.heartbeat.json"
26
+ READY_FILE = "READY.json"
27
+ INSTALL_ATTEMPTS = 3
28
+ INCOMPLETE_GRACE_SECONDS = 3600
29
+
30
+
31
+ def resolve_role(env=None):
32
+ """(role, source) from AIOS_PROCESS_ROLE. Unset: `web` on a Space, `all` elsewhere."""
33
+ env = os.environ if env is None else env
34
+ raw = str(env.get("AIOS_PROCESS_ROLE") or "").strip().lower()
35
+ if raw in ROLES:
36
+ return raw, "AIOS_PROCESS_ROLE"
37
+ space = bool(str(env.get("SPACE_ID") or "").strip())
38
+ source = (f"unknown AIOS_PROCESS_ROLE {raw!r}" if raw else "AIOS_PROCESS_ROLE unset") + (
39
+ " on a Space" if space else " off a Space")
40
+ return ("web" if space else "all"), source
41
+
42
+
43
+ ROLE, ROLE_SOURCE = resolve_role()
44
+ # The data layer cannot import this module, so it reads the resolved role from the environment.
45
+ os.environ["AIOS_PROCESS_ROLE"] = ROLE
46
+ if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web":
47
+ print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True)
48
+ if ROLE == "web":
49
+ # The web process keeps a smaller DuckDB budget. apply_limits reads this at connect time.
50
+ os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "3GB")
51
+ if os.environ.get("AIOS_WEB_DUCKDB_THREADS"):
52
+ os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ["AIOS_WEB_DUCKDB_THREADS"])
53
+
54
+ _LOCK = threading.Lock()
55
+ _STARTED = {}
56
+ _STOP = threading.Event()
57
+ _WORKER = {"proc": None, "pid": None, "restarts": 0, "startedAt": None, "lastExit": None}
58
+ _INSTALLED = {}
59
+ _INSTALL_FAILURES = {}
60
+ LAST_INSTALL = {}
61
+ LAST_INSTALL_ERROR = {}
62
+
63
+
64
+ def role():
65
+ return ROLE
66
+
67
+
68
+ def allows(kind, role_name=None):
69
+ """Whether a job of this kind may start in the role."""
70
+ if kind not in KINDS:
71
+ raise ValueError(f"unknown job kind {kind!r}")
72
+ return kind == "light" or (role_name or ROLE) in ("worker", "all")
73
+
74
+
75
+ def admit(name, kind, role_name=None):
76
+ """allows(), plus one refusal line. For call sites whose callee owns its thread."""
77
+ if allows(kind, role_name):
78
+ return True
79
+ print(f"[process-role] refused {name} ({kind}) in role {role_name or ROLE}", flush=True)
80
+ return False
81
+
82
+
83
+ class _Refused:
84
+ """What thread() returns for a refused job: start() does nothing and returns None."""
85
+
86
+ def __init__(self, name):
87
+ self.name = name
88
+
89
+ def start(self):
90
+ return None
91
+
92
+ def is_alive(self):
93
+ return False
94
+
95
+ def join(self, timeout=None):
96
+ return None
97
+
98
+
99
+ def thread(*, target, name, kind, daemon=True, args=(), kwargs=None, delay=None, role_name=None):
100
+ """An unstarted Thread, or a Timer when delay is given. A refused job gets an inert stand-in."""
101
+ if not admit(name, kind, role_name):
102
+ return _Refused(name)
103
+ if delay is None:
104
+ job = threading.Thread(target=target, args=tuple(args), kwargs=dict(kwargs or {}),
105
+ daemon=daemon, name=name)
106
+ else:
107
+ job = threading.Timer(float(delay), target, args=list(args), kwargs=dict(kwargs or {}))
108
+ job.daemon = daemon
109
+ job.name = name
110
+ with _LOCK:
111
+ _STARTED[name] = {"kind": kind, "at": time.time()}
112
+ return job
113
+
114
+
115
+ def admitted(job):
116
+ """False for the stand-in thread() returns when it refuses a job."""
117
+ return not isinstance(job, _Refused)
118
+
119
+
120
+ def start(name, kind, target, *, daemon=True, args=(), kwargs=None, delay=None, role_name=None):
121
+ """Start one background job as a Thread, or a Timer when delay is given. None when refused."""
122
+ job = thread(target=target, name=name, kind=kind, daemon=daemon, args=args, kwargs=kwargs,
123
+ delay=delay, role_name=role_name)
124
+ if not admitted(job):
125
+ return None
126
+ job.start()
127
+ return job
128
+
129
+
130
+ def started_jobs():
131
+ with _LOCK:
132
+ return {name: dict(row) for name, row in _STARTED.items()}
133
+
134
+
135
+ # ----------------------------------------------------------------------------- the hand-off dir
136
+ def handoff_dir():
137
+ return Path(os.environ.get("AIOS_MIRROR_HANDOFF_DIR") or "/tmp/aios-mirror-handoff")
138
+
139
+
140
+ def _write_json(path, value):
141
+ """Write JSON through a sibling temp file and os.replace, so a reader never sees half of it."""
142
+ path.parent.mkdir(parents=True, exist_ok=True)
143
+ tmp = path.with_name(path.name + f".{os.getpid()}.tmp")
144
+ with open(tmp, "w", encoding="utf-8", newline="") as handle:
145
+ json.dump(value, handle, default=str)
146
+ os.replace(tmp, path)
147
+
148
+
149
+ def _read_json(path):
150
+ try:
151
+ with open(path, encoding="utf-8") as handle:
152
+ return json.load(handle)
153
+ except (OSError, ValueError):
154
+ return None
155
+
156
+
157
+ def write_handoff(connector, tenant_key, manifest, rows, paths, *, root=None, now=None):
158
+ """Keep one published snapshot's parquet for the web process. READY.json is written last."""
159
+ root = Path(root or handoff_dir())
160
+ snapshot = str(manifest.get("snapshot_id") or uuid.uuid4().hex)
161
+ target = root / connector / snapshot
162
+ tables = target / "tables"
163
+ tables.mkdir(parents=True, exist_ok=True)
164
+ kept = {}
165
+ for name, files in paths.items():
166
+ kept[name] = []
167
+ for index, source in enumerate(files):
168
+ copy = tables / f"{name}.{index}.parquet"
169
+ copy.unlink(missing_ok=True)
170
+ try:
171
+ os.link(source, copy)
172
+ except OSError:
173
+ shutil.copyfile(source, copy)
174
+ kept[name].append(str(copy))
175
+ _write_json(target / READY_FILE, {
176
+ "version": 1, "tenant": tenant_key, "connector": connector, "snapshotId": snapshot,
177
+ "completedAt": time.time() if now is None else now, "manifest": manifest,
178
+ "stateRows": [list(row) for row in rows], "paths": kept})
179
+ return target
180
+
181
+
182
+ def ready_handoffs(root=None):
183
+ """{(tenant, connector): [(completedAt, dir, ready), ...]} oldest first. Dirs without READY.json are skipped."""
184
+ root = Path(root or handoff_dir())
185
+ groups = {}
186
+ if not root.is_dir():
187
+ return groups
188
+ for connector_dir in root.iterdir():
189
+ if not connector_dir.is_dir():
190
+ continue
191
+ for snapshot_dir in connector_dir.iterdir():
192
+ ready = _read_json(snapshot_dir / READY_FILE) if snapshot_dir.is_dir() else None
193
+ if not isinstance(ready, dict) or not ready.get("tenant") or not isinstance(ready.get("paths"), dict):
194
+ continue
195
+ key = (str(ready["tenant"]), str(ready.get("connector") or connector_dir.name))
196
+ groups.setdefault(key, []).append((float(ready.get("completedAt") or 0), snapshot_dir, ready))
197
+ for rows in groups.values():
198
+ rows.sort(key=lambda row: row[0])
199
+ return groups
200
+
201
+
202
+ def _install_default(connector, tenant, ready):
203
+ from harness import mirror_store, runtime
204
+ rt = runtime.get_runtime(tenant)
205
+ paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
206
+ installed = mirror_store.install_handoff(connector, tenant, rt, ready["manifest"],
207
+ ready.get("stateRows") or [], paths)
208
+ try:
209
+ import routes_odoo_tables
210
+ routes_odoo_tables.invalidate_sources()
211
+ except Exception as exc: # noqa: BLE001
212
+ print(f"[process-role] read registry refresh failed ({type(exc).__name__})", flush=True)
213
+ return installed
214
+
215
+
216
+ def _remove_dir(path):
217
+ shutil.rmtree(path, ignore_errors=True)
218
+
219
+
220
+ def install_ready(root=None, installer=None, *, now=None):
221
+ """Install the newest READY snapshot per tenant and connector, then delete older hand-offs."""
222
+ root = Path(root or handoff_dir())
223
+ installer = installer or _install_default
224
+ stamp = time.time() if now is None else now
225
+ installed = []
226
+ for key, rows in ready_handoffs(root).items():
227
+ completed_at, newest_dir, ready = rows[-1]
228
+ snapshot = ready.get("snapshotId") or newest_dir.name
229
+ if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS:
230
+ try:
231
+ installer(key[1], key[0], ready)
232
+ except Exception as exc: # noqa: BLE001
233
+ _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
234
+ LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
235
+ attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
236
+ print(f"[process-role] hand-off install {key[0]}/{key[1]} {snapshot} failed "
237
+ f"({type(exc).__name__}: {exc})", flush=True)
238
+ continue
239
+ _INSTALLED[key] = snapshot
240
+ LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
241
+ if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
242
+ LAST_INSTALL_ERROR.clear()
243
+ installed.append(key)
244
+ print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]}", flush=True)
245
+ for _at, older, _ready in rows[:-1]:
246
+ _remove_dir(older)
247
+ # A worker that died mid-copy leaves a dir without READY.json. Drop it after a grace period.
248
+ if root.is_dir():
249
+ for connector_dir in (p for p in root.iterdir() if p.is_dir()):
250
+ for snapshot_dir in (p for p in connector_dir.iterdir() if p.is_dir()):
251
+ if not (snapshot_dir / READY_FILE).exists():
252
+ try:
253
+ old = stamp - snapshot_dir.stat().st_mtime > INCOMPLETE_GRACE_SECONDS
254
+ except OSError:
255
+ old = False
256
+ if old:
257
+ _remove_dir(snapshot_dir)
258
+ return installed
259
+
260
+
261
+ # ----------------------------------------------------------------------------- cross-process desk
262
+ def _forward_request(receipt, connection_id):
263
+ """Web side: hand an accepted Sync now to the worker through the store desk."""
264
+ from core import connector_sync
265
+ try:
266
+ from harness import runtime
267
+ rt = runtime.get_runtime(receipt["tenant"])
268
+ connector_sync.queue_remote_request(rt, receipt, connection_id)
269
+ except Exception as exc: # noqa: BLE001
270
+ print(f"[process-role] sync request hand-off failed ({type(exc).__name__})", flush=True)
271
+ connector_sync.fail_remote_request(
272
+ receipt["provider"], receipt["tenant"],
273
+ "The sync request could not be passed to the sync worker. Try again.")
274
+
275
+
276
+ def _tenants():
277
+ import tenant_scan
278
+ from harness import runtime
279
+ return [(slug, rt) for slug, rt in tenant_scan.live_tenants(runtime) if rt is not None]
280
+
281
+
282
+ def poll_worker_desk():
283
+ """Worker side: pick up event receipts and Sync now hand-offs from the store desk."""
284
+ from core import connector_sync
285
+ for slug, rt in _tenants():
286
+ try:
287
+ connector_sync.restore_sync_requests(rt)
288
+ admitted = connector_sync.poll_remote_requests(rt)
289
+ except Exception as exc: # noqa: BLE001
290
+ print(f"[process-role] sync desk poll refused for {slug} ({type(exc).__name__})", flush=True)
291
+ continue
292
+ for entry in admitted:
293
+ print(f"[process-role] sync request {entry['provider']} for {slug} admitted from the web", flush=True)
294
+ import routes_connectors
295
+ key = (entry["provider"], slug)
296
+ with routes_connectors._SYNC_REQUEST_CONNECTION_LOCK:
297
+ # No connection id means the default connection, never the previous request's.
298
+ if entry.get("connectionId"):
299
+ routes_connectors._SYNC_REQUEST_CONNECTIONS[key] = entry["connectionId"]
300
+ else:
301
+ routes_connectors._SYNC_REQUEST_CONNECTIONS.pop(key, None)
302
+
303
+
304
+ def _worker_desk_loop():
305
+ while not _STOP.wait(POLL_SECONDS):
306
+ try:
307
+ poll_worker_desk()
308
+ except Exception as exc: # noqa: BLE001
309
+ print(f"[process-role] sync desk poll failed ({type(exc).__name__})", flush=True)
310
+
311
+
312
+ def _apply_worker_state():
313
+ """Web side: mirror the worker's request progress and catch-up state into this process."""
314
+ from core import connector_sync
315
+ from harness import mirror_store
316
+ beat = _read_json(handoff_dir() / HEARTBEAT_FILE) or {}
317
+ catch_up = beat.get("catchUp") if isinstance(beat.get("catchUp"), dict) else {}
318
+ resync = beat.get("resync")
319
+ app_main = _running_main()
320
+ if isinstance(resync, dict) and app_main is not None:
321
+ # The status routes read RESYNC_STATE here. Restore errors stay this process's own.
322
+ app_main.RESYNC_STATE.update({k: v for k, v in resync.items() if "estore" not in k})
323
+ for slug, rt in _tenants():
324
+ try:
325
+ connector_sync.apply_remote_progress(rt)
326
+ except Exception as exc: # noqa: BLE001
327
+ print(f"[process-role] sync progress read refused for {slug} ({type(exc).__name__})", flush=True)
328
+ if slug in catch_up:
329
+ mirror_store.apply_catch_up(rt, catch_up[slug])
330
+
331
+
332
+ def _web_watch_loop():
333
+ while not _STOP.wait(POLL_SECONDS):
334
+ try:
335
+ install_ready()
336
+ except Exception as exc: # noqa: BLE001
337
+ print(f"[process-role] hand-off scan failed ({type(exc).__name__}: {exc})", flush=True)
338
+ try:
339
+ _apply_worker_state()
340
+ except Exception as exc: # noqa: BLE001
341
+ print(f"[process-role] worker state read failed ({type(exc).__name__})", flush=True)
342
+
343
+
344
+ # ----------------------------------------------------------------------------- the worker process
345
+ def _running_main():
346
+ return sys.modules.get("api.main") or sys.modules.get("main")
347
+
348
+
349
+ def write_heartbeat(now=None):
350
+ try:
351
+ from harness import mirror_store
352
+ catch_up = mirror_store.catch_up_snapshot()
353
+ except Exception: # noqa: BLE001
354
+ catch_up = {}
355
+ app_main = _running_main()
356
+ resync = dict(getattr(app_main, "RESYNC_STATE", None) or {})
357
+ _write_json(handoff_dir() / HEARTBEAT_FILE,
358
+ {"pid": os.getpid(), "at": time.time() if now is None else now,
359
+ "catchUp": catch_up, "resync": resync})
360
+
361
+
362
+ def _heartbeat_loop(parent_pid):
363
+ while True:
364
+ try:
365
+ write_heartbeat()
366
+ except Exception as exc: # noqa: BLE001
367
+ print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True)
368
+ if parent_pid and os.getppid() != parent_pid:
369
+ # An orphaned worker must not keep publishing after its web process is gone.
370
+ print("[sync-worker] parent process gone; exiting", flush=True)
371
+ os._exit(0)
372
+ if _STOP.wait(HEARTBEAT_SECONDS):
373
+ return
374
+
375
+
376
+ def start_worker_services():
377
+ """Called by sync_worker.py: heartbeat, hand-off writer and the store desk poll."""
378
+ from harness import mirror_store
379
+ mirror_store.register_publish_listener(write_handoff)
380
+ parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
381
+ start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
382
+ start("sync-desk-poll", "heavy", _worker_desk_loop)
383
+
384
+
385
+ def _worker_env():
386
+ env = dict(os.environ)
387
+ env.update(AIOS_PROCESS_ROLE="worker", AIOS_PARENT_PID=str(os.getpid()),
388
+ AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
389
+ for name in ("AIOS_DUCKDB_MEMORY", "AIOS_DUCKDB_THREADS"):
390
+ env.pop(name, None) # the worker sets its own caps
391
+ return env
392
+
393
+
394
+ def _supervise():
395
+ backoff = 5.0
396
+ while not _STOP.is_set():
397
+ started = time.time()
398
+ try:
399
+ proc = subprocess.Popen([sys.executable, "sync_worker.py"], cwd=str(API_DIR), env=_worker_env())
400
+ except OSError as exc:
401
+ print(f"[process-role] sync worker could not start ({type(exc).__name__}: {exc})", flush=True)
402
+ proc = None
403
+ if proc is not None:
404
+ _WORKER.update(proc=proc, pid=proc.pid, startedAt=started)
405
+ print(f"[process-role] sync worker started pid={proc.pid}", flush=True)
406
+ code = proc.wait()
407
+ _WORKER["lastExit"] = code
408
+ else:
409
+ code = None
410
+ if _STOP.is_set():
411
+ return
412
+ if time.time() - started > 600:
413
+ backoff = 5.0
414
+ _WORKER["restarts"] += 1
415
+ print(f"[process-role] sync worker exited code={code}; restart {_WORKER['restarts']} "
416
+ f"in {int(backoff)}s", flush=True)
417
+ if _STOP.wait(backoff):
418
+ return
419
+ backoff = min(backoff * 2, 300.0)
420
+
421
+
422
+ def stop_worker():
423
+ """Stop the supervised worker by its own pid."""
424
+ _STOP.set()
425
+ proc = _WORKER.get("proc")
426
+ if proc is not None and proc.poll() is None:
427
+ proc.terminate()
428
+ try:
429
+ proc.wait(10)
430
+ except subprocess.TimeoutExpired:
431
+ proc.kill()
432
+
433
+
434
+ def start_web_services():
435
+ """Web role only: forward Sync now, supervise the worker, install its snapshots."""
436
+ if ROLE != "web" or _WORKER.get("supervisor") is not None:
437
+ return False
438
+ from core import connector_sync
439
+ connector_sync.register_request_listener(_forward_request)
440
+ _WORKER["supervisor"] = start("sync-worker-supervisor", "light", _supervise)
441
+ start("mirror-handoff-watcher", "light", _web_watch_loop)
442
+ atexit.register(stop_worker)
443
+ return True
444
+
445
+
446
+ def status():
447
+ """{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response."""
448
+ beat = _read_json(handoff_dir() / HEARTBEAT_FILE) if ROLE != "all" else None
449
+ if ROLE == "web":
450
+ proc = _WORKER.get("proc")
451
+ pid = _WORKER.get("pid")
452
+ alive = proc is not None and proc.poll() is None
453
+ elif ROLE == "worker":
454
+ pid, alive = os.getpid(), True
455
+ else:
456
+ pid, alive = None, False
457
+ age = None
458
+ if isinstance(beat, dict) and beat.get("pid") == pid and beat.get("at"):
459
+ age = round(max(0.0, time.time() - float(beat["at"])), 1)
460
+ return {"role": ROLE, "roleSource": ROLE_SOURCE,
461
+ "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
462
+ "restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
463
+ "lastInstallError": dict(LAST_INSTALL_ERROR) or None}}
api/routes_admin.py CHANGED
@@ -2804,7 +2804,16 @@ def settings(session: Session = Depends(require_session)):
2804
  """
2805
  modules = perms.allowed_modules(session.user)
2806
  from core import freeze_notice
 
 
 
 
 
 
2807
  return {
 
 
 
2808
  "freezeNotice": freeze_notice.notice(),
2809
  # The build this Space is running. Behind the session on purpose — see `main.py::health`,
2810
  # which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a
 
2804
  """
2805
  modules = perms.allowed_modules(session.user)
2806
  from core import freeze_notice
2807
+ import process_role
2808
+ roles = process_role.status()
2809
+ worker = roles["worker"]
2810
+ if not perms.is_admin(session.user):
2811
+ # A member sees liveness only, never a process id.
2812
+ worker = {key: worker[key] for key in ("alive", "heartbeatAgeS", "restarts")}
2813
  return {
2814
+ # Which process role answered, and whether its sync worker is alive and beating.
2815
+ "processRole": roles["role"],
2816
+ "worker": worker,
2817
  "freezeNotice": freeze_notice.notice(),
2818
  # The build this Space is running. Behind the session on purpose — see `main.py::health`,
2819
  # which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a
api/routes_connectors.py CHANGED
@@ -2676,9 +2676,12 @@ def _start_database_jobs():
2676
  _materialization_wake.set()
2677
  if _materialization_thread and _materialization_thread.is_alive():
2678
  return
2679
- _materialization_thread = threading.Thread(target=run, daemon=True,
2680
- name="connector-materialization")
2681
- _materialization_thread.start()
 
 
 
2682
 
2683
 
2684
  @router.on_event("shutdown")
@@ -2988,7 +2991,8 @@ def connector_sync_now(connector_key: str, connection_id: str = None,
2988
  # Bind under one API-side lock so a duplicate request cannot race ahead of the
2989
  # accepted request and attach its different connection id to the queued intent.
2990
  receipt = connector_sync.request_sync(connector_key, session.runtime.key,
2991
- requested_by=session.uname)
 
2992
  if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS:
2993
  _SYNC_REQUEST_CONNECTIONS[key] = requested_connection
2994
  receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key]
 
2676
  _materialization_wake.set()
2677
  if _materialization_thread and _materialization_thread.is_alive():
2678
  return
2679
+ import process_role
2680
+ job = process_role.thread(target=run, daemon=True, kind="heavy",
2681
+ name="connector-materialization")
2682
+ if process_role.admitted(job):
2683
+ _materialization_thread = job
2684
+ _materialization_thread.start()
2685
 
2686
 
2687
  @router.on_event("shutdown")
 
2991
  # Bind under one API-side lock so a duplicate request cannot race ahead of the
2992
  # accepted request and attach its different connection id to the queued intent.
2993
  receipt = connector_sync.request_sync(connector_key, session.runtime.key,
2994
+ requested_by=session.uname,
2995
+ connection_id=requested_connection)
2996
  if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS:
2997
  _SYNC_REQUEST_CONNECTIONS[key] = requested_connection
2998
  receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key]
api/routes_platform_admin.py CHANGED
@@ -51,8 +51,12 @@ from deps import Session, err, require_session, users
51
 
52
  @asynccontextmanager
53
  async def _neon_lifespan(app):
54
- _start_neon_sampler()
55
- _start_do_sampler()
 
 
 
 
56
  try:
57
  yield
58
  finally:
 
51
 
52
  @asynccontextmanager
53
  async def _neon_lifespan(app):
54
+ import process_role
55
+ # Light: each sampler makes one billing API read per interval for this process's card.
56
+ if process_role.admit("neon-usage-sampler", "light"):
57
+ _start_neon_sampler()
58
+ if process_role.admit("do-usage-sampler", "light"):
59
+ _start_do_sampler()
60
  try:
61
  yield
62
  finally:
api/routes_sheet.py CHANGED
@@ -1745,6 +1745,51 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str:
1745
  ONE function that turns those pieces into a key, so a future change to the wall's shape cannot
1746
  make the two drift silently apart.
1747
  """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1748
  from core import sheets
1749
  import core.perm_scope as perm_scope
1750
  _refuse_unapplied_audience(session, database)
@@ -1779,8 +1824,8 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str:
1779
  policy = {key: value for key, value in session.user.items()
1780
  if key not in ("username", "name", "email", "password", "password_hash", "epoch",
1781
  "created_at", "updated_at", "last_login")}
1782
- return fingerprint([sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {}),
1783
- binding, policy, memo[database], memo.get("_grants")])
1784
 
1785
 
1786
  def _has_ending(binding):
@@ -2078,13 +2123,21 @@ def _reresolved_for_reader(session: Session, book, *, endings_only=False, recalc
2078
  "cell": legacy.get("snapshot") or {"value": value, "state": "fresh",
2079
  "display": "" if value is None else str(value),
2080
  "refreshed_at": legacy.get("refreshedAt")}}
 
 
 
2081
  stored_answer = False
2082
  if field_refusal:
2083
  _snapshot_cell(cell, _binding_refusal(field_refusal["code"], field_refusal["message"]))
2084
  elif seen[db] and _binding_catching_up(session, binding):
2085
- _snapshot_cell(cell, {**(hit or {}).get("cell", {}),
2086
- **_binding_refusal("store_catching_up",
2087
- "This database is updating. Refresh when it finishes.")})
 
 
 
 
 
2088
  elif hit:
2089
  _snapshot_cell(cell, _public_refresh_snapshot(hit["cell"]))
2090
  stored_answer = True
@@ -2101,7 +2154,8 @@ def _reresolved_for_reader(session: Session, book, *, endings_only=False, recalc
2101
  # not what that protected: an admin has no row wall (`row_scope_applies`), and
2102
  # the stored figure is the owner's own document's last computed answer. A `stale`
2103
  # cell is excluded because it has no answer to keep. Non-owners are unchanged.
2104
- cell.update(_public_refresh_snapshot(cell))
 
2105
  stored_answer = True
2106
  else:
2107
  _snapshot_cell(cell, _binding_refusal("sheet_refresh_required",
@@ -3716,8 +3770,10 @@ def _refresh_scope_cells(book, scope):
3716
 
3717
 
3718
  def _public_refresh_snapshot(snapshot):
3719
- """A refusal can retain a private last-good value but cannot project it as an answer."""
3720
  if snapshot.get("state") == "refused":
 
 
3721
  refusal = snapshot.get("refusal") or {}
3722
  return {**snapshot, **_binding_refusal(refusal.get("code", "sheet_unresolved"),
3723
  refusal.get("message", "Refresh was held."))}
@@ -3832,6 +3888,7 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
3832
  started = perf_counter()
3833
  resolved_count = 0
3834
  held_refusal = None
 
3835
  denied_aggregates = set()
3836
  held_sources, entry_sources = set(), {}
3837
  scope_names = {id(tab): name for tab, name in
@@ -3901,17 +3958,29 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
3901
  "refreshed_at": stamp}
3902
  if limits:
3903
  snapshot["limits"] = limits
3904
- except (HTTPException, sheet_mod.SheetRefusal) as exc:
3905
- info = ({"code": exc.code, "message": exc.message}
3906
- if isinstance(exc, sheet_mod.SheetRefusal)
3907
- else ((exc.detail or {}).get("error") or {}))
3908
- snapshot = _binding_refusal(info.get("code", "sheet_unresolved"),
3909
- info.get("message", "This cell could not be refreshed."))
3910
- # Authored binding refusals are answers; only a failed source holds the batch.
3911
- if (isinstance(exc, HTTPException) and exc.status_code >= 500
3912
- or info.get("code") in ("store_catching_up", "source_unavailable",
3913
- "store_not_ready", "mirror_not_ready", "mirror_mismatch",
3914
- "amazon_mirror_not_loaded")):
 
 
 
 
 
 
 
 
 
 
 
 
3915
  held_refusal = held_refusal or snapshot["refusal"]
3916
  held_sources.update({reading_source} if reading_source else
3917
  {key for key in entry_sources[cell_key] if _database_connected(session, key)} or {db})
@@ -3919,7 +3988,8 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
3919
  snapshot["stamp"] = _binding_answer_stamp(session, book, binding, tab.get("id"), fr)
3920
  _snapshot_cell(cell, snapshot)
3921
  entries.append((cell_key, scope_key, {"tokens": source_tokens,
3922
- "context": context, "cell": deepcopy(snapshot)}))
 
3923
 
3924
  capture.confirm([entry for _, _, entry in entries])
3925
  for cell_key, _, entry in entries:
@@ -3934,8 +4004,8 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
3934
  if held:
3935
  # Retain every old snapshot, but refuse only failed sources and their dependents.
3936
  entries.extend(skipped_entries)
3937
- held_addresses = {key for key, _, entry in entries
3938
- if entry["cell"].get("refusal") or entry_sources[key] & held_sources}
3939
  graph = _read_graph(session, original, fr, None) or _new_read_graph(original, fr)
3940
  # Discover dynamic dependents from accepted values without resolving any database.
3941
  engine = _read_engine(graph, original, fr, None)[0]
@@ -3950,16 +4020,26 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
3950
  {key for key, address in wire.items() if address in held_addresses},
3951
  dependent_keys=dependent_keys)
3952
  held_addresses.update(wire[key] for key in dependent_keys if key in wire)
 
3953
  retained_entries = []
3954
  for key, slot, entry in entries:
3955
  old = cache.get("refreshCells", {}).get(key, {}).get(slot, {})
 
 
 
 
3956
  if key not in held_addresses:
3957
  if old:
3958
  retained_entries.append((key, slot, deepcopy(old)))
3959
  continue
3960
- refusal = entry["cell"].get("refusal") or held_refusal
3961
- entry["cell"] = {"stamp": entry["cell"]["stamp"], **deepcopy(old.get("cell", {})),
3962
- "state": "refused", "refusal": deepcopy(refusal)}
 
 
 
 
 
3963
  entry["tokens"] = deepcopy(old.get("tokens", {}))
3964
  entry["context"] = old.get("context")
3965
  retained_entries.append((key, slot, entry))
@@ -4022,6 +4102,11 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
4022
  refusal = entry["cell"]["refusal"]
4023
  entry.update(tokens=deepcopy(old.get("tokens", {})), context=old.get("context"),
4024
  cell={**deepcopy(old["cell"]), "state": "refused", "refusal": refusal})
 
 
 
 
 
4025
  recalc_ms = round((perf_counter() - started) * 1000.0, 1)
4026
  stats = _read_stats(session) or {}
4027
  print(f"[sheet] refresh wb={workbook_id} cells={'all' if only is None else len(only)} "
@@ -4040,12 +4125,14 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
4040
  if held:
4041
  result["reason"] = held
4042
  result["message"] = held_refusal["message"]
4043
- result["refusals"] = {key: entry["cell"]["refusal"] for key, _, entry in entries
4044
- if key in held_addresses}
 
 
4045
  result["stamps"] = {key: entry["cell"]["stamp"] for key, _, entry in entries
4046
- if key in held_addresses}
4047
  saved = sheets.finish_refresh(store, workbook_id, result, session, held=True,
4048
- addresses=held_addresses.intersection(addresses) if editable else (),
4049
  expect_rev=original.get("rev"))
4050
  elif editable:
4051
  try:
@@ -4906,9 +4993,63 @@ _BINDING_REFUSAL_DISPLAY = {
4906
  "field_unknown": "#FIELD?",
4907
  # HOTFIX 2026-09-24: "needs a Refresh" is not a calculation error and must not read as one.
4908
  "sheet_refresh_required": "#REFRESH",
 
 
 
 
 
4909
  }
4910
 
4911
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4912
  def _binding_refusal(code, message):
4913
  """One refusal shape for compute, refresh and caller-scoped reads of metric cells."""
4914
  return {
@@ -5658,10 +5799,25 @@ def _compute_cell_scoped(workbook_id: str, addr: str, body, session: Session):
5658
  #: a cell left `stale` would shimmer forever, and the reason belongs on the cell.
5659
  detail = getattr(exc, "detail", None) or {}
5660
  info = detail.get("error") if isinstance(detail, dict) else None
5661
- saved.update(_binding_refusal(
5662
- (info or {}).get("code", "sheet_unresolved"),
5663
- (info or {}).get("message", "This cell could not be refreshed."),
5664
- ))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5665
  else:
5666
  limits = resolved["limits"]
5667
  saved["state"] = "fresh"
 
1745
  ONE function that turns those pieces into a key, so a future change to the wall's shape cannot
1746
  make the two drift silently apart.
1747
  """
1748
+ from core.sheet_binding import fingerprint
1749
+ scope, policy, access, grants = _binding_scope_parts(session, database, binding)
1750
+ return fingerprint([scope, binding, policy, access, grants])
1751
+
1752
+
1753
+ #: User keys that name or decorate a person and never change what they may read.
1754
+ _WALL_IDENTITY_KEYS = frozenset(("avatar", "whatsapp"))
1755
+
1756
+
1757
+ def _binding_wall_key(session: Session, database: str, binding: dict):
1758
+ """Fingerprint of the reader's permission wall alone, or None when no other reader may share it.
1759
+
1760
+ Two readers with equal wall keys read the same rows and fields, so one's stored answer is the other's.
1761
+ """
1762
+ from core.sheet_binding import fingerprint
1763
+ try:
1764
+ # Cell inputs and related tables carry walls this key does not see.
1765
+ if _has_cell_inputs(binding):
1766
+ return None
1767
+ scope, policy, access, grants = _binding_scope_parts(session, database, binding)
1768
+ except Exception: # noqa: BLE001
1769
+ return None
1770
+ # An unprovable wall is a random string; a personal workspace makes the value caller specific.
1771
+ if not isinstance(access, list) or access[1] is not None:
1772
+ return None
1773
+ return fingerprint(["wall", scope, binding, access[0], grants,
1774
+ {key: value for key, value in policy.items() if key not in _WALL_IDENTITY_KEYS}])
1775
+
1776
+
1777
+ def _wall_hit(session, cache, cell_key, database, binding):
1778
+ """A stored answer written for another reader whose permission wall equals this reader's."""
1779
+ slots = (cache.get("refreshCells") or {}).get(cell_key)
1780
+ if not isinstance(slots, dict) or not slots:
1781
+ return None
1782
+ wall = _binding_wall_key(session, database, binding)
1783
+ if wall is None:
1784
+ return None
1785
+ for entry in slots.values():
1786
+ if isinstance(entry, dict) and entry.get("wall") == wall and isinstance(entry.get("cell"), dict):
1787
+ return entry
1788
+ return None
1789
+
1790
+
1791
+ def _binding_scope_parts(session: Session, database: str, binding: dict):
1792
+ """`(scope hash, policy, field access, grants)`: the pieces both cache keys fingerprint."""
1793
  from core import sheets
1794
  import core.perm_scope as perm_scope
1795
  _refuse_unapplied_audience(session, database)
 
1824
  policy = {key: value for key, value in session.user.items()
1825
  if key not in ("username", "name", "email", "password", "password_hash", "epoch",
1826
  "created_at", "updated_at", "last_login")}
1827
+ return (sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {}),
1828
+ policy, memo[database], memo.get("_grants"))
1829
 
1830
 
1831
  def _has_ending(binding):
 
2123
  "cell": legacy.get("snapshot") or {"value": value, "state": "fresh",
2124
  "display": "" if value is None else str(value),
2125
  "refreshed_at": legacy.get("refreshedAt")}}
2126
+ if not hit and seen[db]:
2127
+ # A reader with no slot of their own reads the answer stored for an identical wall.
2128
+ hit = _wall_hit(session, cache, str(tab.get("id")) + "!" + addr, db, binding)
2129
  stored_answer = False
2130
  if field_refusal:
2131
  _snapshot_cell(cell, _binding_refusal(field_refusal["code"], field_refusal["message"]))
2132
  elif seen[db] and _binding_catching_up(session, binding):
2133
+ kept = _public_refresh_snapshot(hit["cell"]) if hit else None
2134
+ if kept is not None and kept.get("state") != "refused":
2135
+ # An update in progress keeps the stored value, marked stale.
2136
+ _snapshot_cell(cell, _held_stale(kept, "store_catching_up"))
2137
+ else:
2138
+ _snapshot_cell(cell, {**(hit or {}).get("cell", {}),
2139
+ **_binding_refusal("store_catching_up",
2140
+ "This database is updating. Refresh when it finishes.")})
2141
  elif hit:
2142
  _snapshot_cell(cell, _public_refresh_snapshot(hit["cell"]))
2143
  stored_answer = True
 
2154
  # not what that protected: an admin has no row wall (`row_scope_applies`), and
2155
  # the stored figure is the owner's own document's last computed answer. A `stale`
2156
  # cell is excluded because it has no answer to keep. Non-owners are unchanged.
2157
+ # Copy first: `_snapshot_cell` clears the keys it is about to rewrite on this same cell.
2158
+ _snapshot_cell(cell, deepcopy(_public_refresh_snapshot(cell)))
2159
  stored_answer = True
2160
  else:
2161
  _snapshot_cell(cell, _binding_refusal("sheet_refresh_required",
 
3770
 
3771
 
3772
  def _public_refresh_snapshot(snapshot):
3773
+ """An authored refusal hides a retained value. A transient one serves it as stale."""
3774
  if snapshot.get("state") == "refused":
3775
+ if _held_answer(snapshot):
3776
+ return _held_stale(snapshot)
3777
  refusal = snapshot.get("refusal") or {}
3778
  return {**snapshot, **_binding_refusal(refusal.get("code", "sheet_unresolved"),
3779
  refusal.get("message", "Refresh was held."))}
 
3888
  started = perf_counter()
3889
  resolved_count = 0
3890
  held_refusal = None
3891
+ authored_keys = set()
3892
  denied_aggregates = set()
3893
  held_sources, entry_sources = set(), {}
3894
  scope_names = {id(tab): name for tab, name in
 
3958
  "refreshed_at": stamp}
3959
  if limits:
3960
  snapshot["limits"] = limits
3961
+ except Exception as exc: # noqa: BLE001
3962
+ http_status = exc.status_code if isinstance(exc, HTTPException) else None
3963
+ if isinstance(exc, sheet_mod.SheetRefusal):
3964
+ info = {"code": exc.code, "message": exc.message}
3965
+ elif isinstance(exc, HTTPException):
3966
+ info = (exc.detail if isinstance(exc.detail, dict) else {}).get("error") or {}
3967
+ else:
3968
+ # A timeout, a dropped connection or any unexpected failure is the server's, not the cell's.
3969
+ print(f"[sheet] refresh cell={cell_key} held {type(exc).__name__}: "
3970
+ f"{(str(exc).splitlines() or [''])[0][:200]}", flush=True)
3971
+ info = {"code": "source_unavailable",
3972
+ "message": "The database was busy. This cell updates after the next sync."}
3973
+ code = info.get("code") or "sheet_unresolved"
3974
+ authored = _refusal_authored(code, http_status,
3975
+ named=isinstance(exc, sheet_mod.SheetRefusal))
3976
+ if not authored and code not in _TRANSIENT_REFUSAL_CODES:
3977
+ code, info = "source_unavailable", {
3978
+ "message": "The database was busy. This cell updates after the next sync."}
3979
+ snapshot = _binding_refusal(code, info.get("message", "This cell could not be refreshed."))
3980
+ # Only an authored refusal replaces a stored value. Every other failure holds it.
3981
+ if authored:
3982
+ authored_keys.add(cell_key)
3983
+ else:
3984
  held_refusal = held_refusal or snapshot["refusal"]
3985
  held_sources.update({reading_source} if reading_source else
3986
  {key for key in entry_sources[cell_key] if _database_connected(session, key)} or {db})
 
3988
  snapshot["stamp"] = _binding_answer_stamp(session, book, binding, tab.get("id"), fr)
3989
  _snapshot_cell(cell, snapshot)
3990
  entries.append((cell_key, scope_key, {"tokens": source_tokens,
3991
+ "context": context, "cell": deepcopy(snapshot),
3992
+ "wall": _binding_wall_key(session, db, binding)}))
3993
 
3994
  capture.confirm([entry for _, _, entry in entries])
3995
  for cell_key, _, entry in entries:
 
4004
  if held:
4005
  # Retain every old snapshot, but refuse only failed sources and their dependents.
4006
  entries.extend(skipped_entries)
4007
+ held_addresses = {key for key, _, entry in entries if key not in authored_keys
4008
+ and (entry["cell"].get("refusal") or entry_sources[key] & held_sources)}
4009
  graph = _read_graph(session, original, fr, None) or _new_read_graph(original, fr)
4010
  # Discover dynamic dependents from accepted values without resolving any database.
4011
  engine = _read_engine(graph, original, fr, None)[0]
 
4020
  {key for key, address in wire.items() if address in held_addresses},
4021
  dependent_keys=dependent_keys)
4022
  held_addresses.update(wire[key] for key in dependent_keys if key in wire)
4023
+ held_addresses -= authored_keys
4024
  retained_entries = []
4025
  for key, slot, entry in entries:
4026
  old = cache.get("refreshCells", {}).get(key, {}).get(slot, {})
4027
+ if key in authored_keys:
4028
+ # An authored refusal is the answer and replaces the stored value.
4029
+ retained_entries.append((key, slot, entry))
4030
+ continue
4031
  if key not in held_addresses:
4032
  if old:
4033
  retained_entries.append((key, slot, deepcopy(old)))
4034
  continue
4035
+ kept = deepcopy(old.get("cell", {}))
4036
+ if _held_answer(kept):
4037
+ # A transient failure keeps the stored value, served stale, never refused.
4038
+ entry["cell"] = {"stamp": entry["cell"]["stamp"], **_held_stale(kept)}
4039
+ else:
4040
+ refusal = entry["cell"].get("refusal") or held_refusal
4041
+ entry["cell"] = {"stamp": entry["cell"]["stamp"], **kept,
4042
+ "state": "refused", "refusal": deepcopy(refusal)}
4043
  entry["tokens"] = deepcopy(old.get("tokens", {}))
4044
  entry["context"] = old.get("context")
4045
  retained_entries.append((key, slot, entry))
 
4102
  refusal = entry["cell"]["refusal"]
4103
  entry.update(tokens=deepcopy(old.get("tokens", {})), context=old.get("context"),
4104
  cell={**deepcopy(old["cell"]), "state": "refused", "refusal": refusal})
4105
+ # The private snapshot keeps the wall it was computed under.
4106
+ if "wall" in old:
4107
+ entry["wall"] = old["wall"]
4108
+ else:
4109
+ entry.pop("wall", None)
4110
  recalc_ms = round((perf_counter() - started) * 1000.0, 1)
4111
  stats = _read_stats(session) or {}
4112
  print(f"[sheet] refresh wb={workbook_id} cells={'all' if only is None else len(only)} "
 
4125
  if held:
4126
  result["reason"] = held
4127
  result["message"] = held_refusal["message"]
4128
+ # The document marks held cells with the cause. A GET serves a kept value stale.
4129
+ marked = held_addresses | authored_keys
4130
+ result["refusals"] = {key: entry["cell"].get("refusal") or held_refusal
4131
+ for key, _, entry in entries if key in marked}
4132
  result["stamps"] = {key: entry["cell"]["stamp"] for key, _, entry in entries
4133
+ if key in marked}
4134
  saved = sheets.finish_refresh(store, workbook_id, result, session, held=True,
4135
+ addresses=marked.intersection(addresses) if editable else (),
4136
  expect_rev=original.get("rev"))
4137
  elif editable:
4138
  try:
 
4993
  "field_unknown": "#FIELD?",
4994
  # HOTFIX 2026-09-24: "needs a Refresh" is not a calculation error and must not read as one.
4995
  "sheet_refresh_required": "#REFRESH",
4996
+ # A busy source is not a calculation error either.
4997
+ "source_unavailable": "#UPDATING",
4998
+ "store_not_ready": "#UPDATING",
4999
+ "mirror_not_ready": "#UPDATING",
5000
+ "amazon_mirror_not_loaded": "#UPDATING",
5001
  }
5002
 
5003
 
5004
+ #: Failures of the source or the server, never of the binding. They keep a stored value.
5005
+ _TRANSIENT_REFUSAL_CODES = frozenset((
5006
+ "store_catching_up", "source_unavailable", "store_not_ready", "mirror_not_ready",
5007
+ "mirror_mismatch", "amazon_mirror_not_loaded", "sheet_unresolved", "change_token_unavailable",
5008
+ "sheet_refresh_required", "schema_not_loaded", "grants_changed", "relation_cursor_mismatch",
5009
+ ))
5010
+
5011
+ #: The binding itself is invalid for this reader: a field or database is gone, or access is denied.
5012
+ _AUTHORED_REFUSAL_CODES = frozenset((
5013
+ "field_unreadable", "database_unreadable", "field_unknown", "related_field_unknown",
5014
+ "aggregate_unknown", "unknown_relation", "binding_invalid", "related_invalid", "ending_invalid",
5015
+ "ending_not_supported", "period_invalid", "period_window_invalid", "metric_not_additive",
5016
+ "metric_period_required", "pivot_field_unsupported", "pivot_too_large", "view_missing", "scope_not_applicable", "not_read_through", "folder_missing",
5017
+ "sheet_database_required", "sheet_bindings_required",
5018
+ "database_forbidden", "field_forbidden", "no_access", "permission_denied",
5019
+ ))
5020
+
5021
+
5022
+ def _refusal_authored(code, status=None, named=False):
5023
+ """True only when the binding is invalid. Every other failure is transient and keeps a stored value.
5024
+
5025
+ `named` marks a binding module refusal, which is authored unless its code names a busy source.
5026
+ """
5027
+ if code in _TRANSIENT_REFUSAL_CODES:
5028
+ return False
5029
+ if code in _AUTHORED_REFUSAL_CODES or named:
5030
+ return True
5031
+ # Any other client error names the request; throttling and every server error are transient.
5032
+ return status is not None and 400 <= status < 500 and status not in (408, 423, 425, 429)
5033
+
5034
+
5035
+ def _held_answer(cell):
5036
+ """True when a stored snapshot carries a computed value that a transient failure must not replace."""
5037
+ if not isinstance(cell, dict) or "value" not in cell:
5038
+ return False
5039
+ state = cell.get("state")
5040
+ if state in ("fresh", "stale"):
5041
+ return cell.get("value") is not None or bool(cell.get("refreshed_at"))
5042
+ # A held snapshot written before transient failures were stored `stale` kept its value refused.
5043
+ return (state == "refused" and bool(cell.get("refreshed_at"))
5044
+ and (cell.get("refusal") or {}).get("code") in _TRANSIENT_REFUSAL_CODES)
5045
+
5046
+
5047
+ def _held_stale(cell, reason="refresh_held"):
5048
+ """The stored value served as a stale answer, never as a refusal."""
5049
+ kept = {key: value for key, value in cell.items() if key != "refusal"}
5050
+ return {**kept, "state": "stale", "staleReason": reason}
5051
+
5052
+
5053
  def _binding_refusal(code, message):
5054
  """One refusal shape for compute, refresh and caller-scoped reads of metric cells."""
5055
  return {
 
5799
  #: a cell left `stale` would shimmer forever, and the reason belongs on the cell.
5800
  detail = getattr(exc, "detail", None) or {}
5801
  info = detail.get("error") if isinstance(detail, dict) else None
5802
+ code = (info or {}).get("code", "sheet_unresolved")
5803
+ kept = None
5804
+ if not _refusal_authored(code, exc.status_code):
5805
+ # A transient failure keeps this caller's own stored value, served stale.
5806
+ try:
5807
+ from core.sheet_binding import read_refresh_cache
5808
+ own = ((read_refresh_cache(_store(session), workbook_id).get("refreshCells") or {})
5809
+ .get(tab_ref + "!" + key, {})
5810
+ .get(_binding_scope_key(session, str(binding.get("database") or ""), binding)))
5811
+ except Exception: # noqa: BLE001
5812
+ own = None
5813
+ if isinstance(own, dict) and _held_answer(own.get("cell")):
5814
+ kept = _held_stale(own["cell"])
5815
+ if kept is not None:
5816
+ saved.pop("refusal", None)
5817
+ saved.update(kept)
5818
+ else:
5819
+ saved.update(_binding_refusal(
5820
+ code, (info or {}).get("message", "This cell could not be refreshed.")))
5821
  else:
5822
  limits = resolved["limits"]
5823
  saved["state"] = "fresh"
api/sheet_warm.py CHANGED
@@ -104,15 +104,17 @@ def bound_cells(book) -> set:
104
 
105
 
106
  def cells_needing_warm(view) -> set:
107
- """Bound cells a reader's projection marks stale (any reason) or `sheet_refresh_required`."""
108
  out = set()
109
  for tab in (view or {}).get("sheets") or []:
110
  for addr, cell in (tab.get("cells") or {}).items():
111
  if not _is_bound(cell):
112
  continue
113
  state = cell.get("state")
114
- code = (cell.get("refusal") or {}).get("code")
115
- if state == "stale" or (state == "refused" and code == "sheet_refresh_required"):
 
 
116
  out.add(f"{tab.get('id')}!{addr}")
117
  return out
118
 
@@ -337,7 +339,10 @@ def start():
337
  thread = _STATE["thread"]
338
  if thread is not None and thread.is_alive():
339
  return thread
340
- thread = threading.Thread(target=_run, daemon=True, name="sheet-warm")
 
 
 
341
  _STATE["thread"] = thread
342
  atexit.register(_on_exit)
343
  thread.start()
 
104
 
105
 
106
  def cells_needing_warm(view) -> set:
107
+ """Bound cells a reader's projection marks stale (any reason) or refused for a transient cause."""
108
  out = set()
109
  for tab in (view or {}).get("sheets") or []:
110
  for addr, cell in (tab.get("cells") or {}).items():
111
  if not _is_bound(cell):
112
  continue
113
  state = cell.get("state")
114
+ refusal = cell.get("refusal") or {}
115
+ # A busy-server refusal stored before the held rule still needs a retry; an authored one does not.
116
+ if state == "stale" or (state == "refused" and not routes_sheet._refusal_authored(
117
+ refusal.get("code"), refusal.get("status"))):
118
  out.add(f"{tab.get('id')}!{addr}")
119
  return out
120
 
 
339
  thread = _STATE["thread"]
340
  if thread is not None and thread.is_alive():
341
  return thread
342
+ import process_role
343
+ thread = process_role.thread(target=_run, daemon=True, name="sheet-warm", kind="heavy")
344
+ if not process_role.admitted(thread):
345
+ return None
346
  _STATE["thread"] = thread
347
  atexit.register(_on_exit)
348
  thread.start()
api/sync_worker.py ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The sync worker process: runs every heavy background job and never serves HTTP.
2
+
3
+ Run as `python sync_worker.py` from aios-web/api. The web process spawns and supervises it.
4
+ It keeps its own DuckDB files, so it never opens the web process's royal.duckdb.
5
+ """
6
+ import os
7
+ import sys
8
+ import threading
9
+ from pathlib import Path
10
+
11
+ API_DIR = Path(__file__).resolve().parent
12
+ _RI = Path(os.environ.get("RI_DIR") or (API_DIR.parents[1] / "platform"))
13
+
14
+
15
+ def configure(env=None):
16
+ """Set the worker role, its own DuckDB files and its DuckDB caps. Returns the worker DuckDB path."""
17
+ env = os.environ if env is None else env
18
+ env["AIOS_PROCESS_ROLE"] = "worker"
19
+ web_store = Path(env.get("AIOS_DUCKDB_STORE_DIR") or (_RI / "data" / "store"))
20
+ web_db = Path(env.get("AIOS_DUCKDB_PATH") or (web_store / "royal.duckdb"))
21
+ worker_dir = Path(env.get("AIOS_WORKER_STORE_DIR") or (web_db.parent / "worker"))
22
+ worker_db = worker_dir / web_db.name
23
+ env["AIOS_DUCKDB_STORE_DIR"] = str(worker_dir)
24
+ env["AIOS_DUCKDB_PATH"] = str(worker_db)
25
+ env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "4GB"
26
+ env["AIOS_DUCKDB_THREADS"] = env.get("AIOS_WORKER_DUCKDB_THREADS") or "1"
27
+ return worker_db
28
+
29
+
30
+ def lower_priority(env=None):
31
+ """Drop this process to a low CPU priority, so the web process wins every contested core."""
32
+ env = os.environ if env is None else env
33
+ step = int(env.get("AIOS_WORKER_NICE") or 15)
34
+ if step and hasattr(os, "nice"):
35
+ try:
36
+ return os.nice(step)
37
+ except OSError:
38
+ return None
39
+ return None
40
+
41
+
42
+ def main():
43
+ worker_db = configure()
44
+ worker_db.parent.mkdir(parents=True, exist_ok=True)
45
+ niceness = lower_priority()
46
+ for path in (str(_RI), str(API_DIR)):
47
+ if path not in sys.path:
48
+ sys.path.insert(0, path)
49
+ print(f"[sync-worker] pid={os.getpid()} duckdb={worker_db} "
50
+ f"memory_limit={os.environ['AIOS_DUCKDB_MEMORY']} threads={os.environ['AIOS_DUCKDB_THREADS']} "
51
+ f"nice={niceness}",
52
+ flush=True)
53
+ print("[sync-worker] amazon_store.sync excludes brand analytics reports (memory); "
54
+ "they stay out of this worker too", flush=True)
55
+ from dotenv import load_dotenv
56
+ load_dotenv(_RI / ".env")
57
+ import process_role
58
+ process_role.start_worker_services()
59
+ # Importing the app starts every role-allowed thread at module level. Startup hooks need a call.
60
+ import main as app_main # noqa: F401
61
+ import routes_connectors
62
+ routes_connectors._start_database_jobs()
63
+ print(f"[sync-worker] running: {', '.join(sorted(process_role.started_jobs()))}", flush=True)
64
+ stop = threading.Event()
65
+ while not stop.wait(3600):
66
+ pass
67
+
68
+
69
+ if __name__ == "__main__":
70
+ main()
platform/core/connector_sync.py CHANGED
@@ -32,6 +32,11 @@ _SYNC_WAKE = threading.Event()
32
  _SYNC_DUE_TIMER = None
33
  SYNC_REQUESTS_KEY = "connector_sync_requests_v1"
34
  NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1"
 
 
 
 
 
35
  _EVENT_DEDUPE_SECONDS = 600
36
 
37
 
@@ -52,7 +57,7 @@ def _sync_request_document(current):
52
  document = deepcopy(current) if current is not None else {}
53
  if not isinstance(document, dict):
54
  raise ValueError("The stored sync request desk is unavailable.")
55
- for member in ("requests", "events"):
56
  document.setdefault(member, {})
57
  if not isinstance(document[member], dict):
58
  raise ValueError("The stored sync request desk is unavailable.")
@@ -81,13 +86,16 @@ def _wake_sync_requests(stamp):
81
  _SYNC_WAKE.set()
82
 
83
 
84
- def restore_sync_requests(rt, *, now=None):
85
- """Restore durable events and requeue claims abandoned by a previous process."""
 
 
 
86
  stamp = time.time() if now is None else now
87
  with _SYNC_REQUEST_LOCK:
88
  document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
89
  abandoned = {identity for identity, row in document["requests"].items()
90
- if row.get("state") == "running" and
91
  _SYNC_REQUESTS.get((row.get("provider"), rt.key, identity), {}).get("state") != "running"}
92
  if abandoned:
93
  def restore(current):
@@ -141,9 +149,25 @@ def defer_sync(provider, tenant, not_before):
141
  _SYNC_WAKE.set()
142
 
143
 
 
 
 
 
 
 
144
  def request_sync(provider, tenant, *, requested_by="", now=None, not_before=None,
145
- rt=None, tables=None, event_id=""):
146
  """Queue ordinary work or durably acknowledge one tenant-bound Amazon event."""
 
 
 
 
 
 
 
 
 
 
147
  if provider not in _PROVIDERS:
148
  raise ValueError("This connector is not registered for sync.")
149
  tenant = str(tenant or "").strip()
@@ -306,6 +330,124 @@ def sync_wake_event():
306
  """The event the resident thread waits on between passes."""
307
  return _SYNC_WAKE
308
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
309
  # One receipt per immutable database identity. No credentials, session, or source values.
310
  JOBS_KEY = "connector_materialization_jobs"
311
  MAX_ATTEMPTS = 3
 
32
  _SYNC_DUE_TIMER = None
33
  SYNC_REQUESTS_KEY = "connector_sync_requests_v1"
34
  NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1"
35
+ # Ordinary requests the web process hands to the sync worker live in this member of the desk.
36
+ REMOTE_MEMBER = "ordinary"
37
+ REMOTE_KEEP_SECONDS = 3600
38
+ _REQUEST_LISTENERS = []
39
+ _REMOTE_IDS = {}
40
  _EVENT_DEDUPE_SECONDS = 600
41
 
42
 
 
57
  document = deepcopy(current) if current is not None else {}
58
  if not isinstance(document, dict):
59
  raise ValueError("The stored sync request desk is unavailable.")
60
+ for member in ("requests", "events", REMOTE_MEMBER):
61
  document.setdefault(member, {})
62
  if not isinstance(document[member], dict):
63
  raise ValueError("The stored sync request desk is unavailable.")
 
86
  _SYNC_WAKE.set()
87
 
88
 
89
+ def restore_sync_requests(rt, *, now=None, requeue=True):
90
+ """Restore durable events and requeue claims abandoned by a previous process.
91
+
92
+ requeue=False only reads: a process that never claims must not requeue another one's claim.
93
+ """
94
  stamp = time.time() if now is None else now
95
  with _SYNC_REQUEST_LOCK:
96
  document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
97
  abandoned = {identity for identity, row in document["requests"].items()
98
+ if requeue and row.get("state") == "running" and
99
  _SYNC_REQUESTS.get((row.get("provider"), rt.key, identity), {}).get("state") != "running"}
100
  if abandoned:
101
  def restore(current):
 
149
  _SYNC_WAKE.set()
150
 
151
 
152
+ def register_request_listener(fn):
153
+ """Call fn(receipt, connection_id) after each accepted ordinary request, outside the desk lock."""
154
+ if fn not in _REQUEST_LISTENERS:
155
+ _REQUEST_LISTENERS.append(fn)
156
+
157
+
158
  def request_sync(provider, tenant, *, requested_by="", now=None, not_before=None,
159
+ rt=None, tables=None, event_id="", connection_id=None):
160
  """Queue ordinary work or durably acknowledge one tenant-bound Amazon event."""
161
+ receipt = _request_sync(provider, tenant, requested_by=requested_by, now=now,
162
+ not_before=not_before, rt=rt, tables=tables, event_id=event_id)
163
+ if tables is None and receipt.get("accepted"):
164
+ for listener in list(_REQUEST_LISTENERS):
165
+ listener(dict(receipt), connection_id)
166
+ return receipt
167
+
168
+
169
+ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=None,
170
+ rt=None, tables=None, event_id=""):
171
  if provider not in _PROVIDERS:
172
  raise ValueError("This connector is not registered for sync.")
173
  tenant = str(tenant or "").strip()
 
330
  """The event the resident thread waits on between passes."""
331
  return _SYNC_WAKE
332
 
333
+
334
+ def queue_remote_request(rt, receipt, connection_id=None, *, now=None):
335
+ """Web side: durably hand one accepted ordinary request to the sync worker process."""
336
+ stamp = time.time() if now is None else now
337
+ identity = uuid.uuid4().hex
338
+ entry = {"id": identity, "provider": receipt["provider"], "tenant": rt.key,
339
+ "requestedBy": str(receipt.get("requestedBy") or ""),
340
+ "requestedAt": float(receipt.get("requestedAt") or stamp),
341
+ "notBefore": receipt.get("notBefore"),
342
+ "connectionId": str(connection_id) if connection_id else None,
343
+ "state": "queued", "startedAt": None, "finishedAt": None, "detail": ""}
344
+
345
+ def add(current):
346
+ document = _sync_request_document(current)
347
+ document[REMOTE_MEMBER][identity] = entry
348
+ return document
349
+
350
+ rt.update(SYNC_REQUESTS_KEY, add, flush="sync")
351
+ with _SYNC_REQUEST_LOCK:
352
+ _REMOTE_IDS[(receipt["provider"], rt.key)] = identity
353
+ return identity
354
+
355
+
356
+ def fail_remote_request(provider, tenant, detail, *, now=None):
357
+ """Web side: a hand-off that could not be saved finishes its receipt instead of hanging."""
358
+ with _SYNC_REQUEST_LOCK:
359
+ receipt = _SYNC_REQUESTS.get((provider, tenant))
360
+ if receipt and receipt["state"] == "queued":
361
+ receipt.update(state="failed", detail=str(detail)[:300],
362
+ finishedAt=time.time() if now is None else now)
363
+
364
+
365
+ def _remote_progress(entry, receipt):
366
+ """The worker receipt's progress for one entry, or None when that run does not cover it."""
367
+ started = receipt.get("startedAt") if receipt else None
368
+ if not started or entry["requestedAt"] > started or receipt["state"] not in ("running", "done", "failed"):
369
+ return None
370
+ progress = {"state": receipt["state"], "startedAt": started,
371
+ "finishedAt": receipt.get("finishedAt"), "detail": receipt.get("detail") or ""}
372
+ if receipt.get("firstLoad"):
373
+ progress["firstLoad"] = dict(receipt["firstLoad"])
374
+ return progress
375
+
376
+
377
+ def poll_remote_requests(rt, *, now=None):
378
+ """Worker side: admit queued hand-offs, publish their progress, prune old ones.
379
+
380
+ An entry leaves the queue only when a run that started after it was requested finishes.
381
+ Returns the entries admitted as new local receipts.
382
+ """
383
+ stamp = time.time() if now is None else now
384
+ admitted, changes, pruned = [], {}, []
385
+ with _SYNC_REQUEST_LOCK:
386
+ document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
387
+ for identity, entry in sorted(document[REMOTE_MEMBER].items(),
388
+ key=lambda item: item[1].get("requestedAt") or 0):
389
+ key = (entry.get("provider"), rt.key)
390
+ if entry.get("tenant") != rt.key or key[0] not in _PROVIDERS:
391
+ continue
392
+ if entry["state"] in ("done", "failed"):
393
+ if stamp - float(entry.get("finishedAt") or 0) > REMOTE_KEEP_SECONDS:
394
+ pruned.append(identity)
395
+ continue
396
+ receipt = _SYNC_REQUESTS.get(key)
397
+ progress = _remote_progress(entry, receipt)
398
+ if progress is not None:
399
+ if any(entry.get(name) != value for name, value in progress.items()):
400
+ changes[identity] = progress
401
+ continue
402
+ # A "running" entry with no local run was abandoned by a worker restart: admit it again.
403
+ if (entry["state"] in ("queued", "running")
404
+ and not (receipt and receipt["state"] in ("queued", "running"))):
405
+ local = {"provider": key[0], "tenant": rt.key, "state": "queued",
406
+ "requestedAt": entry["requestedAt"],
407
+ "notBefore": max(float(entry.get("notBefore") or 0),
408
+ _SYNC_NOT_BEFORE.get(key, 0)) or None,
409
+ "startedAt": None, "finishedAt": None, "detail": "",
410
+ "requestedBy": entry.get("requestedBy") or "", "accepted": True}
411
+ if receipt and receipt.get("firstLoad") and local["requestedBy"] == "first load":
412
+ local.update(detail=receipt["detail"], firstLoad=dict(receipt["firstLoad"]))
413
+ _SYNC_REQUESTS[key] = local
414
+ _SYNC_REQUEST_RUNTIMES.setdefault(rt.key, rt)
415
+ admitted.append(deepcopy(entry))
416
+ if changes or pruned:
417
+ def publish(current):
418
+ doc = _sync_request_document(current)
419
+ for identity, progress in changes.items():
420
+ if identity in doc[REMOTE_MEMBER]:
421
+ doc[REMOTE_MEMBER][identity].update(progress)
422
+ for identity in pruned:
423
+ doc[REMOTE_MEMBER].pop(identity, None)
424
+ return doc
425
+ rt.update(SYNC_REQUESTS_KEY, publish, flush="sync")
426
+ if admitted:
427
+ _wake_sync_requests(stamp)
428
+ return admitted
429
+
430
+
431
+ def apply_remote_progress(rt):
432
+ """Web side: copy the worker's published progress onto this process's receipts."""
433
+ document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
434
+ applied = 0
435
+ with _SYNC_REQUEST_LOCK:
436
+ for identity, entry in document[REMOTE_MEMBER].items():
437
+ key = (entry.get("provider"), rt.key)
438
+ receipt = _SYNC_REQUESTS.get(key)
439
+ if _REMOTE_IDS.get(key) != identity or receipt is None:
440
+ continue
441
+ fields = {name: entry.get(name) for name in ("state", "startedAt", "finishedAt", "detail")}
442
+ if entry.get("firstLoad"):
443
+ fields["firstLoad"] = dict(entry["firstLoad"])
444
+ if entry["state"] != "queued" and any(receipt.get(k) != v for k, v in fields.items()):
445
+ receipt.update(fields)
446
+ applied += 1
447
+ # Event receipts the worker claimed or finished, refreshed without requeueing anything.
448
+ _cache_sync_requests(rt, document)
449
+ return applied
450
+
451
  # One receipt per immutable database identity. No credentials, session, or source values.
452
  JOBS_KEY = "connector_materialization_jobs"
453
  MAX_ATTEMPTS = 3
platform/harness/datastore.py CHANGED
@@ -126,11 +126,24 @@ def path_for(tenant_key):
126
  boundary that holds is the operating system's — one file, one tenant.
127
  """
128
  key = str(tenant_key or "").strip().lower()
 
 
129
  if key in ("", "royal-imports"):
130
  return DB_PATH if key == "" else Path(
131
- os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb"))
132
  safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60]
133
- return _STORE_DIR / f"{safe}.duckdb"
 
 
 
 
 
 
 
 
 
 
 
134
 
135
 
136
  def use_path(path):
@@ -444,7 +457,7 @@ def _mirror_connection(tenant_key, rt):
444
  rt.assert_datastore_matches()
445
  return connect(), path
446
  path.parent.mkdir(parents=True, exist_ok=True)
447
- con = duckdb.connect(str(path))
448
  _ensure_sync_state(con)
449
  return con, path
450
 
@@ -851,7 +864,7 @@ def _instance():
851
  with _INSTANCE_LOCK:
852
  if _INSTANCE["con"] is None:
853
  DB_PATH.parent.mkdir(parents=True, exist_ok=True)
854
- _INSTANCE["con"] = duckdb.connect(str(DB_PATH))
855
  return _INSTANCE["con"]
856
 
857
 
 
126
  boundary that holds is the operating system's — one file, one tenant.
127
  """
128
  key = str(tenant_key or "").strip().lower()
129
+ # AIOS_DUCKDB_STORE_DIR moves every tenant's file, so the sync worker never opens the web's.
130
+ store_dir = Path(os.environ.get("AIOS_DUCKDB_STORE_DIR") or _STORE_DIR)
131
  if key in ("", "royal-imports"):
132
  return DB_PATH if key == "" else Path(
133
+ os.environ.get("AIOS_DUCKDB_PATH") or (store_dir / "royal.duckdb"))
134
  safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60]
135
+ return store_dir / f"{safe}.duckdb"
136
+
137
+
138
+ def apply_limits(con):
139
+ """Cap one DuckDB instance from AIOS_DUCKDB_MEMORY and AIOS_DUCKDB_THREADS. Unset keeps the default."""
140
+ memory = str(os.environ.get("AIOS_DUCKDB_MEMORY") or "").strip()
141
+ threads = str(os.environ.get("AIOS_DUCKDB_THREADS") or "").strip()
142
+ if memory and re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", memory, re.IGNORECASE):
143
+ con.execute(f"SET memory_limit='{memory}'")
144
+ if threads.isdigit() and int(threads) > 0:
145
+ con.execute(f"SET threads={int(threads)}")
146
+ return con
147
 
148
 
149
  def use_path(path):
 
457
  rt.assert_datastore_matches()
458
  return connect(), path
459
  path.parent.mkdir(parents=True, exist_ok=True)
460
+ con = apply_limits(duckdb.connect(str(path)))
461
  _ensure_sync_state(con)
462
  return con, path
463
 
 
864
  with _INSTANCE_LOCK:
865
  if _INSTANCE["con"] is None:
866
  DB_PATH.parent.mkdir(parents=True, exist_ok=True)
867
+ _INSTANCE["con"] = apply_limits(duckdb.connect(str(DB_PATH)))
868
  return _INSTANCE["con"]
869
 
870
 
platform/harness/mirror_store.py CHANGED
@@ -44,6 +44,88 @@ _CATCH_UP_LOCK = threading.RLock()
44
  _RESTORE_PROGRESS = threading.local()
45
  _NEWEST_READERS = {}
46
  _PUBLICATION_CHECKS = {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
 
48
 
49
  def register_publication_check(connector, check):
@@ -603,6 +685,15 @@ class MirrorStore:
603
  result = {"state": "local", "problems": [], "manifest": {}, "dirty": set(), "runtime": rt,
604
  "refused_tables": []}
605
  con, _path = self._connect(tenant_key, rt)
 
 
 
 
 
 
 
 
 
606
  _RESTORE_PROGRESS.store = self
607
  try:
608
  if self._has_local(con):
@@ -619,6 +710,7 @@ class MirrorStore:
619
  local_id = datastore._publication(con, self.connector).get("snapshot_id")
620
  if saved.get("snapshot_id") and saved["snapshot_id"] != local_id:
621
  self._install_snapshot(con, saved, rows, paths)
 
622
  result["manifest"] = saved
623
  if not result["manifest"].get("snapshot_id"):
624
  self._after_restore(con, result, log)
@@ -645,6 +737,7 @@ class MirrorStore:
645
  manifest, rows, paths, total, refused = self._validated_snapshot(
646
  tenant_key, rt, con, raw, tmp, allow_partial=True)
647
  self._install_snapshot(con, manifest, rows, paths)
 
648
  result.update(state="partial" if refused else "restored", manifest=manifest, refused_tables=refused)
649
  if refused:
650
  result["problems"].append(f"{self.label} mirror tables refused: " + ", ".join(refused)
@@ -1136,7 +1229,7 @@ def _export_tables(store, con, directory):
1136
  def _open_candidate(store, tenant_key, rt, published):
1137
  """Resume private rows only while their base publication still matches the serving copy."""
1138
  path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
1139
- candidate = datastore.duckdb.connect(str(path))
1140
  datastore._ensure_sync_state(candidate)
1141
  candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_base (identity VARCHAR, started_at VARCHAR)")
1142
  candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_pending (name VARCHAR PRIMARY KEY)")
@@ -1348,6 +1441,12 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1348
  raise
1349
  if odoo:
1350
  note("install", installing)
 
 
 
 
 
 
1351
  if holds is not None and holds["tables"]:
1352
  candidate.execute("ROLLBACK")
1353
  candidate.execute("DELETE FROM _candidate_base")
 
44
  _RESTORE_PROGRESS = threading.local()
45
  _NEWEST_READERS = {}
46
  _PUBLICATION_CHECKS = {}
47
+ _PUBLISH_LISTENERS = []
48
+
49
+
50
+ def register_publish_listener(fn):
51
+ """Call fn(connector, tenant_key, manifest, state_rows, paths) after each local install.
52
+
53
+ The paths are deleted when fn returns, so a listener that keeps them copies them first.
54
+ """
55
+ if fn not in _PUBLISH_LISTENERS:
56
+ _PUBLISH_LISTENERS.append(fn)
57
+
58
+
59
+ def _web_waits_for_worker(connector):
60
+ """True in the web process for every mirror but Odoo: those arrive only from the sync worker."""
61
+ return os.environ.get("AIOS_PROCESS_ROLE") == "web" and connector != "odoo"
62
+
63
+
64
+ def _notify_publish(connector, tenant_key, manifest, rows, paths, log=print):
65
+ """Hand a restored snapshot to the listeners too, so the web process never restores it itself."""
66
+ if not paths or not manifest.get("snapshot_id"):
67
+ return
68
+ for listener in list(_PUBLISH_LISTENERS):
69
+ try:
70
+ listener(connector, tenant_key, manifest, [tuple(row) for row in rows], paths)
71
+ except Exception as exc: # noqa: BLE001
72
+ log(f"{connector} mirror restore hand-off failed ({type(exc).__name__}: {exc})")
73
+
74
+
75
+ def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
76
+ """Install a snapshot another process already published. Returns False if it is already resident."""
77
+ store = for_connector(connector)
78
+ if manifest.get("tenant") not in (None, _tenant_key(tenant_key)):
79
+ raise ValueError("Hand-off manifest belongs to another tenant")
80
+ with store.lock:
81
+ con, _path = store._connect(tenant_key, rt)
82
+ try:
83
+ if (manifest.get("snapshot_id") and
84
+ datastore._publication(con, store.connector).get("snapshot_id") == manifest["snapshot_id"]):
85
+ return False
86
+ store._install_snapshot(con, manifest, [tuple(row) for row in rows], paths)
87
+ newest = _record_newest(con, store.connector)
88
+ finally:
89
+ con.close()
90
+ cache_key = store._mirror_cache_key(tenant_key, rt)
91
+ if _web_waits_for_worker(store.connector):
92
+ # The web process never persists, so the installed hand-off is its restore answer.
93
+ store.cache[cache_key] = {"state": "restored", "problems": [], "manifest": manifest, "dirty": set(),
94
+ "runtime": rt, "refused_tables": []}
95
+ else:
96
+ # The next restore re-reads the stored manifest instead of trusting this process's copy.
97
+ store.cache.pop(cache_key, None)
98
+ with _CATCH_UP_LOCK:
99
+ state = _catch_up_source(rt, store.connector)
100
+ if not state["refused"]:
101
+ state.update(current=True, pass_pending=False, newest=newest, misses=0, behind="")
102
+ return True
103
+
104
+
105
+ _CATCH_UP_SHARED = ("current", "part_total", "passes", "pass_pending", "newest", "refused",
106
+ "misses", "behind", "attempt_at", "missed_at", "miss_reason", "since")
107
+
108
+
109
+ def catch_up_snapshot():
110
+ """This process's catch-up progress keyed by tenant, as JSON for another process."""
111
+ with _CATCH_UP_LOCK:
112
+ return {tenant: {connector: {**{k: state[k] for k in _CATCH_UP_SHARED}, "parts": len(state["parts"])}
113
+ for connector, state in sources.items()}
114
+ for (tenant, _path), sources in _CATCH_UP.items()}
115
+
116
+
117
+ def apply_catch_up(rt, sources):
118
+ """Adopt the sync process's catch-up progress for one tenant's resident mirror."""
119
+ with _CATCH_UP_LOCK:
120
+ for connector, shared in (sources or {}).items():
121
+ state = _catch_up_source(rt, connector)
122
+ refused = state["refused"]
123
+ state.update({k: shared[k] for k in _CATCH_UP_SHARED if k in shared})
124
+ # This process's own restore refusal still stands whatever the sync process reports.
125
+ state["refused"] = bool(refused or state["refused"])
126
+ parts = int(shared.get("parts") or 0)
127
+ if len(state["parts"]) < parts:
128
+ state["parts"] = set(range(parts))
129
 
130
 
131
  def register_publication_check(connector, check):
 
685
  result = {"state": "local", "problems": [], "manifest": {}, "dirty": set(), "runtime": rt,
686
  "refused_tables": []}
687
  con, _path = self._connect(tenant_key, rt)
688
+ if _web_waits_for_worker(self.connector):
689
+ # The sync worker restores this mirror and hands it over; the web process never downloads it.
690
+ try:
691
+ result["state"] = "local" if self._has_local(con) else "absent"
692
+ finally:
693
+ con.close()
694
+ with _CATCH_UP_LOCK:
695
+ _catch_up_source(rt, self.connector)["pass_pending"] = True
696
+ return result
697
  _RESTORE_PROGRESS.store = self
698
  try:
699
  if self._has_local(con):
 
710
  local_id = datastore._publication(con, self.connector).get("snapshot_id")
711
  if saved.get("snapshot_id") and saved["snapshot_id"] != local_id:
712
  self._install_snapshot(con, saved, rows, paths)
713
+ _notify_publish(self.connector, tenant_key, saved, rows, paths, log)
714
  result["manifest"] = saved
715
  if not result["manifest"].get("snapshot_id"):
716
  self._after_restore(con, result, log)
 
737
  manifest, rows, paths, total, refused = self._validated_snapshot(
738
  tenant_key, rt, con, raw, tmp, allow_partial=True)
739
  self._install_snapshot(con, manifest, rows, paths)
740
+ _notify_publish(self.connector, tenant_key, manifest, rows, paths, log)
741
  result.update(state="partial" if refused else "restored", manifest=manifest, refused_tables=refused)
742
  if refused:
743
  result["problems"].append(f"{self.label} mirror tables refused: " + ", ".join(refused)
 
1229
  def _open_candidate(store, tenant_key, rt, published):
1230
  """Resume private rows only while their base publication still matches the serving copy."""
1231
  path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
1232
+ candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
1233
  datastore._ensure_sync_state(candidate)
1234
  candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_base (identity VARCHAR, started_at VARCHAR)")
1235
  candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_pending (name VARCHAR PRIMARY KEY)")
 
1441
  raise
1442
  if odoo:
1443
  note("install", installing)
1444
+ for listener in list(_PUBLISH_LISTENERS):
1445
+ try:
1446
+ listener(store.connector, tenant_key, manifest, store._state_rows(candidate), paths)
1447
+ except Exception as exc:
1448
+ # The snapshot is saved and installed here. A failed hand-off only delays readers.
1449
+ log(f"{store.label} mirror hand-off failed ({type(exc).__name__}: {exc})")
1450
  if holds is not None and holds["tables"]:
1451
  candidate.execute("ROLLBACK")
1452
  candidate.execute("DELETE FROM _candidate_base")
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 53d6cab2185aa5cb904c952ca5aa9ddce217d5d8c8bde9b2bb0ecfb9b73cd4a1
 
1
+ 37d432d140cdd437d7215e992efb2bbe154af782d7202e71f19aabc15857ddc6
web/src/sheet/SheetGrid.tsx CHANGED
@@ -492,7 +492,17 @@ const BINDING_REFUSAL_COPY: Readonly<Record<string, { display: string; message:
492
  // calculation, and painting it `#ERROR!` is what made a working binding look broken.
493
  sheet_refresh_required: { display: "#REFRESH", message: "Press Refresh to calculate this cell for your access." },
494
  store_catching_up: { display: "#UPDATING", message: "This database is updating. Refresh when it finishes." },
 
 
 
 
495
  };
 
 
 
 
 
 
496
  const BINDING_REFUSAL_FALLBACK = { display: "#ERROR!", message: "This cell's database binding could not be calculated." };
497
 
498
  /** R48: old documents may have a blank display, but the grid always owns its refusal copy. */
@@ -1983,7 +1993,7 @@ export default function SheetGrid({
1983
  const shimmerPhase = useRef(0);
1984
  const pendingBoundCells = useMemo(
1985
  () => Object.entries(cells)
1986
- .filter(([, cell]) => cell && cell.kind === "bound" && cell.state === "stale")
1987
  .map(([addr]) => addr),
1988
  [cells],
1989
  );
@@ -6671,7 +6681,7 @@ export default function SheetGrid({
6671
  * `tokenColour` (DESIGN.md), so the softness is made with `globalAlpha` instead.
6672
  */
6673
  const paintShimmer = () => {
6674
- if (!cell || cell.kind !== "bound" || cell.state !== "stale") return;
6675
  const bandWidth = Math.max(24, rect.width * 0.5);
6676
  const head = rect.x - bandWidth + (rect.width + bandWidth) * shimmerPhase.current;
6677
  const slices = 8;
 
492
  // calculation, and painting it `#ERROR!` is what made a working binding look broken.
493
  sheet_refresh_required: { display: "#REFRESH", message: "Press Refresh to calculate this cell for your access." },
494
  store_catching_up: { display: "#UPDATING", message: "This database is updating. Refresh when it finishes." },
495
+ source_unavailable: { display: "#UPDATING", message: "The database was busy. This cell updates after the next sync." },
496
+ store_not_ready: { display: "#UPDATING", message: "The database was busy. This cell updates after the next sync." },
497
+ mirror_not_ready: { display: "#UPDATING", message: "The database was busy. This cell updates after the next sync." },
498
+ amazon_mirror_not_loaded: { display: "#UPDATING", message: "The database was busy. This cell updates after the next sync." },
499
  };
500
+
501
+ /** A bound cell still waiting for an answer. A stale cell with a server reason keeps its value and does not shimmer. */
502
+ function boundWaiting(cell: Cell | null | undefined): cell is Cell {
503
+ return !!cell && cell.kind === "bound" && cell.state === "stale"
504
+ && !(cell as Cell & { staleReason?: string }).staleReason;
505
+ }
506
  const BINDING_REFUSAL_FALLBACK = { display: "#ERROR!", message: "This cell's database binding could not be calculated." };
507
 
508
  /** R48: old documents may have a blank display, but the grid always owns its refusal copy. */
 
1993
  const shimmerPhase = useRef(0);
1994
  const pendingBoundCells = useMemo(
1995
  () => Object.entries(cells)
1996
+ .filter(([, cell]) => boundWaiting(cell))
1997
  .map(([addr]) => addr),
1998
  [cells],
1999
  );
 
6681
  * `tokenColour` (DESIGN.md), so the softness is made with `globalAlpha` instead.
6682
  */
6683
  const paintShimmer = () => {
6684
+ if (!boundWaiting(cell)) return;
6685
  const bandWidth = Math.max(24, rect.width * 0.5);
6686
  const head = rect.x - bandWidth + (rect.width + bandWidth) * shimmerPhase.current;
6687
  const slices = 8;
web/src/sheet/SheetPage.tsx CHANGED
@@ -71,7 +71,7 @@ import { readWorkbookChecks, writeWorkbookChecks } from "./sheetApi";
71
  import type { RailStrata } from "./SheetRail";
72
  import {
73
  computeCell, createdFolderIds, createWorkbook, deleteWorkbook, duplicateSheetTab, reorderSheetTab, patchTabCells, readIndex, readTab, readTabMeta,
74
- applyRefreshScope, isSourceStale, placementOnlyDelta, readWorkbook, refreshCells, refreshScope, retryRefresh, renameTabName, renameWorkbookName, writeIndex,
75
  writePlacements, writeWorkbook,
76
  } from "./sheetApi";
77
  import type { RefreshScope, Result, SheetIndex, SheetMountAddress, WorkbookActivity } from "./sheetApi";
@@ -1175,9 +1175,9 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
1175
  }, [book?.id, opening]);
1176
 
1177
  /**
1178
- * S66-T07 / R15: resolve missing caller slots on any open, including the initial read.
1179
- * Fresh slots and the admin owner's stored answers stay as read (SDLC 65 C2). Only an explicit
1180
- * sheet_refresh_required refusal starts W56-T03's background refresh and pending marks.
1181
  *
1182
  * ⛔ THIS PASS NEVER SETS `busy`. The grid stays usable while a large workbook recomputes,
1183
  * and the existing per-cell stale mark is the progress signal. Its answer is abandoned when a
@@ -1196,7 +1196,7 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
1196
  // (~470 cells, minutes on Daily Sales). Now only the cells that need it are marked, sent to
1197
  // `POST /refresh {cells}`, and merged back one by one; every other cell keeps its answer.
1198
  const needs = loaded ? Object.entries(loaded.cells).filter(([, cell]) => cell.kind === "bound"
1199
- && ((cell.state === "stale" && !isSourceStale(cell))
1200
  || (cell.state === "refused" && cell.refusal?.code === "sheet_refresh_required"))) : [];
1201
  if (needs.length === 0) return;
1202
  const pass = { id: wid, restore: () => restoreMarkedCells() };
@@ -1275,11 +1275,8 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
1275
  const current = bookRef.current;
1276
  const needsCells = current && !tabLoadedRef.current.has(tabKeyOf(current.id, id));
1277
  setPendingTab(needsCells ? { workbookId: current.id, tabId: id, generation } : null);
1278
- if (!current) return;
1279
- if (!needsCells) {
1280
- refreshOpenedWorkbook(current.id, id);
1281
- return;
1282
- }
1283
  const stillCurrent = () => tabNavVerdict(generation, tabNavGenRef.current) === "apply"
1284
  && addressMountedRef.current && !workbookTransitionRef.current
1285
  && bookRef.current?.id === current.id
@@ -1287,7 +1284,6 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
1287
  void (async () => {
1288
  try {
1289
  const loaded = await ensureTabs(current.id, [id], false, stillCurrent);
1290
- if (loaded && stillCurrent()) refreshOpenedWorkbook(current.id, id);
1291
  // Missing tabs and workbooks already restore a real surface through ensureTabs.
1292
  if (!loaded && stillCurrent() && bookRef.current?.sheets.some((tab) => tab.id === id)) {
1293
  setUnreadableAddressTab(id);
@@ -1302,7 +1298,7 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
1302
  setPendingTab((pending) => pending?.generation === generation ? null : pending);
1303
  }
1304
  })();
1305
- }, [currentSheetHash, ensureTabs, rememberOpenWorkbook, refreshOpenedWorkbook]);
1306
 
1307
  // Wave 60 base kit: a hidden or very hidden sheet has no tab, so it is never the open one. When
1308
  // the open tab is hidden (by the View ribbon, an undo, or a workbook that opened on one), the
@@ -2815,7 +2811,6 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
2815
  const live = bookRef.current!;
2816
  if (!address.tab && !address.cell) {
2817
  acceptedAddressRef.current = address;
2818
- refreshOpenedWorkbook(live.id, tabIdRef.current);
2819
  return;
2820
  }
2821
  // A cell-only link narrows the current tab; unknown tabs quietly keep that tab.
@@ -2873,7 +2868,6 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
2873
  // the same tab, while ordinary saves and background refreshes keep the existing mount.
2874
  setAddressSelectionKey((key) => key + 1);
2875
  acceptedAddressRef.current = { ...address, wb: opened.id };
2876
- refreshOpenedWorkbook(opened.id, activeTab);
2877
  } finally {
2878
  workbookTransitionRef.current = false;
2879
  if (addressMountedRef.current) {
@@ -2884,7 +2878,7 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
2884
  }
2885
  })();
2886
  }, [sheetHash, opening, barAddr, adoptWorkbook, ensureTabs, flushBeforeWorkbookChange,
2887
- lastWorkbookId, prepareWorkbookOpen, refreshIndex, refreshOpenedWorkbook, discardMissingWorkbook,
2888
  rememberOpenWorkbook, currentSheetHash, navigateSheetAddress, mergeTab, readAttempt]);
2889
 
2890
  // The workbook can open before its index arrives. Enrich only after the open has settled,
 
71
  import type { RailStrata } from "./SheetRail";
72
  import {
73
  computeCell, createdFolderIds, createWorkbook, deleteWorkbook, duplicateSheetTab, reorderSheetTab, patchTabCells, readIndex, readTab, readTabMeta,
74
+ applyRefreshScope, isSettledStale, placementOnlyDelta, readWorkbook, refreshCells, refreshScope, retryRefresh, renameTabName, renameWorkbookName, writeIndex,
75
  writePlacements, writeWorkbook,
76
  } from "./sheetApi";
77
  import type { RefreshScope, Result, SheetIndex, SheetMountAddress, WorkbookActivity } from "./sheetApi";
 
1175
  }, [book?.id, opening]);
1176
 
1177
  /**
1178
+ * Resolve the bound cells a structural edit or a new tab left without an answer.
1179
+ * ⛔ Opening a workbook or a tab never calls this: an open only reads stored results.
1180
+ * A cell the server serves stale with a reason keeps its stored value and is not resolved here.
1181
  *
1182
  * ⛔ THIS PASS NEVER SETS `busy`. The grid stays usable while a large workbook recomputes,
1183
  * and the existing per-cell stale mark is the progress signal. Its answer is abandoned when a
 
1196
  // (~470 cells, minutes on Daily Sales). Now only the cells that need it are marked, sent to
1197
  // `POST /refresh {cells}`, and merged back one by one; every other cell keeps its answer.
1198
  const needs = loaded ? Object.entries(loaded.cells).filter(([, cell]) => cell.kind === "bound"
1199
+ && ((cell.state === "stale" && !isSettledStale(cell))
1200
  || (cell.state === "refused" && cell.refusal?.code === "sheet_refresh_required"))) : [];
1201
  if (needs.length === 0) return;
1202
  const pass = { id: wid, restore: () => restoreMarkedCells() };
 
1275
  const current = bookRef.current;
1276
  const needsCells = current && !tabLoadedRef.current.has(tabKeyOf(current.id, id));
1277
  setPendingTab(needsCells ? { workbookId: current.id, tabId: id, generation } : null);
1278
+ // Opening a tab only reads. The sync schedule and the Refresh button compute bound cells.
1279
+ if (!current || !needsCells) return;
 
 
 
1280
  const stillCurrent = () => tabNavVerdict(generation, tabNavGenRef.current) === "apply"
1281
  && addressMountedRef.current && !workbookTransitionRef.current
1282
  && bookRef.current?.id === current.id
 
1284
  void (async () => {
1285
  try {
1286
  const loaded = await ensureTabs(current.id, [id], false, stillCurrent);
 
1287
  // Missing tabs and workbooks already restore a real surface through ensureTabs.
1288
  if (!loaded && stillCurrent() && bookRef.current?.sheets.some((tab) => tab.id === id)) {
1289
  setUnreadableAddressTab(id);
 
1298
  setPendingTab((pending) => pending?.generation === generation ? null : pending);
1299
  }
1300
  })();
1301
+ }, [currentSheetHash, ensureTabs, rememberOpenWorkbook]);
1302
 
1303
  // Wave 60 base kit: a hidden or very hidden sheet has no tab, so it is never the open one. When
1304
  // the open tab is hidden (by the View ribbon, an undo, or a workbook that opened on one), the
 
2811
  const live = bookRef.current!;
2812
  if (!address.tab && !address.cell) {
2813
  acceptedAddressRef.current = address;
 
2814
  return;
2815
  }
2816
  // A cell-only link narrows the current tab; unknown tabs quietly keep that tab.
 
2868
  // the same tab, while ordinary saves and background refreshes keep the existing mount.
2869
  setAddressSelectionKey((key) => key + 1);
2870
  acceptedAddressRef.current = { ...address, wb: opened.id };
 
2871
  } finally {
2872
  workbookTransitionRef.current = false;
2873
  if (addressMountedRef.current) {
 
2878
  }
2879
  })();
2880
  }, [sheetHash, opening, barAddr, adoptWorkbook, ensureTabs, flushBeforeWorkbookChange,
2881
+ lastWorkbookId, prepareWorkbookOpen, refreshIndex, discardMissingWorkbook,
2882
  rememberOpenWorkbook, currentSheetHash, navigateSheetAddress, mergeTab, readAttempt]);
2883
 
2884
  // The workbook can open before its index arrives. Enrich only after the open has settled,
web/src/sheet/sheetApi.ts CHANGED
@@ -513,6 +513,10 @@ export const retryRefresh = (id: string, refreshId: string): Promise<Result<Refr
513
  export const isSourceStale = (cell: Cell): boolean =>
514
  (cell as Cell & { staleReason?: string }).staleReason === "source_changed";
515
 
 
 
 
 
516
  export type RefreshScopeResult = { addresses: string[]; recalcMs: number | null } & (
517
  { workbook: Workbook } | { workbook: Pick<Workbook, "id" | "rev">; cells: Record<string, Cell> }
518
  );
 
513
  export const isSourceStale = (cell: Cell): boolean =>
514
  (cell as Cell & { staleReason?: string }).staleReason === "source_changed";
515
 
516
+ /** A stale cell the server explained keeps its stored value: it is settled, not pending. */
517
+ export const isSettledStale = (cell: Cell | null | undefined): boolean =>
518
+ !!cell && cell.state === "stale" && !!(cell as Cell & { staleReason?: string }).staleReason;
519
+
520
  export type RefreshScopeResult = { addresses: string[]; recalcMs: number | null } & (
521
  { workbook: Workbook } | { workbook: Pick<Workbook, "id" | "rev">; cells: Record<string, Cell> }
522
  );