Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/automation_engine.py +5 -2
- api/main.py +56 -26
- api/process_role.py +463 -0
- api/routes_admin.py +9 -0
- api/routes_connectors.py +8 -4
- api/routes_platform_admin.py +6 -2
- api/routes_sheet.py +188 -32
- api/sheet_warm.py +9 -4
- api/sync_worker.py +70 -0
- platform/core/connector_sync.py +147 -5
- platform/harness/datastore.py +17 -4
- platform/harness/mirror_store.py +100 -1
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/sheet/SheetGrid.tsx +12 -2
- web/src/sheet/SheetPage.tsx +9 -15
- web/src/sheet/sheetApi.ts +4 -0
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v83",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "5a3e53380f59 (5a3e53380f59)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v83",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
5a3e53380f59 (5a3e53380f59)
|
api/automation_engine.py
CHANGED
|
@@ -16723,8 +16723,11 @@ def start_scheduler(log=print):
|
|
| 16723 |
return False
|
| 16724 |
if _SCHEDULER[0] is not None:
|
| 16725 |
return False
|
| 16726 |
-
|
| 16727 |
-
|
|
|
|
|
|
|
|
|
|
| 16728 |
_SCHEDULER[0] = th
|
| 16729 |
th.start()
|
| 16730 |
return True
|
|
|
|
| 16723 |
return False
|
| 16724 |
if _SCHEDULER[0] is not None:
|
| 16725 |
return False
|
| 16726 |
+
import process_role
|
| 16727 |
+
th = process_role.thread(target=scheduler_loop, kwargs={"log": log},
|
| 16728 |
+
daemon=True, name="automation-scheduler", kind="heavy")
|
| 16729 |
+
if not process_role.admitted(th):
|
| 16730 |
+
return False
|
| 16731 |
_SCHEDULER[0] = th
|
| 16732 |
th.start()
|
| 16733 |
return True
|
api/main.py
CHANGED
|
@@ -36,6 +36,7 @@ for _p in (str(_RI), str(_HERE.parent)):
|
|
| 36 |
|
| 37 |
from dotenv import load_dotenv # noqa: E402
|
| 38 |
load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
|
|
|
|
| 39 |
|
| 40 |
from fastapi import Body, Depends, FastAPI, Request # noqa: E402
|
| 41 |
from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
|
|
@@ -923,7 +924,12 @@ def _pull_amazon(why, vendor=True):
|
|
| 923 |
|
| 924 |
def _restore_amazon_mirror(why):
|
| 925 |
"""Restore every known tenant's Amazon manifest without moving the resident file."""
|
|
|
|
|
|
|
| 926 |
try:
|
|
|
|
|
|
|
|
|
|
| 927 |
from harness import runtime as _runtime
|
| 928 |
results = [_restore_amazon_mirror_once(why, key) for key in _runtime.known_tenants()]
|
| 929 |
errors = RESYNC_STATE.get("amazonRestoreErrors", {})
|
|
@@ -931,6 +937,8 @@ def _restore_amazon_mirror(why):
|
|
| 931 |
RESYNC_STATE["amazonRestoreError"] = next(iter(errors.values()))
|
| 932 |
else:
|
| 933 |
RESYNC_STATE.pop("amazonRestoreError", None)
|
|
|
|
|
|
|
| 934 |
return all(results) and not errors
|
| 935 |
except Exception as exc: # noqa: BLE001
|
| 936 |
RESYNC_STATE["amazonRestoreError"] = {
|
|
@@ -977,7 +985,9 @@ def _restore_amazon_mirror_once(why, tenant_key):
|
|
| 977 |
if result.get("state") == "refused":
|
| 978 |
raise RuntimeError("; ".join(result.get("problems") or ["Amazon mirror restore refused."]))
|
| 979 |
# Publish measured definitions before any Odoo read can block the resident writer.
|
| 980 |
-
|
|
|
|
|
|
|
| 981 |
rt.datastore_path.resolve() == _ds.DB_PATH.resolve()):
|
| 982 |
state = _rebuild_amazon_relational(why, rt)
|
| 983 |
if state.get("changed") and not state.get("applied") and not state.get("known"):
|
|
@@ -1293,14 +1303,12 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
|
|
| 1293 |
# ⛔ HOTFIX 2026-09-29: NOT every 30 seconds. A pass that landed nothing can only defer
|
| 1294 |
# again until Amazon's next report floor opens, and each of those passes republished
|
| 1295 |
# the mirror's state to the store. Wait for the floor, then continue.
|
| 1296 |
-
import threading as _threading_desk
|
| 1297 |
delay = min(max(floor_wait + 5.0, 60.0), 86400.0)
|
| 1298 |
-
|
| 1299 |
-
|
| 1300 |
-
|
| 1301 |
-
|
| 1302 |
-
|
| 1303 |
-
print(f"[aios-api] amazon first load waits {int(delay)}s for the next report floor")
|
| 1304 |
elif continuing and ok:
|
| 1305 |
# Finish first: the desk refuses a second in-flight intent. The resident loop claims
|
| 1306 |
# the continuation as a DESK-ONLY wake; the Odoo pass keeps its own cadence.
|
|
@@ -2776,14 +2784,17 @@ def _boot_app_migrations(why):
|
|
| 2776 |
def _prewarm():
|
| 2777 |
if _freeze_holds("prewarm"):
|
| 2778 |
return
|
|
|
|
| 2779 |
import time as _t
|
| 2780 |
t0 = _t.time()
|
| 2781 |
# The Odoo seed/sync can run for minutes and is separate from the finite cache warm below.
|
| 2782 |
# Keep it opt-in: Live Pg serves its seeded analytical mirror at boot and lets explicit
|
| 2783 |
# workers perform real sync work, so a routine restart can still scale Neon back to zero.
|
| 2784 |
if os.environ.get("AIOS_BOOT_SYNC") == "1" and os.environ.get("AIOS_STORE_RESYNC") != "1":
|
| 2785 |
-
|
| 2786 |
-
|
|
|
|
|
|
|
| 2787 |
try:
|
| 2788 |
from harness import runtime as _runtime
|
| 2789 |
rt = _runtime.get_runtime("royal-imports")
|
|
@@ -2948,25 +2959,43 @@ def _await_mirror_restore():
|
|
| 2948 |
return
|
| 2949 |
thread = globals().get("_SEED_THREAD")
|
| 2950 |
if thread is None:
|
| 2951 |
-
import
|
| 2952 |
-
|
|
|
|
| 2953 |
globals()["_SEED_THREAD"] = thread
|
| 2954 |
thread.start()
|
| 2955 |
|
| 2956 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2957 |
def _restore_sync_request_desks():
|
| 2958 |
from core import connector_sync
|
| 2959 |
from harness import runtime
|
|
|
|
|
|
|
|
|
|
| 2960 |
for tenant_key in runtime.known_tenants():
|
| 2961 |
try:
|
| 2962 |
-
connector_sync.restore_sync_requests(runtime.get_runtime(tenant_key))
|
| 2963 |
except Exception as exc:
|
| 2964 |
print(f"[aios-api] sync request restore refused for {tenant_key} ({type(exc).__name__})")
|
| 2965 |
|
| 2966 |
|
| 2967 |
def _boot_mirrors():
|
|
|
|
| 2968 |
_restore_sync_request_desks()
|
| 2969 |
_seed_mirror_if_absent()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2970 |
_restore_amazon_mirror("boot")
|
| 2971 |
_restore_declared_mirrors("boot")
|
| 2972 |
|
|
@@ -2997,9 +3026,9 @@ def _restore_declared_mirrors(why):
|
|
| 2997 |
try:
|
| 2998 |
from harness import datastore as _ds_boot
|
| 2999 |
if not data_binding.gate_process() and (data_binding.is_space() or not _ds_boot.DB_PATH.exists()):
|
| 3000 |
-
import
|
| 3001 |
-
_SEED_THREAD =
|
| 3002 |
-
|
| 3003 |
_SEED_THREAD.start()
|
| 3004 |
except Exception as e: # noqa: BLE001
|
| 3005 |
print(f"[aios-api] mirror seed not scheduled: {e}")
|
|
@@ -3012,25 +3041,26 @@ if (not data_binding.gate_process()
|
|
| 3012 |
and os.environ.get("AIOS_ENABLE_QA_TENANT") == "1"
|
| 3013 |
and os.environ.get("STORE_BACKEND") == "pg"
|
| 3014 |
and not _freeze_holds("QA schema provisioning")):
|
| 3015 |
-
import
|
| 3016 |
-
|
| 3017 |
-
|
| 3018 |
|
| 3019 |
if (not data_binding.gate_process()
|
| 3020 |
and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
|
| 3021 |
and (data_binding.is_space() or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
|
| 3022 |
and not _freeze_holds("app migrations")):
|
| 3023 |
-
import
|
| 3024 |
-
|
| 3025 |
-
|
| 3026 |
|
| 3027 |
if os.environ.get("AIOS_BOOT_SYNC") == "1" and freeze_notice.enabled():
|
| 3028 |
_freeze_holds("boot store sync")
|
| 3029 |
|
| 3030 |
if (not data_binding.gate_process()
|
| 3031 |
and os.environ.get("AIOS_PREWARM") == "1" and not _freeze_holds("prewarm")):
|
| 3032 |
-
import
|
| 3033 |
-
|
|
|
|
| 3034 |
|
| 3035 |
# A boot prewarm is finite; the Odoo/store resync loop is resident work, so it stays opt-in.
|
| 3036 |
#
|
|
@@ -3065,7 +3095,7 @@ if (not data_binding.gate_process()
|
|
| 3065 |
# ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
|
| 3066 |
if (not data_binding.gate_process()
|
| 3067 |
and os.environ.get("AIOS_STORE_RESYNC") == "1" and not _freeze_holds("store resync")):
|
| 3068 |
-
import
|
| 3069 |
-
|
| 3070 |
import sheet_warm as _sheet_warm
|
| 3071 |
_sheet_warm.start()
|
|
|
|
| 36 |
|
| 37 |
from dotenv import load_dotenv # noqa: E402
|
| 38 |
load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
|
| 39 |
+
import process_role # noqa: E402 (before any DuckDB connect: the web role sets its memory cap)
|
| 40 |
|
| 41 |
from fastapi import Body, Depends, FastAPI, Request # noqa: E402
|
| 42 |
from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
|
|
|
|
| 924 |
|
| 925 |
def _restore_amazon_mirror(why):
|
| 926 |
"""Restore every known tenant's Amazon manifest without moving the resident file."""
|
| 927 |
+
# One good boot restore per process: the boot seed thread and the resync loop both ask for it.
|
| 928 |
+
boot_done = globals().setdefault("_AMAZON_BOOT_RESTORE", {})
|
| 929 |
try:
|
| 930 |
+
if why == "boot" and boot_done.get("result") is True:
|
| 931 |
+
print("[aios-api] amazon mirror boot restore already done in this process")
|
| 932 |
+
return True
|
| 933 |
from harness import runtime as _runtime
|
| 934 |
results = [_restore_amazon_mirror_once(why, key) for key in _runtime.known_tenants()]
|
| 935 |
errors = RESYNC_STATE.get("amazonRestoreErrors", {})
|
|
|
|
| 937 |
RESYNC_STATE["amazonRestoreError"] = next(iter(errors.values()))
|
| 938 |
else:
|
| 939 |
RESYNC_STATE.pop("amazonRestoreError", None)
|
| 940 |
+
if why == "boot":
|
| 941 |
+
boot_done["result"] = all(results) and not errors
|
| 942 |
return all(results) and not errors
|
| 943 |
except Exception as exc: # noqa: BLE001
|
| 944 |
RESYNC_STATE["amazonRestoreError"] = {
|
|
|
|
| 985 |
if result.get("state") == "refused":
|
| 986 |
raise RuntimeError("; ".join(result.get("problems") or ["Amazon mirror restore refused."]))
|
| 987 |
# Publish measured definitions before any Odoo read can block the resident writer.
|
| 988 |
+
import process_role as _process_role
|
| 989 |
+
# The relational rebuild writes the store, so only the sync worker runs it.
|
| 990 |
+
if (result.get("state") != "absent" and _process_role.allows("heavy") and
|
| 991 |
rt.datastore_path.resolve() == _ds.DB_PATH.resolve()):
|
| 992 |
state = _rebuild_amazon_relational(why, rt)
|
| 993 |
if state.get("changed") and not state.get("applied") and not state.get("known"):
|
|
|
|
| 1303 |
# ⛔ HOTFIX 2026-09-29: NOT every 30 seconds. A pass that landed nothing can only defer
|
| 1304 |
# again until Amazon's next report floor opens, and each of those passes republished
|
| 1305 |
# the mirror's state to the store. Wait for the floor, then continue.
|
|
|
|
| 1306 |
delay = min(max(floor_wait + 5.0, 60.0), 86400.0)
|
| 1307 |
+
import process_role as _process_role
|
| 1308 |
+
if _process_role.start("amazon-first-load-floor", "heavy", connector_sync.request_sync,
|
| 1309 |
+
args=("amazon_sp", tenant), kwargs={"requested_by": "first load"},
|
| 1310 |
+
delay=delay) is not None:
|
| 1311 |
+
print(f"[aios-api] amazon first load waits {int(delay)}s for the next report floor")
|
|
|
|
| 1312 |
elif continuing and ok:
|
| 1313 |
# Finish first: the desk refuses a second in-flight intent. The resident loop claims
|
| 1314 |
# the continuation as a DESK-ONLY wake; the Odoo pass keeps its own cadence.
|
|
|
|
| 2784 |
def _prewarm():
|
| 2785 |
if _freeze_holds("prewarm"):
|
| 2786 |
return
|
| 2787 |
+
import process_role as _process_role
|
| 2788 |
import time as _t
|
| 2789 |
t0 = _t.time()
|
| 2790 |
# The Odoo seed/sync can run for minutes and is separate from the finite cache warm below.
|
| 2791 |
# Keep it opt-in: Live Pg serves its seeded analytical mirror at boot and lets explicit
|
| 2792 |
# workers perform real sync work, so a routine restart can still scale Neon back to zero.
|
| 2793 |
if os.environ.get("AIOS_BOOT_SYNC") == "1" and os.environ.get("AIOS_STORE_RESYNC") != "1":
|
| 2794 |
+
_process_role.start("store-seed-sync", "heavy", _seed_and_sync_store)
|
| 2795 |
+
if _process_role.role() == "worker":
|
| 2796 |
+
# The warm below fills per-process memos, which the sync worker never serves.
|
| 2797 |
+
return
|
| 2798 |
try:
|
| 2799 |
from harness import runtime as _runtime
|
| 2800 |
rt = _runtime.get_runtime("royal-imports")
|
|
|
|
| 2959 |
return
|
| 2960 |
thread = globals().get("_SEED_THREAD")
|
| 2961 |
if thread is None:
|
| 2962 |
+
import process_role as _process_role
|
| 2963 |
+
# Light: the web process restores its resident mirrors so it has data to read.
|
| 2964 |
+
thread = _process_role.thread(target=_boot_mirrors, daemon=True, name="mirror-seed", kind="light")
|
| 2965 |
globals()["_SEED_THREAD"] = thread
|
| 2966 |
thread.start()
|
| 2967 |
|
| 2968 |
|
| 2969 |
+
@app.on_event("startup")
|
| 2970 |
+
def _start_process_role():
|
| 2971 |
+
"""Web role: spawn and supervise the sync worker, forward Sync now, install its snapshots."""
|
| 2972 |
+
if data_binding.gate_process():
|
| 2973 |
+
return
|
| 2974 |
+
import process_role as _process_role
|
| 2975 |
+
_process_role.start_web_services()
|
| 2976 |
+
|
| 2977 |
+
|
| 2978 |
def _restore_sync_request_desks():
|
| 2979 |
from core import connector_sync
|
| 2980 |
from harness import runtime
|
| 2981 |
+
import process_role as _process_role
|
| 2982 |
+
# The web role never claims, so it must not requeue the worker's in-flight claims.
|
| 2983 |
+
requeue = _process_role.role() != "web"
|
| 2984 |
for tenant_key in runtime.known_tenants():
|
| 2985 |
try:
|
| 2986 |
+
connector_sync.restore_sync_requests(runtime.get_runtime(tenant_key), requeue=requeue)
|
| 2987 |
except Exception as exc:
|
| 2988 |
print(f"[aios-api] sync request restore refused for {tenant_key} ({type(exc).__name__})")
|
| 2989 |
|
| 2990 |
|
| 2991 |
def _boot_mirrors():
|
| 2992 |
+
import process_role as _role_boot
|
| 2993 |
_restore_sync_request_desks()
|
| 2994 |
_seed_mirror_if_absent()
|
| 2995 |
+
if _role_boot.role() == "web":
|
| 2996 |
+
# Every connector but Odoo reaches the web process through the worker's hand-off.
|
| 2997 |
+
print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
|
| 2998 |
+
return
|
| 2999 |
_restore_amazon_mirror("boot")
|
| 3000 |
_restore_declared_mirrors("boot")
|
| 3001 |
|
|
|
|
| 3026 |
try:
|
| 3027 |
from harness import datastore as _ds_boot
|
| 3028 |
if not data_binding.gate_process() and (data_binding.is_space() or not _ds_boot.DB_PATH.exists()):
|
| 3029 |
+
import process_role as _role_seed
|
| 3030 |
+
_SEED_THREAD = _role_seed.thread(target=_boot_mirrors, daemon=True, kind="light",
|
| 3031 |
+
name="mirror-seed")
|
| 3032 |
_SEED_THREAD.start()
|
| 3033 |
except Exception as e: # noqa: BLE001
|
| 3034 |
print(f"[aios-api] mirror seed not scheduled: {e}")
|
|
|
|
| 3041 |
and os.environ.get("AIOS_ENABLE_QA_TENANT") == "1"
|
| 3042 |
and os.environ.get("STORE_BACKEND") == "pg"
|
| 3043 |
and not _freeze_holds("QA schema provisioning")):
|
| 3044 |
+
import process_role as _role_qa
|
| 3045 |
+
_role_qa.thread(target=_provision_qa_sandbox, daemon=True, kind="heavy",
|
| 3046 |
+
name="qa-sandbox-schema").start()
|
| 3047 |
|
| 3048 |
if (not data_binding.gate_process()
|
| 3049 |
and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
|
| 3050 |
and (data_binding.is_space() or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
|
| 3051 |
and not _freeze_holds("app migrations")):
|
| 3052 |
+
import process_role as _role_apps
|
| 3053 |
+
_role_apps.thread(target=_boot_app_migrations, args=("boot",), daemon=True, kind="heavy",
|
| 3054 |
+
name="app-migrations").start()
|
| 3055 |
|
| 3056 |
if os.environ.get("AIOS_BOOT_SYNC") == "1" and freeze_notice.enabled():
|
| 3057 |
_freeze_holds("boot store sync")
|
| 3058 |
|
| 3059 |
if (not data_binding.gate_process()
|
| 3060 |
and os.environ.get("AIOS_PREWARM") == "1" and not _freeze_holds("prewarm")):
|
| 3061 |
+
import process_role as _role_prewarm
|
| 3062 |
+
# Light: the cache warm fills this process's memos. Its store sync half is heavy inside.
|
| 3063 |
+
_role_prewarm.thread(target=_prewarm, daemon=True, kind="light", name="prewarm").start()
|
| 3064 |
|
| 3065 |
# A boot prewarm is finite; the Odoo/store resync loop is resident work, so it stays opt-in.
|
| 3066 |
#
|
|
|
|
| 3095 |
# ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
|
| 3096 |
if (not data_binding.gate_process()
|
| 3097 |
and os.environ.get("AIOS_STORE_RESYNC") == "1" and not _freeze_holds("store resync")):
|
| 3098 |
+
import process_role as _role_resync
|
| 3099 |
+
_role_resync.thread(target=_store_resync_loop, daemon=True, kind="heavy", name="store-resync").start()
|
| 3100 |
import sheet_warm as _sheet_warm
|
| 3101 |
_sheet_warm.start()
|
api/process_role.py
ADDED
|
@@ -0,0 +1,463 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One process rule for every database: heavy background work runs only in the sync worker.
|
| 2 |
+
|
| 3 |
+
Roles: `web` serves reads and installs finished mirror snapshots, `worker` runs every heavy job,
|
| 4 |
+
`all` does both in one process (local runs and gates). Kinds: `heavy` moves source data or writes
|
| 5 |
+
the store in the background; `light` serves this process's own reads.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import atexit
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
import shutil
|
| 13 |
+
import subprocess
|
| 14 |
+
import sys
|
| 15 |
+
import threading
|
| 16 |
+
import time
|
| 17 |
+
import uuid
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
ROLES = ("web", "worker", "all")
|
| 21 |
+
KINDS = ("heavy", "light")
|
| 22 |
+
API_DIR = Path(__file__).resolve().parent
|
| 23 |
+
POLL_SECONDS = 15
|
| 24 |
+
HEARTBEAT_SECONDS = 30
|
| 25 |
+
HEARTBEAT_FILE = "worker.heartbeat.json"
|
| 26 |
+
READY_FILE = "READY.json"
|
| 27 |
+
INSTALL_ATTEMPTS = 3
|
| 28 |
+
INCOMPLETE_GRACE_SECONDS = 3600
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def resolve_role(env=None):
|
| 32 |
+
"""(role, source) from AIOS_PROCESS_ROLE. Unset: `web` on a Space, `all` elsewhere."""
|
| 33 |
+
env = os.environ if env is None else env
|
| 34 |
+
raw = str(env.get("AIOS_PROCESS_ROLE") or "").strip().lower()
|
| 35 |
+
if raw in ROLES:
|
| 36 |
+
return raw, "AIOS_PROCESS_ROLE"
|
| 37 |
+
space = bool(str(env.get("SPACE_ID") or "").strip())
|
| 38 |
+
source = (f"unknown AIOS_PROCESS_ROLE {raw!r}" if raw else "AIOS_PROCESS_ROLE unset") + (
|
| 39 |
+
" on a Space" if space else " off a Space")
|
| 40 |
+
return ("web" if space else "all"), source
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
ROLE, ROLE_SOURCE = resolve_role()
|
| 44 |
+
# The data layer cannot import this module, so it reads the resolved role from the environment.
|
| 45 |
+
os.environ["AIOS_PROCESS_ROLE"] = ROLE
|
| 46 |
+
if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web":
|
| 47 |
+
print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True)
|
| 48 |
+
if ROLE == "web":
|
| 49 |
+
# The web process keeps a smaller DuckDB budget. apply_limits reads this at connect time.
|
| 50 |
+
os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "3GB")
|
| 51 |
+
if os.environ.get("AIOS_WEB_DUCKDB_THREADS"):
|
| 52 |
+
os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ["AIOS_WEB_DUCKDB_THREADS"])
|
| 53 |
+
|
| 54 |
+
_LOCK = threading.Lock()
|
| 55 |
+
_STARTED = {}
|
| 56 |
+
_STOP = threading.Event()
|
| 57 |
+
_WORKER = {"proc": None, "pid": None, "restarts": 0, "startedAt": None, "lastExit": None}
|
| 58 |
+
_INSTALLED = {}
|
| 59 |
+
_INSTALL_FAILURES = {}
|
| 60 |
+
LAST_INSTALL = {}
|
| 61 |
+
LAST_INSTALL_ERROR = {}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def role():
|
| 65 |
+
return ROLE
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def allows(kind, role_name=None):
|
| 69 |
+
"""Whether a job of this kind may start in the role."""
|
| 70 |
+
if kind not in KINDS:
|
| 71 |
+
raise ValueError(f"unknown job kind {kind!r}")
|
| 72 |
+
return kind == "light" or (role_name or ROLE) in ("worker", "all")
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def admit(name, kind, role_name=None):
|
| 76 |
+
"""allows(), plus one refusal line. For call sites whose callee owns its thread."""
|
| 77 |
+
if allows(kind, role_name):
|
| 78 |
+
return True
|
| 79 |
+
print(f"[process-role] refused {name} ({kind}) in role {role_name or ROLE}", flush=True)
|
| 80 |
+
return False
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class _Refused:
|
| 84 |
+
"""What thread() returns for a refused job: start() does nothing and returns None."""
|
| 85 |
+
|
| 86 |
+
def __init__(self, name):
|
| 87 |
+
self.name = name
|
| 88 |
+
|
| 89 |
+
def start(self):
|
| 90 |
+
return None
|
| 91 |
+
|
| 92 |
+
def is_alive(self):
|
| 93 |
+
return False
|
| 94 |
+
|
| 95 |
+
def join(self, timeout=None):
|
| 96 |
+
return None
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def thread(*, target, name, kind, daemon=True, args=(), kwargs=None, delay=None, role_name=None):
|
| 100 |
+
"""An unstarted Thread, or a Timer when delay is given. A refused job gets an inert stand-in."""
|
| 101 |
+
if not admit(name, kind, role_name):
|
| 102 |
+
return _Refused(name)
|
| 103 |
+
if delay is None:
|
| 104 |
+
job = threading.Thread(target=target, args=tuple(args), kwargs=dict(kwargs or {}),
|
| 105 |
+
daemon=daemon, name=name)
|
| 106 |
+
else:
|
| 107 |
+
job = threading.Timer(float(delay), target, args=list(args), kwargs=dict(kwargs or {}))
|
| 108 |
+
job.daemon = daemon
|
| 109 |
+
job.name = name
|
| 110 |
+
with _LOCK:
|
| 111 |
+
_STARTED[name] = {"kind": kind, "at": time.time()}
|
| 112 |
+
return job
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def admitted(job):
|
| 116 |
+
"""False for the stand-in thread() returns when it refuses a job."""
|
| 117 |
+
return not isinstance(job, _Refused)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def start(name, kind, target, *, daemon=True, args=(), kwargs=None, delay=None, role_name=None):
|
| 121 |
+
"""Start one background job as a Thread, or a Timer when delay is given. None when refused."""
|
| 122 |
+
job = thread(target=target, name=name, kind=kind, daemon=daemon, args=args, kwargs=kwargs,
|
| 123 |
+
delay=delay, role_name=role_name)
|
| 124 |
+
if not admitted(job):
|
| 125 |
+
return None
|
| 126 |
+
job.start()
|
| 127 |
+
return job
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def started_jobs():
|
| 131 |
+
with _LOCK:
|
| 132 |
+
return {name: dict(row) for name, row in _STARTED.items()}
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
# ----------------------------------------------------------------------------- the hand-off dir
|
| 136 |
+
def handoff_dir():
|
| 137 |
+
return Path(os.environ.get("AIOS_MIRROR_HANDOFF_DIR") or "/tmp/aios-mirror-handoff")
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _write_json(path, value):
|
| 141 |
+
"""Write JSON through a sibling temp file and os.replace, so a reader never sees half of it."""
|
| 142 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 143 |
+
tmp = path.with_name(path.name + f".{os.getpid()}.tmp")
|
| 144 |
+
with open(tmp, "w", encoding="utf-8", newline="") as handle:
|
| 145 |
+
json.dump(value, handle, default=str)
|
| 146 |
+
os.replace(tmp, path)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def _read_json(path):
|
| 150 |
+
try:
|
| 151 |
+
with open(path, encoding="utf-8") as handle:
|
| 152 |
+
return json.load(handle)
|
| 153 |
+
except (OSError, ValueError):
|
| 154 |
+
return None
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def write_handoff(connector, tenant_key, manifest, rows, paths, *, root=None, now=None):
|
| 158 |
+
"""Keep one published snapshot's parquet for the web process. READY.json is written last."""
|
| 159 |
+
root = Path(root or handoff_dir())
|
| 160 |
+
snapshot = str(manifest.get("snapshot_id") or uuid.uuid4().hex)
|
| 161 |
+
target = root / connector / snapshot
|
| 162 |
+
tables = target / "tables"
|
| 163 |
+
tables.mkdir(parents=True, exist_ok=True)
|
| 164 |
+
kept = {}
|
| 165 |
+
for name, files in paths.items():
|
| 166 |
+
kept[name] = []
|
| 167 |
+
for index, source in enumerate(files):
|
| 168 |
+
copy = tables / f"{name}.{index}.parquet"
|
| 169 |
+
copy.unlink(missing_ok=True)
|
| 170 |
+
try:
|
| 171 |
+
os.link(source, copy)
|
| 172 |
+
except OSError:
|
| 173 |
+
shutil.copyfile(source, copy)
|
| 174 |
+
kept[name].append(str(copy))
|
| 175 |
+
_write_json(target / READY_FILE, {
|
| 176 |
+
"version": 1, "tenant": tenant_key, "connector": connector, "snapshotId": snapshot,
|
| 177 |
+
"completedAt": time.time() if now is None else now, "manifest": manifest,
|
| 178 |
+
"stateRows": [list(row) for row in rows], "paths": kept})
|
| 179 |
+
return target
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def ready_handoffs(root=None):
|
| 183 |
+
"""{(tenant, connector): [(completedAt, dir, ready), ...]} oldest first. Dirs without READY.json are skipped."""
|
| 184 |
+
root = Path(root or handoff_dir())
|
| 185 |
+
groups = {}
|
| 186 |
+
if not root.is_dir():
|
| 187 |
+
return groups
|
| 188 |
+
for connector_dir in root.iterdir():
|
| 189 |
+
if not connector_dir.is_dir():
|
| 190 |
+
continue
|
| 191 |
+
for snapshot_dir in connector_dir.iterdir():
|
| 192 |
+
ready = _read_json(snapshot_dir / READY_FILE) if snapshot_dir.is_dir() else None
|
| 193 |
+
if not isinstance(ready, dict) or not ready.get("tenant") or not isinstance(ready.get("paths"), dict):
|
| 194 |
+
continue
|
| 195 |
+
key = (str(ready["tenant"]), str(ready.get("connector") or connector_dir.name))
|
| 196 |
+
groups.setdefault(key, []).append((float(ready.get("completedAt") or 0), snapshot_dir, ready))
|
| 197 |
+
for rows in groups.values():
|
| 198 |
+
rows.sort(key=lambda row: row[0])
|
| 199 |
+
return groups
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def _install_default(connector, tenant, ready):
|
| 203 |
+
from harness import mirror_store, runtime
|
| 204 |
+
rt = runtime.get_runtime(tenant)
|
| 205 |
+
paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
|
| 206 |
+
installed = mirror_store.install_handoff(connector, tenant, rt, ready["manifest"],
|
| 207 |
+
ready.get("stateRows") or [], paths)
|
| 208 |
+
try:
|
| 209 |
+
import routes_odoo_tables
|
| 210 |
+
routes_odoo_tables.invalidate_sources()
|
| 211 |
+
except Exception as exc: # noqa: BLE001
|
| 212 |
+
print(f"[process-role] read registry refresh failed ({type(exc).__name__})", flush=True)
|
| 213 |
+
return installed
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def _remove_dir(path):
|
| 217 |
+
shutil.rmtree(path, ignore_errors=True)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def install_ready(root=None, installer=None, *, now=None):
|
| 221 |
+
"""Install the newest READY snapshot per tenant and connector, then delete older hand-offs."""
|
| 222 |
+
root = Path(root or handoff_dir())
|
| 223 |
+
installer = installer or _install_default
|
| 224 |
+
stamp = time.time() if now is None else now
|
| 225 |
+
installed = []
|
| 226 |
+
for key, rows in ready_handoffs(root).items():
|
| 227 |
+
completed_at, newest_dir, ready = rows[-1]
|
| 228 |
+
snapshot = ready.get("snapshotId") or newest_dir.name
|
| 229 |
+
if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS:
|
| 230 |
+
try:
|
| 231 |
+
installer(key[1], key[0], ready)
|
| 232 |
+
except Exception as exc: # noqa: BLE001
|
| 233 |
+
_INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
|
| 234 |
+
LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
|
| 235 |
+
attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
|
| 236 |
+
print(f"[process-role] hand-off install {key[0]}/{key[1]} {snapshot} failed "
|
| 237 |
+
f"({type(exc).__name__}: {exc})", flush=True)
|
| 238 |
+
continue
|
| 239 |
+
_INSTALLED[key] = snapshot
|
| 240 |
+
LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
|
| 241 |
+
if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
|
| 242 |
+
LAST_INSTALL_ERROR.clear()
|
| 243 |
+
installed.append(key)
|
| 244 |
+
print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]}", flush=True)
|
| 245 |
+
for _at, older, _ready in rows[:-1]:
|
| 246 |
+
_remove_dir(older)
|
| 247 |
+
# A worker that died mid-copy leaves a dir without READY.json. Drop it after a grace period.
|
| 248 |
+
if root.is_dir():
|
| 249 |
+
for connector_dir in (p for p in root.iterdir() if p.is_dir()):
|
| 250 |
+
for snapshot_dir in (p for p in connector_dir.iterdir() if p.is_dir()):
|
| 251 |
+
if not (snapshot_dir / READY_FILE).exists():
|
| 252 |
+
try:
|
| 253 |
+
old = stamp - snapshot_dir.stat().st_mtime > INCOMPLETE_GRACE_SECONDS
|
| 254 |
+
except OSError:
|
| 255 |
+
old = False
|
| 256 |
+
if old:
|
| 257 |
+
_remove_dir(snapshot_dir)
|
| 258 |
+
return installed
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
# ----------------------------------------------------------------------------- cross-process desk
|
| 262 |
+
def _forward_request(receipt, connection_id):
|
| 263 |
+
"""Web side: hand an accepted Sync now to the worker through the store desk."""
|
| 264 |
+
from core import connector_sync
|
| 265 |
+
try:
|
| 266 |
+
from harness import runtime
|
| 267 |
+
rt = runtime.get_runtime(receipt["tenant"])
|
| 268 |
+
connector_sync.queue_remote_request(rt, receipt, connection_id)
|
| 269 |
+
except Exception as exc: # noqa: BLE001
|
| 270 |
+
print(f"[process-role] sync request hand-off failed ({type(exc).__name__})", flush=True)
|
| 271 |
+
connector_sync.fail_remote_request(
|
| 272 |
+
receipt["provider"], receipt["tenant"],
|
| 273 |
+
"The sync request could not be passed to the sync worker. Try again.")
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def _tenants():
|
| 277 |
+
import tenant_scan
|
| 278 |
+
from harness import runtime
|
| 279 |
+
return [(slug, rt) for slug, rt in tenant_scan.live_tenants(runtime) if rt is not None]
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def poll_worker_desk():
|
| 283 |
+
"""Worker side: pick up event receipts and Sync now hand-offs from the store desk."""
|
| 284 |
+
from core import connector_sync
|
| 285 |
+
for slug, rt in _tenants():
|
| 286 |
+
try:
|
| 287 |
+
connector_sync.restore_sync_requests(rt)
|
| 288 |
+
admitted = connector_sync.poll_remote_requests(rt)
|
| 289 |
+
except Exception as exc: # noqa: BLE001
|
| 290 |
+
print(f"[process-role] sync desk poll refused for {slug} ({type(exc).__name__})", flush=True)
|
| 291 |
+
continue
|
| 292 |
+
for entry in admitted:
|
| 293 |
+
print(f"[process-role] sync request {entry['provider']} for {slug} admitted from the web", flush=True)
|
| 294 |
+
import routes_connectors
|
| 295 |
+
key = (entry["provider"], slug)
|
| 296 |
+
with routes_connectors._SYNC_REQUEST_CONNECTION_LOCK:
|
| 297 |
+
# No connection id means the default connection, never the previous request's.
|
| 298 |
+
if entry.get("connectionId"):
|
| 299 |
+
routes_connectors._SYNC_REQUEST_CONNECTIONS[key] = entry["connectionId"]
|
| 300 |
+
else:
|
| 301 |
+
routes_connectors._SYNC_REQUEST_CONNECTIONS.pop(key, None)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def _worker_desk_loop():
|
| 305 |
+
while not _STOP.wait(POLL_SECONDS):
|
| 306 |
+
try:
|
| 307 |
+
poll_worker_desk()
|
| 308 |
+
except Exception as exc: # noqa: BLE001
|
| 309 |
+
print(f"[process-role] sync desk poll failed ({type(exc).__name__})", flush=True)
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def _apply_worker_state():
|
| 313 |
+
"""Web side: mirror the worker's request progress and catch-up state into this process."""
|
| 314 |
+
from core import connector_sync
|
| 315 |
+
from harness import mirror_store
|
| 316 |
+
beat = _read_json(handoff_dir() / HEARTBEAT_FILE) or {}
|
| 317 |
+
catch_up = beat.get("catchUp") if isinstance(beat.get("catchUp"), dict) else {}
|
| 318 |
+
resync = beat.get("resync")
|
| 319 |
+
app_main = _running_main()
|
| 320 |
+
if isinstance(resync, dict) and app_main is not None:
|
| 321 |
+
# The status routes read RESYNC_STATE here. Restore errors stay this process's own.
|
| 322 |
+
app_main.RESYNC_STATE.update({k: v for k, v in resync.items() if "estore" not in k})
|
| 323 |
+
for slug, rt in _tenants():
|
| 324 |
+
try:
|
| 325 |
+
connector_sync.apply_remote_progress(rt)
|
| 326 |
+
except Exception as exc: # noqa: BLE001
|
| 327 |
+
print(f"[process-role] sync progress read refused for {slug} ({type(exc).__name__})", flush=True)
|
| 328 |
+
if slug in catch_up:
|
| 329 |
+
mirror_store.apply_catch_up(rt, catch_up[slug])
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
def _web_watch_loop():
|
| 333 |
+
while not _STOP.wait(POLL_SECONDS):
|
| 334 |
+
try:
|
| 335 |
+
install_ready()
|
| 336 |
+
except Exception as exc: # noqa: BLE001
|
| 337 |
+
print(f"[process-role] hand-off scan failed ({type(exc).__name__}: {exc})", flush=True)
|
| 338 |
+
try:
|
| 339 |
+
_apply_worker_state()
|
| 340 |
+
except Exception as exc: # noqa: BLE001
|
| 341 |
+
print(f"[process-role] worker state read failed ({type(exc).__name__})", flush=True)
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
# ----------------------------------------------------------------------------- the worker process
|
| 345 |
+
def _running_main():
|
| 346 |
+
return sys.modules.get("api.main") or sys.modules.get("main")
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def write_heartbeat(now=None):
|
| 350 |
+
try:
|
| 351 |
+
from harness import mirror_store
|
| 352 |
+
catch_up = mirror_store.catch_up_snapshot()
|
| 353 |
+
except Exception: # noqa: BLE001
|
| 354 |
+
catch_up = {}
|
| 355 |
+
app_main = _running_main()
|
| 356 |
+
resync = dict(getattr(app_main, "RESYNC_STATE", None) or {})
|
| 357 |
+
_write_json(handoff_dir() / HEARTBEAT_FILE,
|
| 358 |
+
{"pid": os.getpid(), "at": time.time() if now is None else now,
|
| 359 |
+
"catchUp": catch_up, "resync": resync})
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def _heartbeat_loop(parent_pid):
|
| 363 |
+
while True:
|
| 364 |
+
try:
|
| 365 |
+
write_heartbeat()
|
| 366 |
+
except Exception as exc: # noqa: BLE001
|
| 367 |
+
print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True)
|
| 368 |
+
if parent_pid and os.getppid() != parent_pid:
|
| 369 |
+
# An orphaned worker must not keep publishing after its web process is gone.
|
| 370 |
+
print("[sync-worker] parent process gone; exiting", flush=True)
|
| 371 |
+
os._exit(0)
|
| 372 |
+
if _STOP.wait(HEARTBEAT_SECONDS):
|
| 373 |
+
return
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
def start_worker_services():
|
| 377 |
+
"""Called by sync_worker.py: heartbeat, hand-off writer and the store desk poll."""
|
| 378 |
+
from harness import mirror_store
|
| 379 |
+
mirror_store.register_publish_listener(write_handoff)
|
| 380 |
+
parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
|
| 381 |
+
start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
|
| 382 |
+
start("sync-desk-poll", "heavy", _worker_desk_loop)
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
def _worker_env():
|
| 386 |
+
env = dict(os.environ)
|
| 387 |
+
env.update(AIOS_PROCESS_ROLE="worker", AIOS_PARENT_PID=str(os.getpid()),
|
| 388 |
+
AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
|
| 389 |
+
for name in ("AIOS_DUCKDB_MEMORY", "AIOS_DUCKDB_THREADS"):
|
| 390 |
+
env.pop(name, None) # the worker sets its own caps
|
| 391 |
+
return env
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def _supervise():
|
| 395 |
+
backoff = 5.0
|
| 396 |
+
while not _STOP.is_set():
|
| 397 |
+
started = time.time()
|
| 398 |
+
try:
|
| 399 |
+
proc = subprocess.Popen([sys.executable, "sync_worker.py"], cwd=str(API_DIR), env=_worker_env())
|
| 400 |
+
except OSError as exc:
|
| 401 |
+
print(f"[process-role] sync worker could not start ({type(exc).__name__}: {exc})", flush=True)
|
| 402 |
+
proc = None
|
| 403 |
+
if proc is not None:
|
| 404 |
+
_WORKER.update(proc=proc, pid=proc.pid, startedAt=started)
|
| 405 |
+
print(f"[process-role] sync worker started pid={proc.pid}", flush=True)
|
| 406 |
+
code = proc.wait()
|
| 407 |
+
_WORKER["lastExit"] = code
|
| 408 |
+
else:
|
| 409 |
+
code = None
|
| 410 |
+
if _STOP.is_set():
|
| 411 |
+
return
|
| 412 |
+
if time.time() - started > 600:
|
| 413 |
+
backoff = 5.0
|
| 414 |
+
_WORKER["restarts"] += 1
|
| 415 |
+
print(f"[process-role] sync worker exited code={code}; restart {_WORKER['restarts']} "
|
| 416 |
+
f"in {int(backoff)}s", flush=True)
|
| 417 |
+
if _STOP.wait(backoff):
|
| 418 |
+
return
|
| 419 |
+
backoff = min(backoff * 2, 300.0)
|
| 420 |
+
|
| 421 |
+
|
| 422 |
+
def stop_worker():
|
| 423 |
+
"""Stop the supervised worker by its own pid."""
|
| 424 |
+
_STOP.set()
|
| 425 |
+
proc = _WORKER.get("proc")
|
| 426 |
+
if proc is not None and proc.poll() is None:
|
| 427 |
+
proc.terminate()
|
| 428 |
+
try:
|
| 429 |
+
proc.wait(10)
|
| 430 |
+
except subprocess.TimeoutExpired:
|
| 431 |
+
proc.kill()
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
def start_web_services():
|
| 435 |
+
"""Web role only: forward Sync now, supervise the worker, install its snapshots."""
|
| 436 |
+
if ROLE != "web" or _WORKER.get("supervisor") is not None:
|
| 437 |
+
return False
|
| 438 |
+
from core import connector_sync
|
| 439 |
+
connector_sync.register_request_listener(_forward_request)
|
| 440 |
+
_WORKER["supervisor"] = start("sync-worker-supervisor", "light", _supervise)
|
| 441 |
+
start("mirror-handoff-watcher", "light", _web_watch_loop)
|
| 442 |
+
atexit.register(stop_worker)
|
| 443 |
+
return True
|
| 444 |
+
|
| 445 |
+
|
| 446 |
+
def status():
|
| 447 |
+
"""{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response."""
|
| 448 |
+
beat = _read_json(handoff_dir() / HEARTBEAT_FILE) if ROLE != "all" else None
|
| 449 |
+
if ROLE == "web":
|
| 450 |
+
proc = _WORKER.get("proc")
|
| 451 |
+
pid = _WORKER.get("pid")
|
| 452 |
+
alive = proc is not None and proc.poll() is None
|
| 453 |
+
elif ROLE == "worker":
|
| 454 |
+
pid, alive = os.getpid(), True
|
| 455 |
+
else:
|
| 456 |
+
pid, alive = None, False
|
| 457 |
+
age = None
|
| 458 |
+
if isinstance(beat, dict) and beat.get("pid") == pid and beat.get("at"):
|
| 459 |
+
age = round(max(0.0, time.time() - float(beat["at"])), 1)
|
| 460 |
+
return {"role": ROLE, "roleSource": ROLE_SOURCE,
|
| 461 |
+
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
| 462 |
+
"restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
|
| 463 |
+
"lastInstallError": dict(LAST_INSTALL_ERROR) or None}}
|
api/routes_admin.py
CHANGED
|
@@ -2804,7 +2804,16 @@ def settings(session: Session = Depends(require_session)):
|
|
| 2804 |
"""
|
| 2805 |
modules = perms.allowed_modules(session.user)
|
| 2806 |
from core import freeze_notice
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2807 |
return {
|
|
|
|
|
|
|
|
|
|
| 2808 |
"freezeNotice": freeze_notice.notice(),
|
| 2809 |
# The build this Space is running. Behind the session on purpose — see `main.py::health`,
|
| 2810 |
# which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a
|
|
|
|
| 2804 |
"""
|
| 2805 |
modules = perms.allowed_modules(session.user)
|
| 2806 |
from core import freeze_notice
|
| 2807 |
+
import process_role
|
| 2808 |
+
roles = process_role.status()
|
| 2809 |
+
worker = roles["worker"]
|
| 2810 |
+
if not perms.is_admin(session.user):
|
| 2811 |
+
# A member sees liveness only, never a process id.
|
| 2812 |
+
worker = {key: worker[key] for key in ("alive", "heartbeatAgeS", "restarts")}
|
| 2813 |
return {
|
| 2814 |
+
# Which process role answered, and whether its sync worker is alive and beating.
|
| 2815 |
+
"processRole": roles["role"],
|
| 2816 |
+
"worker": worker,
|
| 2817 |
"freezeNotice": freeze_notice.notice(),
|
| 2818 |
# The build this Space is running. Behind the session on purpose — see `main.py::health`,
|
| 2819 |
# which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a
|
api/routes_connectors.py
CHANGED
|
@@ -2676,9 +2676,12 @@ def _start_database_jobs():
|
|
| 2676 |
_materialization_wake.set()
|
| 2677 |
if _materialization_thread and _materialization_thread.is_alive():
|
| 2678 |
return
|
| 2679 |
-
|
| 2680 |
-
|
| 2681 |
-
|
|
|
|
|
|
|
|
|
|
| 2682 |
|
| 2683 |
|
| 2684 |
@router.on_event("shutdown")
|
|
@@ -2988,7 +2991,8 @@ def connector_sync_now(connector_key: str, connection_id: str = None,
|
|
| 2988 |
# Bind under one API-side lock so a duplicate request cannot race ahead of the
|
| 2989 |
# accepted request and attach its different connection id to the queued intent.
|
| 2990 |
receipt = connector_sync.request_sync(connector_key, session.runtime.key,
|
| 2991 |
-
requested_by=session.uname
|
|
|
|
| 2992 |
if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS:
|
| 2993 |
_SYNC_REQUEST_CONNECTIONS[key] = requested_connection
|
| 2994 |
receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key]
|
|
|
|
| 2676 |
_materialization_wake.set()
|
| 2677 |
if _materialization_thread and _materialization_thread.is_alive():
|
| 2678 |
return
|
| 2679 |
+
import process_role
|
| 2680 |
+
job = process_role.thread(target=run, daemon=True, kind="heavy",
|
| 2681 |
+
name="connector-materialization")
|
| 2682 |
+
if process_role.admitted(job):
|
| 2683 |
+
_materialization_thread = job
|
| 2684 |
+
_materialization_thread.start()
|
| 2685 |
|
| 2686 |
|
| 2687 |
@router.on_event("shutdown")
|
|
|
|
| 2991 |
# Bind under one API-side lock so a duplicate request cannot race ahead of the
|
| 2992 |
# accepted request and attach its different connection id to the queued intent.
|
| 2993 |
receipt = connector_sync.request_sync(connector_key, session.runtime.key,
|
| 2994 |
+
requested_by=session.uname,
|
| 2995 |
+
connection_id=requested_connection)
|
| 2996 |
if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS:
|
| 2997 |
_SYNC_REQUEST_CONNECTIONS[key] = requested_connection
|
| 2998 |
receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key]
|
api/routes_platform_admin.py
CHANGED
|
@@ -51,8 +51,12 @@ from deps import Session, err, require_session, users
|
|
| 51 |
|
| 52 |
@asynccontextmanager
|
| 53 |
async def _neon_lifespan(app):
|
| 54 |
-
|
| 55 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
try:
|
| 57 |
yield
|
| 58 |
finally:
|
|
|
|
| 51 |
|
| 52 |
@asynccontextmanager
|
| 53 |
async def _neon_lifespan(app):
|
| 54 |
+
import process_role
|
| 55 |
+
# Light: each sampler makes one billing API read per interval for this process's card.
|
| 56 |
+
if process_role.admit("neon-usage-sampler", "light"):
|
| 57 |
+
_start_neon_sampler()
|
| 58 |
+
if process_role.admit("do-usage-sampler", "light"):
|
| 59 |
+
_start_do_sampler()
|
| 60 |
try:
|
| 61 |
yield
|
| 62 |
finally:
|
api/routes_sheet.py
CHANGED
|
@@ -1745,6 +1745,51 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str:
|
|
| 1745 |
ONE function that turns those pieces into a key, so a future change to the wall's shape cannot
|
| 1746 |
make the two drift silently apart.
|
| 1747 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1748 |
from core import sheets
|
| 1749 |
import core.perm_scope as perm_scope
|
| 1750 |
_refuse_unapplied_audience(session, database)
|
|
@@ -1779,8 +1824,8 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str:
|
|
| 1779 |
policy = {key: value for key, value in session.user.items()
|
| 1780 |
if key not in ("username", "name", "email", "password", "password_hash", "epoch",
|
| 1781 |
"created_at", "updated_at", "last_login")}
|
| 1782 |
-
return
|
| 1783 |
-
|
| 1784 |
|
| 1785 |
|
| 1786 |
def _has_ending(binding):
|
|
@@ -2078,13 +2123,21 @@ def _reresolved_for_reader(session: Session, book, *, endings_only=False, recalc
|
|
| 2078 |
"cell": legacy.get("snapshot") or {"value": value, "state": "fresh",
|
| 2079 |
"display": "" if value is None else str(value),
|
| 2080 |
"refreshed_at": legacy.get("refreshedAt")}}
|
|
|
|
|
|
|
|
|
|
| 2081 |
stored_answer = False
|
| 2082 |
if field_refusal:
|
| 2083 |
_snapshot_cell(cell, _binding_refusal(field_refusal["code"], field_refusal["message"]))
|
| 2084 |
elif seen[db] and _binding_catching_up(session, binding):
|
| 2085 |
-
|
| 2086 |
-
|
| 2087 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2088 |
elif hit:
|
| 2089 |
_snapshot_cell(cell, _public_refresh_snapshot(hit["cell"]))
|
| 2090 |
stored_answer = True
|
|
@@ -2101,7 +2154,8 @@ def _reresolved_for_reader(session: Session, book, *, endings_only=False, recalc
|
|
| 2101 |
# not what that protected: an admin has no row wall (`row_scope_applies`), and
|
| 2102 |
# the stored figure is the owner's own document's last computed answer. A `stale`
|
| 2103 |
# cell is excluded because it has no answer to keep. Non-owners are unchanged.
|
| 2104 |
-
cell.
|
|
|
|
| 2105 |
stored_answer = True
|
| 2106 |
else:
|
| 2107 |
_snapshot_cell(cell, _binding_refusal("sheet_refresh_required",
|
|
@@ -3716,8 +3770,10 @@ def _refresh_scope_cells(book, scope):
|
|
| 3716 |
|
| 3717 |
|
| 3718 |
def _public_refresh_snapshot(snapshot):
|
| 3719 |
-
"""
|
| 3720 |
if snapshot.get("state") == "refused":
|
|
|
|
|
|
|
| 3721 |
refusal = snapshot.get("refusal") or {}
|
| 3722 |
return {**snapshot, **_binding_refusal(refusal.get("code", "sheet_unresolved"),
|
| 3723 |
refusal.get("message", "Refresh was held."))}
|
|
@@ -3832,6 +3888,7 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
|
|
| 3832 |
started = perf_counter()
|
| 3833 |
resolved_count = 0
|
| 3834 |
held_refusal = None
|
|
|
|
| 3835 |
denied_aggregates = set()
|
| 3836 |
held_sources, entry_sources = set(), {}
|
| 3837 |
scope_names = {id(tab): name for tab, name in
|
|
@@ -3901,17 +3958,29 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
|
|
| 3901 |
"refreshed_at": stamp}
|
| 3902 |
if limits:
|
| 3903 |
snapshot["limits"] = limits
|
| 3904 |
-
except
|
| 3905 |
-
|
| 3906 |
-
|
| 3907 |
-
|
| 3908 |
-
|
| 3909 |
-
info
|
| 3910 |
-
|
| 3911 |
-
|
| 3912 |
-
|
| 3913 |
-
|
| 3914 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3915 |
held_refusal = held_refusal or snapshot["refusal"]
|
| 3916 |
held_sources.update({reading_source} if reading_source else
|
| 3917 |
{key for key in entry_sources[cell_key] if _database_connected(session, key)} or {db})
|
|
@@ -3919,7 +3988,8 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
|
|
| 3919 |
snapshot["stamp"] = _binding_answer_stamp(session, book, binding, tab.get("id"), fr)
|
| 3920 |
_snapshot_cell(cell, snapshot)
|
| 3921 |
entries.append((cell_key, scope_key, {"tokens": source_tokens,
|
| 3922 |
-
"context": context, "cell": deepcopy(snapshot)
|
|
|
|
| 3923 |
|
| 3924 |
capture.confirm([entry for _, _, entry in entries])
|
| 3925 |
for cell_key, _, entry in entries:
|
|
@@ -3934,8 +4004,8 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
|
|
| 3934 |
if held:
|
| 3935 |
# Retain every old snapshot, but refuse only failed sources and their dependents.
|
| 3936 |
entries.extend(skipped_entries)
|
| 3937 |
-
held_addresses = {key for key, _, entry in entries
|
| 3938 |
-
|
| 3939 |
graph = _read_graph(session, original, fr, None) or _new_read_graph(original, fr)
|
| 3940 |
# Discover dynamic dependents from accepted values without resolving any database.
|
| 3941 |
engine = _read_engine(graph, original, fr, None)[0]
|
|
@@ -3950,16 +4020,26 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
|
|
| 3950 |
{key for key, address in wire.items() if address in held_addresses},
|
| 3951 |
dependent_keys=dependent_keys)
|
| 3952 |
held_addresses.update(wire[key] for key in dependent_keys if key in wire)
|
|
|
|
| 3953 |
retained_entries = []
|
| 3954 |
for key, slot, entry in entries:
|
| 3955 |
old = cache.get("refreshCells", {}).get(key, {}).get(slot, {})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3956 |
if key not in held_addresses:
|
| 3957 |
if old:
|
| 3958 |
retained_entries.append((key, slot, deepcopy(old)))
|
| 3959 |
continue
|
| 3960 |
-
|
| 3961 |
-
|
| 3962 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3963 |
entry["tokens"] = deepcopy(old.get("tokens", {}))
|
| 3964 |
entry["context"] = old.get("context")
|
| 3965 |
retained_entries.append((key, slot, entry))
|
|
@@ -4022,6 +4102,11 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
|
|
| 4022 |
refusal = entry["cell"]["refusal"]
|
| 4023 |
entry.update(tokens=deepcopy(old.get("tokens", {})), context=old.get("context"),
|
| 4024 |
cell={**deepcopy(old["cell"]), "state": "refused", "refusal": refusal})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4025 |
recalc_ms = round((perf_counter() - started) * 1000.0, 1)
|
| 4026 |
stats = _read_stats(session) or {}
|
| 4027 |
print(f"[sheet] refresh wb={workbook_id} cells={'all' if only is None else len(only)} "
|
|
@@ -4040,12 +4125,14 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
|
|
| 4040 |
if held:
|
| 4041 |
result["reason"] = held
|
| 4042 |
result["message"] = held_refusal["message"]
|
| 4043 |
-
|
| 4044 |
-
|
|
|
|
|
|
|
| 4045 |
result["stamps"] = {key: entry["cell"]["stamp"] for key, _, entry in entries
|
| 4046 |
-
if key in
|
| 4047 |
saved = sheets.finish_refresh(store, workbook_id, result, session, held=True,
|
| 4048 |
-
addresses=
|
| 4049 |
expect_rev=original.get("rev"))
|
| 4050 |
elif editable:
|
| 4051 |
try:
|
|
@@ -4906,9 +4993,63 @@ _BINDING_REFUSAL_DISPLAY = {
|
|
| 4906 |
"field_unknown": "#FIELD?",
|
| 4907 |
# HOTFIX 2026-09-24: "needs a Refresh" is not a calculation error and must not read as one.
|
| 4908 |
"sheet_refresh_required": "#REFRESH",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4909 |
}
|
| 4910 |
|
| 4911 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4912 |
def _binding_refusal(code, message):
|
| 4913 |
"""One refusal shape for compute, refresh and caller-scoped reads of metric cells."""
|
| 4914 |
return {
|
|
@@ -5658,10 +5799,25 @@ def _compute_cell_scoped(workbook_id: str, addr: str, body, session: Session):
|
|
| 5658 |
#: a cell left `stale` would shimmer forever, and the reason belongs on the cell.
|
| 5659 |
detail = getattr(exc, "detail", None) or {}
|
| 5660 |
info = detail.get("error") if isinstance(detail, dict) else None
|
| 5661 |
-
|
| 5662 |
-
|
| 5663 |
-
|
| 5664 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5665 |
else:
|
| 5666 |
limits = resolved["limits"]
|
| 5667 |
saved["state"] = "fresh"
|
|
|
|
| 1745 |
ONE function that turns those pieces into a key, so a future change to the wall's shape cannot
|
| 1746 |
make the two drift silently apart.
|
| 1747 |
"""
|
| 1748 |
+
from core.sheet_binding import fingerprint
|
| 1749 |
+
scope, policy, access, grants = _binding_scope_parts(session, database, binding)
|
| 1750 |
+
return fingerprint([scope, binding, policy, access, grants])
|
| 1751 |
+
|
| 1752 |
+
|
| 1753 |
+
#: User keys that name or decorate a person and never change what they may read.
|
| 1754 |
+
_WALL_IDENTITY_KEYS = frozenset(("avatar", "whatsapp"))
|
| 1755 |
+
|
| 1756 |
+
|
| 1757 |
+
def _binding_wall_key(session: Session, database: str, binding: dict):
|
| 1758 |
+
"""Fingerprint of the reader's permission wall alone, or None when no other reader may share it.
|
| 1759 |
+
|
| 1760 |
+
Two readers with equal wall keys read the same rows and fields, so one's stored answer is the other's.
|
| 1761 |
+
"""
|
| 1762 |
+
from core.sheet_binding import fingerprint
|
| 1763 |
+
try:
|
| 1764 |
+
# Cell inputs and related tables carry walls this key does not see.
|
| 1765 |
+
if _has_cell_inputs(binding):
|
| 1766 |
+
return None
|
| 1767 |
+
scope, policy, access, grants = _binding_scope_parts(session, database, binding)
|
| 1768 |
+
except Exception: # noqa: BLE001
|
| 1769 |
+
return None
|
| 1770 |
+
# An unprovable wall is a random string; a personal workspace makes the value caller specific.
|
| 1771 |
+
if not isinstance(access, list) or access[1] is not None:
|
| 1772 |
+
return None
|
| 1773 |
+
return fingerprint(["wall", scope, binding, access[0], grants,
|
| 1774 |
+
{key: value for key, value in policy.items() if key not in _WALL_IDENTITY_KEYS}])
|
| 1775 |
+
|
| 1776 |
+
|
| 1777 |
+
def _wall_hit(session, cache, cell_key, database, binding):
|
| 1778 |
+
"""A stored answer written for another reader whose permission wall equals this reader's."""
|
| 1779 |
+
slots = (cache.get("refreshCells") or {}).get(cell_key)
|
| 1780 |
+
if not isinstance(slots, dict) or not slots:
|
| 1781 |
+
return None
|
| 1782 |
+
wall = _binding_wall_key(session, database, binding)
|
| 1783 |
+
if wall is None:
|
| 1784 |
+
return None
|
| 1785 |
+
for entry in slots.values():
|
| 1786 |
+
if isinstance(entry, dict) and entry.get("wall") == wall and isinstance(entry.get("cell"), dict):
|
| 1787 |
+
return entry
|
| 1788 |
+
return None
|
| 1789 |
+
|
| 1790 |
+
|
| 1791 |
+
def _binding_scope_parts(session: Session, database: str, binding: dict):
|
| 1792 |
+
"""`(scope hash, policy, field access, grants)`: the pieces both cache keys fingerprint."""
|
| 1793 |
from core import sheets
|
| 1794 |
import core.perm_scope as perm_scope
|
| 1795 |
_refuse_unapplied_audience(session, database)
|
|
|
|
| 1824 |
policy = {key: value for key, value in session.user.items()
|
| 1825 |
if key not in ("username", "name", "email", "password", "password_hash", "epoch",
|
| 1826 |
"created_at", "updated_at", "last_login")}
|
| 1827 |
+
return (sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {}),
|
| 1828 |
+
policy, memo[database], memo.get("_grants"))
|
| 1829 |
|
| 1830 |
|
| 1831 |
def _has_ending(binding):
|
|
|
|
| 2123 |
"cell": legacy.get("snapshot") or {"value": value, "state": "fresh",
|
| 2124 |
"display": "" if value is None else str(value),
|
| 2125 |
"refreshed_at": legacy.get("refreshedAt")}}
|
| 2126 |
+
if not hit and seen[db]:
|
| 2127 |
+
# A reader with no slot of their own reads the answer stored for an identical wall.
|
| 2128 |
+
hit = _wall_hit(session, cache, str(tab.get("id")) + "!" + addr, db, binding)
|
| 2129 |
stored_answer = False
|
| 2130 |
if field_refusal:
|
| 2131 |
_snapshot_cell(cell, _binding_refusal(field_refusal["code"], field_refusal["message"]))
|
| 2132 |
elif seen[db] and _binding_catching_up(session, binding):
|
| 2133 |
+
kept = _public_refresh_snapshot(hit["cell"]) if hit else None
|
| 2134 |
+
if kept is not None and kept.get("state") != "refused":
|
| 2135 |
+
# An update in progress keeps the stored value, marked stale.
|
| 2136 |
+
_snapshot_cell(cell, _held_stale(kept, "store_catching_up"))
|
| 2137 |
+
else:
|
| 2138 |
+
_snapshot_cell(cell, {**(hit or {}).get("cell", {}),
|
| 2139 |
+
**_binding_refusal("store_catching_up",
|
| 2140 |
+
"This database is updating. Refresh when it finishes.")})
|
| 2141 |
elif hit:
|
| 2142 |
_snapshot_cell(cell, _public_refresh_snapshot(hit["cell"]))
|
| 2143 |
stored_answer = True
|
|
|
|
| 2154 |
# not what that protected: an admin has no row wall (`row_scope_applies`), and
|
| 2155 |
# the stored figure is the owner's own document's last computed answer. A `stale`
|
| 2156 |
# cell is excluded because it has no answer to keep. Non-owners are unchanged.
|
| 2157 |
+
# Copy first: `_snapshot_cell` clears the keys it is about to rewrite on this same cell.
|
| 2158 |
+
_snapshot_cell(cell, deepcopy(_public_refresh_snapshot(cell)))
|
| 2159 |
stored_answer = True
|
| 2160 |
else:
|
| 2161 |
_snapshot_cell(cell, _binding_refusal("sheet_refresh_required",
|
|
|
|
| 3770 |
|
| 3771 |
|
| 3772 |
def _public_refresh_snapshot(snapshot):
|
| 3773 |
+
"""An authored refusal hides a retained value. A transient one serves it as stale."""
|
| 3774 |
if snapshot.get("state") == "refused":
|
| 3775 |
+
if _held_answer(snapshot):
|
| 3776 |
+
return _held_stale(snapshot)
|
| 3777 |
refusal = snapshot.get("refusal") or {}
|
| 3778 |
return {**snapshot, **_binding_refusal(refusal.get("code", "sheet_unresolved"),
|
| 3779 |
refusal.get("message", "Refresh was held."))}
|
|
|
|
| 3888 |
started = perf_counter()
|
| 3889 |
resolved_count = 0
|
| 3890 |
held_refusal = None
|
| 3891 |
+
authored_keys = set()
|
| 3892 |
denied_aggregates = set()
|
| 3893 |
held_sources, entry_sources = set(), {}
|
| 3894 |
scope_names = {id(tab): name for tab, name in
|
|
|
|
| 3958 |
"refreshed_at": stamp}
|
| 3959 |
if limits:
|
| 3960 |
snapshot["limits"] = limits
|
| 3961 |
+
except Exception as exc: # noqa: BLE001
|
| 3962 |
+
http_status = exc.status_code if isinstance(exc, HTTPException) else None
|
| 3963 |
+
if isinstance(exc, sheet_mod.SheetRefusal):
|
| 3964 |
+
info = {"code": exc.code, "message": exc.message}
|
| 3965 |
+
elif isinstance(exc, HTTPException):
|
| 3966 |
+
info = (exc.detail if isinstance(exc.detail, dict) else {}).get("error") or {}
|
| 3967 |
+
else:
|
| 3968 |
+
# A timeout, a dropped connection or any unexpected failure is the server's, not the cell's.
|
| 3969 |
+
print(f"[sheet] refresh cell={cell_key} held {type(exc).__name__}: "
|
| 3970 |
+
f"{(str(exc).splitlines() or [''])[0][:200]}", flush=True)
|
| 3971 |
+
info = {"code": "source_unavailable",
|
| 3972 |
+
"message": "The database was busy. This cell updates after the next sync."}
|
| 3973 |
+
code = info.get("code") or "sheet_unresolved"
|
| 3974 |
+
authored = _refusal_authored(code, http_status,
|
| 3975 |
+
named=isinstance(exc, sheet_mod.SheetRefusal))
|
| 3976 |
+
if not authored and code not in _TRANSIENT_REFUSAL_CODES:
|
| 3977 |
+
code, info = "source_unavailable", {
|
| 3978 |
+
"message": "The database was busy. This cell updates after the next sync."}
|
| 3979 |
+
snapshot = _binding_refusal(code, info.get("message", "This cell could not be refreshed."))
|
| 3980 |
+
# Only an authored refusal replaces a stored value. Every other failure holds it.
|
| 3981 |
+
if authored:
|
| 3982 |
+
authored_keys.add(cell_key)
|
| 3983 |
+
else:
|
| 3984 |
held_refusal = held_refusal or snapshot["refusal"]
|
| 3985 |
held_sources.update({reading_source} if reading_source else
|
| 3986 |
{key for key in entry_sources[cell_key] if _database_connected(session, key)} or {db})
|
|
|
|
| 3988 |
snapshot["stamp"] = _binding_answer_stamp(session, book, binding, tab.get("id"), fr)
|
| 3989 |
_snapshot_cell(cell, snapshot)
|
| 3990 |
entries.append((cell_key, scope_key, {"tokens": source_tokens,
|
| 3991 |
+
"context": context, "cell": deepcopy(snapshot),
|
| 3992 |
+
"wall": _binding_wall_key(session, db, binding)}))
|
| 3993 |
|
| 3994 |
capture.confirm([entry for _, _, entry in entries])
|
| 3995 |
for cell_key, _, entry in entries:
|
|
|
|
| 4004 |
if held:
|
| 4005 |
# Retain every old snapshot, but refuse only failed sources and their dependents.
|
| 4006 |
entries.extend(skipped_entries)
|
| 4007 |
+
held_addresses = {key for key, _, entry in entries if key not in authored_keys
|
| 4008 |
+
and (entry["cell"].get("refusal") or entry_sources[key] & held_sources)}
|
| 4009 |
graph = _read_graph(session, original, fr, None) or _new_read_graph(original, fr)
|
| 4010 |
# Discover dynamic dependents from accepted values without resolving any database.
|
| 4011 |
engine = _read_engine(graph, original, fr, None)[0]
|
|
|
|
| 4020 |
{key for key, address in wire.items() if address in held_addresses},
|
| 4021 |
dependent_keys=dependent_keys)
|
| 4022 |
held_addresses.update(wire[key] for key in dependent_keys if key in wire)
|
| 4023 |
+
held_addresses -= authored_keys
|
| 4024 |
retained_entries = []
|
| 4025 |
for key, slot, entry in entries:
|
| 4026 |
old = cache.get("refreshCells", {}).get(key, {}).get(slot, {})
|
| 4027 |
+
if key in authored_keys:
|
| 4028 |
+
# An authored refusal is the answer and replaces the stored value.
|
| 4029 |
+
retained_entries.append((key, slot, entry))
|
| 4030 |
+
continue
|
| 4031 |
if key not in held_addresses:
|
| 4032 |
if old:
|
| 4033 |
retained_entries.append((key, slot, deepcopy(old)))
|
| 4034 |
continue
|
| 4035 |
+
kept = deepcopy(old.get("cell", {}))
|
| 4036 |
+
if _held_answer(kept):
|
| 4037 |
+
# A transient failure keeps the stored value, served stale, never refused.
|
| 4038 |
+
entry["cell"] = {"stamp": entry["cell"]["stamp"], **_held_stale(kept)}
|
| 4039 |
+
else:
|
| 4040 |
+
refusal = entry["cell"].get("refusal") or held_refusal
|
| 4041 |
+
entry["cell"] = {"stamp": entry["cell"]["stamp"], **kept,
|
| 4042 |
+
"state": "refused", "refusal": deepcopy(refusal)}
|
| 4043 |
entry["tokens"] = deepcopy(old.get("tokens", {}))
|
| 4044 |
entry["context"] = old.get("context")
|
| 4045 |
retained_entries.append((key, slot, entry))
|
|
|
|
| 4102 |
refusal = entry["cell"]["refusal"]
|
| 4103 |
entry.update(tokens=deepcopy(old.get("tokens", {})), context=old.get("context"),
|
| 4104 |
cell={**deepcopy(old["cell"]), "state": "refused", "refusal": refusal})
|
| 4105 |
+
# The private snapshot keeps the wall it was computed under.
|
| 4106 |
+
if "wall" in old:
|
| 4107 |
+
entry["wall"] = old["wall"]
|
| 4108 |
+
else:
|
| 4109 |
+
entry.pop("wall", None)
|
| 4110 |
recalc_ms = round((perf_counter() - started) * 1000.0, 1)
|
| 4111 |
stats = _read_stats(session) or {}
|
| 4112 |
print(f"[sheet] refresh wb={workbook_id} cells={'all' if only is None else len(only)} "
|
|
|
|
| 4125 |
if held:
|
| 4126 |
result["reason"] = held
|
| 4127 |
result["message"] = held_refusal["message"]
|
| 4128 |
+
# The document marks held cells with the cause. A GET serves a kept value stale.
|
| 4129 |
+
marked = held_addresses | authored_keys
|
| 4130 |
+
result["refusals"] = {key: entry["cell"].get("refusal") or held_refusal
|
| 4131 |
+
for key, _, entry in entries if key in marked}
|
| 4132 |
result["stamps"] = {key: entry["cell"]["stamp"] for key, _, entry in entries
|
| 4133 |
+
if key in marked}
|
| 4134 |
saved = sheets.finish_refresh(store, workbook_id, result, session, held=True,
|
| 4135 |
+
addresses=marked.intersection(addresses) if editable else (),
|
| 4136 |
expect_rev=original.get("rev"))
|
| 4137 |
elif editable:
|
| 4138 |
try:
|
|
|
|
| 4993 |
"field_unknown": "#FIELD?",
|
| 4994 |
# HOTFIX 2026-09-24: "needs a Refresh" is not a calculation error and must not read as one.
|
| 4995 |
"sheet_refresh_required": "#REFRESH",
|
| 4996 |
+
# A busy source is not a calculation error either.
|
| 4997 |
+
"source_unavailable": "#UPDATING",
|
| 4998 |
+
"store_not_ready": "#UPDATING",
|
| 4999 |
+
"mirror_not_ready": "#UPDATING",
|
| 5000 |
+
"amazon_mirror_not_loaded": "#UPDATING",
|
| 5001 |
}
|
| 5002 |
|
| 5003 |
|
| 5004 |
+
#: Failures of the source or the server, never of the binding. They keep a stored value.
|
| 5005 |
+
_TRANSIENT_REFUSAL_CODES = frozenset((
|
| 5006 |
+
"store_catching_up", "source_unavailable", "store_not_ready", "mirror_not_ready",
|
| 5007 |
+
"mirror_mismatch", "amazon_mirror_not_loaded", "sheet_unresolved", "change_token_unavailable",
|
| 5008 |
+
"sheet_refresh_required", "schema_not_loaded", "grants_changed", "relation_cursor_mismatch",
|
| 5009 |
+
))
|
| 5010 |
+
|
| 5011 |
+
#: The binding itself is invalid for this reader: a field or database is gone, or access is denied.
|
| 5012 |
+
_AUTHORED_REFUSAL_CODES = frozenset((
|
| 5013 |
+
"field_unreadable", "database_unreadable", "field_unknown", "related_field_unknown",
|
| 5014 |
+
"aggregate_unknown", "unknown_relation", "binding_invalid", "related_invalid", "ending_invalid",
|
| 5015 |
+
"ending_not_supported", "period_invalid", "period_window_invalid", "metric_not_additive",
|
| 5016 |
+
"metric_period_required", "pivot_field_unsupported", "pivot_too_large", "view_missing", "scope_not_applicable", "not_read_through", "folder_missing",
|
| 5017 |
+
"sheet_database_required", "sheet_bindings_required",
|
| 5018 |
+
"database_forbidden", "field_forbidden", "no_access", "permission_denied",
|
| 5019 |
+
))
|
| 5020 |
+
|
| 5021 |
+
|
| 5022 |
+
def _refusal_authored(code, status=None, named=False):
|
| 5023 |
+
"""True only when the binding is invalid. Every other failure is transient and keeps a stored value.
|
| 5024 |
+
|
| 5025 |
+
`named` marks a binding module refusal, which is authored unless its code names a busy source.
|
| 5026 |
+
"""
|
| 5027 |
+
if code in _TRANSIENT_REFUSAL_CODES:
|
| 5028 |
+
return False
|
| 5029 |
+
if code in _AUTHORED_REFUSAL_CODES or named:
|
| 5030 |
+
return True
|
| 5031 |
+
# Any other client error names the request; throttling and every server error are transient.
|
| 5032 |
+
return status is not None and 400 <= status < 500 and status not in (408, 423, 425, 429)
|
| 5033 |
+
|
| 5034 |
+
|
| 5035 |
+
def _held_answer(cell):
|
| 5036 |
+
"""True when a stored snapshot carries a computed value that a transient failure must not replace."""
|
| 5037 |
+
if not isinstance(cell, dict) or "value" not in cell:
|
| 5038 |
+
return False
|
| 5039 |
+
state = cell.get("state")
|
| 5040 |
+
if state in ("fresh", "stale"):
|
| 5041 |
+
return cell.get("value") is not None or bool(cell.get("refreshed_at"))
|
| 5042 |
+
# A held snapshot written before transient failures were stored `stale` kept its value refused.
|
| 5043 |
+
return (state == "refused" and bool(cell.get("refreshed_at"))
|
| 5044 |
+
and (cell.get("refusal") or {}).get("code") in _TRANSIENT_REFUSAL_CODES)
|
| 5045 |
+
|
| 5046 |
+
|
| 5047 |
+
def _held_stale(cell, reason="refresh_held"):
|
| 5048 |
+
"""The stored value served as a stale answer, never as a refusal."""
|
| 5049 |
+
kept = {key: value for key, value in cell.items() if key != "refusal"}
|
| 5050 |
+
return {**kept, "state": "stale", "staleReason": reason}
|
| 5051 |
+
|
| 5052 |
+
|
| 5053 |
def _binding_refusal(code, message):
|
| 5054 |
"""One refusal shape for compute, refresh and caller-scoped reads of metric cells."""
|
| 5055 |
return {
|
|
|
|
| 5799 |
#: a cell left `stale` would shimmer forever, and the reason belongs on the cell.
|
| 5800 |
detail = getattr(exc, "detail", None) or {}
|
| 5801 |
info = detail.get("error") if isinstance(detail, dict) else None
|
| 5802 |
+
code = (info or {}).get("code", "sheet_unresolved")
|
| 5803 |
+
kept = None
|
| 5804 |
+
if not _refusal_authored(code, exc.status_code):
|
| 5805 |
+
# A transient failure keeps this caller's own stored value, served stale.
|
| 5806 |
+
try:
|
| 5807 |
+
from core.sheet_binding import read_refresh_cache
|
| 5808 |
+
own = ((read_refresh_cache(_store(session), workbook_id).get("refreshCells") or {})
|
| 5809 |
+
.get(tab_ref + "!" + key, {})
|
| 5810 |
+
.get(_binding_scope_key(session, str(binding.get("database") or ""), binding)))
|
| 5811 |
+
except Exception: # noqa: BLE001
|
| 5812 |
+
own = None
|
| 5813 |
+
if isinstance(own, dict) and _held_answer(own.get("cell")):
|
| 5814 |
+
kept = _held_stale(own["cell"])
|
| 5815 |
+
if kept is not None:
|
| 5816 |
+
saved.pop("refusal", None)
|
| 5817 |
+
saved.update(kept)
|
| 5818 |
+
else:
|
| 5819 |
+
saved.update(_binding_refusal(
|
| 5820 |
+
code, (info or {}).get("message", "This cell could not be refreshed.")))
|
| 5821 |
else:
|
| 5822 |
limits = resolved["limits"]
|
| 5823 |
saved["state"] = "fresh"
|
api/sheet_warm.py
CHANGED
|
@@ -104,15 +104,17 @@ def bound_cells(book) -> set:
|
|
| 104 |
|
| 105 |
|
| 106 |
def cells_needing_warm(view) -> set:
|
| 107 |
-
"""Bound cells a reader's projection marks stale (any reason) or
|
| 108 |
out = set()
|
| 109 |
for tab in (view or {}).get("sheets") or []:
|
| 110 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 111 |
if not _is_bound(cell):
|
| 112 |
continue
|
| 113 |
state = cell.get("state")
|
| 114 |
-
|
| 115 |
-
|
|
|
|
|
|
|
| 116 |
out.add(f"{tab.get('id')}!{addr}")
|
| 117 |
return out
|
| 118 |
|
|
@@ -337,7 +339,10 @@ def start():
|
|
| 337 |
thread = _STATE["thread"]
|
| 338 |
if thread is not None and thread.is_alive():
|
| 339 |
return thread
|
| 340 |
-
|
|
|
|
|
|
|
|
|
|
| 341 |
_STATE["thread"] = thread
|
| 342 |
atexit.register(_on_exit)
|
| 343 |
thread.start()
|
|
|
|
| 104 |
|
| 105 |
|
| 106 |
def cells_needing_warm(view) -> set:
|
| 107 |
+
"""Bound cells a reader's projection marks stale (any reason) or refused for a transient cause."""
|
| 108 |
out = set()
|
| 109 |
for tab in (view or {}).get("sheets") or []:
|
| 110 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 111 |
if not _is_bound(cell):
|
| 112 |
continue
|
| 113 |
state = cell.get("state")
|
| 114 |
+
refusal = cell.get("refusal") or {}
|
| 115 |
+
# A busy-server refusal stored before the held rule still needs a retry; an authored one does not.
|
| 116 |
+
if state == "stale" or (state == "refused" and not routes_sheet._refusal_authored(
|
| 117 |
+
refusal.get("code"), refusal.get("status"))):
|
| 118 |
out.add(f"{tab.get('id')}!{addr}")
|
| 119 |
return out
|
| 120 |
|
|
|
|
| 339 |
thread = _STATE["thread"]
|
| 340 |
if thread is not None and thread.is_alive():
|
| 341 |
return thread
|
| 342 |
+
import process_role
|
| 343 |
+
thread = process_role.thread(target=_run, daemon=True, name="sheet-warm", kind="heavy")
|
| 344 |
+
if not process_role.admitted(thread):
|
| 345 |
+
return None
|
| 346 |
_STATE["thread"] = thread
|
| 347 |
atexit.register(_on_exit)
|
| 348 |
thread.start()
|
api/sync_worker.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The sync worker process: runs every heavy background job and never serves HTTP.
|
| 2 |
+
|
| 3 |
+
Run as `python sync_worker.py` from aios-web/api. The web process spawns and supervises it.
|
| 4 |
+
It keeps its own DuckDB files, so it never opens the web process's royal.duckdb.
|
| 5 |
+
"""
|
| 6 |
+
import os
|
| 7 |
+
import sys
|
| 8 |
+
import threading
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
API_DIR = Path(__file__).resolve().parent
|
| 12 |
+
_RI = Path(os.environ.get("RI_DIR") or (API_DIR.parents[1] / "platform"))
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def configure(env=None):
|
| 16 |
+
"""Set the worker role, its own DuckDB files and its DuckDB caps. Returns the worker DuckDB path."""
|
| 17 |
+
env = os.environ if env is None else env
|
| 18 |
+
env["AIOS_PROCESS_ROLE"] = "worker"
|
| 19 |
+
web_store = Path(env.get("AIOS_DUCKDB_STORE_DIR") or (_RI / "data" / "store"))
|
| 20 |
+
web_db = Path(env.get("AIOS_DUCKDB_PATH") or (web_store / "royal.duckdb"))
|
| 21 |
+
worker_dir = Path(env.get("AIOS_WORKER_STORE_DIR") or (web_db.parent / "worker"))
|
| 22 |
+
worker_db = worker_dir / web_db.name
|
| 23 |
+
env["AIOS_DUCKDB_STORE_DIR"] = str(worker_dir)
|
| 24 |
+
env["AIOS_DUCKDB_PATH"] = str(worker_db)
|
| 25 |
+
env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "4GB"
|
| 26 |
+
env["AIOS_DUCKDB_THREADS"] = env.get("AIOS_WORKER_DUCKDB_THREADS") or "1"
|
| 27 |
+
return worker_db
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def lower_priority(env=None):
|
| 31 |
+
"""Drop this process to a low CPU priority, so the web process wins every contested core."""
|
| 32 |
+
env = os.environ if env is None else env
|
| 33 |
+
step = int(env.get("AIOS_WORKER_NICE") or 15)
|
| 34 |
+
if step and hasattr(os, "nice"):
|
| 35 |
+
try:
|
| 36 |
+
return os.nice(step)
|
| 37 |
+
except OSError:
|
| 38 |
+
return None
|
| 39 |
+
return None
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def main():
|
| 43 |
+
worker_db = configure()
|
| 44 |
+
worker_db.parent.mkdir(parents=True, exist_ok=True)
|
| 45 |
+
niceness = lower_priority()
|
| 46 |
+
for path in (str(_RI), str(API_DIR)):
|
| 47 |
+
if path not in sys.path:
|
| 48 |
+
sys.path.insert(0, path)
|
| 49 |
+
print(f"[sync-worker] pid={os.getpid()} duckdb={worker_db} "
|
| 50 |
+
f"memory_limit={os.environ['AIOS_DUCKDB_MEMORY']} threads={os.environ['AIOS_DUCKDB_THREADS']} "
|
| 51 |
+
f"nice={niceness}",
|
| 52 |
+
flush=True)
|
| 53 |
+
print("[sync-worker] amazon_store.sync excludes brand analytics reports (memory); "
|
| 54 |
+
"they stay out of this worker too", flush=True)
|
| 55 |
+
from dotenv import load_dotenv
|
| 56 |
+
load_dotenv(_RI / ".env")
|
| 57 |
+
import process_role
|
| 58 |
+
process_role.start_worker_services()
|
| 59 |
+
# Importing the app starts every role-allowed thread at module level. Startup hooks need a call.
|
| 60 |
+
import main as app_main # noqa: F401
|
| 61 |
+
import routes_connectors
|
| 62 |
+
routes_connectors._start_database_jobs()
|
| 63 |
+
print(f"[sync-worker] running: {', '.join(sorted(process_role.started_jobs()))}", flush=True)
|
| 64 |
+
stop = threading.Event()
|
| 65 |
+
while not stop.wait(3600):
|
| 66 |
+
pass
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
if __name__ == "__main__":
|
| 70 |
+
main()
|
platform/core/connector_sync.py
CHANGED
|
@@ -32,6 +32,11 @@ _SYNC_WAKE = threading.Event()
|
|
| 32 |
_SYNC_DUE_TIMER = None
|
| 33 |
SYNC_REQUESTS_KEY = "connector_sync_requests_v1"
|
| 34 |
NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
_EVENT_DEDUPE_SECONDS = 600
|
| 36 |
|
| 37 |
|
|
@@ -52,7 +57,7 @@ def _sync_request_document(current):
|
|
| 52 |
document = deepcopy(current) if current is not None else {}
|
| 53 |
if not isinstance(document, dict):
|
| 54 |
raise ValueError("The stored sync request desk is unavailable.")
|
| 55 |
-
for member in ("requests", "events"):
|
| 56 |
document.setdefault(member, {})
|
| 57 |
if not isinstance(document[member], dict):
|
| 58 |
raise ValueError("The stored sync request desk is unavailable.")
|
|
@@ -81,13 +86,16 @@ def _wake_sync_requests(stamp):
|
|
| 81 |
_SYNC_WAKE.set()
|
| 82 |
|
| 83 |
|
| 84 |
-
def restore_sync_requests(rt, *, now=None):
|
| 85 |
-
"""Restore durable events and requeue claims abandoned by a previous process.
|
|
|
|
|
|
|
|
|
|
| 86 |
stamp = time.time() if now is None else now
|
| 87 |
with _SYNC_REQUEST_LOCK:
|
| 88 |
document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
|
| 89 |
abandoned = {identity for identity, row in document["requests"].items()
|
| 90 |
-
if row.get("state") == "running" and
|
| 91 |
_SYNC_REQUESTS.get((row.get("provider"), rt.key, identity), {}).get("state") != "running"}
|
| 92 |
if abandoned:
|
| 93 |
def restore(current):
|
|
@@ -141,9 +149,25 @@ def defer_sync(provider, tenant, not_before):
|
|
| 141 |
_SYNC_WAKE.set()
|
| 142 |
|
| 143 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
def request_sync(provider, tenant, *, requested_by="", now=None, not_before=None,
|
| 145 |
-
rt=None, tables=None, event_id=""):
|
| 146 |
"""Queue ordinary work or durably acknowledge one tenant-bound Amazon event."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
if provider not in _PROVIDERS:
|
| 148 |
raise ValueError("This connector is not registered for sync.")
|
| 149 |
tenant = str(tenant or "").strip()
|
|
@@ -306,6 +330,124 @@ def sync_wake_event():
|
|
| 306 |
"""The event the resident thread waits on between passes."""
|
| 307 |
return _SYNC_WAKE
|
| 308 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 309 |
# One receipt per immutable database identity. No credentials, session, or source values.
|
| 310 |
JOBS_KEY = "connector_materialization_jobs"
|
| 311 |
MAX_ATTEMPTS = 3
|
|
|
|
| 32 |
_SYNC_DUE_TIMER = None
|
| 33 |
SYNC_REQUESTS_KEY = "connector_sync_requests_v1"
|
| 34 |
NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1"
|
| 35 |
+
# Ordinary requests the web process hands to the sync worker live in this member of the desk.
|
| 36 |
+
REMOTE_MEMBER = "ordinary"
|
| 37 |
+
REMOTE_KEEP_SECONDS = 3600
|
| 38 |
+
_REQUEST_LISTENERS = []
|
| 39 |
+
_REMOTE_IDS = {}
|
| 40 |
_EVENT_DEDUPE_SECONDS = 600
|
| 41 |
|
| 42 |
|
|
|
|
| 57 |
document = deepcopy(current) if current is not None else {}
|
| 58 |
if not isinstance(document, dict):
|
| 59 |
raise ValueError("The stored sync request desk is unavailable.")
|
| 60 |
+
for member in ("requests", "events", REMOTE_MEMBER):
|
| 61 |
document.setdefault(member, {})
|
| 62 |
if not isinstance(document[member], dict):
|
| 63 |
raise ValueError("The stored sync request desk is unavailable.")
|
|
|
|
| 86 |
_SYNC_WAKE.set()
|
| 87 |
|
| 88 |
|
| 89 |
+
def restore_sync_requests(rt, *, now=None, requeue=True):
|
| 90 |
+
"""Restore durable events and requeue claims abandoned by a previous process.
|
| 91 |
+
|
| 92 |
+
requeue=False only reads: a process that never claims must not requeue another one's claim.
|
| 93 |
+
"""
|
| 94 |
stamp = time.time() if now is None else now
|
| 95 |
with _SYNC_REQUEST_LOCK:
|
| 96 |
document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
|
| 97 |
abandoned = {identity for identity, row in document["requests"].items()
|
| 98 |
+
if requeue and row.get("state") == "running" and
|
| 99 |
_SYNC_REQUESTS.get((row.get("provider"), rt.key, identity), {}).get("state") != "running"}
|
| 100 |
if abandoned:
|
| 101 |
def restore(current):
|
|
|
|
| 149 |
_SYNC_WAKE.set()
|
| 150 |
|
| 151 |
|
| 152 |
+
def register_request_listener(fn):
|
| 153 |
+
"""Call fn(receipt, connection_id) after each accepted ordinary request, outside the desk lock."""
|
| 154 |
+
if fn not in _REQUEST_LISTENERS:
|
| 155 |
+
_REQUEST_LISTENERS.append(fn)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
def request_sync(provider, tenant, *, requested_by="", now=None, not_before=None,
|
| 159 |
+
rt=None, tables=None, event_id="", connection_id=None):
|
| 160 |
"""Queue ordinary work or durably acknowledge one tenant-bound Amazon event."""
|
| 161 |
+
receipt = _request_sync(provider, tenant, requested_by=requested_by, now=now,
|
| 162 |
+
not_before=not_before, rt=rt, tables=tables, event_id=event_id)
|
| 163 |
+
if tables is None and receipt.get("accepted"):
|
| 164 |
+
for listener in list(_REQUEST_LISTENERS):
|
| 165 |
+
listener(dict(receipt), connection_id)
|
| 166 |
+
return receipt
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=None,
|
| 170 |
+
rt=None, tables=None, event_id=""):
|
| 171 |
if provider not in _PROVIDERS:
|
| 172 |
raise ValueError("This connector is not registered for sync.")
|
| 173 |
tenant = str(tenant or "").strip()
|
|
|
|
| 330 |
"""The event the resident thread waits on between passes."""
|
| 331 |
return _SYNC_WAKE
|
| 332 |
|
| 333 |
+
|
| 334 |
+
def queue_remote_request(rt, receipt, connection_id=None, *, now=None):
|
| 335 |
+
"""Web side: durably hand one accepted ordinary request to the sync worker process."""
|
| 336 |
+
stamp = time.time() if now is None else now
|
| 337 |
+
identity = uuid.uuid4().hex
|
| 338 |
+
entry = {"id": identity, "provider": receipt["provider"], "tenant": rt.key,
|
| 339 |
+
"requestedBy": str(receipt.get("requestedBy") or ""),
|
| 340 |
+
"requestedAt": float(receipt.get("requestedAt") or stamp),
|
| 341 |
+
"notBefore": receipt.get("notBefore"),
|
| 342 |
+
"connectionId": str(connection_id) if connection_id else None,
|
| 343 |
+
"state": "queued", "startedAt": None, "finishedAt": None, "detail": ""}
|
| 344 |
+
|
| 345 |
+
def add(current):
|
| 346 |
+
document = _sync_request_document(current)
|
| 347 |
+
document[REMOTE_MEMBER][identity] = entry
|
| 348 |
+
return document
|
| 349 |
+
|
| 350 |
+
rt.update(SYNC_REQUESTS_KEY, add, flush="sync")
|
| 351 |
+
with _SYNC_REQUEST_LOCK:
|
| 352 |
+
_REMOTE_IDS[(receipt["provider"], rt.key)] = identity
|
| 353 |
+
return identity
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
def fail_remote_request(provider, tenant, detail, *, now=None):
|
| 357 |
+
"""Web side: a hand-off that could not be saved finishes its receipt instead of hanging."""
|
| 358 |
+
with _SYNC_REQUEST_LOCK:
|
| 359 |
+
receipt = _SYNC_REQUESTS.get((provider, tenant))
|
| 360 |
+
if receipt and receipt["state"] == "queued":
|
| 361 |
+
receipt.update(state="failed", detail=str(detail)[:300],
|
| 362 |
+
finishedAt=time.time() if now is None else now)
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def _remote_progress(entry, receipt):
|
| 366 |
+
"""The worker receipt's progress for one entry, or None when that run does not cover it."""
|
| 367 |
+
started = receipt.get("startedAt") if receipt else None
|
| 368 |
+
if not started or entry["requestedAt"] > started or receipt["state"] not in ("running", "done", "failed"):
|
| 369 |
+
return None
|
| 370 |
+
progress = {"state": receipt["state"], "startedAt": started,
|
| 371 |
+
"finishedAt": receipt.get("finishedAt"), "detail": receipt.get("detail") or ""}
|
| 372 |
+
if receipt.get("firstLoad"):
|
| 373 |
+
progress["firstLoad"] = dict(receipt["firstLoad"])
|
| 374 |
+
return progress
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def poll_remote_requests(rt, *, now=None):
|
| 378 |
+
"""Worker side: admit queued hand-offs, publish their progress, prune old ones.
|
| 379 |
+
|
| 380 |
+
An entry leaves the queue only when a run that started after it was requested finishes.
|
| 381 |
+
Returns the entries admitted as new local receipts.
|
| 382 |
+
"""
|
| 383 |
+
stamp = time.time() if now is None else now
|
| 384 |
+
admitted, changes, pruned = [], {}, []
|
| 385 |
+
with _SYNC_REQUEST_LOCK:
|
| 386 |
+
document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
|
| 387 |
+
for identity, entry in sorted(document[REMOTE_MEMBER].items(),
|
| 388 |
+
key=lambda item: item[1].get("requestedAt") or 0):
|
| 389 |
+
key = (entry.get("provider"), rt.key)
|
| 390 |
+
if entry.get("tenant") != rt.key or key[0] not in _PROVIDERS:
|
| 391 |
+
continue
|
| 392 |
+
if entry["state"] in ("done", "failed"):
|
| 393 |
+
if stamp - float(entry.get("finishedAt") or 0) > REMOTE_KEEP_SECONDS:
|
| 394 |
+
pruned.append(identity)
|
| 395 |
+
continue
|
| 396 |
+
receipt = _SYNC_REQUESTS.get(key)
|
| 397 |
+
progress = _remote_progress(entry, receipt)
|
| 398 |
+
if progress is not None:
|
| 399 |
+
if any(entry.get(name) != value for name, value in progress.items()):
|
| 400 |
+
changes[identity] = progress
|
| 401 |
+
continue
|
| 402 |
+
# A "running" entry with no local run was abandoned by a worker restart: admit it again.
|
| 403 |
+
if (entry["state"] in ("queued", "running")
|
| 404 |
+
and not (receipt and receipt["state"] in ("queued", "running"))):
|
| 405 |
+
local = {"provider": key[0], "tenant": rt.key, "state": "queued",
|
| 406 |
+
"requestedAt": entry["requestedAt"],
|
| 407 |
+
"notBefore": max(float(entry.get("notBefore") or 0),
|
| 408 |
+
_SYNC_NOT_BEFORE.get(key, 0)) or None,
|
| 409 |
+
"startedAt": None, "finishedAt": None, "detail": "",
|
| 410 |
+
"requestedBy": entry.get("requestedBy") or "", "accepted": True}
|
| 411 |
+
if receipt and receipt.get("firstLoad") and local["requestedBy"] == "first load":
|
| 412 |
+
local.update(detail=receipt["detail"], firstLoad=dict(receipt["firstLoad"]))
|
| 413 |
+
_SYNC_REQUESTS[key] = local
|
| 414 |
+
_SYNC_REQUEST_RUNTIMES.setdefault(rt.key, rt)
|
| 415 |
+
admitted.append(deepcopy(entry))
|
| 416 |
+
if changes or pruned:
|
| 417 |
+
def publish(current):
|
| 418 |
+
doc = _sync_request_document(current)
|
| 419 |
+
for identity, progress in changes.items():
|
| 420 |
+
if identity in doc[REMOTE_MEMBER]:
|
| 421 |
+
doc[REMOTE_MEMBER][identity].update(progress)
|
| 422 |
+
for identity in pruned:
|
| 423 |
+
doc[REMOTE_MEMBER].pop(identity, None)
|
| 424 |
+
return doc
|
| 425 |
+
rt.update(SYNC_REQUESTS_KEY, publish, flush="sync")
|
| 426 |
+
if admitted:
|
| 427 |
+
_wake_sync_requests(stamp)
|
| 428 |
+
return admitted
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def apply_remote_progress(rt):
|
| 432 |
+
"""Web side: copy the worker's published progress onto this process's receipts."""
|
| 433 |
+
document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
|
| 434 |
+
applied = 0
|
| 435 |
+
with _SYNC_REQUEST_LOCK:
|
| 436 |
+
for identity, entry in document[REMOTE_MEMBER].items():
|
| 437 |
+
key = (entry.get("provider"), rt.key)
|
| 438 |
+
receipt = _SYNC_REQUESTS.get(key)
|
| 439 |
+
if _REMOTE_IDS.get(key) != identity or receipt is None:
|
| 440 |
+
continue
|
| 441 |
+
fields = {name: entry.get(name) for name in ("state", "startedAt", "finishedAt", "detail")}
|
| 442 |
+
if entry.get("firstLoad"):
|
| 443 |
+
fields["firstLoad"] = dict(entry["firstLoad"])
|
| 444 |
+
if entry["state"] != "queued" and any(receipt.get(k) != v for k, v in fields.items()):
|
| 445 |
+
receipt.update(fields)
|
| 446 |
+
applied += 1
|
| 447 |
+
# Event receipts the worker claimed or finished, refreshed without requeueing anything.
|
| 448 |
+
_cache_sync_requests(rt, document)
|
| 449 |
+
return applied
|
| 450 |
+
|
| 451 |
# One receipt per immutable database identity. No credentials, session, or source values.
|
| 452 |
JOBS_KEY = "connector_materialization_jobs"
|
| 453 |
MAX_ATTEMPTS = 3
|
platform/harness/datastore.py
CHANGED
|
@@ -126,11 +126,24 @@ def path_for(tenant_key):
|
|
| 126 |
boundary that holds is the operating system's — one file, one tenant.
|
| 127 |
"""
|
| 128 |
key = str(tenant_key or "").strip().lower()
|
|
|
|
|
|
|
| 129 |
if key in ("", "royal-imports"):
|
| 130 |
return DB_PATH if key == "" else Path(
|
| 131 |
-
os.environ.get("AIOS_DUCKDB_PATH") or (
|
| 132 |
safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60]
|
| 133 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
|
| 135 |
|
| 136 |
def use_path(path):
|
|
@@ -444,7 +457,7 @@ def _mirror_connection(tenant_key, rt):
|
|
| 444 |
rt.assert_datastore_matches()
|
| 445 |
return connect(), path
|
| 446 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 447 |
-
con = duckdb.connect(str(path))
|
| 448 |
_ensure_sync_state(con)
|
| 449 |
return con, path
|
| 450 |
|
|
@@ -851,7 +864,7 @@ def _instance():
|
|
| 851 |
with _INSTANCE_LOCK:
|
| 852 |
if _INSTANCE["con"] is None:
|
| 853 |
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 854 |
-
_INSTANCE["con"] = duckdb.connect(str(DB_PATH))
|
| 855 |
return _INSTANCE["con"]
|
| 856 |
|
| 857 |
|
|
|
|
| 126 |
boundary that holds is the operating system's — one file, one tenant.
|
| 127 |
"""
|
| 128 |
key = str(tenant_key or "").strip().lower()
|
| 129 |
+
# AIOS_DUCKDB_STORE_DIR moves every tenant's file, so the sync worker never opens the web's.
|
| 130 |
+
store_dir = Path(os.environ.get("AIOS_DUCKDB_STORE_DIR") or _STORE_DIR)
|
| 131 |
if key in ("", "royal-imports"):
|
| 132 |
return DB_PATH if key == "" else Path(
|
| 133 |
+
os.environ.get("AIOS_DUCKDB_PATH") or (store_dir / "royal.duckdb"))
|
| 134 |
safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60]
|
| 135 |
+
return store_dir / f"{safe}.duckdb"
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def apply_limits(con):
|
| 139 |
+
"""Cap one DuckDB instance from AIOS_DUCKDB_MEMORY and AIOS_DUCKDB_THREADS. Unset keeps the default."""
|
| 140 |
+
memory = str(os.environ.get("AIOS_DUCKDB_MEMORY") or "").strip()
|
| 141 |
+
threads = str(os.environ.get("AIOS_DUCKDB_THREADS") or "").strip()
|
| 142 |
+
if memory and re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", memory, re.IGNORECASE):
|
| 143 |
+
con.execute(f"SET memory_limit='{memory}'")
|
| 144 |
+
if threads.isdigit() and int(threads) > 0:
|
| 145 |
+
con.execute(f"SET threads={int(threads)}")
|
| 146 |
+
return con
|
| 147 |
|
| 148 |
|
| 149 |
def use_path(path):
|
|
|
|
| 457 |
rt.assert_datastore_matches()
|
| 458 |
return connect(), path
|
| 459 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 460 |
+
con = apply_limits(duckdb.connect(str(path)))
|
| 461 |
_ensure_sync_state(con)
|
| 462 |
return con, path
|
| 463 |
|
|
|
|
| 864 |
with _INSTANCE_LOCK:
|
| 865 |
if _INSTANCE["con"] is None:
|
| 866 |
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 867 |
+
_INSTANCE["con"] = apply_limits(duckdb.connect(str(DB_PATH)))
|
| 868 |
return _INSTANCE["con"]
|
| 869 |
|
| 870 |
|
platform/harness/mirror_store.py
CHANGED
|
@@ -44,6 +44,88 @@ _CATCH_UP_LOCK = threading.RLock()
|
|
| 44 |
_RESTORE_PROGRESS = threading.local()
|
| 45 |
_NEWEST_READERS = {}
|
| 46 |
_PUBLICATION_CHECKS = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
|
| 48 |
|
| 49 |
def register_publication_check(connector, check):
|
|
@@ -603,6 +685,15 @@ class MirrorStore:
|
|
| 603 |
result = {"state": "local", "problems": [], "manifest": {}, "dirty": set(), "runtime": rt,
|
| 604 |
"refused_tables": []}
|
| 605 |
con, _path = self._connect(tenant_key, rt)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 606 |
_RESTORE_PROGRESS.store = self
|
| 607 |
try:
|
| 608 |
if self._has_local(con):
|
|
@@ -619,6 +710,7 @@ class MirrorStore:
|
|
| 619 |
local_id = datastore._publication(con, self.connector).get("snapshot_id")
|
| 620 |
if saved.get("snapshot_id") and saved["snapshot_id"] != local_id:
|
| 621 |
self._install_snapshot(con, saved, rows, paths)
|
|
|
|
| 622 |
result["manifest"] = saved
|
| 623 |
if not result["manifest"].get("snapshot_id"):
|
| 624 |
self._after_restore(con, result, log)
|
|
@@ -645,6 +737,7 @@ class MirrorStore:
|
|
| 645 |
manifest, rows, paths, total, refused = self._validated_snapshot(
|
| 646 |
tenant_key, rt, con, raw, tmp, allow_partial=True)
|
| 647 |
self._install_snapshot(con, manifest, rows, paths)
|
|
|
|
| 648 |
result.update(state="partial" if refused else "restored", manifest=manifest, refused_tables=refused)
|
| 649 |
if refused:
|
| 650 |
result["problems"].append(f"{self.label} mirror tables refused: " + ", ".join(refused)
|
|
@@ -1136,7 +1229,7 @@ def _export_tables(store, con, directory):
|
|
| 1136 |
def _open_candidate(store, tenant_key, rt, published):
|
| 1137 |
"""Resume private rows only while their base publication still matches the serving copy."""
|
| 1138 |
path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
|
| 1139 |
-
candidate = datastore.duckdb.connect(str(path))
|
| 1140 |
datastore._ensure_sync_state(candidate)
|
| 1141 |
candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_base (identity VARCHAR, started_at VARCHAR)")
|
| 1142 |
candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_pending (name VARCHAR PRIMARY KEY)")
|
|
@@ -1348,6 +1441,12 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
|
|
| 1348 |
raise
|
| 1349 |
if odoo:
|
| 1350 |
note("install", installing)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1351 |
if holds is not None and holds["tables"]:
|
| 1352 |
candidate.execute("ROLLBACK")
|
| 1353 |
candidate.execute("DELETE FROM _candidate_base")
|
|
|
|
| 44 |
_RESTORE_PROGRESS = threading.local()
|
| 45 |
_NEWEST_READERS = {}
|
| 46 |
_PUBLICATION_CHECKS = {}
|
| 47 |
+
_PUBLISH_LISTENERS = []
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def register_publish_listener(fn):
|
| 51 |
+
"""Call fn(connector, tenant_key, manifest, state_rows, paths) after each local install.
|
| 52 |
+
|
| 53 |
+
The paths are deleted when fn returns, so a listener that keeps them copies them first.
|
| 54 |
+
"""
|
| 55 |
+
if fn not in _PUBLISH_LISTENERS:
|
| 56 |
+
_PUBLISH_LISTENERS.append(fn)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _web_waits_for_worker(connector):
|
| 60 |
+
"""True in the web process for every mirror but Odoo: those arrive only from the sync worker."""
|
| 61 |
+
return os.environ.get("AIOS_PROCESS_ROLE") == "web" and connector != "odoo"
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _notify_publish(connector, tenant_key, manifest, rows, paths, log=print):
|
| 65 |
+
"""Hand a restored snapshot to the listeners too, so the web process never restores it itself."""
|
| 66 |
+
if not paths or not manifest.get("snapshot_id"):
|
| 67 |
+
return
|
| 68 |
+
for listener in list(_PUBLISH_LISTENERS):
|
| 69 |
+
try:
|
| 70 |
+
listener(connector, tenant_key, manifest, [tuple(row) for row in rows], paths)
|
| 71 |
+
except Exception as exc: # noqa: BLE001
|
| 72 |
+
log(f"{connector} mirror restore hand-off failed ({type(exc).__name__}: {exc})")
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
|
| 76 |
+
"""Install a snapshot another process already published. Returns False if it is already resident."""
|
| 77 |
+
store = for_connector(connector)
|
| 78 |
+
if manifest.get("tenant") not in (None, _tenant_key(tenant_key)):
|
| 79 |
+
raise ValueError("Hand-off manifest belongs to another tenant")
|
| 80 |
+
with store.lock:
|
| 81 |
+
con, _path = store._connect(tenant_key, rt)
|
| 82 |
+
try:
|
| 83 |
+
if (manifest.get("snapshot_id") and
|
| 84 |
+
datastore._publication(con, store.connector).get("snapshot_id") == manifest["snapshot_id"]):
|
| 85 |
+
return False
|
| 86 |
+
store._install_snapshot(con, manifest, [tuple(row) for row in rows], paths)
|
| 87 |
+
newest = _record_newest(con, store.connector)
|
| 88 |
+
finally:
|
| 89 |
+
con.close()
|
| 90 |
+
cache_key = store._mirror_cache_key(tenant_key, rt)
|
| 91 |
+
if _web_waits_for_worker(store.connector):
|
| 92 |
+
# The web process never persists, so the installed hand-off is its restore answer.
|
| 93 |
+
store.cache[cache_key] = {"state": "restored", "problems": [], "manifest": manifest, "dirty": set(),
|
| 94 |
+
"runtime": rt, "refused_tables": []}
|
| 95 |
+
else:
|
| 96 |
+
# The next restore re-reads the stored manifest instead of trusting this process's copy.
|
| 97 |
+
store.cache.pop(cache_key, None)
|
| 98 |
+
with _CATCH_UP_LOCK:
|
| 99 |
+
state = _catch_up_source(rt, store.connector)
|
| 100 |
+
if not state["refused"]:
|
| 101 |
+
state.update(current=True, pass_pending=False, newest=newest, misses=0, behind="")
|
| 102 |
+
return True
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
_CATCH_UP_SHARED = ("current", "part_total", "passes", "pass_pending", "newest", "refused",
|
| 106 |
+
"misses", "behind", "attempt_at", "missed_at", "miss_reason", "since")
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def catch_up_snapshot():
|
| 110 |
+
"""This process's catch-up progress keyed by tenant, as JSON for another process."""
|
| 111 |
+
with _CATCH_UP_LOCK:
|
| 112 |
+
return {tenant: {connector: {**{k: state[k] for k in _CATCH_UP_SHARED}, "parts": len(state["parts"])}
|
| 113 |
+
for connector, state in sources.items()}
|
| 114 |
+
for (tenant, _path), sources in _CATCH_UP.items()}
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def apply_catch_up(rt, sources):
|
| 118 |
+
"""Adopt the sync process's catch-up progress for one tenant's resident mirror."""
|
| 119 |
+
with _CATCH_UP_LOCK:
|
| 120 |
+
for connector, shared in (sources or {}).items():
|
| 121 |
+
state = _catch_up_source(rt, connector)
|
| 122 |
+
refused = state["refused"]
|
| 123 |
+
state.update({k: shared[k] for k in _CATCH_UP_SHARED if k in shared})
|
| 124 |
+
# This process's own restore refusal still stands whatever the sync process reports.
|
| 125 |
+
state["refused"] = bool(refused or state["refused"])
|
| 126 |
+
parts = int(shared.get("parts") or 0)
|
| 127 |
+
if len(state["parts"]) < parts:
|
| 128 |
+
state["parts"] = set(range(parts))
|
| 129 |
|
| 130 |
|
| 131 |
def register_publication_check(connector, check):
|
|
|
|
| 685 |
result = {"state": "local", "problems": [], "manifest": {}, "dirty": set(), "runtime": rt,
|
| 686 |
"refused_tables": []}
|
| 687 |
con, _path = self._connect(tenant_key, rt)
|
| 688 |
+
if _web_waits_for_worker(self.connector):
|
| 689 |
+
# The sync worker restores this mirror and hands it over; the web process never downloads it.
|
| 690 |
+
try:
|
| 691 |
+
result["state"] = "local" if self._has_local(con) else "absent"
|
| 692 |
+
finally:
|
| 693 |
+
con.close()
|
| 694 |
+
with _CATCH_UP_LOCK:
|
| 695 |
+
_catch_up_source(rt, self.connector)["pass_pending"] = True
|
| 696 |
+
return result
|
| 697 |
_RESTORE_PROGRESS.store = self
|
| 698 |
try:
|
| 699 |
if self._has_local(con):
|
|
|
|
| 710 |
local_id = datastore._publication(con, self.connector).get("snapshot_id")
|
| 711 |
if saved.get("snapshot_id") and saved["snapshot_id"] != local_id:
|
| 712 |
self._install_snapshot(con, saved, rows, paths)
|
| 713 |
+
_notify_publish(self.connector, tenant_key, saved, rows, paths, log)
|
| 714 |
result["manifest"] = saved
|
| 715 |
if not result["manifest"].get("snapshot_id"):
|
| 716 |
self._after_restore(con, result, log)
|
|
|
|
| 737 |
manifest, rows, paths, total, refused = self._validated_snapshot(
|
| 738 |
tenant_key, rt, con, raw, tmp, allow_partial=True)
|
| 739 |
self._install_snapshot(con, manifest, rows, paths)
|
| 740 |
+
_notify_publish(self.connector, tenant_key, manifest, rows, paths, log)
|
| 741 |
result.update(state="partial" if refused else "restored", manifest=manifest, refused_tables=refused)
|
| 742 |
if refused:
|
| 743 |
result["problems"].append(f"{self.label} mirror tables refused: " + ", ".join(refused)
|
|
|
|
| 1229 |
def _open_candidate(store, tenant_key, rt, published):
|
| 1230 |
"""Resume private rows only while their base publication still matches the serving copy."""
|
| 1231 |
path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
|
| 1232 |
+
candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
|
| 1233 |
datastore._ensure_sync_state(candidate)
|
| 1234 |
candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_base (identity VARCHAR, started_at VARCHAR)")
|
| 1235 |
candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_pending (name VARCHAR PRIMARY KEY)")
|
|
|
|
| 1441 |
raise
|
| 1442 |
if odoo:
|
| 1443 |
note("install", installing)
|
| 1444 |
+
for listener in list(_PUBLISH_LISTENERS):
|
| 1445 |
+
try:
|
| 1446 |
+
listener(store.connector, tenant_key, manifest, store._state_rows(candidate), paths)
|
| 1447 |
+
except Exception as exc:
|
| 1448 |
+
# The snapshot is saved and installed here. A failed hand-off only delays readers.
|
| 1449 |
+
log(f"{store.label} mirror hand-off failed ({type(exc).__name__}: {exc})")
|
| 1450 |
if holds is not None and holds["tables"]:
|
| 1451 |
candidate.execute("ROLLBACK")
|
| 1452 |
candidate.execute("DELETE FROM _candidate_base")
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
37d432d140cdd437d7215e992efb2bbe154af782d7202e71f19aabc15857ddc6
|
web/src/sheet/SheetGrid.tsx
CHANGED
|
@@ -492,7 +492,17 @@ const BINDING_REFUSAL_COPY: Readonly<Record<string, { display: string; message:
|
|
| 492 |
// calculation, and painting it `#ERROR!` is what made a working binding look broken.
|
| 493 |
sheet_refresh_required: { display: "#REFRESH", message: "Press Refresh to calculate this cell for your access." },
|
| 494 |
store_catching_up: { display: "#UPDATING", message: "This database is updating. Refresh when it finishes." },
|
|
|
|
|
|
|
|
|
|
|
|
|
| 495 |
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 496 |
const BINDING_REFUSAL_FALLBACK = { display: "#ERROR!", message: "This cell's database binding could not be calculated." };
|
| 497 |
|
| 498 |
/** R48: old documents may have a blank display, but the grid always owns its refusal copy. */
|
|
@@ -1983,7 +1993,7 @@ export default function SheetGrid({
|
|
| 1983 |
const shimmerPhase = useRef(0);
|
| 1984 |
const pendingBoundCells = useMemo(
|
| 1985 |
() => Object.entries(cells)
|
| 1986 |
-
.filter(([, cell]) => cell
|
| 1987 |
.map(([addr]) => addr),
|
| 1988 |
[cells],
|
| 1989 |
);
|
|
@@ -6671,7 +6681,7 @@ export default function SheetGrid({
|
|
| 6671 |
* `tokenColour` (DESIGN.md), so the softness is made with `globalAlpha` instead.
|
| 6672 |
*/
|
| 6673 |
const paintShimmer = () => {
|
| 6674 |
-
if (!cell
|
| 6675 |
const bandWidth = Math.max(24, rect.width * 0.5);
|
| 6676 |
const head = rect.x - bandWidth + (rect.width + bandWidth) * shimmerPhase.current;
|
| 6677 |
const slices = 8;
|
|
|
|
| 492 |
// calculation, and painting it `#ERROR!` is what made a working binding look broken.
|
| 493 |
sheet_refresh_required: { display: "#REFRESH", message: "Press Refresh to calculate this cell for your access." },
|
| 494 |
store_catching_up: { display: "#UPDATING", message: "This database is updating. Refresh when it finishes." },
|
| 495 |
+
source_unavailable: { display: "#UPDATING", message: "The database was busy. This cell updates after the next sync." },
|
| 496 |
+
store_not_ready: { display: "#UPDATING", message: "The database was busy. This cell updates after the next sync." },
|
| 497 |
+
mirror_not_ready: { display: "#UPDATING", message: "The database was busy. This cell updates after the next sync." },
|
| 498 |
+
amazon_mirror_not_loaded: { display: "#UPDATING", message: "The database was busy. This cell updates after the next sync." },
|
| 499 |
};
|
| 500 |
+
|
| 501 |
+
/** A bound cell still waiting for an answer. A stale cell with a server reason keeps its value and does not shimmer. */
|
| 502 |
+
function boundWaiting(cell: Cell | null | undefined): cell is Cell {
|
| 503 |
+
return !!cell && cell.kind === "bound" && cell.state === "stale"
|
| 504 |
+
&& !(cell as Cell & { staleReason?: string }).staleReason;
|
| 505 |
+
}
|
| 506 |
const BINDING_REFUSAL_FALLBACK = { display: "#ERROR!", message: "This cell's database binding could not be calculated." };
|
| 507 |
|
| 508 |
/** R48: old documents may have a blank display, but the grid always owns its refusal copy. */
|
|
|
|
| 1993 |
const shimmerPhase = useRef(0);
|
| 1994 |
const pendingBoundCells = useMemo(
|
| 1995 |
() => Object.entries(cells)
|
| 1996 |
+
.filter(([, cell]) => boundWaiting(cell))
|
| 1997 |
.map(([addr]) => addr),
|
| 1998 |
[cells],
|
| 1999 |
);
|
|
|
|
| 6681 |
* `tokenColour` (DESIGN.md), so the softness is made with `globalAlpha` instead.
|
| 6682 |
*/
|
| 6683 |
const paintShimmer = () => {
|
| 6684 |
+
if (!boundWaiting(cell)) return;
|
| 6685 |
const bandWidth = Math.max(24, rect.width * 0.5);
|
| 6686 |
const head = rect.x - bandWidth + (rect.width + bandWidth) * shimmerPhase.current;
|
| 6687 |
const slices = 8;
|
web/src/sheet/SheetPage.tsx
CHANGED
|
@@ -71,7 +71,7 @@ import { readWorkbookChecks, writeWorkbookChecks } from "./sheetApi";
|
|
| 71 |
import type { RailStrata } from "./SheetRail";
|
| 72 |
import {
|
| 73 |
computeCell, createdFolderIds, createWorkbook, deleteWorkbook, duplicateSheetTab, reorderSheetTab, patchTabCells, readIndex, readTab, readTabMeta,
|
| 74 |
-
applyRefreshScope,
|
| 75 |
writePlacements, writeWorkbook,
|
| 76 |
} from "./sheetApi";
|
| 77 |
import type { RefreshScope, Result, SheetIndex, SheetMountAddress, WorkbookActivity } from "./sheetApi";
|
|
@@ -1175,9 +1175,9 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
|
|
| 1175 |
}, [book?.id, opening]);
|
| 1176 |
|
| 1177 |
/**
|
| 1178 |
-
*
|
| 1179 |
-
*
|
| 1180 |
-
*
|
| 1181 |
*
|
| 1182 |
* ⛔ THIS PASS NEVER SETS `busy`. The grid stays usable while a large workbook recomputes,
|
| 1183 |
* and the existing per-cell stale mark is the progress signal. Its answer is abandoned when a
|
|
@@ -1196,7 +1196,7 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
|
|
| 1196 |
// (~470 cells, minutes on Daily Sales). Now only the cells that need it are marked, sent to
|
| 1197 |
// `POST /refresh {cells}`, and merged back one by one; every other cell keeps its answer.
|
| 1198 |
const needs = loaded ? Object.entries(loaded.cells).filter(([, cell]) => cell.kind === "bound"
|
| 1199 |
-
&& ((cell.state === "stale" && !
|
| 1200 |
|| (cell.state === "refused" && cell.refusal?.code === "sheet_refresh_required"))) : [];
|
| 1201 |
if (needs.length === 0) return;
|
| 1202 |
const pass = { id: wid, restore: () => restoreMarkedCells() };
|
|
@@ -1275,11 +1275,8 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
|
|
| 1275 |
const current = bookRef.current;
|
| 1276 |
const needsCells = current && !tabLoadedRef.current.has(tabKeyOf(current.id, id));
|
| 1277 |
setPendingTab(needsCells ? { workbookId: current.id, tabId: id, generation } : null);
|
| 1278 |
-
|
| 1279 |
-
if (!needsCells)
|
| 1280 |
-
refreshOpenedWorkbook(current.id, id);
|
| 1281 |
-
return;
|
| 1282 |
-
}
|
| 1283 |
const stillCurrent = () => tabNavVerdict(generation, tabNavGenRef.current) === "apply"
|
| 1284 |
&& addressMountedRef.current && !workbookTransitionRef.current
|
| 1285 |
&& bookRef.current?.id === current.id
|
|
@@ -1287,7 +1284,6 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
|
|
| 1287 |
void (async () => {
|
| 1288 |
try {
|
| 1289 |
const loaded = await ensureTabs(current.id, [id], false, stillCurrent);
|
| 1290 |
-
if (loaded && stillCurrent()) refreshOpenedWorkbook(current.id, id);
|
| 1291 |
// Missing tabs and workbooks already restore a real surface through ensureTabs.
|
| 1292 |
if (!loaded && stillCurrent() && bookRef.current?.sheets.some((tab) => tab.id === id)) {
|
| 1293 |
setUnreadableAddressTab(id);
|
|
@@ -1302,7 +1298,7 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
|
|
| 1302 |
setPendingTab((pending) => pending?.generation === generation ? null : pending);
|
| 1303 |
}
|
| 1304 |
})();
|
| 1305 |
-
}, [currentSheetHash, ensureTabs, rememberOpenWorkbook
|
| 1306 |
|
| 1307 |
// Wave 60 base kit: a hidden or very hidden sheet has no tab, so it is never the open one. When
|
| 1308 |
// the open tab is hidden (by the View ribbon, an undo, or a workbook that opened on one), the
|
|
@@ -2815,7 +2811,6 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
|
|
| 2815 |
const live = bookRef.current!;
|
| 2816 |
if (!address.tab && !address.cell) {
|
| 2817 |
acceptedAddressRef.current = address;
|
| 2818 |
-
refreshOpenedWorkbook(live.id, tabIdRef.current);
|
| 2819 |
return;
|
| 2820 |
}
|
| 2821 |
// A cell-only link narrows the current tab; unknown tabs quietly keep that tab.
|
|
@@ -2873,7 +2868,6 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
|
|
| 2873 |
// the same tab, while ordinary saves and background refreshes keep the existing mount.
|
| 2874 |
setAddressSelectionKey((key) => key + 1);
|
| 2875 |
acceptedAddressRef.current = { ...address, wb: opened.id };
|
| 2876 |
-
refreshOpenedWorkbook(opened.id, activeTab);
|
| 2877 |
} finally {
|
| 2878 |
workbookTransitionRef.current = false;
|
| 2879 |
if (addressMountedRef.current) {
|
|
@@ -2884,7 +2878,7 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN
|
|
| 2884 |
}
|
| 2885 |
})();
|
| 2886 |
}, [sheetHash, opening, barAddr, adoptWorkbook, ensureTabs, flushBeforeWorkbookChange,
|
| 2887 |
-
lastWorkbookId, prepareWorkbookOpen, refreshIndex,
|
| 2888 |
rememberOpenWorkbook, currentSheetHash, navigateSheetAddress, mergeTab, readAttempt]);
|
| 2889 |
|
| 2890 |
// The workbook can open before its index arrives. Enrich only after the open has settled,
|
|
|
|
| 71 |
import type { RailStrata } from "./SheetRail";
|
| 72 |
import {
|
| 73 |
computeCell, createdFolderIds, createWorkbook, deleteWorkbook, duplicateSheetTab, reorderSheetTab, patchTabCells, readIndex, readTab, readTabMeta,
|
| 74 |
+
applyRefreshScope, isSettledStale, placementOnlyDelta, readWorkbook, refreshCells, refreshScope, retryRefresh, renameTabName, renameWorkbookName, writeIndex,
|
| 75 |
writePlacements, writeWorkbook,
|
| 76 |
} from "./sheetApi";
|
| 77 |
import type { RefreshScope, Result, SheetIndex, SheetMountAddress, WorkbookActivity } from "./sheetApi";
|
|
|
|
| 1175 |
}, [book?.id, opening]);
|
| 1176 |
|
| 1177 |
/**
|
| 1178 |
+
* Resolve the bound cells a structural edit or a new tab left without an answer.
|
| 1179 |
+
* ⛔ Opening a workbook or a tab never calls this: an open only reads stored results.
|
| 1180 |
+
* A cell the server serves stale with a reason keeps its stored value and is not resolved here.
|
| 1181 |
*
|
| 1182 |
* ⛔ THIS PASS NEVER SETS `busy`. The grid stays usable while a large workbook recomputes,
|
| 1183 |
* and the existing per-cell stale mark is the progress signal. Its answer is abandoned when a
|
|
|
|
| 1196 |
// (~470 cells, minutes on Daily Sales). Now only the cells that need it are marked, sent to
|
| 1197 |
// `POST /refresh {cells}`, and merged back one by one; every other cell keeps its answer.
|
| 1198 |
const needs = loaded ? Object.entries(loaded.cells).filter(([, cell]) => cell.kind === "bound"
|
| 1199 |
+
&& ((cell.state === "stale" && !isSettledStale(cell))
|
| 1200 |
|| (cell.state === "refused" && cell.refusal?.code === "sheet_refresh_required"))) : [];
|
| 1201 |
if (needs.length === 0) return;
|
| 1202 |
const pass = { id: wid, restore: () => restoreMarkedCells() };
|
|
|
|
| 1275 |
const current = bookRef.current;
|
| 1276 |
const needsCells = current && !tabLoadedRef.current.has(tabKeyOf(current.id, id));
|
| 1277 |
setPendingTab(needsCells ? { workbookId: current.id, tabId: id, generation } : null);
|
| 1278 |
+
// Opening a tab only reads. The sync schedule and the Refresh button compute bound cells.
|
| 1279 |
+
if (!current || !needsCells) return;
|
|
|
|
|
|
|
|
|
|
| 1280 |
const stillCurrent = () => tabNavVerdict(generation, tabNavGenRef.current) === "apply"
|
| 1281 |
&& addressMountedRef.current && !workbookTransitionRef.current
|
| 1282 |
&& bookRef.current?.id === current.id
|
|
|
|
| 1284 |
void (async () => {
|
| 1285 |
try {
|
| 1286 |
const loaded = await ensureTabs(current.id, [id], false, stillCurrent);
|
|
|
|
| 1287 |
// Missing tabs and workbooks already restore a real surface through ensureTabs.
|
| 1288 |
if (!loaded && stillCurrent() && bookRef.current?.sheets.some((tab) => tab.id === id)) {
|
| 1289 |
setUnreadableAddressTab(id);
|
|
|
|
| 1298 |
setPendingTab((pending) => pending?.generation === generation ? null : pending);
|
| 1299 |
}
|
| 1300 |
})();
|
| 1301 |
+
}, [currentSheetHash, ensureTabs, rememberOpenWorkbook]);
|
| 1302 |
|
| 1303 |
// Wave 60 base kit: a hidden or very hidden sheet has no tab, so it is never the open one. When
|
| 1304 |
// the open tab is hidden (by the View ribbon, an undo, or a workbook that opened on one), the
|
|
|
|
| 2811 |
const live = bookRef.current!;
|
| 2812 |
if (!address.tab && !address.cell) {
|
| 2813 |
acceptedAddressRef.current = address;
|
|
|
|
| 2814 |
return;
|
| 2815 |
}
|
| 2816 |
// A cell-only link narrows the current tab; unknown tabs quietly keep that tab.
|
|
|
|
| 2868 |
// the same tab, while ordinary saves and background refreshes keep the existing mount.
|
| 2869 |
setAddressSelectionKey((key) => key + 1);
|
| 2870 |
acceptedAddressRef.current = { ...address, wb: opened.id };
|
|
|
|
| 2871 |
} finally {
|
| 2872 |
workbookTransitionRef.current = false;
|
| 2873 |
if (addressMountedRef.current) {
|
|
|
|
| 2878 |
}
|
| 2879 |
})();
|
| 2880 |
}, [sheetHash, opening, barAddr, adoptWorkbook, ensureTabs, flushBeforeWorkbookChange,
|
| 2881 |
+
lastWorkbookId, prepareWorkbookOpen, refreshIndex, discardMissingWorkbook,
|
| 2882 |
rememberOpenWorkbook, currentSheetHash, navigateSheetAddress, mergeTab, readAttempt]);
|
| 2883 |
|
| 2884 |
// The workbook can open before its index arrives. Enrich only after the open has settled,
|
web/src/sheet/sheetApi.ts
CHANGED
|
@@ -513,6 +513,10 @@ export const retryRefresh = (id: string, refreshId: string): Promise<Result<Refr
|
|
| 513 |
export const isSourceStale = (cell: Cell): boolean =>
|
| 514 |
(cell as Cell & { staleReason?: string }).staleReason === "source_changed";
|
| 515 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 516 |
export type RefreshScopeResult = { addresses: string[]; recalcMs: number | null } & (
|
| 517 |
{ workbook: Workbook } | { workbook: Pick<Workbook, "id" | "rev">; cells: Record<string, Cell> }
|
| 518 |
);
|
|
|
|
| 513 |
export const isSourceStale = (cell: Cell): boolean =>
|
| 514 |
(cell as Cell & { staleReason?: string }).staleReason === "source_changed";
|
| 515 |
|
| 516 |
+
/** A stale cell the server explained keeps its stored value: it is settled, not pending. */
|
| 517 |
+
export const isSettledStale = (cell: Cell | null | undefined): boolean =>
|
| 518 |
+
!!cell && cell.state === "stale" && !!(cell as Cell & { staleReason?: string }).staleReason;
|
| 519 |
+
|
| 520 |
export type RefreshScopeResult = { addresses: string[]; recalcMs: number | null } & (
|
| 521 |
{ workbook: Workbook } | { workbook: Pick<Workbook, "id" | "rev">; cells: Record<string, Cell> }
|
| 522 |
);
|