Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +19 -0
- api/process_role.py +6 -1
- api/request_class.py +276 -0
- api/routes_customers.py +3 -2
- api/routes_sheet.py +422 -6
- api/sheet_warm.py +103 -0
- platform/harness/amazon_store.py +4 -4
- platform/harness/mirror_store.py +19 -5
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/sheet/SheetPage.tsx +2 -2
- web/src/sheet/sheetApi.ts +160 -19
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v83",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "7fe0be77d44f (7fe0be77d44f)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v83",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
7fe0be77d44f (7fe0be77d44f)
|
api/main.py
CHANGED
|
@@ -147,6 +147,8 @@ app = FastAPI(title="AIOS web API")
|
|
| 147 |
# payload to ~10β15% of that. minimum_size spares the tiny acks the overhead.
|
| 148 |
from assistant_stream import StreamGZipMiddleware
|
| 149 |
app.add_middleware(StreamGZipMiddleware, minimum_size=1024)
|
|
|
|
|
|
|
| 150 |
|
| 151 |
|
| 152 |
@app.exception_handler(StarletteHTTPException)
|
|
@@ -2995,11 +2997,28 @@ def _boot_mirrors():
|
|
| 2995 |
if _role_boot.role() == "web":
|
| 2996 |
# Every connector but Odoo reaches the web process through the worker's hand-off.
|
| 2997 |
print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
|
|
|
|
|
|
|
| 2998 |
return
|
| 2999 |
_restore_amazon_mirror("boot")
|
| 3000 |
_restore_declared_mirrors("boot")
|
| 3001 |
|
| 3002 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3003 |
def _restore_declared_mirrors(why):
|
| 3004 |
"""Restore each tenant's declarations after the Odoo seed and Amazon policy adapters."""
|
| 3005 |
from core import connector_spec
|
|
|
|
| 147 |
# payload to ~10β15% of that. minimum_size spares the tiny acks the overhead.
|
| 148 |
from assistant_stream import StreamGZipMiddleware
|
| 149 |
app.add_middleware(StreamGZipMiddleware, minimum_size=1024)
|
| 150 |
+
from request_class import HeavyGate # noqa: E402 (heavy reads queue so light requests never starve)
|
| 151 |
+
app.add_middleware(HeavyGate)
|
| 152 |
|
| 153 |
|
| 154 |
@app.exception_handler(StarletteHTTPException)
|
|
|
|
| 2997 |
if _role_boot.role() == "web":
|
| 2998 |
# Every connector but Odoo reaches the web process through the worker's hand-off.
|
| 2999 |
print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
|
| 3000 |
+
# The biggest read caches are this process's own, so they warm here, before the first reader pays.
|
| 3001 |
+
_role_boot.start("read-warm", "light", _warm_read_pools)
|
| 3002 |
return
|
| 3003 |
_restore_amazon_mirror("boot")
|
| 3004 |
_restore_declared_mirrors("boot")
|
| 3005 |
|
| 3006 |
|
| 3007 |
+
def _warm_read_pools():
|
| 3008 |
+
"""Build the consolidated customer and product pools once after boot, off the request path."""
|
| 3009 |
+
import time as _t
|
| 3010 |
+
t0 = _t.time()
|
| 3011 |
+
try:
|
| 3012 |
+
from harness import runtime as _runtime
|
| 3013 |
+
import routes_products
|
| 3014 |
+
rt = _runtime.get_runtime("royal-imports")
|
| 3015 |
+
routes_customers.warm_default(rt)
|
| 3016 |
+
routes_products._pool_for(rt, None)
|
| 3017 |
+
print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
|
| 3018 |
+
except Exception as exc: # noqa: BLE001
|
| 3019 |
+
print(f"[aios-api] read warm skipped ({type(exc).__name__}: {exc})")
|
| 3020 |
+
|
| 3021 |
+
|
| 3022 |
def _restore_declared_mirrors(why):
|
| 3023 |
"""Restore each tenant's declarations after the Odoo seed and Amazon policy adapters."""
|
| 3024 |
from core import connector_spec
|
api/process_role.py
CHANGED
|
@@ -228,7 +228,7 @@ def install_ready(root=None, installer=None, *, now=None):
|
|
| 228 |
snapshot = ready.get("snapshotId") or newest_dir.name
|
| 229 |
if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS:
|
| 230 |
try:
|
| 231 |
-
installer(key[1], key[0], ready)
|
| 232 |
except Exception as exc: # noqa: BLE001
|
| 233 |
_INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
|
| 234 |
LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
|
|
@@ -237,6 +237,11 @@ def install_ready(root=None, installer=None, *, now=None):
|
|
| 237 |
f"({type(exc).__name__}: {exc})", flush=True)
|
| 238 |
continue
|
| 239 |
_INSTALLED[key] = snapshot
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
|
| 241 |
if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
|
| 242 |
LAST_INSTALL_ERROR.clear()
|
|
|
|
| 228 |
snapshot = ready.get("snapshotId") or newest_dir.name
|
| 229 |
if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS:
|
| 230 |
try:
|
| 231 |
+
changed = installer(key[1], key[0], ready)
|
| 232 |
except Exception as exc: # noqa: BLE001
|
| 233 |
_INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
|
| 234 |
LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
|
|
|
|
| 237 |
f"({type(exc).__name__}: {exc})", flush=True)
|
| 238 |
continue
|
| 239 |
_INSTALLED[key] = snapshot
|
| 240 |
+
if changed is False:
|
| 241 |
+
# Already resident: nothing was rebuilt, so this is not an install.
|
| 242 |
+
for _at, older, _ready in rows[:-1]:
|
| 243 |
+
_remove_dir(older)
|
| 244 |
+
continue
|
| 245 |
LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
|
| 246 |
if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
|
| 247 |
LAST_INSTALL_ERROR.clear()
|
api/request_class.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Heavy request admission: a bounded queue so CPU-heavy reads never starve light requests.
|
| 2 |
+
|
| 3 |
+
One web process on 2 vCPU serves everyone. `classify` names the heavy doors in ONE registry, and
|
| 4 |
+
`HeavyGate` lets at most `AIOS_HEAVY_CONCURRENCY` of them run at once. A heavy request that waits
|
| 5 |
+
longer than `AIOS_HEAVY_QUEUE_SECONDS` gets a 503 `server_busy`. Light requests never wait.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import asyncio
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
import re
|
| 13 |
+
import threading
|
| 14 |
+
from urllib.parse import parse_qs
|
| 15 |
+
|
| 16 |
+
BUSY_CODE = "server_busy"
|
| 17 |
+
BUSY_MESSAGE = "Loopable is busy. Try again in a moment."
|
| 18 |
+
RETRY_AFTER_SECONDS = 5
|
| 19 |
+
MCP_PATH = "/api/v1/external/mcp"
|
| 20 |
+
# The MCP body is peeked to classify it. A larger body is heavy without parsing.
|
| 21 |
+
MCP_PEEK_LIMIT = 1 << 20
|
| 22 |
+
|
| 23 |
+
# FastAPI reads these strings as True for a bool query parameter.
|
| 24 |
+
_TRUE = {"1", "true", "on", "yes", "y", "t"}
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _flag(query, name):
|
| 28 |
+
return any(str(v).strip().lower() in _TRUE for v in query.get(name, []))
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _sheet_read_heavy(query):
|
| 32 |
+
# Only the cell-free metadata read is light. A bare GET or any other `tabs` value reads cells.
|
| 33 |
+
tabs = (query.get("tabs") or [""])[-1]
|
| 34 |
+
return not (tabs == "meta" and not _flag(query, "opening"))
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# (methods, path regex, query predicate or None). A request is heavy when one row matches.
|
| 38 |
+
HEAVY = (
|
| 39 |
+
({"GET"}, r"/api/v1/workspace", lambda q: not _flag(q, "fields_only")),
|
| 40 |
+
({"GET"}, r"/api/v1/products", None),
|
| 41 |
+
({"GET"}, r"/api/v1/customers", None),
|
| 42 |
+
({"GET", "POST"}, r"/api/v1/odoo-tables/[^/]+/rows", None),
|
| 43 |
+
({"GET", "POST"}, r"/api/v1/connected-tables/[^/]+/rows", None),
|
| 44 |
+
({"GET"}, r"/api/v1/tables/[^/]+/rows", None),
|
| 45 |
+
({"GET"}, r"/api/v1/sheets/(?!index$|import$|automation$)[^/]+", _sheet_read_heavy),
|
| 46 |
+
({"GET"}, r"/api/v1/sheets/[^/]+/tabs/[^/]+", None),
|
| 47 |
+
# In the web role a refresh only queues for the sync worker; it computes only in worker or all.
|
| 48 |
+
({"POST"}, r"/api/v1/sheets/[^/]+/refresh", lambda q: os.environ.get("AIOS_PROCESS_ROLE") != "web"),
|
| 49 |
+
({"GET", "POST"}, r"/api/v1/.*(?:/export|-export)(?:/.*)?", None),
|
| 50 |
+
({"POST"}, r"/api/v1/(?:external|typed)/databases/[^/]+/query", None),
|
| 51 |
+
({"POST"}, r"/api/v1/external/explorer/run", None),
|
| 52 |
+
)
|
| 53 |
+
_HEAVY = tuple((methods, re.compile(rx), pred) for methods, rx, pred in HEAVY)
|
| 54 |
+
|
| 55 |
+
# MCP tools/call names that read sheets or query databases.
|
| 56 |
+
MCP_HEAVY_TOOLS = {"sheet", "table", "data"}
|
| 57 |
+
_MCP_LIGHT_ACTIONS = re.compile(r"(?:^|[._])(?:describe|list|guide|schema)(?:$|[._])")
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _mcp_heavy(body):
|
| 61 |
+
"""True when an MCP JSON-RPC body is a tools/call on a sheet or query tool."""
|
| 62 |
+
if body is None:
|
| 63 |
+
return False
|
| 64 |
+
if len(body) > MCP_PEEK_LIMIT:
|
| 65 |
+
return True
|
| 66 |
+
try:
|
| 67 |
+
req = json.loads(body)
|
| 68 |
+
except (ValueError, UnicodeError):
|
| 69 |
+
return False
|
| 70 |
+
calls = req if isinstance(req, list) else [req]
|
| 71 |
+
for call in calls:
|
| 72 |
+
if not isinstance(call, dict) or call.get("method") != "tools/call":
|
| 73 |
+
continue
|
| 74 |
+
params = call.get("params") if isinstance(call.get("params"), dict) else {}
|
| 75 |
+
name = params.get("name")
|
| 76 |
+
if not isinstance(name, str):
|
| 77 |
+
continue
|
| 78 |
+
group, _, dotted = name.partition(".")
|
| 79 |
+
if group not in MCP_HEAVY_TOOLS and not name.startswith("sheet_") and "query" not in name:
|
| 80 |
+
continue
|
| 81 |
+
args = params.get("arguments") if isinstance(params.get("arguments"), dict) else {}
|
| 82 |
+
action = dotted or str(args.get("action") or "")
|
| 83 |
+
if action and _MCP_LIGHT_ACTIONS.search(action) and "query" not in action:
|
| 84 |
+
continue
|
| 85 |
+
return True
|
| 86 |
+
return False
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def classify(method, path, query="", body=None):
|
| 90 |
+
"""Return "heavy" or "light" for one request. `query` is a raw query string or a parsed dict."""
|
| 91 |
+
method = (method or "").upper()
|
| 92 |
+
if isinstance(query, bytes):
|
| 93 |
+
query = query.decode("latin-1")
|
| 94 |
+
q = parse_qs(query, keep_blank_values=True) if isinstance(query, str) else (query or {})
|
| 95 |
+
for methods, rx, pred in _HEAVY:
|
| 96 |
+
if method in methods and rx.fullmatch(path or "") and (pred is None or pred(q)):
|
| 97 |
+
return "heavy"
|
| 98 |
+
if method == "POST" and path == MCP_PATH and _mcp_heavy(body):
|
| 99 |
+
return "heavy"
|
| 100 |
+
return "light"
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
class _Counters:
|
| 104 |
+
"""Process-wide heavy request counters. Middleware instances report into this."""
|
| 105 |
+
|
| 106 |
+
def __init__(self):
|
| 107 |
+
self.lock = threading.Lock()
|
| 108 |
+
self.in_flight = 0
|
| 109 |
+
self.queued = 0
|
| 110 |
+
self.busy_refusals = 0
|
| 111 |
+
self.admitted = 0
|
| 112 |
+
self.limit = None
|
| 113 |
+
self.queue_seconds = None
|
| 114 |
+
|
| 115 |
+
def add(self, name, n):
|
| 116 |
+
with self.lock:
|
| 117 |
+
setattr(self, name, getattr(self, name) + n)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
_COUNTERS = _Counters()
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def status():
|
| 124 |
+
"""Snapshot of heavy request admission for a settings or health surface."""
|
| 125 |
+
c = _COUNTERS
|
| 126 |
+
with c.lock:
|
| 127 |
+
return {"limit": c.limit if c.limit is not None else _env_int("AIOS_HEAVY_CONCURRENCY", 2),
|
| 128 |
+
"queueSeconds": (c.queue_seconds if c.queue_seconds is not None
|
| 129 |
+
else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0)),
|
| 130 |
+
"inFlight": c.in_flight, "queued": c.queued,
|
| 131 |
+
"busyRefusals": c.busy_refusals, "admitted": c.admitted}
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _env_int(name, default):
|
| 135 |
+
try:
|
| 136 |
+
return max(1, int(os.environ.get(name) or default))
|
| 137 |
+
except ValueError:
|
| 138 |
+
return default
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def _env_float(name, default):
|
| 142 |
+
try:
|
| 143 |
+
return max(0.0, float(os.environ.get(name) or default))
|
| 144 |
+
except ValueError:
|
| 145 |
+
return default
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
async def _send_busy(send):
|
| 149 |
+
body = json.dumps({"error": {"code": BUSY_CODE, "message": BUSY_MESSAGE}}).encode()
|
| 150 |
+
await send({"type": "http.response.start", "status": 503,
|
| 151 |
+
"headers": [(b"content-type", b"application/json"),
|
| 152 |
+
(b"content-length", str(len(body)).encode()),
|
| 153 |
+
(b"retry-after", str(RETRY_AFTER_SECONDS).encode()),
|
| 154 |
+
(b"cache-control", b"no-store")]})
|
| 155 |
+
await send({"type": "http.response.body", "body": body, "more_body": False})
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
class HeavyGate:
|
| 159 |
+
"""Pure ASGI middleware. Heavy requests take a permit; light ones pass straight through."""
|
| 160 |
+
|
| 161 |
+
def __init__(self, app, concurrency=None, queue_seconds=None, counters=None):
|
| 162 |
+
self.app = app
|
| 163 |
+
self.concurrency = concurrency or _env_int("AIOS_HEAVY_CONCURRENCY", 2)
|
| 164 |
+
self.queue_seconds = (queue_seconds if queue_seconds is not None
|
| 165 |
+
else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0))
|
| 166 |
+
self.counters = counters or _COUNTERS
|
| 167 |
+
self.counters.limit, self.counters.queue_seconds = self.concurrency, self.queue_seconds
|
| 168 |
+
self._sem = None
|
| 169 |
+
self._loop = None
|
| 170 |
+
|
| 171 |
+
def _semaphore(self):
|
| 172 |
+
# A Semaphore binds to the first loop that waits on it. One per loop.
|
| 173 |
+
loop = asyncio.get_running_loop()
|
| 174 |
+
if self._sem is None or self._loop is not loop:
|
| 175 |
+
self._sem, self._loop = asyncio.Semaphore(self.concurrency), loop
|
| 176 |
+
return self._sem
|
| 177 |
+
|
| 178 |
+
async def __call__(self, scope, receive, send):
|
| 179 |
+
if scope.get("type") != "http":
|
| 180 |
+
return await self.app(scope, receive, send)
|
| 181 |
+
method, path = scope.get("method", ""), scope.get("path", "")
|
| 182 |
+
body_msgs = None
|
| 183 |
+
peek = None
|
| 184 |
+
if method == "POST" and path == MCP_PATH:
|
| 185 |
+
body_msgs, peek = await _read_body(receive, MCP_PEEK_LIMIT)
|
| 186 |
+
if peek is None: # The client left while sending the body.
|
| 187 |
+
return
|
| 188 |
+
if classify(method, path, scope.get("query_string", b""), peek) != "heavy":
|
| 189 |
+
return await self.app(scope, _replay(body_msgs, receive), send)
|
| 190 |
+
if body_msgs is None:
|
| 191 |
+
body_msgs, _ = await _read_body(receive, None)
|
| 192 |
+
if body_msgs is None:
|
| 193 |
+
return
|
| 194 |
+
await self._run_heavy(scope, body_msgs, receive, send)
|
| 195 |
+
|
| 196 |
+
async def _run_heavy(self, scope, body_msgs, receive, send):
|
| 197 |
+
sem = self._semaphore()
|
| 198 |
+
c = self.counters
|
| 199 |
+
c.add("queued", 1)
|
| 200 |
+
acquire = asyncio.ensure_future(sem.acquire())
|
| 201 |
+
gone = asyncio.ensure_future(_wait_disconnect(receive))
|
| 202 |
+
try:
|
| 203 |
+
done, _ = await asyncio.wait({acquire, gone}, timeout=self.queue_seconds,
|
| 204 |
+
return_when=asyncio.FIRST_COMPLETED)
|
| 205 |
+
finally:
|
| 206 |
+
c.add("queued", -1)
|
| 207 |
+
if not acquire.done():
|
| 208 |
+
acquire.cancel()
|
| 209 |
+
if not gone.done():
|
| 210 |
+
gone.cancel()
|
| 211 |
+
# A permit can land in the same tick as a timeout or disconnect. Hold it only if used.
|
| 212 |
+
got = acquire.done() and not acquire.cancelled() and acquire.exception() is None
|
| 213 |
+
if acquire not in done or gone in done:
|
| 214 |
+
if got:
|
| 215 |
+
sem.release()
|
| 216 |
+
if gone not in done:
|
| 217 |
+
c.add("busy_refusals", 1)
|
| 218 |
+
await _send_busy(send)
|
| 219 |
+
return
|
| 220 |
+
released = False
|
| 221 |
+
|
| 222 |
+
def release():
|
| 223 |
+
nonlocal released
|
| 224 |
+
if not released:
|
| 225 |
+
released = True
|
| 226 |
+
c.add("in_flight", -1)
|
| 227 |
+
sem.release()
|
| 228 |
+
|
| 229 |
+
async def send_wrapped(message):
|
| 230 |
+
await send(message)
|
| 231 |
+
if message.get("type") == "http.response.body" and not message.get("more_body"):
|
| 232 |
+
release()
|
| 233 |
+
|
| 234 |
+
c.add("in_flight", 1)
|
| 235 |
+
c.add("admitted", 1)
|
| 236 |
+
try:
|
| 237 |
+
await self.app(scope, _replay(body_msgs, receive), send_wrapped)
|
| 238 |
+
finally:
|
| 239 |
+
release()
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
async def _read_body(receive, limit):
|
| 243 |
+
"""Buffer the request body. Returns (messages, bytes or None if the client disconnected)."""
|
| 244 |
+
msgs, chunks, size = [], [], 0
|
| 245 |
+
while True:
|
| 246 |
+
message = await receive()
|
| 247 |
+
if message.get("type") == "http.disconnect":
|
| 248 |
+
return None, None
|
| 249 |
+
msgs.append(message)
|
| 250 |
+
chunk = message.get("body", b"") or b""
|
| 251 |
+
# Keep the chunk that crosses the limit, so an oversized body reads as oversized.
|
| 252 |
+
if limit is None or size <= limit:
|
| 253 |
+
chunks.append(chunk)
|
| 254 |
+
size += len(chunk)
|
| 255 |
+
if not message.get("more_body"):
|
| 256 |
+
return msgs, b"".join(chunks)
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
async def _wait_disconnect(receive):
|
| 260 |
+
while True:
|
| 261 |
+
message = await receive()
|
| 262 |
+
if message.get("type") == "http.disconnect":
|
| 263 |
+
return True
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def _replay(msgs, receive):
|
| 267 |
+
"""A receive that hands back the buffered body first, then defers to the real one."""
|
| 268 |
+
if not msgs:
|
| 269 |
+
return receive
|
| 270 |
+
pending = list(msgs)
|
| 271 |
+
|
| 272 |
+
async def replay():
|
| 273 |
+
if pending:
|
| 274 |
+
return pending.pop(0)
|
| 275 |
+
return await receive()
|
| 276 |
+
return replay
|
api/routes_customers.py
CHANGED
|
@@ -41,8 +41,9 @@ MODULE = "customer_data"
|
|
| 41 |
|
| 42 |
_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
|
| 43 |
|
| 44 |
-
#
|
| 45 |
-
|
|
|
|
| 46 |
_grid_assembly_lock = threading.Lock()
|
| 47 |
_grid_assembly_inflight = {}
|
| 48 |
_grid_assembly_enabled = True
|
|
|
|
| 41 |
|
| 42 |
_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
|
| 43 |
|
| 44 |
+
# Waiters share one cold build (about 4 min on 2 vCPU); a private rebuild per waiter multiplies it.
|
| 45 |
+
# Bounded under the client's 110 s read deadline; request_class caps how many heavy reads wait.
|
| 46 |
+
_GRID_ASSEMBLY_WAIT_SECONDS = 100.0
|
| 47 |
_grid_assembly_lock = threading.Lock()
|
| 48 |
_grid_assembly_inflight = {}
|
| 49 |
_grid_assembly_enabled = True
|
api/routes_sheet.py
CHANGED
|
@@ -2995,10 +2995,17 @@ def read_sheet(workbook_id: str, tabs: str = "",
|
|
| 2995 |
target = next((tab for tab in visible if tab["id"] == open_tab),
|
| 2996 |
visible[0] if visible else None)
|
| 2997 |
if target is not None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2998 |
target["cells"] = sheets.get_workbook_tab_cells(
|
| 2999 |
_store(session), workbook_id, target["id"], expected_rev=book.get("rev"))
|
| 3000 |
target.pop("cellCount", None)
|
| 3001 |
-
|
|
|
|
|
|
|
| 3002 |
return out
|
| 3003 |
except sheets.SheetRefusal as exc:
|
| 3004 |
raise _refuse(exc)
|
|
@@ -3154,6 +3161,8 @@ def read_sheet_refresh_history(workbook_id: str, session: Session = Depends(requ
|
|
| 3154 |
|
| 3155 |
@router.post("/sheets/{workbook_id}/refresh-history/{refresh_id}/retry")
|
| 3156 |
def retry_sheet_refresh(workbook_id: str, refresh_id: str, session: Session = Depends(require_session)):
|
|
|
|
|
|
|
| 3157 |
with _sheet_request_scope(session):
|
| 3158 |
return _refresh_sheet_scoped(workbook_id, session, retry_id=refresh_id)
|
| 3159 |
|
|
@@ -3198,13 +3207,22 @@ def read_sheet_tab(workbook_id: str, tab: str,
|
|
| 3198 |
raise _refuse(exc)
|
| 3199 |
|
| 3200 |
|
| 3201 |
-
def _serve_open_tab(session, book, target):
|
| 3202 |
-
"""Serve the selected tab and its dependency closure under one request's fresh walls.
|
|
|
|
|
|
|
|
|
|
| 3203 |
from core import sheets
|
| 3204 |
|
| 3205 |
def load_tab(tab_id):
|
| 3206 |
-
|
| 3207 |
expected_rev=book.get("rev"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3208 |
|
| 3209 |
with _sheet_request_scope(session):
|
| 3210 |
served = _read_tab_for(session, book, target, load_tab=load_tab)
|
|
@@ -3676,9 +3694,397 @@ def unshare_sheet(workbook_id: str, who: str, session: Session = Depends(require
|
|
| 3676 |
# `drill` with real resolution through the mirror seam; W49-T03 supplies formula evaluation.
|
| 3677 |
|
| 3678 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3679 |
@router.post("/sheets/{workbook_id}/refresh")
|
| 3680 |
-
def
|
| 3681 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3682 |
"""C2/R4: Refresh resolves only databases whose token or binding inputs changed.
|
| 3683 |
|
| 3684 |
Viewers refresh their own server cache. Only editors save the workbook document.
|
|
@@ -3782,6 +4188,16 @@ def _public_refresh_snapshot(snapshot):
|
|
| 3782 |
|
| 3783 |
def _refresh_sheet_scoped(workbook_id: str, session: Session, *, only=None, force=False,
|
| 3784 |
scope=None, retry_id=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3785 |
from core import sheets
|
| 3786 |
store = _store(session)
|
| 3787 |
tenant_keys = (getattr(store, "key", None), getattr(getattr(store, "tenant", None), "key", None))
|
|
|
|
| 2995 |
target = next((tab for tab in visible if tab["id"] == open_tab),
|
| 2996 |
visible[0] if visible else None)
|
| 2997 |
if target is not None:
|
| 2998 |
+
args = (str(tabs or ""), bool(opening), str(open_tab or ""), target["id"])
|
| 2999 |
+
cached = _open_cache_get(session, book, args)
|
| 3000 |
+
if cached is not None:
|
| 3001 |
+
out["opened"] = cached
|
| 3002 |
+
return out
|
| 3003 |
target["cells"] = sheets.get_workbook_tab_cells(
|
| 3004 |
_store(session), workbook_id, target["id"], expected_rev=book.get("rev"))
|
| 3005 |
target.pop("cellCount", None)
|
| 3006 |
+
databases = set()
|
| 3007 |
+
out["opened"] = _serve_open_tab(session, book, target, databases=databases)
|
| 3008 |
+
_open_cache_put(session, book, args, out["opened"], databases)
|
| 3009 |
return out
|
| 3010 |
except sheets.SheetRefusal as exc:
|
| 3011 |
raise _refuse(exc)
|
|
|
|
| 3161 |
|
| 3162 |
@router.post("/sheets/{workbook_id}/refresh-history/{refresh_id}/retry")
|
| 3163 |
def retry_sheet_refresh(workbook_id: str, refresh_id: str, session: Session = Depends(require_session)):
|
| 3164 |
+
if _refresh_desk_queues(session):
|
| 3165 |
+
return _queue_refresh(workbook_id, session, {"kind": "retry", "retryId": str(refresh_id)})
|
| 3166 |
with _sheet_request_scope(session):
|
| 3167 |
return _refresh_sheet_scoped(workbook_id, session, retry_id=refresh_id)
|
| 3168 |
|
|
|
|
| 3207 |
raise _refuse(exc)
|
| 3208 |
|
| 3209 |
|
| 3210 |
+
def _serve_open_tab(session, book, target, *, databases=None):
|
| 3211 |
+
"""Serve the selected tab and its dependency closure under one request's fresh walls.
|
| 3212 |
+
|
| 3213 |
+
`databases`, when given, collects every database a served or loaded tab binds.
|
| 3214 |
+
"""
|
| 3215 |
from core import sheets
|
| 3216 |
|
| 3217 |
def load_tab(tab_id):
|
| 3218 |
+
cells = sheets.get_workbook_tab_cells(_store(session), book["id"], tab_id,
|
| 3219 |
expected_rev=book.get("rev"))
|
| 3220 |
+
if databases is not None:
|
| 3221 |
+
databases.update(_bound_databases(cells))
|
| 3222 |
+
return cells
|
| 3223 |
+
|
| 3224 |
+
if databases is not None:
|
| 3225 |
+
databases.update(_bound_databases(target.get("cells")))
|
| 3226 |
|
| 3227 |
with _sheet_request_scope(session):
|
| 3228 |
served = _read_tab_for(session, book, target, load_tab=load_tab)
|
|
|
|
| 3694 |
# `drill` with real resolution through the mirror seam; W49-T03 supplies formula evaluation.
|
| 3695 |
|
| 3696 |
|
| 3697 |
+
# ββ the refresh desk: the web role queues a person's refresh and the sync worker runs it βββββββββ
|
| 3698 |
+
import time as _desk_time # noqa: E402
|
| 3699 |
+
|
| 3700 |
+
#: One store document per tenant holds the queued refreshes. `sheet_warm` drains it in the worker.
|
| 3701 |
+
REFRESH_REQUESTS_KEY = "sheet_refresh_requests_v1"
|
| 3702 |
+
#: A request still running after this long goes back in the queue once, then fails.
|
| 3703 |
+
REFRESH_STALE_SECONDS = 15 * 60
|
| 3704 |
+
REFRESH_MAX_ATTEMPTS = 2
|
| 3705 |
+
#: A finished request stays readable this long, so a polling client always finds its answer.
|
| 3706 |
+
REFRESH_KEEP_SECONDS = 3600
|
| 3707 |
+
_REFRESH_FAILED = {"code": "sheet_unresolved", "message": "This workbook could not be refreshed."}
|
| 3708 |
+
_REFRESH_TIMEOUT = {"code": "sheet_refresh_timeout",
|
| 3709 |
+
"message": "This refresh took too long. Try again in a few minutes."}
|
| 3710 |
+
|
| 3711 |
+
|
| 3712 |
+
def _refresh_desk_queues(session):
|
| 3713 |
+
"""True in the web role for a person's own session. Read per call, so a gate can patch the role."""
|
| 3714 |
+
import process_role
|
| 3715 |
+
if process_role.role() != "web":
|
| 3716 |
+
return False
|
| 3717 |
+
claims = getattr(session, "claims", None) or {}
|
| 3718 |
+
# An external grant narrows by claim and the worker rebuilds no claims, so it resolves here.
|
| 3719 |
+
return not any(str(key).startswith("external") and value for key, value in claims.items())
|
| 3720 |
+
|
| 3721 |
+
|
| 3722 |
+
def _refresh_body_args(body):
|
| 3723 |
+
"""The refresh arguments the route accepts, as one plain request entry."""
|
| 3724 |
+
if isinstance(body, dict) and "scope" in body:
|
| 3725 |
+
return {"kind": "scope", "scope": body["scope"]}
|
| 3726 |
+
wanted = body.get("cells") if isinstance(body, dict) else None
|
| 3727 |
+
if isinstance(wanted, list):
|
| 3728 |
+
return {"kind": "cells",
|
| 3729 |
+
"cells": sorted({str(item) for item in wanted if isinstance(item, str) and "!" in item})}
|
| 3730 |
+
return {"kind": "workbook"}
|
| 3731 |
+
|
| 3732 |
+
|
| 3733 |
+
def _refresh_desk_merge(old, new):
|
| 3734 |
+
"""One request that covers both, or None when the two cannot share a run."""
|
| 3735 |
+
if old == new:
|
| 3736 |
+
return old
|
| 3737 |
+
kinds = {old.get("kind"), new.get("kind")}
|
| 3738 |
+
if "retry" in kinds:
|
| 3739 |
+
return None
|
| 3740 |
+
if kinds == {"cells"}:
|
| 3741 |
+
return {"kind": "cells", "cells": sorted(set(old.get("cells") or ()) | set(new.get("cells") or ()))}
|
| 3742 |
+
if kinds == {"cells", "workbook"}:
|
| 3743 |
+
return {"kind": "workbook"}
|
| 3744 |
+
return {"kind": "scope", "scope": {"workbook": True}}
|
| 3745 |
+
|
| 3746 |
+
|
| 3747 |
+
def _refresh_desk_rows(doc, now):
|
| 3748 |
+
"""The live request rows. A finished row older than REFRESH_KEEP_SECONDS falls out."""
|
| 3749 |
+
rows = (doc or {}).get("requests") if isinstance(doc, dict) else None
|
| 3750 |
+
return {key: dict(row) for key, row in (rows or {}).items() if isinstance(row, dict)
|
| 3751 |
+
and not (row.get("state") in ("done", "failed")
|
| 3752 |
+
and now - float(row.get("finishedTs") or 0) > REFRESH_KEEP_SECONDS)}
|
| 3753 |
+
|
| 3754 |
+
|
| 3755 |
+
def _refresh_desk_order(row):
|
| 3756 |
+
return (float(row.get("requestedTs") or 0), str(row.get("id") or ""))
|
| 3757 |
+
|
| 3758 |
+
|
| 3759 |
+
def _desk_iso(ts):
|
| 3760 |
+
from datetime import datetime, timezone
|
| 3761 |
+
return datetime.fromtimestamp(ts, timezone.utc).replace(microsecond=0).isoformat()
|
| 3762 |
+
|
| 3763 |
+
|
| 3764 |
+
def _refresh_desk_enqueue(store, tenant, workbook, reader, args, now=None, path=None):
|
| 3765 |
+
"""Queue one request, or fold it into the same reader's queued request for this workbook."""
|
| 3766 |
+
now = _desk_time.time() if now is None else now
|
| 3767 |
+
made = {}
|
| 3768 |
+
|
| 3769 |
+
def apply(current):
|
| 3770 |
+
rows = _refresh_desk_rows(current, now)
|
| 3771 |
+
for row in sorted(rows.values(), key=_refresh_desk_order):
|
| 3772 |
+
if (row.get("state") == "queued" and row.get("workbook") == workbook
|
| 3773 |
+
and row.get("reader") == reader):
|
| 3774 |
+
merged = _refresh_desk_merge(row.get("args") or {}, args)
|
| 3775 |
+
if merged is not None:
|
| 3776 |
+
row["args"] = merged
|
| 3777 |
+
row["coalesced"] = int(row.get("coalesced") or 0) + 1
|
| 3778 |
+
rows[row["id"]] = row
|
| 3779 |
+
made["row"] = dict(row)
|
| 3780 |
+
return {"requests": rows}
|
| 3781 |
+
row = {"id": uuid.uuid4().hex, "tenant": tenant, "workbook": workbook, "reader": reader,
|
| 3782 |
+
"path": str(path or workbook), "args": args, "state": "queued", "attempts": 0,
|
| 3783 |
+
"requestedTs": now,
|
| 3784 |
+
"requestedAt": _desk_iso(now), "finishedAt": None, "error": None}
|
| 3785 |
+
rows[row["id"]] = row
|
| 3786 |
+
made["row"] = dict(row)
|
| 3787 |
+
return {"requests": rows}
|
| 3788 |
+
|
| 3789 |
+
store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
|
| 3790 |
+
return made["row"]
|
| 3791 |
+
|
| 3792 |
+
|
| 3793 |
+
def _refresh_desk_claim(store, now=None):
|
| 3794 |
+
"""Mark the oldest queued request running and return it. None while one is already running."""
|
| 3795 |
+
now = _desk_time.time() if now is None else now
|
| 3796 |
+
claimed = {}
|
| 3797 |
+
|
| 3798 |
+
def stale(row):
|
| 3799 |
+
return row.get("state") == "running" and now - float(row.get("startedTs") or 0) > REFRESH_STALE_SECONDS
|
| 3800 |
+
|
| 3801 |
+
# Read first: an idle poll must never write the store.
|
| 3802 |
+
rows = _refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now).values()
|
| 3803 |
+
busy = any(row.get("state") == "running" and not stale(row) for row in rows)
|
| 3804 |
+
if not any(stale(row) for row in rows) and (busy or not any(row.get("state") == "queued" for row in rows)):
|
| 3805 |
+
return None
|
| 3806 |
+
|
| 3807 |
+
def apply(current):
|
| 3808 |
+
rows = _refresh_desk_rows(current, now)
|
| 3809 |
+
for row in rows.values():
|
| 3810 |
+
if stale(row):
|
| 3811 |
+
if int(row.get("attempts") or 0) < REFRESH_MAX_ATTEMPTS:
|
| 3812 |
+
row["state"] = "queued"
|
| 3813 |
+
else:
|
| 3814 |
+
row.update(state="failed", finishedTs=now, finishedAt=_desk_iso(now),
|
| 3815 |
+
error=dict(_REFRESH_TIMEOUT))
|
| 3816 |
+
if not any(row.get("state") == "running" for row in rows.values()):
|
| 3817 |
+
queued = sorted((row for row in rows.values() if row.get("state") == "queued"),
|
| 3818 |
+
key=_refresh_desk_order)
|
| 3819 |
+
if queued:
|
| 3820 |
+
row = queued[0]
|
| 3821 |
+
row.update(state="running", startedTs=now, startedAt=_desk_iso(now),
|
| 3822 |
+
attempts=int(row.get("attempts") or 0) + 1)
|
| 3823 |
+
claimed["row"] = dict(row)
|
| 3824 |
+
return {"requests": rows}
|
| 3825 |
+
|
| 3826 |
+
store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
|
| 3827 |
+
return claimed.get("row")
|
| 3828 |
+
|
| 3829 |
+
|
| 3830 |
+
def _refresh_desk_finish(store, request_id, error=None, now=None):
|
| 3831 |
+
"""Mark one running request done, or failed with a public `{code, message}`."""
|
| 3832 |
+
now = _desk_time.time() if now is None else now
|
| 3833 |
+
|
| 3834 |
+
def apply(current):
|
| 3835 |
+
rows = _refresh_desk_rows(current, now)
|
| 3836 |
+
row = rows.get(request_id)
|
| 3837 |
+
if row is not None and row.get("state") == "running":
|
| 3838 |
+
row.update(state="failed" if error else "done", finishedTs=now,
|
| 3839 |
+
finishedAt=_desk_iso(now), error=dict(error) if error else None)
|
| 3840 |
+
return {"requests": rows}
|
| 3841 |
+
|
| 3842 |
+
store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
|
| 3843 |
+
|
| 3844 |
+
|
| 3845 |
+
def _refresh_desk_entry(store, workbook_id, reader, request_id):
|
| 3846 |
+
"""The caller's own request row, or None for an unknown id or another person's request."""
|
| 3847 |
+
doc = store.get(REFRESH_REQUESTS_KEY, fresh=True) or {}
|
| 3848 |
+
row = ((doc.get("requests") or {}) if isinstance(doc, dict) else {}).get(str(request_id or ""))
|
| 3849 |
+
if (not isinstance(row, dict) or not reader or row.get("reader") != reader
|
| 3850 |
+
or str(workbook_id) not in (row.get("workbook"), row.get("path"))):
|
| 3851 |
+
return None
|
| 3852 |
+
return row
|
| 3853 |
+
|
| 3854 |
+
|
| 3855 |
+
def _refresh_desk_pending(store):
|
| 3856 |
+
"""True when the document holds a queued or running request."""
|
| 3857 |
+
doc = store.get(REFRESH_REQUESTS_KEY, fresh=True) or {}
|
| 3858 |
+
rows = (doc.get("requests") or {}) if isinstance(doc, dict) else {}
|
| 3859 |
+
return any(isinstance(row, dict) and row.get("state") in ("queued", "running")
|
| 3860 |
+
for row in rows.values())
|
| 3861 |
+
|
| 3862 |
+
|
| 3863 |
+
def _queue_refresh(workbook_id, session, args):
|
| 3864 |
+
"""Check the caller and workbook as the in-process refresh does, then queue it. Answers 202."""
|
| 3865 |
+
from core import sheets
|
| 3866 |
+
from fastapi.responses import JSONResponse
|
| 3867 |
+
store = _store(session)
|
| 3868 |
+
tenant_keys = (getattr(store, "key", None), getattr(getattr(store, "tenant", None), "key", None))
|
| 3869 |
+
if any(key and key != session.tenant for key in tenant_keys):
|
| 3870 |
+
raise err(409, "tenant_mismatch", "The caller and workbook tenants differ.")
|
| 3871 |
+
try:
|
| 3872 |
+
book = sheets.get_workbook(store, workbook_id)
|
| 3873 |
+
_require_reader(session, book)
|
| 3874 |
+
sheets._refresh_caller(store, book, session)
|
| 3875 |
+
if args.get("kind") == "scope":
|
| 3876 |
+
_refresh_scope_cells(book, args.get("scope"))
|
| 3877 |
+
except sheets.SheetRefusal as exc:
|
| 3878 |
+
raise _refuse(exc)
|
| 3879 |
+
row = _refresh_desk_enqueue(store, session.tenant, str(book.get("id") or workbook_id),
|
| 3880 |
+
session.uname, args, path=workbook_id)
|
| 3881 |
+
return JSONResponse(status_code=202, content={"queued": True, "requestId": row["id"],
|
| 3882 |
+
"state": row["state"]})
|
| 3883 |
+
|
| 3884 |
+
|
| 3885 |
+
def _refresh_desk_error(exc):
|
| 3886 |
+
"""The public `{code, message}` a failed request carries."""
|
| 3887 |
+
from core import sheets
|
| 3888 |
+
if isinstance(exc, sheets.SheetRefusal):
|
| 3889 |
+
return {"code": str(exc.code), "message": str(exc.message or _REFRESH_FAILED["message"])}
|
| 3890 |
+
if isinstance(exc, HTTPException):
|
| 3891 |
+
info = ((exc.detail or {}).get("error") if isinstance(exc.detail, dict) else None) or {}
|
| 3892 |
+
return {"code": str(info.get("code") or _REFRESH_FAILED["code"]),
|
| 3893 |
+
"message": str(info.get("message") or _REFRESH_FAILED["message"])}
|
| 3894 |
+
return dict(_REFRESH_FAILED)
|
| 3895 |
+
|
| 3896 |
+
|
| 3897 |
+
def _refresh_desk_run(session, row):
|
| 3898 |
+
"""Run one claimed request as its reader through the in-process refresh resolver."""
|
| 3899 |
+
from core import sheets
|
| 3900 |
+
args = row.get("args") or {}
|
| 3901 |
+
workbook = row["workbook"]
|
| 3902 |
+
if args.get("kind") == "scope":
|
| 3903 |
+
return sheets.refresh_scope(session.runtime, workbook, args.get("scope"), session)
|
| 3904 |
+
with _sheet_request_scope(session):
|
| 3905 |
+
if args.get("kind") == "retry":
|
| 3906 |
+
return _refresh_sheet_scoped(workbook, session, retry_id=args.get("retryId"))
|
| 3907 |
+
only = set(args.get("cells") or ()) if args.get("kind") == "cells" else None
|
| 3908 |
+
return _refresh_sheet_scoped(workbook, session, only=only)
|
| 3909 |
+
|
| 3910 |
+
|
| 3911 |
+
# ββ the opening cache: a repeat open of the same tab at the same state is served from memory βββββ
|
| 3912 |
+
#: Entries kept, least recently used out first.
|
| 3913 |
+
OPEN_CACHE_ENTRIES = 64
|
| 3914 |
+
#: A cached open older than this recomputes. It bounds what a key cannot see, such as a write
|
| 3915 |
+
#: another process made to a store whose revision counter is per process.
|
| 3916 |
+
OPEN_CACHE_TTL_SECONDS = 120.0
|
| 3917 |
+
_OPEN_CACHE = OrderedDict()
|
| 3918 |
+
_OPEN_CACHE_LOCK = threading.Lock()
|
| 3919 |
+
#: User record keys that change on sign in and never change what the reader may see.
|
| 3920 |
+
_OPEN_CACHE_USER_NOISE = frozenset(("last_login", "updated_at", "epoch", "avatar", "whatsapp"))
|
| 3921 |
+
|
| 3922 |
+
|
| 3923 |
+
def _open_cache_enabled():
|
| 3924 |
+
"""On in the web role only. `AIOS_SHEET_OPEN_CACHE=0` turns it off there too."""
|
| 3925 |
+
import process_role
|
| 3926 |
+
return (process_role.role() == "web"
|
| 3927 |
+
and os.environ.get("AIOS_SHEET_OPEN_CACHE", "1").strip() != "0")
|
| 3928 |
+
|
| 3929 |
+
|
| 3930 |
+
def _install_generation():
|
| 3931 |
+
"""The serving mirror's install generation, or None when it cannot be read."""
|
| 3932 |
+
try:
|
| 3933 |
+
from harness import datastore
|
| 3934 |
+
return datastore.mirror_generation()
|
| 3935 |
+
except Exception: # noqa: BLE001
|
| 3936 |
+
return None
|
| 3937 |
+
|
| 3938 |
+
|
| 3939 |
+
def _doc_token(store, key):
|
| 3940 |
+
"""A cheap change token for one store document, or None when none can be read."""
|
| 3941 |
+
from core.sheet_binding import fingerprint
|
| 3942 |
+
revision = getattr(store, "revision", None)
|
| 3943 |
+
try:
|
| 3944 |
+
if callable(revision):
|
| 3945 |
+
return "rev:" + fingerprint(revision(key))
|
| 3946 |
+
return "doc:" + fingerprint(store.get(key) or {})
|
| 3947 |
+
except Exception: # noqa: BLE001
|
| 3948 |
+
return None
|
| 3949 |
+
|
| 3950 |
+
|
| 3951 |
+
def _bound_databases(cells):
|
| 3952 |
+
"""Every database name a tab's bindings mention, related databases included."""
|
| 3953 |
+
found = set()
|
| 3954 |
+
|
| 3955 |
+
def walk(value):
|
| 3956 |
+
if isinstance(value, dict):
|
| 3957 |
+
for key, item in value.items():
|
| 3958 |
+
if key == "database" and isinstance(item, str) and item:
|
| 3959 |
+
found.add(item)
|
| 3960 |
+
else:
|
| 3961 |
+
walk(item)
|
| 3962 |
+
elif isinstance(value, list):
|
| 3963 |
+
for item in value:
|
| 3964 |
+
walk(item)
|
| 3965 |
+
|
| 3966 |
+
for cell in (cells or {}).values() if isinstance(cells, dict) else ():
|
| 3967 |
+
if isinstance(cell, dict) and cell.get("binding"):
|
| 3968 |
+
walk(cell.get("binding"))
|
| 3969 |
+
return found
|
| 3970 |
+
|
| 3971 |
+
|
| 3972 |
+
def _open_cache_key(session, book, args):
|
| 3973 |
+
"""Everything an opened tab's body depends on for one reader, or None when it cannot be keyed."""
|
| 3974 |
+
from core import shares, sheets
|
| 3975 |
+
from core.sheet_binding import fingerprint
|
| 3976 |
+
store = _store(session)
|
| 3977 |
+
workbook = str(book.get("id") or "")
|
| 3978 |
+
refresh_doc = _doc_token(store, sheets._scope_cache_key(workbook))
|
| 3979 |
+
grants_doc = _doc_token(store, shares.SHARES_KEY)
|
| 3980 |
+
if not workbook or not session.uname or refresh_doc is None or grants_doc is None:
|
| 3981 |
+
return None
|
| 3982 |
+
policy = {key: value for key, value in (session.user or {}).items()
|
| 3983 |
+
if key not in _OPEN_CACHE_USER_NOISE}
|
| 3984 |
+
return (str(session.tenant), workbook, book.get("rev"), session.uname, fingerprint(policy),
|
| 3985 |
+
refresh_doc, grants_doc, json.dumps(_install_generation(), default=str),
|
| 3986 |
+
_sheet_today(session), tuple(args))
|
| 3987 |
+
|
| 3988 |
+
|
| 3989 |
+
def _open_cache_sources(session, databases):
|
| 3990 |
+
"""`{database: change token}`, or None when any token cannot be read."""
|
| 3991 |
+
from core import sheet_binding
|
| 3992 |
+
out = {}
|
| 3993 |
+
for database in sorted(databases):
|
| 3994 |
+
try:
|
| 3995 |
+
import harness.mirror_tokens # noqa: F401
|
| 3996 |
+
out[database] = sheet_binding.database_change_token(_store(session), database)
|
| 3997 |
+
except Exception: # noqa: BLE001
|
| 3998 |
+
return None
|
| 3999 |
+
if not out[database]:
|
| 4000 |
+
return None
|
| 4001 |
+
return out
|
| 4002 |
+
|
| 4003 |
+
|
| 4004 |
+
def _open_body_transient(body):
|
| 4005 |
+
"""True when a bound cell holds a busy server refusal, which must never stick in the cache."""
|
| 4006 |
+
for cell in (((body or {}).get("tab") or {}).get("cells") or {}).values():
|
| 4007 |
+
refusal = (cell or {}).get("refusal") or {} if isinstance(cell, dict) else {}
|
| 4008 |
+
if (isinstance(cell, dict) and cell.get("kind") == "bound" and cell.get("state") == "refused"
|
| 4009 |
+
and not _refusal_authored(refusal.get("code"), refusal.get("status"))):
|
| 4010 |
+
return True
|
| 4011 |
+
return False
|
| 4012 |
+
|
| 4013 |
+
|
| 4014 |
+
def _open_cache_get(session, book, args):
|
| 4015 |
+
"""A deep copy of the cached opened tab for this reader at this state, or None."""
|
| 4016 |
+
from copy import deepcopy
|
| 4017 |
+
if not _open_cache_enabled():
|
| 4018 |
+
return None
|
| 4019 |
+
key = _open_cache_key(session, book, args)
|
| 4020 |
+
if key is None:
|
| 4021 |
+
return None
|
| 4022 |
+
with _OPEN_CACHE_LOCK:
|
| 4023 |
+
entry = _OPEN_CACHE.get(key)
|
| 4024 |
+
if entry is not None and _desk_time.monotonic() - entry["at"] > OPEN_CACHE_TTL_SECONDS:
|
| 4025 |
+
_OPEN_CACHE.pop(key, None)
|
| 4026 |
+
entry = None
|
| 4027 |
+
if entry is not None:
|
| 4028 |
+
_OPEN_CACHE.move_to_end(key)
|
| 4029 |
+
if entry is None:
|
| 4030 |
+
return None
|
| 4031 |
+
if _open_cache_sources(session, entry["sources"]) != entry["sources"]:
|
| 4032 |
+
with _OPEN_CACHE_LOCK:
|
| 4033 |
+
_OPEN_CACHE.pop(key, None)
|
| 4034 |
+
return None
|
| 4035 |
+
return deepcopy(entry["body"])
|
| 4036 |
+
|
| 4037 |
+
|
| 4038 |
+
def _open_cache_put(session, book, args, body, databases):
|
| 4039 |
+
"""Keep one served opened tab under the key computed after the serve."""
|
| 4040 |
+
from copy import deepcopy
|
| 4041 |
+
if not _open_cache_enabled() or not isinstance(body, dict) or _open_body_transient(body):
|
| 4042 |
+
return
|
| 4043 |
+
key = _open_cache_key(session, book, args)
|
| 4044 |
+
sources = _open_cache_sources(session, databases) if key is not None else None
|
| 4045 |
+
if sources is None:
|
| 4046 |
+
return
|
| 4047 |
+
with _OPEN_CACHE_LOCK:
|
| 4048 |
+
_OPEN_CACHE[key] = {"body": deepcopy(body), "sources": sources, "at": _desk_time.monotonic()}
|
| 4049 |
+
_OPEN_CACHE.move_to_end(key)
|
| 4050 |
+
while len(_OPEN_CACHE) > OPEN_CACHE_ENTRIES:
|
| 4051 |
+
_OPEN_CACHE.popitem(last=False)
|
| 4052 |
+
|
| 4053 |
+
|
| 4054 |
+
def _open_cache_drop(tenant, workbook_id):
|
| 4055 |
+
"""Forget every cached open of one workbook in this process."""
|
| 4056 |
+
with _OPEN_CACHE_LOCK:
|
| 4057 |
+
for key in [key for key in _OPEN_CACHE if key[0] == str(tenant) and key[1] == str(workbook_id)]:
|
| 4058 |
+
_OPEN_CACHE.pop(key, None)
|
| 4059 |
+
|
| 4060 |
+
|
| 4061 |
@router.post("/sheets/{workbook_id}/refresh")
|
| 4062 |
+
def refresh_sheet_door(workbook_id: str, body: dict = Body(default=None),
|
| 4063 |
+
session: Session = Depends(require_session)):
|
| 4064 |
+
"""The HTTP refresh door. The web role queues the work for the sync worker and answers 202."""
|
| 4065 |
+
if _refresh_desk_queues(session):
|
| 4066 |
+
return _queue_refresh(workbook_id, session, _refresh_body_args(body))
|
| 4067 |
+
return refresh_sheet(workbook_id, body, session=session)
|
| 4068 |
+
|
| 4069 |
+
|
| 4070 |
+
@router.get("/sheets/{workbook_id}/refresh-status")
|
| 4071 |
+
def refresh_sheet_status(workbook_id: str, requestId: str = "",
|
| 4072 |
+
session: Session = Depends(require_session)):
|
| 4073 |
+
"""One queued refresh's state, readable only by the person who asked for it."""
|
| 4074 |
+
entry = _refresh_desk_entry(_store(session), workbook_id, session.uname, requestId)
|
| 4075 |
+
if entry is None:
|
| 4076 |
+
raise err(404, "sheet_refresh_missing", "That refresh is unavailable.")
|
| 4077 |
+
if entry.get("state") in ("done", "failed"):
|
| 4078 |
+
_open_cache_drop(session.tenant, workbook_id)
|
| 4079 |
+
out = {"requestId": requestId, "state": entry.get("state"),
|
| 4080 |
+
"requestedAt": entry.get("requestedAt"), "finishedAt": entry.get("finishedAt")}
|
| 4081 |
+
if entry.get("state") == "failed":
|
| 4082 |
+
out["error"] = entry.get("error") or {"code": "sheet_unresolved",
|
| 4083 |
+
"message": "This workbook could not be refreshed."}
|
| 4084 |
+
return out
|
| 4085 |
+
|
| 4086 |
+
|
| 4087 |
+
def refresh_sheet(workbook_id: str, body: dict = None, session: Session = None):
|
| 4088 |
"""C2/R4: Refresh resolves only databases whose token or binding inputs changed.
|
| 4089 |
|
| 4090 |
Viewers refresh their own server cache. Only editors save the workbook document.
|
|
|
|
| 4188 |
|
| 4189 |
def _refresh_sheet_scoped(workbook_id: str, session: Session, *, only=None, force=False,
|
| 4190 |
scope=None, retry_id=None):
|
| 4191 |
+
try:
|
| 4192 |
+
return _refresh_sheet_scoped_inner(workbook_id, session, only=only, force=force,
|
| 4193 |
+
scope=scope, retry_id=retry_id)
|
| 4194 |
+
finally:
|
| 4195 |
+
# A refresh moves stored answers, so this process forgets its cached opens of the workbook.
|
| 4196 |
+
_open_cache_drop(session.tenant, workbook_id)
|
| 4197 |
+
|
| 4198 |
+
|
| 4199 |
+
def _refresh_sheet_scoped_inner(workbook_id: str, session: Session, *, only=None, force=False,
|
| 4200 |
+
scope=None, retry_id=None):
|
| 4201 |
from core import sheets
|
| 4202 |
store = _store(session)
|
| 4203 |
tenant_keys = (getattr(store, "key", None), getattr(getattr(store, "tenant", None), "key", None))
|
api/sheet_warm.py
CHANGED
|
@@ -330,8 +330,111 @@ def _on_exit():
|
|
| 330 |
_limit(progress["reason"], progress["tenant"], progress["left"], "shutdown")
|
| 331 |
|
| 332 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 333 |
def start():
|
| 334 |
"""Start the one "sheet-warm" daemon thread when enabled. Returns it, or None."""
|
|
|
|
|
|
|
| 335 |
if not enabled():
|
| 336 |
print("[sheet-warm] off: AIOS_SHEET_WARM=0", flush=True)
|
| 337 |
return None
|
|
|
|
| 330 |
_limit(progress["reason"], progress["tenant"], progress["left"], "shutdown")
|
| 331 |
|
| 332 |
|
| 333 |
+
# ββ the refresh desk: person refreshes the web role queued, run here one at a time ββββββββββββββ
|
| 334 |
+
#: Seconds between two reads of each tenant's request document.
|
| 335 |
+
DESK_POLL_SECONDS = 5.0
|
| 336 |
+
_DESK = {"thread": None, "seen": {}}
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def _desk_tenants():
|
| 340 |
+
# Active tenants only: an inactive one has no runtime and would fail every 5 s.
|
| 341 |
+
import tenant_scan
|
| 342 |
+
from harness import runtime
|
| 343 |
+
return [slug for slug, rt in tenant_scan.live_tenants(runtime) if rt is not None]
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
def _desk_skip(rt, tenant):
|
| 347 |
+
"""True when a cross process revision proves the document did not move since an empty read."""
|
| 348 |
+
seen = _DESK["seen"].get(tenant)
|
| 349 |
+
if seen is None:
|
| 350 |
+
return False
|
| 351 |
+
try:
|
| 352 |
+
token = rt.revision(routes_sheet.REFRESH_REQUESTS_KEY)
|
| 353 |
+
except Exception: # noqa: BLE001
|
| 354 |
+
return False
|
| 355 |
+
text = token.get("token") if isinstance(token, dict) else token
|
| 356 |
+
# Only the Postgres revision sees another process's write. Any other token reads the document.
|
| 357 |
+
return str(text or "").startswith("pg:") and token == seen
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def desk_run_one(rt, tenant):
|
| 361 |
+
"""Claim and run the oldest queued refresh of one tenant. Returns True when one ran."""
|
| 362 |
+
row = routes_sheet._refresh_desk_claim(rt)
|
| 363 |
+
if row is None:
|
| 364 |
+
return False
|
| 365 |
+
started = perf_counter()
|
| 366 |
+
error = None
|
| 367 |
+
try:
|
| 368 |
+
book = sheets.get_workbook(rt, row["workbook"])
|
| 369 |
+
rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt)
|
| 370 |
+
session = reader_session(rt, tenant, row.get("reader"), book, rec)
|
| 371 |
+
if session is None:
|
| 372 |
+
error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
|
| 373 |
+
else:
|
| 374 |
+
routes_sheet._refresh_desk_run(session, row)
|
| 375 |
+
except Exception as exc: # noqa: BLE001
|
| 376 |
+
error = routes_sheet._refresh_desk_error(exc)
|
| 377 |
+
routes_sheet._refresh_desk_finish(rt, row["id"], error)
|
| 378 |
+
print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} "
|
| 379 |
+
f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}"
|
| 380 |
+
+ (f" code={error['code']}" if error else ""), flush=True)
|
| 381 |
+
return True
|
| 382 |
+
|
| 383 |
+
|
| 384 |
+
def desk_drain(tenant):
|
| 385 |
+
"""Run every queued refresh of one tenant, oldest first. Returns how many ran."""
|
| 386 |
+
from harness import runtime
|
| 387 |
+
rt = runtime.get_runtime(tenant)
|
| 388 |
+
if _desk_skip(rt, tenant):
|
| 389 |
+
return 0
|
| 390 |
+
# The revision is taken BEFORE the reads, so a request queued between them still moves it.
|
| 391 |
+
try:
|
| 392 |
+
before = rt.revision(routes_sheet.REFRESH_REQUESTS_KEY)
|
| 393 |
+
except Exception: # noqa: BLE001
|
| 394 |
+
before = None
|
| 395 |
+
ran = 0
|
| 396 |
+
while not _STOP.is_set() and desk_run_one(rt, tenant):
|
| 397 |
+
ran += 1
|
| 398 |
+
if not ran and before is not None and not routes_sheet._refresh_desk_pending(rt):
|
| 399 |
+
_DESK["seen"][tenant] = before
|
| 400 |
+
else:
|
| 401 |
+
_DESK["seen"].pop(tenant, None)
|
| 402 |
+
return ran
|
| 403 |
+
|
| 404 |
+
|
| 405 |
+
def _desk_loop():
|
| 406 |
+
while not _STOP.is_set():
|
| 407 |
+
try:
|
| 408 |
+
tenants = _desk_tenants()
|
| 409 |
+
except Exception as exc: # noqa: BLE001
|
| 410 |
+
print(f"[sheet-desk] tenant list failed {type(exc).__name__}: {exc}", flush=True)
|
| 411 |
+
tenants = []
|
| 412 |
+
for tenant in tenants:
|
| 413 |
+
if _STOP.is_set():
|
| 414 |
+
return
|
| 415 |
+
try:
|
| 416 |
+
desk_drain(tenant)
|
| 417 |
+
except Exception as exc: # noqa: BLE001
|
| 418 |
+
print(f"[sheet-desk] tenant={tenant} failed {type(exc).__name__}: {exc}", flush=True)
|
| 419 |
+
_STOP.wait(DESK_POLL_SECONDS)
|
| 420 |
+
|
| 421 |
+
|
| 422 |
+
def start_desk():
|
| 423 |
+
"""Start the one "sheet-refresh-desk" thread where heavy jobs run. Returns it, or None."""
|
| 424 |
+
with _LOCK:
|
| 425 |
+
thread = _DESK["thread"]
|
| 426 |
+
if thread is not None and thread.is_alive():
|
| 427 |
+
return thread
|
| 428 |
+
import process_role
|
| 429 |
+
thread = process_role.start("sheet-refresh-desk", "heavy", _desk_loop)
|
| 430 |
+
_DESK["thread"] = thread
|
| 431 |
+
return thread
|
| 432 |
+
|
| 433 |
+
|
| 434 |
def start():
|
| 435 |
"""Start the one "sheet-warm" daemon thread when enabled. Returns it, or None."""
|
| 436 |
+
# Person refreshes never depend on AIOS_SHEET_WARM, so the desk starts first.
|
| 437 |
+
start_desk()
|
| 438 |
if not enabled():
|
| 439 |
print("[sheet-warm] off: AIOS_SHEET_WARM=0", flush=True)
|
| 440 |
return None
|
platform/harness/amazon_store.py
CHANGED
|
@@ -2863,8 +2863,8 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
|
|
| 2863 |
def part_max_bytes(self):
|
| 2864 |
return MIRROR_PART_MAX_BYTES
|
| 2865 |
|
| 2866 |
-
def _install_snapshot(self, con, manifest, rows, paths):
|
| 2867 |
-
result = super()._install_snapshot(con, manifest, rows, paths)
|
| 2868 |
_note_published() # C2: AFTER_SYNC runs only for a pass that published.
|
| 2869 |
return result
|
| 2870 |
|
|
@@ -2891,8 +2891,8 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
|
|
| 2891 |
_assert_personal_storage(con, name)
|
| 2892 |
return super().persist_mirror(tenant_key, rt, con, changed_tables, report=report, log=log, **kwargs)
|
| 2893 |
|
| 2894 |
-
def _install_snapshot(self, con, manifest, rows, paths):
|
| 2895 |
-
super()._install_snapshot(con, manifest, rows, paths)
|
| 2896 |
refresh_sku_keywords(con)
|
| 2897 |
|
| 2898 |
def _publication_holds(self, report, waiting):
|
|
|
|
| 2863 |
def part_max_bytes(self):
|
| 2864 |
return MIRROR_PART_MAX_BYTES
|
| 2865 |
|
| 2866 |
+
def _install_snapshot(self, con, manifest, rows, paths, only=None):
|
| 2867 |
+
result = super()._install_snapshot(con, manifest, rows, paths, only=only)
|
| 2868 |
_note_published() # C2: AFTER_SYNC runs only for a pass that published.
|
| 2869 |
return result
|
| 2870 |
|
|
|
|
| 2891 |
_assert_personal_storage(con, name)
|
| 2892 |
return super().persist_mirror(tenant_key, rt, con, changed_tables, report=report, log=log, **kwargs)
|
| 2893 |
|
| 2894 |
+
def _install_snapshot(self, con, manifest, rows, paths, only=None):
|
| 2895 |
+
super()._install_snapshot(con, manifest, rows, paths, only=only)
|
| 2896 |
refresh_sku_keywords(con)
|
| 2897 |
|
| 2898 |
def _publication_holds(self, report, waiting):
|
platform/harness/mirror_store.py
CHANGED
|
@@ -72,6 +72,14 @@ def _notify_publish(connector, tenant_key, manifest, rows, paths, log=print):
|
|
| 72 |
log(f"{connector} mirror restore hand-off failed ({type(exc).__name__}: {exc})")
|
| 73 |
|
| 74 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
|
| 76 |
"""Install a snapshot another process already published. Returns False if it is already resident."""
|
| 77 |
store = for_connector(connector)
|
|
@@ -80,10 +88,11 @@ def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
|
|
| 80 |
with store.lock:
|
| 81 |
con, _path = store._connect(tenant_key, rt)
|
| 82 |
try:
|
| 83 |
-
|
| 84 |
-
|
| 85 |
return False
|
| 86 |
-
|
|
|
|
| 87 |
newest = _record_newest(con, store.connector)
|
| 88 |
finally:
|
| 89 |
con.close()
|
|
@@ -647,13 +656,18 @@ class MirrorStore:
|
|
| 647 |
return paths, sorted(refused)
|
| 648 |
|
| 649 |
|
| 650 |
-
def _install_snapshot(self, con, manifest, rows, paths):
|
| 651 |
-
"""Commit one source's tables, sync decisions and identity together.
|
|
|
|
|
|
|
|
|
|
| 652 |
con.execute("BEGIN TRANSACTION")
|
| 653 |
try:
|
| 654 |
for name in self._tables(con) - set(paths):
|
| 655 |
con.execute(f"DROP TABLE {_quoted_identifier(name)}")
|
| 656 |
for name, files in paths.items():
|
|
|
|
|
|
|
| 657 |
con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
|
| 658 |
f"SELECT * FROM {_parquet_source(files)}")
|
| 659 |
old_rows = self._state_rows(con)
|
|
|
|
| 72 |
log(f"{connector} mirror restore hand-off failed ({type(exc).__name__}: {exc})")
|
| 73 |
|
| 74 |
|
| 75 |
+
def changed_tables(previous, manifest, resident, paths):
|
| 76 |
+
"""Tables a hand-off must rebuild: new, missing locally, or with different stored content."""
|
| 77 |
+
before = (previous or {}).get("tables") or {}
|
| 78 |
+
after = (manifest or {}).get("tables") or {}
|
| 79 |
+
return {name for name in paths
|
| 80 |
+
if name not in resident or name not in before or name not in after or before[name] != after[name]}
|
| 81 |
+
|
| 82 |
+
|
| 83 |
def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
|
| 84 |
"""Install a snapshot another process already published. Returns False if it is already resident."""
|
| 85 |
store = for_connector(connector)
|
|
|
|
| 88 |
with store.lock:
|
| 89 |
con, _path = store._connect(tenant_key, rt)
|
| 90 |
try:
|
| 91 |
+
previous = datastore._publication(con, store.connector)
|
| 92 |
+
if manifest.get("snapshot_id") and previous.get("snapshot_id") == manifest["snapshot_id"]:
|
| 93 |
return False
|
| 94 |
+
only = changed_tables(previous, manifest, store._tables(con), paths)
|
| 95 |
+
store._install_snapshot(con, manifest, [tuple(row) for row in rows], paths, only=only)
|
| 96 |
newest = _record_newest(con, store.connector)
|
| 97 |
finally:
|
| 98 |
con.close()
|
|
|
|
| 656 |
return paths, sorted(refused)
|
| 657 |
|
| 658 |
|
| 659 |
+
def _install_snapshot(self, con, manifest, rows, paths, only=None):
|
| 660 |
+
"""Commit one source's tables, sync decisions and identity together.
|
| 661 |
+
|
| 662 |
+
`only` names the tables to rebuild; the others in `paths` are already resident and identical.
|
| 663 |
+
"""
|
| 664 |
con.execute("BEGIN TRANSACTION")
|
| 665 |
try:
|
| 666 |
for name in self._tables(con) - set(paths):
|
| 667 |
con.execute(f"DROP TABLE {_quoted_identifier(name)}")
|
| 668 |
for name, files in paths.items():
|
| 669 |
+
if only is not None and name not in only:
|
| 670 |
+
continue
|
| 671 |
con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
|
| 672 |
f"SELECT * FROM {_parquet_source(files)}")
|
| 673 |
old_rows = self._state_rows(con)
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
5161eb153e223f8d3e8a42b7280ccf54bb15dccfc64a86ffd6f43d20cfce9929
|
web/src/sheet/SheetPage.tsx
CHANGED
|
@@ -67,7 +67,7 @@ import SheetActivity from "./SheetActivity";
|
|
| 67 |
import SheetCatchUp from "./SheetCatchUp";
|
| 68 |
import RefreshHistory from "./RefreshHistory";
|
| 69 |
import WorkbookChecks from "./WorkbookChecks";
|
| 70 |
-
import { readWorkbookChecks, writeWorkbookChecks } from "./sheetApi";
|
| 71 |
import type { RailStrata } from "./SheetRail";
|
| 72 |
import {
|
| 73 |
computeCell, createdFolderIds, createWorkbook, deleteWorkbook, duplicateSheetTab, reorderSheetTab, patchTabCells, readIndex, readTab, readTabMeta,
|
|
@@ -284,7 +284,7 @@ type WorkbookOpening = Workbook & { openingTab?: SheetTab & { rev: number } };
|
|
| 284 |
|
| 285 |
async function readWorkbookOpening(id: string, tab?: string): Promise<Result<WorkbookOpening>> {
|
| 286 |
try {
|
| 287 |
-
const response = await
|
| 288 |
+ (tab ? `&open_tab=${encodeURIComponent(tab)}` : ""), { credentials: "same-origin" });
|
| 289 |
const body = await response.json().catch(() => null) as {
|
| 290 |
workbook?: Workbook;
|
|
|
|
| 67 |
import SheetCatchUp from "./SheetCatchUp";
|
| 68 |
import RefreshHistory from "./RefreshHistory";
|
| 69 |
import WorkbookChecks from "./WorkbookChecks";
|
| 70 |
+
import { fetchRetryingBusy, readWorkbookChecks, writeWorkbookChecks } from "./sheetApi";
|
| 71 |
import type { RailStrata } from "./SheetRail";
|
| 72 |
import {
|
| 73 |
computeCell, createdFolderIds, createWorkbook, deleteWorkbook, duplicateSheetTab, reorderSheetTab, patchTabCells, readIndex, readTab, readTabMeta,
|
|
|
|
| 284 |
|
| 285 |
async function readWorkbookOpening(id: string, tab?: string): Promise<Result<WorkbookOpening>> {
|
| 286 |
try {
|
| 287 |
+
const response = await fetchRetryingBusy(`/api/v1/sheets/${encodeURIComponent(id)}?tabs=meta&opening=true`
|
| 288 |
+ (tab ? `&open_tab=${encodeURIComponent(tab)}` : ""), { credentials: "same-origin" });
|
| 289 |
const body = await response.json().catch(() => null) as {
|
| 290 |
workbook?: Workbook;
|
web/src/sheet/sheetApi.ts
CHANGED
|
@@ -89,14 +89,42 @@ const errorOf = (body: unknown): { code: string; message: string; rev_current?:
|
|
| 89 |
};
|
| 90 |
};
|
| 91 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
async function call<T>(
|
| 93 |
path: string,
|
| 94 |
init: RequestInit,
|
| 95 |
read: (body: unknown) => T,
|
|
|
|
| 96 |
): Promise<Result<T>> {
|
| 97 |
let res: Response;
|
| 98 |
try {
|
| 99 |
-
|
|
|
|
|
|
|
| 100 |
} catch {
|
| 101 |
return { ok: false, status: 0, code: "unreachable", message: "Cannot reach the server." };
|
| 102 |
}
|
|
@@ -267,7 +295,7 @@ export const createWorkbook = (name = ""): Promise<Result<Workbook>> =>
|
|
| 267 |
* different rows of the same database are never handed each other's cached number.
|
| 268 |
*/
|
| 269 |
export const readWorkbook = (id: string): Promise<Result<Workbook>> =>
|
| 270 |
-
call(`/sheets/${encodeURIComponent(id)}`, { method: "GET" }, asWorkbook);
|
| 271 |
|
| 272 |
/** Counts and attribution only, checked against the workbook's current permissions. */
|
| 273 |
export const readSheetActivity = (id: string, signal?: AbortSignal): Promise<Result<SheetActivityResult>> =>
|
|
@@ -429,7 +457,7 @@ export const patchTabCells = (
|
|
| 429 |
|
| 430 |
/** The workbook's identity, revision and tab list, without the cells. */
|
| 431 |
export const readTabMeta = (id: string): Promise<Result<Workbook>> =>
|
| 432 |
-
call(`/sheets/${encodeURIComponent(id)}?tabs=meta`, { method: "GET" }, asWorkbook);
|
| 433 |
|
| 434 |
/**
|
| 435 |
* β IT CAN ANSWER `null`, AND THE CALLER MUST SAY SO. A body this parser does not recognise is
|
|
@@ -449,6 +477,7 @@ export const readTab = (id: string, tab: string): Promise<Result<SheetTab | null
|
|
| 449 |
`/sheets/${encodeURIComponent(id)}/tabs/${encodeURIComponent(tab)}`,
|
| 450 |
{ method: "GET" },
|
| 451 |
asTab,
|
|
|
|
| 452 |
);
|
| 453 |
|
| 454 |
// ββ the doors W49-T04 fills in βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -506,8 +535,8 @@ export const readRefreshHistory = (id: string, signal?: AbortSignal): Promise<Re
|
|
| 506 |
});
|
| 507 |
|
| 508 |
export const retryRefresh = (id: string, refreshId: string): Promise<Result<RefreshScopeResult>> =>
|
| 509 |
-
|
| 510 |
-
{ method: "POST" }, refreshScopeResponse);
|
| 511 |
|
| 512 |
/** Source changes keep the old value until an explicit refresh. */
|
| 513 |
export const isSourceStale = (cell: Cell): boolean =>
|
|
@@ -536,6 +565,81 @@ function refreshScopeResponse(body: unknown): RefreshScopeResult {
|
|
| 536 |
: { workbook: asWorkbook(body) }) };
|
| 537 |
}
|
| 538 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 539 |
/** Share one scope's pending request within the caller's mounted page. */
|
| 540 |
export const refreshScope = (
|
| 541 |
id: string,
|
|
@@ -545,7 +649,8 @@ export const refreshScope = (
|
|
| 545 |
const key = JSON.stringify([id, scope]);
|
| 546 |
const pending = inFlight?.get(key);
|
| 547 |
if (pending) return pending;
|
| 548 |
-
const request =
|
|
|
|
| 549 |
.finally(() => { inFlight?.delete(key); });
|
| 550 |
inFlight?.set(key, request);
|
| 551 |
return request;
|
|
@@ -558,12 +663,15 @@ export const refreshWorkbook = (
|
|
| 558 |
): Promise<Result<RefreshResult>> => {
|
| 559 |
const pending = inFlight?.get(id);
|
| 560 |
if (pending) return pending;
|
| 561 |
-
const request =
|
| 562 |
workbook: asWorkbook(body),
|
| 563 |
recalcMs: typeof (body as { recalc_ms?: unknown } | null)?.recalc_ms === "number"
|
| 564 |
? (body as { recalc_ms: number }).recalc_ms
|
| 565 |
: null,
|
| 566 |
-
})
|
|
|
|
|
|
|
|
|
|
| 567 |
inFlight?.set(id, request);
|
| 568 |
return request;
|
| 569 |
};
|
|
@@ -596,18 +704,51 @@ export interface RefreshedCell extends CellComputed {
|
|
| 596 |
export const refreshCells = (
|
| 597 |
id: string, cells: readonly string[],
|
| 598 |
): Promise<Result<{ rev: number | null; cells: Record<string, RefreshedCell> }>> =>
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 608 |
}
|
| 609 |
-
|
| 610 |
-
}
|
|
|
|
| 611 |
|
| 612 |
/**
|
| 613 |
* W54-T32 (instruction 9): what `POST /sheets/{id}/cells/{addr}/compute` answers.
|
|
|
|
| 89 |
};
|
| 90 |
};
|
| 91 |
|
| 92 |
+
/** A queued heavy request the server could not start in time is retried this many times. */
|
| 93 |
+
export const BUSY_RETRIES = 3;
|
| 94 |
+
/** Seconds to wait before a retry when the server sends no Retry-After header. */
|
| 95 |
+
export const BUSY_DEFAULT_WAIT_S = 5;
|
| 96 |
+
|
| 97 |
+
/**
|
| 98 |
+
* fetch, but a 503 `server_busy` waits its Retry-After seconds and tries again, at most
|
| 99 |
+
* BUSY_RETRIES times. The last answer is returned as it came, so the caller's failure copy shows.
|
| 100 |
+
*/
|
| 101 |
+
export async function fetchRetryingBusy(
|
| 102 |
+
input: string, init?: RequestInit,
|
| 103 |
+
wait: (ms: number) => Promise<void> = (ms) => new Promise((done) => { setTimeout(done, ms); }),
|
| 104 |
+
): Promise<Response> {
|
| 105 |
+
for (let attempt = 0; ; attempt += 1) {
|
| 106 |
+
const res = await fetch(input, init);
|
| 107 |
+
if (res.status !== 503 || attempt >= BUSY_RETRIES) return res;
|
| 108 |
+
const body = (await res.clone().json().catch(() => null)) as { error?: { code?: string } } | null;
|
| 109 |
+
if (body?.error?.code !== "server_busy") return res;
|
| 110 |
+
const header = res.headers.get("Retry-After");
|
| 111 |
+
const seconds = header === null || !Number.isFinite(Number(header)) || Number(header) < 0
|
| 112 |
+
? BUSY_DEFAULT_WAIT_S : Number(header);
|
| 113 |
+
await wait(seconds * 1000);
|
| 114 |
+
}
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
async function call<T>(
|
| 118 |
path: string,
|
| 119 |
init: RequestInit,
|
| 120 |
read: (body: unknown) => T,
|
| 121 |
+
retryBusy = false,
|
| 122 |
): Promise<Result<T>> {
|
| 123 |
let res: Response;
|
| 124 |
try {
|
| 125 |
+
const request = { credentials: CREDENTIALS, ...init };
|
| 126 |
+
res = retryBusy ? await fetchRetryingBusy(`${API_V1}${path}`, request)
|
| 127 |
+
: await fetch(`${API_V1}${path}`, request);
|
| 128 |
} catch {
|
| 129 |
return { ok: false, status: 0, code: "unreachable", message: "Cannot reach the server." };
|
| 130 |
}
|
|
|
|
| 295 |
* different rows of the same database are never handed each other's cached number.
|
| 296 |
*/
|
| 297 |
export const readWorkbook = (id: string): Promise<Result<Workbook>> =>
|
| 298 |
+
call(`/sheets/${encodeURIComponent(id)}`, { method: "GET" }, asWorkbook, true);
|
| 299 |
|
| 300 |
/** Counts and attribution only, checked against the workbook's current permissions. */
|
| 301 |
export const readSheetActivity = (id: string, signal?: AbortSignal): Promise<Result<SheetActivityResult>> =>
|
|
|
|
| 457 |
|
| 458 |
/** The workbook's identity, revision and tab list, without the cells. */
|
| 459 |
export const readTabMeta = (id: string): Promise<Result<Workbook>> =>
|
| 460 |
+
call(`/sheets/${encodeURIComponent(id)}?tabs=meta`, { method: "GET" }, asWorkbook, true);
|
| 461 |
|
| 462 |
/**
|
| 463 |
* β IT CAN ANSWER `null`, AND THE CALLER MUST SAY SO. A body this parser does not recognise is
|
|
|
|
| 477 |
`/sheets/${encodeURIComponent(id)}/tabs/${encodeURIComponent(tab)}`,
|
| 478 |
{ method: "GET" },
|
| 479 |
asTab,
|
| 480 |
+
true,
|
| 481 |
);
|
| 482 |
|
| 483 |
// ββ the doors W49-T04 fills in βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 535 |
});
|
| 536 |
|
| 537 |
export const retryRefresh = (id: string, refreshId: string): Promise<Result<RefreshScopeResult>> =>
|
| 538 |
+
postRefresh(id, `/sheets/${encodeURIComponent(id)}/refresh-history/${encodeURIComponent(refreshId)}/retry`,
|
| 539 |
+
{ method: "POST" }, refreshScopeResponse, () => reloadWholeWorkbook(id));
|
| 540 |
|
| 541 |
/** Source changes keep the old value until an explicit refresh. */
|
| 542 |
export const isSourceStale = (cell: Cell): boolean =>
|
|
|
|
| 565 |
: { workbook: asWorkbook(body) }) };
|
| 566 |
}
|
| 567 |
|
| 568 |
+
// ββ queued refresh: the web server hands a refresh to its sync worker and answers 202 βββββββββ
|
| 569 |
+
|
| 570 |
+
/** Seconds between two status reads of a queued refresh. */
|
| 571 |
+
const REFRESH_POLL_MS = 3000;
|
| 572 |
+
/** A queued refresh still unfinished after this long reads as a failure, never a spinner forever. */
|
| 573 |
+
const REFRESH_WAIT_MS = 20 * 60 * 1000;
|
| 574 |
+
|
| 575 |
+
type QueuedRefresh = { queued: true; requestId: string };
|
| 576 |
+
|
| 577 |
+
const isQueued = (body: unknown): body is QueuedRefresh =>
|
| 578 |
+
!!body && typeof body === "object" && (body as { queued?: unknown }).queued === true
|
| 579 |
+
&& typeof (body as { requestId?: unknown }).requestId === "string";
|
| 580 |
+
|
| 581 |
+
const pause = (ms: number) => new Promise<void>((done) => { setTimeout(done, ms); });
|
| 582 |
+
|
| 583 |
+
/** Poll one queued refresh until the worker marks it done or failed. */
|
| 584 |
+
async function awaitQueuedRefresh(id: string, requestId: string): Promise<Result<true>> {
|
| 585 |
+
const path = `/sheets/${encodeURIComponent(id)}/refresh-status?requestId=${encodeURIComponent(requestId)}`;
|
| 586 |
+
const deadline = Date.now() + REFRESH_WAIT_MS;
|
| 587 |
+
while (Date.now() < deadline) {
|
| 588 |
+
await pause(REFRESH_POLL_MS);
|
| 589 |
+
const got = await call(path, { method: "GET" }, (body) => body as {
|
| 590 |
+
state?: string; error?: { code?: string; message?: string } });
|
| 591 |
+
if (!got.ok) {
|
| 592 |
+
// A dropped or busy status read is not an answer: the worker may still be refreshing.
|
| 593 |
+
if (got.status === 0 || got.code === "server_busy") continue;
|
| 594 |
+
return got;
|
| 595 |
+
}
|
| 596 |
+
if (got.value.state === "done") return { ok: true, value: true };
|
| 597 |
+
if (got.value.state === "failed") {
|
| 598 |
+
return { ok: false, status: 500, code: got.value.error?.code || "sheet_unresolved",
|
| 599 |
+
message: got.value.error?.message || "This workbook could not be refreshed." };
|
| 600 |
+
}
|
| 601 |
+
}
|
| 602 |
+
return { ok: false, status: 0, code: "sheet_refresh_timeout",
|
| 603 |
+
message: "This refresh is taking longer than usual. Reopen the workbook in a few minutes." };
|
| 604 |
+
}
|
| 605 |
+
|
| 606 |
+
/**
|
| 607 |
+
* POST one refresh. A 200 reads as today; a 202 waits for the worker, then runs `reload`, so every
|
| 608 |
+
* caller keeps its own result shape and its existing busy state.
|
| 609 |
+
*/
|
| 610 |
+
async function postRefresh<T>(
|
| 611 |
+
id: string, path: string, init: RequestInit, read: (body: unknown) => T,
|
| 612 |
+
reload: () => Promise<Result<T>>,
|
| 613 |
+
): Promise<Result<T>> {
|
| 614 |
+
const first = await call(path, init, (body) => body, true);
|
| 615 |
+
if (!first.ok) return first;
|
| 616 |
+
if (!isQueued(first.value)) return { ok: true, value: read(first.value) };
|
| 617 |
+
const settled = await awaitQueuedRefresh(id, first.value.requestId);
|
| 618 |
+
if (!settled.ok) return settled;
|
| 619 |
+
return reload();
|
| 620 |
+
}
|
| 621 |
+
|
| 622 |
+
/** The workbook after a queued refresh: the whole document, as a workbook scope answers. */
|
| 623 |
+
const reloadWholeWorkbook = async (id: string): Promise<Result<RefreshScopeResult>> => {
|
| 624 |
+
const got = await readWorkbook(id);
|
| 625 |
+
return got.ok ? { ok: true, value: { addresses: [], recalcMs: null, workbook: got.value } } : got;
|
| 626 |
+
};
|
| 627 |
+
|
| 628 |
+
/** One tab's cells after a queued cell or range refresh, keyed `<tab id>!<ADDR>`. */
|
| 629 |
+
const reloadScopeTab = async (id: string, scope: RefreshScope): Promise<Result<RefreshScopeResult>> => {
|
| 630 |
+
const target = "cell" in scope ? scope.cell : "range" in scope ? scope.range : "";
|
| 631 |
+
const tab = target.includes("!") ? target.slice(0, target.lastIndexOf("!")) : "";
|
| 632 |
+
if (!tab) return reloadWholeWorkbook(id);
|
| 633 |
+
const got = await readTab(id, tab);
|
| 634 |
+
if (!got.ok) return got;
|
| 635 |
+
if (!got.value) return reloadWholeWorkbook(id);
|
| 636 |
+
const found = got.value as SheetTab & { rev?: number };
|
| 637 |
+
const cells = Object.fromEntries(Object.entries(found.cells ?? {})
|
| 638 |
+
.filter(([, cell]) => !!cell).map(([addr, cell]) => [`${found.id}!${addr}`, cell as Cell]));
|
| 639 |
+
return { ok: true, value: { addresses: [], recalcMs: null, cells,
|
| 640 |
+
workbook: { id, rev: typeof found.rev === "number" ? found.rev : 0 } } };
|
| 641 |
+
};
|
| 642 |
+
|
| 643 |
/** Share one scope's pending request within the caller's mounted page. */
|
| 644 |
export const refreshScope = (
|
| 645 |
id: string,
|
|
|
|
| 649 |
const key = JSON.stringify([id, scope]);
|
| 650 |
const pending = inFlight?.get(key);
|
| 651 |
if (pending) return pending;
|
| 652 |
+
const request = postRefresh(id, `/sheets/${encodeURIComponent(id)}/refresh`, sends("POST", { scope }),
|
| 653 |
+
refreshScopeResponse, () => ("workbook" in scope ? reloadWholeWorkbook(id) : reloadScopeTab(id, scope)))
|
| 654 |
.finally(() => { inFlight?.delete(key); });
|
| 655 |
inFlight?.set(key, request);
|
| 656 |
return request;
|
|
|
|
| 663 |
): Promise<Result<RefreshResult>> => {
|
| 664 |
const pending = inFlight?.get(id);
|
| 665 |
if (pending) return pending;
|
| 666 |
+
const request = postRefresh(id, `/sheets/${encodeURIComponent(id)}/refresh`, { method: "POST" }, (body) => ({
|
| 667 |
workbook: asWorkbook(body),
|
| 668 |
recalcMs: typeof (body as { recalc_ms?: unknown } | null)?.recalc_ms === "number"
|
| 669 |
? (body as { recalc_ms: number }).recalc_ms
|
| 670 |
: null,
|
| 671 |
+
}), async (): Promise<Result<RefreshResult>> => {
|
| 672 |
+
const got = await readWorkbook(id);
|
| 673 |
+
return got.ok ? { ok: true, value: { workbook: got.value, recalcMs: null } } : got;
|
| 674 |
+
}).finally(() => { inFlight?.delete(id); });
|
| 675 |
inFlight?.set(id, request);
|
| 676 |
return request;
|
| 677 |
};
|
|
|
|
| 704 |
export const refreshCells = (
|
| 705 |
id: string, cells: readonly string[],
|
| 706 |
): Promise<Result<{ rev: number | null; cells: Record<string, RefreshedCell> }>> =>
|
| 707 |
+
postRefresh(id, `/sheets/${encodeURIComponent(id)}/refresh`, sends("POST", { cells }), readRefreshedCells,
|
| 708 |
+
() => reloadRefreshedCells(id, cells));
|
| 709 |
+
|
| 710 |
+
/** What `POST /refresh {cells}` answers, as refreshed cells keyed `<tab id>!<ADDR>`. */
|
| 711 |
+
function readRefreshedCells(body: unknown): { rev: number | null; cells: Record<string, RefreshedCell> } {
|
| 712 |
+
const raw = (body as { workbook?: { rev?: unknown }; cells?: unknown } | null) ?? {};
|
| 713 |
+
const rev = typeof raw.workbook?.rev === "number" ? raw.workbook.rev : null;
|
| 714 |
+
const out: Record<string, RefreshedCell> = {};
|
| 715 |
+
for (const [key, row] of Object.entries(asComputed(raw.cells))) {
|
| 716 |
+
const extra = ((raw.cells as Record<string, Record<string, unknown>>)[key]) ?? {};
|
| 717 |
+
out[key] = { ...row,
|
| 718 |
+
...(typeof extra.refreshed_at === "string" ? { refreshed_at: extra.refreshed_at } : {}),
|
| 719 |
+
...(Array.isArray(extra.limits) ? { limits: extra.limits } : {}) };
|
| 720 |
+
}
|
| 721 |
+
return { rev, cells: out };
|
| 722 |
+
}
|
| 723 |
+
|
| 724 |
+
/** After a queued cell refresh, read each named tab again and answer the named cells from it. */
|
| 725 |
+
async function reloadRefreshedCells(
|
| 726 |
+
id: string, cells: readonly string[],
|
| 727 |
+
): Promise<Result<{ rev: number | null; cells: Record<string, RefreshedCell> }>> {
|
| 728 |
+
const tabs = [...new Set(cells.filter((key) => key.includes("!"))
|
| 729 |
+
.map((key) => key.slice(0, key.lastIndexOf("!"))))];
|
| 730 |
+
const out: Record<string, RefreshedCell> = {};
|
| 731 |
+
let rev: number | null = null;
|
| 732 |
+
for (const tabId of tabs) {
|
| 733 |
+
const got = await readTab(id, tabId);
|
| 734 |
+
if (!got.ok) return got;
|
| 735 |
+
const found = got.value as (SheetTab & { rev?: number }) | null;
|
| 736 |
+
if (!found) continue;
|
| 737 |
+
if (typeof found.rev === "number") rev = Math.max(rev ?? found.rev, found.rev);
|
| 738 |
+
for (const key of cells) {
|
| 739 |
+
if (!key.startsWith(`${tabId}!`)) continue;
|
| 740 |
+
const cell = found.cells?.[key.slice(tabId.length + 1)] as
|
| 741 |
+
(Cell & { refreshed_at?: string; limits?: unknown[] }) | null | undefined;
|
| 742 |
+
if (!cell) continue;
|
| 743 |
+
out[key] = { value: cell.value ?? null, display: typeof cell.display === "string" ? cell.display : "",
|
| 744 |
+
state: typeof cell.state === "string" ? cell.state : "fresh",
|
| 745 |
+
...(cell.refusal ? { refusal: cell.refusal } : {}),
|
| 746 |
+
...(typeof cell.refreshed_at === "string" ? { refreshed_at: cell.refreshed_at } : {}),
|
| 747 |
+
...(Array.isArray(cell.limits) ? { limits: cell.limits } : {}) };
|
| 748 |
}
|
| 749 |
+
}
|
| 750 |
+
return { ok: true, value: { rev, cells: out } };
|
| 751 |
+
}
|
| 752 |
|
| 753 |
/**
|
| 754 |
* W54-T32 (instruction 9): what `POST /sheets/{id}/cells/{addr}/compute` answers.
|