fsanyoto commited on
Commit
519aa2c
Β·
verified Β·
1 Parent(s): 7e9780d

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "dcc41725b463 (dcc41725b463)",
3
  "releases": [
4
  {
5
  "version": "v83",
 
1
  {
2
+ "current": "7fe0be77d44f (7fe0be77d44f)",
3
  "releases": [
4
  {
5
  "version": "v83",
VERSION CHANGED
@@ -1 +1 @@
1
- dcc41725b463 (dcc41725b463)
 
1
+ 7fe0be77d44f (7fe0be77d44f)
api/main.py CHANGED
@@ -147,6 +147,8 @@ app = FastAPI(title="AIOS web API")
147
  # payload to ~10–15% of that. minimum_size spares the tiny acks the overhead.
148
  from assistant_stream import StreamGZipMiddleware
149
  app.add_middleware(StreamGZipMiddleware, minimum_size=1024)
 
 
150
 
151
 
152
  @app.exception_handler(StarletteHTTPException)
@@ -2995,11 +2997,28 @@ def _boot_mirrors():
2995
  if _role_boot.role() == "web":
2996
  # Every connector but Odoo reaches the web process through the worker's hand-off.
2997
  print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
 
 
2998
  return
2999
  _restore_amazon_mirror("boot")
3000
  _restore_declared_mirrors("boot")
3001
 
3002
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3003
  def _restore_declared_mirrors(why):
3004
  """Restore each tenant's declarations after the Odoo seed and Amazon policy adapters."""
3005
  from core import connector_spec
 
147
  # payload to ~10–15% of that. minimum_size spares the tiny acks the overhead.
148
  from assistant_stream import StreamGZipMiddleware
149
  app.add_middleware(StreamGZipMiddleware, minimum_size=1024)
150
+ from request_class import HeavyGate # noqa: E402 (heavy reads queue so light requests never starve)
151
+ app.add_middleware(HeavyGate)
152
 
153
 
154
  @app.exception_handler(StarletteHTTPException)
 
2997
  if _role_boot.role() == "web":
2998
  # Every connector but Odoo reaches the web process through the worker's hand-off.
2999
  print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
3000
+ # The biggest read caches are this process's own, so they warm here, before the first reader pays.
3001
+ _role_boot.start("read-warm", "light", _warm_read_pools)
3002
  return
3003
  _restore_amazon_mirror("boot")
3004
  _restore_declared_mirrors("boot")
3005
 
3006
 
3007
+ def _warm_read_pools():
3008
+ """Build the consolidated customer and product pools once after boot, off the request path."""
3009
+ import time as _t
3010
+ t0 = _t.time()
3011
+ try:
3012
+ from harness import runtime as _runtime
3013
+ import routes_products
3014
+ rt = _runtime.get_runtime("royal-imports")
3015
+ routes_customers.warm_default(rt)
3016
+ routes_products._pool_for(rt, None)
3017
+ print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
3018
+ except Exception as exc: # noqa: BLE001
3019
+ print(f"[aios-api] read warm skipped ({type(exc).__name__}: {exc})")
3020
+
3021
+
3022
  def _restore_declared_mirrors(why):
3023
  """Restore each tenant's declarations after the Odoo seed and Amazon policy adapters."""
3024
  from core import connector_spec
api/process_role.py CHANGED
@@ -228,7 +228,7 @@ def install_ready(root=None, installer=None, *, now=None):
228
  snapshot = ready.get("snapshotId") or newest_dir.name
229
  if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS:
230
  try:
231
- installer(key[1], key[0], ready)
232
  except Exception as exc: # noqa: BLE001
233
  _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
234
  LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
@@ -237,6 +237,11 @@ def install_ready(root=None, installer=None, *, now=None):
237
  f"({type(exc).__name__}: {exc})", flush=True)
238
  continue
239
  _INSTALLED[key] = snapshot
 
 
 
 
 
240
  LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
241
  if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
242
  LAST_INSTALL_ERROR.clear()
 
228
  snapshot = ready.get("snapshotId") or newest_dir.name
229
  if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS:
230
  try:
231
+ changed = installer(key[1], key[0], ready)
232
  except Exception as exc: # noqa: BLE001
233
  _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
234
  LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
 
237
  f"({type(exc).__name__}: {exc})", flush=True)
238
  continue
239
  _INSTALLED[key] = snapshot
240
+ if changed is False:
241
+ # Already resident: nothing was rebuilt, so this is not an install.
242
+ for _at, older, _ready in rows[:-1]:
243
+ _remove_dir(older)
244
+ continue
245
  LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
246
  if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
247
  LAST_INSTALL_ERROR.clear()
api/request_class.py ADDED
@@ -0,0 +1,276 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Heavy request admission: a bounded queue so CPU-heavy reads never starve light requests.
2
+
3
+ One web process on 2 vCPU serves everyone. `classify` names the heavy doors in ONE registry, and
4
+ `HeavyGate` lets at most `AIOS_HEAVY_CONCURRENCY` of them run at once. A heavy request that waits
5
+ longer than `AIOS_HEAVY_QUEUE_SECONDS` gets a 503 `server_busy`. Light requests never wait.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import asyncio
10
+ import json
11
+ import os
12
+ import re
13
+ import threading
14
+ from urllib.parse import parse_qs
15
+
16
+ BUSY_CODE = "server_busy"
17
+ BUSY_MESSAGE = "Loopable is busy. Try again in a moment."
18
+ RETRY_AFTER_SECONDS = 5
19
+ MCP_PATH = "/api/v1/external/mcp"
20
+ # The MCP body is peeked to classify it. A larger body is heavy without parsing.
21
+ MCP_PEEK_LIMIT = 1 << 20
22
+
23
+ # FastAPI reads these strings as True for a bool query parameter.
24
+ _TRUE = {"1", "true", "on", "yes", "y", "t"}
25
+
26
+
27
+ def _flag(query, name):
28
+ return any(str(v).strip().lower() in _TRUE for v in query.get(name, []))
29
+
30
+
31
+ def _sheet_read_heavy(query):
32
+ # Only the cell-free metadata read is light. A bare GET or any other `tabs` value reads cells.
33
+ tabs = (query.get("tabs") or [""])[-1]
34
+ return not (tabs == "meta" and not _flag(query, "opening"))
35
+
36
+
37
+ # (methods, path regex, query predicate or None). A request is heavy when one row matches.
38
+ HEAVY = (
39
+ ({"GET"}, r"/api/v1/workspace", lambda q: not _flag(q, "fields_only")),
40
+ ({"GET"}, r"/api/v1/products", None),
41
+ ({"GET"}, r"/api/v1/customers", None),
42
+ ({"GET", "POST"}, r"/api/v1/odoo-tables/[^/]+/rows", None),
43
+ ({"GET", "POST"}, r"/api/v1/connected-tables/[^/]+/rows", None),
44
+ ({"GET"}, r"/api/v1/tables/[^/]+/rows", None),
45
+ ({"GET"}, r"/api/v1/sheets/(?!index$|import$|automation$)[^/]+", _sheet_read_heavy),
46
+ ({"GET"}, r"/api/v1/sheets/[^/]+/tabs/[^/]+", None),
47
+ # In the web role a refresh only queues for the sync worker; it computes only in worker or all.
48
+ ({"POST"}, r"/api/v1/sheets/[^/]+/refresh", lambda q: os.environ.get("AIOS_PROCESS_ROLE") != "web"),
49
+ ({"GET", "POST"}, r"/api/v1/.*(?:/export|-export)(?:/.*)?", None),
50
+ ({"POST"}, r"/api/v1/(?:external|typed)/databases/[^/]+/query", None),
51
+ ({"POST"}, r"/api/v1/external/explorer/run", None),
52
+ )
53
+ _HEAVY = tuple((methods, re.compile(rx), pred) for methods, rx, pred in HEAVY)
54
+
55
+ # MCP tools/call names that read sheets or query databases.
56
+ MCP_HEAVY_TOOLS = {"sheet", "table", "data"}
57
+ _MCP_LIGHT_ACTIONS = re.compile(r"(?:^|[._])(?:describe|list|guide|schema)(?:$|[._])")
58
+
59
+
60
+ def _mcp_heavy(body):
61
+ """True when an MCP JSON-RPC body is a tools/call on a sheet or query tool."""
62
+ if body is None:
63
+ return False
64
+ if len(body) > MCP_PEEK_LIMIT:
65
+ return True
66
+ try:
67
+ req = json.loads(body)
68
+ except (ValueError, UnicodeError):
69
+ return False
70
+ calls = req if isinstance(req, list) else [req]
71
+ for call in calls:
72
+ if not isinstance(call, dict) or call.get("method") != "tools/call":
73
+ continue
74
+ params = call.get("params") if isinstance(call.get("params"), dict) else {}
75
+ name = params.get("name")
76
+ if not isinstance(name, str):
77
+ continue
78
+ group, _, dotted = name.partition(".")
79
+ if group not in MCP_HEAVY_TOOLS and not name.startswith("sheet_") and "query" not in name:
80
+ continue
81
+ args = params.get("arguments") if isinstance(params.get("arguments"), dict) else {}
82
+ action = dotted or str(args.get("action") or "")
83
+ if action and _MCP_LIGHT_ACTIONS.search(action) and "query" not in action:
84
+ continue
85
+ return True
86
+ return False
87
+
88
+
89
+ def classify(method, path, query="", body=None):
90
+ """Return "heavy" or "light" for one request. `query` is a raw query string or a parsed dict."""
91
+ method = (method or "").upper()
92
+ if isinstance(query, bytes):
93
+ query = query.decode("latin-1")
94
+ q = parse_qs(query, keep_blank_values=True) if isinstance(query, str) else (query or {})
95
+ for methods, rx, pred in _HEAVY:
96
+ if method in methods and rx.fullmatch(path or "") and (pred is None or pred(q)):
97
+ return "heavy"
98
+ if method == "POST" and path == MCP_PATH and _mcp_heavy(body):
99
+ return "heavy"
100
+ return "light"
101
+
102
+
103
+ class _Counters:
104
+ """Process-wide heavy request counters. Middleware instances report into this."""
105
+
106
+ def __init__(self):
107
+ self.lock = threading.Lock()
108
+ self.in_flight = 0
109
+ self.queued = 0
110
+ self.busy_refusals = 0
111
+ self.admitted = 0
112
+ self.limit = None
113
+ self.queue_seconds = None
114
+
115
+ def add(self, name, n):
116
+ with self.lock:
117
+ setattr(self, name, getattr(self, name) + n)
118
+
119
+
120
+ _COUNTERS = _Counters()
121
+
122
+
123
+ def status():
124
+ """Snapshot of heavy request admission for a settings or health surface."""
125
+ c = _COUNTERS
126
+ with c.lock:
127
+ return {"limit": c.limit if c.limit is not None else _env_int("AIOS_HEAVY_CONCURRENCY", 2),
128
+ "queueSeconds": (c.queue_seconds if c.queue_seconds is not None
129
+ else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0)),
130
+ "inFlight": c.in_flight, "queued": c.queued,
131
+ "busyRefusals": c.busy_refusals, "admitted": c.admitted}
132
+
133
+
134
+ def _env_int(name, default):
135
+ try:
136
+ return max(1, int(os.environ.get(name) or default))
137
+ except ValueError:
138
+ return default
139
+
140
+
141
+ def _env_float(name, default):
142
+ try:
143
+ return max(0.0, float(os.environ.get(name) or default))
144
+ except ValueError:
145
+ return default
146
+
147
+
148
+ async def _send_busy(send):
149
+ body = json.dumps({"error": {"code": BUSY_CODE, "message": BUSY_MESSAGE}}).encode()
150
+ await send({"type": "http.response.start", "status": 503,
151
+ "headers": [(b"content-type", b"application/json"),
152
+ (b"content-length", str(len(body)).encode()),
153
+ (b"retry-after", str(RETRY_AFTER_SECONDS).encode()),
154
+ (b"cache-control", b"no-store")]})
155
+ await send({"type": "http.response.body", "body": body, "more_body": False})
156
+
157
+
158
+ class HeavyGate:
159
+ """Pure ASGI middleware. Heavy requests take a permit; light ones pass straight through."""
160
+
161
+ def __init__(self, app, concurrency=None, queue_seconds=None, counters=None):
162
+ self.app = app
163
+ self.concurrency = concurrency or _env_int("AIOS_HEAVY_CONCURRENCY", 2)
164
+ self.queue_seconds = (queue_seconds if queue_seconds is not None
165
+ else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0))
166
+ self.counters = counters or _COUNTERS
167
+ self.counters.limit, self.counters.queue_seconds = self.concurrency, self.queue_seconds
168
+ self._sem = None
169
+ self._loop = None
170
+
171
+ def _semaphore(self):
172
+ # A Semaphore binds to the first loop that waits on it. One per loop.
173
+ loop = asyncio.get_running_loop()
174
+ if self._sem is None or self._loop is not loop:
175
+ self._sem, self._loop = asyncio.Semaphore(self.concurrency), loop
176
+ return self._sem
177
+
178
+ async def __call__(self, scope, receive, send):
179
+ if scope.get("type") != "http":
180
+ return await self.app(scope, receive, send)
181
+ method, path = scope.get("method", ""), scope.get("path", "")
182
+ body_msgs = None
183
+ peek = None
184
+ if method == "POST" and path == MCP_PATH:
185
+ body_msgs, peek = await _read_body(receive, MCP_PEEK_LIMIT)
186
+ if peek is None: # The client left while sending the body.
187
+ return
188
+ if classify(method, path, scope.get("query_string", b""), peek) != "heavy":
189
+ return await self.app(scope, _replay(body_msgs, receive), send)
190
+ if body_msgs is None:
191
+ body_msgs, _ = await _read_body(receive, None)
192
+ if body_msgs is None:
193
+ return
194
+ await self._run_heavy(scope, body_msgs, receive, send)
195
+
196
+ async def _run_heavy(self, scope, body_msgs, receive, send):
197
+ sem = self._semaphore()
198
+ c = self.counters
199
+ c.add("queued", 1)
200
+ acquire = asyncio.ensure_future(sem.acquire())
201
+ gone = asyncio.ensure_future(_wait_disconnect(receive))
202
+ try:
203
+ done, _ = await asyncio.wait({acquire, gone}, timeout=self.queue_seconds,
204
+ return_when=asyncio.FIRST_COMPLETED)
205
+ finally:
206
+ c.add("queued", -1)
207
+ if not acquire.done():
208
+ acquire.cancel()
209
+ if not gone.done():
210
+ gone.cancel()
211
+ # A permit can land in the same tick as a timeout or disconnect. Hold it only if used.
212
+ got = acquire.done() and not acquire.cancelled() and acquire.exception() is None
213
+ if acquire not in done or gone in done:
214
+ if got:
215
+ sem.release()
216
+ if gone not in done:
217
+ c.add("busy_refusals", 1)
218
+ await _send_busy(send)
219
+ return
220
+ released = False
221
+
222
+ def release():
223
+ nonlocal released
224
+ if not released:
225
+ released = True
226
+ c.add("in_flight", -1)
227
+ sem.release()
228
+
229
+ async def send_wrapped(message):
230
+ await send(message)
231
+ if message.get("type") == "http.response.body" and not message.get("more_body"):
232
+ release()
233
+
234
+ c.add("in_flight", 1)
235
+ c.add("admitted", 1)
236
+ try:
237
+ await self.app(scope, _replay(body_msgs, receive), send_wrapped)
238
+ finally:
239
+ release()
240
+
241
+
242
+ async def _read_body(receive, limit):
243
+ """Buffer the request body. Returns (messages, bytes or None if the client disconnected)."""
244
+ msgs, chunks, size = [], [], 0
245
+ while True:
246
+ message = await receive()
247
+ if message.get("type") == "http.disconnect":
248
+ return None, None
249
+ msgs.append(message)
250
+ chunk = message.get("body", b"") or b""
251
+ # Keep the chunk that crosses the limit, so an oversized body reads as oversized.
252
+ if limit is None or size <= limit:
253
+ chunks.append(chunk)
254
+ size += len(chunk)
255
+ if not message.get("more_body"):
256
+ return msgs, b"".join(chunks)
257
+
258
+
259
+ async def _wait_disconnect(receive):
260
+ while True:
261
+ message = await receive()
262
+ if message.get("type") == "http.disconnect":
263
+ return True
264
+
265
+
266
+ def _replay(msgs, receive):
267
+ """A receive that hands back the buffered body first, then defers to the real one."""
268
+ if not msgs:
269
+ return receive
270
+ pending = list(msgs)
271
+
272
+ async def replay():
273
+ if pending:
274
+ return pending.pop(0)
275
+ return await receive()
276
+ return replay
api/routes_customers.py CHANGED
@@ -41,8 +41,9 @@ MODULE = "customer_data"
41
 
42
  _CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
43
 
44
- # A slow producer must not indefinitely hold another threadpool request; timeout recomputes.
45
- _GRID_ASSEMBLY_WAIT_SECONDS = 10.0
 
46
  _grid_assembly_lock = threading.Lock()
47
  _grid_assembly_inflight = {}
48
  _grid_assembly_enabled = True
 
41
 
42
  _CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
43
 
44
+ # Waiters share one cold build (about 4 min on 2 vCPU); a private rebuild per waiter multiplies it.
45
+ # Bounded under the client's 110 s read deadline; request_class caps how many heavy reads wait.
46
+ _GRID_ASSEMBLY_WAIT_SECONDS = 100.0
47
  _grid_assembly_lock = threading.Lock()
48
  _grid_assembly_inflight = {}
49
  _grid_assembly_enabled = True
api/routes_sheet.py CHANGED
@@ -2995,10 +2995,17 @@ def read_sheet(workbook_id: str, tabs: str = "",
2995
  target = next((tab for tab in visible if tab["id"] == open_tab),
2996
  visible[0] if visible else None)
2997
  if target is not None:
 
 
 
 
 
2998
  target["cells"] = sheets.get_workbook_tab_cells(
2999
  _store(session), workbook_id, target["id"], expected_rev=book.get("rev"))
3000
  target.pop("cellCount", None)
3001
- out["opened"] = _serve_open_tab(session, book, target)
 
 
3002
  return out
3003
  except sheets.SheetRefusal as exc:
3004
  raise _refuse(exc)
@@ -3154,6 +3161,8 @@ def read_sheet_refresh_history(workbook_id: str, session: Session = Depends(requ
3154
 
3155
  @router.post("/sheets/{workbook_id}/refresh-history/{refresh_id}/retry")
3156
  def retry_sheet_refresh(workbook_id: str, refresh_id: str, session: Session = Depends(require_session)):
 
 
3157
  with _sheet_request_scope(session):
3158
  return _refresh_sheet_scoped(workbook_id, session, retry_id=refresh_id)
3159
 
@@ -3198,13 +3207,22 @@ def read_sheet_tab(workbook_id: str, tab: str,
3198
  raise _refuse(exc)
3199
 
3200
 
3201
- def _serve_open_tab(session, book, target):
3202
- """Serve the selected tab and its dependency closure under one request's fresh walls."""
 
 
 
3203
  from core import sheets
3204
 
3205
  def load_tab(tab_id):
3206
- return sheets.get_workbook_tab_cells(_store(session), book["id"], tab_id,
3207
  expected_rev=book.get("rev"))
 
 
 
 
 
 
3208
 
3209
  with _sheet_request_scope(session):
3210
  served = _read_tab_for(session, book, target, load_tab=load_tab)
@@ -3676,9 +3694,397 @@ def unshare_sheet(workbook_id: str, who: str, session: Session = Depends(require
3676
  # `drill` with real resolution through the mirror seam; W49-T03 supplies formula evaluation.
3677
 
3678
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3679
  @router.post("/sheets/{workbook_id}/refresh")
3680
- def refresh_sheet(workbook_id: str, body: dict = Body(default=None),
3681
- session: Session = Depends(require_session)):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3682
  """C2/R4: Refresh resolves only databases whose token or binding inputs changed.
3683
 
3684
  Viewers refresh their own server cache. Only editors save the workbook document.
@@ -3782,6 +4188,16 @@ def _public_refresh_snapshot(snapshot):
3782
 
3783
  def _refresh_sheet_scoped(workbook_id: str, session: Session, *, only=None, force=False,
3784
  scope=None, retry_id=None):
 
 
 
 
 
 
 
 
 
 
3785
  from core import sheets
3786
  store = _store(session)
3787
  tenant_keys = (getattr(store, "key", None), getattr(getattr(store, "tenant", None), "key", None))
 
2995
  target = next((tab for tab in visible if tab["id"] == open_tab),
2996
  visible[0] if visible else None)
2997
  if target is not None:
2998
+ args = (str(tabs or ""), bool(opening), str(open_tab or ""), target["id"])
2999
+ cached = _open_cache_get(session, book, args)
3000
+ if cached is not None:
3001
+ out["opened"] = cached
3002
+ return out
3003
  target["cells"] = sheets.get_workbook_tab_cells(
3004
  _store(session), workbook_id, target["id"], expected_rev=book.get("rev"))
3005
  target.pop("cellCount", None)
3006
+ databases = set()
3007
+ out["opened"] = _serve_open_tab(session, book, target, databases=databases)
3008
+ _open_cache_put(session, book, args, out["opened"], databases)
3009
  return out
3010
  except sheets.SheetRefusal as exc:
3011
  raise _refuse(exc)
 
3161
 
3162
  @router.post("/sheets/{workbook_id}/refresh-history/{refresh_id}/retry")
3163
  def retry_sheet_refresh(workbook_id: str, refresh_id: str, session: Session = Depends(require_session)):
3164
+ if _refresh_desk_queues(session):
3165
+ return _queue_refresh(workbook_id, session, {"kind": "retry", "retryId": str(refresh_id)})
3166
  with _sheet_request_scope(session):
3167
  return _refresh_sheet_scoped(workbook_id, session, retry_id=refresh_id)
3168
 
 
3207
  raise _refuse(exc)
3208
 
3209
 
3210
+ def _serve_open_tab(session, book, target, *, databases=None):
3211
+ """Serve the selected tab and its dependency closure under one request's fresh walls.
3212
+
3213
+ `databases`, when given, collects every database a served or loaded tab binds.
3214
+ """
3215
  from core import sheets
3216
 
3217
  def load_tab(tab_id):
3218
+ cells = sheets.get_workbook_tab_cells(_store(session), book["id"], tab_id,
3219
  expected_rev=book.get("rev"))
3220
+ if databases is not None:
3221
+ databases.update(_bound_databases(cells))
3222
+ return cells
3223
+
3224
+ if databases is not None:
3225
+ databases.update(_bound_databases(target.get("cells")))
3226
 
3227
  with _sheet_request_scope(session):
3228
  served = _read_tab_for(session, book, target, load_tab=load_tab)
 
3694
  # `drill` with real resolution through the mirror seam; W49-T03 supplies formula evaluation.
3695
 
3696
 
3697
+ # ── the refresh desk: the web role queues a person's refresh and the sync worker runs it ─────────
3698
+ import time as _desk_time # noqa: E402
3699
+
3700
+ #: One store document per tenant holds the queued refreshes. `sheet_warm` drains it in the worker.
3701
+ REFRESH_REQUESTS_KEY = "sheet_refresh_requests_v1"
3702
+ #: A request still running after this long goes back in the queue once, then fails.
3703
+ REFRESH_STALE_SECONDS = 15 * 60
3704
+ REFRESH_MAX_ATTEMPTS = 2
3705
+ #: A finished request stays readable this long, so a polling client always finds its answer.
3706
+ REFRESH_KEEP_SECONDS = 3600
3707
+ _REFRESH_FAILED = {"code": "sheet_unresolved", "message": "This workbook could not be refreshed."}
3708
+ _REFRESH_TIMEOUT = {"code": "sheet_refresh_timeout",
3709
+ "message": "This refresh took too long. Try again in a few minutes."}
3710
+
3711
+
3712
+ def _refresh_desk_queues(session):
3713
+ """True in the web role for a person's own session. Read per call, so a gate can patch the role."""
3714
+ import process_role
3715
+ if process_role.role() != "web":
3716
+ return False
3717
+ claims = getattr(session, "claims", None) or {}
3718
+ # An external grant narrows by claim and the worker rebuilds no claims, so it resolves here.
3719
+ return not any(str(key).startswith("external") and value for key, value in claims.items())
3720
+
3721
+
3722
+ def _refresh_body_args(body):
3723
+ """The refresh arguments the route accepts, as one plain request entry."""
3724
+ if isinstance(body, dict) and "scope" in body:
3725
+ return {"kind": "scope", "scope": body["scope"]}
3726
+ wanted = body.get("cells") if isinstance(body, dict) else None
3727
+ if isinstance(wanted, list):
3728
+ return {"kind": "cells",
3729
+ "cells": sorted({str(item) for item in wanted if isinstance(item, str) and "!" in item})}
3730
+ return {"kind": "workbook"}
3731
+
3732
+
3733
+ def _refresh_desk_merge(old, new):
3734
+ """One request that covers both, or None when the two cannot share a run."""
3735
+ if old == new:
3736
+ return old
3737
+ kinds = {old.get("kind"), new.get("kind")}
3738
+ if "retry" in kinds:
3739
+ return None
3740
+ if kinds == {"cells"}:
3741
+ return {"kind": "cells", "cells": sorted(set(old.get("cells") or ()) | set(new.get("cells") or ()))}
3742
+ if kinds == {"cells", "workbook"}:
3743
+ return {"kind": "workbook"}
3744
+ return {"kind": "scope", "scope": {"workbook": True}}
3745
+
3746
+
3747
+ def _refresh_desk_rows(doc, now):
3748
+ """The live request rows. A finished row older than REFRESH_KEEP_SECONDS falls out."""
3749
+ rows = (doc or {}).get("requests") if isinstance(doc, dict) else None
3750
+ return {key: dict(row) for key, row in (rows or {}).items() if isinstance(row, dict)
3751
+ and not (row.get("state") in ("done", "failed")
3752
+ and now - float(row.get("finishedTs") or 0) > REFRESH_KEEP_SECONDS)}
3753
+
3754
+
3755
+ def _refresh_desk_order(row):
3756
+ return (float(row.get("requestedTs") or 0), str(row.get("id") or ""))
3757
+
3758
+
3759
+ def _desk_iso(ts):
3760
+ from datetime import datetime, timezone
3761
+ return datetime.fromtimestamp(ts, timezone.utc).replace(microsecond=0).isoformat()
3762
+
3763
+
3764
+ def _refresh_desk_enqueue(store, tenant, workbook, reader, args, now=None, path=None):
3765
+ """Queue one request, or fold it into the same reader's queued request for this workbook."""
3766
+ now = _desk_time.time() if now is None else now
3767
+ made = {}
3768
+
3769
+ def apply(current):
3770
+ rows = _refresh_desk_rows(current, now)
3771
+ for row in sorted(rows.values(), key=_refresh_desk_order):
3772
+ if (row.get("state") == "queued" and row.get("workbook") == workbook
3773
+ and row.get("reader") == reader):
3774
+ merged = _refresh_desk_merge(row.get("args") or {}, args)
3775
+ if merged is not None:
3776
+ row["args"] = merged
3777
+ row["coalesced"] = int(row.get("coalesced") or 0) + 1
3778
+ rows[row["id"]] = row
3779
+ made["row"] = dict(row)
3780
+ return {"requests": rows}
3781
+ row = {"id": uuid.uuid4().hex, "tenant": tenant, "workbook": workbook, "reader": reader,
3782
+ "path": str(path or workbook), "args": args, "state": "queued", "attempts": 0,
3783
+ "requestedTs": now,
3784
+ "requestedAt": _desk_iso(now), "finishedAt": None, "error": None}
3785
+ rows[row["id"]] = row
3786
+ made["row"] = dict(row)
3787
+ return {"requests": rows}
3788
+
3789
+ store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
3790
+ return made["row"]
3791
+
3792
+
3793
+ def _refresh_desk_claim(store, now=None):
3794
+ """Mark the oldest queued request running and return it. None while one is already running."""
3795
+ now = _desk_time.time() if now is None else now
3796
+ claimed = {}
3797
+
3798
+ def stale(row):
3799
+ return row.get("state") == "running" and now - float(row.get("startedTs") or 0) > REFRESH_STALE_SECONDS
3800
+
3801
+ # Read first: an idle poll must never write the store.
3802
+ rows = _refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now).values()
3803
+ busy = any(row.get("state") == "running" and not stale(row) for row in rows)
3804
+ if not any(stale(row) for row in rows) and (busy or not any(row.get("state") == "queued" for row in rows)):
3805
+ return None
3806
+
3807
+ def apply(current):
3808
+ rows = _refresh_desk_rows(current, now)
3809
+ for row in rows.values():
3810
+ if stale(row):
3811
+ if int(row.get("attempts") or 0) < REFRESH_MAX_ATTEMPTS:
3812
+ row["state"] = "queued"
3813
+ else:
3814
+ row.update(state="failed", finishedTs=now, finishedAt=_desk_iso(now),
3815
+ error=dict(_REFRESH_TIMEOUT))
3816
+ if not any(row.get("state") == "running" for row in rows.values()):
3817
+ queued = sorted((row for row in rows.values() if row.get("state") == "queued"),
3818
+ key=_refresh_desk_order)
3819
+ if queued:
3820
+ row = queued[0]
3821
+ row.update(state="running", startedTs=now, startedAt=_desk_iso(now),
3822
+ attempts=int(row.get("attempts") or 0) + 1)
3823
+ claimed["row"] = dict(row)
3824
+ return {"requests": rows}
3825
+
3826
+ store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
3827
+ return claimed.get("row")
3828
+
3829
+
3830
+ def _refresh_desk_finish(store, request_id, error=None, now=None):
3831
+ """Mark one running request done, or failed with a public `{code, message}`."""
3832
+ now = _desk_time.time() if now is None else now
3833
+
3834
+ def apply(current):
3835
+ rows = _refresh_desk_rows(current, now)
3836
+ row = rows.get(request_id)
3837
+ if row is not None and row.get("state") == "running":
3838
+ row.update(state="failed" if error else "done", finishedTs=now,
3839
+ finishedAt=_desk_iso(now), error=dict(error) if error else None)
3840
+ return {"requests": rows}
3841
+
3842
+ store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
3843
+
3844
+
3845
+ def _refresh_desk_entry(store, workbook_id, reader, request_id):
3846
+ """The caller's own request row, or None for an unknown id or another person's request."""
3847
+ doc = store.get(REFRESH_REQUESTS_KEY, fresh=True) or {}
3848
+ row = ((doc.get("requests") or {}) if isinstance(doc, dict) else {}).get(str(request_id or ""))
3849
+ if (not isinstance(row, dict) or not reader or row.get("reader") != reader
3850
+ or str(workbook_id) not in (row.get("workbook"), row.get("path"))):
3851
+ return None
3852
+ return row
3853
+
3854
+
3855
+ def _refresh_desk_pending(store):
3856
+ """True when the document holds a queued or running request."""
3857
+ doc = store.get(REFRESH_REQUESTS_KEY, fresh=True) or {}
3858
+ rows = (doc.get("requests") or {}) if isinstance(doc, dict) else {}
3859
+ return any(isinstance(row, dict) and row.get("state") in ("queued", "running")
3860
+ for row in rows.values())
3861
+
3862
+
3863
+ def _queue_refresh(workbook_id, session, args):
3864
+ """Check the caller and workbook as the in-process refresh does, then queue it. Answers 202."""
3865
+ from core import sheets
3866
+ from fastapi.responses import JSONResponse
3867
+ store = _store(session)
3868
+ tenant_keys = (getattr(store, "key", None), getattr(getattr(store, "tenant", None), "key", None))
3869
+ if any(key and key != session.tenant for key in tenant_keys):
3870
+ raise err(409, "tenant_mismatch", "The caller and workbook tenants differ.")
3871
+ try:
3872
+ book = sheets.get_workbook(store, workbook_id)
3873
+ _require_reader(session, book)
3874
+ sheets._refresh_caller(store, book, session)
3875
+ if args.get("kind") == "scope":
3876
+ _refresh_scope_cells(book, args.get("scope"))
3877
+ except sheets.SheetRefusal as exc:
3878
+ raise _refuse(exc)
3879
+ row = _refresh_desk_enqueue(store, session.tenant, str(book.get("id") or workbook_id),
3880
+ session.uname, args, path=workbook_id)
3881
+ return JSONResponse(status_code=202, content={"queued": True, "requestId": row["id"],
3882
+ "state": row["state"]})
3883
+
3884
+
3885
+ def _refresh_desk_error(exc):
3886
+ """The public `{code, message}` a failed request carries."""
3887
+ from core import sheets
3888
+ if isinstance(exc, sheets.SheetRefusal):
3889
+ return {"code": str(exc.code), "message": str(exc.message or _REFRESH_FAILED["message"])}
3890
+ if isinstance(exc, HTTPException):
3891
+ info = ((exc.detail or {}).get("error") if isinstance(exc.detail, dict) else None) or {}
3892
+ return {"code": str(info.get("code") or _REFRESH_FAILED["code"]),
3893
+ "message": str(info.get("message") or _REFRESH_FAILED["message"])}
3894
+ return dict(_REFRESH_FAILED)
3895
+
3896
+
3897
+ def _refresh_desk_run(session, row):
3898
+ """Run one claimed request as its reader through the in-process refresh resolver."""
3899
+ from core import sheets
3900
+ args = row.get("args") or {}
3901
+ workbook = row["workbook"]
3902
+ if args.get("kind") == "scope":
3903
+ return sheets.refresh_scope(session.runtime, workbook, args.get("scope"), session)
3904
+ with _sheet_request_scope(session):
3905
+ if args.get("kind") == "retry":
3906
+ return _refresh_sheet_scoped(workbook, session, retry_id=args.get("retryId"))
3907
+ only = set(args.get("cells") or ()) if args.get("kind") == "cells" else None
3908
+ return _refresh_sheet_scoped(workbook, session, only=only)
3909
+
3910
+
3911
+ # ── the opening cache: a repeat open of the same tab at the same state is served from memory ─────
3912
+ #: Entries kept, least recently used out first.
3913
+ OPEN_CACHE_ENTRIES = 64
3914
+ #: A cached open older than this recomputes. It bounds what a key cannot see, such as a write
3915
+ #: another process made to a store whose revision counter is per process.
3916
+ OPEN_CACHE_TTL_SECONDS = 120.0
3917
+ _OPEN_CACHE = OrderedDict()
3918
+ _OPEN_CACHE_LOCK = threading.Lock()
3919
+ #: User record keys that change on sign in and never change what the reader may see.
3920
+ _OPEN_CACHE_USER_NOISE = frozenset(("last_login", "updated_at", "epoch", "avatar", "whatsapp"))
3921
+
3922
+
3923
+ def _open_cache_enabled():
3924
+ """On in the web role only. `AIOS_SHEET_OPEN_CACHE=0` turns it off there too."""
3925
+ import process_role
3926
+ return (process_role.role() == "web"
3927
+ and os.environ.get("AIOS_SHEET_OPEN_CACHE", "1").strip() != "0")
3928
+
3929
+
3930
+ def _install_generation():
3931
+ """The serving mirror's install generation, or None when it cannot be read."""
3932
+ try:
3933
+ from harness import datastore
3934
+ return datastore.mirror_generation()
3935
+ except Exception: # noqa: BLE001
3936
+ return None
3937
+
3938
+
3939
+ def _doc_token(store, key):
3940
+ """A cheap change token for one store document, or None when none can be read."""
3941
+ from core.sheet_binding import fingerprint
3942
+ revision = getattr(store, "revision", None)
3943
+ try:
3944
+ if callable(revision):
3945
+ return "rev:" + fingerprint(revision(key))
3946
+ return "doc:" + fingerprint(store.get(key) or {})
3947
+ except Exception: # noqa: BLE001
3948
+ return None
3949
+
3950
+
3951
+ def _bound_databases(cells):
3952
+ """Every database name a tab's bindings mention, related databases included."""
3953
+ found = set()
3954
+
3955
+ def walk(value):
3956
+ if isinstance(value, dict):
3957
+ for key, item in value.items():
3958
+ if key == "database" and isinstance(item, str) and item:
3959
+ found.add(item)
3960
+ else:
3961
+ walk(item)
3962
+ elif isinstance(value, list):
3963
+ for item in value:
3964
+ walk(item)
3965
+
3966
+ for cell in (cells or {}).values() if isinstance(cells, dict) else ():
3967
+ if isinstance(cell, dict) and cell.get("binding"):
3968
+ walk(cell.get("binding"))
3969
+ return found
3970
+
3971
+
3972
+ def _open_cache_key(session, book, args):
3973
+ """Everything an opened tab's body depends on for one reader, or None when it cannot be keyed."""
3974
+ from core import shares, sheets
3975
+ from core.sheet_binding import fingerprint
3976
+ store = _store(session)
3977
+ workbook = str(book.get("id") or "")
3978
+ refresh_doc = _doc_token(store, sheets._scope_cache_key(workbook))
3979
+ grants_doc = _doc_token(store, shares.SHARES_KEY)
3980
+ if not workbook or not session.uname or refresh_doc is None or grants_doc is None:
3981
+ return None
3982
+ policy = {key: value for key, value in (session.user or {}).items()
3983
+ if key not in _OPEN_CACHE_USER_NOISE}
3984
+ return (str(session.tenant), workbook, book.get("rev"), session.uname, fingerprint(policy),
3985
+ refresh_doc, grants_doc, json.dumps(_install_generation(), default=str),
3986
+ _sheet_today(session), tuple(args))
3987
+
3988
+
3989
+ def _open_cache_sources(session, databases):
3990
+ """`{database: change token}`, or None when any token cannot be read."""
3991
+ from core import sheet_binding
3992
+ out = {}
3993
+ for database in sorted(databases):
3994
+ try:
3995
+ import harness.mirror_tokens # noqa: F401
3996
+ out[database] = sheet_binding.database_change_token(_store(session), database)
3997
+ except Exception: # noqa: BLE001
3998
+ return None
3999
+ if not out[database]:
4000
+ return None
4001
+ return out
4002
+
4003
+
4004
+ def _open_body_transient(body):
4005
+ """True when a bound cell holds a busy server refusal, which must never stick in the cache."""
4006
+ for cell in (((body or {}).get("tab") or {}).get("cells") or {}).values():
4007
+ refusal = (cell or {}).get("refusal") or {} if isinstance(cell, dict) else {}
4008
+ if (isinstance(cell, dict) and cell.get("kind") == "bound" and cell.get("state") == "refused"
4009
+ and not _refusal_authored(refusal.get("code"), refusal.get("status"))):
4010
+ return True
4011
+ return False
4012
+
4013
+
4014
+ def _open_cache_get(session, book, args):
4015
+ """A deep copy of the cached opened tab for this reader at this state, or None."""
4016
+ from copy import deepcopy
4017
+ if not _open_cache_enabled():
4018
+ return None
4019
+ key = _open_cache_key(session, book, args)
4020
+ if key is None:
4021
+ return None
4022
+ with _OPEN_CACHE_LOCK:
4023
+ entry = _OPEN_CACHE.get(key)
4024
+ if entry is not None and _desk_time.monotonic() - entry["at"] > OPEN_CACHE_TTL_SECONDS:
4025
+ _OPEN_CACHE.pop(key, None)
4026
+ entry = None
4027
+ if entry is not None:
4028
+ _OPEN_CACHE.move_to_end(key)
4029
+ if entry is None:
4030
+ return None
4031
+ if _open_cache_sources(session, entry["sources"]) != entry["sources"]:
4032
+ with _OPEN_CACHE_LOCK:
4033
+ _OPEN_CACHE.pop(key, None)
4034
+ return None
4035
+ return deepcopy(entry["body"])
4036
+
4037
+
4038
+ def _open_cache_put(session, book, args, body, databases):
4039
+ """Keep one served opened tab under the key computed after the serve."""
4040
+ from copy import deepcopy
4041
+ if not _open_cache_enabled() or not isinstance(body, dict) or _open_body_transient(body):
4042
+ return
4043
+ key = _open_cache_key(session, book, args)
4044
+ sources = _open_cache_sources(session, databases) if key is not None else None
4045
+ if sources is None:
4046
+ return
4047
+ with _OPEN_CACHE_LOCK:
4048
+ _OPEN_CACHE[key] = {"body": deepcopy(body), "sources": sources, "at": _desk_time.monotonic()}
4049
+ _OPEN_CACHE.move_to_end(key)
4050
+ while len(_OPEN_CACHE) > OPEN_CACHE_ENTRIES:
4051
+ _OPEN_CACHE.popitem(last=False)
4052
+
4053
+
4054
+ def _open_cache_drop(tenant, workbook_id):
4055
+ """Forget every cached open of one workbook in this process."""
4056
+ with _OPEN_CACHE_LOCK:
4057
+ for key in [key for key in _OPEN_CACHE if key[0] == str(tenant) and key[1] == str(workbook_id)]:
4058
+ _OPEN_CACHE.pop(key, None)
4059
+
4060
+
4061
  @router.post("/sheets/{workbook_id}/refresh")
4062
+ def refresh_sheet_door(workbook_id: str, body: dict = Body(default=None),
4063
+ session: Session = Depends(require_session)):
4064
+ """The HTTP refresh door. The web role queues the work for the sync worker and answers 202."""
4065
+ if _refresh_desk_queues(session):
4066
+ return _queue_refresh(workbook_id, session, _refresh_body_args(body))
4067
+ return refresh_sheet(workbook_id, body, session=session)
4068
+
4069
+
4070
+ @router.get("/sheets/{workbook_id}/refresh-status")
4071
+ def refresh_sheet_status(workbook_id: str, requestId: str = "",
4072
+ session: Session = Depends(require_session)):
4073
+ """One queued refresh's state, readable only by the person who asked for it."""
4074
+ entry = _refresh_desk_entry(_store(session), workbook_id, session.uname, requestId)
4075
+ if entry is None:
4076
+ raise err(404, "sheet_refresh_missing", "That refresh is unavailable.")
4077
+ if entry.get("state") in ("done", "failed"):
4078
+ _open_cache_drop(session.tenant, workbook_id)
4079
+ out = {"requestId": requestId, "state": entry.get("state"),
4080
+ "requestedAt": entry.get("requestedAt"), "finishedAt": entry.get("finishedAt")}
4081
+ if entry.get("state") == "failed":
4082
+ out["error"] = entry.get("error") or {"code": "sheet_unresolved",
4083
+ "message": "This workbook could not be refreshed."}
4084
+ return out
4085
+
4086
+
4087
+ def refresh_sheet(workbook_id: str, body: dict = None, session: Session = None):
4088
  """C2/R4: Refresh resolves only databases whose token or binding inputs changed.
4089
 
4090
  Viewers refresh their own server cache. Only editors save the workbook document.
 
4188
 
4189
  def _refresh_sheet_scoped(workbook_id: str, session: Session, *, only=None, force=False,
4190
  scope=None, retry_id=None):
4191
+ try:
4192
+ return _refresh_sheet_scoped_inner(workbook_id, session, only=only, force=force,
4193
+ scope=scope, retry_id=retry_id)
4194
+ finally:
4195
+ # A refresh moves stored answers, so this process forgets its cached opens of the workbook.
4196
+ _open_cache_drop(session.tenant, workbook_id)
4197
+
4198
+
4199
+ def _refresh_sheet_scoped_inner(workbook_id: str, session: Session, *, only=None, force=False,
4200
+ scope=None, retry_id=None):
4201
  from core import sheets
4202
  store = _store(session)
4203
  tenant_keys = (getattr(store, "key", None), getattr(getattr(store, "tenant", None), "key", None))
api/sheet_warm.py CHANGED
@@ -330,8 +330,111 @@ def _on_exit():
330
  _limit(progress["reason"], progress["tenant"], progress["left"], "shutdown")
331
 
332
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
333
  def start():
334
  """Start the one "sheet-warm" daemon thread when enabled. Returns it, or None."""
 
 
335
  if not enabled():
336
  print("[sheet-warm] off: AIOS_SHEET_WARM=0", flush=True)
337
  return None
 
330
  _limit(progress["reason"], progress["tenant"], progress["left"], "shutdown")
331
 
332
 
333
+ # ── the refresh desk: person refreshes the web role queued, run here one at a time ──────────────
334
+ #: Seconds between two reads of each tenant's request document.
335
+ DESK_POLL_SECONDS = 5.0
336
+ _DESK = {"thread": None, "seen": {}}
337
+
338
+
339
+ def _desk_tenants():
340
+ # Active tenants only: an inactive one has no runtime and would fail every 5 s.
341
+ import tenant_scan
342
+ from harness import runtime
343
+ return [slug for slug, rt in tenant_scan.live_tenants(runtime) if rt is not None]
344
+
345
+
346
+ def _desk_skip(rt, tenant):
347
+ """True when a cross process revision proves the document did not move since an empty read."""
348
+ seen = _DESK["seen"].get(tenant)
349
+ if seen is None:
350
+ return False
351
+ try:
352
+ token = rt.revision(routes_sheet.REFRESH_REQUESTS_KEY)
353
+ except Exception: # noqa: BLE001
354
+ return False
355
+ text = token.get("token") if isinstance(token, dict) else token
356
+ # Only the Postgres revision sees another process's write. Any other token reads the document.
357
+ return str(text or "").startswith("pg:") and token == seen
358
+
359
+
360
+ def desk_run_one(rt, tenant):
361
+ """Claim and run the oldest queued refresh of one tenant. Returns True when one ran."""
362
+ row = routes_sheet._refresh_desk_claim(rt)
363
+ if row is None:
364
+ return False
365
+ started = perf_counter()
366
+ error = None
367
+ try:
368
+ book = sheets.get_workbook(rt, row["workbook"])
369
+ rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt)
370
+ session = reader_session(rt, tenant, row.get("reader"), book, rec)
371
+ if session is None:
372
+ error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
373
+ else:
374
+ routes_sheet._refresh_desk_run(session, row)
375
+ except Exception as exc: # noqa: BLE001
376
+ error = routes_sheet._refresh_desk_error(exc)
377
+ routes_sheet._refresh_desk_finish(rt, row["id"], error)
378
+ print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} "
379
+ f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}"
380
+ + (f" code={error['code']}" if error else ""), flush=True)
381
+ return True
382
+
383
+
384
+ def desk_drain(tenant):
385
+ """Run every queued refresh of one tenant, oldest first. Returns how many ran."""
386
+ from harness import runtime
387
+ rt = runtime.get_runtime(tenant)
388
+ if _desk_skip(rt, tenant):
389
+ return 0
390
+ # The revision is taken BEFORE the reads, so a request queued between them still moves it.
391
+ try:
392
+ before = rt.revision(routes_sheet.REFRESH_REQUESTS_KEY)
393
+ except Exception: # noqa: BLE001
394
+ before = None
395
+ ran = 0
396
+ while not _STOP.is_set() and desk_run_one(rt, tenant):
397
+ ran += 1
398
+ if not ran and before is not None and not routes_sheet._refresh_desk_pending(rt):
399
+ _DESK["seen"][tenant] = before
400
+ else:
401
+ _DESK["seen"].pop(tenant, None)
402
+ return ran
403
+
404
+
405
+ def _desk_loop():
406
+ while not _STOP.is_set():
407
+ try:
408
+ tenants = _desk_tenants()
409
+ except Exception as exc: # noqa: BLE001
410
+ print(f"[sheet-desk] tenant list failed {type(exc).__name__}: {exc}", flush=True)
411
+ tenants = []
412
+ for tenant in tenants:
413
+ if _STOP.is_set():
414
+ return
415
+ try:
416
+ desk_drain(tenant)
417
+ except Exception as exc: # noqa: BLE001
418
+ print(f"[sheet-desk] tenant={tenant} failed {type(exc).__name__}: {exc}", flush=True)
419
+ _STOP.wait(DESK_POLL_SECONDS)
420
+
421
+
422
+ def start_desk():
423
+ """Start the one "sheet-refresh-desk" thread where heavy jobs run. Returns it, or None."""
424
+ with _LOCK:
425
+ thread = _DESK["thread"]
426
+ if thread is not None and thread.is_alive():
427
+ return thread
428
+ import process_role
429
+ thread = process_role.start("sheet-refresh-desk", "heavy", _desk_loop)
430
+ _DESK["thread"] = thread
431
+ return thread
432
+
433
+
434
  def start():
435
  """Start the one "sheet-warm" daemon thread when enabled. Returns it, or None."""
436
+ # Person refreshes never depend on AIOS_SHEET_WARM, so the desk starts first.
437
+ start_desk()
438
  if not enabled():
439
  print("[sheet-warm] off: AIOS_SHEET_WARM=0", flush=True)
440
  return None
platform/harness/amazon_store.py CHANGED
@@ -2863,8 +2863,8 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
2863
  def part_max_bytes(self):
2864
  return MIRROR_PART_MAX_BYTES
2865
 
2866
- def _install_snapshot(self, con, manifest, rows, paths):
2867
- result = super()._install_snapshot(con, manifest, rows, paths)
2868
  _note_published() # C2: AFTER_SYNC runs only for a pass that published.
2869
  return result
2870
 
@@ -2891,8 +2891,8 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
2891
  _assert_personal_storage(con, name)
2892
  return super().persist_mirror(tenant_key, rt, con, changed_tables, report=report, log=log, **kwargs)
2893
 
2894
- def _install_snapshot(self, con, manifest, rows, paths):
2895
- super()._install_snapshot(con, manifest, rows, paths)
2896
  refresh_sku_keywords(con)
2897
 
2898
  def _publication_holds(self, report, waiting):
 
2863
  def part_max_bytes(self):
2864
  return MIRROR_PART_MAX_BYTES
2865
 
2866
+ def _install_snapshot(self, con, manifest, rows, paths, only=None):
2867
+ result = super()._install_snapshot(con, manifest, rows, paths, only=only)
2868
  _note_published() # C2: AFTER_SYNC runs only for a pass that published.
2869
  return result
2870
 
 
2891
  _assert_personal_storage(con, name)
2892
  return super().persist_mirror(tenant_key, rt, con, changed_tables, report=report, log=log, **kwargs)
2893
 
2894
+ def _install_snapshot(self, con, manifest, rows, paths, only=None):
2895
+ super()._install_snapshot(con, manifest, rows, paths, only=only)
2896
  refresh_sku_keywords(con)
2897
 
2898
  def _publication_holds(self, report, waiting):
platform/harness/mirror_store.py CHANGED
@@ -72,6 +72,14 @@ def _notify_publish(connector, tenant_key, manifest, rows, paths, log=print):
72
  log(f"{connector} mirror restore hand-off failed ({type(exc).__name__}: {exc})")
73
 
74
 
 
 
 
 
 
 
 
 
75
  def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
76
  """Install a snapshot another process already published. Returns False if it is already resident."""
77
  store = for_connector(connector)
@@ -80,10 +88,11 @@ def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
80
  with store.lock:
81
  con, _path = store._connect(tenant_key, rt)
82
  try:
83
- if (manifest.get("snapshot_id") and
84
- datastore._publication(con, store.connector).get("snapshot_id") == manifest["snapshot_id"]):
85
  return False
86
- store._install_snapshot(con, manifest, [tuple(row) for row in rows], paths)
 
87
  newest = _record_newest(con, store.connector)
88
  finally:
89
  con.close()
@@ -647,13 +656,18 @@ class MirrorStore:
647
  return paths, sorted(refused)
648
 
649
 
650
- def _install_snapshot(self, con, manifest, rows, paths):
651
- """Commit one source's tables, sync decisions and identity together."""
 
 
 
652
  con.execute("BEGIN TRANSACTION")
653
  try:
654
  for name in self._tables(con) - set(paths):
655
  con.execute(f"DROP TABLE {_quoted_identifier(name)}")
656
  for name, files in paths.items():
 
 
657
  con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
658
  f"SELECT * FROM {_parquet_source(files)}")
659
  old_rows = self._state_rows(con)
 
72
  log(f"{connector} mirror restore hand-off failed ({type(exc).__name__}: {exc})")
73
 
74
 
75
+ def changed_tables(previous, manifest, resident, paths):
76
+ """Tables a hand-off must rebuild: new, missing locally, or with different stored content."""
77
+ before = (previous or {}).get("tables") or {}
78
+ after = (manifest or {}).get("tables") or {}
79
+ return {name for name in paths
80
+ if name not in resident or name not in before or name not in after or before[name] != after[name]}
81
+
82
+
83
  def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
84
  """Install a snapshot another process already published. Returns False if it is already resident."""
85
  store = for_connector(connector)
 
88
  with store.lock:
89
  con, _path = store._connect(tenant_key, rt)
90
  try:
91
+ previous = datastore._publication(con, store.connector)
92
+ if manifest.get("snapshot_id") and previous.get("snapshot_id") == manifest["snapshot_id"]:
93
  return False
94
+ only = changed_tables(previous, manifest, store._tables(con), paths)
95
+ store._install_snapshot(con, manifest, [tuple(row) for row in rows], paths, only=only)
96
  newest = _record_newest(con, store.connector)
97
  finally:
98
  con.close()
 
656
  return paths, sorted(refused)
657
 
658
 
659
+ def _install_snapshot(self, con, manifest, rows, paths, only=None):
660
+ """Commit one source's tables, sync decisions and identity together.
661
+
662
+ `only` names the tables to rebuild; the others in `paths` are already resident and identical.
663
+ """
664
  con.execute("BEGIN TRANSACTION")
665
  try:
666
  for name in self._tables(con) - set(paths):
667
  con.execute(f"DROP TABLE {_quoted_identifier(name)}")
668
  for name, files in paths.items():
669
+ if only is not None and name not in only:
670
+ continue
671
  con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
672
  f"SELECT * FROM {_parquet_source(files)}")
673
  old_rows = self._state_rows(con)
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 2dcea8b5389fc55fa58b10312a30dc5e1566e11621c407b26d43e969fd6b2c00
 
1
+ 5161eb153e223f8d3e8a42b7280ccf54bb15dccfc64a86ffd6f43d20cfce9929
web/src/sheet/SheetPage.tsx CHANGED
@@ -67,7 +67,7 @@ import SheetActivity from "./SheetActivity";
67
  import SheetCatchUp from "./SheetCatchUp";
68
  import RefreshHistory from "./RefreshHistory";
69
  import WorkbookChecks from "./WorkbookChecks";
70
- import { readWorkbookChecks, writeWorkbookChecks } from "./sheetApi";
71
  import type { RailStrata } from "./SheetRail";
72
  import {
73
  computeCell, createdFolderIds, createWorkbook, deleteWorkbook, duplicateSheetTab, reorderSheetTab, patchTabCells, readIndex, readTab, readTabMeta,
@@ -284,7 +284,7 @@ type WorkbookOpening = Workbook & { openingTab?: SheetTab & { rev: number } };
284
 
285
  async function readWorkbookOpening(id: string, tab?: string): Promise<Result<WorkbookOpening>> {
286
  try {
287
- const response = await fetch(`/api/v1/sheets/${encodeURIComponent(id)}?tabs=meta&opening=true`
288
  + (tab ? `&open_tab=${encodeURIComponent(tab)}` : ""), { credentials: "same-origin" });
289
  const body = await response.json().catch(() => null) as {
290
  workbook?: Workbook;
 
67
  import SheetCatchUp from "./SheetCatchUp";
68
  import RefreshHistory from "./RefreshHistory";
69
  import WorkbookChecks from "./WorkbookChecks";
70
+ import { fetchRetryingBusy, readWorkbookChecks, writeWorkbookChecks } from "./sheetApi";
71
  import type { RailStrata } from "./SheetRail";
72
  import {
73
  computeCell, createdFolderIds, createWorkbook, deleteWorkbook, duplicateSheetTab, reorderSheetTab, patchTabCells, readIndex, readTab, readTabMeta,
 
284
 
285
  async function readWorkbookOpening(id: string, tab?: string): Promise<Result<WorkbookOpening>> {
286
  try {
287
+ const response = await fetchRetryingBusy(`/api/v1/sheets/${encodeURIComponent(id)}?tabs=meta&opening=true`
288
  + (tab ? `&open_tab=${encodeURIComponent(tab)}` : ""), { credentials: "same-origin" });
289
  const body = await response.json().catch(() => null) as {
290
  workbook?: Workbook;
web/src/sheet/sheetApi.ts CHANGED
@@ -89,14 +89,42 @@ const errorOf = (body: unknown): { code: string; message: string; rev_current?:
89
  };
90
  };
91
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
92
  async function call<T>(
93
  path: string,
94
  init: RequestInit,
95
  read: (body: unknown) => T,
 
96
  ): Promise<Result<T>> {
97
  let res: Response;
98
  try {
99
- res = await fetch(`${API_V1}${path}`, { credentials: CREDENTIALS, ...init });
 
 
100
  } catch {
101
  return { ok: false, status: 0, code: "unreachable", message: "Cannot reach the server." };
102
  }
@@ -267,7 +295,7 @@ export const createWorkbook = (name = ""): Promise<Result<Workbook>> =>
267
  * different rows of the same database are never handed each other's cached number.
268
  */
269
  export const readWorkbook = (id: string): Promise<Result<Workbook>> =>
270
- call(`/sheets/${encodeURIComponent(id)}`, { method: "GET" }, asWorkbook);
271
 
272
  /** Counts and attribution only, checked against the workbook's current permissions. */
273
  export const readSheetActivity = (id: string, signal?: AbortSignal): Promise<Result<SheetActivityResult>> =>
@@ -429,7 +457,7 @@ export const patchTabCells = (
429
 
430
  /** The workbook's identity, revision and tab list, without the cells. */
431
  export const readTabMeta = (id: string): Promise<Result<Workbook>> =>
432
- call(`/sheets/${encodeURIComponent(id)}?tabs=meta`, { method: "GET" }, asWorkbook);
433
 
434
  /**
435
  * ⚠ IT CAN ANSWER `null`, AND THE CALLER MUST SAY SO. A body this parser does not recognise is
@@ -449,6 +477,7 @@ export const readTab = (id: string, tab: string): Promise<Result<SheetTab | null
449
  `/sheets/${encodeURIComponent(id)}/tabs/${encodeURIComponent(tab)}`,
450
  { method: "GET" },
451
  asTab,
 
452
  );
453
 
454
  // ── the doors W49-T04 fills in ───────────────────────────────────────────────────────────────
@@ -506,8 +535,8 @@ export const readRefreshHistory = (id: string, signal?: AbortSignal): Promise<Re
506
  });
507
 
508
  export const retryRefresh = (id: string, refreshId: string): Promise<Result<RefreshScopeResult>> =>
509
- call(`/sheets/${encodeURIComponent(id)}/refresh-history/${encodeURIComponent(refreshId)}/retry`,
510
- { method: "POST" }, refreshScopeResponse);
511
 
512
  /** Source changes keep the old value until an explicit refresh. */
513
  export const isSourceStale = (cell: Cell): boolean =>
@@ -536,6 +565,81 @@ function refreshScopeResponse(body: unknown): RefreshScopeResult {
536
  : { workbook: asWorkbook(body) }) };
537
  }
538
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
539
  /** Share one scope's pending request within the caller's mounted page. */
540
  export const refreshScope = (
541
  id: string,
@@ -545,7 +649,8 @@ export const refreshScope = (
545
  const key = JSON.stringify([id, scope]);
546
  const pending = inFlight?.get(key);
547
  if (pending) return pending;
548
- const request = call(`/sheets/${encodeURIComponent(id)}/refresh`, sends("POST", { scope }), refreshScopeResponse)
 
549
  .finally(() => { inFlight?.delete(key); });
550
  inFlight?.set(key, request);
551
  return request;
@@ -558,12 +663,15 @@ export const refreshWorkbook = (
558
  ): Promise<Result<RefreshResult>> => {
559
  const pending = inFlight?.get(id);
560
  if (pending) return pending;
561
- const request = call(`/sheets/${encodeURIComponent(id)}/refresh`, { method: "POST" }, (body) => ({
562
  workbook: asWorkbook(body),
563
  recalcMs: typeof (body as { recalc_ms?: unknown } | null)?.recalc_ms === "number"
564
  ? (body as { recalc_ms: number }).recalc_ms
565
  : null,
566
- })).finally(() => { inFlight?.delete(id); });
 
 
 
567
  inFlight?.set(id, request);
568
  return request;
569
  };
@@ -596,18 +704,51 @@ export interface RefreshedCell extends CellComputed {
596
  export const refreshCells = (
597
  id: string, cells: readonly string[],
598
  ): Promise<Result<{ rev: number | null; cells: Record<string, RefreshedCell> }>> =>
599
- call(`/sheets/${encodeURIComponent(id)}/refresh`, sends("POST", { cells }), (body) => {
600
- const raw = (body as { workbook?: { rev?: unknown }; cells?: unknown } | null) ?? {};
601
- const rev = typeof raw.workbook?.rev === "number" ? raw.workbook.rev : null;
602
- const out: Record<string, RefreshedCell> = {};
603
- for (const [key, row] of Object.entries(asComputed(raw.cells))) {
604
- const extra = ((raw.cells as Record<string, Record<string, unknown>>)[key]) ?? {};
605
- out[key] = { ...row,
606
- ...(typeof extra.refreshed_at === "string" ? { refreshed_at: extra.refreshed_at } : {}),
607
- ...(Array.isArray(extra.limits) ? { limits: extra.limits } : {}) };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
608
  }
609
- return { rev, cells: out };
610
- });
 
611
 
612
  /**
613
  * W54-T32 (instruction 9): what `POST /sheets/{id}/cells/{addr}/compute` answers.
 
89
  };
90
  };
91
 
92
+ /** A queued heavy request the server could not start in time is retried this many times. */
93
+ export const BUSY_RETRIES = 3;
94
+ /** Seconds to wait before a retry when the server sends no Retry-After header. */
95
+ export const BUSY_DEFAULT_WAIT_S = 5;
96
+
97
+ /**
98
+ * fetch, but a 503 `server_busy` waits its Retry-After seconds and tries again, at most
99
+ * BUSY_RETRIES times. The last answer is returned as it came, so the caller's failure copy shows.
100
+ */
101
+ export async function fetchRetryingBusy(
102
+ input: string, init?: RequestInit,
103
+ wait: (ms: number) => Promise<void> = (ms) => new Promise((done) => { setTimeout(done, ms); }),
104
+ ): Promise<Response> {
105
+ for (let attempt = 0; ; attempt += 1) {
106
+ const res = await fetch(input, init);
107
+ if (res.status !== 503 || attempt >= BUSY_RETRIES) return res;
108
+ const body = (await res.clone().json().catch(() => null)) as { error?: { code?: string } } | null;
109
+ if (body?.error?.code !== "server_busy") return res;
110
+ const header = res.headers.get("Retry-After");
111
+ const seconds = header === null || !Number.isFinite(Number(header)) || Number(header) < 0
112
+ ? BUSY_DEFAULT_WAIT_S : Number(header);
113
+ await wait(seconds * 1000);
114
+ }
115
+ }
116
+
117
  async function call<T>(
118
  path: string,
119
  init: RequestInit,
120
  read: (body: unknown) => T,
121
+ retryBusy = false,
122
  ): Promise<Result<T>> {
123
  let res: Response;
124
  try {
125
+ const request = { credentials: CREDENTIALS, ...init };
126
+ res = retryBusy ? await fetchRetryingBusy(`${API_V1}${path}`, request)
127
+ : await fetch(`${API_V1}${path}`, request);
128
  } catch {
129
  return { ok: false, status: 0, code: "unreachable", message: "Cannot reach the server." };
130
  }
 
295
  * different rows of the same database are never handed each other's cached number.
296
  */
297
  export const readWorkbook = (id: string): Promise<Result<Workbook>> =>
298
+ call(`/sheets/${encodeURIComponent(id)}`, { method: "GET" }, asWorkbook, true);
299
 
300
  /** Counts and attribution only, checked against the workbook's current permissions. */
301
  export const readSheetActivity = (id: string, signal?: AbortSignal): Promise<Result<SheetActivityResult>> =>
 
457
 
458
  /** The workbook's identity, revision and tab list, without the cells. */
459
  export const readTabMeta = (id: string): Promise<Result<Workbook>> =>
460
+ call(`/sheets/${encodeURIComponent(id)}?tabs=meta`, { method: "GET" }, asWorkbook, true);
461
 
462
  /**
463
  * ⚠ IT CAN ANSWER `null`, AND THE CALLER MUST SAY SO. A body this parser does not recognise is
 
477
  `/sheets/${encodeURIComponent(id)}/tabs/${encodeURIComponent(tab)}`,
478
  { method: "GET" },
479
  asTab,
480
+ true,
481
  );
482
 
483
  // ── the doors W49-T04 fills in ───────────────────────────────────────────────────────────────
 
535
  });
536
 
537
  export const retryRefresh = (id: string, refreshId: string): Promise<Result<RefreshScopeResult>> =>
538
+ postRefresh(id, `/sheets/${encodeURIComponent(id)}/refresh-history/${encodeURIComponent(refreshId)}/retry`,
539
+ { method: "POST" }, refreshScopeResponse, () => reloadWholeWorkbook(id));
540
 
541
  /** Source changes keep the old value until an explicit refresh. */
542
  export const isSourceStale = (cell: Cell): boolean =>
 
565
  : { workbook: asWorkbook(body) }) };
566
  }
567
 
568
+ // ── queued refresh: the web server hands a refresh to its sync worker and answers 202 ─────────
569
+
570
+ /** Seconds between two status reads of a queued refresh. */
571
+ const REFRESH_POLL_MS = 3000;
572
+ /** A queued refresh still unfinished after this long reads as a failure, never a spinner forever. */
573
+ const REFRESH_WAIT_MS = 20 * 60 * 1000;
574
+
575
+ type QueuedRefresh = { queued: true; requestId: string };
576
+
577
+ const isQueued = (body: unknown): body is QueuedRefresh =>
578
+ !!body && typeof body === "object" && (body as { queued?: unknown }).queued === true
579
+ && typeof (body as { requestId?: unknown }).requestId === "string";
580
+
581
+ const pause = (ms: number) => new Promise<void>((done) => { setTimeout(done, ms); });
582
+
583
+ /** Poll one queued refresh until the worker marks it done or failed. */
584
+ async function awaitQueuedRefresh(id: string, requestId: string): Promise<Result<true>> {
585
+ const path = `/sheets/${encodeURIComponent(id)}/refresh-status?requestId=${encodeURIComponent(requestId)}`;
586
+ const deadline = Date.now() + REFRESH_WAIT_MS;
587
+ while (Date.now() < deadline) {
588
+ await pause(REFRESH_POLL_MS);
589
+ const got = await call(path, { method: "GET" }, (body) => body as {
590
+ state?: string; error?: { code?: string; message?: string } });
591
+ if (!got.ok) {
592
+ // A dropped or busy status read is not an answer: the worker may still be refreshing.
593
+ if (got.status === 0 || got.code === "server_busy") continue;
594
+ return got;
595
+ }
596
+ if (got.value.state === "done") return { ok: true, value: true };
597
+ if (got.value.state === "failed") {
598
+ return { ok: false, status: 500, code: got.value.error?.code || "sheet_unresolved",
599
+ message: got.value.error?.message || "This workbook could not be refreshed." };
600
+ }
601
+ }
602
+ return { ok: false, status: 0, code: "sheet_refresh_timeout",
603
+ message: "This refresh is taking longer than usual. Reopen the workbook in a few minutes." };
604
+ }
605
+
606
+ /**
607
+ * POST one refresh. A 200 reads as today; a 202 waits for the worker, then runs `reload`, so every
608
+ * caller keeps its own result shape and its existing busy state.
609
+ */
610
+ async function postRefresh<T>(
611
+ id: string, path: string, init: RequestInit, read: (body: unknown) => T,
612
+ reload: () => Promise<Result<T>>,
613
+ ): Promise<Result<T>> {
614
+ const first = await call(path, init, (body) => body, true);
615
+ if (!first.ok) return first;
616
+ if (!isQueued(first.value)) return { ok: true, value: read(first.value) };
617
+ const settled = await awaitQueuedRefresh(id, first.value.requestId);
618
+ if (!settled.ok) return settled;
619
+ return reload();
620
+ }
621
+
622
+ /** The workbook after a queued refresh: the whole document, as a workbook scope answers. */
623
+ const reloadWholeWorkbook = async (id: string): Promise<Result<RefreshScopeResult>> => {
624
+ const got = await readWorkbook(id);
625
+ return got.ok ? { ok: true, value: { addresses: [], recalcMs: null, workbook: got.value } } : got;
626
+ };
627
+
628
+ /** One tab's cells after a queued cell or range refresh, keyed `<tab id>!<ADDR>`. */
629
+ const reloadScopeTab = async (id: string, scope: RefreshScope): Promise<Result<RefreshScopeResult>> => {
630
+ const target = "cell" in scope ? scope.cell : "range" in scope ? scope.range : "";
631
+ const tab = target.includes("!") ? target.slice(0, target.lastIndexOf("!")) : "";
632
+ if (!tab) return reloadWholeWorkbook(id);
633
+ const got = await readTab(id, tab);
634
+ if (!got.ok) return got;
635
+ if (!got.value) return reloadWholeWorkbook(id);
636
+ const found = got.value as SheetTab & { rev?: number };
637
+ const cells = Object.fromEntries(Object.entries(found.cells ?? {})
638
+ .filter(([, cell]) => !!cell).map(([addr, cell]) => [`${found.id}!${addr}`, cell as Cell]));
639
+ return { ok: true, value: { addresses: [], recalcMs: null, cells,
640
+ workbook: { id, rev: typeof found.rev === "number" ? found.rev : 0 } } };
641
+ };
642
+
643
  /** Share one scope's pending request within the caller's mounted page. */
644
  export const refreshScope = (
645
  id: string,
 
649
  const key = JSON.stringify([id, scope]);
650
  const pending = inFlight?.get(key);
651
  if (pending) return pending;
652
+ const request = postRefresh(id, `/sheets/${encodeURIComponent(id)}/refresh`, sends("POST", { scope }),
653
+ refreshScopeResponse, () => ("workbook" in scope ? reloadWholeWorkbook(id) : reloadScopeTab(id, scope)))
654
  .finally(() => { inFlight?.delete(key); });
655
  inFlight?.set(key, request);
656
  return request;
 
663
  ): Promise<Result<RefreshResult>> => {
664
  const pending = inFlight?.get(id);
665
  if (pending) return pending;
666
+ const request = postRefresh(id, `/sheets/${encodeURIComponent(id)}/refresh`, { method: "POST" }, (body) => ({
667
  workbook: asWorkbook(body),
668
  recalcMs: typeof (body as { recalc_ms?: unknown } | null)?.recalc_ms === "number"
669
  ? (body as { recalc_ms: number }).recalc_ms
670
  : null,
671
+ }), async (): Promise<Result<RefreshResult>> => {
672
+ const got = await readWorkbook(id);
673
+ return got.ok ? { ok: true, value: { workbook: got.value, recalcMs: null } } : got;
674
+ }).finally(() => { inFlight?.delete(id); });
675
  inFlight?.set(id, request);
676
  return request;
677
  };
 
704
  export const refreshCells = (
705
  id: string, cells: readonly string[],
706
  ): Promise<Result<{ rev: number | null; cells: Record<string, RefreshedCell> }>> =>
707
+ postRefresh(id, `/sheets/${encodeURIComponent(id)}/refresh`, sends("POST", { cells }), readRefreshedCells,
708
+ () => reloadRefreshedCells(id, cells));
709
+
710
+ /** What `POST /refresh {cells}` answers, as refreshed cells keyed `<tab id>!<ADDR>`. */
711
+ function readRefreshedCells(body: unknown): { rev: number | null; cells: Record<string, RefreshedCell> } {
712
+ const raw = (body as { workbook?: { rev?: unknown }; cells?: unknown } | null) ?? {};
713
+ const rev = typeof raw.workbook?.rev === "number" ? raw.workbook.rev : null;
714
+ const out: Record<string, RefreshedCell> = {};
715
+ for (const [key, row] of Object.entries(asComputed(raw.cells))) {
716
+ const extra = ((raw.cells as Record<string, Record<string, unknown>>)[key]) ?? {};
717
+ out[key] = { ...row,
718
+ ...(typeof extra.refreshed_at === "string" ? { refreshed_at: extra.refreshed_at } : {}),
719
+ ...(Array.isArray(extra.limits) ? { limits: extra.limits } : {}) };
720
+ }
721
+ return { rev, cells: out };
722
+ }
723
+
724
+ /** After a queued cell refresh, read each named tab again and answer the named cells from it. */
725
+ async function reloadRefreshedCells(
726
+ id: string, cells: readonly string[],
727
+ ): Promise<Result<{ rev: number | null; cells: Record<string, RefreshedCell> }>> {
728
+ const tabs = [...new Set(cells.filter((key) => key.includes("!"))
729
+ .map((key) => key.slice(0, key.lastIndexOf("!"))))];
730
+ const out: Record<string, RefreshedCell> = {};
731
+ let rev: number | null = null;
732
+ for (const tabId of tabs) {
733
+ const got = await readTab(id, tabId);
734
+ if (!got.ok) return got;
735
+ const found = got.value as (SheetTab & { rev?: number }) | null;
736
+ if (!found) continue;
737
+ if (typeof found.rev === "number") rev = Math.max(rev ?? found.rev, found.rev);
738
+ for (const key of cells) {
739
+ if (!key.startsWith(`${tabId}!`)) continue;
740
+ const cell = found.cells?.[key.slice(tabId.length + 1)] as
741
+ (Cell & { refreshed_at?: string; limits?: unknown[] }) | null | undefined;
742
+ if (!cell) continue;
743
+ out[key] = { value: cell.value ?? null, display: typeof cell.display === "string" ? cell.display : "",
744
+ state: typeof cell.state === "string" ? cell.state : "fresh",
745
+ ...(cell.refusal ? { refusal: cell.refusal } : {}),
746
+ ...(typeof cell.refreshed_at === "string" ? { refreshed_at: cell.refreshed_at } : {}),
747
+ ...(Array.isArray(cell.limits) ? { limits: cell.limits } : {}) };
748
  }
749
+ }
750
+ return { ok: true, value: { rev, cells: out } };
751
+ }
752
 
753
  /**
754
  * W54-T32 (instruction 9): what `POST /sheets/{id}/cells/{addr}/compute` answers.