fsanyoto commited on
Commit
bd7fcd1
·
verified ·
1 Parent(s): f811b22

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "31302488a",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "fe5f25c0f",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 31302488a
 
1
+ fe5f25c0f
api/amazon_relational.py CHANGED
@@ -470,11 +470,18 @@ def _read_through(cur, key, count):
470
  return count * size > min(ut.MAX_TABLE_BYTES, AMAZON_STORED_BYTES)
471
 
472
 
 
 
 
 
473
  def missing_source(key):
474
- return {"missing": (
475
- f"The Amazon mirror in this container has not loaded Amazon "
476
- f"{key.removeprefix(PREFIX).replace('_', ' ')} yet. "
477
- "The next Amazon sync loads it. Nothing is lost.")}
 
 
 
478
 
479
 
480
  def _text(value):
 
470
  return count * size > min(ut.MAX_TABLE_BYTES, AMAZON_STORED_BYTES)
471
 
472
 
473
+ #: Set by the boot's resync loop until its Amazon restore ends (`main._restore_amazon_mirror`).
474
+ RESTORE_PENDING = {"since": None}
475
+
476
+
477
  def missing_source(key):
478
+ """The ONE sentence every door gives for an Amazon table this server's mirror does not hold yet."""
479
+ label = key.removeprefix(PREFIX).replace("_", " ")
480
+ if RESTORE_PENDING.get("since") is not None:
481
+ return {"missing": f"Amazon {label} is still loading after a server restart, "
482
+ "so ask again in 5 minutes."}
483
+ return {"missing": f"Amazon {label} is still loading from Amazon, "
484
+ "so ask again after the next Amazon sync finishes."}
485
 
486
 
487
  def _text(value):
api/assistant_mirror.py CHANGED
@@ -93,6 +93,33 @@ def open_cursor(session, table_key=None):
93
  return cur, None
94
 
95
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
  def binding(cur, table_key):
97
  """The resolved read-through spec for one connected grid, or None when it has no binding.
98
 
@@ -340,7 +367,7 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t
340
  # ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short.
341
  first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
342
  where=where, params=tuple(params), order_by=spec["id"],
343
- limit=datastore.WINDOW_MAX, cur=cur, **frm)
344
  total = int(first["total"])
345
  if total > ASSISTANT_READ_MAX:
346
  raise MirrorRefusal(
@@ -358,7 +385,8 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t
358
  more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?"
359
  nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
360
  where=more_where, params=tuple(params) + (last,),
361
- order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, **frm)
 
362
  page = list(nxt["rows"])
363
  if not page:
364
  break
@@ -481,7 +509,7 @@ def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree
481
  frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
482
  win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where,
483
  params=tuple(params), order_by=spec["id"], offset=int(offset),
484
- limit=int(limit), cur=cur, **frm)
485
  stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
486
  defn=defn if isinstance(defn, dict) else None)
487
  rows = rot._pool_from_window(spec, win, stamp=stamp)
 
93
  return cur, None
94
 
95
 
96
+ #: The one sentence for a mirror-served table this deployment has no binding for at all.
97
+ NO_BINDING = ("this database is read live from its connector mirror, and this deployment has no "
98
+ "read-through binding for it yet, so it holds no rows the assistant can read")
99
+
100
+
101
+ def unavailable_reason(table_key, definition=None):
102
+ """The ONE sentence `list_databases` and `query_database` both give when the mirror holds no rows.
103
+
104
+ A bound table whose mirror tables are absent is loading (a restart starts from the boot seed);
105
+ a table with no binding at all gets `NO_BINDING`.
106
+ """
107
+ key = str(table_key or "")
108
+ try:
109
+ if key.startswith("ut_amzn_"):
110
+ import amazon_relational as amazon
111
+ if key in amazon.SOURCE:
112
+ return amazon.missing_source(key)["missing"]
113
+ spec = _rot()._sources().get(key)
114
+ if spec and not spec.get("missing"):
115
+ label = str((definition or {}).get("label") or key)
116
+ return (f"{label} is still loading into this server after a restart, "
117
+ "so ask again in an hour.")
118
+ except Exception: # noqa: BLE001
119
+ pass
120
+ return NO_BINDING
121
+
122
+
123
  def binding(cur, table_key):
124
  """The resolved read-through spec for one connected grid, or None when it has no binding.
125
 
 
367
  # ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short.
368
  first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
369
  where=where, params=tuple(params), order_by=spec["id"],
370
+ limit=datastore.WINDOW_MAX, cur=cur, page_first=True, **frm)
371
  total = int(first["total"])
372
  if total > ASSISTANT_READ_MAX:
373
  raise MirrorRefusal(
 
385
  more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?"
386
  nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
387
  where=more_where, params=tuple(params) + (last,),
388
+ order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur,
389
+ page_first=True, **frm)
390
  page = list(nxt["rows"])
391
  if not page:
392
  break
 
509
  frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
510
  win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where,
511
  params=tuple(params), order_by=spec["id"], offset=int(offset),
512
+ limit=int(limit), cur=cur, page_first=True, **frm)
513
  stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
514
  defn=defn if isinstance(defn, dict) else None)
515
  rows = rot._pool_from_window(spec, win, stamp=stamp)
api/deps.py CHANGED
@@ -492,8 +492,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
492
  spec = assistant_mirror.binding(cur, database)
493
  if spec is None:
494
  _assistant_refuse(409, "assistant_snapshot_unavailable",
495
- "this database is served live from its connector and this deployment "
496
- "has no read-through binding for it yet, so it holds no rows to read")
497
  fields = assistant_mirror.answerable_fields(database, spec, declared)
498
  if not fields:
499
  _assistant_refuse(409, "assistant_snapshot_unavailable",
@@ -1028,11 +1027,15 @@ def assistant_source_status(session: Session, databases=None):
1028
  import assistant_mirror
1029
  cur, refusal = assistant_mirror.open_cursor(session)
1030
  mirror_cur = None if refusal is not None else cur
 
 
1031
  except Exception: # noqa: BLE001
1032
  mirror_cur = None
1033
  return mirror_cur
1034
 
1035
  amazon_cur = {}
 
 
1036
 
1037
  def _amazon_mirror(key):
1038
  """ONE Amazon cursor for the batch, chosen by `_table_cursor` for an Amazon key."""
@@ -1041,6 +1044,8 @@ def assistant_source_status(session: Session, databases=None):
1041
  import assistant_mirror
1042
  cur, refusal = assistant_mirror.open_cursor(session, key)
1043
  amazon_cur["cur"] = None if refusal is not None else cur
 
 
1044
  except Exception: # noqa: BLE001
1045
  amazon_cur["cur"] = None
1046
  return amazon_cur["cur"]
@@ -1165,8 +1170,9 @@ def assistant_source_status(session: Session, databases=None):
1165
  # first-sync gate, so "not open right now" is never Odoo's backfill answering for Amazon.
1166
  cur = _amazon_mirror(key) if key.startswith("ut_amzn_") else _mirror()
1167
  if cur is None:
1168
- row["reason"] = ("this database is read live from its connector mirror, and that "
1169
- "mirror is not open on this deployment right now; try again shortly")
 
1170
  continue
1171
  # ⛔ `is_bound`, NOT `binding` - THE AVAILABILITY QUESTION, NOT THE READ ONE, and the
1172
  # difference is 300 ms PER KEY here. Resolving a full spec runs `excluded_ids`, a scan of
@@ -1180,9 +1186,8 @@ def assistant_source_status(session: Session, databases=None):
1180
  if bound:
1181
  row["answerable"] = True
1182
  else:
1183
- row["reason"] = ("this database is read live from its connector mirror, and this "
1184
- "deployment has no read-through binding for it yet, so it holds no "
1185
- "rows the assistant can read")
1186
  return out
1187
 
1188
 
 
492
  spec = assistant_mirror.binding(cur, database)
493
  if spec is None:
494
  _assistant_refuse(409, "assistant_snapshot_unavailable",
495
+ assistant_mirror.unavailable_reason(database, definition))
 
496
  fields = assistant_mirror.answerable_fields(database, spec, declared)
497
  if not fields:
498
  _assistant_refuse(409, "assistant_snapshot_unavailable",
 
1027
  import assistant_mirror
1028
  cur, refusal = assistant_mirror.open_cursor(session)
1029
  mirror_cur = None if refusal is not None else cur
1030
+ if refusal is not None:
1031
+ closed_reason["odoo"] = refusal.message
1032
  except Exception: # noqa: BLE001
1033
  mirror_cur = None
1034
  return mirror_cur
1035
 
1036
  amazon_cur = {}
1037
+ #: The cursor refusal's own message, so this list and `query_database` say one sentence.
1038
+ closed_reason = {}
1039
 
1040
  def _amazon_mirror(key):
1041
  """ONE Amazon cursor for the batch, chosen by `_table_cursor` for an Amazon key."""
 
1044
  import assistant_mirror
1045
  cur, refusal = assistant_mirror.open_cursor(session, key)
1046
  amazon_cur["cur"] = None if refusal is not None else cur
1047
+ if refusal is not None:
1048
+ closed_reason["amazon"] = refusal.message
1049
  except Exception: # noqa: BLE001
1050
  amazon_cur["cur"] = None
1051
  return amazon_cur["cur"]
 
1170
  # first-sync gate, so "not open right now" is never Odoo's backfill answering for Amazon.
1171
  cur = _amazon_mirror(key) if key.startswith("ut_amzn_") else _mirror()
1172
  if cur is None:
1173
+ row["reason"] = (closed_reason.get("amazon" if key.startswith("ut_amzn_") else "odoo")
1174
+ or "this database is read live from its connector mirror, and that "
1175
+ "mirror is not open on this deployment right now; try again shortly")
1176
  continue
1177
  # ⛔ `is_bound`, NOT `binding` - THE AVAILABILITY QUESTION, NOT THE READ ONE, and the
1178
  # difference is 300 ms PER KEY here. Resolving a full spec runs `excluded_ids`, a scan of
 
1186
  if bound:
1187
  row["answerable"] = True
1188
  else:
1189
+ import assistant_mirror
1190
+ row["reason"] = assistant_mirror.unavailable_reason(key, definition)
 
1191
  return out
1192
 
1193
 
api/main.py CHANGED
@@ -898,6 +898,40 @@ def _pull_amazon(why, vendor=True):
898
  return False
899
 
900
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
901
  class _SyncNowRefused(Exception):
902
  """An application-authored, credential-free sentence for a Sync now receipt."""
903
 
@@ -1926,7 +1960,21 @@ def _resync_mirror_pass(manual, passes, entities=None):
1926
  # relational rebuild, which `manual` says nothing about.
1927
  if manual:
1928
  return None, None, None
 
 
 
 
 
 
 
 
 
 
 
1929
  sync_report = _ds.sync_all(entities=entities, log=lambda *a, **k: None)
 
 
 
1930
  # Wave 21: purge hard-deleted rows the cursor sync cannot see (the boot pass's comment has
1931
  # the measured case); without it deleted Odoo lines inflate every sum on the mirror FOREVER.
1932
  # ⚠ INSIDE the non-manual branch: a reconcile is a pass over the mirror this loop was
@@ -1941,6 +1989,19 @@ def _resync_mirror_pass(manual, passes, entities=None):
1941
  return None, None, _classify_resync_error(e)
1942
 
1943
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1944
  def _classify_resync_error(exc):
1945
  """D-314, the half that fits inside this loop: a failed pass leaves a MARK, not just a line.
1946
 
@@ -2183,7 +2244,15 @@ def _store_resync_loop():
2183
  # restores Amazon's saved mirror. The store-touching half (Meta and Amazon vendor pulls,
2184
  # the relational rebuild) waits for the pass that is actually due.
2185
  if not passes:
 
 
 
 
 
2186
  _await_boot_seed()
 
 
 
2187
  _boot_catch_up = ((not passes or catch_up_retries)
2188
  and _resync_boot_recent(last_good_pass_at, _t.time(), _every))
2189
  # ⛔⛔ HOTFIX 2026-09-24 — A DESK WAKE IS NOT A SCHEDULED PASS.
 
898
  return False
899
 
900
 
901
+ def _restore_amazon_mirror(why):
902
+ """Restore Amazon's saved mirror and republish its registry: store reads only, no vendor call."""
903
+ try:
904
+ return _restore_amazon_mirror_once(why)
905
+ finally:
906
+ try:
907
+ import amazon_relational as _ar
908
+ _ar.RESTORE_PENDING["since"] = None
909
+ except Exception: # noqa: BLE001
910
+ pass
911
+
912
+
913
+ def _restore_amazon_mirror_once(why):
914
+ try:
915
+ import time as _t
916
+ from harness import amazon_store as _amazon
917
+ from harness import runtime as _runtime
918
+ import routes_odoo_tables as _rot
919
+ tenant_key = "royal-imports"
920
+ rt = _runtime.get_runtime(tenant_key)
921
+ if not _amazon.credential(tenant_key, rt=rt):
922
+ return False
923
+ started = _t.time()
924
+ result = _amazon.restore_mirror(tenant_key, rt)
925
+ _rot.invalidate_sources()
926
+ _rot._sources()
927
+ print(f"[aios-api] amazon mirror {result.get('state')} ({why}) in "
928
+ f"{_t.time() - started:.0f}s, before the Odoo catch-up")
929
+ return True
930
+ except Exception as exc: # noqa: BLE001
931
+ print(f"[aios-api] amazon mirror restore FAILED ({why}): {type(exc).__name__}: {exc}")
932
+ return False
933
+
934
+
935
  class _SyncNowRefused(Exception):
936
  """An application-authored, credential-free sentence for a Sync now receipt."""
937
 
 
1960
  # relational rebuild, which `manual` says nothing about.
1961
  if manual:
1962
  return None, None, None
1963
+ absent = _absent_mirror_entities(_ds, entities)
1964
+ first = {}
1965
+ if absent:
1966
+ # Tables the boot seed lacks first: minutes, where the whole catch-up runs an hour.
1967
+ try:
1968
+ first = _ds.sync_all(entities=absent, log=lambda *a, **k: None)
1969
+ print(f"[aios-api] store resync: synced {', '.join(absent)} first, "
1970
+ f"absent from the boot seed")
1971
+ except Exception as exc: # noqa: BLE001
1972
+ print(f"[aios-api] store resync: the absent tables first failed "
1973
+ f"({type(exc).__name__}); the full pass below still syncs them")
1974
  sync_report = _ds.sync_all(entities=entities, log=lambda *a, **k: None)
1975
+ for key, row in first.items():
1976
+ if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict):
1977
+ sync_report[key]["pulled"] = sync_report[key].get("pulled", 0) + row.get("pulled", 0)
1978
  # Wave 21: purge hard-deleted rows the cursor sync cannot see (the boot pass's comment has
1979
  # the measured case); without it deleted Odoo lines inflate every sum on the mirror FOREVER.
1980
  # ⚠ INSIDE the non-manual branch: a reconcile is a pass over the mirror this loop was
 
1989
  return None, None, _classify_resync_error(e)
1990
 
1991
 
1992
+ def _absent_mirror_entities(ds, entities=None):
1993
+ """The mirror entities this pass syncs that have no table in the DuckDB file yet, in sync order."""
1994
+ try:
1995
+ cur = ds.connect().cursor()
1996
+ try:
1997
+ return [key for key in (list(ds.ENTITIES) if entities is None else list(entities))
1998
+ if key in ds.ENTITIES and not ds.columns_of(key, cur=cur)]
1999
+ finally:
2000
+ cur.close()
2001
+ except Exception: # noqa: BLE001
2002
+ return []
2003
+
2004
+
2005
  def _classify_resync_error(exc):
2006
  """D-314, the half that fits inside this loop: a failed pass leaves a MARK, not just a line.
2007
 
 
2244
  # restores Amazon's saved mirror. The store-touching half (Meta and Amazon vendor pulls,
2245
  # the relational rebuild) waits for the pass that is actually due.
2246
  if not passes:
2247
+ try:
2248
+ import amazon_relational as _ar
2249
+ _ar.RESTORE_PENDING["since"] = _t.time()
2250
+ except Exception: # noqa: BLE001
2251
+ pass
2252
  _await_boot_seed()
2253
+ # Amazon's saved mirror first: minutes of store reads, where the Odoo catch-up below
2254
+ # runs about an hour after a restart and Amazon answered nothing until it ended.
2255
+ _restore_amazon_mirror("boot")
2256
  _boot_catch_up = ((not passes or catch_up_retries)
2257
  and _resync_boot_recent(last_good_pass_at, _t.time(), _every))
2258
  # ⛔⛔ HOTFIX 2026-09-24 — A DESK WAKE IS NOT A SCHEDULED PASS.
api/routes_odoo_tables.py CHANGED
@@ -3657,7 +3657,7 @@ def odoo_table_rows(table_key: str,
3657
  select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
3658
  win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
3659
  where=where, params=tuple(params), order_by=order,
3660
- offset=offset, limit=(limit or None), cur=cur, **frm)
3661
  if win["clamped"]:
3662
  limits.append({
3663
  "subject": "limit", "effect": "window_clamped",
 
3657
  select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
3658
  win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
3659
  where=where, params=tuple(params), order_by=order,
3660
+ offset=offset, limit=(limit or None), cur=cur, page_first=True, **frm)
3661
  if win["clamped"]:
3662
  limits.append({
3663
  "subject": "limit", "effect": "window_clamped",
platform/harness/datastore.py CHANGED
@@ -601,8 +601,13 @@ WINDOW_DEFAULT = 200
601
  _IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
602
 
603
 
 
 
 
 
604
  def window(table=None, select=None, where=None, params=(), order_by=None,
605
- offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None, group_by=None):
 
606
  """One page of a mirror table PLUS the true count of everything the same predicate matches.
607
 
608
  Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int,
@@ -626,6 +631,9 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
626
  `count(DISTINCT ...)`, which is a projection, not a second meaning for this field.
627
  ⚠ It defaults to None and emits nothing then, so every existing caller's two statements are
628
  byte-identical to what they were before this argument existed.
 
 
 
629
  """
630
  if bool(table) == bool(from_sql):
631
  raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
@@ -661,10 +669,19 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
661
  total = int(cur.execute(f"SELECT count(*) FROM {name}{tail}", params).fetchone()[0] or 0)
662
  grp = str(group_by or "").strip()
663
  fold = f" GROUP BY {grp}" if grp else ""
664
- got = cur.execute(
665
- f"SELECT {proj} FROM {name}{tail}{fold} "
666
- f"ORDER BY {order} LIMIT {int(limit)} OFFSET {int(offset)}",
667
- params).fetchall()
 
 
 
 
 
 
 
 
 
668
  cols = [d[0] for d in (cur.description or [])]
669
  return {"rows": [tuple(r) for r in got], "columns": cols,
670
  "total": total, "offset": offset, "limit": limit, "clamped": clamped}
 
601
  _IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
602
 
603
 
604
+ #: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen.
605
+ _SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE)
606
+
607
+
608
  def window(table=None, select=None, where=None, params=(), order_by=None,
609
+ offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None, group_by=None,
610
+ page_first=False):
611
  """One page of a mirror table PLUS the true count of everything the same predicate matches.
612
 
613
  Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int,
 
631
  `count(DISTINCT ...)`, which is a projection, not a second meaning for this field.
632
  ⚠ It defaults to None and emits nothing then, so every existing caller's two statements are
633
  byte-identical to what they were before this argument existed.
634
+
635
+ `page_first` (a row page whose `order_by` is an expression over the table's own columns) picks
636
+ the page's rows before the projection runs, when only the projection holds a subquery.
637
  """
638
  if bool(table) == bool(from_sql):
639
  raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
 
669
  total = int(cur.execute(f"SELECT count(*) FROM {name}{tail}", params).fetchone()[0] or 0)
670
  grp = str(group_by or "").strip()
671
  fold = f" GROUP BY {grp}" if grp else ""
672
+ if (page_first and table and not grp and _SUBQUERY.search(proj)
673
+ and not _SUBQUERY.search(f"{pred} {order}")):
674
+ # A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page
675
+ # of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71).
676
+ got = cur.execute(
677
+ f"SELECT {proj} FROM (SELECT * FROM {name}{tail} ORDER BY {order} "
678
+ f"LIMIT {int(limit)} OFFSET {int(offset)}) AS {name} ORDER BY {order}",
679
+ params).fetchall()
680
+ else:
681
+ got = cur.execute(
682
+ f"SELECT {proj} FROM {name}{tail}{fold} "
683
+ f"ORDER BY {order} LIMIT {int(limit)} OFFSET {int(offset)}",
684
+ params).fetchall()
685
  cols = [d[0] for d in (cur.description or [])]
686
  return {"rows": [tuple(r) for r in got], "columns": cols,
687
  "total": total, "offset": offset, "limit": limit, "clamped": clamped}
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 3a063f9e9e49991823ca608e4f24205ada7577cd2883b7fe73fb9bd8f96738ca
 
1
+ 4073c7bb3f0821ae686c33783c33d634088e1ecb973bedefd18728b446704658