Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +38 -4
- api/test_odoo_rank_refresh.py +23 -4
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "17b3233",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
17b3233
|
api/main.py
CHANGED
|
@@ -1942,6 +1942,25 @@ def _project_desk(outcome, tenant):
|
|
| 1942 |
print(f"[aios-api] sync now projection failed: {type(e).__name__}: {e}")
|
| 1943 |
|
| 1944 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1945 |
def _resync_boot_recent(last_pass_at, now, every):
|
| 1946 |
"""Did the last scheduled pass finish less than one cadence ago? Pure; None means no."""
|
| 1947 |
try:
|
|
@@ -2009,6 +2028,7 @@ def _store_resync_loop():
|
|
| 2009 |
last_relational_success_at = _memory.get("fullRebuildAt")
|
| 2010 |
last_source_fingerprints = _memory.get("fingerprints")
|
| 2011 |
last_good_pass_at = _memory.get("passAt") or _memory.get("lastGoodSyncAt")
|
|
|
|
| 2012 |
last_meta_fingerprint = None
|
| 2013 |
last_meta_applied_at = None
|
| 2014 |
# β AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
|
|
@@ -2103,7 +2123,10 @@ def _store_resync_loop():
|
|
| 2103 |
# the disposable DuckDB mirror up (Odoo reads, no store; the dashboards read it, W58) and
|
| 2104 |
# restores Amazon's saved mirror. The store-touching half (Meta and Amazon vendor pulls,
|
| 2105 |
# the relational rebuild) waits for the pass that is actually due.
|
| 2106 |
-
|
|
|
|
|
|
|
|
|
|
| 2107 |
# ββ HOTFIX 2026-09-24 β A DESK WAKE IS NOT A SCHEDULED PASS.
|
| 2108 |
# An Amazon first load requeues itself after every bounded chunk, and each requeue SETS the
|
| 2109 |
# wake event. This loop used to answer every wake with the whole scheduled pass: an Odoo
|
|
@@ -2147,8 +2170,18 @@ def _store_resync_loop():
|
|
| 2147 |
f"{_err['type']}: {_err['message']}")
|
| 2148 |
if _boot_catch_up and not _odoo_requested:
|
| 2149 |
# Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
|
| 2150 |
-
|
|
|
|
| 2151 |
_age = max(0, int(_t.time() - last_good_pass_at))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2152 |
print(f"[aios-api] store resync: restarted {_age}s after the last scheduled pass, so "
|
| 2153 |
f"only the local mirror was caught up and no store table was rewritten; the "
|
| 2154 |
f"next scheduled pass is due in {max(0, _every - _age)}s")
|
|
@@ -2406,8 +2439,9 @@ try:
|
|
| 2406 |
from harness import datastore as _ds_boot
|
| 2407 |
if not _ds_boot.DB_PATH.exists():
|
| 2408 |
import threading as _threading_seed
|
| 2409 |
-
_threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True,
|
| 2410 |
-
|
|
|
|
| 2411 |
except Exception as e: # noqa: BLE001
|
| 2412 |
print(f"[aios-api] mirror seed not scheduled: {e}")
|
| 2413 |
|
|
|
|
| 1942 |
print(f"[aios-api] sync now projection failed: {type(e).__name__}: {e}")
|
| 1943 |
|
| 1944 |
|
| 1945 |
+
#: HOTFIX 2026-09-29: how often, and how many times, a failed restart catch-up is retried. It
|
| 1946 |
+
#: touches no store (DuckDB and Odoo only), so retrying is free for Neon and DO; waiting for the
|
| 1947 |
+
#: next daily pass instead would leave the dashboards on the seed for up to a day.
|
| 1948 |
+
_CATCH_UP_RETRY_SECONDS = 300
|
| 1949 |
+
_CATCH_UP_RETRIES = 12
|
| 1950 |
+
|
| 1951 |
+
|
| 1952 |
+
def _await_boot_seed(timeout=900):
|
| 1953 |
+
"""The first resync pass must not race the boot seed's file install.
|
| 1954 |
+
|
| 1955 |
+
Staging 2026-09-29 14:07:44Z: the seed landed and the first pass died on DuckDB
|
| 1956 |
+
`ConnectionException` in the same second. The seed thread exists only on a fresh disk.
|
| 1957 |
+
"""
|
| 1958 |
+
thread = globals().get("_SEED_THREAD")
|
| 1959 |
+
if thread is not None and thread.is_alive():
|
| 1960 |
+
print("[aios-api] store resync waits for the boot seed to finish installing")
|
| 1961 |
+
thread.join(timeout)
|
| 1962 |
+
|
| 1963 |
+
|
| 1964 |
def _resync_boot_recent(last_pass_at, now, every):
|
| 1965 |
"""Did the last scheduled pass finish less than one cadence ago? Pure; None means no."""
|
| 1966 |
try:
|
|
|
|
| 2028 |
last_relational_success_at = _memory.get("fullRebuildAt")
|
| 2029 |
last_source_fingerprints = _memory.get("fingerprints")
|
| 2030 |
last_good_pass_at = _memory.get("passAt") or _memory.get("lastGoodSyncAt")
|
| 2031 |
+
catch_up_retries = 0
|
| 2032 |
last_meta_fingerprint = None
|
| 2033 |
last_meta_applied_at = None
|
| 2034 |
# β AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
|
|
|
|
| 2123 |
# the disposable DuckDB mirror up (Odoo reads, no store; the dashboards read it, W58) and
|
| 2124 |
# restores Amazon's saved mirror. The store-touching half (Meta and Amazon vendor pulls,
|
| 2125 |
# the relational rebuild) waits for the pass that is actually due.
|
| 2126 |
+
if not passes:
|
| 2127 |
+
_await_boot_seed()
|
| 2128 |
+
_boot_catch_up = ((not passes or catch_up_retries)
|
| 2129 |
+
and _resync_boot_recent(last_good_pass_at, _t.time(), _every))
|
| 2130 |
# ββ HOTFIX 2026-09-24 β A DESK WAKE IS NOT A SCHEDULED PASS.
|
| 2131 |
# An Amazon first load requeues itself after every bounded chunk, and each requeue SETS the
|
| 2132 |
# wake event. This loop used to answer every wake with the whole scheduled pass: an Odoo
|
|
|
|
| 2170 |
f"{_err['type']}: {_err['message']}")
|
| 2171 |
if _boot_catch_up and not _odoo_requested:
|
| 2172 |
# Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
|
| 2173 |
+
if not catch_up_retries:
|
| 2174 |
+
_pull_amazon("boot", False)
|
| 2175 |
_age = max(0, int(_t.time() - last_good_pass_at))
|
| 2176 |
+
if _err is not None and catch_up_retries < _CATCH_UP_RETRIES:
|
| 2177 |
+
# The catch-up failed: retry it soon (it is free for the store), never the store half.
|
| 2178 |
+
catch_up_retries += 1
|
| 2179 |
+
print(f"[aios-api] store resync: the restart catch-up failed ({_err['type']}); "
|
| 2180 |
+
f"retry {catch_up_retries} of {_CATCH_UP_RETRIES} in "
|
| 2181 |
+
f"{_CATCH_UP_RETRY_SECONDS}s, and no store table was rewritten")
|
| 2182 |
+
last_full_pass_at = _t.time() - _every + _CATCH_UP_RETRY_SECONDS
|
| 2183 |
+
continue
|
| 2184 |
+
catch_up_retries = 0
|
| 2185 |
print(f"[aios-api] store resync: restarted {_age}s after the last scheduled pass, so "
|
| 2186 |
f"only the local mirror was caught up and no store table was rewritten; the "
|
| 2187 |
f"next scheduled pass is due in {max(0, _every - _age)}s")
|
|
|
|
| 2439 |
from harness import datastore as _ds_boot
|
| 2440 |
if not _ds_boot.DB_PATH.exists():
|
| 2441 |
import threading as _threading_seed
|
| 2442 |
+
_SEED_THREAD = _threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True,
|
| 2443 |
+
name="mirror-seed")
|
| 2444 |
+
_SEED_THREAD.start()
|
| 2445 |
except Exception as e: # noqa: BLE001
|
| 2446 |
print(f"[aios-api] mirror seed not scheduled: {e}")
|
| 2447 |
|
api/test_odoo_rank_refresh.py
CHANGED
|
@@ -477,8 +477,9 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 477 |
self.test_rank_change_behind_cursor_reaches_customer_reader()
|
| 478 |
|
| 479 |
# ββ HOTFIX 2026-09-29 (owner): "only sync daily at the fastest", and only what changed ββββββ
|
| 480 |
-
def run_daily_loop(self, memory, current, *, now=10_000.0, cadence=86400, manual=False
|
| 481 |
-
|
|
|
|
| 482 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 483 |
functions = {'_store_resync_loop', '_resync_rebuild_reason',
|
| 484 |
'_meta_rebuild_force_reason', '_resync_boot_recent'}
|
|
@@ -490,12 +491,19 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 490 |
for target in node.targets)]
|
| 491 |
events, sleeps = [], []
|
| 492 |
|
|
|
|
|
|
|
| 493 |
def sleep(seconds):
|
| 494 |
sleeps.append(seconds)
|
| 495 |
-
|
|
|
|
|
|
|
| 496 |
|
| 497 |
def sync(manual_flag, passes):
|
| 498 |
events.append(('sync', passes))
|
|
|
|
|
|
|
|
|
|
| 499 |
return (None if manual_flag else {'res_partner': {'phase': 'live', 'pulled': 0}},
|
| 500 |
None if manual_flag else {}, None)
|
| 501 |
|
|
@@ -505,7 +513,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 505 |
|
| 506 |
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
|
| 507 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 508 |
-
ns = dict(RESYNC_STATE={}, _RESYNC_NOTICE=[None],
|
| 509 |
_resolve_resync_cadence=lambda: dict(
|
| 510 |
resolvedSeconds=cadence, manual=manual, tenant='synthetic',
|
| 511 |
tenantSource='test', tenantSeconds=cadence, envSeconds=cadence,
|
|
@@ -545,6 +553,17 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 545 |
self.assertEqual(events, [('sync', 1), ('amazon', 'boot', False)])
|
| 546 |
self.assertEqual(sleeps, [86400 - 3600])
|
| 547 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 548 |
def test_restart_after_the_day_runs_the_scheduled_pass(self):
|
| 549 |
fps = {'res_partner': (3, 7)}
|
| 550 |
events, _ = self.run_daily_loop(
|
|
|
|
| 477 |
self.test_rank_change_behind_cursor_reaches_customer_reader()
|
| 478 |
|
| 479 |
# ββ HOTFIX 2026-09-29 (owner): "only sync daily at the fastest", and only what changed ββββββ
|
| 480 |
+
def run_daily_loop(self, memory, current, *, now=10_000.0, cadence=86400, manual=False,
|
| 481 |
+
errors=(), max_sleeps=1):
|
| 482 |
+
"""Drive the real loop with a persisted memory; stop at its `max_sleeps`-th wait."""
|
| 483 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 484 |
functions = {'_store_resync_loop', '_resync_rebuild_reason',
|
| 485 |
'_meta_rebuild_force_reason', '_resync_boot_recent'}
|
|
|
|
| 491 |
for target in node.targets)]
|
| 492 |
events, sleeps = [], []
|
| 493 |
|
| 494 |
+
failures = list(errors)
|
| 495 |
+
|
| 496 |
def sleep(seconds):
|
| 497 |
sleeps.append(seconds)
|
| 498 |
+
if len(sleeps) >= max_sleeps:
|
| 499 |
+
raise StopIteration('bounded synthetic resident loop')
|
| 500 |
+
return False
|
| 501 |
|
| 502 |
def sync(manual_flag, passes):
|
| 503 |
events.append(('sync', passes))
|
| 504 |
+
err = failures.pop(0) if failures else None
|
| 505 |
+
if err is not None:
|
| 506 |
+
return None, None, err
|
| 507 |
return (None if manual_flag else {'res_partner': {'phase': 'live', 'pulled': 0}},
|
| 508 |
None if manual_flag else {}, None)
|
| 509 |
|
|
|
|
| 513 |
|
| 514 |
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
|
| 515 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 516 |
+
ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None],
|
| 517 |
_resolve_resync_cadence=lambda: dict(
|
| 518 |
resolvedSeconds=cadence, manual=manual, tenant='synthetic',
|
| 519 |
tenantSource='test', tenantSeconds=cadence, envSeconds=cadence,
|
|
|
|
| 553 |
self.assertEqual(events, [('sync', 1), ('amazon', 'boot', False)])
|
| 554 |
self.assertEqual(sleeps, [86400 - 3600])
|
| 555 |
|
| 556 |
+
def test_failed_restart_catch_up_retries_soon_without_store_work(self):
|
| 557 |
+
fps = {'res_partner': (3, 7)}
|
| 558 |
+
err = {'kind': 'transient', 'type': 'ConnectionException', 'message': 'seed race',
|
| 559 |
+
'stage': 'mirrorPull', 'code': 'mirror_pull_failed', 'meaning': 'retry'}
|
| 560 |
+
events, sleeps = self.run_daily_loop(
|
| 561 |
+
{'passAt': 10_000.0 - 3600, 'fullRebuildAt': 10_000.0 - 3600, 'fingerprints': fps},
|
| 562 |
+
{'res_partner': (4, 9)}, errors=[err], max_sleeps=2)
|
| 563 |
+
# One restore, a 300 s retry of the catch-up alone, then the remainder of the day.
|
| 564 |
+
self.assertEqual(events, [('sync', 1), ('amazon', 'boot', False), ('sync', 2)])
|
| 565 |
+
self.assertEqual(sleeps, [300, 86400 - 3600])
|
| 566 |
+
|
| 567 |
def test_restart_after_the_day_runs_the_scheduled_pass(self):
|
| 568 |
fps = {'res_partner': (3, 7)}
|
| 569 |
events, _ = self.run_daily_loop(
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
5e56fa8158b38c68a1552f96737f541cf2fc237aa2b59fe332d17a28051dc21a
|