Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +54 -23
- api/odoo_relational.py +41 -9
- api/test_odoo_rank_refresh.py +85 -25
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "7b8ed049e",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
7b8ed049e
|
api/main.py
CHANGED
|
@@ -1862,7 +1862,7 @@ def _sweep_automation_schemas(why):
|
|
| 1862 |
|
| 1863 |
def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
|
| 1864 |
tenant=None, cadence_seconds=None, table_keys=None,
|
| 1865 |
-
source_counts=None):
|
| 1866 |
"""Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
|
| 1867 |
'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
|
| 1868 |
could have asked" are different diagnoses.
|
|
@@ -2151,7 +2151,9 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 2151 |
# β AFTER Meta and inside the same tenant binding β see `_rebuild_amazon_relational`.
|
| 2152 |
# It carries no fingerprint here on purpose: this branch has just done a full Odoo
|
| 2153 |
# rebuild, so a skip would be optimising the rarest pass at the cost of a stale grid.
|
| 2154 |
-
|
|
|
|
|
|
|
| 2155 |
# The isolated cells/rollups/meta handlers above are deliberately allowed to report and
|
| 2156 |
# continue. This return value answers the narrower resync-policy question: did the Odoo
|
| 2157 |
# table refresh itself complete, so a confirmed zero-change pass may be skipped?
|
|
@@ -2789,6 +2791,17 @@ def _resync_slot_pass_write(at, field="slotPassAt"):
|
|
| 2789 |
"a restart before the next slot may run this one again")
|
| 2790 |
|
| 2791 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2792 |
def _resync_confirm_fresh(tenant, cadence_seconds):
|
| 2793 |
"""A pass that proved the mirror equals the last apply is a good sync: stamp both freshness
|
| 2794 |
readers (the connector's `lastGoodSyncAt` and the per-database health), in two writes."""
|
|
@@ -2951,6 +2964,7 @@ def _store_resync_loop():
|
|
| 2951 |
# H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
|
| 2952 |
slot_resume_checked = False
|
| 2953 |
_slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
|
|
|
|
| 2954 |
# H1: a held Odoo backfill that moved queues a bounded Odoo-only pass, at most 24 per slot day.
|
| 2955 |
_backfill_read = globals().get("_odoo_backfill_checkpoints")
|
| 2956 |
backfill_next_at = None
|
|
@@ -3062,18 +3076,27 @@ def _store_resync_loop():
|
|
| 3062 |
connector_sync.sync_wake_event().clear()
|
| 3063 |
_slot_now = _t.time()
|
| 3064 |
_slot_due = _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
|
|
|
|
|
|
|
| 3065 |
if not slot_resume_checked:
|
| 3066 |
slot_resume_checked = True
|
| 3067 |
-
# H2: a claim with no end is a pass a restart killed. Resume it
|
| 3068 |
_killed = _rel_slot.slot_pass_killed(_slot_now, _slot, _memory)
|
| 3069 |
if _killed is not None and not _slot_due:
|
| 3070 |
_slot_due = True
|
|
|
|
|
|
|
| 3071 |
_slot_write(_killed, "slotResumedFor")
|
|
|
|
| 3072 |
print(f"[aios-api] store resync: the {_slot['at']} {_slot['tz']} slot pass "
|
| 3073 |
f"started at {_rel_slot.slot_iso(_memory['slotPassAt'])} never finished; "
|
| 3074 |
-
f"running it
|
| 3075 |
-
|
| 3076 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3077 |
else:
|
| 3078 |
RESYNC_STATE.pop("nextPassAt", None)
|
| 3079 |
# ββ HOTFIX 2026-09-29 β A RESTART INSIDE THE DAILY WINDOW IS NOT A SYNC. The owner
|
|
@@ -3274,18 +3297,7 @@ def _store_resync_loop():
|
|
| 3274 |
_pulls["meta"] = _t.time()
|
| 3275 |
_resync_memory_write(last_good_pass_at, last_relational_success_at,
|
| 3276 |
last_source_fingerprints, dict(_pulls))
|
| 3277 |
-
|
| 3278 |
-
_pull_amazon("boot", False)
|
| 3279 |
-
else:
|
| 3280 |
-
# The daily slot pass refreshes every Amazon table: a late extra pass yesterday must not defer it.
|
| 3281 |
-
RESYNC_STATE["slotPass"] = _slot_claimed
|
| 3282 |
-
try:
|
| 3283 |
-
_pull_amazon("resync")
|
| 3284 |
-
finally:
|
| 3285 |
-
RESYNC_STATE["slotPass"] = False
|
| 3286 |
-
_pulls["amazon"] = _t.time()
|
| 3287 |
-
_resync_memory_write(last_good_pass_at, last_relational_success_at,
|
| 3288 |
-
last_source_fingerprints, dict(_pulls))
|
| 3289 |
_meta_now = _t.time()
|
| 3290 |
_meta_reason = _meta_rebuild_force_reason(
|
| 3291 |
_meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now)
|
|
@@ -3339,6 +3351,8 @@ def _store_resync_loop():
|
|
| 3339 |
_rebuild_scope["table_keys"] = _table_keys
|
| 3340 |
if _source_counts and (_rebuild_variadic or "source_counts" in _rebuild_params):
|
| 3341 |
_rebuild_scope["source_counts"] = _source_counts
|
|
|
|
|
|
|
| 3342 |
if _rebuild_odoo_relational(
|
| 3343 |
"resync", _meta_previous, _meta_state, **_rebuild_scope):
|
| 3344 |
last_source_fingerprints = source_fingerprints or None
|
|
@@ -3362,14 +3376,31 @@ def _store_resync_loop():
|
|
| 3362 |
from harness import runtime as _runtime
|
| 3363 |
_rt_resync = _runtime.get_runtime(_cad["tenant"])
|
| 3364 |
_meta_state = _rebuild_meta_relational("resync", _rt_resync, _meta_previous)
|
| 3365 |
-
# A zero Odoo report says nothing about Amazon either. Same reasoning, same pass,
|
| 3366 |
-
# and it inherits the tenant binding the line above just established.
|
| 3367 |
-
_amzn_state = _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous)
|
| 3368 |
-
last_amzn_fingerprint = (_amzn_state["fingerprint"]
|
| 3369 |
-
if _amzn_state.get("known") else None)
|
| 3370 |
except Exception as e: # noqa: BLE001
|
| 3371 |
print(f"[aios-api] meta relational refresh failed (resync): "
|
| 3372 |
f"{type(e).__name__}: {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3373 |
if _meta_state.get("known"):
|
| 3374 |
last_meta_fingerprint = _meta_state["fingerprint"]
|
| 3375 |
if _meta_state.get("applied"):
|
|
|
|
| 1862 |
|
| 1863 |
def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
|
| 1864 |
tenant=None, cadence_seconds=None, table_keys=None,
|
| 1865 |
+
source_counts=None, amazon_relational=True):
|
| 1866 |
"""Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
|
| 1867 |
'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
|
| 1868 |
could have asked" are different diagnoses.
|
|
|
|
| 2151 |
# β AFTER Meta and inside the same tenant binding β see `_rebuild_amazon_relational`.
|
| 2152 |
# It carries no fingerprint here on purpose: this branch has just done a full Odoo
|
| 2153 |
# rebuild, so a skip would be optimising the rarest pass at the cost of a stale grid.
|
| 2154 |
+
# The resync loop passes `amazon_relational=False` and projects Amazon after its own pull.
|
| 2155 |
+
if amazon_relational:
|
| 2156 |
+
_rebuild_amazon_relational(why, _rt)
|
| 2157 |
# The isolated cells/rollups/meta handlers above are deliberately allowed to report and
|
| 2158 |
# continue. This return value answers the narrower resync-policy question: did the Odoo
|
| 2159 |
# table refresh itself complete, so a confirmed zero-change pass may be skipped?
|
|
|
|
| 2791 |
"a restart before the next slot may run this one again")
|
| 2792 |
|
| 2793 |
|
| 2794 |
+
def _resync_slot_exhausted_write(message):
|
| 2795 |
+
"""Write why the slot pass stopped resuming on the Odoo card. Best effort, never raises."""
|
| 2796 |
+
try:
|
| 2797 |
+
from core import user_tables as _ut_slot
|
| 2798 |
+
from harness import runtime as _runtime_slot
|
| 2799 |
+
_ut_slot.record_odoo_sync_status(
|
| 2800 |
+
phase="error", errors=[message], st=_runtime_slot.get_runtime(_resync_tenant()[0]))
|
| 2801 |
+
except Exception as e: # noqa: BLE001
|
| 2802 |
+
print(f"[aios-api] store resync slot cause not saved ({type(e).__name__}): {e}")
|
| 2803 |
+
|
| 2804 |
+
|
| 2805 |
def _resync_confirm_fresh(tenant, cadence_seconds):
|
| 2806 |
"""A pass that proved the mirror equals the last apply is a good sync: stamp both freshness
|
| 2807 |
readers (the connector's `lastGoodSyncAt` and the per-database health), in two writes."""
|
|
|
|
| 2964 |
# H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
|
| 2965 |
slot_resume_checked = False
|
| 2966 |
_slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
|
| 2967 |
+
_slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
|
| 2968 |
# H1: a held Odoo backfill that moved queues a bounded Odoo-only pass, at most 24 per slot day.
|
| 2969 |
_backfill_read = globals().get("_odoo_backfill_checkpoints")
|
| 2970 |
backfill_next_at = None
|
|
|
|
| 3076 |
connector_sync.sync_wake_event().clear()
|
| 3077 |
_slot_now = _t.time()
|
| 3078 |
_slot_due = _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
|
| 3079 |
+
# The scheduled pass after the one this cycle may run.
|
| 3080 |
+
RESYNC_STATE["nextPassAt"] = _rel_slot.slot_iso(_rel_slot.slot_after(_slot_now, _slot))
|
| 3081 |
if not slot_resume_checked:
|
| 3082 |
slot_resume_checked = True
|
| 3083 |
+
# H2: a claim with no end is a pass a restart killed. Resume it up to SLOT_RESUME_LIMIT times.
|
| 3084 |
_killed = _rel_slot.slot_pass_killed(_slot_now, _slot, _memory)
|
| 3085 |
if _killed is not None and not _slot_due:
|
| 3086 |
_slot_due = True
|
| 3087 |
+
_n = ((_memory.get("slotResumes") or 0) + 1
|
| 3088 |
+
if (_memory.get("slotResumedFor") or 0) >= _killed else 1)
|
| 3089 |
_slot_write(_killed, "slotResumedFor")
|
| 3090 |
+
_slot_write(_n, "slotResumes")
|
| 3091 |
print(f"[aios-api] store resync: the {_slot['at']} {_slot['tz']} slot pass "
|
| 3092 |
f"started at {_rel_slot.slot_iso(_memory['slotPassAt'])} never finished; "
|
| 3093 |
+
f"running it again now (resume {_n} of {_rel_slot.SLOT_RESUME_LIMIT})")
|
| 3094 |
+
elif _rel_slot.slot_pass_exhausted(_slot_now, _slot, _memory) is not None:
|
| 3095 |
+
_cause = (f"slot_pass_interrupted: the {_slot['at']} {_slot['tz']} slot pass "
|
| 3096 |
+
f"was interrupted by {_rel_slot.SLOT_RESUME_LIMIT + 1} restarts and "
|
| 3097 |
+
f"waits for the next slot at {RESYNC_STATE['nextPassAt']}")
|
| 3098 |
+
_slot_exhausted_write(_cause)
|
| 3099 |
+
print(f"[aios-api] store resync: {_cause}")
|
| 3100 |
else:
|
| 3101 |
RESYNC_STATE.pop("nextPassAt", None)
|
| 3102 |
# ββ HOTFIX 2026-09-29 β A RESTART INSIDE THE DAILY WINDOW IS NOT A SYNC. The owner
|
|
|
|
| 3297 |
_pulls["meta"] = _t.time()
|
| 3298 |
_resync_memory_write(last_good_pass_at, last_relational_success_at,
|
| 3299 |
last_source_fingerprints, dict(_pulls))
|
| 3300 |
+
# The Odoo publication never waits on the Amazon pull or its callbacks (live 2026-10-31: 2 h 21 min).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3301 |
_meta_now = _t.time()
|
| 3302 |
_meta_reason = _meta_rebuild_force_reason(
|
| 3303 |
_meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now)
|
|
|
|
| 3351 |
_rebuild_scope["table_keys"] = _table_keys
|
| 3352 |
if _source_counts and (_rebuild_variadic or "source_counts" in _rebuild_params):
|
| 3353 |
_rebuild_scope["source_counts"] = _source_counts
|
| 3354 |
+
if _rebuild_variadic or "amazon_relational" in _rebuild_params:
|
| 3355 |
+
_rebuild_scope["amazon_relational"] = False
|
| 3356 |
if _rebuild_odoo_relational(
|
| 3357 |
"resync", _meta_previous, _meta_state, **_rebuild_scope):
|
| 3358 |
last_source_fingerprints = source_fingerprints or None
|
|
|
|
| 3376 |
from harness import runtime as _runtime
|
| 3377 |
_rt_resync = _runtime.get_runtime(_cad["tenant"])
|
| 3378 |
_meta_state = _rebuild_meta_relational("resync", _rt_resync, _meta_previous)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3379 |
except Exception as e: # noqa: BLE001
|
| 3380 |
print(f"[aios-api] meta relational refresh failed (resync): "
|
| 3381 |
f"{type(e).__name__}: {e}")
|
| 3382 |
+
if _resumed.get("amazon"):
|
| 3383 |
+
_pull_amazon("boot", False)
|
| 3384 |
+
else:
|
| 3385 |
+
# The daily slot pass refreshes every Amazon table: a late extra pass yesterday must not defer it.
|
| 3386 |
+
RESYNC_STATE["slotPass"] = _slot_claimed
|
| 3387 |
+
try:
|
| 3388 |
+
_pull_amazon("resync")
|
| 3389 |
+
finally:
|
| 3390 |
+
RESYNC_STATE["slotPass"] = False
|
| 3391 |
+
_pulls["amazon"] = _t.time()
|
| 3392 |
+
_resync_memory_write(last_good_pass_at, last_relational_success_at,
|
| 3393 |
+
last_source_fingerprints, dict(_pulls))
|
| 3394 |
+
# A zero Odoo report says nothing about Amazon either, so its projection runs after its pull in both branches.
|
| 3395 |
+
try:
|
| 3396 |
+
from harness import runtime as _runtime
|
| 3397 |
+
_rt_resync = _runtime.get_runtime(_cad["tenant"])
|
| 3398 |
+
_amzn_state = _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous)
|
| 3399 |
+
last_amzn_fingerprint = (_amzn_state["fingerprint"]
|
| 3400 |
+
if _amzn_state.get("known") else None)
|
| 3401 |
+
except Exception as e: # noqa: BLE001
|
| 3402 |
+
print(f"[aios-api] amazon relational refresh failed (resync): "
|
| 3403 |
+
f"{type(e).__name__}: {e}")
|
| 3404 |
if _meta_state.get("known"):
|
| 3405 |
last_meta_fingerprint = _meta_state["fingerprint"]
|
| 3406 |
if _meta_state.get("applied"):
|
api/odoo_relational.py
CHANGED
|
@@ -5888,7 +5888,7 @@ def slot_due(now, slot, last_pass_at):
|
|
| 5888 |
|
| 5889 |
|
| 5890 |
def slot_pass_killed(now, slot, memory):
|
| 5891 |
-
"""The latest slot's epoch when its claimed pass never ended and
|
| 5892 |
|
| 5893 |
A `passAt` at or after the claim also ends it, which covers memory from older code.
|
| 5894 |
"""
|
|
@@ -5901,8 +5901,32 @@ def slot_pass_killed(now, slot, memory):
|
|
| 5901 |
if ends and max(ends) >= float(claim):
|
| 5902 |
return None
|
| 5903 |
resumed = memory.get("slotResumedFor")
|
| 5904 |
-
# Storm guard: a slot pass killed
|
| 5905 |
-
if resumed is not None and float(resumed) >= newest:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5906 |
return None
|
| 5907 |
return newest
|
| 5908 |
|
|
@@ -6068,7 +6092,8 @@ def decode_fingerprints(stored):
|
|
| 6068 |
def read_resync_memory(rt):
|
| 6069 |
"""What the last container's resync loop knew, or empty values. Never raises."""
|
| 6070 |
empty = {"passAt": None, "fullRebuildAt": None, "fingerprints": None, "lastGoodSyncAt": None,
|
| 6071 |
-
"pulls": None, "slotPassAt": None, "slotDoneAt": None, "slotResumedFor": None
|
|
|
|
| 6072 |
try:
|
| 6073 |
cur = rt.get(CONFIG_KEY) if rt is not None else None
|
| 6074 |
except Exception: # noqa: BLE001
|
|
@@ -6085,7 +6110,8 @@ def read_resync_memory(rt):
|
|
| 6085 |
if isinstance(pulls, dict) and pulls else None),
|
| 6086 |
"slotPassAt": _memory_epoch(memory.get("slotPassAt")),
|
| 6087 |
"slotDoneAt": _memory_epoch(memory.get("slotDoneAt")),
|
| 6088 |
-
"slotResumedFor": _memory_epoch(memory.get("slotResumedFor"))
|
|
|
|
| 6089 |
|
| 6090 |
|
| 6091 |
def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints, pulls=None):
|
|
@@ -6114,16 +6140,22 @@ def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints, pulls=No
|
|
| 6114 |
return memory
|
| 6115 |
|
| 6116 |
|
| 6117 |
-
#: The slot stamps: a pass's start, its end (any outcome),
|
| 6118 |
-
SLOT_MEMORY_FIELDS = ("slotPassAt", "slotDoneAt", "slotResumedFor")
|
|
|
|
|
|
|
|
|
|
| 6119 |
|
| 6120 |
|
| 6121 |
def record_slot_pass(rt, at, field="slotPassAt"):
|
| 6122 |
"""Merge one slot stamp (ISO) into the resync memory, by default the start of the last slot pass."""
|
| 6123 |
if field not in SLOT_MEMORY_FIELDS:
|
| 6124 |
raise ValueError(f"not a slot stamp: {field}")
|
| 6125 |
-
|
| 6126 |
-
|
|
|
|
|
|
|
|
|
|
| 6127 |
|
| 6128 |
def _up(cur):
|
| 6129 |
cur = dict(cur) if isinstance(cur, dict) else {}
|
|
|
|
| 5888 |
|
| 5889 |
|
| 5890 |
def slot_pass_killed(now, slot, memory):
|
| 5891 |
+
"""The latest slot's epoch when its claimed pass never ended and has a resume left, else None.
|
| 5892 |
|
| 5893 |
A `passAt` at or after the claim also ends it, which covers memory from older code.
|
| 5894 |
"""
|
|
|
|
| 5901 |
if ends and max(ends) >= float(claim):
|
| 5902 |
return None
|
| 5903 |
resumed = memory.get("slotResumedFor")
|
| 5904 |
+
# Storm guard: a slot pass killed after SLOT_RESUME_LIMIT resumes waits for the next slot.
|
| 5905 |
+
if resumed is not None and float(resumed) >= newest and _slot_resumes(memory) >= SLOT_RESUME_LIMIT:
|
| 5906 |
+
return None
|
| 5907 |
+
return newest
|
| 5908 |
+
|
| 5909 |
+
|
| 5910 |
+
def _slot_resumes(memory):
|
| 5911 |
+
"""How many boots resumed the slot named by `slotResumedFor`, 0 when unreadable."""
|
| 5912 |
+
try:
|
| 5913 |
+
return int(memory.get("slotResumes") or 0)
|
| 5914 |
+
except (TypeError, ValueError):
|
| 5915 |
+
return 0
|
| 5916 |
+
|
| 5917 |
+
|
| 5918 |
+
def slot_pass_exhausted(now, slot, memory):
|
| 5919 |
+
"""The newest slot's epoch when its killed pass used every resume, else None."""
|
| 5920 |
+
memory = memory if isinstance(memory, dict) else {}
|
| 5921 |
+
claim = memory.get("slotPassAt")
|
| 5922 |
+
newest = slot_before(now, slot)
|
| 5923 |
+
if claim is None or float(claim) < newest:
|
| 5924 |
+
return None
|
| 5925 |
+
ends = [float(at) for at in (memory.get("slotDoneAt"), memory.get("passAt")) if at is not None]
|
| 5926 |
+
if ends and max(ends) >= float(claim):
|
| 5927 |
+
return None
|
| 5928 |
+
resumed = memory.get("slotResumedFor")
|
| 5929 |
+
if resumed is None or float(resumed) < newest or _slot_resumes(memory) < SLOT_RESUME_LIMIT:
|
| 5930 |
return None
|
| 5931 |
return newest
|
| 5932 |
|
|
|
|
| 6092 |
def read_resync_memory(rt):
|
| 6093 |
"""What the last container's resync loop knew, or empty values. Never raises."""
|
| 6094 |
empty = {"passAt": None, "fullRebuildAt": None, "fingerprints": None, "lastGoodSyncAt": None,
|
| 6095 |
+
"pulls": None, "slotPassAt": None, "slotDoneAt": None, "slotResumedFor": None,
|
| 6096 |
+
"slotResumes": 0}
|
| 6097 |
try:
|
| 6098 |
cur = rt.get(CONFIG_KEY) if rt is not None else None
|
| 6099 |
except Exception: # noqa: BLE001
|
|
|
|
| 6110 |
if isinstance(pulls, dict) and pulls else None),
|
| 6111 |
"slotPassAt": _memory_epoch(memory.get("slotPassAt")),
|
| 6112 |
"slotDoneAt": _memory_epoch(memory.get("slotDoneAt")),
|
| 6113 |
+
"slotResumedFor": _memory_epoch(memory.get("slotResumedFor")),
|
| 6114 |
+
"slotResumes": _slot_resumes(memory)}
|
| 6115 |
|
| 6116 |
|
| 6117 |
def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints, pulls=None):
|
|
|
|
| 6140 |
return memory
|
| 6141 |
|
| 6142 |
|
| 6143 |
+
#: The slot stamps: a pass's start, its end (any outcome), the slot whose killed pass was resumed, and its resume count.
|
| 6144 |
+
SLOT_MEMORY_FIELDS = ("slotPassAt", "slotDoneAt", "slotResumedFor", "slotResumes")
|
| 6145 |
+
|
| 6146 |
+
#: A boot resumes a killed slot pass at most this many times, then the slot waits and the card names the cause.
|
| 6147 |
+
SLOT_RESUME_LIMIT = 3
|
| 6148 |
|
| 6149 |
|
| 6150 |
def record_slot_pass(rt, at, field="slotPassAt"):
|
| 6151 |
"""Merge one slot stamp (ISO) into the resync memory, by default the start of the last slot pass."""
|
| 6152 |
if field not in SLOT_MEMORY_FIELDS:
|
| 6153 |
raise ValueError(f"not a slot stamp: {field}")
|
| 6154 |
+
if field == "slotResumes":
|
| 6155 |
+
stamp = int(at)
|
| 6156 |
+
else:
|
| 6157 |
+
stamp = (_dt.datetime.fromtimestamp(float(at), _dt.timezone.utc).isoformat()
|
| 6158 |
+
if isinstance(at, (int, float)) else str(at))
|
| 6159 |
|
| 6160 |
def _up(cur):
|
| 6161 |
cur = dict(cur) if isinstance(cur, dict) else {}
|
api/test_odoo_rank_refresh.py
CHANGED
|
@@ -584,8 +584,8 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 584 |
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000,
|
| 585 |
'fingerprints': fps}, fps)
|
| 586 |
self.assertEqual([e[0] for e in events],
|
| 587 |
-
['sync', 'meta', 'memory', '
|
| 588 |
-
self.assertEqual(events[
|
| 589 |
self.assertEqual(events[-1], ('memory', 10_000.0, 10_000.0 - 90_000, fps, None))
|
| 590 |
|
| 591 |
# ββ HOTFIX sdlc74 #3 (owner): "we retired rebooting sync at deployment" βββββββββββββββββββββ
|
|
@@ -595,13 +595,13 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 595 |
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old},
|
| 596 |
new, rebuild_ok=False)
|
| 597 |
kinds = [e[0] for e in events]
|
| 598 |
-
self.assertEqual(kinds, ['sync', 'meta', 'memory', '
|
| 599 |
-
# The
|
| 600 |
-
#
|
| 601 |
self.assertEqual(events[2], ('memory', 10_000.0 - 90_000, 10_000.0 - 90_000, old,
|
| 602 |
{'startedAt': 10_000.0, 'meta': 10_000.0}))
|
| 603 |
both = {'startedAt': 10_000.0, 'meta': 10_000.0, 'amazon': 10_000.0}
|
| 604 |
-
self.assertEqual(events[
|
| 605 |
self.assertEqual(events[-1][-1], both)
|
| 606 |
|
| 607 |
def test_restart_after_an_unfinished_pass_calls_no_vendor_and_rebuilds(self):
|
|
@@ -612,9 +612,9 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 612 |
'amazon': 10_000.0 - 600}}, new)
|
| 613 |
# No Meta pull and no Amazon vendor pull: Amazon's saved mirror is restored, the mirror
|
| 614 |
# catches up, and the rebuild that never reached the store runs.
|
| 615 |
-
self.assertEqual([e[0] for e in events], ['sync', '
|
| 616 |
-
self.assertEqual(events[
|
| 617 |
-
self.assertIn('ut_odoo_orders', events[
|
| 618 |
# The pass ends good at the time its pulls ended, and its checkpoint is cleared.
|
| 619 |
self.assertEqual(events[-1], ('memory', 10_000.0 - 600, 10_000.0 - 90_000, new, None))
|
| 620 |
# The next scheduled pass is one cadence after the unfinished pass STARTED.
|
|
@@ -626,9 +626,9 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 626 |
events, _ = self.run_daily_loop(
|
| 627 |
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old,
|
| 628 |
'pulls': {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400}}, new)
|
| 629 |
-
self.assertEqual([e[0] for e in events], ['sync', '
|
| 630 |
-
self.assertEqual(events[
|
| 631 |
-
self.assertEqual(events[
|
| 632 |
'amazon': 10_000.0})
|
| 633 |
|
| 634 |
def test_an_unfinished_pass_older_than_the_cadence_is_not_resumed(self):
|
|
@@ -771,7 +771,8 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 771 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 772 |
# The loop reads both slot helpers by name, so they are extracted here explicitly.
|
| 773 |
functions = {'_store_resync_loop', '_resync_rebuild_reason', '_meta_rebuild_force_reason',
|
| 774 |
-
'_resync_boot_recent', '_resolve_resync_slot', '_resync_slot_pass_write'
|
|
|
|
| 775 |
nodes = [node for node in source.body
|
| 776 |
if isinstance(node, ast.FunctionDef) and node.name in functions
|
| 777 |
or isinstance(node, ast.Assign) and any(
|
|
@@ -902,17 +903,33 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 902 |
dict(base, slotDoneAt=yesterday), boot=boot, max_sleeps=1)
|
| 903 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 904 |
memory = rel.read_resync_memory(self.slot_store)
|
| 905 |
-
self.assertEqual((memory['slotPassAt'], memory['slotDoneAt'], memory['slotResumedFor']
|
| 906 |
-
|
| 907 |
-
|
| 908 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 909 |
events, sleeps, odoo = self.run_slot_loop(
|
| 910 |
-
dict(base, slotPassAt=boot, slotDoneAt=yesterday,
|
| 911 |
-
|
| 912 |
-
|
| 913 |
-
self.assertEqual((odoo, sleeps), ([], [self.utc('2026-10-25T06:00:01Z')
|
| 914 |
-
- self.utc('2026-10-24T06:50:00Z')]))
|
| 915 |
self.assertNotIn('never finished', self.loop_output)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 916 |
|
| 917 |
def test_h2_staging_memory_after_a_deploy_killed_the_slot_pass(self):
|
| 918 |
"""H2: claim 06:00:01Z, passAt the day before, boot 06:07Z: due once, then not due"""
|
|
@@ -923,8 +940,9 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 923 |
'slotDoneAt': None, 'slotResumedFor': None}
|
| 924 |
self.assertFalse(rel.slot_due(boot, slot, claim))
|
| 925 |
self.assertEqual(rel.slot_pass_killed(boot, slot, memory), self.utc('2026-10-27T06:00:00Z'))
|
| 926 |
-
resumed = dict(memory, slotResumedFor=self.utc('2026-10-27T06:00:00Z'))
|
| 927 |
-
self.
|
|
|
|
| 928 |
# The next day's slot is due by the ordinary rule, so the guard holds for one slot only.
|
| 929 |
self.assertTrue(rel.slot_due(self.utc('2026-10-28T06:00:00Z'), slot, claim))
|
| 930 |
# The loop runs the killed pass once at boot and logs the one resume line.
|
|
@@ -933,9 +951,51 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 933 |
boot=boot, max_sleeps=1)
|
| 934 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 935 |
self.assertIn('the 13:00 Asia/Jakarta slot pass started at 2026-10-27T06:00:01Z never '
|
| 936 |
-
'finished; running it
|
| 937 |
self.assertEqual(sleeps, [self.utc('2026-10-28T06:00:01Z') - boot])
|
| 938 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 939 |
def test_h73_a_tenant_without_a_slot_keeps_the_cadence_loop(self):
|
| 940 |
"""H73 d: not tenant #0 and no stored time: the first pass runs at boot, then a cadence"""
|
| 941 |
import odoo_relational as rel
|
|
|
|
| 584 |
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000,
|
| 585 |
'fingerprints': fps}, fps)
|
| 586 |
self.assertEqual([e[0] for e in events],
|
| 587 |
+
['sync', 'meta', 'memory', 'fresh', 'amazon', 'memory', 'memory'])
|
| 588 |
+
self.assertEqual(events[4], ('amazon', 'resync'))
|
| 589 |
self.assertEqual(events[-1], ('memory', 10_000.0, 10_000.0 - 90_000, fps, None))
|
| 590 |
|
| 591 |
# ββ HOTFIX sdlc74 #3 (owner): "we retired rebooting sync at deployment" βββββββββββββββββββββ
|
|
|
|
| 595 |
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old},
|
| 596 |
new, rebuild_ok=False)
|
| 597 |
kinds = [e[0] for e in events]
|
| 598 |
+
self.assertEqual(kinds, ['sync', 'meta', 'memory', 'rebuild', 'amazon', 'memory', 'memory'])
|
| 599 |
+
# The Meta checkpoint changes none of the three facts. The Amazon checkpoint after a bad
|
| 600 |
+
# rebuild keeps recovery due, plus the pulls the pass already made.
|
| 601 |
self.assertEqual(events[2], ('memory', 10_000.0 - 90_000, 10_000.0 - 90_000, old,
|
| 602 |
{'startedAt': 10_000.0, 'meta': 10_000.0}))
|
| 603 |
both = {'startedAt': 10_000.0, 'meta': 10_000.0, 'amazon': 10_000.0}
|
| 604 |
+
self.assertEqual(events[5], ('memory', 10_000.0 - 90_000, None, old, both))
|
| 605 |
self.assertEqual(events[-1][-1], both)
|
| 606 |
|
| 607 |
def test_restart_after_an_unfinished_pass_calls_no_vendor_and_rebuilds(self):
|
|
|
|
| 612 |
'amazon': 10_000.0 - 600}}, new)
|
| 613 |
# No Meta pull and no Amazon vendor pull: Amazon's saved mirror is restored, the mirror
|
| 614 |
# catches up, and the rebuild that never reached the store runs.
|
| 615 |
+
self.assertEqual([e[0] for e in events], ['sync', 'rebuild', 'amazon', 'memory'])
|
| 616 |
+
self.assertEqual(events[2], ('amazon', 'boot', False))
|
| 617 |
+
self.assertIn('ut_odoo_orders', events[1][1])
|
| 618 |
# The pass ends good at the time its pulls ended, and its checkpoint is cleared.
|
| 619 |
self.assertEqual(events[-1], ('memory', 10_000.0 - 600, 10_000.0 - 90_000, new, None))
|
| 620 |
# The next scheduled pass is one cadence after the unfinished pass STARTED.
|
|
|
|
| 626 |
events, _ = self.run_daily_loop(
|
| 627 |
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old,
|
| 628 |
'pulls': {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400}}, new)
|
| 629 |
+
self.assertEqual([e[0] for e in events], ['sync', 'rebuild', 'amazon', 'memory', 'memory'])
|
| 630 |
+
self.assertEqual(events[2], ('amazon', 'resync'))
|
| 631 |
+
self.assertEqual(events[3][-1], {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400,
|
| 632 |
'amazon': 10_000.0})
|
| 633 |
|
| 634 |
def test_an_unfinished_pass_older_than_the_cadence_is_not_resumed(self):
|
|
|
|
| 771 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 772 |
# The loop reads both slot helpers by name, so they are extracted here explicitly.
|
| 773 |
functions = {'_store_resync_loop', '_resync_rebuild_reason', '_meta_rebuild_force_reason',
|
| 774 |
+
'_resync_boot_recent', '_resolve_resync_slot', '_resync_slot_pass_write',
|
| 775 |
+
'_resync_slot_exhausted_write'}
|
| 776 |
nodes = [node for node in source.body
|
| 777 |
if isinstance(node, ast.FunctionDef) and node.name in functions
|
| 778 |
or isinstance(node, ast.Assign) and any(
|
|
|
|
| 903 |
dict(base, slotDoneAt=yesterday), boot=boot, max_sleeps=1)
|
| 904 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 905 |
memory = rel.read_resync_memory(self.slot_store)
|
| 906 |
+
self.assertEqual((memory['slotPassAt'], memory['slotDoneAt'], memory['slotResumedFor'],
|
| 907 |
+
memory['slotResumes']),
|
| 908 |
+
(boot, boot, self.utc('2026-10-24T06:00:00Z'), 1))
|
| 909 |
+
self.assertIn('never finished; running it again now (resume 1 of 3)', self.loop_output)
|
| 910 |
+
# Killed again: the second and third kills resume too, and each boot counts its resume.
|
| 911 |
+
slot_at = self.utc('2026-10-24T06:00:00Z')
|
| 912 |
+
for count in (1, 2):
|
| 913 |
+
again = self.utc('2026-10-24T06:50:00Z') + 600 * count
|
| 914 |
+
events, sleeps, odoo = self.run_slot_loop(
|
| 915 |
+
dict(base, slotPassAt=boot, slotDoneAt=yesterday, slotResumedFor=slot_at,
|
| 916 |
+
slotResumes=count), boot=again, max_sleeps=1)
|
| 917 |
+
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [again])
|
| 918 |
+
self.assertEqual(rel.read_resync_memory(self.slot_store)['slotResumes'], count + 1)
|
| 919 |
+
self.assertIn(f'running it again now (resume {count + 1} of 3)', self.loop_output)
|
| 920 |
+
# The fourth boot runs no pass, and the card names the cause until the next slot.
|
| 921 |
+
fourth = self.utc('2026-10-24T07:30:00Z')
|
| 922 |
events, sleeps, odoo = self.run_slot_loop(
|
| 923 |
+
dict(base, slotPassAt=boot, slotDoneAt=yesterday, slotResumedFor=slot_at,
|
| 924 |
+
slotResumes=3), boot=fourth, max_sleeps=1)
|
| 925 |
+
self.assertEqual((odoo, sleeps), ([], [self.utc('2026-10-25T06:00:01Z') - fourth]))
|
|
|
|
|
|
|
| 926 |
self.assertNotIn('never finished', self.loop_output)
|
| 927 |
+
self.assertIn('slot_pass_interrupted: the 13:00 Asia/Jakarta slot pass was interrupted by '
|
| 928 |
+
'4 restarts and waits for the next slot at 2026-10-25T06:00:00Z',
|
| 929 |
+
self.loop_output)
|
| 930 |
+
failure = rel.sync_freshness(self.slot_store, now=fourth)['failure']
|
| 931 |
+
self.assertEqual(failure['code'], 'slot_pass_interrupted')
|
| 932 |
+
self.assertNotRegex(failure['message'], '[\u2013\u2014]')
|
| 933 |
|
| 934 |
def test_h2_staging_memory_after_a_deploy_killed_the_slot_pass(self):
|
| 935 |
"""H2: claim 06:00:01Z, passAt the day before, boot 06:07Z: due once, then not due"""
|
|
|
|
| 940 |
'slotDoneAt': None, 'slotResumedFor': None}
|
| 941 |
self.assertFalse(rel.slot_due(boot, slot, claim))
|
| 942 |
self.assertEqual(rel.slot_pass_killed(boot, slot, memory), self.utc('2026-10-27T06:00:00Z'))
|
| 943 |
+
resumed = dict(memory, slotResumedFor=self.utc('2026-10-27T06:00:00Z'), slotResumes=1)
|
| 944 |
+
self.assertEqual(rel.slot_pass_killed(boot, slot, resumed), self.utc('2026-10-27T06:00:00Z'))
|
| 945 |
+
self.assertIsNone(rel.slot_pass_killed(boot, slot, dict(resumed, slotResumes=3)))
|
| 946 |
# The next day's slot is due by the ordinary rule, so the guard holds for one slot only.
|
| 947 |
self.assertTrue(rel.slot_due(self.utc('2026-10-28T06:00:00Z'), slot, claim))
|
| 948 |
# The loop runs the killed pass once at boot and logs the one resume line.
|
|
|
|
| 951 |
boot=boot, max_sleeps=1)
|
| 952 |
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 953 |
self.assertIn('the 13:00 Asia/Jakarta slot pass started at 2026-10-27T06:00:01Z never '
|
| 954 |
+
'finished; running it again now (resume 1 of 3)', self.loop_output)
|
| 955 |
self.assertEqual(sleeps, [self.utc('2026-10-28T06:00:01Z') - boot])
|
| 956 |
|
| 957 |
+
def test_h1_slot_resumes_are_bounded_and_reset_by_a_new_slot(self):
|
| 958 |
+
"""H1: a killed slot pass resumes while its count is under 3, then waits; a new slot starts over"""
|
| 959 |
+
import odoo_relational as rel
|
| 960 |
+
slot = rel.sync_slot(rel.read_config(self.SlotStore('royal-imports')))
|
| 961 |
+
slot_at, boot = self.utc('2026-10-27T06:00:00Z'), self.utc('2026-10-27T06:52:00Z')
|
| 962 |
+
memory = {'slotPassAt': self.utc('2026-10-27T06:43:17Z'), 'slotDoneAt': None,
|
| 963 |
+
'passAt': self.utc('2026-10-26T06:59:00Z'), 'slotResumedFor': slot_at}
|
| 964 |
+
self.assertEqual(rel.SLOT_RESUME_LIMIT, 3)
|
| 965 |
+
for count, killed in ((0, slot_at), (2, slot_at), (3, None)):
|
| 966 |
+
state = dict(memory, slotResumes=count)
|
| 967 |
+
self.assertEqual(rel.slot_pass_killed(boot, slot, state), killed)
|
| 968 |
+
self.assertEqual(rel.slot_pass_exhausted(boot, slot, state),
|
| 969 |
+
None if killed else slot_at)
|
| 970 |
+
# A claim for a new slot resumes again, whatever the old slot's count.
|
| 971 |
+
new_slot = self.utc('2026-10-28T06:00:00Z')
|
| 972 |
+
state = dict(memory, slotPassAt=new_slot + 1, slotResumes=3)
|
| 973 |
+
self.assertEqual(rel.slot_pass_killed(new_slot + 600, slot, state), new_slot)
|
| 974 |
+
self.assertIsNone(rel.slot_pass_exhausted(new_slot + 600, slot, state))
|
| 975 |
+
# An ended pass is neither resumed nor exhausted.
|
| 976 |
+
ended = dict(memory, slotResumes=3, slotDoneAt=boot - 60)
|
| 977 |
+
self.assertIsNone(rel.slot_pass_killed(boot, slot, ended))
|
| 978 |
+
self.assertIsNone(rel.slot_pass_exhausted(boot, slot, ended))
|
| 979 |
+
# The count round-trips through the store as an int.
|
| 980 |
+
rt = self.SlotStore('royal-imports')
|
| 981 |
+
self.assertEqual(rel.read_resync_memory(rt)['slotResumes'], 0)
|
| 982 |
+
rel.record_slot_pass(rt, 2, 'slotResumes')
|
| 983 |
+
self.assertEqual(rel.read_resync_memory(rt)['slotResumes'], 2)
|
| 984 |
+
|
| 985 |
+
def test_h2_the_slot_pass_publishes_odoo_before_the_amazon_pull(self):
|
| 986 |
+
"""H2: live 10-31: the claimed slot pass rebuilds or confirms Odoo before it pulls Amazon"""
|
| 987 |
+
last = self.utc('2026-10-23T06:05:00Z')
|
| 988 |
+
events, _sleeps, _odoo = self.run_slot_loop(
|
| 989 |
+
{'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
|
| 990 |
+
boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=2)
|
| 991 |
+
slot = self.utc('2026-10-24T06:00:00Z')
|
| 992 |
+
kinds = [e[0] for e in events if e[1] > slot]
|
| 993 |
+
amazon = [i for i, e in enumerate(e for e in events if e[1] > slot)
|
| 994 |
+
if e[0] == 'amazon' and e[2:3] == ('resync',)]
|
| 995 |
+
odoo = [i for i, kind in enumerate(kinds) if kind in ('rebuild', 'fresh')]
|
| 996 |
+
self.assertTrue(amazon and odoo, kinds)
|
| 997 |
+
self.assertLess(odoo[0], amazon[0], kinds)
|
| 998 |
+
|
| 999 |
def test_h73_a_tenant_without_a_slot_keeps_the_cadence_loop(self):
|
| 1000 |
"""H73 d: not tenant #0 and no stored time: the first pass runs at boot, then a cadence"""
|
| 1001 |
import odoo_relational as rel
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
e392538c2037e3fe77958006bfe1446187d4280556b55a3afdeecde76bdd6973
|