fsanyoto commited on
Commit
3ebcf0f
Β·
verified Β·
1 Parent(s): 8f5ad99

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "77c4d6035",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "7b8ed049e",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 77c4d6035
 
1
+ 7b8ed049e
api/main.py CHANGED
@@ -1862,7 +1862,7 @@ def _sweep_automation_schemas(why):
1862
 
1863
  def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
1864
  tenant=None, cadence_seconds=None, table_keys=None,
1865
- source_counts=None):
1866
  """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
1867
  'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
1868
  could have asked" are different diagnoses.
@@ -2151,7 +2151,9 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
2151
  # ⚠ AFTER Meta and inside the same tenant binding β€” see `_rebuild_amazon_relational`.
2152
  # It carries no fingerprint here on purpose: this branch has just done a full Odoo
2153
  # rebuild, so a skip would be optimising the rarest pass at the cost of a stale grid.
2154
- _rebuild_amazon_relational(why, _rt)
 
 
2155
  # The isolated cells/rollups/meta handlers above are deliberately allowed to report and
2156
  # continue. This return value answers the narrower resync-policy question: did the Odoo
2157
  # table refresh itself complete, so a confirmed zero-change pass may be skipped?
@@ -2789,6 +2791,17 @@ def _resync_slot_pass_write(at, field="slotPassAt"):
2789
  "a restart before the next slot may run this one again")
2790
 
2791
 
 
 
 
 
 
 
 
 
 
 
 
2792
  def _resync_confirm_fresh(tenant, cadence_seconds):
2793
  """A pass that proved the mirror equals the last apply is a good sync: stamp both freshness
2794
  readers (the connector's `lastGoodSyncAt` and the per-database health), in two writes."""
@@ -2951,6 +2964,7 @@ def _store_resync_loop():
2951
  # H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
2952
  slot_resume_checked = False
2953
  _slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
 
2954
  # H1: a held Odoo backfill that moved queues a bounded Odoo-only pass, at most 24 per slot day.
2955
  _backfill_read = globals().get("_odoo_backfill_checkpoints")
2956
  backfill_next_at = None
@@ -3062,18 +3076,27 @@ def _store_resync_loop():
3062
  connector_sync.sync_wake_event().clear()
3063
  _slot_now = _t.time()
3064
  _slot_due = _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
 
 
3065
  if not slot_resume_checked:
3066
  slot_resume_checked = True
3067
- # H2: a claim with no end is a pass a restart killed. Resume it once per slot.
3068
  _killed = _rel_slot.slot_pass_killed(_slot_now, _slot, _memory)
3069
  if _killed is not None and not _slot_due:
3070
  _slot_due = True
 
 
3071
  _slot_write(_killed, "slotResumedFor")
 
3072
  print(f"[aios-api] store resync: the {_slot['at']} {_slot['tz']} slot pass "
3073
  f"started at {_rel_slot.slot_iso(_memory['slotPassAt'])} never finished; "
3074
- f"running it once now")
3075
- # The scheduled pass after the one this cycle may run.
3076
- RESYNC_STATE["nextPassAt"] = _rel_slot.slot_iso(_rel_slot.slot_after(_slot_now, _slot))
 
 
 
 
3077
  else:
3078
  RESYNC_STATE.pop("nextPassAt", None)
3079
  # β›”β›” HOTFIX 2026-09-29 β€” A RESTART INSIDE THE DAILY WINDOW IS NOT A SYNC. The owner
@@ -3274,18 +3297,7 @@ def _store_resync_loop():
3274
  _pulls["meta"] = _t.time()
3275
  _resync_memory_write(last_good_pass_at, last_relational_success_at,
3276
  last_source_fingerprints, dict(_pulls))
3277
- if _resumed.get("amazon"):
3278
- _pull_amazon("boot", False)
3279
- else:
3280
- # The daily slot pass refreshes every Amazon table: a late extra pass yesterday must not defer it.
3281
- RESYNC_STATE["slotPass"] = _slot_claimed
3282
- try:
3283
- _pull_amazon("resync")
3284
- finally:
3285
- RESYNC_STATE["slotPass"] = False
3286
- _pulls["amazon"] = _t.time()
3287
- _resync_memory_write(last_good_pass_at, last_relational_success_at,
3288
- last_source_fingerprints, dict(_pulls))
3289
  _meta_now = _t.time()
3290
  _meta_reason = _meta_rebuild_force_reason(
3291
  _meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now)
@@ -3339,6 +3351,8 @@ def _store_resync_loop():
3339
  _rebuild_scope["table_keys"] = _table_keys
3340
  if _source_counts and (_rebuild_variadic or "source_counts" in _rebuild_params):
3341
  _rebuild_scope["source_counts"] = _source_counts
 
 
3342
  if _rebuild_odoo_relational(
3343
  "resync", _meta_previous, _meta_state, **_rebuild_scope):
3344
  last_source_fingerprints = source_fingerprints or None
@@ -3362,14 +3376,31 @@ def _store_resync_loop():
3362
  from harness import runtime as _runtime
3363
  _rt_resync = _runtime.get_runtime(_cad["tenant"])
3364
  _meta_state = _rebuild_meta_relational("resync", _rt_resync, _meta_previous)
3365
- # A zero Odoo report says nothing about Amazon either. Same reasoning, same pass,
3366
- # and it inherits the tenant binding the line above just established.
3367
- _amzn_state = _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous)
3368
- last_amzn_fingerprint = (_amzn_state["fingerprint"]
3369
- if _amzn_state.get("known") else None)
3370
  except Exception as e: # noqa: BLE001
3371
  print(f"[aios-api] meta relational refresh failed (resync): "
3372
  f"{type(e).__name__}: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3373
  if _meta_state.get("known"):
3374
  last_meta_fingerprint = _meta_state["fingerprint"]
3375
  if _meta_state.get("applied"):
 
1862
 
1863
  def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
1864
  tenant=None, cadence_seconds=None, table_keys=None,
1865
+ source_counts=None, amazon_relational=True):
1866
  """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
1867
  'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
1868
  could have asked" are different diagnoses.
 
2151
  # ⚠ AFTER Meta and inside the same tenant binding β€” see `_rebuild_amazon_relational`.
2152
  # It carries no fingerprint here on purpose: this branch has just done a full Odoo
2153
  # rebuild, so a skip would be optimising the rarest pass at the cost of a stale grid.
2154
+ # The resync loop passes `amazon_relational=False` and projects Amazon after its own pull.
2155
+ if amazon_relational:
2156
+ _rebuild_amazon_relational(why, _rt)
2157
  # The isolated cells/rollups/meta handlers above are deliberately allowed to report and
2158
  # continue. This return value answers the narrower resync-policy question: did the Odoo
2159
  # table refresh itself complete, so a confirmed zero-change pass may be skipped?
 
2791
  "a restart before the next slot may run this one again")
2792
 
2793
 
2794
+ def _resync_slot_exhausted_write(message):
2795
+ """Write why the slot pass stopped resuming on the Odoo card. Best effort, never raises."""
2796
+ try:
2797
+ from core import user_tables as _ut_slot
2798
+ from harness import runtime as _runtime_slot
2799
+ _ut_slot.record_odoo_sync_status(
2800
+ phase="error", errors=[message], st=_runtime_slot.get_runtime(_resync_tenant()[0]))
2801
+ except Exception as e: # noqa: BLE001
2802
+ print(f"[aios-api] store resync slot cause not saved ({type(e).__name__}): {e}")
2803
+
2804
+
2805
  def _resync_confirm_fresh(tenant, cadence_seconds):
2806
  """A pass that proved the mirror equals the last apply is a good sync: stamp both freshness
2807
  readers (the connector's `lastGoodSyncAt` and the per-database health), in two writes."""
 
2964
  # H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
2965
  slot_resume_checked = False
2966
  _slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
2967
+ _slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
2968
  # H1: a held Odoo backfill that moved queues a bounded Odoo-only pass, at most 24 per slot day.
2969
  _backfill_read = globals().get("_odoo_backfill_checkpoints")
2970
  backfill_next_at = None
 
3076
  connector_sync.sync_wake_event().clear()
3077
  _slot_now = _t.time()
3078
  _slot_due = _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
3079
+ # The scheduled pass after the one this cycle may run.
3080
+ RESYNC_STATE["nextPassAt"] = _rel_slot.slot_iso(_rel_slot.slot_after(_slot_now, _slot))
3081
  if not slot_resume_checked:
3082
  slot_resume_checked = True
3083
+ # H2: a claim with no end is a pass a restart killed. Resume it up to SLOT_RESUME_LIMIT times.
3084
  _killed = _rel_slot.slot_pass_killed(_slot_now, _slot, _memory)
3085
  if _killed is not None and not _slot_due:
3086
  _slot_due = True
3087
+ _n = ((_memory.get("slotResumes") or 0) + 1
3088
+ if (_memory.get("slotResumedFor") or 0) >= _killed else 1)
3089
  _slot_write(_killed, "slotResumedFor")
3090
+ _slot_write(_n, "slotResumes")
3091
  print(f"[aios-api] store resync: the {_slot['at']} {_slot['tz']} slot pass "
3092
  f"started at {_rel_slot.slot_iso(_memory['slotPassAt'])} never finished; "
3093
+ f"running it again now (resume {_n} of {_rel_slot.SLOT_RESUME_LIMIT})")
3094
+ elif _rel_slot.slot_pass_exhausted(_slot_now, _slot, _memory) is not None:
3095
+ _cause = (f"slot_pass_interrupted: the {_slot['at']} {_slot['tz']} slot pass "
3096
+ f"was interrupted by {_rel_slot.SLOT_RESUME_LIMIT + 1} restarts and "
3097
+ f"waits for the next slot at {RESYNC_STATE['nextPassAt']}")
3098
+ _slot_exhausted_write(_cause)
3099
+ print(f"[aios-api] store resync: {_cause}")
3100
  else:
3101
  RESYNC_STATE.pop("nextPassAt", None)
3102
  # β›”β›” HOTFIX 2026-09-29 β€” A RESTART INSIDE THE DAILY WINDOW IS NOT A SYNC. The owner
 
3297
  _pulls["meta"] = _t.time()
3298
  _resync_memory_write(last_good_pass_at, last_relational_success_at,
3299
  last_source_fingerprints, dict(_pulls))
3300
+ # The Odoo publication never waits on the Amazon pull or its callbacks (live 2026-10-31: 2 h 21 min).
 
 
 
 
 
 
 
 
 
 
 
3301
  _meta_now = _t.time()
3302
  _meta_reason = _meta_rebuild_force_reason(
3303
  _meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now)
 
3351
  _rebuild_scope["table_keys"] = _table_keys
3352
  if _source_counts and (_rebuild_variadic or "source_counts" in _rebuild_params):
3353
  _rebuild_scope["source_counts"] = _source_counts
3354
+ if _rebuild_variadic or "amazon_relational" in _rebuild_params:
3355
+ _rebuild_scope["amazon_relational"] = False
3356
  if _rebuild_odoo_relational(
3357
  "resync", _meta_previous, _meta_state, **_rebuild_scope):
3358
  last_source_fingerprints = source_fingerprints or None
 
3376
  from harness import runtime as _runtime
3377
  _rt_resync = _runtime.get_runtime(_cad["tenant"])
3378
  _meta_state = _rebuild_meta_relational("resync", _rt_resync, _meta_previous)
 
 
 
 
 
3379
  except Exception as e: # noqa: BLE001
3380
  print(f"[aios-api] meta relational refresh failed (resync): "
3381
  f"{type(e).__name__}: {e}")
3382
+ if _resumed.get("amazon"):
3383
+ _pull_amazon("boot", False)
3384
+ else:
3385
+ # The daily slot pass refreshes every Amazon table: a late extra pass yesterday must not defer it.
3386
+ RESYNC_STATE["slotPass"] = _slot_claimed
3387
+ try:
3388
+ _pull_amazon("resync")
3389
+ finally:
3390
+ RESYNC_STATE["slotPass"] = False
3391
+ _pulls["amazon"] = _t.time()
3392
+ _resync_memory_write(last_good_pass_at, last_relational_success_at,
3393
+ last_source_fingerprints, dict(_pulls))
3394
+ # A zero Odoo report says nothing about Amazon either, so its projection runs after its pull in both branches.
3395
+ try:
3396
+ from harness import runtime as _runtime
3397
+ _rt_resync = _runtime.get_runtime(_cad["tenant"])
3398
+ _amzn_state = _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous)
3399
+ last_amzn_fingerprint = (_amzn_state["fingerprint"]
3400
+ if _amzn_state.get("known") else None)
3401
+ except Exception as e: # noqa: BLE001
3402
+ print(f"[aios-api] amazon relational refresh failed (resync): "
3403
+ f"{type(e).__name__}: {e}")
3404
  if _meta_state.get("known"):
3405
  last_meta_fingerprint = _meta_state["fingerprint"]
3406
  if _meta_state.get("applied"):
api/odoo_relational.py CHANGED
@@ -5888,7 +5888,7 @@ def slot_due(now, slot, last_pass_at):
5888
 
5889
 
5890
  def slot_pass_killed(now, slot, memory):
5891
- """The latest slot's epoch when its claimed pass never ended and was not resumed, else None.
5892
 
5893
  A `passAt` at or after the claim also ends it, which covers memory from older code.
5894
  """
@@ -5901,8 +5901,32 @@ def slot_pass_killed(now, slot, memory):
5901
  if ends and max(ends) >= float(claim):
5902
  return None
5903
  resumed = memory.get("slotResumedFor")
5904
- # Storm guard: a slot pass killed again after its one resume waits for the next slot.
5905
- if resumed is not None and float(resumed) >= newest:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5906
  return None
5907
  return newest
5908
 
@@ -6068,7 +6092,8 @@ def decode_fingerprints(stored):
6068
  def read_resync_memory(rt):
6069
  """What the last container's resync loop knew, or empty values. Never raises."""
6070
  empty = {"passAt": None, "fullRebuildAt": None, "fingerprints": None, "lastGoodSyncAt": None,
6071
- "pulls": None, "slotPassAt": None, "slotDoneAt": None, "slotResumedFor": None}
 
6072
  try:
6073
  cur = rt.get(CONFIG_KEY) if rt is not None else None
6074
  except Exception: # noqa: BLE001
@@ -6085,7 +6110,8 @@ def read_resync_memory(rt):
6085
  if isinstance(pulls, dict) and pulls else None),
6086
  "slotPassAt": _memory_epoch(memory.get("slotPassAt")),
6087
  "slotDoneAt": _memory_epoch(memory.get("slotDoneAt")),
6088
- "slotResumedFor": _memory_epoch(memory.get("slotResumedFor"))}
 
6089
 
6090
 
6091
  def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints, pulls=None):
@@ -6114,16 +6140,22 @@ def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints, pulls=No
6114
  return memory
6115
 
6116
 
6117
- #: The slot stamps: a pass's start, its end (any outcome), and the slot whose killed pass was resumed.
6118
- SLOT_MEMORY_FIELDS = ("slotPassAt", "slotDoneAt", "slotResumedFor")
 
 
 
6119
 
6120
 
6121
  def record_slot_pass(rt, at, field="slotPassAt"):
6122
  """Merge one slot stamp (ISO) into the resync memory, by default the start of the last slot pass."""
6123
  if field not in SLOT_MEMORY_FIELDS:
6124
  raise ValueError(f"not a slot stamp: {field}")
6125
- stamp = (_dt.datetime.fromtimestamp(float(at), _dt.timezone.utc).isoformat()
6126
- if isinstance(at, (int, float)) else str(at))
 
 
 
6127
 
6128
  def _up(cur):
6129
  cur = dict(cur) if isinstance(cur, dict) else {}
 
5888
 
5889
 
5890
  def slot_pass_killed(now, slot, memory):
5891
+ """The latest slot's epoch when its claimed pass never ended and has a resume left, else None.
5892
 
5893
  A `passAt` at or after the claim also ends it, which covers memory from older code.
5894
  """
 
5901
  if ends and max(ends) >= float(claim):
5902
  return None
5903
  resumed = memory.get("slotResumedFor")
5904
+ # Storm guard: a slot pass killed after SLOT_RESUME_LIMIT resumes waits for the next slot.
5905
+ if resumed is not None and float(resumed) >= newest and _slot_resumes(memory) >= SLOT_RESUME_LIMIT:
5906
+ return None
5907
+ return newest
5908
+
5909
+
5910
+ def _slot_resumes(memory):
5911
+ """How many boots resumed the slot named by `slotResumedFor`, 0 when unreadable."""
5912
+ try:
5913
+ return int(memory.get("slotResumes") or 0)
5914
+ except (TypeError, ValueError):
5915
+ return 0
5916
+
5917
+
5918
+ def slot_pass_exhausted(now, slot, memory):
5919
+ """The newest slot's epoch when its killed pass used every resume, else None."""
5920
+ memory = memory if isinstance(memory, dict) else {}
5921
+ claim = memory.get("slotPassAt")
5922
+ newest = slot_before(now, slot)
5923
+ if claim is None or float(claim) < newest:
5924
+ return None
5925
+ ends = [float(at) for at in (memory.get("slotDoneAt"), memory.get("passAt")) if at is not None]
5926
+ if ends and max(ends) >= float(claim):
5927
+ return None
5928
+ resumed = memory.get("slotResumedFor")
5929
+ if resumed is None or float(resumed) < newest or _slot_resumes(memory) < SLOT_RESUME_LIMIT:
5930
  return None
5931
  return newest
5932
 
 
6092
  def read_resync_memory(rt):
6093
  """What the last container's resync loop knew, or empty values. Never raises."""
6094
  empty = {"passAt": None, "fullRebuildAt": None, "fingerprints": None, "lastGoodSyncAt": None,
6095
+ "pulls": None, "slotPassAt": None, "slotDoneAt": None, "slotResumedFor": None,
6096
+ "slotResumes": 0}
6097
  try:
6098
  cur = rt.get(CONFIG_KEY) if rt is not None else None
6099
  except Exception: # noqa: BLE001
 
6110
  if isinstance(pulls, dict) and pulls else None),
6111
  "slotPassAt": _memory_epoch(memory.get("slotPassAt")),
6112
  "slotDoneAt": _memory_epoch(memory.get("slotDoneAt")),
6113
+ "slotResumedFor": _memory_epoch(memory.get("slotResumedFor")),
6114
+ "slotResumes": _slot_resumes(memory)}
6115
 
6116
 
6117
  def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints, pulls=None):
 
6140
  return memory
6141
 
6142
 
6143
+ #: The slot stamps: a pass's start, its end (any outcome), the slot whose killed pass was resumed, and its resume count.
6144
+ SLOT_MEMORY_FIELDS = ("slotPassAt", "slotDoneAt", "slotResumedFor", "slotResumes")
6145
+
6146
+ #: A boot resumes a killed slot pass at most this many times, then the slot waits and the card names the cause.
6147
+ SLOT_RESUME_LIMIT = 3
6148
 
6149
 
6150
  def record_slot_pass(rt, at, field="slotPassAt"):
6151
  """Merge one slot stamp (ISO) into the resync memory, by default the start of the last slot pass."""
6152
  if field not in SLOT_MEMORY_FIELDS:
6153
  raise ValueError(f"not a slot stamp: {field}")
6154
+ if field == "slotResumes":
6155
+ stamp = int(at)
6156
+ else:
6157
+ stamp = (_dt.datetime.fromtimestamp(float(at), _dt.timezone.utc).isoformat()
6158
+ if isinstance(at, (int, float)) else str(at))
6159
 
6160
  def _up(cur):
6161
  cur = dict(cur) if isinstance(cur, dict) else {}
api/test_odoo_rank_refresh.py CHANGED
@@ -584,8 +584,8 @@ class RankRefreshTests(unittest.TestCase):
584
  {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000,
585
  'fingerprints': fps}, fps)
586
  self.assertEqual([e[0] for e in events],
587
- ['sync', 'meta', 'memory', 'amazon', 'memory', 'fresh', 'memory'])
588
- self.assertEqual(events[3], ('amazon', 'resync'))
589
  self.assertEqual(events[-1], ('memory', 10_000.0, 10_000.0 - 90_000, fps, None))
590
 
591
  # ── HOTFIX sdlc74 #3 (owner): "we retired rebooting sync at deployment" ─────────────────────
@@ -595,13 +595,13 @@ class RankRefreshTests(unittest.TestCase):
595
  {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old},
596
  new, rebuild_ok=False)
597
  kinds = [e[0] for e in events]
598
- self.assertEqual(kinds, ['sync', 'meta', 'memory', 'amazon', 'memory', 'rebuild', 'memory'])
599
- # The checkpoints change none of the three facts: a death in the rebuild leaves them as
600
- # they were, plus the pulls the pass already made.
601
  self.assertEqual(events[2], ('memory', 10_000.0 - 90_000, 10_000.0 - 90_000, old,
602
  {'startedAt': 10_000.0, 'meta': 10_000.0}))
603
  both = {'startedAt': 10_000.0, 'meta': 10_000.0, 'amazon': 10_000.0}
604
- self.assertEqual(events[4], ('memory', 10_000.0 - 90_000, 10_000.0 - 90_000, old, both))
605
  self.assertEqual(events[-1][-1], both)
606
 
607
  def test_restart_after_an_unfinished_pass_calls_no_vendor_and_rebuilds(self):
@@ -612,9 +612,9 @@ class RankRefreshTests(unittest.TestCase):
612
  'amazon': 10_000.0 - 600}}, new)
613
  # No Meta pull and no Amazon vendor pull: Amazon's saved mirror is restored, the mirror
614
  # catches up, and the rebuild that never reached the store runs.
615
- self.assertEqual([e[0] for e in events], ['sync', 'amazon', 'rebuild', 'memory'])
616
- self.assertEqual(events[1], ('amazon', 'boot', False))
617
- self.assertIn('ut_odoo_orders', events[2][1])
618
  # The pass ends good at the time its pulls ended, and its checkpoint is cleared.
619
  self.assertEqual(events[-1], ('memory', 10_000.0 - 600, 10_000.0 - 90_000, new, None))
620
  # The next scheduled pass is one cadence after the unfinished pass STARTED.
@@ -626,9 +626,9 @@ class RankRefreshTests(unittest.TestCase):
626
  events, _ = self.run_daily_loop(
627
  {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old,
628
  'pulls': {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400}}, new)
629
- self.assertEqual([e[0] for e in events], ['sync', 'amazon', 'memory', 'rebuild', 'memory'])
630
- self.assertEqual(events[1], ('amazon', 'resync'))
631
- self.assertEqual(events[2][-1], {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400,
632
  'amazon': 10_000.0})
633
 
634
  def test_an_unfinished_pass_older_than_the_cadence_is_not_resumed(self):
@@ -771,7 +771,8 @@ class RankRefreshTests(unittest.TestCase):
771
  source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
772
  # The loop reads both slot helpers by name, so they are extracted here explicitly.
773
  functions = {'_store_resync_loop', '_resync_rebuild_reason', '_meta_rebuild_force_reason',
774
- '_resync_boot_recent', '_resolve_resync_slot', '_resync_slot_pass_write'}
 
775
  nodes = [node for node in source.body
776
  if isinstance(node, ast.FunctionDef) and node.name in functions
777
  or isinstance(node, ast.Assign) and any(
@@ -902,17 +903,33 @@ class RankRefreshTests(unittest.TestCase):
902
  dict(base, slotDoneAt=yesterday), boot=boot, max_sleeps=1)
903
  self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
904
  memory = rel.read_resync_memory(self.slot_store)
905
- self.assertEqual((memory['slotPassAt'], memory['slotDoneAt'], memory['slotResumedFor']),
906
- (boot, boot, self.utc('2026-10-24T06:00:00Z')))
907
- self.assertIn('never finished; running it once now', self.loop_output)
908
- # Killed again after its one resume: no Odoo call until the next slot.
 
 
 
 
 
 
 
 
 
 
 
 
909
  events, sleeps, odoo = self.run_slot_loop(
910
- dict(base, slotPassAt=boot, slotDoneAt=yesterday,
911
- slotResumedFor=self.utc('2026-10-24T06:00:00Z')),
912
- boot=self.utc('2026-10-24T06:50:00Z'), max_sleeps=1)
913
- self.assertEqual((odoo, sleeps), ([], [self.utc('2026-10-25T06:00:01Z')
914
- - self.utc('2026-10-24T06:50:00Z')]))
915
  self.assertNotIn('never finished', self.loop_output)
 
 
 
 
 
 
916
 
917
  def test_h2_staging_memory_after_a_deploy_killed_the_slot_pass(self):
918
  """H2: claim 06:00:01Z, passAt the day before, boot 06:07Z: due once, then not due"""
@@ -923,8 +940,9 @@ class RankRefreshTests(unittest.TestCase):
923
  'slotDoneAt': None, 'slotResumedFor': None}
924
  self.assertFalse(rel.slot_due(boot, slot, claim))
925
  self.assertEqual(rel.slot_pass_killed(boot, slot, memory), self.utc('2026-10-27T06:00:00Z'))
926
- resumed = dict(memory, slotResumedFor=self.utc('2026-10-27T06:00:00Z'))
927
- self.assertIsNone(rel.slot_pass_killed(boot, slot, resumed))
 
928
  # The next day's slot is due by the ordinary rule, so the guard holds for one slot only.
929
  self.assertTrue(rel.slot_due(self.utc('2026-10-28T06:00:00Z'), slot, claim))
930
  # The loop runs the killed pass once at boot and logs the one resume line.
@@ -933,9 +951,51 @@ class RankRefreshTests(unittest.TestCase):
933
  boot=boot, max_sleeps=1)
934
  self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
935
  self.assertIn('the 13:00 Asia/Jakarta slot pass started at 2026-10-27T06:00:01Z never '
936
- 'finished; running it once now', self.loop_output)
937
  self.assertEqual(sleeps, [self.utc('2026-10-28T06:00:01Z') - boot])
938
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
939
  def test_h73_a_tenant_without_a_slot_keeps_the_cadence_loop(self):
940
  """H73 d: not tenant #0 and no stored time: the first pass runs at boot, then a cadence"""
941
  import odoo_relational as rel
 
584
  {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000,
585
  'fingerprints': fps}, fps)
586
  self.assertEqual([e[0] for e in events],
587
+ ['sync', 'meta', 'memory', 'fresh', 'amazon', 'memory', 'memory'])
588
+ self.assertEqual(events[4], ('amazon', 'resync'))
589
  self.assertEqual(events[-1], ('memory', 10_000.0, 10_000.0 - 90_000, fps, None))
590
 
591
  # ── HOTFIX sdlc74 #3 (owner): "we retired rebooting sync at deployment" ─────────────────────
 
595
  {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old},
596
  new, rebuild_ok=False)
597
  kinds = [e[0] for e in events]
598
+ self.assertEqual(kinds, ['sync', 'meta', 'memory', 'rebuild', 'amazon', 'memory', 'memory'])
599
+ # The Meta checkpoint changes none of the three facts. The Amazon checkpoint after a bad
600
+ # rebuild keeps recovery due, plus the pulls the pass already made.
601
  self.assertEqual(events[2], ('memory', 10_000.0 - 90_000, 10_000.0 - 90_000, old,
602
  {'startedAt': 10_000.0, 'meta': 10_000.0}))
603
  both = {'startedAt': 10_000.0, 'meta': 10_000.0, 'amazon': 10_000.0}
604
+ self.assertEqual(events[5], ('memory', 10_000.0 - 90_000, None, old, both))
605
  self.assertEqual(events[-1][-1], both)
606
 
607
  def test_restart_after_an_unfinished_pass_calls_no_vendor_and_rebuilds(self):
 
612
  'amazon': 10_000.0 - 600}}, new)
613
  # No Meta pull and no Amazon vendor pull: Amazon's saved mirror is restored, the mirror
614
  # catches up, and the rebuild that never reached the store runs.
615
+ self.assertEqual([e[0] for e in events], ['sync', 'rebuild', 'amazon', 'memory'])
616
+ self.assertEqual(events[2], ('amazon', 'boot', False))
617
+ self.assertIn('ut_odoo_orders', events[1][1])
618
  # The pass ends good at the time its pulls ended, and its checkpoint is cleared.
619
  self.assertEqual(events[-1], ('memory', 10_000.0 - 600, 10_000.0 - 90_000, new, None))
620
  # The next scheduled pass is one cadence after the unfinished pass STARTED.
 
626
  events, _ = self.run_daily_loop(
627
  {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old,
628
  'pulls': {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400}}, new)
629
+ self.assertEqual([e[0] for e in events], ['sync', 'rebuild', 'amazon', 'memory', 'memory'])
630
+ self.assertEqual(events[2], ('amazon', 'resync'))
631
+ self.assertEqual(events[3][-1], {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400,
632
  'amazon': 10_000.0})
633
 
634
  def test_an_unfinished_pass_older_than_the_cadence_is_not_resumed(self):
 
771
  source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
772
  # The loop reads both slot helpers by name, so they are extracted here explicitly.
773
  functions = {'_store_resync_loop', '_resync_rebuild_reason', '_meta_rebuild_force_reason',
774
+ '_resync_boot_recent', '_resolve_resync_slot', '_resync_slot_pass_write',
775
+ '_resync_slot_exhausted_write'}
776
  nodes = [node for node in source.body
777
  if isinstance(node, ast.FunctionDef) and node.name in functions
778
  or isinstance(node, ast.Assign) and any(
 
903
  dict(base, slotDoneAt=yesterday), boot=boot, max_sleeps=1)
904
  self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
905
  memory = rel.read_resync_memory(self.slot_store)
906
+ self.assertEqual((memory['slotPassAt'], memory['slotDoneAt'], memory['slotResumedFor'],
907
+ memory['slotResumes']),
908
+ (boot, boot, self.utc('2026-10-24T06:00:00Z'), 1))
909
+ self.assertIn('never finished; running it again now (resume 1 of 3)', self.loop_output)
910
+ # Killed again: the second and third kills resume too, and each boot counts its resume.
911
+ slot_at = self.utc('2026-10-24T06:00:00Z')
912
+ for count in (1, 2):
913
+ again = self.utc('2026-10-24T06:50:00Z') + 600 * count
914
+ events, sleeps, odoo = self.run_slot_loop(
915
+ dict(base, slotPassAt=boot, slotDoneAt=yesterday, slotResumedFor=slot_at,
916
+ slotResumes=count), boot=again, max_sleeps=1)
917
+ self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [again])
918
+ self.assertEqual(rel.read_resync_memory(self.slot_store)['slotResumes'], count + 1)
919
+ self.assertIn(f'running it again now (resume {count + 1} of 3)', self.loop_output)
920
+ # The fourth boot runs no pass, and the card names the cause until the next slot.
921
+ fourth = self.utc('2026-10-24T07:30:00Z')
922
  events, sleeps, odoo = self.run_slot_loop(
923
+ dict(base, slotPassAt=boot, slotDoneAt=yesterday, slotResumedFor=slot_at,
924
+ slotResumes=3), boot=fourth, max_sleeps=1)
925
+ self.assertEqual((odoo, sleeps), ([], [self.utc('2026-10-25T06:00:01Z') - fourth]))
 
 
926
  self.assertNotIn('never finished', self.loop_output)
927
+ self.assertIn('slot_pass_interrupted: the 13:00 Asia/Jakarta slot pass was interrupted by '
928
+ '4 restarts and waits for the next slot at 2026-10-25T06:00:00Z',
929
+ self.loop_output)
930
+ failure = rel.sync_freshness(self.slot_store, now=fourth)['failure']
931
+ self.assertEqual(failure['code'], 'slot_pass_interrupted')
932
+ self.assertNotRegex(failure['message'], '[\u2013\u2014]')
933
 
934
  def test_h2_staging_memory_after_a_deploy_killed_the_slot_pass(self):
935
  """H2: claim 06:00:01Z, passAt the day before, boot 06:07Z: due once, then not due"""
 
940
  'slotDoneAt': None, 'slotResumedFor': None}
941
  self.assertFalse(rel.slot_due(boot, slot, claim))
942
  self.assertEqual(rel.slot_pass_killed(boot, slot, memory), self.utc('2026-10-27T06:00:00Z'))
943
+ resumed = dict(memory, slotResumedFor=self.utc('2026-10-27T06:00:00Z'), slotResumes=1)
944
+ self.assertEqual(rel.slot_pass_killed(boot, slot, resumed), self.utc('2026-10-27T06:00:00Z'))
945
+ self.assertIsNone(rel.slot_pass_killed(boot, slot, dict(resumed, slotResumes=3)))
946
  # The next day's slot is due by the ordinary rule, so the guard holds for one slot only.
947
  self.assertTrue(rel.slot_due(self.utc('2026-10-28T06:00:00Z'), slot, claim))
948
  # The loop runs the killed pass once at boot and logs the one resume line.
 
951
  boot=boot, max_sleeps=1)
952
  self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
953
  self.assertIn('the 13:00 Asia/Jakarta slot pass started at 2026-10-27T06:00:01Z never '
954
+ 'finished; running it again now (resume 1 of 3)', self.loop_output)
955
  self.assertEqual(sleeps, [self.utc('2026-10-28T06:00:01Z') - boot])
956
 
957
+ def test_h1_slot_resumes_are_bounded_and_reset_by_a_new_slot(self):
958
+ """H1: a killed slot pass resumes while its count is under 3, then waits; a new slot starts over"""
959
+ import odoo_relational as rel
960
+ slot = rel.sync_slot(rel.read_config(self.SlotStore('royal-imports')))
961
+ slot_at, boot = self.utc('2026-10-27T06:00:00Z'), self.utc('2026-10-27T06:52:00Z')
962
+ memory = {'slotPassAt': self.utc('2026-10-27T06:43:17Z'), 'slotDoneAt': None,
963
+ 'passAt': self.utc('2026-10-26T06:59:00Z'), 'slotResumedFor': slot_at}
964
+ self.assertEqual(rel.SLOT_RESUME_LIMIT, 3)
965
+ for count, killed in ((0, slot_at), (2, slot_at), (3, None)):
966
+ state = dict(memory, slotResumes=count)
967
+ self.assertEqual(rel.slot_pass_killed(boot, slot, state), killed)
968
+ self.assertEqual(rel.slot_pass_exhausted(boot, slot, state),
969
+ None if killed else slot_at)
970
+ # A claim for a new slot resumes again, whatever the old slot's count.
971
+ new_slot = self.utc('2026-10-28T06:00:00Z')
972
+ state = dict(memory, slotPassAt=new_slot + 1, slotResumes=3)
973
+ self.assertEqual(rel.slot_pass_killed(new_slot + 600, slot, state), new_slot)
974
+ self.assertIsNone(rel.slot_pass_exhausted(new_slot + 600, slot, state))
975
+ # An ended pass is neither resumed nor exhausted.
976
+ ended = dict(memory, slotResumes=3, slotDoneAt=boot - 60)
977
+ self.assertIsNone(rel.slot_pass_killed(boot, slot, ended))
978
+ self.assertIsNone(rel.slot_pass_exhausted(boot, slot, ended))
979
+ # The count round-trips through the store as an int.
980
+ rt = self.SlotStore('royal-imports')
981
+ self.assertEqual(rel.read_resync_memory(rt)['slotResumes'], 0)
982
+ rel.record_slot_pass(rt, 2, 'slotResumes')
983
+ self.assertEqual(rel.read_resync_memory(rt)['slotResumes'], 2)
984
+
985
+ def test_h2_the_slot_pass_publishes_odoo_before_the_amazon_pull(self):
986
+ """H2: live 10-31: the claimed slot pass rebuilds or confirms Odoo before it pulls Amazon"""
987
+ last = self.utc('2026-10-23T06:05:00Z')
988
+ events, _sleeps, _odoo = self.run_slot_loop(
989
+ {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}},
990
+ boot=self.utc('2026-10-24T02:30:00Z'), max_sleeps=2)
991
+ slot = self.utc('2026-10-24T06:00:00Z')
992
+ kinds = [e[0] for e in events if e[1] > slot]
993
+ amazon = [i for i, e in enumerate(e for e in events if e[1] > slot)
994
+ if e[0] == 'amazon' and e[2:3] == ('resync',)]
995
+ odoo = [i for i, kind in enumerate(kinds) if kind in ('rebuild', 'fresh')]
996
+ self.assertTrue(amazon and odoo, kinds)
997
+ self.assertLess(odoo[0], amazon[0], kinds)
998
+
999
  def test_h73_a_tenant_without_a_slot_keeps_the_cadence_loop(self):
1000
  """H73 d: not tenant #0 and no stored time: the first pass runs at boot, then a cadence"""
1001
  import odoo_relational as rel
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 81d94080ecd642aa38b6dc30b8034ab3f052e3de73a16c4a8b24f8b10cf55273
 
1
+ e392538c2037e3fe77958006bfe1446187d4280556b55a3afdeecde76bdd6973