fsanyoto commited on
Commit
1bee71d
·
verified ·
1 Parent(s): 3cde246

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "b4cbcd3c8f9f",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "99129353b8d3",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- b4cbcd3c8f9f
 
1
+ 99129353b8d3
api/main.py CHANGED
@@ -1385,10 +1385,16 @@ def _pull_amazon(why, vendor=True):
1385
  _status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
1386
  raise
1387
  _amazon_status.record_pass_start(rt, "amazon_sp")
 
 
 
 
 
 
1388
  try:
1389
  rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
1390
  slot_pass=bool(RESYNC_STATE.get("slotPass")),
1391
- log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}")))
1392
  except Exception:
1393
  _status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
1394
  raise
@@ -3005,6 +3011,9 @@ def _resync_mirror_pass(manual, passes, entities=None):
3005
  def _resync_log(*parts, **_kw):
3006
  # One timestamped line per step. A logging error never changes the pass.
3007
  try:
 
 
 
3008
  stamp = _pass_dt.now(_pass_tz.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
3009
  print(f"[aios-api] {stamp} odoo pass: {' '.join(str(p) for p in parts)}", flush=True)
3010
  except Exception: # noqa: BLE001
@@ -3583,6 +3592,8 @@ def _store_resync_loop():
3583
  slot_restored = False
3584
  # H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
3585
  slot_resume_checked = False
 
 
3586
  _slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
3587
  _idle_wait = globals().get("_sync_idle_wait") or (lambda event, seconds: event.wait(seconds))
3588
  _slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
@@ -3594,6 +3605,8 @@ def _store_resync_loop():
3594
  backfill_spacing_seconds = 60.0
3595
  backfill_cap = None
3596
  while True:
 
 
3597
  if backfill_next_at is not None and backfill_cap is not None and _t.time() >= backfill_cap:
3598
  backfill_next_at = None
3599
  # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
@@ -3728,6 +3741,18 @@ def _store_resync_loop():
3728
  if _slot_interrupted_clear():
3729
  print(f"[aios-api] store resync: a good pass ended after the {_slot['at']} "
3730
  f"{_slot['tz']} slot; cleared the slot_pass_interrupted failure")
 
 
 
 
 
 
 
 
 
 
 
 
3731
  else:
3732
  RESYNC_STATE.pop("nextPassAt", None)
3733
  _cadence_due = not passes or _t.time() - last_full_pass_at >= _every
@@ -3858,6 +3883,7 @@ def _store_resync_loop():
3858
  "the next admitted pass resumes the unfinished Odoo cohort")
3859
  if _slot_claimed:
3860
  _slot_write(_t.time(), "slotDoneAt")
 
3861
  # H1: progress is a changed unfinished checkpoint in the candidate's `_sync_state`.
3862
  _backfill_after = _backfill_read(_cad["tenant"]) if _backfill_read else None
3863
  _moved = ([row for row in _backfill_after if row not in _backfill_before]
@@ -4083,6 +4109,7 @@ def _store_resync_loop():
4083
  _reason is not None or bool(_meta_state.get("applied")))
4084
  if _slot_claimed:
4085
  _slot_write(_t.time(), "slotDoneAt")
 
4086
 
4087
 
4088
  def _boot_app_migrations(why, tenant=None):
@@ -4306,10 +4333,17 @@ def _prewarm():
4306
  #: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a
4307
  #: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause.
4308
  MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""}
 
 
4309
 
4310
 
4311
  def _seed_mirror_if_absent():
4312
  """Restore tenant manifests before considering the resident tenant's legacy seed."""
 
 
 
 
 
4313
  from harness import datastore as _ds
4314
  from harness import runtime as _runtime
4315
  MIRROR_SEED["attempted"] = True
@@ -4325,14 +4359,18 @@ def _seed_mirror_if_absent():
4325
 
4326
  def restore(tenant_key, rt):
4327
  try:
4328
- if _ds.odoo_mirror_restored(tenant_key, rt):
4329
- return
4330
- result = _ds.restore_odoo_mirror(tenant_key, rt)
4331
- except Exception as exc:
4332
- result = refused(tenant_key, exc)
4333
- if result["state"] == "refused":
4334
- MIRROR_SEED["cause"] = "; ".join(result["problems"])
4335
- MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
 
 
 
 
4336
 
4337
  def runtime_failed(tenant_key, exc):
4338
  MIRROR_SEED["cause"] = "; ".join(refused(tenant_key, exc)["problems"])
@@ -4343,9 +4381,31 @@ def _seed_mirror_if_absent():
4343
  restore, name="odoo-mirror-restore", tenants=_runtime.known_tenants(),
4344
  get_runtime=tenant_scan.runtime_lookup(_runtime, "odoo-mirror-restore",
4345
  on_error=runtime_failed), kind="light")
4346
- for job in jobs.values():
4347
- job.join() # The legacy seed below decides on the restored manifests.
4348
- if _ds.ensure_seed():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4349
  MIRROR_SEED["seeded"] = True
4350
  print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
4351
  return True
 
1385
  _status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
1386
  raise
1387
  _amazon_status.record_pass_start(rt, "amazon_sp")
1388
+
1389
+ def _amazon_log(message):
1390
+ # Every Amazon pull step line is progress for the liveness check.
1391
+ getattr(_amazon_role, "note_progress", lambda _step: None)("amazon sync")
1392
+ print(f"[aios-api] amazon sync ({why}): {message}")
1393
+
1394
  try:
1395
  rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
1396
  slot_pass=bool(RESYNC_STATE.get("slotPass")),
1397
+ log=_amazon_log))
1398
  except Exception:
1399
  _status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
1400
  raise
 
3011
  def _resync_log(*parts, **_kw):
3012
  # One timestamped line per step. A logging error never changes the pass.
3013
  try:
3014
+ import process_role as _pass_role
3015
+ # Every Odoo pass step line is progress for the liveness check.
3016
+ getattr(_pass_role, "note_progress", lambda _step: None)("odoo pass")
3017
  stamp = _pass_dt.now(_pass_tz.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
3018
  print(f"[aios-api] {stamp} odoo pass: {' '.join(str(p) for p in parts)}", flush=True)
3019
  except Exception: # noqa: BLE001
 
3592
  slot_restored = False
3593
  # H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
3594
  slot_resume_checked = False
3595
+ # T2: an ended slot pass is checked once for a same-day rerun; the boot memory counts as one.
3596
+ slot_rerun_check = True
3597
  _slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
3598
  _idle_wait = globals().get("_sync_idle_wait") or (lambda event, seconds: event.wait(seconds))
3599
  _slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
 
3605
  backfill_spacing_seconds = 60.0
3606
  backfill_cap = None
3607
  while True:
3608
+ # Each cycle of a running loop is progress; the boot seed wait above is not.
3609
+ getattr(_loop_role, "note_progress", lambda _step: None)("resync idle tick")
3610
  if backfill_next_at is not None and backfill_cap is not None and _t.time() >= backfill_cap:
3611
  backfill_next_at = None
3612
  # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
 
3741
  if _slot_interrupted_clear():
3742
  print(f"[aios-api] store resync: a good pass ended after the {_slot['at']} "
3743
  f"{_slot['tz']} slot; cleared the slot_pass_interrupted failure")
3744
+ # T2: a slot pass that ended without a good sync reruns once the same day; a queued backfill goes first.
3745
+ if slot_rerun_check and not _slot_due and backfill_next_at is None:
3746
+ slot_rerun_check = False
3747
+ _rerun_memory = _memory if not passes else _resync_memory_read()
3748
+ _rerun = _rel_slot.slot_pass_rerun(_slot_now, _slot, _rerun_memory)
3749
+ if _rerun is not None:
3750
+ _slot_due = True
3751
+ # The resume pair marks the slot, so neither a rerun nor a resume repeats it.
3752
+ _slot_write(_rerun, "slotResumedFor")
3753
+ _slot_write(1, "slotResumes")
3754
+ print(f"[aios-api] store resync: the {_slot['at']} {_slot['tz']} slot pass ended "
3755
+ f"without a good sync; rerunning it once today")
3756
  else:
3757
  RESYNC_STATE.pop("nextPassAt", None)
3758
  _cadence_due = not passes or _t.time() - last_full_pass_at >= _every
 
3883
  "the next admitted pass resumes the unfinished Odoo cohort")
3884
  if _slot_claimed:
3885
  _slot_write(_t.time(), "slotDoneAt")
3886
+ slot_rerun_check = True
3887
  # H1: progress is a changed unfinished checkpoint in the candidate's `_sync_state`.
3888
  _backfill_after = _backfill_read(_cad["tenant"]) if _backfill_read else None
3889
  _moved = ([row for row in _backfill_after if row not in _backfill_before]
 
4109
  _reason is not None or bool(_meta_state.get("applied")))
4110
  if _slot_claimed:
4111
  _slot_write(_t.time(), "slotDoneAt")
4112
+ slot_rerun_check = True
4113
 
4114
 
4115
  def _boot_app_migrations(why, tenant=None):
 
4333
  #: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a
4334
  #: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause.
4335
  MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""}
4336
+ #: One shared deadline for every boot seed restore job, so the seed thread always ends.
4337
+ _BOOT_SEED_DEADLINE_SECONDS = 1200
4338
 
4339
 
4340
  def _seed_mirror_if_absent():
4341
  """Restore tenant manifests before considering the resident tenant's legacy seed."""
4342
+ import os as _seed_os
4343
+ import sys as _seed_sys
4344
+ import time as _seed_time
4345
+ import traceback as _seed_tb
4346
+ import process_role as _seed_role
4347
  from harness import datastore as _ds
4348
  from harness import runtime as _runtime
4349
  MIRROR_SEED["attempted"] = True
 
4359
 
4360
  def restore(tenant_key, rt):
4361
  try:
4362
+ try:
4363
+ if _ds.odoo_mirror_restored(tenant_key, rt):
4364
+ return
4365
+ result = _ds.restore_odoo_mirror(tenant_key, rt)
4366
+ except Exception as exc:
4367
+ result = refused(tenant_key, exc)
4368
+ if result["state"] == "refused":
4369
+ MIRROR_SEED["cause"] = "; ".join(result["problems"])
4370
+ MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
4371
+ finally:
4372
+ # A finished tenant restore is progress for the liveness check.
4373
+ getattr(_seed_role, "note_progress", lambda _step: None)("boot seed restore")
4374
 
4375
  def runtime_failed(tenant_key, exc):
4376
  MIRROR_SEED["cause"] = "; ".join(refused(tenant_key, exc)["problems"])
 
4381
  restore, name="odoo-mirror-restore", tenants=_runtime.known_tenants(),
4382
  get_runtime=tenant_scan.runtime_lookup(_runtime, "odoo-mirror-restore",
4383
  on_error=runtime_failed), kind="light")
4384
+ try:
4385
+ deadline = float(_seed_os.environ.get("AIOS_BOOT_SEED_DEADLINE_S")
4386
+ or globals().get("_BOOT_SEED_DEADLINE_SECONDS", 1200))
4387
+ except ValueError:
4388
+ deadline = 1200.0
4389
+ deadline_at = _seed_time.monotonic() + deadline
4390
+ stuck = []
4391
+ for tenant_key, job in jobs.items():
4392
+ # Every join shares one deadline: a restore that never returns cannot hold the seed thread.
4393
+ job.join(max(0.0, deadline_at - _seed_time.monotonic()))
4394
+ if not getattr(job, "is_alive", lambda: False)():
4395
+ continue
4396
+ name = getattr(job, "name", None) or f"odoo-mirror-restore:{tenant_key}"
4397
+ frame = _seed_sys._current_frames().get(getattr(job, "ident", None))
4398
+ where = ", ".join(f"{_seed_os.path.basename(fs.filename)}:{fs.lineno} {fs.name}"
4399
+ for fs in (_seed_tb.extract_stack(frame)[-3:] if frame else ())) or "unknown"
4400
+ print(f"[aios-api] boot seed restore {name} passed its {deadline:g} s deadline at "
4401
+ f"{where}; the worker continues without it", flush=True)
4402
+ stuck.append(name)
4403
+ if stuck:
4404
+ # The legacy seed never races a restore that may still be installing.
4405
+ MIRROR_SEED["cause"] = f"the boot seed restore {', '.join(stuck)} passed its {deadline:g} s deadline"
4406
+ MIRROR_SEED["recommendation"] = "check the store and the mirror hand off, then restart"
4407
+ return False
4408
+ if _ds.ensure_seed(): # The legacy seed decides on the restored manifests.
4409
  MIRROR_SEED["seeded"] = True
4410
  print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
4411
  return True
api/odoo_relational.py CHANGED
@@ -6055,6 +6055,28 @@ def slot_pass_killed(now, slot, memory):
6055
  return newest
6056
 
6057
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6058
  def _slot_resumes(memory):
6059
  """How many boots resumed the slot named by `slotResumedFor`, 0 when unreadable."""
6060
  try:
 
6055
  return newest
6056
 
6057
 
6058
+ def slot_pass_rerun(now, slot, memory):
6059
+ """The newest slot's epoch when its claimed pass ended without a good sync and no resume ran for it, else None.
6060
+
6061
+ The caller writes the resume pair for that slot, so a slot reruns at most once. Never raises.
6062
+ """
6063
+ memory = memory if isinstance(memory, dict) else {}
6064
+ try:
6065
+ newest = slot_before(now, slot)
6066
+ if slot_pass_satisfied(now, slot, memory):
6067
+ return None
6068
+ claim, done = _good_epoch(memory.get("slotPassAt")), _good_epoch(memory.get("slotDoneAt"))
6069
+ if claim is None or claim < newest or done is None or done < claim:
6070
+ return None
6071
+ resumed = _good_epoch(memory.get("slotResumedFor"))
6072
+ # Any resume of this slot, a rerun or a killed-pass resume, spends its one rerun.
6073
+ if resumed is not None and resumed >= newest:
6074
+ return None
6075
+ return newest
6076
+ except Exception: # noqa: BLE001
6077
+ return None
6078
+
6079
+
6080
  def _slot_resumes(memory):
6081
  """How many boots resumed the slot named by `slotResumedFor`, 0 when unreadable."""
6082
  try:
api/process_role.py CHANGED
@@ -95,6 +95,9 @@ LAST_INSTALL_ERROR = {}
95
  # The web's last read of the store: the worker heartbeat and each tenant's published mirror ids.
96
  _STORED = {"beat": None, "mirrors": {}}
97
  _RESYNC_APPLIED = {"at": ""}
 
 
 
98
 
99
 
100
  def role():
@@ -1031,6 +1034,21 @@ def _running_main():
1031
  _CATCH_UP_SENT = {}
1032
 
1033
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1034
  def write_heartbeat(now=None, get_runtime=None):
1035
  """Worker side: write the heartbeat, each tenant's catch-up and RESYNC_STATE to the C-status store.
1036
 
@@ -1041,7 +1059,8 @@ def write_heartbeat(now=None, get_runtime=None):
1041
  return False
1042
  get_runtime = get_runtime or _status_runtime
1043
  beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
1044
- "at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE)}
 
1045
  rt = get_runtime(writer_tenant())
1046
  import worker_provenance
1047
  attestation = worker_provenance.heartbeat_attestation(rt, beat)
@@ -1115,6 +1134,7 @@ def start_worker_services():
1115
  from harness import mirror_store
1116
  mirror_store.register_publish_listener(record_publication)
1117
  parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
 
1118
  start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
1119
  # Before the first heavy job: each one now waits until this worker holds the writer lease.
1120
  arm_writer_lease()
 
95
  # The web's last read of the store: the worker heartbeat and each tenant's published mirror ids.
96
  _STORED = {"beat": None, "mirrors": {}}
97
  _RESYNC_APPLIED = {"at": ""}
98
+ # The worker's last pass step: the web judges liveness by its age, never by HF stage alone.
99
+ _PROGRESS = {"last": None}
100
+ _PROGRESS_LOCK = threading.Lock()
101
 
102
 
103
  def role():
 
1034
  _CATCH_UP_SENT = {}
1035
 
1036
 
1037
+ def note_progress(step):
1038
+ """Worker side: record that a pass step ran. Memory only: it never raises and never does I/O."""
1039
+ try:
1040
+ with _PROGRESS_LOCK:
1041
+ _PROGRESS["last"] = {"step": str(step)[:80], "at": time.time()}
1042
+ except Exception: # noqa: BLE001
1043
+ pass
1044
+
1045
+
1046
+ def _progress_copy():
1047
+ with _PROGRESS_LOCK:
1048
+ last = _PROGRESS["last"]
1049
+ return dict(last) if last else None
1050
+
1051
+
1052
  def write_heartbeat(now=None, get_runtime=None):
1053
  """Worker side: write the heartbeat, each tenant's catch-up and RESYNC_STATE to the C-status store.
1054
 
 
1059
  return False
1060
  get_runtime = get_runtime or _status_runtime
1061
  beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
1062
+ "at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE),
1063
+ "progress": _progress_copy()}
1064
  rt = get_runtime(writer_tenant())
1065
  import worker_provenance
1066
  attestation = worker_provenance.heartbeat_attestation(rt, beat)
 
1134
  from harness import mirror_store
1135
  mirror_store.register_publish_listener(record_publication)
1136
  parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
1137
+ note_progress("boot")
1138
  start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
1139
  # Before the first heavy job: each one now waits until this worker holds the writer lease.
1140
  arm_writer_lease()
api/sync_runtime.py CHANGED
@@ -54,6 +54,8 @@ WEB_SECTION = "web"
54
  ALIVE_STAGES = ("RUNNING", "SCHEDULING")
55
  #: A job record older than its timeout plus this margin is dead, whatever inspect_job answers.
56
  JOB_MARGIN_SECONDS = 600
 
 
57
  #: A --once wait at most this long runs as a plain wait. A longer wait first checks the desk.
58
  IDLE_GRACE_SECONDS = 30.0
59
  #: The longest sleep a --once job stores as its next pass. A day keeps a lost slot from stalling.
@@ -421,6 +423,28 @@ def job_alive(worker, api, now, namespace=None):
421
  return stage in ALIVE_STAGES, stage.lower() or "unknown"
422
 
423
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
424
  def pass_due(worker, rt, now):
425
  """(due, why): a due desk request, or a stored nextPassAt at or before now, or none stored."""
426
  try:
@@ -507,6 +531,19 @@ def tick(wanted=None, *, get_runtime=None, api=None, now=None):
507
  {"version": version, "at": stamp(stamp_now)}):
508
  _WEB_WRITTEN[slug] = version
509
  live, why = job_alive(worker, api, stamp_now, namespace)
 
 
 
 
 
 
 
 
 
 
 
 
 
510
  record = process_role._FLEET.setdefault(slug, {"pid": None, "jobs": 0, "job": None})
511
  record.update(alive=live, aliveWhy=why)
512
  if live:
 
54
  ALIVE_STAGES = ("RUNNING", "SCHEDULING")
55
  #: A job record older than its timeout plus this margin is dead, whatever inspect_job answers.
56
  JOB_MARGIN_SECONDS = 600
57
+ #: A live resident job with no pass step for this long is hung: the web cancels and replaces it.
58
+ PROGRESS_STALE_SECONDS = float(os.environ.get("AIOS_SYNC_PROGRESS_STALE_SECONDS") or 1800)
59
  #: A --once wait at most this long runs as a plain wait. A longer wait first checks the desk.
60
  IDLE_GRACE_SECONDS = 30.0
61
  #: The longest sleep a --once job stores as its next pass. A day keeps a lost slot from stalling.
 
423
  return stage in ALIVE_STAGES, stage.lower() or "unknown"
424
 
425
 
426
+ def job_hung(worker, now, why):
427
+ """(hung, age, step): a job alive by beat or RUNNING stage whose last pass step is older than the budget."""
428
+ if why not in ("heartbeat", "running"):
429
+ # SCHEDULING, an unread stage and an unknown one are never judged hung.
430
+ return False, None, None
431
+ beat = worker.get("heartbeat") if isinstance(worker.get("heartbeat"), dict) else {}
432
+ progress = beat.get("progress") if isinstance(beat.get("progress"), dict) else None
433
+ job = worker.get(JOB_SECTION) if isinstance(worker.get(JOB_SECTION), dict) else {}
434
+ submitted = epoch(job.get("submittedAt"))
435
+ try:
436
+ stepped = float(progress["at"]) if progress else None
437
+ except (KeyError, TypeError, ValueError):
438
+ stepped = None
439
+ # Progress never predates the job's own submission, so an older job's beat cannot condemn its successor.
440
+ marks = [mark for mark in (stepped, submitted) if mark is not None]
441
+ if not job.get("id") or not marks:
442
+ return False, None, None
443
+ age = now - max(marks)
444
+ step = str(progress.get("step") or "unknown") if progress else "none reported"
445
+ return age > PROGRESS_STALE_SECONDS, int(round(age)), step
446
+
447
+
448
  def pass_due(worker, rt, now):
449
  """(due, why): a due desk request, or a stored nextPassAt at or before now, or none stored."""
450
  try:
 
531
  {"version": version, "at": stamp(stamp_now)}):
532
  _WEB_WRITTEN[slug] = version
533
  live, why = job_alive(worker, api, stamp_now, namespace)
534
+ hung, age, step = job_hung(worker, stamp_now, why) if live and mode == "resident" else (False, None, None)
535
+ if hung and api is not None:
536
+ job_id = worker[JOB_SECTION]["id"]
537
+ try:
538
+ api.cancel_job(job_id=job_id, namespace=namespace)
539
+ except Exception as exc: # noqa: BLE001
540
+ # A failed cancel keeps the slot taken: no replacement until a cancel succeeds.
541
+ print(f"[sync-runtime] cancel of hung sync job {job_id} failed ({type(exc).__name__}); "
542
+ f"it counts as alive this tick", flush=True)
543
+ else:
544
+ print(f"[sync-runtime] cancelled hung sync job {job_id}: no progress for {age} s "
545
+ f"(last step {step})", flush=True)
546
+ live, why = False, "cancelled hung"
547
  record = process_role._FLEET.setdefault(slug, {"pid": None, "jobs": 0, "job": None})
548
  record.update(alive=live, aliveWhy=why)
549
  if live:
api/test_odoo_rank_refresh.py CHANGED
@@ -1180,14 +1180,15 @@ class RankRefreshTests(unittest.TestCase):
1180
  self.assertNotIn(('amazon', boot, 'boot', False), events)
1181
 
1182
  def test_h73_a_restart_inside_a_slot_pass_does_not_run_it_twice(self):
1183
- """H2: a slot pass that ended is not re-run; one killed before its end runs once, never twice"""
1184
  import odoo_relational as rel
1185
  stamp = self.utc('2026-10-24T06:00:01Z')
1186
  yesterday = self.utc('2026-10-23T06:30:00Z')
1187
  base = {'passAt': yesterday, 'fullRebuildAt': yesterday, 'slotPassAt': stamp,
1188
  'fingerprints': {'res_partner': (3, 7)}}
1189
- # Ended: `slotDoneAt` after the claim, or (older memory) `passAt` after it. No Odoo call.
1190
- for ended in ({'slotDoneAt': self.utc('2026-10-24T06:30:00Z')},
 
1191
  {'passAt': self.utc('2026-10-24T06:30:00Z')}):
1192
  events, sleeps, odoo = self.run_slot_loop(
1193
  dict(base, **ended), boot=self.utc('2026-10-24T06:40:00Z'), max_sleeps=1)
@@ -1661,5 +1662,27 @@ class RankRefreshTests(unittest.TestCase):
1661
  0 <= firsts['fsanyoto/loopable'] - self.utc('2026-10-24T07:00:00Z') <= 1, firsts)
1662
 
1663
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1664
  if __name__ == '__main__':
1665
  unittest.main()
 
1180
  self.assertNotIn(('amazon', boot, 'boot', False), events)
1181
 
1182
  def test_h73_a_restart_inside_a_slot_pass_does_not_run_it_twice(self):
1183
+ """H2: an ended slot pass whose one rerun is spent is not re-run; one killed before its end runs once, never twice"""
1184
  import odoo_relational as rel
1185
  stamp = self.utc('2026-10-24T06:00:01Z')
1186
  yesterday = self.utc('2026-10-23T06:30:00Z')
1187
  base = {'passAt': yesterday, 'fullRebuildAt': yesterday, 'slotPassAt': stamp,
1188
  'fingerprints': {'res_partner': (3, 7)}}
1189
+ # Ended: `slotDoneAt` after the claim with its rerun spent, or (older memory) `passAt` after it. No Odoo call.
1190
+ for ended in ({'slotDoneAt': self.utc('2026-10-24T06:30:00Z'),
1191
+ 'slotResumedFor': self.utc('2026-10-24T06:00:00Z'), 'slotResumes': 1},
1192
  {'passAt': self.utc('2026-10-24T06:30:00Z')}):
1193
  events, sleeps, odoo = self.run_slot_loop(
1194
  dict(base, **ended), boot=self.utc('2026-10-24T06:40:00Z'), max_sleeps=1)
 
1662
  0 <= firsts['fsanyoto/loopable'] - self.utc('2026-10-24T07:00:00Z') <= 1, firsts)
1663
 
1664
 
1665
+ def test_t2_an_ended_slot_pass_without_a_good_sync_reruns_once_the_same_day(self):
1666
+ """T2: ended without a good sync reruns once; a spent rerun or a good pass does not"""
1667
+ import odoo_relational as rel
1668
+ slot = rel.sync_slot(rel.read_config(self.SlotStore('royal-imports')))
1669
+ slot_at, boot = self.utc('2026-10-24T06:00:00Z'), self.utc('2026-10-24T06:40:00Z')
1670
+ yesterday = self.utc('2026-10-23T06:30:00Z')
1671
+ ended = {'passAt': yesterday, 'fullRebuildAt': yesterday, 'slotPassAt': slot_at + 1,
1672
+ 'slotDoneAt': self.utc('2026-10-24T06:30:00Z'), 'fingerprints': {'res_partner': (3, 7)}}
1673
+ self.assertEqual(rel.slot_pass_rerun(boot, slot, ended), slot_at)
1674
+ self.assertIsNone(rel.slot_pass_rerun(boot, slot, dict(ended, slotResumedFor=slot_at, slotResumes=1)))
1675
+ self.assertIsNone(rel.slot_pass_rerun(boot, slot, dict(ended, lastGoodSyncAt=slot_at + 600)))
1676
+ self.assertIsNone(rel.slot_pass_rerun(boot, slot, dict(ended, slotDoneAt=yesterday)))
1677
+ self.assertIsNone(rel.slot_pass_rerun(boot, slot, None))
1678
+ # The loop reruns it at boot, outside the slot window, and spends the resume pair.
1679
+ events, sleeps, odoo = self.run_slot_loop(ended, boot=boot, max_sleeps=1)
1680
+ self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
1681
+ self.assertIn('the 13:00 Asia/Jakarta slot pass ended without a good sync; rerunning it once today',
1682
+ self.loop_output)
1683
+ memory = rel.read_resync_memory(self.slot_store)
1684
+ self.assertEqual((memory['slotResumedFor'], memory['slotResumes']), (slot_at, 1))
1685
+
1686
+
1687
  if __name__ == '__main__':
1688
  unittest.main()
platform/core/received_changes.py CHANGED
@@ -11,10 +11,13 @@ import os
11
  from pathlib import Path
12
  import tempfile
13
  import threading
 
14
 
15
 
16
  STORE_KEY = "received_changes_v1"
17
  MAX_EVENTS = 2000
 
 
18
  BASELINE_CHUNK_BYTES = 8 * 1024 * 1024
19
  PROJECTION_VERSION = 1
20
  _LISTENERS = []
@@ -215,7 +218,17 @@ def _baseline_lock(root):
215
  msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
216
  else:
217
  import fcntl
218
- fcntl.flock(handle, fcntl.LOCK_EX)
 
 
 
 
 
 
 
 
 
 
219
  try:
220
  yield
221
  finally:
@@ -393,15 +406,26 @@ def _move_baseline(rt, identity, expected, updated):
393
  raise BaselineUnavailable("The observation reference was not saved.")
394
 
395
 
396
- def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot):
397
- cursor = con.cursor()
 
 
 
 
 
 
 
 
398
  try:
399
- cursor.execute("SELECT coalesce(b.row_id, a.row_id), b.image, a.image "
400
- "FROM read_parquet(?) b FULL OUTER JOIN read_parquet(?) a "
401
- "USING (row_id) WHERE b.image IS DISTINCT FROM a.image "
402
- "ORDER BY coalesce(b.row_id, a.row_id)", [str(baseline), str(current)])
403
  while batch := cursor.fetchmany(256):
404
  for row_id, before_json, after_json in batch:
 
 
 
 
 
405
  before, after = json.loads(before_json or "{}"), json.loads(after_json or "{}")
406
  changes = {}
407
  for key, (_sql, formatter) in spec["cols"].items():
@@ -414,11 +438,12 @@ def _compare_baseline(rt, source, database, spec, con, baseline, current, snapsh
414
  operation = "created" if before_json is None else "deleted" if after_json is None else "updated"
415
  publish(rt, source=source, database=database, row_id=row_id, changes=changes,
416
  receipt_id=snapshot, operation=operation)
 
417
  finally:
418
  cursor.close()
419
 
420
 
421
- def _snapshot(rt, source, database, spec, con, root, manifest):
422
  """Compare verified durable observations in batches, recovering interrupted comparisons first."""
423
  if not spec.get("cols"):
424
  return
@@ -443,7 +468,8 @@ def _snapshot(rt, source, database, spec, con, root, manifest):
443
  observed = _restore_baseline(rt, identity, binding, pending["pending"], current, con)
444
  if pending["current"]:
445
  _restore_baseline(rt, identity, binding, pending["current"], baseline, con)
446
- _compare_baseline(rt, source, database, spec, con, baseline, current, observed["snapshot"])
 
447
  committed = {"current": pending["pending"], "pending": None}
448
  _move_baseline(rt, identity, pending, committed)
449
  return committed
@@ -486,13 +512,20 @@ def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
486
  raise ValueError("Received-change runtime belongs to another tenant.")
487
  root = Path(rt.datastore_path).with_suffix(".received-changes")
488
  with _baseline_lock(root):
 
 
 
489
  con = datastore.apply_limits(datastore.duckdb.connect(":memory:"))
490
  try:
491
  for name, files in paths.items():
492
  con.read_parquet([str(path) for path in files], union_by_name=True).create_view(str(name))
493
  for database, spec in resolver(rt, con).items():
 
 
 
 
494
  try:
495
- _snapshot(rt, connector, str(database), spec, con, root, manifest)
496
  except Exception as exc:
497
  _LOG.warning("Received-change projection deferred for %s: %s",
498
  database, type(exc).__name__)
@@ -700,6 +733,83 @@ def selftest():
700
  check("cross-tenant publication refused", True)
701
  else:
702
  check("cross-tenant publication refused", False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
703
  finally:
704
  con.close()
705
  finally:
 
11
  from pathlib import Path
12
  import tempfile
13
  import threading
14
+ import time
15
 
16
 
17
  STORE_KEY = "received_changes_v1"
18
  MAX_EVENTS = 2000
19
+ JOURNAL_DEADLINE_SECONDS = 120
20
+ BASELINE_LOCK_SECONDS = 300
21
  BASELINE_CHUNK_BYTES = 8 * 1024 * 1024
22
  PROJECTION_VERSION = 1
23
  _LISTENERS = []
 
218
  msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
219
  else:
220
  import fcntl
221
+ # A held lock fails the hand-off after a bounded wait instead of hanging the worker.
222
+ wait = float(os.environ.get("AIOS_RECEIVED_CHANGE_LOCK_S") or BASELINE_LOCK_SECONDS)
223
+ end = time.monotonic() + wait
224
+ while True:
225
+ try:
226
+ fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
227
+ break
228
+ except BlockingIOError:
229
+ if time.monotonic() >= end:
230
+ raise TimeoutError(f"The received-change baseline lock stayed busy for {wait:g} s.")
231
+ time.sleep(0.05)
232
  try:
233
  yield
234
  finally:
 
406
  raise BaselineUnavailable("The observation reference was not saved.")
407
 
408
 
409
+ def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
410
+ diff = ("FROM read_parquet(?) b FULL OUTER JOIN read_parquet(?) a "
411
+ "USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
412
+ count = con.execute("SELECT count(*) " + diff, [str(baseline), str(current)]).fetchone()[0]
413
+ # A diff wider than the retained window cannot be replayed, so it is recorded as one bulk change.
414
+ if count > MAX_EVENTS:
415
+ print(f"[received-changes] {database}: {count} changed rows exceed the journal's "
416
+ f"{MAX_EVENTS}-event window; recorded as a bulk change, rows not journaled", flush=True)
417
+ return
418
+ cursor, done = con.cursor(), 0
419
  try:
420
+ cursor.execute("SELECT coalesce(b.row_id, a.row_id), b.image, a.image " + diff
421
+ + " ORDER BY coalesce(b.row_id, a.row_id)", [str(baseline), str(current)])
 
 
422
  while batch := cursor.fetchmany(256):
423
  for row_id, before_json, after_json in batch:
424
+ # Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
425
+ if deadline is not None and time.monotonic() >= deadline[0]:
426
+ print(f"[received-changes] {database}: journal deadline {deadline[1]:g} s passed "
427
+ f"after {done} of {count} rows; the rest are not journaled", flush=True)
428
+ return
429
  before, after = json.loads(before_json or "{}"), json.loads(after_json or "{}")
430
  changes = {}
431
  for key, (_sql, formatter) in spec["cols"].items():
 
438
  operation = "created" if before_json is None else "deleted" if after_json is None else "updated"
439
  publish(rt, source=source, database=database, row_id=row_id, changes=changes,
440
  receipt_id=snapshot, operation=operation)
441
+ done += 1
442
  finally:
443
  cursor.close()
444
 
445
 
446
+ def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
447
  """Compare verified durable observations in batches, recovering interrupted comparisons first."""
448
  if not spec.get("cols"):
449
  return
 
468
  observed = _restore_baseline(rt, identity, binding, pending["pending"], current, con)
469
  if pending["current"]:
470
  _restore_baseline(rt, identity, binding, pending["current"], baseline, con)
471
+ _compare_baseline(rt, source, database, spec, con, baseline, current, observed["snapshot"],
472
+ deadline)
473
  committed = {"current": pending["pending"], "pending": None}
474
  _move_baseline(rt, identity, pending, committed)
475
  return committed
 
512
  raise ValueError("Received-change runtime belongs to another tenant.")
513
  root = Path(rt.datastore_path).with_suffix(".received-changes")
514
  with _baseline_lock(root):
515
+ # One journal budget per publication, shared by its databases.
516
+ seconds = float(os.environ.get("AIOS_RECEIVED_CHANGE_DEADLINE_S") or JOURNAL_DEADLINE_SECONDS)
517
+ deadline = (time.monotonic() + seconds, seconds)
518
  con = datastore.apply_limits(datastore.duckdb.connect(":memory:"))
519
  try:
520
  for name, files in paths.items():
521
  con.read_parquet([str(path) for path in files], union_by_name=True).create_view(str(name))
522
  for database, spec in resolver(rt, con).items():
523
+ # A skipped database keeps its stored baseline, so the next publication compares again.
524
+ if time.monotonic() >= deadline[0]:
525
+ print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
526
+ continue
527
  try:
528
+ _snapshot(rt, connector, str(database), spec, con, root, manifest, deadline)
529
  except Exception as exc:
530
  _LOG.warning("Received-change projection deferred for %s: %s",
531
  database, type(exc).__name__)
 
733
  check("cross-tenant publication refused", True)
734
  else:
735
  check("cross-tenant publication refused", False)
736
+
737
+ import contextlib
738
+ import io
739
+ bound = FakeRuntime("fixture-bound", directory)
740
+ bulk_path = Path(directory) / "bulk.parquet"
741
+ con.execute("CREATE TABLE bulk_rows (native_id BIGINT, status VARCHAR)")
742
+ con.execute("INSERT INTO bulk_rows SELECT range, 'open' FROM range(1, 9)")
743
+ bulk_spec = {"table": "bulk_rows", "id": "native_id", "cols": {"status": ("status", str)}}
744
+
745
+ def bound_snapshot(databases, identity, second):
746
+ register_projection("odoo", lambda _rt, _cur: {name: bulk_spec for name in databases})
747
+ con.execute("COPY bulk_rows TO ? (FORMAT PARQUET)", [str(bulk_path)])
748
+ manifest = {"tenant": bound.key, "snapshot_id": identity,
749
+ "completed_at": f"2026-11-17T00:00:{second:02d}Z"}
750
+ output = io.StringIO()
751
+ with patch.object(runtime, "get_runtime", return_value=bound), \
752
+ contextlib.redirect_stdout(output):
753
+ mirror_hook("odoo", bound.key, manifest, [], {"bulk_rows": [bulk_path]})
754
+ return output.getvalue()
755
+
756
+ def bound_state(database):
757
+ identity = hashlib.sha256(json.dumps([bound.key, "odoo", database]).encode()).hexdigest()
758
+ state = _observation_state(bound, identity)[0]
759
+ reference = state["current"]
760
+ manifest = json.loads(_read_blob(bound, _blob_path(identity, "manifests", reference["sha256"]),
761
+ reference["sha256"], reference["bytes"]))
762
+ return state, manifest["snapshot"]
763
+
764
+ bound_snapshot(("ut_bulk", "ut_bulk_twin"), "bound-initial", 10)
765
+ twin_before = bound_state("ut_bulk_twin")
766
+ con.execute("UPDATE bulk_rows SET status='done'")
767
+ calls = []
768
+
769
+ def slow_publish(*_args, **_kwargs):
770
+ calls.append(1)
771
+ time.sleep(0.2)
772
+
773
+ started = time.monotonic()
774
+ with patch(__name__ + ".publish", side_effect=slow_publish), \
775
+ patch.dict(os.environ, {"AIOS_RECEIVED_CHANGE_DEADLINE_S": "0.5"}):
776
+ printed = bound_snapshot(("ut_bulk", "ut_bulk_twin"), "bound-stall", 11)
777
+ state, committed = bound_state("ut_bulk")
778
+ check("stalled journal stops at its deadline", time.monotonic() - started < 1.2
779
+ and 0 < len(calls) < 8 and "ut_bulk: journal deadline 0.5 s passed after "
780
+ + f"{len(calls)} of 8 rows; the rest are not journaled" in printed)
781
+ check("stalled journal commits its baseline", committed == "bound-stall"
782
+ and state["pending"] is None)
783
+ check("database after the deadline keeps its baseline",
784
+ "ut_bulk_twin: skipped, journal deadline passed" in printed
785
+ and bound_state("ut_bulk_twin") == twin_before)
786
+ con.execute("UPDATE bulk_rows SET status='bulk' WHERE native_id <= 3")
787
+ calls.clear()
788
+ with patch(__name__ + ".publish", side_effect=slow_publish), \
789
+ patch(__name__ + ".MAX_EVENTS", 1):
790
+ printed = bound_snapshot(("ut_bulk",), "bound-bulk", 12)
791
+ state, committed = bound_state("ut_bulk")
792
+ check("bulk change journals no rows", not calls and "ut_bulk: 3 changed rows exceed "
793
+ "the journal's 1-event window; recorded as a bulk change, rows not journaled" in printed)
794
+ check("bulk change commits its baseline", committed == "bound-bulk"
795
+ and state["pending"] is None)
796
+ if os.name != "nt":
797
+ import fcntl
798
+ lock_root = Path(directory) / "lock-fixture"
799
+ lock_root.mkdir()
800
+ with (lock_root / "lock").open("a+b") as holder:
801
+ fcntl.flock(holder, fcntl.LOCK_EX)
802
+ started = time.monotonic()
803
+ try:
804
+ with patch.dict(os.environ, {"AIOS_RECEIVED_CHANGE_LOCK_S": "0.2"}), \
805
+ _baseline_lock(lock_root):
806
+ pass
807
+ except TimeoutError as exc:
808
+ check("held baseline lock times out", time.monotonic() - started < 1.0
809
+ and "0.2 s" in str(exc))
810
+ else:
811
+ check("held baseline lock times out", False)
812
+ fcntl.flock(holder, fcntl.LOCK_UN)
813
  finally:
814
  con.close()
815
  finally:
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 2489684f9351f94e00784ec384d8a83630ff24f0d18c3d2f83ea4254c7906309
 
1
+ afc4252504d8e071ec0937b2e6a7ad64c1e00860885ad0596112eaca89d79eea