Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +72 -12
- api/odoo_relational.py +22 -0
- api/process_role.py +21 -1
- api/sync_runtime.py +37 -0
- api/test_odoo_rank_refresh.py +26 -3
- platform/core/received_changes.py +120 -10
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "99129353b8d3",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
99129353b8d3
|
api/main.py
CHANGED
|
@@ -1385,10 +1385,16 @@ def _pull_amazon(why, vendor=True):
|
|
| 1385 |
_status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
|
| 1386 |
raise
|
| 1387 |
_amazon_status.record_pass_start(rt, "amazon_sp")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1388 |
try:
|
| 1389 |
rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
|
| 1390 |
slot_pass=bool(RESYNC_STATE.get("slotPass")),
|
| 1391 |
-
log=
|
| 1392 |
except Exception:
|
| 1393 |
_status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
|
| 1394 |
raise
|
|
@@ -3005,6 +3011,9 @@ def _resync_mirror_pass(manual, passes, entities=None):
|
|
| 3005 |
def _resync_log(*parts, **_kw):
|
| 3006 |
# One timestamped line per step. A logging error never changes the pass.
|
| 3007 |
try:
|
|
|
|
|
|
|
|
|
|
| 3008 |
stamp = _pass_dt.now(_pass_tz.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 3009 |
print(f"[aios-api] {stamp} odoo pass: {' '.join(str(p) for p in parts)}", flush=True)
|
| 3010 |
except Exception: # noqa: BLE001
|
|
@@ -3583,6 +3592,8 @@ def _store_resync_loop():
|
|
| 3583 |
slot_restored = False
|
| 3584 |
# H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
|
| 3585 |
slot_resume_checked = False
|
|
|
|
|
|
|
| 3586 |
_slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
|
| 3587 |
_idle_wait = globals().get("_sync_idle_wait") or (lambda event, seconds: event.wait(seconds))
|
| 3588 |
_slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
|
|
@@ -3594,6 +3605,8 @@ def _store_resync_loop():
|
|
| 3594 |
backfill_spacing_seconds = 60.0
|
| 3595 |
backfill_cap = None
|
| 3596 |
while True:
|
|
|
|
|
|
|
| 3597 |
if backfill_next_at is not None and backfill_cap is not None and _t.time() >= backfill_cap:
|
| 3598 |
backfill_next_at = None
|
| 3599 |
# ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
|
|
@@ -3728,6 +3741,18 @@ def _store_resync_loop():
|
|
| 3728 |
if _slot_interrupted_clear():
|
| 3729 |
print(f"[aios-api] store resync: a good pass ended after the {_slot['at']} "
|
| 3730 |
f"{_slot['tz']} slot; cleared the slot_pass_interrupted failure")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3731 |
else:
|
| 3732 |
RESYNC_STATE.pop("nextPassAt", None)
|
| 3733 |
_cadence_due = not passes or _t.time() - last_full_pass_at >= _every
|
|
@@ -3858,6 +3883,7 @@ def _store_resync_loop():
|
|
| 3858 |
"the next admitted pass resumes the unfinished Odoo cohort")
|
| 3859 |
if _slot_claimed:
|
| 3860 |
_slot_write(_t.time(), "slotDoneAt")
|
|
|
|
| 3861 |
# H1: progress is a changed unfinished checkpoint in the candidate's `_sync_state`.
|
| 3862 |
_backfill_after = _backfill_read(_cad["tenant"]) if _backfill_read else None
|
| 3863 |
_moved = ([row for row in _backfill_after if row not in _backfill_before]
|
|
@@ -4083,6 +4109,7 @@ def _store_resync_loop():
|
|
| 4083 |
_reason is not None or bool(_meta_state.get("applied")))
|
| 4084 |
if _slot_claimed:
|
| 4085 |
_slot_write(_t.time(), "slotDoneAt")
|
|
|
|
| 4086 |
|
| 4087 |
|
| 4088 |
def _boot_app_migrations(why, tenant=None):
|
|
@@ -4306,10 +4333,17 @@ def _prewarm():
|
|
| 4306 |
#: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a
|
| 4307 |
#: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause.
|
| 4308 |
MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""}
|
|
|
|
|
|
|
| 4309 |
|
| 4310 |
|
| 4311 |
def _seed_mirror_if_absent():
|
| 4312 |
"""Restore tenant manifests before considering the resident tenant's legacy seed."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4313 |
from harness import datastore as _ds
|
| 4314 |
from harness import runtime as _runtime
|
| 4315 |
MIRROR_SEED["attempted"] = True
|
|
@@ -4325,14 +4359,18 @@ def _seed_mirror_if_absent():
|
|
| 4325 |
|
| 4326 |
def restore(tenant_key, rt):
|
| 4327 |
try:
|
| 4328 |
-
|
| 4329 |
-
|
| 4330 |
-
|
| 4331 |
-
|
| 4332 |
-
|
| 4333 |
-
|
| 4334 |
-
|
| 4335 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4336 |
|
| 4337 |
def runtime_failed(tenant_key, exc):
|
| 4338 |
MIRROR_SEED["cause"] = "; ".join(refused(tenant_key, exc)["problems"])
|
|
@@ -4343,9 +4381,31 @@ def _seed_mirror_if_absent():
|
|
| 4343 |
restore, name="odoo-mirror-restore", tenants=_runtime.known_tenants(),
|
| 4344 |
get_runtime=tenant_scan.runtime_lookup(_runtime, "odoo-mirror-restore",
|
| 4345 |
on_error=runtime_failed), kind="light")
|
| 4346 |
-
|
| 4347 |
-
|
| 4348 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4349 |
MIRROR_SEED["seeded"] = True
|
| 4350 |
print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
|
| 4351 |
return True
|
|
|
|
| 1385 |
_status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
|
| 1386 |
raise
|
| 1387 |
_amazon_status.record_pass_start(rt, "amazon_sp")
|
| 1388 |
+
|
| 1389 |
+
def _amazon_log(message):
|
| 1390 |
+
# Every Amazon pull step line is progress for the liveness check.
|
| 1391 |
+
getattr(_amazon_role, "note_progress", lambda _step: None)("amazon sync")
|
| 1392 |
+
print(f"[aios-api] amazon sync ({why}): {message}")
|
| 1393 |
+
|
| 1394 |
try:
|
| 1395 |
rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
|
| 1396 |
slot_pass=bool(RESYNC_STATE.get("slotPass")),
|
| 1397 |
+
log=_amazon_log))
|
| 1398 |
except Exception:
|
| 1399 |
_status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
|
| 1400 |
raise
|
|
|
|
| 3011 |
def _resync_log(*parts, **_kw):
|
| 3012 |
# One timestamped line per step. A logging error never changes the pass.
|
| 3013 |
try:
|
| 3014 |
+
import process_role as _pass_role
|
| 3015 |
+
# Every Odoo pass step line is progress for the liveness check.
|
| 3016 |
+
getattr(_pass_role, "note_progress", lambda _step: None)("odoo pass")
|
| 3017 |
stamp = _pass_dt.now(_pass_tz.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 3018 |
print(f"[aios-api] {stamp} odoo pass: {' '.join(str(p) for p in parts)}", flush=True)
|
| 3019 |
except Exception: # noqa: BLE001
|
|
|
|
| 3592 |
slot_restored = False
|
| 3593 |
# H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
|
| 3594 |
slot_resume_checked = False
|
| 3595 |
+
# T2: an ended slot pass is checked once for a same-day rerun; the boot memory counts as one.
|
| 3596 |
+
slot_rerun_check = True
|
| 3597 |
_slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
|
| 3598 |
_idle_wait = globals().get("_sync_idle_wait") or (lambda event, seconds: event.wait(seconds))
|
| 3599 |
_slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
|
|
|
|
| 3605 |
backfill_spacing_seconds = 60.0
|
| 3606 |
backfill_cap = None
|
| 3607 |
while True:
|
| 3608 |
+
# Each cycle of a running loop is progress; the boot seed wait above is not.
|
| 3609 |
+
getattr(_loop_role, "note_progress", lambda _step: None)("resync idle tick")
|
| 3610 |
if backfill_next_at is not None and backfill_cap is not None and _t.time() >= backfill_cap:
|
| 3611 |
backfill_next_at = None
|
| 3612 |
# ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
|
|
|
|
| 3741 |
if _slot_interrupted_clear():
|
| 3742 |
print(f"[aios-api] store resync: a good pass ended after the {_slot['at']} "
|
| 3743 |
f"{_slot['tz']} slot; cleared the slot_pass_interrupted failure")
|
| 3744 |
+
# T2: a slot pass that ended without a good sync reruns once the same day; a queued backfill goes first.
|
| 3745 |
+
if slot_rerun_check and not _slot_due and backfill_next_at is None:
|
| 3746 |
+
slot_rerun_check = False
|
| 3747 |
+
_rerun_memory = _memory if not passes else _resync_memory_read()
|
| 3748 |
+
_rerun = _rel_slot.slot_pass_rerun(_slot_now, _slot, _rerun_memory)
|
| 3749 |
+
if _rerun is not None:
|
| 3750 |
+
_slot_due = True
|
| 3751 |
+
# The resume pair marks the slot, so neither a rerun nor a resume repeats it.
|
| 3752 |
+
_slot_write(_rerun, "slotResumedFor")
|
| 3753 |
+
_slot_write(1, "slotResumes")
|
| 3754 |
+
print(f"[aios-api] store resync: the {_slot['at']} {_slot['tz']} slot pass ended "
|
| 3755 |
+
f"without a good sync; rerunning it once today")
|
| 3756 |
else:
|
| 3757 |
RESYNC_STATE.pop("nextPassAt", None)
|
| 3758 |
_cadence_due = not passes or _t.time() - last_full_pass_at >= _every
|
|
|
|
| 3883 |
"the next admitted pass resumes the unfinished Odoo cohort")
|
| 3884 |
if _slot_claimed:
|
| 3885 |
_slot_write(_t.time(), "slotDoneAt")
|
| 3886 |
+
slot_rerun_check = True
|
| 3887 |
# H1: progress is a changed unfinished checkpoint in the candidate's `_sync_state`.
|
| 3888 |
_backfill_after = _backfill_read(_cad["tenant"]) if _backfill_read else None
|
| 3889 |
_moved = ([row for row in _backfill_after if row not in _backfill_before]
|
|
|
|
| 4109 |
_reason is not None or bool(_meta_state.get("applied")))
|
| 4110 |
if _slot_claimed:
|
| 4111 |
_slot_write(_t.time(), "slotDoneAt")
|
| 4112 |
+
slot_rerun_check = True
|
| 4113 |
|
| 4114 |
|
| 4115 |
def _boot_app_migrations(why, tenant=None):
|
|
|
|
| 4333 |
#: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a
|
| 4334 |
#: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause.
|
| 4335 |
MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""}
|
| 4336 |
+
#: One shared deadline for every boot seed restore job, so the seed thread always ends.
|
| 4337 |
+
_BOOT_SEED_DEADLINE_SECONDS = 1200
|
| 4338 |
|
| 4339 |
|
| 4340 |
def _seed_mirror_if_absent():
|
| 4341 |
"""Restore tenant manifests before considering the resident tenant's legacy seed."""
|
| 4342 |
+
import os as _seed_os
|
| 4343 |
+
import sys as _seed_sys
|
| 4344 |
+
import time as _seed_time
|
| 4345 |
+
import traceback as _seed_tb
|
| 4346 |
+
import process_role as _seed_role
|
| 4347 |
from harness import datastore as _ds
|
| 4348 |
from harness import runtime as _runtime
|
| 4349 |
MIRROR_SEED["attempted"] = True
|
|
|
|
| 4359 |
|
| 4360 |
def restore(tenant_key, rt):
|
| 4361 |
try:
|
| 4362 |
+
try:
|
| 4363 |
+
if _ds.odoo_mirror_restored(tenant_key, rt):
|
| 4364 |
+
return
|
| 4365 |
+
result = _ds.restore_odoo_mirror(tenant_key, rt)
|
| 4366 |
+
except Exception as exc:
|
| 4367 |
+
result = refused(tenant_key, exc)
|
| 4368 |
+
if result["state"] == "refused":
|
| 4369 |
+
MIRROR_SEED["cause"] = "; ".join(result["problems"])
|
| 4370 |
+
MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
|
| 4371 |
+
finally:
|
| 4372 |
+
# A finished tenant restore is progress for the liveness check.
|
| 4373 |
+
getattr(_seed_role, "note_progress", lambda _step: None)("boot seed restore")
|
| 4374 |
|
| 4375 |
def runtime_failed(tenant_key, exc):
|
| 4376 |
MIRROR_SEED["cause"] = "; ".join(refused(tenant_key, exc)["problems"])
|
|
|
|
| 4381 |
restore, name="odoo-mirror-restore", tenants=_runtime.known_tenants(),
|
| 4382 |
get_runtime=tenant_scan.runtime_lookup(_runtime, "odoo-mirror-restore",
|
| 4383 |
on_error=runtime_failed), kind="light")
|
| 4384 |
+
try:
|
| 4385 |
+
deadline = float(_seed_os.environ.get("AIOS_BOOT_SEED_DEADLINE_S")
|
| 4386 |
+
or globals().get("_BOOT_SEED_DEADLINE_SECONDS", 1200))
|
| 4387 |
+
except ValueError:
|
| 4388 |
+
deadline = 1200.0
|
| 4389 |
+
deadline_at = _seed_time.monotonic() + deadline
|
| 4390 |
+
stuck = []
|
| 4391 |
+
for tenant_key, job in jobs.items():
|
| 4392 |
+
# Every join shares one deadline: a restore that never returns cannot hold the seed thread.
|
| 4393 |
+
job.join(max(0.0, deadline_at - _seed_time.monotonic()))
|
| 4394 |
+
if not getattr(job, "is_alive", lambda: False)():
|
| 4395 |
+
continue
|
| 4396 |
+
name = getattr(job, "name", None) or f"odoo-mirror-restore:{tenant_key}"
|
| 4397 |
+
frame = _seed_sys._current_frames().get(getattr(job, "ident", None))
|
| 4398 |
+
where = ", ".join(f"{_seed_os.path.basename(fs.filename)}:{fs.lineno} {fs.name}"
|
| 4399 |
+
for fs in (_seed_tb.extract_stack(frame)[-3:] if frame else ())) or "unknown"
|
| 4400 |
+
print(f"[aios-api] boot seed restore {name} passed its {deadline:g} s deadline at "
|
| 4401 |
+
f"{where}; the worker continues without it", flush=True)
|
| 4402 |
+
stuck.append(name)
|
| 4403 |
+
if stuck:
|
| 4404 |
+
# The legacy seed never races a restore that may still be installing.
|
| 4405 |
+
MIRROR_SEED["cause"] = f"the boot seed restore {', '.join(stuck)} passed its {deadline:g} s deadline"
|
| 4406 |
+
MIRROR_SEED["recommendation"] = "check the store and the mirror hand off, then restart"
|
| 4407 |
+
return False
|
| 4408 |
+
if _ds.ensure_seed(): # The legacy seed decides on the restored manifests.
|
| 4409 |
MIRROR_SEED["seeded"] = True
|
| 4410 |
print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
|
| 4411 |
return True
|
api/odoo_relational.py
CHANGED
|
@@ -6055,6 +6055,28 @@ def slot_pass_killed(now, slot, memory):
|
|
| 6055 |
return newest
|
| 6056 |
|
| 6057 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6058 |
def _slot_resumes(memory):
|
| 6059 |
"""How many boots resumed the slot named by `slotResumedFor`, 0 when unreadable."""
|
| 6060 |
try:
|
|
|
|
| 6055 |
return newest
|
| 6056 |
|
| 6057 |
|
| 6058 |
+
def slot_pass_rerun(now, slot, memory):
|
| 6059 |
+
"""The newest slot's epoch when its claimed pass ended without a good sync and no resume ran for it, else None.
|
| 6060 |
+
|
| 6061 |
+
The caller writes the resume pair for that slot, so a slot reruns at most once. Never raises.
|
| 6062 |
+
"""
|
| 6063 |
+
memory = memory if isinstance(memory, dict) else {}
|
| 6064 |
+
try:
|
| 6065 |
+
newest = slot_before(now, slot)
|
| 6066 |
+
if slot_pass_satisfied(now, slot, memory):
|
| 6067 |
+
return None
|
| 6068 |
+
claim, done = _good_epoch(memory.get("slotPassAt")), _good_epoch(memory.get("slotDoneAt"))
|
| 6069 |
+
if claim is None or claim < newest or done is None or done < claim:
|
| 6070 |
+
return None
|
| 6071 |
+
resumed = _good_epoch(memory.get("slotResumedFor"))
|
| 6072 |
+
# Any resume of this slot, a rerun or a killed-pass resume, spends its one rerun.
|
| 6073 |
+
if resumed is not None and resumed >= newest:
|
| 6074 |
+
return None
|
| 6075 |
+
return newest
|
| 6076 |
+
except Exception: # noqa: BLE001
|
| 6077 |
+
return None
|
| 6078 |
+
|
| 6079 |
+
|
| 6080 |
def _slot_resumes(memory):
|
| 6081 |
"""How many boots resumed the slot named by `slotResumedFor`, 0 when unreadable."""
|
| 6082 |
try:
|
api/process_role.py
CHANGED
|
@@ -95,6 +95,9 @@ LAST_INSTALL_ERROR = {}
|
|
| 95 |
# The web's last read of the store: the worker heartbeat and each tenant's published mirror ids.
|
| 96 |
_STORED = {"beat": None, "mirrors": {}}
|
| 97 |
_RESYNC_APPLIED = {"at": ""}
|
|
|
|
|
|
|
|
|
|
| 98 |
|
| 99 |
|
| 100 |
def role():
|
|
@@ -1031,6 +1034,21 @@ def _running_main():
|
|
| 1031 |
_CATCH_UP_SENT = {}
|
| 1032 |
|
| 1033 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1034 |
def write_heartbeat(now=None, get_runtime=None):
|
| 1035 |
"""Worker side: write the heartbeat, each tenant's catch-up and RESYNC_STATE to the C-status store.
|
| 1036 |
|
|
@@ -1041,7 +1059,8 @@ def write_heartbeat(now=None, get_runtime=None):
|
|
| 1041 |
return False
|
| 1042 |
get_runtime = get_runtime or _status_runtime
|
| 1043 |
beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
|
| 1044 |
-
"at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE)
|
|
|
|
| 1045 |
rt = get_runtime(writer_tenant())
|
| 1046 |
import worker_provenance
|
| 1047 |
attestation = worker_provenance.heartbeat_attestation(rt, beat)
|
|
@@ -1115,6 +1134,7 @@ def start_worker_services():
|
|
| 1115 |
from harness import mirror_store
|
| 1116 |
mirror_store.register_publish_listener(record_publication)
|
| 1117 |
parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
|
|
|
|
| 1118 |
start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
|
| 1119 |
# Before the first heavy job: each one now waits until this worker holds the writer lease.
|
| 1120 |
arm_writer_lease()
|
|
|
|
| 95 |
# The web's last read of the store: the worker heartbeat and each tenant's published mirror ids.
|
| 96 |
_STORED = {"beat": None, "mirrors": {}}
|
| 97 |
_RESYNC_APPLIED = {"at": ""}
|
| 98 |
+
# The worker's last pass step: the web judges liveness by its age, never by HF stage alone.
|
| 99 |
+
_PROGRESS = {"last": None}
|
| 100 |
+
_PROGRESS_LOCK = threading.Lock()
|
| 101 |
|
| 102 |
|
| 103 |
def role():
|
|
|
|
| 1034 |
_CATCH_UP_SENT = {}
|
| 1035 |
|
| 1036 |
|
| 1037 |
+
def note_progress(step):
|
| 1038 |
+
"""Worker side: record that a pass step ran. Memory only: it never raises and never does I/O."""
|
| 1039 |
+
try:
|
| 1040 |
+
with _PROGRESS_LOCK:
|
| 1041 |
+
_PROGRESS["last"] = {"step": str(step)[:80], "at": time.time()}
|
| 1042 |
+
except Exception: # noqa: BLE001
|
| 1043 |
+
pass
|
| 1044 |
+
|
| 1045 |
+
|
| 1046 |
+
def _progress_copy():
|
| 1047 |
+
with _PROGRESS_LOCK:
|
| 1048 |
+
last = _PROGRESS["last"]
|
| 1049 |
+
return dict(last) if last else None
|
| 1050 |
+
|
| 1051 |
+
|
| 1052 |
def write_heartbeat(now=None, get_runtime=None):
|
| 1053 |
"""Worker side: write the heartbeat, each tenant's catch-up and RESYNC_STATE to the C-status store.
|
| 1054 |
|
|
|
|
| 1059 |
return False
|
| 1060 |
get_runtime = get_runtime or _status_runtime
|
| 1061 |
beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
|
| 1062 |
+
"at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE),
|
| 1063 |
+
"progress": _progress_copy()}
|
| 1064 |
rt = get_runtime(writer_tenant())
|
| 1065 |
import worker_provenance
|
| 1066 |
attestation = worker_provenance.heartbeat_attestation(rt, beat)
|
|
|
|
| 1134 |
from harness import mirror_store
|
| 1135 |
mirror_store.register_publish_listener(record_publication)
|
| 1136 |
parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
|
| 1137 |
+
note_progress("boot")
|
| 1138 |
start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
|
| 1139 |
# Before the first heavy job: each one now waits until this worker holds the writer lease.
|
| 1140 |
arm_writer_lease()
|
api/sync_runtime.py
CHANGED
|
@@ -54,6 +54,8 @@ WEB_SECTION = "web"
|
|
| 54 |
ALIVE_STAGES = ("RUNNING", "SCHEDULING")
|
| 55 |
#: A job record older than its timeout plus this margin is dead, whatever inspect_job answers.
|
| 56 |
JOB_MARGIN_SECONDS = 600
|
|
|
|
|
|
|
| 57 |
#: A --once wait at most this long runs as a plain wait. A longer wait first checks the desk.
|
| 58 |
IDLE_GRACE_SECONDS = 30.0
|
| 59 |
#: The longest sleep a --once job stores as its next pass. A day keeps a lost slot from stalling.
|
|
@@ -421,6 +423,28 @@ def job_alive(worker, api, now, namespace=None):
|
|
| 421 |
return stage in ALIVE_STAGES, stage.lower() or "unknown"
|
| 422 |
|
| 423 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 424 |
def pass_due(worker, rt, now):
|
| 425 |
"""(due, why): a due desk request, or a stored nextPassAt at or before now, or none stored."""
|
| 426 |
try:
|
|
@@ -507,6 +531,19 @@ def tick(wanted=None, *, get_runtime=None, api=None, now=None):
|
|
| 507 |
{"version": version, "at": stamp(stamp_now)}):
|
| 508 |
_WEB_WRITTEN[slug] = version
|
| 509 |
live, why = job_alive(worker, api, stamp_now, namespace)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 510 |
record = process_role._FLEET.setdefault(slug, {"pid": None, "jobs": 0, "job": None})
|
| 511 |
record.update(alive=live, aliveWhy=why)
|
| 512 |
if live:
|
|
|
|
| 54 |
ALIVE_STAGES = ("RUNNING", "SCHEDULING")
|
| 55 |
#: A job record older than its timeout plus this margin is dead, whatever inspect_job answers.
|
| 56 |
JOB_MARGIN_SECONDS = 600
|
| 57 |
+
#: A live resident job with no pass step for this long is hung: the web cancels and replaces it.
|
| 58 |
+
PROGRESS_STALE_SECONDS = float(os.environ.get("AIOS_SYNC_PROGRESS_STALE_SECONDS") or 1800)
|
| 59 |
#: A --once wait at most this long runs as a plain wait. A longer wait first checks the desk.
|
| 60 |
IDLE_GRACE_SECONDS = 30.0
|
| 61 |
#: The longest sleep a --once job stores as its next pass. A day keeps a lost slot from stalling.
|
|
|
|
| 423 |
return stage in ALIVE_STAGES, stage.lower() or "unknown"
|
| 424 |
|
| 425 |
|
| 426 |
+
def job_hung(worker, now, why):
|
| 427 |
+
"""(hung, age, step): a job alive by beat or RUNNING stage whose last pass step is older than the budget."""
|
| 428 |
+
if why not in ("heartbeat", "running"):
|
| 429 |
+
# SCHEDULING, an unread stage and an unknown one are never judged hung.
|
| 430 |
+
return False, None, None
|
| 431 |
+
beat = worker.get("heartbeat") if isinstance(worker.get("heartbeat"), dict) else {}
|
| 432 |
+
progress = beat.get("progress") if isinstance(beat.get("progress"), dict) else None
|
| 433 |
+
job = worker.get(JOB_SECTION) if isinstance(worker.get(JOB_SECTION), dict) else {}
|
| 434 |
+
submitted = epoch(job.get("submittedAt"))
|
| 435 |
+
try:
|
| 436 |
+
stepped = float(progress["at"]) if progress else None
|
| 437 |
+
except (KeyError, TypeError, ValueError):
|
| 438 |
+
stepped = None
|
| 439 |
+
# Progress never predates the job's own submission, so an older job's beat cannot condemn its successor.
|
| 440 |
+
marks = [mark for mark in (stepped, submitted) if mark is not None]
|
| 441 |
+
if not job.get("id") or not marks:
|
| 442 |
+
return False, None, None
|
| 443 |
+
age = now - max(marks)
|
| 444 |
+
step = str(progress.get("step") or "unknown") if progress else "none reported"
|
| 445 |
+
return age > PROGRESS_STALE_SECONDS, int(round(age)), step
|
| 446 |
+
|
| 447 |
+
|
| 448 |
def pass_due(worker, rt, now):
|
| 449 |
"""(due, why): a due desk request, or a stored nextPassAt at or before now, or none stored."""
|
| 450 |
try:
|
|
|
|
| 531 |
{"version": version, "at": stamp(stamp_now)}):
|
| 532 |
_WEB_WRITTEN[slug] = version
|
| 533 |
live, why = job_alive(worker, api, stamp_now, namespace)
|
| 534 |
+
hung, age, step = job_hung(worker, stamp_now, why) if live and mode == "resident" else (False, None, None)
|
| 535 |
+
if hung and api is not None:
|
| 536 |
+
job_id = worker[JOB_SECTION]["id"]
|
| 537 |
+
try:
|
| 538 |
+
api.cancel_job(job_id=job_id, namespace=namespace)
|
| 539 |
+
except Exception as exc: # noqa: BLE001
|
| 540 |
+
# A failed cancel keeps the slot taken: no replacement until a cancel succeeds.
|
| 541 |
+
print(f"[sync-runtime] cancel of hung sync job {job_id} failed ({type(exc).__name__}); "
|
| 542 |
+
f"it counts as alive this tick", flush=True)
|
| 543 |
+
else:
|
| 544 |
+
print(f"[sync-runtime] cancelled hung sync job {job_id}: no progress for {age} s "
|
| 545 |
+
f"(last step {step})", flush=True)
|
| 546 |
+
live, why = False, "cancelled hung"
|
| 547 |
record = process_role._FLEET.setdefault(slug, {"pid": None, "jobs": 0, "job": None})
|
| 548 |
record.update(alive=live, aliveWhy=why)
|
| 549 |
if live:
|
api/test_odoo_rank_refresh.py
CHANGED
|
@@ -1180,14 +1180,15 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1180 |
self.assertNotIn(('amazon', boot, 'boot', False), events)
|
| 1181 |
|
| 1182 |
def test_h73_a_restart_inside_a_slot_pass_does_not_run_it_twice(self):
|
| 1183 |
-
"""H2:
|
| 1184 |
import odoo_relational as rel
|
| 1185 |
stamp = self.utc('2026-10-24T06:00:01Z')
|
| 1186 |
yesterday = self.utc('2026-10-23T06:30:00Z')
|
| 1187 |
base = {'passAt': yesterday, 'fullRebuildAt': yesterday, 'slotPassAt': stamp,
|
| 1188 |
'fingerprints': {'res_partner': (3, 7)}}
|
| 1189 |
-
# Ended: `slotDoneAt` after the claim, or (older memory) `passAt` after it. No Odoo call.
|
| 1190 |
-
for ended in ({'slotDoneAt': self.utc('2026-10-24T06:30:00Z')
|
|
|
|
| 1191 |
{'passAt': self.utc('2026-10-24T06:30:00Z')}):
|
| 1192 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1193 |
dict(base, **ended), boot=self.utc('2026-10-24T06:40:00Z'), max_sleeps=1)
|
|
@@ -1661,5 +1662,27 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1661 |
0 <= firsts['fsanyoto/loopable'] - self.utc('2026-10-24T07:00:00Z') <= 1, firsts)
|
| 1662 |
|
| 1663 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1664 |
if __name__ == '__main__':
|
| 1665 |
unittest.main()
|
|
|
|
| 1180 |
self.assertNotIn(('amazon', boot, 'boot', False), events)
|
| 1181 |
|
| 1182 |
def test_h73_a_restart_inside_a_slot_pass_does_not_run_it_twice(self):
|
| 1183 |
+
"""H2: an ended slot pass whose one rerun is spent is not re-run; one killed before its end runs once, never twice"""
|
| 1184 |
import odoo_relational as rel
|
| 1185 |
stamp = self.utc('2026-10-24T06:00:01Z')
|
| 1186 |
yesterday = self.utc('2026-10-23T06:30:00Z')
|
| 1187 |
base = {'passAt': yesterday, 'fullRebuildAt': yesterday, 'slotPassAt': stamp,
|
| 1188 |
'fingerprints': {'res_partner': (3, 7)}}
|
| 1189 |
+
# Ended: `slotDoneAt` after the claim with its rerun spent, or (older memory) `passAt` after it. No Odoo call.
|
| 1190 |
+
for ended in ({'slotDoneAt': self.utc('2026-10-24T06:30:00Z'),
|
| 1191 |
+
'slotResumedFor': self.utc('2026-10-24T06:00:00Z'), 'slotResumes': 1},
|
| 1192 |
{'passAt': self.utc('2026-10-24T06:30:00Z')}):
|
| 1193 |
events, sleeps, odoo = self.run_slot_loop(
|
| 1194 |
dict(base, **ended), boot=self.utc('2026-10-24T06:40:00Z'), max_sleeps=1)
|
|
|
|
| 1662 |
0 <= firsts['fsanyoto/loopable'] - self.utc('2026-10-24T07:00:00Z') <= 1, firsts)
|
| 1663 |
|
| 1664 |
|
| 1665 |
+
def test_t2_an_ended_slot_pass_without_a_good_sync_reruns_once_the_same_day(self):
|
| 1666 |
+
"""T2: ended without a good sync reruns once; a spent rerun or a good pass does not"""
|
| 1667 |
+
import odoo_relational as rel
|
| 1668 |
+
slot = rel.sync_slot(rel.read_config(self.SlotStore('royal-imports')))
|
| 1669 |
+
slot_at, boot = self.utc('2026-10-24T06:00:00Z'), self.utc('2026-10-24T06:40:00Z')
|
| 1670 |
+
yesterday = self.utc('2026-10-23T06:30:00Z')
|
| 1671 |
+
ended = {'passAt': yesterday, 'fullRebuildAt': yesterday, 'slotPassAt': slot_at + 1,
|
| 1672 |
+
'slotDoneAt': self.utc('2026-10-24T06:30:00Z'), 'fingerprints': {'res_partner': (3, 7)}}
|
| 1673 |
+
self.assertEqual(rel.slot_pass_rerun(boot, slot, ended), slot_at)
|
| 1674 |
+
self.assertIsNone(rel.slot_pass_rerun(boot, slot, dict(ended, slotResumedFor=slot_at, slotResumes=1)))
|
| 1675 |
+
self.assertIsNone(rel.slot_pass_rerun(boot, slot, dict(ended, lastGoodSyncAt=slot_at + 600)))
|
| 1676 |
+
self.assertIsNone(rel.slot_pass_rerun(boot, slot, dict(ended, slotDoneAt=yesterday)))
|
| 1677 |
+
self.assertIsNone(rel.slot_pass_rerun(boot, slot, None))
|
| 1678 |
+
# The loop reruns it at boot, outside the slot window, and spends the resume pair.
|
| 1679 |
+
events, sleeps, odoo = self.run_slot_loop(ended, boot=boot, max_sleeps=1)
|
| 1680 |
+
self.assertEqual([e[1] for e in odoo if e[0] == 'mirror'], [boot])
|
| 1681 |
+
self.assertIn('the 13:00 Asia/Jakarta slot pass ended without a good sync; rerunning it once today',
|
| 1682 |
+
self.loop_output)
|
| 1683 |
+
memory = rel.read_resync_memory(self.slot_store)
|
| 1684 |
+
self.assertEqual((memory['slotResumedFor'], memory['slotResumes']), (slot_at, 1))
|
| 1685 |
+
|
| 1686 |
+
|
| 1687 |
if __name__ == '__main__':
|
| 1688 |
unittest.main()
|
platform/core/received_changes.py
CHANGED
|
@@ -11,10 +11,13 @@ import os
|
|
| 11 |
from pathlib import Path
|
| 12 |
import tempfile
|
| 13 |
import threading
|
|
|
|
| 14 |
|
| 15 |
|
| 16 |
STORE_KEY = "received_changes_v1"
|
| 17 |
MAX_EVENTS = 2000
|
|
|
|
|
|
|
| 18 |
BASELINE_CHUNK_BYTES = 8 * 1024 * 1024
|
| 19 |
PROJECTION_VERSION = 1
|
| 20 |
_LISTENERS = []
|
|
@@ -215,7 +218,17 @@ def _baseline_lock(root):
|
|
| 215 |
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
|
| 216 |
else:
|
| 217 |
import fcntl
|
| 218 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 219 |
try:
|
| 220 |
yield
|
| 221 |
finally:
|
|
@@ -393,15 +406,26 @@ def _move_baseline(rt, identity, expected, updated):
|
|
| 393 |
raise BaselineUnavailable("The observation reference was not saved.")
|
| 394 |
|
| 395 |
|
| 396 |
-
def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot):
|
| 397 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 398 |
try:
|
| 399 |
-
cursor.execute("SELECT coalesce(b.row_id, a.row_id), b.image, a.image "
|
| 400 |
-
"
|
| 401 |
-
"USING (row_id) WHERE b.image IS DISTINCT FROM a.image "
|
| 402 |
-
"ORDER BY coalesce(b.row_id, a.row_id)", [str(baseline), str(current)])
|
| 403 |
while batch := cursor.fetchmany(256):
|
| 404 |
for row_id, before_json, after_json in batch:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 405 |
before, after = json.loads(before_json or "{}"), json.loads(after_json or "{}")
|
| 406 |
changes = {}
|
| 407 |
for key, (_sql, formatter) in spec["cols"].items():
|
|
@@ -414,11 +438,12 @@ def _compare_baseline(rt, source, database, spec, con, baseline, current, snapsh
|
|
| 414 |
operation = "created" if before_json is None else "deleted" if after_json is None else "updated"
|
| 415 |
publish(rt, source=source, database=database, row_id=row_id, changes=changes,
|
| 416 |
receipt_id=snapshot, operation=operation)
|
|
|
|
| 417 |
finally:
|
| 418 |
cursor.close()
|
| 419 |
|
| 420 |
|
| 421 |
-
def _snapshot(rt, source, database, spec, con, root, manifest):
|
| 422 |
"""Compare verified durable observations in batches, recovering interrupted comparisons first."""
|
| 423 |
if not spec.get("cols"):
|
| 424 |
return
|
|
@@ -443,7 +468,8 @@ def _snapshot(rt, source, database, spec, con, root, manifest):
|
|
| 443 |
observed = _restore_baseline(rt, identity, binding, pending["pending"], current, con)
|
| 444 |
if pending["current"]:
|
| 445 |
_restore_baseline(rt, identity, binding, pending["current"], baseline, con)
|
| 446 |
-
_compare_baseline(rt, source, database, spec, con, baseline, current, observed["snapshot"]
|
|
|
|
| 447 |
committed = {"current": pending["pending"], "pending": None}
|
| 448 |
_move_baseline(rt, identity, pending, committed)
|
| 449 |
return committed
|
|
@@ -486,13 +512,20 @@ def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
|
|
| 486 |
raise ValueError("Received-change runtime belongs to another tenant.")
|
| 487 |
root = Path(rt.datastore_path).with_suffix(".received-changes")
|
| 488 |
with _baseline_lock(root):
|
|
|
|
|
|
|
|
|
|
| 489 |
con = datastore.apply_limits(datastore.duckdb.connect(":memory:"))
|
| 490 |
try:
|
| 491 |
for name, files in paths.items():
|
| 492 |
con.read_parquet([str(path) for path in files], union_by_name=True).create_view(str(name))
|
| 493 |
for database, spec in resolver(rt, con).items():
|
|
|
|
|
|
|
|
|
|
|
|
|
| 494 |
try:
|
| 495 |
-
_snapshot(rt, connector, str(database), spec, con, root, manifest)
|
| 496 |
except Exception as exc:
|
| 497 |
_LOG.warning("Received-change projection deferred for %s: %s",
|
| 498 |
database, type(exc).__name__)
|
|
@@ -700,6 +733,83 @@ def selftest():
|
|
| 700 |
check("cross-tenant publication refused", True)
|
| 701 |
else:
|
| 702 |
check("cross-tenant publication refused", False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 703 |
finally:
|
| 704 |
con.close()
|
| 705 |
finally:
|
|
|
|
| 11 |
from pathlib import Path
|
| 12 |
import tempfile
|
| 13 |
import threading
|
| 14 |
+
import time
|
| 15 |
|
| 16 |
|
| 17 |
STORE_KEY = "received_changes_v1"
|
| 18 |
MAX_EVENTS = 2000
|
| 19 |
+
JOURNAL_DEADLINE_SECONDS = 120
|
| 20 |
+
BASELINE_LOCK_SECONDS = 300
|
| 21 |
BASELINE_CHUNK_BYTES = 8 * 1024 * 1024
|
| 22 |
PROJECTION_VERSION = 1
|
| 23 |
_LISTENERS = []
|
|
|
|
| 218 |
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
|
| 219 |
else:
|
| 220 |
import fcntl
|
| 221 |
+
# A held lock fails the hand-off after a bounded wait instead of hanging the worker.
|
| 222 |
+
wait = float(os.environ.get("AIOS_RECEIVED_CHANGE_LOCK_S") or BASELINE_LOCK_SECONDS)
|
| 223 |
+
end = time.monotonic() + wait
|
| 224 |
+
while True:
|
| 225 |
+
try:
|
| 226 |
+
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
| 227 |
+
break
|
| 228 |
+
except BlockingIOError:
|
| 229 |
+
if time.monotonic() >= end:
|
| 230 |
+
raise TimeoutError(f"The received-change baseline lock stayed busy for {wait:g} s.")
|
| 231 |
+
time.sleep(0.05)
|
| 232 |
try:
|
| 233 |
yield
|
| 234 |
finally:
|
|
|
|
| 406 |
raise BaselineUnavailable("The observation reference was not saved.")
|
| 407 |
|
| 408 |
|
| 409 |
+
def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
|
| 410 |
+
diff = ("FROM read_parquet(?) b FULL OUTER JOIN read_parquet(?) a "
|
| 411 |
+
"USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
|
| 412 |
+
count = con.execute("SELECT count(*) " + diff, [str(baseline), str(current)]).fetchone()[0]
|
| 413 |
+
# A diff wider than the retained window cannot be replayed, so it is recorded as one bulk change.
|
| 414 |
+
if count > MAX_EVENTS:
|
| 415 |
+
print(f"[received-changes] {database}: {count} changed rows exceed the journal's "
|
| 416 |
+
f"{MAX_EVENTS}-event window; recorded as a bulk change, rows not journaled", flush=True)
|
| 417 |
+
return
|
| 418 |
+
cursor, done = con.cursor(), 0
|
| 419 |
try:
|
| 420 |
+
cursor.execute("SELECT coalesce(b.row_id, a.row_id), b.image, a.image " + diff
|
| 421 |
+
+ " ORDER BY coalesce(b.row_id, a.row_id)", [str(baseline), str(current)])
|
|
|
|
|
|
|
| 422 |
while batch := cursor.fetchmany(256):
|
| 423 |
for row_id, before_json, after_json in batch:
|
| 424 |
+
# Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
|
| 425 |
+
if deadline is not None and time.monotonic() >= deadline[0]:
|
| 426 |
+
print(f"[received-changes] {database}: journal deadline {deadline[1]:g} s passed "
|
| 427 |
+
f"after {done} of {count} rows; the rest are not journaled", flush=True)
|
| 428 |
+
return
|
| 429 |
before, after = json.loads(before_json or "{}"), json.loads(after_json or "{}")
|
| 430 |
changes = {}
|
| 431 |
for key, (_sql, formatter) in spec["cols"].items():
|
|
|
|
| 438 |
operation = "created" if before_json is None else "deleted" if after_json is None else "updated"
|
| 439 |
publish(rt, source=source, database=database, row_id=row_id, changes=changes,
|
| 440 |
receipt_id=snapshot, operation=operation)
|
| 441 |
+
done += 1
|
| 442 |
finally:
|
| 443 |
cursor.close()
|
| 444 |
|
| 445 |
|
| 446 |
+
def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
|
| 447 |
"""Compare verified durable observations in batches, recovering interrupted comparisons first."""
|
| 448 |
if not spec.get("cols"):
|
| 449 |
return
|
|
|
|
| 468 |
observed = _restore_baseline(rt, identity, binding, pending["pending"], current, con)
|
| 469 |
if pending["current"]:
|
| 470 |
_restore_baseline(rt, identity, binding, pending["current"], baseline, con)
|
| 471 |
+
_compare_baseline(rt, source, database, spec, con, baseline, current, observed["snapshot"],
|
| 472 |
+
deadline)
|
| 473 |
committed = {"current": pending["pending"], "pending": None}
|
| 474 |
_move_baseline(rt, identity, pending, committed)
|
| 475 |
return committed
|
|
|
|
| 512 |
raise ValueError("Received-change runtime belongs to another tenant.")
|
| 513 |
root = Path(rt.datastore_path).with_suffix(".received-changes")
|
| 514 |
with _baseline_lock(root):
|
| 515 |
+
# One journal budget per publication, shared by its databases.
|
| 516 |
+
seconds = float(os.environ.get("AIOS_RECEIVED_CHANGE_DEADLINE_S") or JOURNAL_DEADLINE_SECONDS)
|
| 517 |
+
deadline = (time.monotonic() + seconds, seconds)
|
| 518 |
con = datastore.apply_limits(datastore.duckdb.connect(":memory:"))
|
| 519 |
try:
|
| 520 |
for name, files in paths.items():
|
| 521 |
con.read_parquet([str(path) for path in files], union_by_name=True).create_view(str(name))
|
| 522 |
for database, spec in resolver(rt, con).items():
|
| 523 |
+
# A skipped database keeps its stored baseline, so the next publication compares again.
|
| 524 |
+
if time.monotonic() >= deadline[0]:
|
| 525 |
+
print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
|
| 526 |
+
continue
|
| 527 |
try:
|
| 528 |
+
_snapshot(rt, connector, str(database), spec, con, root, manifest, deadline)
|
| 529 |
except Exception as exc:
|
| 530 |
_LOG.warning("Received-change projection deferred for %s: %s",
|
| 531 |
database, type(exc).__name__)
|
|
|
|
| 733 |
check("cross-tenant publication refused", True)
|
| 734 |
else:
|
| 735 |
check("cross-tenant publication refused", False)
|
| 736 |
+
|
| 737 |
+
import contextlib
|
| 738 |
+
import io
|
| 739 |
+
bound = FakeRuntime("fixture-bound", directory)
|
| 740 |
+
bulk_path = Path(directory) / "bulk.parquet"
|
| 741 |
+
con.execute("CREATE TABLE bulk_rows (native_id BIGINT, status VARCHAR)")
|
| 742 |
+
con.execute("INSERT INTO bulk_rows SELECT range, 'open' FROM range(1, 9)")
|
| 743 |
+
bulk_spec = {"table": "bulk_rows", "id": "native_id", "cols": {"status": ("status", str)}}
|
| 744 |
+
|
| 745 |
+
def bound_snapshot(databases, identity, second):
|
| 746 |
+
register_projection("odoo", lambda _rt, _cur: {name: bulk_spec for name in databases})
|
| 747 |
+
con.execute("COPY bulk_rows TO ? (FORMAT PARQUET)", [str(bulk_path)])
|
| 748 |
+
manifest = {"tenant": bound.key, "snapshot_id": identity,
|
| 749 |
+
"completed_at": f"2026-11-17T00:00:{second:02d}Z"}
|
| 750 |
+
output = io.StringIO()
|
| 751 |
+
with patch.object(runtime, "get_runtime", return_value=bound), \
|
| 752 |
+
contextlib.redirect_stdout(output):
|
| 753 |
+
mirror_hook("odoo", bound.key, manifest, [], {"bulk_rows": [bulk_path]})
|
| 754 |
+
return output.getvalue()
|
| 755 |
+
|
| 756 |
+
def bound_state(database):
|
| 757 |
+
identity = hashlib.sha256(json.dumps([bound.key, "odoo", database]).encode()).hexdigest()
|
| 758 |
+
state = _observation_state(bound, identity)[0]
|
| 759 |
+
reference = state["current"]
|
| 760 |
+
manifest = json.loads(_read_blob(bound, _blob_path(identity, "manifests", reference["sha256"]),
|
| 761 |
+
reference["sha256"], reference["bytes"]))
|
| 762 |
+
return state, manifest["snapshot"]
|
| 763 |
+
|
| 764 |
+
bound_snapshot(("ut_bulk", "ut_bulk_twin"), "bound-initial", 10)
|
| 765 |
+
twin_before = bound_state("ut_bulk_twin")
|
| 766 |
+
con.execute("UPDATE bulk_rows SET status='done'")
|
| 767 |
+
calls = []
|
| 768 |
+
|
| 769 |
+
def slow_publish(*_args, **_kwargs):
|
| 770 |
+
calls.append(1)
|
| 771 |
+
time.sleep(0.2)
|
| 772 |
+
|
| 773 |
+
started = time.monotonic()
|
| 774 |
+
with patch(__name__ + ".publish", side_effect=slow_publish), \
|
| 775 |
+
patch.dict(os.environ, {"AIOS_RECEIVED_CHANGE_DEADLINE_S": "0.5"}):
|
| 776 |
+
printed = bound_snapshot(("ut_bulk", "ut_bulk_twin"), "bound-stall", 11)
|
| 777 |
+
state, committed = bound_state("ut_bulk")
|
| 778 |
+
check("stalled journal stops at its deadline", time.monotonic() - started < 1.2
|
| 779 |
+
and 0 < len(calls) < 8 and "ut_bulk: journal deadline 0.5 s passed after "
|
| 780 |
+
+ f"{len(calls)} of 8 rows; the rest are not journaled" in printed)
|
| 781 |
+
check("stalled journal commits its baseline", committed == "bound-stall"
|
| 782 |
+
and state["pending"] is None)
|
| 783 |
+
check("database after the deadline keeps its baseline",
|
| 784 |
+
"ut_bulk_twin: skipped, journal deadline passed" in printed
|
| 785 |
+
and bound_state("ut_bulk_twin") == twin_before)
|
| 786 |
+
con.execute("UPDATE bulk_rows SET status='bulk' WHERE native_id <= 3")
|
| 787 |
+
calls.clear()
|
| 788 |
+
with patch(__name__ + ".publish", side_effect=slow_publish), \
|
| 789 |
+
patch(__name__ + ".MAX_EVENTS", 1):
|
| 790 |
+
printed = bound_snapshot(("ut_bulk",), "bound-bulk", 12)
|
| 791 |
+
state, committed = bound_state("ut_bulk")
|
| 792 |
+
check("bulk change journals no rows", not calls and "ut_bulk: 3 changed rows exceed "
|
| 793 |
+
"the journal's 1-event window; recorded as a bulk change, rows not journaled" in printed)
|
| 794 |
+
check("bulk change commits its baseline", committed == "bound-bulk"
|
| 795 |
+
and state["pending"] is None)
|
| 796 |
+
if os.name != "nt":
|
| 797 |
+
import fcntl
|
| 798 |
+
lock_root = Path(directory) / "lock-fixture"
|
| 799 |
+
lock_root.mkdir()
|
| 800 |
+
with (lock_root / "lock").open("a+b") as holder:
|
| 801 |
+
fcntl.flock(holder, fcntl.LOCK_EX)
|
| 802 |
+
started = time.monotonic()
|
| 803 |
+
try:
|
| 804 |
+
with patch.dict(os.environ, {"AIOS_RECEIVED_CHANGE_LOCK_S": "0.2"}), \
|
| 805 |
+
_baseline_lock(lock_root):
|
| 806 |
+
pass
|
| 807 |
+
except TimeoutError as exc:
|
| 808 |
+
check("held baseline lock times out", time.monotonic() - started < 1.0
|
| 809 |
+
and "0.2 s" in str(exc))
|
| 810 |
+
else:
|
| 811 |
+
check("held baseline lock times out", False)
|
| 812 |
+
fcntl.flock(holder, fcntl.LOCK_UN)
|
| 813 |
finally:
|
| 814 |
con.close()
|
| 815 |
finally:
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
afc4252504d8e071ec0937b2e6a7ad64c1e00860885ad0596112eaca89d79eea
|