Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- Dockerfile +1 -0
- RELEASES.json +1 -1
- VERSION +1 -1
- api/process_role.py +34 -3
- api/sheet_warm.py +23 -3
- api/sync_worker.py +2 -0
- api/worker_provenance.py +428 -0
- platform/harness/datastore.py +8 -0
- platform/harness/mirror_store.py +7 -3
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
Dockerfile
CHANGED
|
@@ -61,6 +61,7 @@ COPY requirements.txt ./
|
|
| 61 |
RUN python -m pip install --no-cache-dir --disable-pip-version-check --upgrade "pip==25.0.1" \
|
| 62 |
&& python -m pip install --no-cache-dir --disable-pip-version-check -r requirements.txt
|
| 63 |
COPY api/ ./api/
|
|
|
|
| 64 |
# the reused data layer (modules/core/harness)
|
| 65 |
# ⛔ COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction:
|
| 66 |
# it parsed `# the reused data layer ...` as extra SOURCE PATHS and the first build ever run
|
|
|
|
| 61 |
RUN python -m pip install --no-cache-dir --disable-pip-version-check --upgrade "pip==25.0.1" \
|
| 62 |
&& python -m pip install --no-cache-dir --disable-pip-version-check -r requirements.txt
|
| 63 |
COPY api/ ./api/
|
| 64 |
+
COPY VERSION ./VERSION
|
| 65 |
# the reused data layer (modules/core/harness)
|
| 66 |
# ⛔ COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction:
|
| 67 |
# it parsed `# the reused data layer ...` as extra SOURCE PATHS and the first build ever run
|
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "b9a552ad82b",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
b9a552ad82b
|
api/process_role.py
CHANGED
|
@@ -883,11 +883,17 @@ def stored_worker(rt):
|
|
| 883 |
return connector_sync.status(rt, WORKER_STATUS, fresh=True) or {}
|
| 884 |
|
| 885 |
|
|
|
|
|
|
|
|
|
|
| 886 |
def stored_heartbeat(tenant=None):
|
| 887 |
"""The worker heartbeat in the store, or None. A store that does not answer reads as None."""
|
| 888 |
try:
|
| 889 |
-
|
|
|
|
|
|
|
| 890 |
except Exception: # noqa: BLE001
|
|
|
|
| 891 |
return None
|
| 892 |
return beat if isinstance(beat, dict) else None
|
| 893 |
|
|
@@ -1005,7 +1011,27 @@ def write_heartbeat(now=None, get_runtime=None):
|
|
| 1005 |
get_runtime = get_runtime or _status_runtime
|
| 1006 |
beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
|
| 1007 |
"at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE)}
|
| 1008 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1009 |
try:
|
| 1010 |
from harness import mirror_store
|
| 1011 |
catch_up = mirror_store.catch_up_snapshot()
|
|
@@ -1163,7 +1189,11 @@ def status():
|
|
| 1163 |
|
| 1164 |
A serving role reads the worker from its stored heartbeat, never from a child process (D-11183).
|
| 1165 |
"""
|
|
|
|
| 1166 |
beat = stored_heartbeat() if ROLE != "all" else None
|
|
|
|
|
|
|
|
|
|
| 1167 |
age = beat_age(beat) if beat is not None else None
|
| 1168 |
if ROLE == "worker":
|
| 1169 |
pid, alive = os.getpid(), True
|
|
@@ -1184,5 +1214,6 @@ def status():
|
|
| 1184 |
"tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
|
| 1185 |
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
| 1186 |
"restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
|
| 1187 |
-
"lastInstallError": dict(LAST_INSTALL_ERROR) or None
|
|
|
|
| 1188 |
"writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
|
|
|
|
| 883 |
return connector_sync.status(rt, WORKER_STATUS, fresh=True) or {}
|
| 884 |
|
| 885 |
|
| 886 |
+
_HEARTBEAT_READ = threading.local()
|
| 887 |
+
|
| 888 |
+
|
| 889 |
def stored_heartbeat(tenant=None):
|
| 890 |
"""The worker heartbeat in the store, or None. A store that does not answer reads as None."""
|
| 891 |
try:
|
| 892 |
+
worker = stored_worker(_status_runtime(tenant))
|
| 893 |
+
_HEARTBEAT_READ.worker = worker
|
| 894 |
+
beat = worker.get("heartbeat")
|
| 895 |
except Exception: # noqa: BLE001
|
| 896 |
+
_HEARTBEAT_READ.worker = {}
|
| 897 |
return None
|
| 898 |
return beat if isinstance(beat, dict) else None
|
| 899 |
|
|
|
|
| 1011 |
get_runtime = get_runtime or _status_runtime
|
| 1012 |
beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
|
| 1013 |
"at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE)}
|
| 1014 |
+
rt = get_runtime(writer_tenant())
|
| 1015 |
+
import worker_provenance
|
| 1016 |
+
attestation = worker_provenance.heartbeat_attestation(rt, beat)
|
| 1017 |
+
|
| 1018 |
+
def change(entry):
|
| 1019 |
+
job = entry.get("job") or {}
|
| 1020 |
+
actual_job = os.environ.get("JOB_ID")
|
| 1021 |
+
if job.get("id") and (actual_job != job["id"] or job.get("tenant") not in (None, writer_tenant())):
|
| 1022 |
+
raise ValueError("stale_worker_heartbeat")
|
| 1023 |
+
entry["heartbeat"] = {**(entry.get("heartbeat") or {}), **beat}
|
| 1024 |
+
# Hash the final stored heartbeat, including fields from an earlier schema.
|
| 1025 |
+
entry["attestation"] = None
|
| 1026 |
+
if attestation and actual_job == job.get("id") and job.get("tenant") == writer_tenant():
|
| 1027 |
+
try:
|
| 1028 |
+
worker_provenance.bounded(entry["heartbeat"])
|
| 1029 |
+
entry["attestation"] = worker_provenance.bounded({**attestation, "heartbeatSha256": worker_provenance.sha(
|
| 1030 |
+
worker_provenance.encoded(entry["heartbeat"]))})
|
| 1031 |
+
except Exception:
|
| 1032 |
+
pass
|
| 1033 |
+
|
| 1034 |
+
written = connector_sync._status_update(rt, WORKER_STATUS, None, change)
|
| 1035 |
try:
|
| 1036 |
from harness import mirror_store
|
| 1037 |
catch_up = mirror_store.catch_up_snapshot()
|
|
|
|
| 1189 |
|
| 1190 |
A serving role reads the worker from its stored heartbeat, never from a child process (D-11183).
|
| 1191 |
"""
|
| 1192 |
+
_HEARTBEAT_READ.worker = {}
|
| 1193 |
beat = stored_heartbeat() if ROLE != "all" else None
|
| 1194 |
+
stored = _HEARTBEAT_READ.worker
|
| 1195 |
+
beat = beat if isinstance(beat, dict) else None
|
| 1196 |
+
import worker_provenance
|
| 1197 |
age = beat_age(beat) if beat is not None else None
|
| 1198 |
if ROLE == "worker":
|
| 1199 |
pid, alive = os.getpid(), True
|
|
|
|
| 1214 |
"tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
|
| 1215 |
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
| 1216 |
"restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
|
| 1217 |
+
"lastInstallError": dict(LAST_INSTALL_ERROR) or None,
|
| 1218 |
+
"attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None},
|
| 1219 |
"writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
|
api/sheet_warm.py
CHANGED
|
@@ -516,7 +516,23 @@ def _desk_result_log(row, tenant, session, before, result, *, admitted=None):
|
|
| 516 |
"""A logging failure supplies no proof and never changes the desk's business outcome."""
|
| 517 |
try:
|
| 518 |
capture = desk_result_capture(row, tenant, session, before, result, admitted=admitted)
|
| 519 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 520 |
except Exception: # noqa: BLE001
|
| 521 |
pass
|
| 522 |
|
|
@@ -542,7 +558,7 @@ def desk_run_one(rt, tenant, generation=None):
|
|
| 542 |
"deadline": time.time() + budget,
|
| 543 |
"thread": threading.current_thread()}
|
| 544 |
error = None
|
| 545 |
-
book, session, result, admitted = None, None, None, None
|
| 546 |
try:
|
| 547 |
try:
|
| 548 |
with request_timing.budget_scope(deadline):
|
|
@@ -553,18 +569,22 @@ def desk_run_one(rt, tenant, generation=None):
|
|
| 553 |
error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
|
| 554 |
else:
|
| 555 |
admitted = _desk_input_identity(book)
|
|
|
|
|
|
|
| 556 |
result = routes_sheet._refresh_desk_run(session, row)
|
| 557 |
except Exception as exc: # noqa: BLE001
|
| 558 |
timed_out = (isinstance(exc, request_timing.RequestDeadlineExceeded)
|
| 559 |
or time.monotonic() >= deadline)
|
| 560 |
error = dict(routes_sheet._REFRESH_TIMEOUT) if timed_out else routes_sheet._refresh_desk_error(exc)
|
| 561 |
-
_desk_result_log(row, tenant, session, book, result, admitted=admitted)
|
| 562 |
owned = routes_sheet._refresh_desk_finish(rt, row["id"], error, claim=claim)
|
| 563 |
if owned is False:
|
| 564 |
with _LOCK:
|
| 565 |
run = _TENANT_RUNS.get(tenant)
|
| 566 |
if run is not None and run["thread"] is threading.current_thread():
|
| 567 |
_FENCED_RUNNERS.add(threading.current_thread())
|
|
|
|
|
|
|
| 568 |
finally:
|
| 569 |
if claim:
|
| 570 |
routes_sheet._REFRESH_LIVE.pop(claim, None)
|
|
|
|
| 516 |
"""A logging failure supplies no proof and never changes the desk's business outcome."""
|
| 517 |
try:
|
| 518 |
capture = desk_result_capture(row, tenant, session, before, result, admitted=admitted)
|
| 519 |
+
raw = refresh_proof.bounded_json(capture, refresh_proof.MAX_LINE_BYTES)
|
| 520 |
+
print(DESK_RESULT_PREFIX + raw, flush=True)
|
| 521 |
+
return capture, raw
|
| 522 |
+
except Exception: # noqa: BLE001
|
| 523 |
+
return None
|
| 524 |
+
|
| 525 |
+
|
| 526 |
+
def _desk_attestation_log(rt, captured, provenance):
|
| 527 |
+
"""Emit optional proof only after the desk records its business outcome."""
|
| 528 |
+
if not captured or not provenance:
|
| 529 |
+
return
|
| 530 |
+
try:
|
| 531 |
+
import worker_provenance
|
| 532 |
+
capture, raw = captured
|
| 533 |
+
companion = worker_provenance.desk_companion(rt, capture, raw.encode("utf-8"), provenance)
|
| 534 |
+
if companion:
|
| 535 |
+
print(worker_provenance.PREFIX + worker_provenance.encoded(companion).decode("utf-8"), flush=True)
|
| 536 |
except Exception: # noqa: BLE001
|
| 537 |
pass
|
| 538 |
|
|
|
|
| 558 |
"deadline": time.time() + budget,
|
| 559 |
"thread": threading.current_thread()}
|
| 560 |
error = None
|
| 561 |
+
book, session, result, admitted, provenance = None, None, None, None, None
|
| 562 |
try:
|
| 563 |
try:
|
| 564 |
with request_timing.budget_scope(deadline):
|
|
|
|
| 569 |
error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
|
| 570 |
else:
|
| 571 |
admitted = _desk_input_identity(book)
|
| 572 |
+
import worker_provenance
|
| 573 |
+
provenance = worker_provenance.evidence(rt)
|
| 574 |
result = routes_sheet._refresh_desk_run(session, row)
|
| 575 |
except Exception as exc: # noqa: BLE001
|
| 576 |
timed_out = (isinstance(exc, request_timing.RequestDeadlineExceeded)
|
| 577 |
or time.monotonic() >= deadline)
|
| 578 |
error = dict(routes_sheet._REFRESH_TIMEOUT) if timed_out else routes_sheet._refresh_desk_error(exc)
|
| 579 |
+
captured = _desk_result_log(row, tenant, session, book, result, admitted=admitted)
|
| 580 |
owned = routes_sheet._refresh_desk_finish(rt, row["id"], error, claim=claim)
|
| 581 |
if owned is False:
|
| 582 |
with _LOCK:
|
| 583 |
run = _TENANT_RUNS.get(tenant)
|
| 584 |
if run is not None and run["thread"] is threading.current_thread():
|
| 585 |
_FENCED_RUNNERS.add(threading.current_thread())
|
| 586 |
+
else:
|
| 587 |
+
_desk_attestation_log(rt, captured, provenance)
|
| 588 |
finally:
|
| 589 |
if claim:
|
| 590 |
routes_sheet._REFRESH_LIVE.pop(claim, None)
|
api/sync_worker.py
CHANGED
|
@@ -123,6 +123,8 @@ def main(argv=None):
|
|
| 123 |
for path in (str(_RI), str(API_DIR)):
|
| 124 |
if path not in sys.path:
|
| 125 |
sys.path.insert(0, path)
|
|
|
|
|
|
|
| 126 |
tenant = os.environ.get("AIOS_SYNC_WRITER_TENANT") or "royal-imports"
|
| 127 |
print(f"[sync-worker] pid={os.getpid()} tenant={tenant} "
|
| 128 |
f"scope={os.environ.get('AIOS_WORKER_SCOPE') or 'primary'} duckdb={worker_db} "
|
|
|
|
| 123 |
for path in (str(_RI), str(API_DIR)):
|
| 124 |
if path not in sys.path:
|
| 125 |
sys.path.insert(0, path)
|
| 126 |
+
import worker_provenance
|
| 127 |
+
worker_provenance.boot_source()
|
| 128 |
tenant = os.environ.get("AIOS_SYNC_WRITER_TENANT") or "royal-imports"
|
| 129 |
print(f"[sync-worker] pid={os.getpid()} tenant={tenant} "
|
| 130 |
f"scope={os.environ.get('AIOS_WORKER_SCOPE') or 'primary'} duckdb={worker_db} "
|
api/worker_provenance.py
ADDED
|
@@ -0,0 +1,428 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Bound installed source and active store evidence without changing Refresh outcomes."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import hashlib
|
| 5 |
+
import contextlib
|
| 6 |
+
import dataclasses
|
| 7 |
+
import inspect
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
import re
|
| 12 |
+
import stat
|
| 13 |
+
import sys
|
| 14 |
+
import time
|
| 15 |
+
from types import CodeType, FunctionType, ModuleType
|
| 16 |
+
|
| 17 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 18 |
+
MAX_BYTES = 8192
|
| 19 |
+
MAX_FILES = 4096
|
| 20 |
+
MAX_FILE_BYTES = 4 * 1024 * 1024
|
| 21 |
+
MAX_SOURCE_BYTES = 64 * 1024 * 1024
|
| 22 |
+
SOURCE_SECONDS = 5
|
| 23 |
+
COMPILE_SECONDS = 30
|
| 24 |
+
MAX_CODES = 131072
|
| 25 |
+
PREFIX = "[sheet-worker-attestation] "
|
| 26 |
+
MANDATORY = ("api/sheet_warm.py", "api/process_role.py", "api/sync_runtime.py",
|
| 27 |
+
"platform/core/refresh_proof.py", "api/worker_provenance.py", "api/sync_worker.py")
|
| 28 |
+
_BOOT_SOURCE = None
|
| 29 |
+
_BOOT_FILES = {}
|
| 30 |
+
_BOOT_CODES = {}
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def encoded(value):
|
| 34 |
+
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
| 35 |
+
allow_nan=False).encode("utf-8")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def sha(raw):
|
| 39 |
+
return hashlib.sha256(raw).hexdigest()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def bounded(value):
|
| 43 |
+
if len(encoded(value)) + len(PREFIX) + 1 > MAX_BYTES:
|
| 44 |
+
raise ValueError("worker_attestation_capacity")
|
| 45 |
+
return value
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def read_file(path, limit):
|
| 49 |
+
"""Read one regular file without following a final symlink."""
|
| 50 |
+
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
| 51 |
+
with os.fdopen(descriptor, "rb") as stream:
|
| 52 |
+
before = os.fstat(stream.fileno())
|
| 53 |
+
if not stat.S_ISREG(before.st_mode) or before.st_size > limit:
|
| 54 |
+
raise ValueError("worker_source_file_capacity_or_type")
|
| 55 |
+
raw = stream.read(limit + 1)
|
| 56 |
+
after = os.fstat(stream.fileno())
|
| 57 |
+
if len(raw) > limit or (before.st_size, before.st_mtime_ns, before.st_ino) != (
|
| 58 |
+
after.st_size, after.st_mtime_ns, after.st_ino):
|
| 59 |
+
raise ValueError("worker_source_read_drift")
|
| 60 |
+
return raw
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def source_summary(files, total):
|
| 64 |
+
if not set(MANDATORY) <= files.keys() or len(files) > MAX_FILES or total > MAX_SOURCE_BYTES:
|
| 65 |
+
raise ValueError("worker_source_missing_or_capacity")
|
| 66 |
+
return {"files": {path: files[path] for path in MANDATORY},
|
| 67 |
+
"manifestSha256": sha(encoded(files)), "fileCount": len(files), "bytes": total}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def source_files(root=None):
|
| 71 |
+
"""Read the complete bounded installed source inventory."""
|
| 72 |
+
root = Path(root or ROOT)
|
| 73 |
+
if root.is_symlink() or not root.is_dir():
|
| 74 |
+
raise ValueError("worker_source_root_invalid")
|
| 75 |
+
version_raw = read_file(root / "VERSION", 128)
|
| 76 |
+
version = version_raw.decode("ascii").strip()
|
| 77 |
+
if not re.fullmatch(r"[0-9a-f]{7,40}", version):
|
| 78 |
+
raise ValueError("worker_installed_version_missing")
|
| 79 |
+
started, files, total = time.monotonic(), {"VERSION": sha(version_raw)}, len(version_raw)
|
| 80 |
+
entries_seen = 0
|
| 81 |
+
for area in ("api", "platform"):
|
| 82 |
+
directory = root / area
|
| 83 |
+
if directory.is_symlink() or not directory.is_dir():
|
| 84 |
+
raise ValueError("worker_source_directory_missing")
|
| 85 |
+
for parent, dirs, names in os.walk(directory, followlinks=False):
|
| 86 |
+
entries_seen += len(dirs) + len(names)
|
| 87 |
+
if entries_seen > MAX_FILES * 4 or time.monotonic() - started > SOURCE_SECONDS:
|
| 88 |
+
raise ValueError("worker_source_tree_capacity_or_deadline")
|
| 89 |
+
for name in dirs:
|
| 90 |
+
if (Path(parent) / name).is_symlink():
|
| 91 |
+
raise ValueError("worker_source_symlink")
|
| 92 |
+
for name in sorted(names):
|
| 93 |
+
if not name.endswith(".py"):
|
| 94 |
+
continue
|
| 95 |
+
path = Path(parent) / name
|
| 96 |
+
if path.is_symlink() or time.monotonic() - started > SOURCE_SECONDS:
|
| 97 |
+
raise ValueError("worker_source_symlink_or_deadline")
|
| 98 |
+
if len(files) >= MAX_FILES:
|
| 99 |
+
raise ValueError("worker_source_capacity")
|
| 100 |
+
raw = read_file(path, min(MAX_FILE_BYTES, MAX_SOURCE_BYTES - total))
|
| 101 |
+
total += len(raw)
|
| 102 |
+
files[path.relative_to(root).as_posix()] = sha(raw)
|
| 103 |
+
return version, files, total
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def installed_source(root=None):
|
| 107 |
+
"""Hash every installed API/platform Python path and the installed VERSION."""
|
| 108 |
+
version, files, total = source_files(root)
|
| 109 |
+
return version, source_summary(files, total)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def code_identity(code):
|
| 113 |
+
return code.replace(co_filename="", co_consts=tuple(
|
| 114 |
+
code_identity(value) if isinstance(value, CodeType) else value for value in code.co_consts))
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def function_codes(function, owner=None):
|
| 118 |
+
"""Retain decorator origins and attribute dataclass-generated methods to their stdlib factory."""
|
| 119 |
+
seen = set()
|
| 120 |
+
while function is not None:
|
| 121 |
+
if id(function) in seen or len(seen) >= 32:
|
| 122 |
+
raise ValueError("worker_loaded_wrapper_capacity")
|
| 123 |
+
seen.add(id(function))
|
| 124 |
+
if isinstance(function, FunctionType):
|
| 125 |
+
code = function.__code__
|
| 126 |
+
generated = (owner is not None and dataclasses.is_dataclass(owner) and code.co_filename == "<string>"
|
| 127 |
+
and code.co_qualname == "__create_fn__.<locals>." + function.__name__
|
| 128 |
+
and function.__name__ in ("__init__", "__repr__", "__eq__", "__lt__", "__le__",
|
| 129 |
+
"__gt__", "__ge__", "__hash__", "__setattr__", "__delattr__"))
|
| 130 |
+
yield dataclasses._create_fn.__code__ if generated else code
|
| 131 |
+
function = getattr(function, "__wrapped__", None)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def definitions(module):
|
| 135 |
+
"""Yield Python definitions owned by a module, including adapter methods."""
|
| 136 |
+
for value in tuple(vars(module).values()):
|
| 137 |
+
if ((isinstance(value, FunctionType) or type(value).__module__ == "functools")
|
| 138 |
+
and getattr(value, "__module__", None) == module.__name__):
|
| 139 |
+
yield from function_codes(value)
|
| 140 |
+
elif isinstance(value, type) and value.__module__ == module.__name__:
|
| 141 |
+
for member in tuple(vars(value).values()):
|
| 142 |
+
if isinstance(member, (staticmethod, classmethod)):
|
| 143 |
+
member = member.__func__
|
| 144 |
+
members = (member.fget, member.fset, member.fdel) if isinstance(member, property) else (member,)
|
| 145 |
+
for function in members:
|
| 146 |
+
if isinstance(function, FunctionType) and function.__module__ == module.__name__:
|
| 147 |
+
yield from function_codes(function, value)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def verified_code(code):
|
| 151 |
+
"""Verify application code or a known stdlib decorator factory without trusting a filename alone."""
|
| 152 |
+
for library, factory in ((contextlib, contextlib.contextmanager), (contextlib, contextlib.asynccontextmanager),
|
| 153 |
+
(dataclasses, dataclasses._create_fn),
|
| 154 |
+
(dataclasses, dataclasses._recursive_repr)):
|
| 155 |
+
if code.co_filename != library.__file__:
|
| 156 |
+
continue
|
| 157 |
+
if (library.__spec__.origin != library.__file__
|
| 158 |
+
or Path(library.__file__).parent != Path(os.__file__).parent
|
| 159 |
+
or not Path(library.__file__).is_file() or Path(library.__file__).is_symlink()):
|
| 160 |
+
raise ValueError("worker_stdlib_factory_origin")
|
| 161 |
+
pending = [factory.__code__]
|
| 162 |
+
while pending:
|
| 163 |
+
original = pending.pop()
|
| 164 |
+
if code == original:
|
| 165 |
+
return
|
| 166 |
+
pending.extend(value for value in original.co_consts if isinstance(value, CodeType))
|
| 167 |
+
path = source_path(code.co_filename)
|
| 168 |
+
if code_identity(code) not in _BOOT_CODES[path]:
|
| 169 |
+
raise ValueError("worker_loaded_code_drift")
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def source_path(path):
|
| 173 |
+
"""Require an existing source path inside the installed tree without symlink components."""
|
| 174 |
+
path, root = Path(path).absolute(), ROOT.absolute()
|
| 175 |
+
relative = path.relative_to(root)
|
| 176 |
+
if any((root.joinpath(*relative.parts[:i])).is_symlink() for i in range(len(relative.parts) + 1)):
|
| 177 |
+
raise ValueError("worker_loaded_origin_symlink")
|
| 178 |
+
if relative.as_posix() not in _BOOT_FILES or path.suffix != ".py" or not path.is_file():
|
| 179 |
+
raise ValueError("worker_loaded_source_missing")
|
| 180 |
+
return relative.as_posix()
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def module_origin(module, expected=None):
|
| 184 |
+
"""Bind a loaded module and its defining code to the pinned installed inventory."""
|
| 185 |
+
path = source_path(module.__file__)
|
| 186 |
+
spec = getattr(module, "__spec__", None)
|
| 187 |
+
if ((spec is None and module.__name__ != "__main__")
|
| 188 |
+
or (spec is not None and source_path(spec.origin) != path) or (expected and expected != path)):
|
| 189 |
+
raise ValueError("worker_loaded_origin_mismatch")
|
| 190 |
+
if sha(read_file(ROOT / path, MAX_FILE_BYTES)) != _BOOT_FILES[path]:
|
| 191 |
+
raise ValueError("worker_loaded_source_drift")
|
| 192 |
+
for code in definitions(module):
|
| 193 |
+
verified_code(code)
|
| 194 |
+
return path
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def loaded_origins():
|
| 198 |
+
"""Check application modules, namespaces and aliases without adding volatile identity fields."""
|
| 199 |
+
names, directories, started = {}, {}, time.monotonic()
|
| 200 |
+
for path in _BOOT_CODES:
|
| 201 |
+
area, relative = path.split("/", 1)
|
| 202 |
+
parts = relative.removesuffix(".py").split("/")
|
| 203 |
+
if parts[-1] == "__init__":
|
| 204 |
+
parts.pop()
|
| 205 |
+
if parts:
|
| 206 |
+
names[".".join(parts)] = path
|
| 207 |
+
names[area + "." + ".".join(parts)] = path
|
| 208 |
+
parents = relative.split("/")[:-1]
|
| 209 |
+
for i in range(len(parents) + 1):
|
| 210 |
+
directory = area + ("/" + "/".join(parents[:i]) if i else "")
|
| 211 |
+
directories[directory.replace("/", ".")] = directory
|
| 212 |
+
if i:
|
| 213 |
+
directories[".".join(parents[:i])] = directory
|
| 214 |
+
signatures = set().union(*_BOOT_CODES.values())
|
| 215 |
+
modules = tuple(sys.modules.items())
|
| 216 |
+
if len(modules) > MAX_FILES * 4:
|
| 217 |
+
raise ValueError("worker_loaded_module_capacity")
|
| 218 |
+
for name, module in modules:
|
| 219 |
+
if time.monotonic() - started > SOURCE_SECONDS:
|
| 220 |
+
raise ValueError("worker_loaded_origin_deadline")
|
| 221 |
+
if not isinstance(module, ModuleType):
|
| 222 |
+
continue
|
| 223 |
+
spec = getattr(module, "__spec__", None)
|
| 224 |
+
labels = (name, module.__name__, getattr(spec, "name", ""))
|
| 225 |
+
expected = next((names[label] for label in labels if label in names), None)
|
| 226 |
+
namespace = next((directories[label] for label in labels if label in directories), None)
|
| 227 |
+
paths = [getattr(module, "__file__", None), getattr(spec, "origin", None)]
|
| 228 |
+
rooted = any(isinstance(path, str) and Path(path).is_relative_to(ROOT) for path in paths)
|
| 229 |
+
owned = tuple(definitions(module))
|
| 230 |
+
alias = any(code_identity(code) in signatures for code in owned)
|
| 231 |
+
# Python's platform.py is a stdlib module, not the application's platform namespace.
|
| 232 |
+
if not (expected or rooted or alias or (namespace and hasattr(module, "__path__"))):
|
| 233 |
+
continue
|
| 234 |
+
if not paths[0] and namespace and spec is not None and spec.origin is None:
|
| 235 |
+
locations = tuple(getattr(spec, "submodule_search_locations", ()) or ())
|
| 236 |
+
if not locations or set(locations) != set(module.__path__):
|
| 237 |
+
raise ValueError("worker_loaded_namespace_unproven")
|
| 238 |
+
for location in locations:
|
| 239 |
+
directory = Path(location).absolute()
|
| 240 |
+
if (directory != ROOT / namespace or not directory.is_dir()
|
| 241 |
+
or directory.resolve() != directory):
|
| 242 |
+
raise ValueError("worker_loaded_namespace_origin")
|
| 243 |
+
continue
|
| 244 |
+
module_origin(module, expected)
|
| 245 |
+
frame, depth = sys._getframe(), 0
|
| 246 |
+
try:
|
| 247 |
+
while frame is not None:
|
| 248 |
+
depth += 1
|
| 249 |
+
if depth > 128:
|
| 250 |
+
raise ValueError("worker_loaded_stack_capacity")
|
| 251 |
+
name = frame.f_globals.get("__name__", "")
|
| 252 |
+
code, spec = frame.f_code, frame.f_globals.get("__spec__")
|
| 253 |
+
signature = code_identity(code)
|
| 254 |
+
rooted = Path(code.co_filename).is_relative_to(ROOT)
|
| 255 |
+
if name in names or rooted or signature in signatures:
|
| 256 |
+
path = source_path(frame.f_globals.get("__file__"))
|
| 257 |
+
if (source_path(code.co_filename) != path or (spec is None and name != "__main__")
|
| 258 |
+
or (spec is not None and source_path(spec.origin) != path)
|
| 259 |
+
or (name in names and names[name] != path) or signature not in _BOOT_CODES[path]):
|
| 260 |
+
raise ValueError("worker_executing_origin_mismatch")
|
| 261 |
+
frame = frame.f_back
|
| 262 |
+
finally:
|
| 263 |
+
del frame
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def boot_source():
|
| 267 |
+
"""Pin installed bytes once before worker services start, including an unproven boot."""
|
| 268 |
+
global _BOOT_SOURCE, _BOOT_FILES, _BOOT_CODES
|
| 269 |
+
if _BOOT_SOURCE is None:
|
| 270 |
+
try:
|
| 271 |
+
path = Path(__file__)
|
| 272 |
+
if path.is_symlink() or path.parent.is_symlink():
|
| 273 |
+
raise ValueError("worker_source_symlink")
|
| 274 |
+
version, _BOOT_FILES, total = source_files()
|
| 275 |
+
_BOOT_CODES = {}
|
| 276 |
+
started, code_count = time.monotonic(), 0
|
| 277 |
+
for relative, digest in _BOOT_FILES.items():
|
| 278 |
+
if not relative.endswith(".py"):
|
| 279 |
+
continue
|
| 280 |
+
raw = read_file(ROOT / relative, MAX_FILE_BYTES)
|
| 281 |
+
if sha(raw) != digest or time.monotonic() - started > COMPILE_SECONDS:
|
| 282 |
+
raise ValueError("worker_boot_code_drift_or_deadline")
|
| 283 |
+
codes, pending = set(), [compile(raw, str(ROOT / relative), "exec", dont_inherit=True)]
|
| 284 |
+
while pending:
|
| 285 |
+
code_count += 1
|
| 286 |
+
if code_count > MAX_CODES or time.monotonic() - started > COMPILE_SECONDS:
|
| 287 |
+
raise ValueError("worker_boot_code_capacity_or_deadline")
|
| 288 |
+
code = pending.pop()
|
| 289 |
+
codes.add(code_identity(code))
|
| 290 |
+
pending.extend(value for value in code.co_consts if isinstance(value, CodeType))
|
| 291 |
+
_BOOT_CODES[relative] = codes
|
| 292 |
+
loaded_origins()
|
| 293 |
+
_BOOT_SOURCE = version, source_summary(_BOOT_FILES, total)
|
| 294 |
+
except Exception:
|
| 295 |
+
_BOOT_SOURCE = False
|
| 296 |
+
return _BOOT_SOURCE
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def store_identity(rt):
|
| 300 |
+
"""Identify the adapter in use and fingerprint a checked live PostgreSQL connection."""
|
| 301 |
+
if not boot_source():
|
| 302 |
+
raise ValueError("worker_adapter_source_unproven")
|
| 303 |
+
tenant = str(getattr(rt, "key", ""))
|
| 304 |
+
backend = rt._document_store() if callable(getattr(rt, "_document_store", None)) else rt
|
| 305 |
+
cls = type(backend)
|
| 306 |
+
defining = sys.modules.get(cls.__module__)
|
| 307 |
+
if not isinstance(defining, ModuleType):
|
| 308 |
+
raise ValueError("worker_adapter_module_missing")
|
| 309 |
+
adapter_path = module_origin(defining)
|
| 310 |
+
if source_path(inspect.getfile(cls)) != adapter_path:
|
| 311 |
+
raise ValueError("worker_adapter_origin_mismatch")
|
| 312 |
+
for member in vars(cls).values():
|
| 313 |
+
if isinstance(member, FunctionType) and (source_path(member.__code__.co_filename) != adapter_path
|
| 314 |
+
or code_identity(member.__code__) not in _BOOT_CODES[adapter_path]):
|
| 315 |
+
raise ValueError("worker_adapter_code_drift")
|
| 316 |
+
# An installed fixture remains memory-backed even when DATABASE_URL is configured.
|
| 317 |
+
fake = sys.modules.get("_fake_store")
|
| 318 |
+
if fake is not None and cls is getattr(fake, "FakeStore", None):
|
| 319 |
+
return {"backend": "memory", "tenant": tenant,
|
| 320 |
+
"adapter": cls.__module__ + "." + cls.__qualname__,
|
| 321 |
+
"adapterSha256": sha(read_file(Path(inspect.getfile(cls)), MAX_FILE_BYTES))}
|
| 322 |
+
from core import store_pg
|
| 323 |
+
from psycopg.conninfo import conninfo_to_dict
|
| 324 |
+
if cls is not store_pg.PgStore or backend.tenant_slug != tenant:
|
| 325 |
+
raise ValueError("worker_active_store_unproven")
|
| 326 |
+
store_pg.check_write(tenant, "worker provenance")
|
| 327 |
+
expected = conninfo_to_dict(store_pg._url() or "")
|
| 328 |
+
if not all(expected.get(key) for key in ("host", "port", "dbname")):
|
| 329 |
+
raise ValueError("worker_active_endpoint_missing")
|
| 330 |
+
with store_pg._pool().connection(timeout=1.0) as connection:
|
| 331 |
+
info = connection.info
|
| 332 |
+
actual = {key: str(getattr(info, key)) for key in ("host", "port", "dbname")}
|
| 333 |
+
if any(actual[key] != str(expected[key]) for key in actual) or not connection.pgconn.ssl_in_use:
|
| 334 |
+
raise ValueError("worker_active_endpoint_mismatch")
|
| 335 |
+
return {"backend": "pg", "tenant": tenant, "hostSha256": sha(actual["host"].lower().encode())[:12],
|
| 336 |
+
"port": int(actual["port"]), "databaseSha256": sha(actual["dbname"].encode()),
|
| 337 |
+
"schemaSha256": sha(store_pg._schema_for(tenant).encode()), "tls": True}
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def evidence(rt):
|
| 341 |
+
"""Return fresh bounded evidence or None without affecting worker health."""
|
| 342 |
+
try:
|
| 343 |
+
import process_role
|
| 344 |
+
from core import data_binding
|
| 345 |
+
tenant, job = str(getattr(rt, "key", "")), os.environ.get("JOB_ID", "")
|
| 346 |
+
space = data_binding.deployment_id()
|
| 347 |
+
dataset = os.environ.get("OS_DATA_REPO", "")
|
| 348 |
+
if (process_role.role() != "worker" or tenant != process_role.writer_tenant()
|
| 349 |
+
or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", job)
|
| 350 |
+
or not re.fullmatch(r"[A-Za-z0-9_-]+/[A-Za-z0-9_-]+", space)
|
| 351 |
+
or os.environ.get("SPACE_ID", space) != space
|
| 352 |
+
or (os.environ.get("RI_DIR") and Path(os.environ["RI_DIR"]).resolve() != (ROOT / "platform").resolve())
|
| 353 |
+
or (dataset and not re.fullmatch(r"[A-Za-z0-9_-]+/[A-Za-z0-9_.-]+", dataset))
|
| 354 |
+
or os.environ.get("AIOS_SYNC_JOB_MODE") != "resident"):
|
| 355 |
+
return None
|
| 356 |
+
boot = boot_source()
|
| 357 |
+
if not boot:
|
| 358 |
+
return None
|
| 359 |
+
current = installed_source()
|
| 360 |
+
if current != boot:
|
| 361 |
+
return None
|
| 362 |
+
version, source = current
|
| 363 |
+
loaded_origins()
|
| 364 |
+
store = store_identity(rt)
|
| 365 |
+
loaded_origins()
|
| 366 |
+
if installed_source() != boot:
|
| 367 |
+
return None
|
| 368 |
+
return bounded({"at": time.time(), "pid": os.getpid(), "spaceId": space, "jobId": job,
|
| 369 |
+
"tenant": tenant, "version": version, "source": source, "store": store,
|
| 370 |
+
"role": process_role.role(), "scope": process_role.worker_scope(),
|
| 371 |
+
"deployment": data_binding.deployment_id(), "mode": "resident",
|
| 372 |
+
"dataset": dataset,
|
| 373 |
+
"policy": {"productionOverride": str(os.environ.get("AIOS_ALLOW_PRODUCTION_DATA", "")).strip().lower()
|
| 374 |
+
not in ("", "0", "false", "no"),
|
| 375 |
+
"directDatabase": bool(os.environ.get("DATABASE_URL_DIRECT")),
|
| 376 |
+
"executableOverride": any((os.environ.get("ENV"), os.environ.get("BASH_ENV"),
|
| 377 |
+
os.environ.get("PYTHONPATH"), os.environ.get("PYTHONHOME"),
|
| 378 |
+
os.environ.get("LD_PRELOAD"), os.environ.get("LD_LIBRARY_PATH")))}})
|
| 379 |
+
except Exception:
|
| 380 |
+
return None
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def stable(value):
|
| 384 |
+
return {key: item for key, item in value.items() if key != "at"} if value else None
|
| 385 |
+
|
| 386 |
+
|
| 387 |
+
def desk_companion(rt, capture, raw, before):
|
| 388 |
+
"""Bind this original capture to unchanged source and active-worker evidence."""
|
| 389 |
+
try:
|
| 390 |
+
import process_role
|
| 391 |
+
if not before:
|
| 392 |
+
return None
|
| 393 |
+
current = evidence(rt)
|
| 394 |
+
if not current or stable(before) != stable(current):
|
| 395 |
+
return None
|
| 396 |
+
stored = process_role.stored_worker(rt)
|
| 397 |
+
if ((stored.get("job") or {}).get("id") != current["jobId"]
|
| 398 |
+
or (stored.get("job") or {}).get("tenant") != current["tenant"]
|
| 399 |
+
or capture.get("version") != current["version"]):
|
| 400 |
+
return None
|
| 401 |
+
return bounded({**current, "at": time.time(), "schema": "sheet-worker-attestation/v1", "resultSha256": sha(raw),
|
| 402 |
+
**{key: capture[key] for key in ("requestId", "executionId", "tenant", "reader", "workbook")}})
|
| 403 |
+
except Exception:
|
| 404 |
+
return None
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
def heartbeat_attestation(rt, beat):
|
| 408 |
+
try:
|
| 409 |
+
value = evidence(rt)
|
| 410 |
+
bounded(beat)
|
| 411 |
+
return bounded({**value, "schema": "sync-worker-attestation/v1", "heartbeatSha256": sha(encoded(beat))}) if value else None
|
| 412 |
+
except Exception:
|
| 413 |
+
return None
|
| 414 |
+
|
| 415 |
+
|
| 416 |
+
def visible(stored):
|
| 417 |
+
"""Expose proof only for the same fresh stored heartbeat and job."""
|
| 418 |
+
try:
|
| 419 |
+
value, beat, job = stored["attestation"], stored["heartbeat"], stored["job"]
|
| 420 |
+
bounded(value)
|
| 421 |
+
age = time.time() - beat["at"]
|
| 422 |
+
if (value["jobId"] == job["id"] and value["tenant"] == job["tenant"] == beat["tenant"]
|
| 423 |
+
and value["pid"] == beat["pid"] and 0 <= age < 120
|
| 424 |
+
and value["heartbeatSha256"] == sha(encoded(beat))):
|
| 425 |
+
return value
|
| 426 |
+
except Exception:
|
| 427 |
+
pass
|
| 428 |
+
return None
|
platform/harness/datastore.py
CHANGED
|
@@ -804,6 +804,14 @@ def _odoo_mirror_store():
|
|
| 804 |
def _owns_state(self, entity):
|
| 805 |
return entity.split(".", 1)[0] in self.spec.tables
|
| 806 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 807 |
def _validated_snapshot(self, tenant_key, rt, con, raw, tmp, **kwargs):
|
| 808 |
# Odoo never advertises a partial restore or replaces it with a seed.
|
| 809 |
snapshot = super()._validated_snapshot(tenant_key, rt, con, raw, tmp,
|
|
|
|
| 804 |
def _owns_state(self, entity):
|
| 805 |
return entity.split(".", 1)[0] in self.spec.tables
|
| 806 |
|
| 807 |
+
def _restore_complete(self, tables, rows, con=None):
|
| 808 |
+
"""Require live sync entities and installed receipts for their required column backfills."""
|
| 809 |
+
entities = set(tables) & set(self.spec.tables)
|
| 810 |
+
return (con is not None and super()._restore_complete(entities, rows)
|
| 811 |
+
and all(row[1] == "live" for row in rows if row[0] in entities)
|
| 812 |
+
and all(_required_columns_ready(con, key, ENTITIES.get(key) or {})
|
| 813 |
+
for key in entities))
|
| 814 |
+
|
| 815 |
def _validated_snapshot(self, tenant_key, rt, con, raw, tmp, **kwargs):
|
| 816 |
# Odoo never advertises a partial restore or replaces it with a seed.
|
| 817 |
snapshot = super()._validated_snapshot(tenant_key, rt, con, raw, tmp,
|
platform/harness/mirror_store.py
CHANGED
|
@@ -547,6 +547,12 @@ class MirrorStore:
|
|
| 547 |
def _state_complete(self, rows):
|
| 548 |
return True
|
| 549 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 550 |
def _log_restored(self, paths, total, rows, log):
|
| 551 |
log(f"{self.label} mirror restored from the store: {len(paths)} tables, {total:,} rows")
|
| 552 |
|
|
@@ -886,9 +892,7 @@ class MirrorStore:
|
|
| 886 |
# Trap: the healed local rows differ from the published snapshot until the
|
| 887 |
# next publish; each restore heals again and a second heal changes nothing.
|
| 888 |
self._heal_published(con, result, log)
|
| 889 |
-
|
| 890 |
-
current = (not refused and bool(paths) and set(paths) <= set(table_states)
|
| 891 |
-
and all(row[1] in {"live", "done"} for row in rows))
|
| 892 |
newest = _record_newest(con, self.connector)
|
| 893 |
with _CATCH_UP_LOCK:
|
| 894 |
_catch_up_source(rt, self.connector).update(
|
|
|
|
| 547 |
def _state_complete(self, rows):
|
| 548 |
return True
|
| 549 |
|
| 550 |
+
def _restore_complete(self, tables, rows, con=None):
|
| 551 |
+
"""Require a finished marker for every installed table unless its connector owns another policy."""
|
| 552 |
+
table_states = {row[0]: row[1] for row in rows if row[0] in tables}
|
| 553 |
+
return (bool(tables) and set(tables) <= set(table_states)
|
| 554 |
+
and all(row[1] in {"live", "done"} for row in rows))
|
| 555 |
+
|
| 556 |
def _log_restored(self, paths, total, rows, log):
|
| 557 |
log(f"{self.label} mirror restored from the store: {len(paths)} tables, {total:,} rows")
|
| 558 |
|
|
|
|
| 892 |
# Trap: the healed local rows differ from the published snapshot until the
|
| 893 |
# next publish; each restore heals again and a second heal changes nothing.
|
| 894 |
self._heal_published(con, result, log)
|
| 895 |
+
current = not refused and self._restore_complete(paths, rows, con=con)
|
|
|
|
|
|
|
| 896 |
newest = _record_newest(con, self.connector)
|
| 897 |
with _CATCH_UP_LOCK:
|
| 898 |
_catch_up_source(rt, self.connector).update(
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
3146b695594322bf63b85303beb8150a938e2e80648416de20fbee2970e1bdb0
|