fsanyoto commited on
Commit
79a4edf
·
verified ·
1 Parent(s): ac97ffa

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
Dockerfile CHANGED
@@ -61,6 +61,7 @@ COPY requirements.txt ./
61
  RUN python -m pip install --no-cache-dir --disable-pip-version-check --upgrade "pip==25.0.1" \
62
  && python -m pip install --no-cache-dir --disable-pip-version-check -r requirements.txt
63
  COPY api/ ./api/
 
64
  # the reused data layer (modules/core/harness)
65
  # ⛔ COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction:
66
  # it parsed `# the reused data layer ...` as extra SOURCE PATHS and the first build ever run
 
61
  RUN python -m pip install --no-cache-dir --disable-pip-version-check --upgrade "pip==25.0.1" \
62
  && python -m pip install --no-cache-dir --disable-pip-version-check -r requirements.txt
63
  COPY api/ ./api/
64
+ COPY VERSION ./VERSION
65
  # the reused data layer (modules/core/harness)
66
  # ⛔ COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction:
67
  # it parsed `# the reused data layer ...` as extra SOURCE PATHS and the first build ever run
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "b0197f374",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "b9a552ad82b",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- b0197f374
 
1
+ b9a552ad82b
api/process_role.py CHANGED
@@ -883,11 +883,17 @@ def stored_worker(rt):
883
  return connector_sync.status(rt, WORKER_STATUS, fresh=True) or {}
884
 
885
 
 
 
 
886
  def stored_heartbeat(tenant=None):
887
  """The worker heartbeat in the store, or None. A store that does not answer reads as None."""
888
  try:
889
- beat = stored_worker(_status_runtime(tenant)).get("heartbeat")
 
 
890
  except Exception: # noqa: BLE001
 
891
  return None
892
  return beat if isinstance(beat, dict) else None
893
 
@@ -1005,7 +1011,27 @@ def write_heartbeat(now=None, get_runtime=None):
1005
  get_runtime = get_runtime or _status_runtime
1006
  beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
1007
  "at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE)}
1008
- written = connector_sync.update_status(get_runtime(writer_tenant()), WORKER_STATUS, "heartbeat", beat)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1009
  try:
1010
  from harness import mirror_store
1011
  catch_up = mirror_store.catch_up_snapshot()
@@ -1163,7 +1189,11 @@ def status():
1163
 
1164
  A serving role reads the worker from its stored heartbeat, never from a child process (D-11183).
1165
  """
 
1166
  beat = stored_heartbeat() if ROLE != "all" else None
 
 
 
1167
  age = beat_age(beat) if beat is not None else None
1168
  if ROLE == "worker":
1169
  pid, alive = os.getpid(), True
@@ -1184,5 +1214,6 @@ def status():
1184
  "tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
1185
  "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
1186
  "restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
1187
- "lastInstallError": dict(LAST_INSTALL_ERROR) or None},
 
1188
  "writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
 
883
  return connector_sync.status(rt, WORKER_STATUS, fresh=True) or {}
884
 
885
 
886
+ _HEARTBEAT_READ = threading.local()
887
+
888
+
889
  def stored_heartbeat(tenant=None):
890
  """The worker heartbeat in the store, or None. A store that does not answer reads as None."""
891
  try:
892
+ worker = stored_worker(_status_runtime(tenant))
893
+ _HEARTBEAT_READ.worker = worker
894
+ beat = worker.get("heartbeat")
895
  except Exception: # noqa: BLE001
896
+ _HEARTBEAT_READ.worker = {}
897
  return None
898
  return beat if isinstance(beat, dict) else None
899
 
 
1011
  get_runtime = get_runtime or _status_runtime
1012
  beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
1013
  "at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE)}
1014
+ rt = get_runtime(writer_tenant())
1015
+ import worker_provenance
1016
+ attestation = worker_provenance.heartbeat_attestation(rt, beat)
1017
+
1018
+ def change(entry):
1019
+ job = entry.get("job") or {}
1020
+ actual_job = os.environ.get("JOB_ID")
1021
+ if job.get("id") and (actual_job != job["id"] or job.get("tenant") not in (None, writer_tenant())):
1022
+ raise ValueError("stale_worker_heartbeat")
1023
+ entry["heartbeat"] = {**(entry.get("heartbeat") or {}), **beat}
1024
+ # Hash the final stored heartbeat, including fields from an earlier schema.
1025
+ entry["attestation"] = None
1026
+ if attestation and actual_job == job.get("id") and job.get("tenant") == writer_tenant():
1027
+ try:
1028
+ worker_provenance.bounded(entry["heartbeat"])
1029
+ entry["attestation"] = worker_provenance.bounded({**attestation, "heartbeatSha256": worker_provenance.sha(
1030
+ worker_provenance.encoded(entry["heartbeat"]))})
1031
+ except Exception:
1032
+ pass
1033
+
1034
+ written = connector_sync._status_update(rt, WORKER_STATUS, None, change)
1035
  try:
1036
  from harness import mirror_store
1037
  catch_up = mirror_store.catch_up_snapshot()
 
1189
 
1190
  A serving role reads the worker from its stored heartbeat, never from a child process (D-11183).
1191
  """
1192
+ _HEARTBEAT_READ.worker = {}
1193
  beat = stored_heartbeat() if ROLE != "all" else None
1194
+ stored = _HEARTBEAT_READ.worker
1195
+ beat = beat if isinstance(beat, dict) else None
1196
+ import worker_provenance
1197
  age = beat_age(beat) if beat is not None else None
1198
  if ROLE == "worker":
1199
  pid, alive = os.getpid(), True
 
1214
  "tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
1215
  "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
1216
  "restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
1217
+ "lastInstallError": dict(LAST_INSTALL_ERROR) or None,
1218
+ "attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None},
1219
  "writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
api/sheet_warm.py CHANGED
@@ -516,7 +516,23 @@ def _desk_result_log(row, tenant, session, before, result, *, admitted=None):
516
  """A logging failure supplies no proof and never changes the desk's business outcome."""
517
  try:
518
  capture = desk_result_capture(row, tenant, session, before, result, admitted=admitted)
519
- print(DESK_RESULT_PREFIX + refresh_proof.bounded_json(capture, refresh_proof.MAX_LINE_BYTES), flush=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
520
  except Exception: # noqa: BLE001
521
  pass
522
 
@@ -542,7 +558,7 @@ def desk_run_one(rt, tenant, generation=None):
542
  "deadline": time.time() + budget,
543
  "thread": threading.current_thread()}
544
  error = None
545
- book, session, result, admitted = None, None, None, None
546
  try:
547
  try:
548
  with request_timing.budget_scope(deadline):
@@ -553,18 +569,22 @@ def desk_run_one(rt, tenant, generation=None):
553
  error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
554
  else:
555
  admitted = _desk_input_identity(book)
 
 
556
  result = routes_sheet._refresh_desk_run(session, row)
557
  except Exception as exc: # noqa: BLE001
558
  timed_out = (isinstance(exc, request_timing.RequestDeadlineExceeded)
559
  or time.monotonic() >= deadline)
560
  error = dict(routes_sheet._REFRESH_TIMEOUT) if timed_out else routes_sheet._refresh_desk_error(exc)
561
- _desk_result_log(row, tenant, session, book, result, admitted=admitted)
562
  owned = routes_sheet._refresh_desk_finish(rt, row["id"], error, claim=claim)
563
  if owned is False:
564
  with _LOCK:
565
  run = _TENANT_RUNS.get(tenant)
566
  if run is not None and run["thread"] is threading.current_thread():
567
  _FENCED_RUNNERS.add(threading.current_thread())
 
 
568
  finally:
569
  if claim:
570
  routes_sheet._REFRESH_LIVE.pop(claim, None)
 
516
  """A logging failure supplies no proof and never changes the desk's business outcome."""
517
  try:
518
  capture = desk_result_capture(row, tenant, session, before, result, admitted=admitted)
519
+ raw = refresh_proof.bounded_json(capture, refresh_proof.MAX_LINE_BYTES)
520
+ print(DESK_RESULT_PREFIX + raw, flush=True)
521
+ return capture, raw
522
+ except Exception: # noqa: BLE001
523
+ return None
524
+
525
+
526
+ def _desk_attestation_log(rt, captured, provenance):
527
+ """Emit optional proof only after the desk records its business outcome."""
528
+ if not captured or not provenance:
529
+ return
530
+ try:
531
+ import worker_provenance
532
+ capture, raw = captured
533
+ companion = worker_provenance.desk_companion(rt, capture, raw.encode("utf-8"), provenance)
534
+ if companion:
535
+ print(worker_provenance.PREFIX + worker_provenance.encoded(companion).decode("utf-8"), flush=True)
536
  except Exception: # noqa: BLE001
537
  pass
538
 
 
558
  "deadline": time.time() + budget,
559
  "thread": threading.current_thread()}
560
  error = None
561
+ book, session, result, admitted, provenance = None, None, None, None, None
562
  try:
563
  try:
564
  with request_timing.budget_scope(deadline):
 
569
  error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
570
  else:
571
  admitted = _desk_input_identity(book)
572
+ import worker_provenance
573
+ provenance = worker_provenance.evidence(rt)
574
  result = routes_sheet._refresh_desk_run(session, row)
575
  except Exception as exc: # noqa: BLE001
576
  timed_out = (isinstance(exc, request_timing.RequestDeadlineExceeded)
577
  or time.monotonic() >= deadline)
578
  error = dict(routes_sheet._REFRESH_TIMEOUT) if timed_out else routes_sheet._refresh_desk_error(exc)
579
+ captured = _desk_result_log(row, tenant, session, book, result, admitted=admitted)
580
  owned = routes_sheet._refresh_desk_finish(rt, row["id"], error, claim=claim)
581
  if owned is False:
582
  with _LOCK:
583
  run = _TENANT_RUNS.get(tenant)
584
  if run is not None and run["thread"] is threading.current_thread():
585
  _FENCED_RUNNERS.add(threading.current_thread())
586
+ else:
587
+ _desk_attestation_log(rt, captured, provenance)
588
  finally:
589
  if claim:
590
  routes_sheet._REFRESH_LIVE.pop(claim, None)
api/sync_worker.py CHANGED
@@ -123,6 +123,8 @@ def main(argv=None):
123
  for path in (str(_RI), str(API_DIR)):
124
  if path not in sys.path:
125
  sys.path.insert(0, path)
 
 
126
  tenant = os.environ.get("AIOS_SYNC_WRITER_TENANT") or "royal-imports"
127
  print(f"[sync-worker] pid={os.getpid()} tenant={tenant} "
128
  f"scope={os.environ.get('AIOS_WORKER_SCOPE') or 'primary'} duckdb={worker_db} "
 
123
  for path in (str(_RI), str(API_DIR)):
124
  if path not in sys.path:
125
  sys.path.insert(0, path)
126
+ import worker_provenance
127
+ worker_provenance.boot_source()
128
  tenant = os.environ.get("AIOS_SYNC_WRITER_TENANT") or "royal-imports"
129
  print(f"[sync-worker] pid={os.getpid()} tenant={tenant} "
130
  f"scope={os.environ.get('AIOS_WORKER_SCOPE') or 'primary'} duckdb={worker_db} "
api/worker_provenance.py ADDED
@@ -0,0 +1,428 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Bound installed source and active store evidence without changing Refresh outcomes."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ import contextlib
6
+ import dataclasses
7
+ import inspect
8
+ import json
9
+ import os
10
+ from pathlib import Path
11
+ import re
12
+ import stat
13
+ import sys
14
+ import time
15
+ from types import CodeType, FunctionType, ModuleType
16
+
17
+ ROOT = Path(__file__).resolve().parents[1]
18
+ MAX_BYTES = 8192
19
+ MAX_FILES = 4096
20
+ MAX_FILE_BYTES = 4 * 1024 * 1024
21
+ MAX_SOURCE_BYTES = 64 * 1024 * 1024
22
+ SOURCE_SECONDS = 5
23
+ COMPILE_SECONDS = 30
24
+ MAX_CODES = 131072
25
+ PREFIX = "[sheet-worker-attestation] "
26
+ MANDATORY = ("api/sheet_warm.py", "api/process_role.py", "api/sync_runtime.py",
27
+ "platform/core/refresh_proof.py", "api/worker_provenance.py", "api/sync_worker.py")
28
+ _BOOT_SOURCE = None
29
+ _BOOT_FILES = {}
30
+ _BOOT_CODES = {}
31
+
32
+
33
+ def encoded(value):
34
+ return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
35
+ allow_nan=False).encode("utf-8")
36
+
37
+
38
+ def sha(raw):
39
+ return hashlib.sha256(raw).hexdigest()
40
+
41
+
42
+ def bounded(value):
43
+ if len(encoded(value)) + len(PREFIX) + 1 > MAX_BYTES:
44
+ raise ValueError("worker_attestation_capacity")
45
+ return value
46
+
47
+
48
+ def read_file(path, limit):
49
+ """Read one regular file without following a final symlink."""
50
+ descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
51
+ with os.fdopen(descriptor, "rb") as stream:
52
+ before = os.fstat(stream.fileno())
53
+ if not stat.S_ISREG(before.st_mode) or before.st_size > limit:
54
+ raise ValueError("worker_source_file_capacity_or_type")
55
+ raw = stream.read(limit + 1)
56
+ after = os.fstat(stream.fileno())
57
+ if len(raw) > limit or (before.st_size, before.st_mtime_ns, before.st_ino) != (
58
+ after.st_size, after.st_mtime_ns, after.st_ino):
59
+ raise ValueError("worker_source_read_drift")
60
+ return raw
61
+
62
+
63
+ def source_summary(files, total):
64
+ if not set(MANDATORY) <= files.keys() or len(files) > MAX_FILES or total > MAX_SOURCE_BYTES:
65
+ raise ValueError("worker_source_missing_or_capacity")
66
+ return {"files": {path: files[path] for path in MANDATORY},
67
+ "manifestSha256": sha(encoded(files)), "fileCount": len(files), "bytes": total}
68
+
69
+
70
+ def source_files(root=None):
71
+ """Read the complete bounded installed source inventory."""
72
+ root = Path(root or ROOT)
73
+ if root.is_symlink() or not root.is_dir():
74
+ raise ValueError("worker_source_root_invalid")
75
+ version_raw = read_file(root / "VERSION", 128)
76
+ version = version_raw.decode("ascii").strip()
77
+ if not re.fullmatch(r"[0-9a-f]{7,40}", version):
78
+ raise ValueError("worker_installed_version_missing")
79
+ started, files, total = time.monotonic(), {"VERSION": sha(version_raw)}, len(version_raw)
80
+ entries_seen = 0
81
+ for area in ("api", "platform"):
82
+ directory = root / area
83
+ if directory.is_symlink() or not directory.is_dir():
84
+ raise ValueError("worker_source_directory_missing")
85
+ for parent, dirs, names in os.walk(directory, followlinks=False):
86
+ entries_seen += len(dirs) + len(names)
87
+ if entries_seen > MAX_FILES * 4 or time.monotonic() - started > SOURCE_SECONDS:
88
+ raise ValueError("worker_source_tree_capacity_or_deadline")
89
+ for name in dirs:
90
+ if (Path(parent) / name).is_symlink():
91
+ raise ValueError("worker_source_symlink")
92
+ for name in sorted(names):
93
+ if not name.endswith(".py"):
94
+ continue
95
+ path = Path(parent) / name
96
+ if path.is_symlink() or time.monotonic() - started > SOURCE_SECONDS:
97
+ raise ValueError("worker_source_symlink_or_deadline")
98
+ if len(files) >= MAX_FILES:
99
+ raise ValueError("worker_source_capacity")
100
+ raw = read_file(path, min(MAX_FILE_BYTES, MAX_SOURCE_BYTES - total))
101
+ total += len(raw)
102
+ files[path.relative_to(root).as_posix()] = sha(raw)
103
+ return version, files, total
104
+
105
+
106
+ def installed_source(root=None):
107
+ """Hash every installed API/platform Python path and the installed VERSION."""
108
+ version, files, total = source_files(root)
109
+ return version, source_summary(files, total)
110
+
111
+
112
+ def code_identity(code):
113
+ return code.replace(co_filename="", co_consts=tuple(
114
+ code_identity(value) if isinstance(value, CodeType) else value for value in code.co_consts))
115
+
116
+
117
+ def function_codes(function, owner=None):
118
+ """Retain decorator origins and attribute dataclass-generated methods to their stdlib factory."""
119
+ seen = set()
120
+ while function is not None:
121
+ if id(function) in seen or len(seen) >= 32:
122
+ raise ValueError("worker_loaded_wrapper_capacity")
123
+ seen.add(id(function))
124
+ if isinstance(function, FunctionType):
125
+ code = function.__code__
126
+ generated = (owner is not None and dataclasses.is_dataclass(owner) and code.co_filename == "<string>"
127
+ and code.co_qualname == "__create_fn__.<locals>." + function.__name__
128
+ and function.__name__ in ("__init__", "__repr__", "__eq__", "__lt__", "__le__",
129
+ "__gt__", "__ge__", "__hash__", "__setattr__", "__delattr__"))
130
+ yield dataclasses._create_fn.__code__ if generated else code
131
+ function = getattr(function, "__wrapped__", None)
132
+
133
+
134
+ def definitions(module):
135
+ """Yield Python definitions owned by a module, including adapter methods."""
136
+ for value in tuple(vars(module).values()):
137
+ if ((isinstance(value, FunctionType) or type(value).__module__ == "functools")
138
+ and getattr(value, "__module__", None) == module.__name__):
139
+ yield from function_codes(value)
140
+ elif isinstance(value, type) and value.__module__ == module.__name__:
141
+ for member in tuple(vars(value).values()):
142
+ if isinstance(member, (staticmethod, classmethod)):
143
+ member = member.__func__
144
+ members = (member.fget, member.fset, member.fdel) if isinstance(member, property) else (member,)
145
+ for function in members:
146
+ if isinstance(function, FunctionType) and function.__module__ == module.__name__:
147
+ yield from function_codes(function, value)
148
+
149
+
150
+ def verified_code(code):
151
+ """Verify application code or a known stdlib decorator factory without trusting a filename alone."""
152
+ for library, factory in ((contextlib, contextlib.contextmanager), (contextlib, contextlib.asynccontextmanager),
153
+ (dataclasses, dataclasses._create_fn),
154
+ (dataclasses, dataclasses._recursive_repr)):
155
+ if code.co_filename != library.__file__:
156
+ continue
157
+ if (library.__spec__.origin != library.__file__
158
+ or Path(library.__file__).parent != Path(os.__file__).parent
159
+ or not Path(library.__file__).is_file() or Path(library.__file__).is_symlink()):
160
+ raise ValueError("worker_stdlib_factory_origin")
161
+ pending = [factory.__code__]
162
+ while pending:
163
+ original = pending.pop()
164
+ if code == original:
165
+ return
166
+ pending.extend(value for value in original.co_consts if isinstance(value, CodeType))
167
+ path = source_path(code.co_filename)
168
+ if code_identity(code) not in _BOOT_CODES[path]:
169
+ raise ValueError("worker_loaded_code_drift")
170
+
171
+
172
+ def source_path(path):
173
+ """Require an existing source path inside the installed tree without symlink components."""
174
+ path, root = Path(path).absolute(), ROOT.absolute()
175
+ relative = path.relative_to(root)
176
+ if any((root.joinpath(*relative.parts[:i])).is_symlink() for i in range(len(relative.parts) + 1)):
177
+ raise ValueError("worker_loaded_origin_symlink")
178
+ if relative.as_posix() not in _BOOT_FILES or path.suffix != ".py" or not path.is_file():
179
+ raise ValueError("worker_loaded_source_missing")
180
+ return relative.as_posix()
181
+
182
+
183
+ def module_origin(module, expected=None):
184
+ """Bind a loaded module and its defining code to the pinned installed inventory."""
185
+ path = source_path(module.__file__)
186
+ spec = getattr(module, "__spec__", None)
187
+ if ((spec is None and module.__name__ != "__main__")
188
+ or (spec is not None and source_path(spec.origin) != path) or (expected and expected != path)):
189
+ raise ValueError("worker_loaded_origin_mismatch")
190
+ if sha(read_file(ROOT / path, MAX_FILE_BYTES)) != _BOOT_FILES[path]:
191
+ raise ValueError("worker_loaded_source_drift")
192
+ for code in definitions(module):
193
+ verified_code(code)
194
+ return path
195
+
196
+
197
+ def loaded_origins():
198
+ """Check application modules, namespaces and aliases without adding volatile identity fields."""
199
+ names, directories, started = {}, {}, time.monotonic()
200
+ for path in _BOOT_CODES:
201
+ area, relative = path.split("/", 1)
202
+ parts = relative.removesuffix(".py").split("/")
203
+ if parts[-1] == "__init__":
204
+ parts.pop()
205
+ if parts:
206
+ names[".".join(parts)] = path
207
+ names[area + "." + ".".join(parts)] = path
208
+ parents = relative.split("/")[:-1]
209
+ for i in range(len(parents) + 1):
210
+ directory = area + ("/" + "/".join(parents[:i]) if i else "")
211
+ directories[directory.replace("/", ".")] = directory
212
+ if i:
213
+ directories[".".join(parents[:i])] = directory
214
+ signatures = set().union(*_BOOT_CODES.values())
215
+ modules = tuple(sys.modules.items())
216
+ if len(modules) > MAX_FILES * 4:
217
+ raise ValueError("worker_loaded_module_capacity")
218
+ for name, module in modules:
219
+ if time.monotonic() - started > SOURCE_SECONDS:
220
+ raise ValueError("worker_loaded_origin_deadline")
221
+ if not isinstance(module, ModuleType):
222
+ continue
223
+ spec = getattr(module, "__spec__", None)
224
+ labels = (name, module.__name__, getattr(spec, "name", ""))
225
+ expected = next((names[label] for label in labels if label in names), None)
226
+ namespace = next((directories[label] for label in labels if label in directories), None)
227
+ paths = [getattr(module, "__file__", None), getattr(spec, "origin", None)]
228
+ rooted = any(isinstance(path, str) and Path(path).is_relative_to(ROOT) for path in paths)
229
+ owned = tuple(definitions(module))
230
+ alias = any(code_identity(code) in signatures for code in owned)
231
+ # Python's platform.py is a stdlib module, not the application's platform namespace.
232
+ if not (expected or rooted or alias or (namespace and hasattr(module, "__path__"))):
233
+ continue
234
+ if not paths[0] and namespace and spec is not None and spec.origin is None:
235
+ locations = tuple(getattr(spec, "submodule_search_locations", ()) or ())
236
+ if not locations or set(locations) != set(module.__path__):
237
+ raise ValueError("worker_loaded_namespace_unproven")
238
+ for location in locations:
239
+ directory = Path(location).absolute()
240
+ if (directory != ROOT / namespace or not directory.is_dir()
241
+ or directory.resolve() != directory):
242
+ raise ValueError("worker_loaded_namespace_origin")
243
+ continue
244
+ module_origin(module, expected)
245
+ frame, depth = sys._getframe(), 0
246
+ try:
247
+ while frame is not None:
248
+ depth += 1
249
+ if depth > 128:
250
+ raise ValueError("worker_loaded_stack_capacity")
251
+ name = frame.f_globals.get("__name__", "")
252
+ code, spec = frame.f_code, frame.f_globals.get("__spec__")
253
+ signature = code_identity(code)
254
+ rooted = Path(code.co_filename).is_relative_to(ROOT)
255
+ if name in names or rooted or signature in signatures:
256
+ path = source_path(frame.f_globals.get("__file__"))
257
+ if (source_path(code.co_filename) != path or (spec is None and name != "__main__")
258
+ or (spec is not None and source_path(spec.origin) != path)
259
+ or (name in names and names[name] != path) or signature not in _BOOT_CODES[path]):
260
+ raise ValueError("worker_executing_origin_mismatch")
261
+ frame = frame.f_back
262
+ finally:
263
+ del frame
264
+
265
+
266
+ def boot_source():
267
+ """Pin installed bytes once before worker services start, including an unproven boot."""
268
+ global _BOOT_SOURCE, _BOOT_FILES, _BOOT_CODES
269
+ if _BOOT_SOURCE is None:
270
+ try:
271
+ path = Path(__file__)
272
+ if path.is_symlink() or path.parent.is_symlink():
273
+ raise ValueError("worker_source_symlink")
274
+ version, _BOOT_FILES, total = source_files()
275
+ _BOOT_CODES = {}
276
+ started, code_count = time.monotonic(), 0
277
+ for relative, digest in _BOOT_FILES.items():
278
+ if not relative.endswith(".py"):
279
+ continue
280
+ raw = read_file(ROOT / relative, MAX_FILE_BYTES)
281
+ if sha(raw) != digest or time.monotonic() - started > COMPILE_SECONDS:
282
+ raise ValueError("worker_boot_code_drift_or_deadline")
283
+ codes, pending = set(), [compile(raw, str(ROOT / relative), "exec", dont_inherit=True)]
284
+ while pending:
285
+ code_count += 1
286
+ if code_count > MAX_CODES or time.monotonic() - started > COMPILE_SECONDS:
287
+ raise ValueError("worker_boot_code_capacity_or_deadline")
288
+ code = pending.pop()
289
+ codes.add(code_identity(code))
290
+ pending.extend(value for value in code.co_consts if isinstance(value, CodeType))
291
+ _BOOT_CODES[relative] = codes
292
+ loaded_origins()
293
+ _BOOT_SOURCE = version, source_summary(_BOOT_FILES, total)
294
+ except Exception:
295
+ _BOOT_SOURCE = False
296
+ return _BOOT_SOURCE
297
+
298
+
299
+ def store_identity(rt):
300
+ """Identify the adapter in use and fingerprint a checked live PostgreSQL connection."""
301
+ if not boot_source():
302
+ raise ValueError("worker_adapter_source_unproven")
303
+ tenant = str(getattr(rt, "key", ""))
304
+ backend = rt._document_store() if callable(getattr(rt, "_document_store", None)) else rt
305
+ cls = type(backend)
306
+ defining = sys.modules.get(cls.__module__)
307
+ if not isinstance(defining, ModuleType):
308
+ raise ValueError("worker_adapter_module_missing")
309
+ adapter_path = module_origin(defining)
310
+ if source_path(inspect.getfile(cls)) != adapter_path:
311
+ raise ValueError("worker_adapter_origin_mismatch")
312
+ for member in vars(cls).values():
313
+ if isinstance(member, FunctionType) and (source_path(member.__code__.co_filename) != adapter_path
314
+ or code_identity(member.__code__) not in _BOOT_CODES[adapter_path]):
315
+ raise ValueError("worker_adapter_code_drift")
316
+ # An installed fixture remains memory-backed even when DATABASE_URL is configured.
317
+ fake = sys.modules.get("_fake_store")
318
+ if fake is not None and cls is getattr(fake, "FakeStore", None):
319
+ return {"backend": "memory", "tenant": tenant,
320
+ "adapter": cls.__module__ + "." + cls.__qualname__,
321
+ "adapterSha256": sha(read_file(Path(inspect.getfile(cls)), MAX_FILE_BYTES))}
322
+ from core import store_pg
323
+ from psycopg.conninfo import conninfo_to_dict
324
+ if cls is not store_pg.PgStore or backend.tenant_slug != tenant:
325
+ raise ValueError("worker_active_store_unproven")
326
+ store_pg.check_write(tenant, "worker provenance")
327
+ expected = conninfo_to_dict(store_pg._url() or "")
328
+ if not all(expected.get(key) for key in ("host", "port", "dbname")):
329
+ raise ValueError("worker_active_endpoint_missing")
330
+ with store_pg._pool().connection(timeout=1.0) as connection:
331
+ info = connection.info
332
+ actual = {key: str(getattr(info, key)) for key in ("host", "port", "dbname")}
333
+ if any(actual[key] != str(expected[key]) for key in actual) or not connection.pgconn.ssl_in_use:
334
+ raise ValueError("worker_active_endpoint_mismatch")
335
+ return {"backend": "pg", "tenant": tenant, "hostSha256": sha(actual["host"].lower().encode())[:12],
336
+ "port": int(actual["port"]), "databaseSha256": sha(actual["dbname"].encode()),
337
+ "schemaSha256": sha(store_pg._schema_for(tenant).encode()), "tls": True}
338
+
339
+
340
+ def evidence(rt):
341
+ """Return fresh bounded evidence or None without affecting worker health."""
342
+ try:
343
+ import process_role
344
+ from core import data_binding
345
+ tenant, job = str(getattr(rt, "key", "")), os.environ.get("JOB_ID", "")
346
+ space = data_binding.deployment_id()
347
+ dataset = os.environ.get("OS_DATA_REPO", "")
348
+ if (process_role.role() != "worker" or tenant != process_role.writer_tenant()
349
+ or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", job)
350
+ or not re.fullmatch(r"[A-Za-z0-9_-]+/[A-Za-z0-9_-]+", space)
351
+ or os.environ.get("SPACE_ID", space) != space
352
+ or (os.environ.get("RI_DIR") and Path(os.environ["RI_DIR"]).resolve() != (ROOT / "platform").resolve())
353
+ or (dataset and not re.fullmatch(r"[A-Za-z0-9_-]+/[A-Za-z0-9_.-]+", dataset))
354
+ or os.environ.get("AIOS_SYNC_JOB_MODE") != "resident"):
355
+ return None
356
+ boot = boot_source()
357
+ if not boot:
358
+ return None
359
+ current = installed_source()
360
+ if current != boot:
361
+ return None
362
+ version, source = current
363
+ loaded_origins()
364
+ store = store_identity(rt)
365
+ loaded_origins()
366
+ if installed_source() != boot:
367
+ return None
368
+ return bounded({"at": time.time(), "pid": os.getpid(), "spaceId": space, "jobId": job,
369
+ "tenant": tenant, "version": version, "source": source, "store": store,
370
+ "role": process_role.role(), "scope": process_role.worker_scope(),
371
+ "deployment": data_binding.deployment_id(), "mode": "resident",
372
+ "dataset": dataset,
373
+ "policy": {"productionOverride": str(os.environ.get("AIOS_ALLOW_PRODUCTION_DATA", "")).strip().lower()
374
+ not in ("", "0", "false", "no"),
375
+ "directDatabase": bool(os.environ.get("DATABASE_URL_DIRECT")),
376
+ "executableOverride": any((os.environ.get("ENV"), os.environ.get("BASH_ENV"),
377
+ os.environ.get("PYTHONPATH"), os.environ.get("PYTHONHOME"),
378
+ os.environ.get("LD_PRELOAD"), os.environ.get("LD_LIBRARY_PATH")))}})
379
+ except Exception:
380
+ return None
381
+
382
+
383
+ def stable(value):
384
+ return {key: item for key, item in value.items() if key != "at"} if value else None
385
+
386
+
387
+ def desk_companion(rt, capture, raw, before):
388
+ """Bind this original capture to unchanged source and active-worker evidence."""
389
+ try:
390
+ import process_role
391
+ if not before:
392
+ return None
393
+ current = evidence(rt)
394
+ if not current or stable(before) != stable(current):
395
+ return None
396
+ stored = process_role.stored_worker(rt)
397
+ if ((stored.get("job") or {}).get("id") != current["jobId"]
398
+ or (stored.get("job") or {}).get("tenant") != current["tenant"]
399
+ or capture.get("version") != current["version"]):
400
+ return None
401
+ return bounded({**current, "at": time.time(), "schema": "sheet-worker-attestation/v1", "resultSha256": sha(raw),
402
+ **{key: capture[key] for key in ("requestId", "executionId", "tenant", "reader", "workbook")}})
403
+ except Exception:
404
+ return None
405
+
406
+
407
+ def heartbeat_attestation(rt, beat):
408
+ try:
409
+ value = evidence(rt)
410
+ bounded(beat)
411
+ return bounded({**value, "schema": "sync-worker-attestation/v1", "heartbeatSha256": sha(encoded(beat))}) if value else None
412
+ except Exception:
413
+ return None
414
+
415
+
416
+ def visible(stored):
417
+ """Expose proof only for the same fresh stored heartbeat and job."""
418
+ try:
419
+ value, beat, job = stored["attestation"], stored["heartbeat"], stored["job"]
420
+ bounded(value)
421
+ age = time.time() - beat["at"]
422
+ if (value["jobId"] == job["id"] and value["tenant"] == job["tenant"] == beat["tenant"]
423
+ and value["pid"] == beat["pid"] and 0 <= age < 120
424
+ and value["heartbeatSha256"] == sha(encoded(beat))):
425
+ return value
426
+ except Exception:
427
+ pass
428
+ return None
platform/harness/datastore.py CHANGED
@@ -804,6 +804,14 @@ def _odoo_mirror_store():
804
  def _owns_state(self, entity):
805
  return entity.split(".", 1)[0] in self.spec.tables
806
 
 
 
 
 
 
 
 
 
807
  def _validated_snapshot(self, tenant_key, rt, con, raw, tmp, **kwargs):
808
  # Odoo never advertises a partial restore or replaces it with a seed.
809
  snapshot = super()._validated_snapshot(tenant_key, rt, con, raw, tmp,
 
804
  def _owns_state(self, entity):
805
  return entity.split(".", 1)[0] in self.spec.tables
806
 
807
+ def _restore_complete(self, tables, rows, con=None):
808
+ """Require live sync entities and installed receipts for their required column backfills."""
809
+ entities = set(tables) & set(self.spec.tables)
810
+ return (con is not None and super()._restore_complete(entities, rows)
811
+ and all(row[1] == "live" for row in rows if row[0] in entities)
812
+ and all(_required_columns_ready(con, key, ENTITIES.get(key) or {})
813
+ for key in entities))
814
+
815
  def _validated_snapshot(self, tenant_key, rt, con, raw, tmp, **kwargs):
816
  # Odoo never advertises a partial restore or replaces it with a seed.
817
  snapshot = super()._validated_snapshot(tenant_key, rt, con, raw, tmp,
platform/harness/mirror_store.py CHANGED
@@ -547,6 +547,12 @@ class MirrorStore:
547
  def _state_complete(self, rows):
548
  return True
549
 
 
 
 
 
 
 
550
  def _log_restored(self, paths, total, rows, log):
551
  log(f"{self.label} mirror restored from the store: {len(paths)} tables, {total:,} rows")
552
 
@@ -886,9 +892,7 @@ class MirrorStore:
886
  # Trap: the healed local rows differ from the published snapshot until the
887
  # next publish; each restore heals again and a second heal changes nothing.
888
  self._heal_published(con, result, log)
889
- table_states = {row[0]: row[1] for row in rows if row[0] in paths}
890
- current = (not refused and bool(paths) and set(paths) <= set(table_states)
891
- and all(row[1] in {"live", "done"} for row in rows))
892
  newest = _record_newest(con, self.connector)
893
  with _CATCH_UP_LOCK:
894
  _catch_up_source(rt, self.connector).update(
 
547
  def _state_complete(self, rows):
548
  return True
549
 
550
+ def _restore_complete(self, tables, rows, con=None):
551
+ """Require a finished marker for every installed table unless its connector owns another policy."""
552
+ table_states = {row[0]: row[1] for row in rows if row[0] in tables}
553
+ return (bool(tables) and set(tables) <= set(table_states)
554
+ and all(row[1] in {"live", "done"} for row in rows))
555
+
556
  def _log_restored(self, paths, total, rows, log):
557
  log(f"{self.label} mirror restored from the store: {len(paths)} tables, {total:,} rows")
558
 
 
892
  # Trap: the healed local rows differ from the published snapshot until the
893
  # next publish; each restore heals again and a second heal changes nothing.
894
  self._heal_published(con, result, log)
895
+ current = not refused and self._restore_complete(paths, rows, con=con)
 
 
896
  newest = _record_newest(con, self.connector)
897
  with _CATCH_UP_LOCK:
898
  _catch_up_source(rt, self.connector).update(
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 5961efa485d5f97b5e51b75dbb137cc7e396465d94dfb66eff86cc268de083d7
 
1
+ 3146b695594322bf63b85303beb8150a938e2e80648416de20fbee2970e1bdb0