fsanyoto commited on
Commit
ed522b3
·
verified ·
1 Parent(s): 9ad4b87

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "29a7652e48b",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "6ca559b5c8c",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 29a7652e48b
 
1
+ 6ca559b5c8c
platform/harness/amazon_store.py CHANGED
@@ -1741,6 +1741,54 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
1741
  def _connect(self, tenant_key, rt):
1742
  return _connect_tenant_mirror(tenant_key, rt)
1743
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1744
  def _before_restore(self, tenant_key, rt):
1745
  obligation = retention_check(tenant_key, rt=rt, raise_overdue=True)
1746
  if obligation["earliestTriggerAt"]:
@@ -3098,9 +3146,34 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
3098
  expected.append("settlements")
3099
  with _mirrors.sync_mirror("amazon", tenant_key, rt, report=report, log=log, expected_tables=expected):
3100
  report.update(_sync_candidate(tenant_key, **options))
 
3101
  return report
3102
 
3103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3104
  def _sync_credential(tenant_key, rt):
3105
  """Refuse unusable credentials before storage or vendor work at either sync entry."""
3106
  fields = credential(tenant_key, rt)
 
1741
  def _connect(self, tenant_key, rt):
1742
  return _connect_tenant_mirror(tenant_key, rt)
1743
 
1744
+ def _publication_holds(self, report, waiting):
1745
+ """Publish what a refresh pass completed; hold each unfinished table at its previous rows.
1746
+
1747
+ A first-load pass (it reports a state) keeps SDLC 73's whole-source publication. A problem no table owns
1748
+ holds the pass.
1749
+ """
1750
+ if report.get("state") in {"partial", "failed", "done"}:
1751
+ return None
1752
+ rows = report.get("tables") or {}
1753
+ unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
1754
+ unfinished |= {name for name, row in rows.items() if isinstance(row, dict) and row.get("state") != "disabled"
1755
+ and not _mirrors._table_sync_complete("amazon", row)}
1756
+ # Head and lines come from one document, so neither publishes without the other.
1757
+ pairs = {**SHARED_REPORT_SIBLINGS, **{line: head for head, line in SHARED_REPORT_SIBLINGS.items()}}
1758
+ unfinished |= {pairs[name] for name in unfinished if name in pairs}
1759
+ owners = {key for name in set(rows) | unfinished for key in (name, table_name(name))}
1760
+ blocked = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []
1761
+ if str(problem).split(":", 1)[0].strip() not in owners]
1762
+ held = {}
1763
+ for name in sorted(unfinished):
1764
+ row = rows.get(name) if isinstance(rows.get(name), dict) else {}
1765
+ state = " ".join(str(part) for part in (row.get("state") or "not reached in this pass",
1766
+ row.get("amazonStatus")) if part)
1767
+ held[name] = state + (f": {row['reason']}" if row.get("reason") else "")
1768
+ return {"held": held, "tables": {table_name(name) for name in held}, "blocked": blocked}
1769
+
1770
+ def _held_state_rows(self, rows, tables):
1771
+ """A held table's data markers: watermark, floor, proofs, limits, role results and documents.
1772
+
1773
+ Its status, tick, refusal and pending-report markers describe the pass, so they publish with it.
1774
+ """
1775
+ tables = set(tables)
1776
+ names = sorted({table_name(t) for t in TABLES} | tables, key=lambda name: (-len(name), name))
1777
+
1778
+ def owner(entity):
1779
+ if entity.startswith(entity_key("settlement_doc_")):
1780
+ return table_name("settlements")
1781
+ for family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "reconcile_", "limit_"):
1782
+ for name in names:
1783
+ marker = entity_key(family + name[len(PREFIX):])
1784
+ if entity == marker or entity.startswith(marker + "_"):
1785
+ return name
1786
+ if entity in names:
1787
+ return None
1788
+ return next((name for name in names if entity.startswith(name + "_")), None)
1789
+
1790
+ return [row for row in rows if owner(str(row[0])) in tables]
1791
+
1792
  def _before_restore(self, tenant_key, rt):
1793
  obligation = retention_check(tenant_key, rt=rt, raise_overdue=True)
1794
  if obligation["earliestTriggerAt"]:
 
3146
  expected.append("settlements")
3147
  with _mirrors.sync_mirror("amazon", tenant_key, rt, report=report, log=log, expected_tables=expected):
3148
  report.update(_sync_candidate(tenant_key, **options))
3149
+ _refuse_unreconciled(tenant_key, rt, report, log)
3150
  return report
3151
 
3152
 
3153
+ def _refuse_unreconciled(tenant_key, rt, report, log):
3154
+ """Standing rule 8: a table synced in this pass whose reconciliation refuses keeps its previous rows."""
3155
+ synced = [table for table, row in (report.get("tables") or {}).items()
3156
+ if isinstance(row, dict) and row.get("state", "done") == "done" and "rows" in row]
3157
+ if not synced or report.get("state") in {"partial", "failed", "done"}:
3158
+ return
3159
+ try:
3160
+ checked = validate(tenant_key, rt=rt, log=lambda *_: None)
3161
+ except Exception as exc: # noqa: BLE001
3162
+ report["problems"].append(f"Amazon reconciliation unavailable ({type(exc).__name__}); "
3163
+ "this pass publishes nothing new until it runs.")
3164
+ log(report["problems"][-1])
3165
+ return
3166
+ con, _path = _connect_tenant_mirror(tenant_key, rt)
3167
+ for table in synced:
3168
+ if (checked.get(table) or {}).get("state") != "refused":
3169
+ continue
3170
+ reason = "reconciliation refused: " + str(checked[table].get("reason") or "no reason given")
3171
+ report["tables"][table] = {**report["tables"][table], "state": "refused", "reason": reason}
3172
+ report["problems"].append(f"{table}: {reason}")
3173
+ _record_state(con, entity_key(table), "refused", 0, reason)
3174
+ log(f"{table}: {reason}")
3175
+
3176
+
3177
  def _sync_credential(tenant_key, rt):
3178
  """Refuse unusable credentials before storage or vendor work at either sync entry."""
3179
  fields = credential(tenant_key, rt)
platform/harness/mirror_store.py CHANGED
@@ -139,7 +139,12 @@ def _sync_result(connector, report):
139
  for row in report.values())
140
  return completed, completed and all(row["phase"] == "live" for row in report.values())
141
  if report.get("problems") or report.get("audit_problems"):
142
- return False, False
 
 
 
 
 
143
  rows = report.get("tables") or {}
144
  if connector == "meta":
145
  completed = bool(rows) and not any(row.get("capped") for row in rows.values())
@@ -165,7 +170,8 @@ def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwa
165
  completed, current = _sync_result(connector, report)
166
  current = current and _catch_up_complete and Path(rt.datastore_path).exists()
167
  newest = None
168
- if current:
 
169
  try:
170
  con, _ = datastore._mirror_connection(rt.key, rt)
171
  try:
@@ -179,6 +185,8 @@ def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwa
179
  state = _catch_up_source(rt, connector)
180
  if completed:
181
  state["passes"] += 1
 
 
182
  if current and not state["refused"]:
183
  state.update(current=True, pass_pending=False, newest=newest or {})
184
  return report
@@ -300,6 +308,15 @@ class MirrorStore:
300
  def _log_restored(self, paths, total, rows, log):
301
  log(f"{self.label} mirror restored from the store: {len(paths)} tables, {total:,} rows")
302
 
 
 
 
 
 
 
 
 
 
303
  def _mirror_cache_key(self, tenant_key, rt):
304
  key = _tenant_key(tenant_key)
305
  if rt is None or _tenant_key(getattr(rt, "key", None)) != key:
@@ -1044,6 +1061,41 @@ def _open_candidate(store, tenant_key, rt, published):
1044
  raise
1045
 
1046
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1047
  @contextmanager
1048
  def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_publish=None,
1049
  expected_tables=None):
@@ -1080,6 +1132,7 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1080
  published, _path = store._connect(tenant_key, rt)
1081
  candidate = None
1082
  installed = False
 
1083
  save_started = None
1084
  try:
1085
  opened = time.monotonic()
@@ -1107,20 +1160,31 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1107
  or (row.get("state") == "disabled" and name not in pending))}
1108
  if done:
1109
  candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", [[name] for name in done])
1110
- if candidate.execute("SELECT 1 FROM _candidate_pending LIMIT 1").fetchone():
 
 
 
1111
  if odoo:
1112
  note("held, tables still pending", names=lambda: [row[0] for row in candidate.execute(
1113
  "SELECT name FROM _candidate_pending").fetchall()])
1114
  return
1115
  if report is not None:
1116
  completed, current = _sync_result(store.connector, report)
1117
- if not (completed if store.connector == "meta" else current):
1118
  if odoo:
1119
  note(f"held, the pass is not current (completed={completed}), entities not live",
1120
  names=lambda: [f"{name}={row.get('phase') if isinstance(row, dict) else row}"
1121
  for name, row in report.items()
1122
  if not (isinstance(row, dict) and row.get("phase") == "live")])
 
 
 
 
1123
  return
 
 
 
 
1124
  if store.connector == "odoo" and any(row[1] not in {"live", "done"}
1125
  for row in store._state_rows(candidate)):
1126
  note("held, sync state not live or done", names=lambda: [
@@ -1182,6 +1246,13 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1182
  raise
1183
  if odoo:
1184
  note("install", installing)
 
 
 
 
 
 
 
1185
  finally:
1186
  if save_started is not None and store.connector == "odoo":
1187
  status = datastore.ODOO_MIRROR_STATUS.setdefault(str(Path(rt.datastore_path).resolve()), {})
@@ -1189,7 +1260,7 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1189
  print(f"[datastore] Odoo mirror save {tenant_key}: {status['save']}")
1190
  if candidate is not None:
1191
  candidate.close()
1192
- if installed:
1193
  path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
1194
  path.unlink(missing_ok=True)
1195
  published.close()
 
139
  for row in report.values())
140
  return completed, completed and all(row["phase"] == "live" for row in report.values())
141
  if report.get("problems") or report.get("audit_problems"):
142
+ # A problem owned by one held table publishes the rest; it never makes the pass current.
143
+ holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None
144
+ if holds is None or holds["blocked"]:
145
+ return False, False
146
+ rows = report.get("tables") or {}
147
+ return bool(rows) and any(_table_sync_complete(connector, row) for row in rows.values()), False
148
  rows = report.get("tables") or {}
149
  if connector == "meta":
150
  completed = bool(rows) and not any(row.get("capped") for row in rows.values())
 
170
  completed, current = _sync_result(connector, report)
171
  current = current and _catch_up_complete and Path(rt.datastore_path).exists()
172
  newest = None
173
+ # An Amazon pass that published part of the source still moves the measured record dates.
174
+ if current or (completed and connector == "amazon" and Path(rt.datastore_path).exists()):
175
  try:
176
  con, _ = datastore._mirror_connection(rt.key, rt)
177
  try:
 
185
  state = _catch_up_source(rt, connector)
186
  if completed:
187
  state["passes"] += 1
188
+ if newest and not state["refused"]:
189
+ state["newest"] = newest
190
  if current and not state["refused"]:
191
  state.update(current=True, pass_pending=False, newest=newest or {})
192
  return report
 
308
  def _log_restored(self, paths, total, rows, log):
309
  log(f"{self.label} mirror restored from the store: {len(paths)} tables, {total:,} rows")
310
 
311
+ def _publication_holds(self, report, waiting):
312
+ """None publishes the whole source or nothing. A connector may return
313
+ {"held": {name: reason}, "tables": held mirror tables, "blocked": [problems that hold the pass]}."""
314
+ return None
315
+
316
+ def _held_state_rows(self, rows, tables):
317
+ """The state rows that describe a held table's data, so they ride with its previous rows."""
318
+ return [row for row in rows if row[0] in tables or row[0].split(".", 1)[0] in tables]
319
+
320
  def _mirror_cache_key(self, tenant_key, rt):
321
  key = _tenant_key(tenant_key)
322
  if rt is None or _tenant_key(getattr(rt, "key", None)) != key:
 
1061
  raise
1062
 
1063
 
1064
+ def _hold_previous(store, candidate, published, tables):
1065
+ """Give each held table its published rows and data markers, inside the candidate's open transaction."""
1066
+ prior = store._tables(published)
1067
+ with tempfile.TemporaryDirectory(prefix="mirror-held-") as tmp:
1068
+ for name in sorted(tables):
1069
+ if name not in prior:
1070
+ candidate.execute(f"DROP TABLE IF EXISTS {_quoted_identifier(name)}")
1071
+ continue
1072
+ path = Path(tmp) / (name + ".parquet")
1073
+ published.execute(f"COPY {_quoted_identifier(name)} TO ? (FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
1074
+ candidate.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
1075
+ f"SELECT * FROM {_parquet_source([path])}")
1076
+ private = store._held_state_rows(store._state_rows(candidate), tables)
1077
+ if private:
1078
+ candidate.executemany("DELETE FROM _sync_state WHERE entity=?", [[row[0]] for row in private])
1079
+ previous = store._held_state_rows(store._state_rows(published), tables)
1080
+ if previous:
1081
+ candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous)
1082
+
1083
+
1084
+ def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None):
1085
+ """One log line: what a pass published and what it held, each with its reason."""
1086
+ rows = report.get("tables") or {}
1087
+ synced = sorted(name for name in done if "rows" in (rows.get(name) or {}))
1088
+ synced_text = ", ".join(f"{name} {rows[name]['rows']:,} rows" if isinstance(rows[name]["rows"], int)
1089
+ else name for name in synced) or "none"
1090
+ held = "; ".join(f"{name} ({str(reason)[:160]})" for name, reason in sorted(holds["held"].items())) or "none"
1091
+ if snapshot_id:
1092
+ text = (f"{store.label} publication {snapshot_id}: published {len(synced)} synced ({synced_text}) and "
1093
+ f"{len(set(done) - set(synced))} current; held {len(holds['held'])} at their previous rows: {held}")
1094
+ else:
1095
+ text = f"{store.label} publication held, the previous snapshot stays: {why}; unfinished: {held}"
1096
+ return text.replace("—", "-").replace("–", "-").replace("\n", " ").replace("\r", " ")
1097
+
1098
+
1099
  @contextmanager
1100
  def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_publish=None,
1101
  expected_tables=None):
 
1132
  published, _path = store._connect(tenant_key, rt)
1133
  candidate = None
1134
  installed = False
1135
+ kept = False
1136
  save_started = None
1137
  try:
1138
  opened = time.monotonic()
 
1160
  or (row.get("state") == "disabled" and name not in pending))}
1161
  if done:
1162
  candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", [[name] for name in done])
1163
+ # A connector policy may publish what completed and hold each unfinished table at its previous rows.
1164
+ holds = None if report is None or odoo else store._publication_holds(report, {
1165
+ row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()})
1166
+ if holds is None and candidate.execute("SELECT 1 FROM _candidate_pending LIMIT 1").fetchone():
1167
  if odoo:
1168
  note("held, tables still pending", names=lambda: [row[0] for row in candidate.execute(
1169
  "SELECT name FROM _candidate_pending").fetchall()])
1170
  return
1171
  if report is not None:
1172
  completed, current = _sync_result(store.connector, report)
1173
+ if not (completed if store.connector == "meta" or holds is not None else current):
1174
  if odoo:
1175
  note(f"held, the pass is not current (completed={completed}), entities not live",
1176
  names=lambda: [f"{name}={row.get('phase') if isinstance(row, dict) else row}"
1177
  for name, row in report.items()
1178
  if not (isinstance(row, dict) and row.get("phase") == "live")])
1179
+ if holds is not None:
1180
+ log(_publication_line(store, report, done, holds, why=(
1181
+ "a problem no table owns: " + "; ".join(str(p)[:200] for p in holds["blocked"][:3])
1182
+ if holds["blocked"] else "no table completed in this pass")))
1183
  return
1184
+ if holds is not None and holds["tables"]:
1185
+ # Rolled back after publication, so the candidate keeps each held table's private progress.
1186
+ candidate.execute("BEGIN TRANSACTION")
1187
+ _hold_previous(store, candidate, published, holds["tables"])
1188
  if store.connector == "odoo" and any(row[1] not in {"live", "done"}
1189
  for row in store._state_rows(candidate)):
1190
  note("held, sync state not live or done", names=lambda: [
 
1246
  raise
1247
  if odoo:
1248
  note("install", installing)
1249
+ if holds is not None and holds["tables"]:
1250
+ candidate.execute("ROLLBACK")
1251
+ candidate.execute("DELETE FROM _candidate_base")
1252
+ candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [manifest["snapshot_id"], started])
1253
+ kept = True
1254
+ if holds is not None:
1255
+ log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"]))
1256
  finally:
1257
  if save_started is not None and store.connector == "odoo":
1258
  status = datastore.ODOO_MIRROR_STATUS.setdefault(str(Path(rt.datastore_path).resolve()), {})
 
1260
  print(f"[datastore] Odoo mirror save {tenant_key}: {status['save']}")
1261
  if candidate is not None:
1262
  candidate.close()
1263
+ if installed and not kept:
1264
  path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
1265
  path.unlink(missing_ok=True)
1266
  published.close()
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 2f102ed1020fc34d07db65c051a23d76fd97fb3805bd7642fdd43afb5d39a456
 
1
+ df8d6ac42e6acaf4afc92e5742282a91ee0d63b5c1777613b9a0e9f701b7c1df