fsanyoto commited on
Commit
06f41c9
·
verified ·
1 Parent(s): 4b32bf5

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "28dea583f9",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "a11b7f74a",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 28dea583f9
 
1
+ a11b7f74a
platform/harness/mirror_store.py CHANGED
@@ -37,6 +37,8 @@ _KEY_LEDGER = "keys.json"
37
  _TABLE_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.parquet")
38
  _PART_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)\.p([0-9a-z-]{1,48})\.([0-9a-f]{16})(?:\.sealed-v1)?\.parquet")
39
  _STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.json")
 
 
40
  _PART_COLUMNS = ("_period_start", "month", "_snapshot_date", "posted_date", "purchase_date",
41
  "shipment_date", "date", "snapshot_date", "last_updated_date", "settlement_start_date")
42
  ORPHAN_GRACE = timedelta(hours=24)
@@ -726,10 +728,59 @@ class MirrorStore:
726
  return key
727
 
728
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
729
  def _is_mirror_blob_key(self, key):
730
  match = _TABLE_BLOB_RE.fullmatch(str(key)) or _PART_BLOB_RE.fullmatch(str(key))
731
  return bool((match and self._valid_table(match.group(1)))
732
- or _STATE_BLOB_RE.fullmatch(str(key)))
733
 
734
 
735
  def _manifest_keys(self, manifest, *, strict=True):
@@ -747,7 +798,8 @@ class MirrorStore:
747
  state = manifest.get("sync_state")
748
  if isinstance(state, dict):
749
  try:
750
- out[self._state_blob_key(state, strict=strict)] = state.get("sha256")
 
751
  except (ValueError, AttributeError, TypeError):
752
  pass
753
  return out
@@ -805,8 +857,9 @@ class MirrorStore:
805
  raise ValueError("Mirror publication identity or times are invalid")
806
  allow_partial = False
807
  state_meta = manifest["sync_state"]
808
- state_bytes = self._checked_mirror_blob(rt, self._state_blob_key(state_meta), state_meta)
809
- rows = json.loads(state_bytes)
 
810
  if (not isinstance(rows, list) or len(rows) != state_meta.get("rows")
811
  or not self._state_complete(rows) or any(not isinstance(row, list) or len(row) != 6
812
  or not isinstance(row[0], str) or not self._owns_state(row[0]) for row in rows)
@@ -1351,18 +1404,16 @@ class MirrorStore:
1351
  continue
1352
  pending.append((key, path))
1353
  rows = self._state_rows(con)
1354
- state = _mirror_json(rows)
1355
- state_meta = {**_blob_metadata(state), "rows": len(rows)}
1356
- state_meta["key"] = _content_key("sync_state", suffix + ".json", state_meta)
1357
  published = None
1358
  if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
1359
  or manifest["tables"] != old.get("tables", {})
1360
  or old.get("version") != MANIFEST_VERSION):
1361
- state_path = None
1362
- if state_meta["key"] not in named:
1363
- state_path = Path(tmp) / "sync_state.json"
1364
- state_path.write_bytes(state)
1365
- pending.append((state_meta["key"], state_path))
1366
  if any(path.stat().st_size > part_limit for _key, path in pending):
1367
  raise ValueError("mirror part exceeds the byte limit")
1368
  if pending:
 
37
  _TABLE_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.parquet")
38
  _PART_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)\.p([0-9a-z-]{1,48})\.([0-9a-f]{16})(?:\.sealed-v1)?\.parquet")
39
  _STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.json")
40
+ #: State over one part's byte limit is saved as consecutive row parts, as a table is.
41
+ _STATE_PART_RE = re.compile(r"sync_state\.p([0-9]{1,6})\.([0-9a-f]{16})(?:\.sealed-v1)?\.json")
42
  _PART_COLUMNS = ("_period_start", "month", "_snapshot_date", "posted_date", "purchase_date",
43
  "shipment_date", "date", "snapshot_date", "last_updated_date", "settlement_start_date")
44
  ORPHAN_GRACE = timedelta(hours=24)
 
728
  return key
729
 
730
 
731
+ def _state_blob_keys(self, metadata, *, strict=True):
732
+ """[(key, metadata)] for the manifest's state entry: its row parts, or its one blob."""
733
+ if not (isinstance(metadata, dict) and "parts" in metadata):
734
+ return [(self._state_blob_key(metadata, strict=strict), metadata)]
735
+ parts = metadata["parts"]
736
+ if not isinstance(parts, list) or not parts:
737
+ raise ValueError(f"{self.label} mirror manifest names an empty state part list")
738
+ out = []
739
+ for part in parts:
740
+ key = str(part.get("key") or "") if isinstance(part, dict) else ""
741
+ match = _STATE_PART_RE.fullmatch(key)
742
+ if not match or (strict and not str(part.get("sha256") or "").startswith(match.group(2))):
743
+ raise ValueError(f"{self.label} mirror manifest names an invalid state part key")
744
+ out.append((key, part))
745
+ return out
746
+
747
+
748
+ @staticmethod
749
+ def _state_entry(rows, limit, suffix):
750
+ """(manifest state entry, [(key, bytes)]): one blob while the rows fit `limit`, else row parts."""
751
+ state = _mirror_json(rows)
752
+ if len(state) <= limit:
753
+ meta = {**_blob_metadata(state), "rows": len(rows)}
754
+ meta["key"] = _content_key("sync_state", suffix + ".json", meta)
755
+ return meta, [(meta["key"], state)]
756
+ parts, blobs, chunk, size = [], [], [], 2
757
+
758
+ def close():
759
+ data = _mirror_json(chunk)
760
+ meta = {**_blob_metadata(data), "rows": len(chunk), "part": len(parts)}
761
+ meta["key"] = _content_key(f"sync_state.p{len(parts)}", suffix + ".json", meta)
762
+ parts.append(meta)
763
+ blobs.append((meta["key"], data))
764
+
765
+ for row in rows:
766
+ width = len(_mirror_json(row)) + 1
767
+ if chunk and size + width > limit:
768
+ close()
769
+ chunk, size = [], 2
770
+ chunk.append(row)
771
+ size += width
772
+ if chunk:
773
+ close()
774
+ # A single row wider than `limit` stays one part, and the pending byte check names it.
775
+ combined = hashlib.sha256("".join(p["sha256"] for p in parts).encode()).hexdigest()
776
+ return ({"parts": parts, "rows": len(rows), "sha256": combined,
777
+ "bytes": sum(p["bytes"] for p in parts)}, blobs)
778
+
779
+
780
  def _is_mirror_blob_key(self, key):
781
  match = _TABLE_BLOB_RE.fullmatch(str(key)) or _PART_BLOB_RE.fullmatch(str(key))
782
  return bool((match and self._valid_table(match.group(1)))
783
+ or _STATE_BLOB_RE.fullmatch(str(key)) or _STATE_PART_RE.fullmatch(str(key)))
784
 
785
 
786
  def _manifest_keys(self, manifest, *, strict=True):
 
798
  state = manifest.get("sync_state")
799
  if isinstance(state, dict):
800
  try:
801
+ for key, blob in self._state_blob_keys(state, strict=strict):
802
+ out[key] = blob.get("sha256")
803
  except (ValueError, AttributeError, TypeError):
804
  pass
805
  return out
 
857
  raise ValueError("Mirror publication identity or times are invalid")
858
  allow_partial = False
859
  state_meta = manifest["sync_state"]
860
+ parts = [json.loads(self._checked_mirror_blob(rt, key, blob))
861
+ for key, blob in self._state_blob_keys(state_meta)]
862
+ rows = [row for part in parts for row in part] if all(isinstance(p, list) for p in parts) else None
863
  if (not isinstance(rows, list) or len(rows) != state_meta.get("rows")
864
  or not self._state_complete(rows) or any(not isinstance(row, list) or len(row) != 6
865
  or not isinstance(row[0], str) or not self._owns_state(row[0]) for row in rows)
 
1404
  continue
1405
  pending.append((key, path))
1406
  rows = self._state_rows(con)
1407
+ state_meta, state_blobs = self._state_entry(rows, part_limit, suffix)
 
 
1408
  published = None
1409
  if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
1410
  or manifest["tables"] != old.get("tables", {})
1411
  or old.get("version") != MANIFEST_VERSION):
1412
+ for index, (state_key, state) in enumerate(state_blobs):
1413
+ if state_key not in named:
1414
+ state_path = Path(tmp) / f"sync_state.{index}.json"
1415
+ state_path.write_bytes(state)
1416
+ pending.append((state_key, state_path))
1417
  if any(path.stat().st_size > part_limit for _key, path in pending):
1418
  raise ValueError("mirror part exceeds the byte limit")
1419
  if pending:
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- b0081f6b8447ab0b643dd2a93d4019e06717448cd94dc78abb6a103639c41202
 
1
+ 626a0389547c6145534965b5672a408a1f8f558942ccd51d6aafaedb5b2085f4