Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- platform/harness/mirror_store.py +63 -12
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "a11b7f74a",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
a11b7f74a
|
platform/harness/mirror_store.py
CHANGED
|
@@ -37,6 +37,8 @@ _KEY_LEDGER = "keys.json"
|
|
| 37 |
_TABLE_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.parquet")
|
| 38 |
_PART_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)\.p([0-9a-z-]{1,48})\.([0-9a-f]{16})(?:\.sealed-v1)?\.parquet")
|
| 39 |
_STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.json")
|
|
|
|
|
|
|
| 40 |
_PART_COLUMNS = ("_period_start", "month", "_snapshot_date", "posted_date", "purchase_date",
|
| 41 |
"shipment_date", "date", "snapshot_date", "last_updated_date", "settlement_start_date")
|
| 42 |
ORPHAN_GRACE = timedelta(hours=24)
|
|
@@ -726,10 +728,59 @@ class MirrorStore:
|
|
| 726 |
return key
|
| 727 |
|
| 728 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 729 |
def _is_mirror_blob_key(self, key):
|
| 730 |
match = _TABLE_BLOB_RE.fullmatch(str(key)) or _PART_BLOB_RE.fullmatch(str(key))
|
| 731 |
return bool((match and self._valid_table(match.group(1)))
|
| 732 |
-
or _STATE_BLOB_RE.fullmatch(str(key)))
|
| 733 |
|
| 734 |
|
| 735 |
def _manifest_keys(self, manifest, *, strict=True):
|
|
@@ -747,7 +798,8 @@ class MirrorStore:
|
|
| 747 |
state = manifest.get("sync_state")
|
| 748 |
if isinstance(state, dict):
|
| 749 |
try:
|
| 750 |
-
|
|
|
|
| 751 |
except (ValueError, AttributeError, TypeError):
|
| 752 |
pass
|
| 753 |
return out
|
|
@@ -805,8 +857,9 @@ class MirrorStore:
|
|
| 805 |
raise ValueError("Mirror publication identity or times are invalid")
|
| 806 |
allow_partial = False
|
| 807 |
state_meta = manifest["sync_state"]
|
| 808 |
-
|
| 809 |
-
|
|
|
|
| 810 |
if (not isinstance(rows, list) or len(rows) != state_meta.get("rows")
|
| 811 |
or not self._state_complete(rows) or any(not isinstance(row, list) or len(row) != 6
|
| 812 |
or not isinstance(row[0], str) or not self._owns_state(row[0]) for row in rows)
|
|
@@ -1351,18 +1404,16 @@ class MirrorStore:
|
|
| 1351 |
continue
|
| 1352 |
pending.append((key, path))
|
| 1353 |
rows = self._state_rows(con)
|
| 1354 |
-
|
| 1355 |
-
state_meta = {**_blob_metadata(state), "rows": len(rows)}
|
| 1356 |
-
state_meta["key"] = _content_key("sync_state", suffix + ".json", state_meta)
|
| 1357 |
published = None
|
| 1358 |
if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
|
| 1359 |
or manifest["tables"] != old.get("tables", {})
|
| 1360 |
or old.get("version") != MANIFEST_VERSION):
|
| 1361 |
-
|
| 1362 |
-
|
| 1363 |
-
|
| 1364 |
-
|
| 1365 |
-
|
| 1366 |
if any(path.stat().st_size > part_limit for _key, path in pending):
|
| 1367 |
raise ValueError("mirror part exceeds the byte limit")
|
| 1368 |
if pending:
|
|
|
|
| 37 |
_TABLE_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.parquet")
|
| 38 |
_PART_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)\.p([0-9a-z-]{1,48})\.([0-9a-f]{16})(?:\.sealed-v1)?\.parquet")
|
| 39 |
_STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.json")
|
| 40 |
+
#: State over one part's byte limit is saved as consecutive row parts, as a table is.
|
| 41 |
+
_STATE_PART_RE = re.compile(r"sync_state\.p([0-9]{1,6})\.([0-9a-f]{16})(?:\.sealed-v1)?\.json")
|
| 42 |
_PART_COLUMNS = ("_period_start", "month", "_snapshot_date", "posted_date", "purchase_date",
|
| 43 |
"shipment_date", "date", "snapshot_date", "last_updated_date", "settlement_start_date")
|
| 44 |
ORPHAN_GRACE = timedelta(hours=24)
|
|
|
|
| 728 |
return key
|
| 729 |
|
| 730 |
|
| 731 |
+
def _state_blob_keys(self, metadata, *, strict=True):
|
| 732 |
+
"""[(key, metadata)] for the manifest's state entry: its row parts, or its one blob."""
|
| 733 |
+
if not (isinstance(metadata, dict) and "parts" in metadata):
|
| 734 |
+
return [(self._state_blob_key(metadata, strict=strict), metadata)]
|
| 735 |
+
parts = metadata["parts"]
|
| 736 |
+
if not isinstance(parts, list) or not parts:
|
| 737 |
+
raise ValueError(f"{self.label} mirror manifest names an empty state part list")
|
| 738 |
+
out = []
|
| 739 |
+
for part in parts:
|
| 740 |
+
key = str(part.get("key") or "") if isinstance(part, dict) else ""
|
| 741 |
+
match = _STATE_PART_RE.fullmatch(key)
|
| 742 |
+
if not match or (strict and not str(part.get("sha256") or "").startswith(match.group(2))):
|
| 743 |
+
raise ValueError(f"{self.label} mirror manifest names an invalid state part key")
|
| 744 |
+
out.append((key, part))
|
| 745 |
+
return out
|
| 746 |
+
|
| 747 |
+
|
| 748 |
+
@staticmethod
|
| 749 |
+
def _state_entry(rows, limit, suffix):
|
| 750 |
+
"""(manifest state entry, [(key, bytes)]): one blob while the rows fit `limit`, else row parts."""
|
| 751 |
+
state = _mirror_json(rows)
|
| 752 |
+
if len(state) <= limit:
|
| 753 |
+
meta = {**_blob_metadata(state), "rows": len(rows)}
|
| 754 |
+
meta["key"] = _content_key("sync_state", suffix + ".json", meta)
|
| 755 |
+
return meta, [(meta["key"], state)]
|
| 756 |
+
parts, blobs, chunk, size = [], [], [], 2
|
| 757 |
+
|
| 758 |
+
def close():
|
| 759 |
+
data = _mirror_json(chunk)
|
| 760 |
+
meta = {**_blob_metadata(data), "rows": len(chunk), "part": len(parts)}
|
| 761 |
+
meta["key"] = _content_key(f"sync_state.p{len(parts)}", suffix + ".json", meta)
|
| 762 |
+
parts.append(meta)
|
| 763 |
+
blobs.append((meta["key"], data))
|
| 764 |
+
|
| 765 |
+
for row in rows:
|
| 766 |
+
width = len(_mirror_json(row)) + 1
|
| 767 |
+
if chunk and size + width > limit:
|
| 768 |
+
close()
|
| 769 |
+
chunk, size = [], 2
|
| 770 |
+
chunk.append(row)
|
| 771 |
+
size += width
|
| 772 |
+
if chunk:
|
| 773 |
+
close()
|
| 774 |
+
# A single row wider than `limit` stays one part, and the pending byte check names it.
|
| 775 |
+
combined = hashlib.sha256("".join(p["sha256"] for p in parts).encode()).hexdigest()
|
| 776 |
+
return ({"parts": parts, "rows": len(rows), "sha256": combined,
|
| 777 |
+
"bytes": sum(p["bytes"] for p in parts)}, blobs)
|
| 778 |
+
|
| 779 |
+
|
| 780 |
def _is_mirror_blob_key(self, key):
|
| 781 |
match = _TABLE_BLOB_RE.fullmatch(str(key)) or _PART_BLOB_RE.fullmatch(str(key))
|
| 782 |
return bool((match and self._valid_table(match.group(1)))
|
| 783 |
+
or _STATE_BLOB_RE.fullmatch(str(key)) or _STATE_PART_RE.fullmatch(str(key)))
|
| 784 |
|
| 785 |
|
| 786 |
def _manifest_keys(self, manifest, *, strict=True):
|
|
|
|
| 798 |
state = manifest.get("sync_state")
|
| 799 |
if isinstance(state, dict):
|
| 800 |
try:
|
| 801 |
+
for key, blob in self._state_blob_keys(state, strict=strict):
|
| 802 |
+
out[key] = blob.get("sha256")
|
| 803 |
except (ValueError, AttributeError, TypeError):
|
| 804 |
pass
|
| 805 |
return out
|
|
|
|
| 857 |
raise ValueError("Mirror publication identity or times are invalid")
|
| 858 |
allow_partial = False
|
| 859 |
state_meta = manifest["sync_state"]
|
| 860 |
+
parts = [json.loads(self._checked_mirror_blob(rt, key, blob))
|
| 861 |
+
for key, blob in self._state_blob_keys(state_meta)]
|
| 862 |
+
rows = [row for part in parts for row in part] if all(isinstance(p, list) for p in parts) else None
|
| 863 |
if (not isinstance(rows, list) or len(rows) != state_meta.get("rows")
|
| 864 |
or not self._state_complete(rows) or any(not isinstance(row, list) or len(row) != 6
|
| 865 |
or not isinstance(row[0], str) or not self._owns_state(row[0]) for row in rows)
|
|
|
|
| 1404 |
continue
|
| 1405 |
pending.append((key, path))
|
| 1406 |
rows = self._state_rows(con)
|
| 1407 |
+
state_meta, state_blobs = self._state_entry(rows, part_limit, suffix)
|
|
|
|
|
|
|
| 1408 |
published = None
|
| 1409 |
if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
|
| 1410 |
or manifest["tables"] != old.get("tables", {})
|
| 1411 |
or old.get("version") != MANIFEST_VERSION):
|
| 1412 |
+
for index, (state_key, state) in enumerate(state_blobs):
|
| 1413 |
+
if state_key not in named:
|
| 1414 |
+
state_path = Path(tmp) / f"sync_state.{index}.json"
|
| 1415 |
+
state_path.write_bytes(state)
|
| 1416 |
+
pending.append((state_key, state_path))
|
| 1417 |
if any(path.stat().st_size > part_limit for _key, path in pending):
|
| 1418 |
raise ValueError("mirror part exceeds the byte limit")
|
| 1419 |
if pending:
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
626a0389547c6145534965b5672a408a1f8f558942ccd51d6aafaedb5b2085f4
|