Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +38 -4
- api/test_odoo_rank_refresh.py +32 -9
- platform/core/store_pg.py +116 -26
- platform/harness/amazon_store.py +230 -18
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "17b3233 (17b3233)",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
17b3233 (17b3233)
|
api/main.py
CHANGED
|
@@ -1942,6 +1942,25 @@ def _project_desk(outcome, tenant):
|
|
| 1942 |
print(f"[aios-api] sync now projection failed: {type(e).__name__}: {e}")
|
| 1943 |
|
| 1944 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1945 |
def _resync_boot_recent(last_pass_at, now, every):
|
| 1946 |
"""Did the last scheduled pass finish less than one cadence ago? Pure; None means no."""
|
| 1947 |
try:
|
|
@@ -2009,6 +2028,7 @@ def _store_resync_loop():
|
|
| 2009 |
last_relational_success_at = _memory.get("fullRebuildAt")
|
| 2010 |
last_source_fingerprints = _memory.get("fingerprints")
|
| 2011 |
last_good_pass_at = _memory.get("passAt") or _memory.get("lastGoodSyncAt")
|
|
|
|
| 2012 |
last_meta_fingerprint = None
|
| 2013 |
last_meta_applied_at = None
|
| 2014 |
# ⚠ AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
|
|
@@ -2103,7 +2123,10 @@ def _store_resync_loop():
|
|
| 2103 |
# the disposable DuckDB mirror up (Odoo reads, no store; the dashboards read it, W58) and
|
| 2104 |
# restores Amazon's saved mirror. The store-touching half (Meta and Amazon vendor pulls,
|
| 2105 |
# the relational rebuild) waits for the pass that is actually due.
|
| 2106 |
-
|
|
|
|
|
|
|
|
|
|
| 2107 |
# ⛔⛔ HOTFIX 2026-09-24 — A DESK WAKE IS NOT A SCHEDULED PASS.
|
| 2108 |
# An Amazon first load requeues itself after every bounded chunk, and each requeue SETS the
|
| 2109 |
# wake event. This loop used to answer every wake with the whole scheduled pass: an Odoo
|
|
@@ -2147,8 +2170,18 @@ def _store_resync_loop():
|
|
| 2147 |
f"{_err['type']}: {_err['message']}")
|
| 2148 |
if _boot_catch_up and not _odoo_requested:
|
| 2149 |
# Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
|
| 2150 |
-
|
|
|
|
| 2151 |
_age = max(0, int(_t.time() - last_good_pass_at))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2152 |
print(f"[aios-api] store resync: restarted {_age}s after the last scheduled pass, so "
|
| 2153 |
f"only the local mirror was caught up and no store table was rewritten; the "
|
| 2154 |
f"next scheduled pass is due in {max(0, _every - _age)}s")
|
|
@@ -2406,8 +2439,9 @@ try:
|
|
| 2406 |
from harness import datastore as _ds_boot
|
| 2407 |
if not _ds_boot.DB_PATH.exists():
|
| 2408 |
import threading as _threading_seed
|
| 2409 |
-
_threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True,
|
| 2410 |
-
|
|
|
|
| 2411 |
except Exception as e: # noqa: BLE001
|
| 2412 |
print(f"[aios-api] mirror seed not scheduled: {e}")
|
| 2413 |
|
|
|
|
| 1942 |
print(f"[aios-api] sync now projection failed: {type(e).__name__}: {e}")
|
| 1943 |
|
| 1944 |
|
| 1945 |
+
#: HOTFIX 2026-09-29: how often, and how many times, a failed restart catch-up is retried. It
|
| 1946 |
+
#: touches no store (DuckDB and Odoo only), so retrying is free for Neon and DO; waiting for the
|
| 1947 |
+
#: next daily pass instead would leave the dashboards on the seed for up to a day.
|
| 1948 |
+
_CATCH_UP_RETRY_SECONDS = 300
|
| 1949 |
+
_CATCH_UP_RETRIES = 12
|
| 1950 |
+
|
| 1951 |
+
|
| 1952 |
+
def _await_boot_seed(timeout=900):
|
| 1953 |
+
"""The first resync pass must not race the boot seed's file install.
|
| 1954 |
+
|
| 1955 |
+
Staging 2026-09-29 14:07:44Z: the seed landed and the first pass died on DuckDB
|
| 1956 |
+
`ConnectionException` in the same second. The seed thread exists only on a fresh disk.
|
| 1957 |
+
"""
|
| 1958 |
+
thread = globals().get("_SEED_THREAD")
|
| 1959 |
+
if thread is not None and thread.is_alive():
|
| 1960 |
+
print("[aios-api] store resync waits for the boot seed to finish installing")
|
| 1961 |
+
thread.join(timeout)
|
| 1962 |
+
|
| 1963 |
+
|
| 1964 |
def _resync_boot_recent(last_pass_at, now, every):
|
| 1965 |
"""Did the last scheduled pass finish less than one cadence ago? Pure; None means no."""
|
| 1966 |
try:
|
|
|
|
| 2028 |
last_relational_success_at = _memory.get("fullRebuildAt")
|
| 2029 |
last_source_fingerprints = _memory.get("fingerprints")
|
| 2030 |
last_good_pass_at = _memory.get("passAt") or _memory.get("lastGoodSyncAt")
|
| 2031 |
+
catch_up_retries = 0
|
| 2032 |
last_meta_fingerprint = None
|
| 2033 |
last_meta_applied_at = None
|
| 2034 |
# ⚠ AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
|
|
|
|
| 2123 |
# the disposable DuckDB mirror up (Odoo reads, no store; the dashboards read it, W58) and
|
| 2124 |
# restores Amazon's saved mirror. The store-touching half (Meta and Amazon vendor pulls,
|
| 2125 |
# the relational rebuild) waits for the pass that is actually due.
|
| 2126 |
+
if not passes:
|
| 2127 |
+
_await_boot_seed()
|
| 2128 |
+
_boot_catch_up = ((not passes or catch_up_retries)
|
| 2129 |
+
and _resync_boot_recent(last_good_pass_at, _t.time(), _every))
|
| 2130 |
# ⛔⛔ HOTFIX 2026-09-24 — A DESK WAKE IS NOT A SCHEDULED PASS.
|
| 2131 |
# An Amazon first load requeues itself after every bounded chunk, and each requeue SETS the
|
| 2132 |
# wake event. This loop used to answer every wake with the whole scheduled pass: an Odoo
|
|
|
|
| 2170 |
f"{_err['type']}: {_err['message']}")
|
| 2171 |
if _boot_catch_up and not _odoo_requested:
|
| 2172 |
# Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
|
| 2173 |
+
if not catch_up_retries:
|
| 2174 |
+
_pull_amazon("boot", False)
|
| 2175 |
_age = max(0, int(_t.time() - last_good_pass_at))
|
| 2176 |
+
if _err is not None and catch_up_retries < _CATCH_UP_RETRIES:
|
| 2177 |
+
# The catch-up failed: retry it soon (it is free for the store), never the store half.
|
| 2178 |
+
catch_up_retries += 1
|
| 2179 |
+
print(f"[aios-api] store resync: the restart catch-up failed ({_err['type']}); "
|
| 2180 |
+
f"retry {catch_up_retries} of {_CATCH_UP_RETRIES} in "
|
| 2181 |
+
f"{_CATCH_UP_RETRY_SECONDS}s, and no store table was rewritten")
|
| 2182 |
+
last_full_pass_at = _t.time() - _every + _CATCH_UP_RETRY_SECONDS
|
| 2183 |
+
continue
|
| 2184 |
+
catch_up_retries = 0
|
| 2185 |
print(f"[aios-api] store resync: restarted {_age}s after the last scheduled pass, so "
|
| 2186 |
f"only the local mirror was caught up and no store table was rewritten; the "
|
| 2187 |
f"next scheduled pass is due in {max(0, _every - _age)}s")
|
|
|
|
| 2439 |
from harness import datastore as _ds_boot
|
| 2440 |
if not _ds_boot.DB_PATH.exists():
|
| 2441 |
import threading as _threading_seed
|
| 2442 |
+
_SEED_THREAD = _threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True,
|
| 2443 |
+
name="mirror-seed")
|
| 2444 |
+
_SEED_THREAD.start()
|
| 2445 |
except Exception as e: # noqa: BLE001
|
| 2446 |
print(f"[aios-api] mirror seed not scheduled: {e}")
|
| 2447 |
|
api/test_odoo_rank_refresh.py
CHANGED
|
@@ -188,7 +188,10 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 188 |
with duckdb.connect(self.path) as con:
|
| 189 |
projection[:] = sorted(r[rel.JOIN_KEY] for r in rel.read_customers(con))
|
| 190 |
return True
|
| 191 |
-
|
|
|
|
|
|
|
|
|
|
| 192 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 193 |
ns = dict(RESYNC_STATE={}, _RESYNC_NOTICE=[None],
|
| 194 |
_resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False,
|
|
@@ -291,21 +294,21 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 291 |
attempts, projection, sleeps, reports = self.observe_resident_recovery({2})
|
| 292 |
self.assertEqual(reports, [0, 1, 0, 0])
|
| 293 |
self.assertEqual(attempts, [1, 2, 3])
|
| 294 |
-
self.assertEqual(projection, [1, 2])
|
| 295 |
self.assertEqual(sleeps, [60] * 3)
|
| 296 |
|
| 297 |
def test_repeated_rank_rebuild_failure_retries_at_configured_cadence(self):
|
| 298 |
attempts, projection, sleeps, reports = self.observe_resident_recovery({2, 3})
|
| 299 |
self.assertEqual(reports, [0, 1, 0, 0, 0])
|
| 300 |
self.assertEqual(attempts, [1, 2, 3, 4])
|
| 301 |
-
self.assertEqual(projection, [1, 2])
|
| 302 |
self.assertEqual(sleeps, [60] * 4)
|
| 303 |
|
| 304 |
def test_successful_rank_rebuild_still_skips_unchanged_pass(self):
|
| 305 |
attempts, projection, sleeps, reports = self.observe_resident_recovery(set())
|
| 306 |
self.assertEqual(reports, [0, 1, 0])
|
| 307 |
self.assertEqual(attempts, [1, 2])
|
| 308 |
-
self.assertEqual(projection, [1, 2])
|
| 309 |
self.assertEqual(sleeps, [60] * 2)
|
| 310 |
|
| 311 |
def test_unchanged_refresh_remains_a_confirmed_noop(self):
|
|
@@ -376,7 +379,8 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 376 |
with duckdb.connect(self.path) as con:
|
| 377 |
con.execute('CREATE TABLE sale_order (partner_id INTEGER, state VARCHAR)')
|
| 378 |
con.execute('CREATE TABLE account_move (partner_id INTEGER, state VARCHAR, move_type VARCHAR)')
|
| 379 |
-
|
|
|
|
| 380 |
self.assertEqual(ds._state(con, 'res_partner')['cursor_wd'], '2026-09-03 01:02:25')
|
| 381 |
|
| 382 |
def test_completed_refresh_repeats_and_clears_rank(self):
|
|
@@ -473,8 +477,9 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 473 |
self.test_rank_change_behind_cursor_reaches_customer_reader()
|
| 474 |
|
| 475 |
# ── HOTFIX 2026-09-29 (owner): "only sync daily at the fastest", and only what changed ──────
|
| 476 |
-
def run_daily_loop(self, memory, current, *, now=10_000.0, cadence=86400, manual=False
|
| 477 |
-
|
|
|
|
| 478 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 479 |
functions = {'_store_resync_loop', '_resync_rebuild_reason',
|
| 480 |
'_meta_rebuild_force_reason', '_resync_boot_recent'}
|
|
@@ -486,12 +491,19 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 486 |
for target in node.targets)]
|
| 487 |
events, sleeps = [], []
|
| 488 |
|
|
|
|
|
|
|
| 489 |
def sleep(seconds):
|
| 490 |
sleeps.append(seconds)
|
| 491 |
-
|
|
|
|
|
|
|
| 492 |
|
| 493 |
def sync(manual_flag, passes):
|
| 494 |
events.append(('sync', passes))
|
|
|
|
|
|
|
|
|
|
| 495 |
return (None if manual_flag else {'res_partner': {'phase': 'live', 'pulled': 0}},
|
| 496 |
None if manual_flag else {}, None)
|
| 497 |
|
|
@@ -501,7 +513,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 501 |
|
| 502 |
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
|
| 503 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 504 |
-
ns = dict(RESYNC_STATE={}, _RESYNC_NOTICE=[None],
|
| 505 |
_resolve_resync_cadence=lambda: dict(
|
| 506 |
resolvedSeconds=cadence, manual=manual, tenant='synthetic',
|
| 507 |
tenantSource='test', tenantSeconds=cadence, envSeconds=cadence,
|
|
@@ -541,6 +553,17 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 541 |
self.assertEqual(events, [('sync', 1), ('amazon', 'boot', False)])
|
| 542 |
self.assertEqual(sleeps, [86400 - 3600])
|
| 543 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 544 |
def test_restart_after_the_day_runs_the_scheduled_pass(self):
|
| 545 |
fps = {'res_partner': (3, 7)}
|
| 546 |
events, _ = self.run_daily_loop(
|
|
|
|
| 188 |
with duckdb.connect(self.path) as con:
|
| 189 |
projection[:] = sorted(r[rel.JOIN_KEY] for r in rel.read_customers(con))
|
| 190 |
return True
|
| 191 |
+
# Time moves at the SLEEP, not inside a pass: the loop's wait is the cadence minus the pass's
|
| 192 |
+
# own duration (hotfix 2026-09-29), so a clock that jumped a whole cadence inside
|
| 193 |
+
# every pass would ask for a zero wait.
|
| 194 |
+
clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + cycle[0] * 60,
|
| 195 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 196 |
ns = dict(RESYNC_STATE={}, _RESYNC_NOTICE=[None],
|
| 197 |
_resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False,
|
|
|
|
| 294 |
attempts, projection, sleeps, reports = self.observe_resident_recovery({2})
|
| 295 |
self.assertEqual(reports, [0, 1, 0, 0])
|
| 296 |
self.assertEqual(attempts, [1, 2, 3])
|
| 297 |
+
self.assertEqual(projection, [1, 2, 3]) # rank > 0 across both active states (W29 R12)
|
| 298 |
self.assertEqual(sleeps, [60] * 3)
|
| 299 |
|
| 300 |
def test_repeated_rank_rebuild_failure_retries_at_configured_cadence(self):
|
| 301 |
attempts, projection, sleeps, reports = self.observe_resident_recovery({2, 3})
|
| 302 |
self.assertEqual(reports, [0, 1, 0, 0, 0])
|
| 303 |
self.assertEqual(attempts, [1, 2, 3, 4])
|
| 304 |
+
self.assertEqual(projection, [1, 2, 3]) # rank > 0 across both active states (W29 R12)
|
| 305 |
self.assertEqual(sleeps, [60] * 4)
|
| 306 |
|
| 307 |
def test_successful_rank_rebuild_still_skips_unchanged_pass(self):
|
| 308 |
attempts, projection, sleeps, reports = self.observe_resident_recovery(set())
|
| 309 |
self.assertEqual(reports, [0, 1, 0])
|
| 310 |
self.assertEqual(attempts, [1, 2])
|
| 311 |
+
self.assertEqual(projection, [1, 2, 3]) # rank > 0 across both active states (W29 R12)
|
| 312 |
self.assertEqual(sleeps, [60] * 2)
|
| 313 |
|
| 314 |
def test_unchanged_refresh_remains_a_confirmed_noop(self):
|
|
|
|
| 379 |
with duckdb.connect(self.path) as con:
|
| 380 |
con.execute('CREATE TABLE sale_order (partner_id INTEGER, state VARCHAR)')
|
| 381 |
con.execute('CREATE TABLE account_move (partner_id INTEGER, state VARCHAR, move_type VARCHAR)')
|
| 382 |
+
# Partner 3 is archived with rank 2: a customer under W29 R12 (rank > 0, both states).
|
| 383 |
+
self.assertEqual({r[rel.JOIN_KEY] for r in rel.read_customers(con)}, {1, 2, 3})
|
| 384 |
self.assertEqual(ds._state(con, 'res_partner')['cursor_wd'], '2026-09-03 01:02:25')
|
| 385 |
|
| 386 |
def test_completed_refresh_repeats_and_clears_rank(self):
|
|
|
|
| 477 |
self.test_rank_change_behind_cursor_reaches_customer_reader()
|
| 478 |
|
| 479 |
# ── HOTFIX 2026-09-29 (owner): "only sync daily at the fastest", and only what changed ──────
|
| 480 |
+
def run_daily_loop(self, memory, current, *, now=10_000.0, cadence=86400, manual=False,
|
| 481 |
+
errors=(), max_sleeps=1):
|
| 482 |
+
"""Drive the real loop with a persisted memory; stop at its `max_sleeps`-th wait."""
|
| 483 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 484 |
functions = {'_store_resync_loop', '_resync_rebuild_reason',
|
| 485 |
'_meta_rebuild_force_reason', '_resync_boot_recent'}
|
|
|
|
| 491 |
for target in node.targets)]
|
| 492 |
events, sleeps = [], []
|
| 493 |
|
| 494 |
+
failures = list(errors)
|
| 495 |
+
|
| 496 |
def sleep(seconds):
|
| 497 |
sleeps.append(seconds)
|
| 498 |
+
if len(sleeps) >= max_sleeps:
|
| 499 |
+
raise StopIteration('bounded synthetic resident loop')
|
| 500 |
+
return False
|
| 501 |
|
| 502 |
def sync(manual_flag, passes):
|
| 503 |
events.append(('sync', passes))
|
| 504 |
+
err = failures.pop(0) if failures else None
|
| 505 |
+
if err is not None:
|
| 506 |
+
return None, None, err
|
| 507 |
return (None if manual_flag else {'res_partner': {'phase': 'live', 'pulled': 0}},
|
| 508 |
None if manual_flag else {}, None)
|
| 509 |
|
|
|
|
| 513 |
|
| 514 |
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
|
| 515 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 516 |
+
ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None],
|
| 517 |
_resolve_resync_cadence=lambda: dict(
|
| 518 |
resolvedSeconds=cadence, manual=manual, tenant='synthetic',
|
| 519 |
tenantSource='test', tenantSeconds=cadence, envSeconds=cadence,
|
|
|
|
| 553 |
self.assertEqual(events, [('sync', 1), ('amazon', 'boot', False)])
|
| 554 |
self.assertEqual(sleeps, [86400 - 3600])
|
| 555 |
|
| 556 |
+
def test_failed_restart_catch_up_retries_soon_without_store_work(self):
|
| 557 |
+
fps = {'res_partner': (3, 7)}
|
| 558 |
+
err = {'kind': 'transient', 'type': 'ConnectionException', 'message': 'seed race',
|
| 559 |
+
'stage': 'mirrorPull', 'code': 'mirror_pull_failed', 'meaning': 'retry'}
|
| 560 |
+
events, sleeps = self.run_daily_loop(
|
| 561 |
+
{'passAt': 10_000.0 - 3600, 'fullRebuildAt': 10_000.0 - 3600, 'fingerprints': fps},
|
| 562 |
+
{'res_partner': (4, 9)}, errors=[err], max_sleeps=2)
|
| 563 |
+
# One restore, a 300 s retry of the catch-up alone, then the remainder of the day.
|
| 564 |
+
self.assertEqual(events, [('sync', 1), ('amazon', 'boot', False), ('sync', 2)])
|
| 565 |
+
self.assertEqual(sleeps, [300, 86400 - 3600])
|
| 566 |
+
|
| 567 |
def test_restart_after_the_day_runs_the_scheduled_pass(self):
|
| 568 |
fps = {'res_partner': (3, 7)}
|
| 569 |
events, _ = self.run_daily_loop(
|
platform/core/store_pg.py
CHANGED
|
@@ -379,18 +379,92 @@ def _q(template, kind, tenant_slug=None):
|
|
| 379 |
bare=sql.Identifier(kind))
|
| 380 |
|
| 381 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 382 |
def get(name, fresh=False, tenant_slug=None):
|
| 383 |
"""The value stored under `name`, or `{}` when the key is genuinely absent.
|
| 384 |
|
| 385 |
`fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and
|
| 386 |
-
there is no cache here — every read is a SELECT
|
| 387 |
-
branch on the backend.
|
| 388 |
"""
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
|
|
|
|
|
|
|
|
|
| 394 |
|
| 395 |
|
| 396 |
def get_projection(name, drop=(), tenant_slug=None):
|
|
@@ -411,18 +485,23 @@ def get_projection(name, drop=(), tenant_slug=None):
|
|
| 411 |
import core.store as _hf # noqa: PLC0415 — lazy: _project lives with the class
|
| 412 |
|
| 413 |
drops = tuple(str(d) for d in drop)
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 426 |
return _hf._project(data, frozenset(drops))
|
| 427 |
|
| 428 |
|
|
@@ -602,13 +681,16 @@ def put(name, data, tenant_slug=None):
|
|
| 602 |
# pool stubbed to raise, "refused" and "reached the database" are two distinguishable answers,
|
| 603 |
# which is what makes the negative control in `verify_store_pg.py` bite without a server.
|
| 604 |
check_write(tenant_slug, f'put {name!r}')
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
|
|
|
|
|
|
|
|
|
| 612 |
return data
|
| 613 |
|
| 614 |
|
|
@@ -654,6 +736,14 @@ def update(name, fn, flush='sync', tenant_slug=None, expect_rev=None, audit_row=
|
|
| 654 |
what it loses is locality, not rows.
|
| 655 |
"""
|
| 656 |
check_write(tenant_slug, f'update {name!r}') # D-353 — see `put`, and before `_pool()`
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 657 |
with _pool().connection() as con:
|
| 658 |
with con.transaction():
|
| 659 |
row = con.execute(
|
|
|
|
| 379 |
bare=sql.Identifier(kind))
|
| 380 |
|
| 381 |
|
| 382 |
+
#: ⭐⭐ D-2640 (hotfix 2026-09-29): IDENTICAL CONCURRENT READS SHARE ONE SELECT.
|
| 383 |
+
#:
|
| 384 |
+
#: Measured on the 2 GB DigitalOcean staging node on 2026-09-28: 14 backends each reading the whole
|
| 385 |
+
#: `user_tables` document (57 MB of JSON text) at once, every one detoasting it and building its own
|
| 386 |
+
#: copy, drove the node to 94% swap and 100% CPU. A burst of requests arriving during a sync asks for
|
| 387 |
+
#: the SAME document. So a read that finds an identical read already in flight (same tenant, key and
|
| 388 |
+
#: projection) waits for it and parses the same TEXT itself, instead of opening another backend.
|
| 389 |
+
#: ⚠ NOT A CACHE: nothing outlives the SELECT that produced it, so `fresh` still means nothing here.
|
| 390 |
+
#: ⚠ READ-YOUR-WRITES HOLDS: every `put`/`update` in this process bumps the key's generation when it
|
| 391 |
+
#: finishes, and a read never joins a flight started under an older generation.
|
| 392 |
+
#: ⚠ Each caller gets its OWN parsed object (the shared thing is an immutable str), because callers
|
| 393 |
+
#: mutate what `get` returns.
|
| 394 |
+
_READ_FLIGHTS = {}
|
| 395 |
+
_WRITE_GENERATION = {}
|
| 396 |
+
_READ_FLIGHT_LOCK = threading.Lock()
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
class _ReadFlight:
|
| 400 |
+
__slots__ = ('done', 'text', 'error', 'generation')
|
| 401 |
+
|
| 402 |
+
def __init__(self, generation):
|
| 403 |
+
self.done = threading.Event()
|
| 404 |
+
self.text = None
|
| 405 |
+
self.error = None
|
| 406 |
+
self.generation = generation
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
def _bump_write_generation(name, tenant_slug):
|
| 410 |
+
key = (_effective_slug(tenant_slug), str(name))
|
| 411 |
+
with _READ_FLIGHT_LOCK:
|
| 412 |
+
_WRITE_GENERATION[key] = _WRITE_GENERATION.get(key, 0) + 1
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
def _shared_read(name, drops, tenant_slug, fetch_text):
|
| 416 |
+
"""`fetch_text()` once for every concurrent identical read. -> the value's JSON text, or None."""
|
| 417 |
+
doc = (_effective_slug(tenant_slug), str(name))
|
| 418 |
+
key = doc + (drops,)
|
| 419 |
+
with _READ_FLIGHT_LOCK:
|
| 420 |
+
generation = _WRITE_GENERATION.get(doc, 0)
|
| 421 |
+
flight = _READ_FLIGHTS.get(key)
|
| 422 |
+
leader = flight is None or flight.generation != generation
|
| 423 |
+
if leader:
|
| 424 |
+
flight = _ReadFlight(generation)
|
| 425 |
+
_READ_FLIGHTS[key] = flight
|
| 426 |
+
if not leader:
|
| 427 |
+
flight.done.wait()
|
| 428 |
+
if flight.error is not None:
|
| 429 |
+
raise RuntimeError(f'shared store read of {name!r} failed: '
|
| 430 |
+
f'{type(flight.error).__name__}') from flight.error
|
| 431 |
+
return flight.text
|
| 432 |
+
try:
|
| 433 |
+
flight.text = fetch_text()
|
| 434 |
+
except BaseException as exc:
|
| 435 |
+
flight.error = exc
|
| 436 |
+
raise
|
| 437 |
+
finally:
|
| 438 |
+
with _READ_FLIGHT_LOCK:
|
| 439 |
+
if _READ_FLIGHTS.get(key) is flight:
|
| 440 |
+
del _READ_FLIGHTS[key]
|
| 441 |
+
flight.done.set()
|
| 442 |
+
return flight.text
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def _parsed(text):
|
| 446 |
+
"""The caller's own object from the shared JSON text (`None` = the key is absent)."""
|
| 447 |
+
if text is None:
|
| 448 |
+
return {}
|
| 449 |
+
value = json.loads(text) if isinstance(text, (str, bytes, bytearray)) else text
|
| 450 |
+
return dict(value) if isinstance(value, dict) else value
|
| 451 |
+
|
| 452 |
+
|
| 453 |
def get(name, fresh=False, tenant_slug=None):
|
| 454 |
"""The value stored under `name`, or `{}` when the key is genuinely absent.
|
| 455 |
|
| 456 |
`fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and
|
| 457 |
+
there is no cache here — every read is a SELECT (shared only with identical reads already in
|
| 458 |
+
flight, D-2640). Keeping the parameter means callers do not branch on the backend.
|
| 459 |
"""
|
| 460 |
+
def fetch():
|
| 461 |
+
with _pool().connection() as con:
|
| 462 |
+
row = con.execute(
|
| 463 |
+
_q('SELECT value::text FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
|
| 464 |
+
(str(name),)).fetchone()
|
| 465 |
+
return row[0] if row else None
|
| 466 |
+
|
| 467 |
+
return _parsed(_shared_read(name, None, tenant_slug, fetch))
|
| 468 |
|
| 469 |
|
| 470 |
def get_projection(name, drop=(), tenant_slug=None):
|
|
|
|
| 485 |
import core.store as _hf # noqa: PLC0415 — lazy: _project lives with the class
|
| 486 |
|
| 487 |
drops = tuple(str(d) for d in drop)
|
| 488 |
+
|
| 489 |
+
def fetch():
|
| 490 |
+
with _pool().connection() as con:
|
| 491 |
+
row = con.execute(
|
| 492 |
+
_q('SELECT (CASE '
|
| 493 |
+
"WHEN jsonb_typeof(kv.value) = 'object' THEN "
|
| 494 |
+
"COALESCE((SELECT jsonb_object_agg(e.entry_key, "
|
| 495 |
+
"CASE WHEN jsonb_typeof(e.entry_value) = 'object' "
|
| 496 |
+
'THEN e.entry_value - %s::text[] ELSE e.entry_value END) '
|
| 497 |
+
'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), '
|
| 498 |
+
"'{{}}'::jsonb) ELSE kv.value END)::text FROM {tbl} AS kv WHERE kv.key = %s",
|
| 499 |
+
'store_kv', tenant_slug),
|
| 500 |
+
(list(drops), str(name))).fetchone()
|
| 501 |
+
return row[0] if row else None
|
| 502 |
+
|
| 503 |
+
# D-2640: the projection is the read 14 concurrent backends were making; share it too.
|
| 504 |
+
data = _parsed(_shared_read(name, tuple(sorted(drops)), tenant_slug, fetch))
|
| 505 |
return _hf._project(data, frozenset(drops))
|
| 506 |
|
| 507 |
|
|
|
|
| 681 |
# pool stubbed to raise, "refused" and "reached the database" are two distinguishable answers,
|
| 682 |
# which is what makes the negative control in `verify_store_pg.py` bite without a server.
|
| 683 |
check_write(tenant_slug, f'put {name!r}')
|
| 684 |
+
try:
|
| 685 |
+
with _pool().connection() as con:
|
| 686 |
+
payload = _payload(name, data, con, tenant_slug)
|
| 687 |
+
con.execute(
|
| 688 |
+
_q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
|
| 689 |
+
'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
|
| 690 |
+
'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
|
| 691 |
+
(str(name), payload))
|
| 692 |
+
finally:
|
| 693 |
+
_bump_write_generation(name, tenant_slug) # D-2640: no read joins an older flight
|
| 694 |
return data
|
| 695 |
|
| 696 |
|
|
|
|
| 736 |
what it loses is locality, not rows.
|
| 737 |
"""
|
| 738 |
check_write(tenant_slug, f'update {name!r}') # D-353 — see `put`, and before `_pool()`
|
| 739 |
+
try:
|
| 740 |
+
return _update_locked(name, fn, tenant_slug, expect_rev, audit_row)
|
| 741 |
+
finally:
|
| 742 |
+
_bump_write_generation(name, tenant_slug) # D-2640: no read joins an older flight
|
| 743 |
+
|
| 744 |
+
|
| 745 |
+
def _update_locked(name, fn, tenant_slug, expect_rev, audit_row):
|
| 746 |
+
"""`update`'s transaction body (split out so the D-2640 generation bump wraps all of it)."""
|
| 747 |
with _pool().connection() as con:
|
| 748 |
with con.transaction():
|
| 749 |
row = con.execute(
|
platform/harness/amazon_store.py
CHANGED
|
@@ -1702,11 +1702,26 @@ def _blob_metadata(data):
|
|
| 1702 |
#: restores; v2 names each blob's CONTENT key, so a manifest only ever names complete blobs and a
|
| 1703 |
#: key it names is never overwritten. `keys.json` is the write-ahead ledger of every key ever
|
| 1704 |
#: uploaded: the store has no list-by-prefix, so without it an orphan is unpurgeable.
|
| 1705 |
-
|
| 1706 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1707 |
_MANIFEST_KEY = "manifest.json"
|
| 1708 |
_KEY_LEDGER = "keys.json"
|
| 1709 |
_TABLE_BLOB_RE = re.compile(r"(amzn_[a-z0-9_]+)(?:\.([0-9a-f]{16}))?\.parquet")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1710 |
_STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?\.json")
|
| 1711 |
#: A ledger key no manifest names is a killed checkpoint's orphan once it is this old. A live
|
| 1712 |
#: checkpoint publishes within minutes, so the grace never races an in-flight writer.
|
|
@@ -1734,6 +1749,28 @@ def _table_blob_key(name, metadata, *, strict=True):
|
|
| 1734 |
return key
|
| 1735 |
|
| 1736 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1737 |
def _state_blob_key(metadata, *, strict=True):
|
| 1738 |
key = str(metadata.get("key") or "sync_state.json")
|
| 1739 |
match = _STATE_BLOB_RE.fullmatch(key)
|
|
@@ -1744,7 +1781,8 @@ def _state_blob_key(metadata, *, strict=True):
|
|
| 1744 |
|
| 1745 |
|
| 1746 |
def _is_mirror_blob_key(key):
|
| 1747 |
-
return bool(_TABLE_BLOB_RE.fullmatch(str(key)) or
|
|
|
|
| 1748 |
|
| 1749 |
|
| 1750 |
def _manifest_keys(manifest, *, strict=True):
|
|
@@ -1755,7 +1793,8 @@ def _manifest_keys(manifest, *, strict=True):
|
|
| 1755 |
tables = manifest.get("tables")
|
| 1756 |
for name, metadata in (tables.items() if isinstance(tables, dict) else ()):
|
| 1757 |
try:
|
| 1758 |
-
|
|
|
|
| 1759 |
except (ValueError, AttributeError, TypeError):
|
| 1760 |
continue
|
| 1761 |
state = manifest.get("sync_state")
|
|
@@ -1820,20 +1859,32 @@ def _validated_snapshot(tenant_key, rt, con, raw, tmp):
|
|
| 1820 |
for name, metadata in manifest["tables"].items():
|
| 1821 |
if not re.fullmatch(r"amzn_[a-z0-9_]+", name) or not isinstance(metadata, dict):
|
| 1822 |
raise ValueError("Amazon mirror manifest contains an invalid table name")
|
| 1823 |
-
|
| 1824 |
-
|
| 1825 |
-
|
| 1826 |
-
|
| 1827 |
-
|
| 1828 |
-
|
| 1829 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1830 |
if type(metadata.get("rows")) is not int or count != metadata["rows"]:
|
| 1831 |
raise ValueError("Amazon mirror parquet row count mismatch")
|
| 1832 |
-
paths[name] =
|
| 1833 |
total += count
|
| 1834 |
return manifest, rows, paths, total
|
| 1835 |
|
| 1836 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1837 |
def _snapshot_tables():
|
| 1838 |
"""Current-state tables: every report or API the client declares `snapshot`."""
|
| 1839 |
cx = _client()
|
|
@@ -1944,9 +1995,9 @@ def restore_mirror(tenant_key, rt, log=None):
|
|
| 1944 |
manifest, rows, paths, total = _validated_snapshot(tenant_key, rt, con, raw, tmp)
|
| 1945 |
con.execute("BEGIN TRANSACTION")
|
| 1946 |
try:
|
| 1947 |
-
for name,
|
| 1948 |
con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
|
| 1949 |
-
"SELECT * FROM
|
| 1950 |
con.executemany("INSERT INTO _sync_state "
|
| 1951 |
"(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)", rows)
|
| 1952 |
con.execute("COMMIT")
|
|
@@ -2047,6 +2098,99 @@ def _retire_blobs(tenant_key, rt, published, previous, log, *, orphans=True):
|
|
| 2047 |
return 0
|
| 2048 |
|
| 2049 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2050 |
def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print):
|
| 2051 |
"""Checkpoint changed tables and all Amazon markers, publishing the manifest last.
|
| 2052 |
|
|
@@ -2081,16 +2225,28 @@ def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print):
|
|
| 2081 |
changed, pending = [], []
|
| 2082 |
with tempfile.TemporaryDirectory(prefix="amazon-persist-") as tmp:
|
| 2083 |
# A legacy entry (no content key) is a candidate too: the first checkpoint migrates.
|
| 2084 |
-
legacy = {name for name, meta in manifest["tables"].items()
|
|
|
|
| 2085 |
for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables):
|
| 2086 |
if not re.fullmatch(r"amzn_[a-z0-9_]+", name):
|
| 2087 |
raise ValueError("Amazon mirror table name is invalid")
|
|
|
|
| 2088 |
path = Path(tmp) / (name + ".parquet")
|
| 2089 |
-
|
| 2090 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2091 |
data = path.read_bytes()
|
| 2092 |
metadata = _blob_metadata(data)
|
| 2093 |
-
prior = manifest["tables"].get(name, {})
|
| 2094 |
if metadata["sha256"] == prior.get("sha256") and prior.get("key"):
|
| 2095 |
continue
|
| 2096 |
key = _content_key(name, ".parquet", metadata)
|
|
@@ -4325,6 +4481,62 @@ def mirror_selftest(log=print, check=None):
|
|
| 4325 |
_purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)])
|
| 4326 |
ck("purge deletes every stored version including the orphan", lambda:
|
| 4327 |
orphans and not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4328 |
finally:
|
| 4329 |
forget()
|
| 4330 |
con.close()
|
|
|
|
| 1702 |
#: restores; v2 names each blob's CONTENT key, so a manifest only ever names complete blobs and a
|
| 1703 |
#: key it names is never overwritten. `keys.json` is the write-ahead ledger of every key ever
|
| 1704 |
#: uploaded: the store has no list-by-prefix, so without it an orphan is unpurgeable.
|
| 1705 |
+
#: ⭐⭐ v3 (D-2639, owner 2026-09-29 "make sure the chunking works"): a table larger than
|
| 1706 |
+
#: `MIRROR_PART_MAX_BYTES` is stored as PARTS, one per month of its date column (split further
|
| 1707 |
+
#: when a month is still too big), each under its own content key. A checkpoint re-hashes every
|
| 1708 |
+
#: part of a changed table and uploads only the parts whose bytes changed, so a day of new orders
|
| 1709 |
+
#: rewrites the current month's part instead of the whole 89 MB `amzn_transactions` value that
|
| 1710 |
+
#: swap-thrashed the DigitalOcean node. Tables under the cap keep the v2 single-blob entry.
|
| 1711 |
+
#: ⚠ A build older than v3 cannot read a v3 manifest and would ask for a first load.
|
| 1712 |
+
MANIFEST_VERSIONS = (1, 2, 3)
|
| 1713 |
+
MANIFEST_VERSION = 3
|
| 1714 |
_MANIFEST_KEY = "manifest.json"
|
| 1715 |
_KEY_LEDGER = "keys.json"
|
| 1716 |
_TABLE_BLOB_RE = re.compile(r"(amzn_[a-z0-9_]+)(?:\.([0-9a-f]{16}))?\.parquet")
|
| 1717 |
+
#: v3: ONE part of a chunked table, always content-keyed: `amzn_<t>.p<part>.<sha16>.parquet`.
|
| 1718 |
+
_PART_BLOB_RE = re.compile(r"(amzn_[a-z0-9_]+)\.p([0-9a-z-]{1,48})\.([0-9a-f]{16})\.parquet")
|
| 1719 |
+
#: No single mirror blob the checkpoint writes exceeds this (D-2639's exit names 8 MB).
|
| 1720 |
+
MIRROR_PART_MAX_BYTES = 8 * 1024 * 1024
|
| 1721 |
+
#: The column a chunked table is split by, first match wins; a table with none of them is split
|
| 1722 |
+
#: into a fixed number of hash buckets instead (bounded, though every bucket then changes).
|
| 1723 |
+
_PART_COLUMNS = ("_period_start", "month", "_snapshot_date", "posted_date", "purchase_date",
|
| 1724 |
+
"shipment_date", "date", "snapshot_date", "last_updated_date", "settlement_start_date")
|
| 1725 |
_STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?\.json")
|
| 1726 |
#: A ledger key no manifest names is a killed checkpoint's orphan once it is this old. A live
|
| 1727 |
#: checkpoint publishes within minutes, so the grace never races an in-flight writer.
|
|
|
|
| 1749 |
return key
|
| 1750 |
|
| 1751 |
|
| 1752 |
+
def _part_blob_key(name, part, *, strict=True):
|
| 1753 |
+
"""The content key of ONE part of a chunked table (v3)."""
|
| 1754 |
+
if not isinstance(part, dict):
|
| 1755 |
+
raise ValueError("Amazon mirror manifest names an invalid table part")
|
| 1756 |
+
key = str(part.get("key") or "")
|
| 1757 |
+
match = _PART_BLOB_RE.fullmatch(key)
|
| 1758 |
+
if not match or match.group(1) != name or (
|
| 1759 |
+
strict and not str(part.get("sha256") or "").startswith(match.group(3))):
|
| 1760 |
+
raise ValueError("Amazon mirror manifest names an invalid table part key")
|
| 1761 |
+
return key
|
| 1762 |
+
|
| 1763 |
+
|
| 1764 |
+
def _entry_blob_keys(name, metadata, *, strict=True):
|
| 1765 |
+
"""[(key, metadata)] for one manifest table entry: its parts (v3) or its one blob."""
|
| 1766 |
+
if isinstance(metadata, dict) and "parts" in metadata:
|
| 1767 |
+
parts = metadata["parts"]
|
| 1768 |
+
if not isinstance(parts, list) or not parts:
|
| 1769 |
+
raise ValueError("Amazon mirror manifest names an empty part list")
|
| 1770 |
+
return [(_part_blob_key(name, part, strict=strict), part) for part in parts]
|
| 1771 |
+
return [(_table_blob_key(name, metadata, strict=strict), metadata)]
|
| 1772 |
+
|
| 1773 |
+
|
| 1774 |
def _state_blob_key(metadata, *, strict=True):
|
| 1775 |
key = str(metadata.get("key") or "sync_state.json")
|
| 1776 |
match = _STATE_BLOB_RE.fullmatch(key)
|
|
|
|
| 1781 |
|
| 1782 |
|
| 1783 |
def _is_mirror_blob_key(key):
|
| 1784 |
+
return bool(_TABLE_BLOB_RE.fullmatch(str(key)) or _PART_BLOB_RE.fullmatch(str(key))
|
| 1785 |
+
or _STATE_BLOB_RE.fullmatch(str(key)))
|
| 1786 |
|
| 1787 |
|
| 1788 |
def _manifest_keys(manifest, *, strict=True):
|
|
|
|
| 1793 |
tables = manifest.get("tables")
|
| 1794 |
for name, metadata in (tables.items() if isinstance(tables, dict) else ()):
|
| 1795 |
try:
|
| 1796 |
+
for key, blob in _entry_blob_keys(str(name), metadata, strict=strict):
|
| 1797 |
+
out[key] = blob.get("sha256")
|
| 1798 |
except (ValueError, AttributeError, TypeError):
|
| 1799 |
continue
|
| 1800 |
state = manifest.get("sync_state")
|
|
|
|
| 1859 |
for name, metadata in manifest["tables"].items():
|
| 1860 |
if not re.fullmatch(r"amzn_[a-z0-9_]+", name) or not isinstance(metadata, dict):
|
| 1861 |
raise ValueError("Amazon mirror manifest contains an invalid table name")
|
| 1862 |
+
files, count = [], 0
|
| 1863 |
+
for index, (key, blob) in enumerate(_entry_blob_keys(name, metadata)):
|
| 1864 |
+
data = _checked_mirror_blob(rt, key, blob)
|
| 1865 |
+
path = Path(tmp) / f"{name}.{index:04d}.parquet"
|
| 1866 |
+
path.write_bytes(data)
|
| 1867 |
+
try:
|
| 1868 |
+
part_rows = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0]
|
| 1869 |
+
except Exception as exc: # noqa: BLE001
|
| 1870 |
+
raise ValueError("Amazon mirror parquet is unreadable") from exc
|
| 1871 |
+
if "parts" in metadata and (type(blob.get("rows")) is not int or part_rows != blob["rows"]):
|
| 1872 |
+
raise ValueError("Amazon mirror parquet part row count mismatch")
|
| 1873 |
+
files.append(str(path))
|
| 1874 |
+
count += part_rows
|
| 1875 |
if type(metadata.get("rows")) is not int or count != metadata["rows"]:
|
| 1876 |
raise ValueError("Amazon mirror parquet row count mismatch")
|
| 1877 |
+
paths[name] = files
|
| 1878 |
total += count
|
| 1879 |
return manifest, rows, paths, total
|
| 1880 |
|
| 1881 |
|
| 1882 |
+
def _parquet_source(files):
|
| 1883 |
+
"""A `read_parquet` source over one or many validated local files (our own temp paths)."""
|
| 1884 |
+
quoted = ", ".join("'" + str(path).replace("'", "''") + "'" for path in files)
|
| 1885 |
+
return f"read_parquet([{quoted}], union_by_name=true)"
|
| 1886 |
+
|
| 1887 |
+
|
| 1888 |
def _snapshot_tables():
|
| 1889 |
"""Current-state tables: every report or API the client declares `snapshot`."""
|
| 1890 |
cx = _client()
|
|
|
|
| 1995 |
manifest, rows, paths, total = _validated_snapshot(tenant_key, rt, con, raw, tmp)
|
| 1996 |
con.execute("BEGIN TRANSACTION")
|
| 1997 |
try:
|
| 1998 |
+
for name, files in paths.items():
|
| 1999 |
con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
|
| 2000 |
+
f"SELECT * FROM {_parquet_source(files)}")
|
| 2001 |
con.executemany("INSERT INTO _sync_state "
|
| 2002 |
"(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)", rows)
|
| 2003 |
con.execute("COMMIT")
|
|
|
|
| 2098 |
return 0
|
| 2099 |
|
| 2100 |
|
| 2101 |
+
def _part_slug(value):
|
| 2102 |
+
slug = re.sub(r"[^0-9a-z-]", "-", str(value).lower())[:40].strip("-")
|
| 2103 |
+
return slug or "none"
|
| 2104 |
+
|
| 2105 |
+
|
| 2106 |
+
def _sql_literal(value):
|
| 2107 |
+
return "'" + str(value).replace("'", "''") + "'"
|
| 2108 |
+
|
| 2109 |
+
|
| 2110 |
+
def _copy_part(con, name, where, path):
|
| 2111 |
+
"""One part, sorted so unchanged rows give byte-identical parquet. `where` is fully literal:
|
| 2112 |
+
a parameter inside `COPY (subquery)` is not portable, and the values are escaped."""
|
| 2113 |
+
con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)} AS t WHERE {where} ORDER BY ALL) "
|
| 2114 |
+
"TO ? (FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
|
| 2115 |
+
return path.stat().st_size
|
| 2116 |
+
|
| 2117 |
+
|
| 2118 |
+
def _table_parts(con, name, tmp):
|
| 2119 |
+
"""Write `name` as bounded parts under `tmp`. -> [(slug, path, rows)], every file under the cap.
|
| 2120 |
+
|
| 2121 |
+
⭐ SPLIT BY MONTH OF THE TABLE'S DATE COLUMN, because Amazon's incremental windows land in the
|
| 2122 |
+
newest months: yesterday's orders change the current month's part and leave history's bytes
|
| 2123 |
+
identical. `ORDER BY ALL` makes an unchanged part byte-identical from one checkpoint to the
|
| 2124 |
+
next, which is what lets its content key be reused rather than re-uploaded. A month still over
|
| 2125 |
+
the cap is split into `hash(row) % k` sub-parts (k a power of two, so the split is stable).
|
| 2126 |
+
"""
|
| 2127 |
+
cols = [row[0] for row in con.execute(
|
| 2128 |
+
"SELECT column_name FROM information_schema.columns WHERE table_name = ? "
|
| 2129 |
+
"ORDER BY ordinal_position", [name]).fetchall()]
|
| 2130 |
+
column = next((c for c in _PART_COLUMNS if c in cols), None)
|
| 2131 |
+
if column:
|
| 2132 |
+
expr = f"coalesce(substr(CAST(t.{_quoted_identifier(column)} AS VARCHAR), 1, 7), 'none')"
|
| 2133 |
+
groups = [row[0] for row in con.execute(
|
| 2134 |
+
f"SELECT DISTINCT {expr} AS g FROM {_quoted_identifier(name)} AS t ORDER BY g").fetchall()]
|
| 2135 |
+
groups = groups or ["none"] # an emptied table is still one (empty) part
|
| 2136 |
+
else:
|
| 2137 |
+
total = max(1, con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0])
|
| 2138 |
+
buckets = 2
|
| 2139 |
+
while buckets < 4096 and total / buckets > 50000:
|
| 2140 |
+
buckets *= 2
|
| 2141 |
+
expr = f"CAST(hash(t) % {buckets} AS VARCHAR)"
|
| 2142 |
+
groups = [str(b) for b in range(buckets)]
|
| 2143 |
+
out = []
|
| 2144 |
+
for group in groups:
|
| 2145 |
+
slug = _part_slug(group)
|
| 2146 |
+
path = tmp / f"{name}.p{slug}.parquet"
|
| 2147 |
+
size = _copy_part(con, name, f"{expr} = {_sql_literal(group)}", path)
|
| 2148 |
+
if size <= MIRROR_PART_MAX_BYTES:
|
| 2149 |
+
out.append((slug, path, None))
|
| 2150 |
+
continue
|
| 2151 |
+
k = 2
|
| 2152 |
+
while True:
|
| 2153 |
+
files, fits = [], True
|
| 2154 |
+
for j in range(k):
|
| 2155 |
+
sub = tmp / f"{name}.p{slug}-s{j}.parquet"
|
| 2156 |
+
files.append((f"{slug}-s{j}", sub))
|
| 2157 |
+
if _copy_part(con, name, f"{expr} = {_sql_literal(group)} AND hash(t) % {k} = {j}",
|
| 2158 |
+
sub) > MIRROR_PART_MAX_BYTES:
|
| 2159 |
+
fits = False
|
| 2160 |
+
if fits or k >= 1024:
|
| 2161 |
+
out.extend((s, f, None) for s, f in files)
|
| 2162 |
+
break
|
| 2163 |
+
k *= 2
|
| 2164 |
+
return [(slug, path, con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0])
|
| 2165 |
+
for slug, path, _rows in out]
|
| 2166 |
+
|
| 2167 |
+
|
| 2168 |
+
def _chunked_entry(con, name, tmp, prior, named):
|
| 2169 |
+
"""A v3 manifest entry for `name`, plus [(key, path)] of the parts that must be uploaded.
|
| 2170 |
+
|
| 2171 |
+
Returns (None, []) when every part's bytes equal the published entry's (nothing to write).
|
| 2172 |
+
A part whose sha the previous entry already names keeps that key and is not uploaded again.
|
| 2173 |
+
"""
|
| 2174 |
+
known = {}
|
| 2175 |
+
for part in (prior.get("parts") or []) if isinstance(prior, dict) else []:
|
| 2176 |
+
if isinstance(part, dict) and part.get("sha256") and part.get("key"):
|
| 2177 |
+
known[part["sha256"]] = part["key"]
|
| 2178 |
+
parts, fresh = [], []
|
| 2179 |
+
for slug, path, rows in _table_parts(con, name, tmp):
|
| 2180 |
+
metadata = _blob_metadata(path.read_bytes())
|
| 2181 |
+
key = known.get(metadata["sha256"]) or _content_key(f"{name}.p{slug}", ".parquet", metadata)
|
| 2182 |
+
parts.append({"part": slug, "key": key, "rows": int(rows), **metadata})
|
| 2183 |
+
if key not in known.values() and key not in named:
|
| 2184 |
+
fresh.append((key, path))
|
| 2185 |
+
combined = hashlib.sha256("".join(p["sha256"] for p in parts).encode()).hexdigest()
|
| 2186 |
+
if prior.get("parts") and prior.get("sha256") == combined:
|
| 2187 |
+
return None, []
|
| 2188 |
+
entry = {"parts": parts, "rows": sum(p["rows"] for p in parts), "sha256": combined,
|
| 2189 |
+
"bytes": sum(p["bytes"] for p in parts),
|
| 2190 |
+
"written_at": _iso(datetime.now(timezone.utc))}
|
| 2191 |
+
return entry, fresh
|
| 2192 |
+
|
| 2193 |
+
|
| 2194 |
def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print):
|
| 2195 |
"""Checkpoint changed tables and all Amazon markers, publishing the manifest last.
|
| 2196 |
|
|
|
|
| 2225 |
changed, pending = [], []
|
| 2226 |
with tempfile.TemporaryDirectory(prefix="amazon-persist-") as tmp:
|
| 2227 |
# A legacy entry (no content key) is a candidate too: the first checkpoint migrates.
|
| 2228 |
+
legacy = {name for name, meta in manifest["tables"].items()
|
| 2229 |
+
if not meta.get("key") and "parts" not in meta}
|
| 2230 |
for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables):
|
| 2231 |
if not re.fullmatch(r"amzn_[a-z0-9_]+", name):
|
| 2232 |
raise ValueError("Amazon mirror table name is invalid")
|
| 2233 |
+
prior = manifest["tables"].get(name, {})
|
| 2234 |
path = Path(tmp) / (name + ".parquet")
|
| 2235 |
+
size = None
|
| 2236 |
+
if "parts" not in prior:
|
| 2237 |
+
con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
|
| 2238 |
+
"(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
|
| 2239 |
+
size = path.stat().st_size
|
| 2240 |
+
if size is None or size > MIRROR_PART_MAX_BYTES:
|
| 2241 |
+
entry, fresh = _chunked_entry(con, name, Path(tmp), prior, named)
|
| 2242 |
+
if entry is None:
|
| 2243 |
+
continue
|
| 2244 |
+
manifest["tables"][name] = entry
|
| 2245 |
+
changed.append(name)
|
| 2246 |
+
pending.extend(fresh)
|
| 2247 |
+
continue
|
| 2248 |
data = path.read_bytes()
|
| 2249 |
metadata = _blob_metadata(data)
|
|
|
|
| 2250 |
if metadata["sha256"] == prior.get("sha256") and prior.get("key"):
|
| 2251 |
continue
|
| 2252 |
key = _content_key(name, ".parquet", metadata)
|
|
|
|
| 4481 |
_purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)])
|
| 4482 |
ck("purge deletes every stored version including the orphan", lambda:
|
| 4483 |
orphans and not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"})
|
| 4484 |
+
|
| 4485 |
+
# ⭐⭐ D-2639 (owner 2026-09-29 "make sure the chunking works"): a table over the cap is
|
| 4486 |
+
# stored as month parts, none over the cap, and a checkpoint uploads only changed parts.
|
| 4487 |
+
module = sys.modules[__name__]
|
| 4488 |
+
con.execute("CREATE OR REPLACE TABLE amzn_transactions AS SELECT "
|
| 4489 |
+
"'2026-0' || (1 + i % 3) || '-15' AS posted_date, i AS id, md5(i::VARCHAR) AS a, "
|
| 4490 |
+
"md5((i * 7)::VARCHAR) AS b FROM range(9000) t(i)")
|
| 4491 |
+
cap = 60000
|
| 4492 |
+
with patch.object(module, "MIRROR_PART_MAX_BYTES", 10 ** 9):
|
| 4493 |
+
persist({name for name, _kind in _amazon_objects(con)})
|
| 4494 |
+
whole = json.loads(rt.blobs[manifest_path])["tables"]["amzn_transactions"]
|
| 4495 |
+
ck("NC: under a huge cap the same table is one blob, and that blob is over the real cap",
|
| 4496 |
+
lambda: "parts" not in whole and whole["bytes"] > cap)
|
| 4497 |
+
|
| 4498 |
+
def is_part(path):
|
| 4499 |
+
return bool(re.search(r"\.p[0-9a-z-]+\.[0-9a-f]{16}\.parquet$", path))
|
| 4500 |
+
|
| 4501 |
+
def txn_keys():
|
| 4502 |
+
entry = json.loads(rt.blobs[manifest_path])["tables"]["amzn_transactions"]
|
| 4503 |
+
return [part["key"] for part in entry.get("parts", [])]
|
| 4504 |
+
|
| 4505 |
+
with patch.object(module, "MIRROR_PART_MAX_BYTES", cap):
|
| 4506 |
+
start = len(rt.uploads)
|
| 4507 |
+
chunk_report = persist({"amzn_transactions"})
|
| 4508 |
+
if chunk_report["problems"]:
|
| 4509 |
+
log(" chunked persist problems: " + " | ".join(chunk_report["problems"]))
|
| 4510 |
+
saved = json.loads(rt.blobs[manifest_path])
|
| 4511 |
+
entry = saved["tables"]["amzn_transactions"]
|
| 4512 |
+
written = [p for p in rt.uploads[start:] if is_part(p)]
|
| 4513 |
+
ck("a table over the cap is saved as parts, none over the cap, rows intact (v3)", lambda:
|
| 4514 |
+
saved["version"] == 3 and len(entry["parts"]) >= 3 and written
|
| 4515 |
+
and all(len(rt.blobs[p]) <= cap for p in written)
|
| 4516 |
+
and entry["rows"] == 9000 == sum(part["rows"] for part in entry["parts"])
|
| 4517 |
+
and {p["part"][:7] for p in entry["parts"]} == {"2026-01", "2026-02", "2026-03"})
|
| 4518 |
+
before = txn_keys()
|
| 4519 |
+
start = len(rt.uploads)
|
| 4520 |
+
persist({"amzn_transactions"})
|
| 4521 |
+
ck("an unchanged chunked table uploads no part again", lambda:
|
| 4522 |
+
not [p for p in rt.uploads[start:] if is_part(p)] and txn_keys() == before)
|
| 4523 |
+
con.execute("INSERT INTO amzn_transactions SELECT '2026-03-20', 9000 + i, "
|
| 4524 |
+
"md5(('x' || i)::VARCHAR), 'new' FROM range(50) t(i)")
|
| 4525 |
+
start = len(rt.uploads)
|
| 4526 |
+
persist({"amzn_transactions"})
|
| 4527 |
+
moved = [p for p in rt.uploads[start:] if is_part(p)]
|
| 4528 |
+
ck("new rows in one month upload only that month's parts", lambda:
|
| 4529 |
+
moved and all(".p2026-03" in p for p in moved)
|
| 4530 |
+
and set(txn_keys()) - set(before) == {p[len(MIRROR_PREFIX):] for p in moved}
|
| 4531 |
+
and [k for k in before if ".p2026-01" in k] == [k for k in txn_keys() if ".p2026-01" in k])
|
| 4532 |
+
expected = dump()
|
| 4533 |
+
wipe()
|
| 4534 |
+
restored = restore()
|
| 4535 |
+
ck("a chunked snapshot restores every row exactly", lambda:
|
| 4536 |
+
restored["state"] == "restored" and dump() == expected)
|
| 4537 |
+
_purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)])
|
| 4538 |
+
ck("purge deletes every part of a chunked table", lambda:
|
| 4539 |
+
not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"})
|
| 4540 |
finally:
|
| 4541 |
forget()
|
| 4542 |
con.close()
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
8a504ee35407fb8885493ae89b253f1a55072c7b026dd7e7861a13023e5a9182
|