fsanyoto commited on
Commit
d4b0913
·
verified ·
1 Parent(s): 9c91770

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "e454dec (e454dec)",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "17b3233 (17b3233)",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- e454dec (e454dec)
 
1
+ 17b3233 (17b3233)
api/main.py CHANGED
@@ -1942,6 +1942,25 @@ def _project_desk(outcome, tenant):
1942
  print(f"[aios-api] sync now projection failed: {type(e).__name__}: {e}")
1943
 
1944
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1945
  def _resync_boot_recent(last_pass_at, now, every):
1946
  """Did the last scheduled pass finish less than one cadence ago? Pure; None means no."""
1947
  try:
@@ -2009,6 +2028,7 @@ def _store_resync_loop():
2009
  last_relational_success_at = _memory.get("fullRebuildAt")
2010
  last_source_fingerprints = _memory.get("fingerprints")
2011
  last_good_pass_at = _memory.get("passAt") or _memory.get("lastGoodSyncAt")
 
2012
  last_meta_fingerprint = None
2013
  last_meta_applied_at = None
2014
  # ⚠ AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
@@ -2103,7 +2123,10 @@ def _store_resync_loop():
2103
  # the disposable DuckDB mirror up (Odoo reads, no store; the dashboards read it, W58) and
2104
  # restores Amazon's saved mirror. The store-touching half (Meta and Amazon vendor pulls,
2105
  # the relational rebuild) waits for the pass that is actually due.
2106
- _boot_catch_up = not passes and _resync_boot_recent(last_good_pass_at, _t.time(), _every)
 
 
 
2107
  # ⛔⛔ HOTFIX 2026-09-24 — A DESK WAKE IS NOT A SCHEDULED PASS.
2108
  # An Amazon first load requeues itself after every bounded chunk, and each requeue SETS the
2109
  # wake event. This loop used to answer every wake with the whole scheduled pass: an Odoo
@@ -2147,8 +2170,18 @@ def _store_resync_loop():
2147
  f"{_err['type']}: {_err['message']}")
2148
  if _boot_catch_up and not _odoo_requested:
2149
  # Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
2150
- _pull_amazon("boot", False)
 
2151
  _age = max(0, int(_t.time() - last_good_pass_at))
 
 
 
 
 
 
 
 
 
2152
  print(f"[aios-api] store resync: restarted {_age}s after the last scheduled pass, so "
2153
  f"only the local mirror was caught up and no store table was rewritten; the "
2154
  f"next scheduled pass is due in {max(0, _every - _age)}s")
@@ -2406,8 +2439,9 @@ try:
2406
  from harness import datastore as _ds_boot
2407
  if not _ds_boot.DB_PATH.exists():
2408
  import threading as _threading_seed
2409
- _threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True,
2410
- name="mirror-seed").start()
 
2411
  except Exception as e: # noqa: BLE001
2412
  print(f"[aios-api] mirror seed not scheduled: {e}")
2413
 
 
1942
  print(f"[aios-api] sync now projection failed: {type(e).__name__}: {e}")
1943
 
1944
 
1945
+ #: HOTFIX 2026-09-29: how often, and how many times, a failed restart catch-up is retried. It
1946
+ #: touches no store (DuckDB and Odoo only), so retrying is free for Neon and DO; waiting for the
1947
+ #: next daily pass instead would leave the dashboards on the seed for up to a day.
1948
+ _CATCH_UP_RETRY_SECONDS = 300
1949
+ _CATCH_UP_RETRIES = 12
1950
+
1951
+
1952
+ def _await_boot_seed(timeout=900):
1953
+ """The first resync pass must not race the boot seed's file install.
1954
+
1955
+ Staging 2026-09-29 14:07:44Z: the seed landed and the first pass died on DuckDB
1956
+ `ConnectionException` in the same second. The seed thread exists only on a fresh disk.
1957
+ """
1958
+ thread = globals().get("_SEED_THREAD")
1959
+ if thread is not None and thread.is_alive():
1960
+ print("[aios-api] store resync waits for the boot seed to finish installing")
1961
+ thread.join(timeout)
1962
+
1963
+
1964
  def _resync_boot_recent(last_pass_at, now, every):
1965
  """Did the last scheduled pass finish less than one cadence ago? Pure; None means no."""
1966
  try:
 
2028
  last_relational_success_at = _memory.get("fullRebuildAt")
2029
  last_source_fingerprints = _memory.get("fingerprints")
2030
  last_good_pass_at = _memory.get("passAt") or _memory.get("lastGoodSyncAt")
2031
+ catch_up_retries = 0
2032
  last_meta_fingerprint = None
2033
  last_meta_applied_at = None
2034
  # ⚠ AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
 
2123
  # the disposable DuckDB mirror up (Odoo reads, no store; the dashboards read it, W58) and
2124
  # restores Amazon's saved mirror. The store-touching half (Meta and Amazon vendor pulls,
2125
  # the relational rebuild) waits for the pass that is actually due.
2126
+ if not passes:
2127
+ _await_boot_seed()
2128
+ _boot_catch_up = ((not passes or catch_up_retries)
2129
+ and _resync_boot_recent(last_good_pass_at, _t.time(), _every))
2130
  # ⛔⛔ HOTFIX 2026-09-24 — A DESK WAKE IS NOT A SCHEDULED PASS.
2131
  # An Amazon first load requeues itself after every bounded chunk, and each requeue SETS the
2132
  # wake event. This loop used to answer every wake with the whole scheduled pass: an Odoo
 
2170
  f"{_err['type']}: {_err['message']}")
2171
  if _boot_catch_up and not _odoo_requested:
2172
  # Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
2173
+ if not catch_up_retries:
2174
+ _pull_amazon("boot", False)
2175
  _age = max(0, int(_t.time() - last_good_pass_at))
2176
+ if _err is not None and catch_up_retries < _CATCH_UP_RETRIES:
2177
+ # The catch-up failed: retry it soon (it is free for the store), never the store half.
2178
+ catch_up_retries += 1
2179
+ print(f"[aios-api] store resync: the restart catch-up failed ({_err['type']}); "
2180
+ f"retry {catch_up_retries} of {_CATCH_UP_RETRIES} in "
2181
+ f"{_CATCH_UP_RETRY_SECONDS}s, and no store table was rewritten")
2182
+ last_full_pass_at = _t.time() - _every + _CATCH_UP_RETRY_SECONDS
2183
+ continue
2184
+ catch_up_retries = 0
2185
  print(f"[aios-api] store resync: restarted {_age}s after the last scheduled pass, so "
2186
  f"only the local mirror was caught up and no store table was rewritten; the "
2187
  f"next scheduled pass is due in {max(0, _every - _age)}s")
 
2439
  from harness import datastore as _ds_boot
2440
  if not _ds_boot.DB_PATH.exists():
2441
  import threading as _threading_seed
2442
+ _SEED_THREAD = _threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True,
2443
+ name="mirror-seed")
2444
+ _SEED_THREAD.start()
2445
  except Exception as e: # noqa: BLE001
2446
  print(f"[aios-api] mirror seed not scheduled: {e}")
2447
 
api/test_odoo_rank_refresh.py CHANGED
@@ -188,7 +188,10 @@ class RankRefreshTests(unittest.TestCase):
188
  with duckdb.connect(self.path) as con:
189
  projection[:] = sorted(r[rel.JOIN_KEY] for r in rel.read_customers(con))
190
  return True
191
- clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + current_pass[0] * 60,
 
 
 
192
  strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
193
  ns = dict(RESYNC_STATE={}, _RESYNC_NOTICE=[None],
194
  _resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False,
@@ -291,21 +294,21 @@ class RankRefreshTests(unittest.TestCase):
291
  attempts, projection, sleeps, reports = self.observe_resident_recovery({2})
292
  self.assertEqual(reports, [0, 1, 0, 0])
293
  self.assertEqual(attempts, [1, 2, 3])
294
- self.assertEqual(projection, [1, 2])
295
  self.assertEqual(sleeps, [60] * 3)
296
 
297
  def test_repeated_rank_rebuild_failure_retries_at_configured_cadence(self):
298
  attempts, projection, sleeps, reports = self.observe_resident_recovery({2, 3})
299
  self.assertEqual(reports, [0, 1, 0, 0, 0])
300
  self.assertEqual(attempts, [1, 2, 3, 4])
301
- self.assertEqual(projection, [1, 2])
302
  self.assertEqual(sleeps, [60] * 4)
303
 
304
  def test_successful_rank_rebuild_still_skips_unchanged_pass(self):
305
  attempts, projection, sleeps, reports = self.observe_resident_recovery(set())
306
  self.assertEqual(reports, [0, 1, 0])
307
  self.assertEqual(attempts, [1, 2])
308
- self.assertEqual(projection, [1, 2])
309
  self.assertEqual(sleeps, [60] * 2)
310
 
311
  def test_unchanged_refresh_remains_a_confirmed_noop(self):
@@ -376,7 +379,8 @@ class RankRefreshTests(unittest.TestCase):
376
  with duckdb.connect(self.path) as con:
377
  con.execute('CREATE TABLE sale_order (partner_id INTEGER, state VARCHAR)')
378
  con.execute('CREATE TABLE account_move (partner_id INTEGER, state VARCHAR, move_type VARCHAR)')
379
- self.assertEqual({r[rel.JOIN_KEY] for r in rel.read_customers(con)}, {1, 2})
 
380
  self.assertEqual(ds._state(con, 'res_partner')['cursor_wd'], '2026-09-03 01:02:25')
381
 
382
  def test_completed_refresh_repeats_and_clears_rank(self):
@@ -473,8 +477,9 @@ class RankRefreshTests(unittest.TestCase):
473
  self.test_rank_change_behind_cursor_reaches_customer_reader()
474
 
475
  # ── HOTFIX 2026-09-29 (owner): "only sync daily at the fastest", and only what changed ──────
476
- def run_daily_loop(self, memory, current, *, now=10_000.0, cadence=86400, manual=False):
477
- """Drive the real loop for ONE pass with a persisted memory; stop at its first wait."""
 
478
  source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
479
  functions = {'_store_resync_loop', '_resync_rebuild_reason',
480
  '_meta_rebuild_force_reason', '_resync_boot_recent'}
@@ -486,12 +491,19 @@ class RankRefreshTests(unittest.TestCase):
486
  for target in node.targets)]
487
  events, sleeps = [], []
488
 
 
 
489
  def sleep(seconds):
490
  sleeps.append(seconds)
491
- raise StopIteration('bounded synthetic resident loop')
 
 
492
 
493
  def sync(manual_flag, passes):
494
  events.append(('sync', passes))
 
 
 
495
  return (None if manual_flag else {'res_partner': {'phase': 'live', 'pulled': 0}},
496
  None if manual_flag else {}, None)
497
 
@@ -501,7 +513,7 @@ class RankRefreshTests(unittest.TestCase):
501
 
502
  clock = SimpleNamespace(sleep=sleep, time=lambda: now,
503
  strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
504
- ns = dict(RESYNC_STATE={}, _RESYNC_NOTICE=[None],
505
  _resolve_resync_cadence=lambda: dict(
506
  resolvedSeconds=cadence, manual=manual, tenant='synthetic',
507
  tenantSource='test', tenantSeconds=cadence, envSeconds=cadence,
@@ -541,6 +553,17 @@ class RankRefreshTests(unittest.TestCase):
541
  self.assertEqual(events, [('sync', 1), ('amazon', 'boot', False)])
542
  self.assertEqual(sleeps, [86400 - 3600])
543
 
 
 
 
 
 
 
 
 
 
 
 
544
  def test_restart_after_the_day_runs_the_scheduled_pass(self):
545
  fps = {'res_partner': (3, 7)}
546
  events, _ = self.run_daily_loop(
 
188
  with duckdb.connect(self.path) as con:
189
  projection[:] = sorted(r[rel.JOIN_KEY] for r in rel.read_customers(con))
190
  return True
191
+ # Time moves at the SLEEP, not inside a pass: the loop's wait is the cadence minus the pass's
192
+ # own duration (hotfix 2026-09-29), so a clock that jumped a whole cadence inside
193
+ # every pass would ask for a zero wait.
194
+ clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + cycle[0] * 60,
195
  strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
196
  ns = dict(RESYNC_STATE={}, _RESYNC_NOTICE=[None],
197
  _resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False,
 
294
  attempts, projection, sleeps, reports = self.observe_resident_recovery({2})
295
  self.assertEqual(reports, [0, 1, 0, 0])
296
  self.assertEqual(attempts, [1, 2, 3])
297
+ self.assertEqual(projection, [1, 2, 3]) # rank > 0 across both active states (W29 R12)
298
  self.assertEqual(sleeps, [60] * 3)
299
 
300
  def test_repeated_rank_rebuild_failure_retries_at_configured_cadence(self):
301
  attempts, projection, sleeps, reports = self.observe_resident_recovery({2, 3})
302
  self.assertEqual(reports, [0, 1, 0, 0, 0])
303
  self.assertEqual(attempts, [1, 2, 3, 4])
304
+ self.assertEqual(projection, [1, 2, 3]) # rank > 0 across both active states (W29 R12)
305
  self.assertEqual(sleeps, [60] * 4)
306
 
307
  def test_successful_rank_rebuild_still_skips_unchanged_pass(self):
308
  attempts, projection, sleeps, reports = self.observe_resident_recovery(set())
309
  self.assertEqual(reports, [0, 1, 0])
310
  self.assertEqual(attempts, [1, 2])
311
+ self.assertEqual(projection, [1, 2, 3]) # rank > 0 across both active states (W29 R12)
312
  self.assertEqual(sleeps, [60] * 2)
313
 
314
  def test_unchanged_refresh_remains_a_confirmed_noop(self):
 
379
  with duckdb.connect(self.path) as con:
380
  con.execute('CREATE TABLE sale_order (partner_id INTEGER, state VARCHAR)')
381
  con.execute('CREATE TABLE account_move (partner_id INTEGER, state VARCHAR, move_type VARCHAR)')
382
+ # Partner 3 is archived with rank 2: a customer under W29 R12 (rank > 0, both states).
383
+ self.assertEqual({r[rel.JOIN_KEY] for r in rel.read_customers(con)}, {1, 2, 3})
384
  self.assertEqual(ds._state(con, 'res_partner')['cursor_wd'], '2026-09-03 01:02:25')
385
 
386
  def test_completed_refresh_repeats_and_clears_rank(self):
 
477
  self.test_rank_change_behind_cursor_reaches_customer_reader()
478
 
479
  # ── HOTFIX 2026-09-29 (owner): "only sync daily at the fastest", and only what changed ──────
480
+ def run_daily_loop(self, memory, current, *, now=10_000.0, cadence=86400, manual=False,
481
+ errors=(), max_sleeps=1):
482
+ """Drive the real loop with a persisted memory; stop at its `max_sleeps`-th wait."""
483
  source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
484
  functions = {'_store_resync_loop', '_resync_rebuild_reason',
485
  '_meta_rebuild_force_reason', '_resync_boot_recent'}
 
491
  for target in node.targets)]
492
  events, sleeps = [], []
493
 
494
+ failures = list(errors)
495
+
496
  def sleep(seconds):
497
  sleeps.append(seconds)
498
+ if len(sleeps) >= max_sleeps:
499
+ raise StopIteration('bounded synthetic resident loop')
500
+ return False
501
 
502
  def sync(manual_flag, passes):
503
  events.append(('sync', passes))
504
+ err = failures.pop(0) if failures else None
505
+ if err is not None:
506
+ return None, None, err
507
  return (None if manual_flag else {'res_partner': {'phase': 'live', 'pulled': 0}},
508
  None if manual_flag else {}, None)
509
 
 
513
 
514
  clock = SimpleNamespace(sleep=sleep, time=lambda: now,
515
  strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
516
+ ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None],
517
  _resolve_resync_cadence=lambda: dict(
518
  resolvedSeconds=cadence, manual=manual, tenant='synthetic',
519
  tenantSource='test', tenantSeconds=cadence, envSeconds=cadence,
 
553
  self.assertEqual(events, [('sync', 1), ('amazon', 'boot', False)])
554
  self.assertEqual(sleeps, [86400 - 3600])
555
 
556
+ def test_failed_restart_catch_up_retries_soon_without_store_work(self):
557
+ fps = {'res_partner': (3, 7)}
558
+ err = {'kind': 'transient', 'type': 'ConnectionException', 'message': 'seed race',
559
+ 'stage': 'mirrorPull', 'code': 'mirror_pull_failed', 'meaning': 'retry'}
560
+ events, sleeps = self.run_daily_loop(
561
+ {'passAt': 10_000.0 - 3600, 'fullRebuildAt': 10_000.0 - 3600, 'fingerprints': fps},
562
+ {'res_partner': (4, 9)}, errors=[err], max_sleeps=2)
563
+ # One restore, a 300 s retry of the catch-up alone, then the remainder of the day.
564
+ self.assertEqual(events, [('sync', 1), ('amazon', 'boot', False), ('sync', 2)])
565
+ self.assertEqual(sleeps, [300, 86400 - 3600])
566
+
567
  def test_restart_after_the_day_runs_the_scheduled_pass(self):
568
  fps = {'res_partner': (3, 7)}
569
  events, _ = self.run_daily_loop(
platform/core/store_pg.py CHANGED
@@ -379,18 +379,92 @@ def _q(template, kind, tenant_slug=None):
379
  bare=sql.Identifier(kind))
380
 
381
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
382
  def get(name, fresh=False, tenant_slug=None):
383
  """The value stored under `name`, or `{}` when the key is genuinely absent.
384
 
385
  `fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and
386
- there is no cache here — every read is a SELECT. Keeping the parameter means callers do not
387
- branch on the backend.
388
  """
389
- with _pool().connection() as con:
390
- row = con.execute(
391
- _q('SELECT value FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
392
- (str(name),)).fetchone()
393
- return dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
 
 
 
394
 
395
 
396
  def get_projection(name, drop=(), tenant_slug=None):
@@ -411,18 +485,23 @@ def get_projection(name, drop=(), tenant_slug=None):
411
  import core.store as _hf # noqa: PLC0415 — lazy: _project lives with the class
412
 
413
  drops = tuple(str(d) for d in drop)
414
- with _pool().connection() as con:
415
- row = con.execute(
416
- _q('SELECT CASE '
417
- "WHEN jsonb_typeof(kv.value) = 'object' THEN "
418
- "COALESCE((SELECT jsonb_object_agg(e.entry_key, "
419
- "CASE WHEN jsonb_typeof(e.entry_value) = 'object' "
420
- 'THEN e.entry_value - %s::text[] ELSE e.entry_value END) '
421
- 'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), '
422
- "'{{}}'::jsonb) ELSE kv.value END FROM {tbl} AS kv WHERE kv.key = %s",
423
- 'store_kv', tenant_slug),
424
- (list(drops), str(name))).fetchone()
425
- data = dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
 
 
 
 
 
426
  return _hf._project(data, frozenset(drops))
427
 
428
 
@@ -602,13 +681,16 @@ def put(name, data, tenant_slug=None):
602
  # pool stubbed to raise, "refused" and "reached the database" are two distinguishable answers,
603
  # which is what makes the negative control in `verify_store_pg.py` bite without a server.
604
  check_write(tenant_slug, f'put {name!r}')
605
- with _pool().connection() as con:
606
- payload = _payload(name, data, con, tenant_slug)
607
- con.execute(
608
- _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
609
- 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
610
- 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
611
- (str(name), payload))
 
 
 
612
  return data
613
 
614
 
@@ -654,6 +736,14 @@ def update(name, fn, flush='sync', tenant_slug=None, expect_rev=None, audit_row=
654
  what it loses is locality, not rows.
655
  """
656
  check_write(tenant_slug, f'update {name!r}') # D-353 — see `put`, and before `_pool()`
 
 
 
 
 
 
 
 
657
  with _pool().connection() as con:
658
  with con.transaction():
659
  row = con.execute(
 
379
  bare=sql.Identifier(kind))
380
 
381
 
382
+ #: ⭐⭐ D-2640 (hotfix 2026-09-29): IDENTICAL CONCURRENT READS SHARE ONE SELECT.
383
+ #:
384
+ #: Measured on the 2 GB DigitalOcean staging node on 2026-09-28: 14 backends each reading the whole
385
+ #: `user_tables` document (57 MB of JSON text) at once, every one detoasting it and building its own
386
+ #: copy, drove the node to 94% swap and 100% CPU. A burst of requests arriving during a sync asks for
387
+ #: the SAME document. So a read that finds an identical read already in flight (same tenant, key and
388
+ #: projection) waits for it and parses the same TEXT itself, instead of opening another backend.
389
+ #: ⚠ NOT A CACHE: nothing outlives the SELECT that produced it, so `fresh` still means nothing here.
390
+ #: ⚠ READ-YOUR-WRITES HOLDS: every `put`/`update` in this process bumps the key's generation when it
391
+ #: finishes, and a read never joins a flight started under an older generation.
392
+ #: ⚠ Each caller gets its OWN parsed object (the shared thing is an immutable str), because callers
393
+ #: mutate what `get` returns.
394
+ _READ_FLIGHTS = {}
395
+ _WRITE_GENERATION = {}
396
+ _READ_FLIGHT_LOCK = threading.Lock()
397
+
398
+
399
+ class _ReadFlight:
400
+ __slots__ = ('done', 'text', 'error', 'generation')
401
+
402
+ def __init__(self, generation):
403
+ self.done = threading.Event()
404
+ self.text = None
405
+ self.error = None
406
+ self.generation = generation
407
+
408
+
409
+ def _bump_write_generation(name, tenant_slug):
410
+ key = (_effective_slug(tenant_slug), str(name))
411
+ with _READ_FLIGHT_LOCK:
412
+ _WRITE_GENERATION[key] = _WRITE_GENERATION.get(key, 0) + 1
413
+
414
+
415
+ def _shared_read(name, drops, tenant_slug, fetch_text):
416
+ """`fetch_text()` once for every concurrent identical read. -> the value's JSON text, or None."""
417
+ doc = (_effective_slug(tenant_slug), str(name))
418
+ key = doc + (drops,)
419
+ with _READ_FLIGHT_LOCK:
420
+ generation = _WRITE_GENERATION.get(doc, 0)
421
+ flight = _READ_FLIGHTS.get(key)
422
+ leader = flight is None or flight.generation != generation
423
+ if leader:
424
+ flight = _ReadFlight(generation)
425
+ _READ_FLIGHTS[key] = flight
426
+ if not leader:
427
+ flight.done.wait()
428
+ if flight.error is not None:
429
+ raise RuntimeError(f'shared store read of {name!r} failed: '
430
+ f'{type(flight.error).__name__}') from flight.error
431
+ return flight.text
432
+ try:
433
+ flight.text = fetch_text()
434
+ except BaseException as exc:
435
+ flight.error = exc
436
+ raise
437
+ finally:
438
+ with _READ_FLIGHT_LOCK:
439
+ if _READ_FLIGHTS.get(key) is flight:
440
+ del _READ_FLIGHTS[key]
441
+ flight.done.set()
442
+ return flight.text
443
+
444
+
445
+ def _parsed(text):
446
+ """The caller's own object from the shared JSON text (`None` = the key is absent)."""
447
+ if text is None:
448
+ return {}
449
+ value = json.loads(text) if isinstance(text, (str, bytes, bytearray)) else text
450
+ return dict(value) if isinstance(value, dict) else value
451
+
452
+
453
  def get(name, fresh=False, tenant_slug=None):
454
  """The value stored under `name`, or `{}` when the key is genuinely absent.
455
 
456
  `fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and
457
+ there is no cache here — every read is a SELECT (shared only with identical reads already in
458
+ flight, D-2640). Keeping the parameter means callers do not branch on the backend.
459
  """
460
+ def fetch():
461
+ with _pool().connection() as con:
462
+ row = con.execute(
463
+ _q('SELECT value::text FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
464
+ (str(name),)).fetchone()
465
+ return row[0] if row else None
466
+
467
+ return _parsed(_shared_read(name, None, tenant_slug, fetch))
468
 
469
 
470
  def get_projection(name, drop=(), tenant_slug=None):
 
485
  import core.store as _hf # noqa: PLC0415 — lazy: _project lives with the class
486
 
487
  drops = tuple(str(d) for d in drop)
488
+
489
+ def fetch():
490
+ with _pool().connection() as con:
491
+ row = con.execute(
492
+ _q('SELECT (CASE '
493
+ "WHEN jsonb_typeof(kv.value) = 'object' THEN "
494
+ "COALESCE((SELECT jsonb_object_agg(e.entry_key, "
495
+ "CASE WHEN jsonb_typeof(e.entry_value) = 'object' "
496
+ 'THEN e.entry_value - %s::text[] ELSE e.entry_value END) '
497
+ 'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), '
498
+ "'{{}}'::jsonb) ELSE kv.value END)::text FROM {tbl} AS kv WHERE kv.key = %s",
499
+ 'store_kv', tenant_slug),
500
+ (list(drops), str(name))).fetchone()
501
+ return row[0] if row else None
502
+
503
+ # D-2640: the projection is the read 14 concurrent backends were making; share it too.
504
+ data = _parsed(_shared_read(name, tuple(sorted(drops)), tenant_slug, fetch))
505
  return _hf._project(data, frozenset(drops))
506
 
507
 
 
681
  # pool stubbed to raise, "refused" and "reached the database" are two distinguishable answers,
682
  # which is what makes the negative control in `verify_store_pg.py` bite without a server.
683
  check_write(tenant_slug, f'put {name!r}')
684
+ try:
685
+ with _pool().connection() as con:
686
+ payload = _payload(name, data, con, tenant_slug)
687
+ con.execute(
688
+ _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
689
+ 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
690
+ 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
691
+ (str(name), payload))
692
+ finally:
693
+ _bump_write_generation(name, tenant_slug) # D-2640: no read joins an older flight
694
  return data
695
 
696
 
 
736
  what it loses is locality, not rows.
737
  """
738
  check_write(tenant_slug, f'update {name!r}') # D-353 — see `put`, and before `_pool()`
739
+ try:
740
+ return _update_locked(name, fn, tenant_slug, expect_rev, audit_row)
741
+ finally:
742
+ _bump_write_generation(name, tenant_slug) # D-2640: no read joins an older flight
743
+
744
+
745
+ def _update_locked(name, fn, tenant_slug, expect_rev, audit_row):
746
+ """`update`'s transaction body (split out so the D-2640 generation bump wraps all of it)."""
747
  with _pool().connection() as con:
748
  with con.transaction():
749
  row = con.execute(
platform/harness/amazon_store.py CHANGED
@@ -1702,11 +1702,26 @@ def _blob_metadata(data):
1702
  #: restores; v2 names each blob's CONTENT key, so a manifest only ever names complete blobs and a
1703
  #: key it names is never overwritten. `keys.json` is the write-ahead ledger of every key ever
1704
  #: uploaded: the store has no list-by-prefix, so without it an orphan is unpurgeable.
1705
- MANIFEST_VERSIONS = (1, 2)
1706
- MANIFEST_VERSION = 2
 
 
 
 
 
 
 
1707
  _MANIFEST_KEY = "manifest.json"
1708
  _KEY_LEDGER = "keys.json"
1709
  _TABLE_BLOB_RE = re.compile(r"(amzn_[a-z0-9_]+)(?:\.([0-9a-f]{16}))?\.parquet")
 
 
 
 
 
 
 
 
1710
  _STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?\.json")
1711
  #: A ledger key no manifest names is a killed checkpoint's orphan once it is this old. A live
1712
  #: checkpoint publishes within minutes, so the grace never races an in-flight writer.
@@ -1734,6 +1749,28 @@ def _table_blob_key(name, metadata, *, strict=True):
1734
  return key
1735
 
1736
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1737
  def _state_blob_key(metadata, *, strict=True):
1738
  key = str(metadata.get("key") or "sync_state.json")
1739
  match = _STATE_BLOB_RE.fullmatch(key)
@@ -1744,7 +1781,8 @@ def _state_blob_key(metadata, *, strict=True):
1744
 
1745
 
1746
  def _is_mirror_blob_key(key):
1747
- return bool(_TABLE_BLOB_RE.fullmatch(str(key)) or _STATE_BLOB_RE.fullmatch(str(key)))
 
1748
 
1749
 
1750
  def _manifest_keys(manifest, *, strict=True):
@@ -1755,7 +1793,8 @@ def _manifest_keys(manifest, *, strict=True):
1755
  tables = manifest.get("tables")
1756
  for name, metadata in (tables.items() if isinstance(tables, dict) else ()):
1757
  try:
1758
- out[_table_blob_key(str(name), metadata, strict=strict)] = metadata.get("sha256")
 
1759
  except (ValueError, AttributeError, TypeError):
1760
  continue
1761
  state = manifest.get("sync_state")
@@ -1820,20 +1859,32 @@ def _validated_snapshot(tenant_key, rt, con, raw, tmp):
1820
  for name, metadata in manifest["tables"].items():
1821
  if not re.fullmatch(r"amzn_[a-z0-9_]+", name) or not isinstance(metadata, dict):
1822
  raise ValueError("Amazon mirror manifest contains an invalid table name")
1823
- data = _checked_mirror_blob(rt, _table_blob_key(name, metadata), metadata)
1824
- path = Path(tmp) / (name + ".parquet")
1825
- path.write_bytes(data)
1826
- try:
1827
- count = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0]
1828
- except Exception as exc: # noqa: BLE001
1829
- raise ValueError("Amazon mirror parquet is unreadable") from exc
 
 
 
 
 
 
1830
  if type(metadata.get("rows")) is not int or count != metadata["rows"]:
1831
  raise ValueError("Amazon mirror parquet row count mismatch")
1832
- paths[name] = str(path)
1833
  total += count
1834
  return manifest, rows, paths, total
1835
 
1836
 
 
 
 
 
 
 
1837
  def _snapshot_tables():
1838
  """Current-state tables: every report or API the client declares `snapshot`."""
1839
  cx = _client()
@@ -1944,9 +1995,9 @@ def restore_mirror(tenant_key, rt, log=None):
1944
  manifest, rows, paths, total = _validated_snapshot(tenant_key, rt, con, raw, tmp)
1945
  con.execute("BEGIN TRANSACTION")
1946
  try:
1947
- for name, path in paths.items():
1948
  con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
1949
- "SELECT * FROM read_parquet(?)", [path])
1950
  con.executemany("INSERT INTO _sync_state "
1951
  "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)", rows)
1952
  con.execute("COMMIT")
@@ -2047,6 +2098,99 @@ def _retire_blobs(tenant_key, rt, published, previous, log, *, orphans=True):
2047
  return 0
2048
 
2049
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2050
  def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print):
2051
  """Checkpoint changed tables and all Amazon markers, publishing the manifest last.
2052
 
@@ -2081,16 +2225,28 @@ def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print):
2081
  changed, pending = [], []
2082
  with tempfile.TemporaryDirectory(prefix="amazon-persist-") as tmp:
2083
  # A legacy entry (no content key) is a candidate too: the first checkpoint migrates.
2084
- legacy = {name for name, meta in manifest["tables"].items() if not meta.get("key")}
 
2085
  for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables):
2086
  if not re.fullmatch(r"amzn_[a-z0-9_]+", name):
2087
  raise ValueError("Amazon mirror table name is invalid")
 
2088
  path = Path(tmp) / (name + ".parquet")
2089
- con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
2090
- "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
 
 
 
 
 
 
 
 
 
 
 
2091
  data = path.read_bytes()
2092
  metadata = _blob_metadata(data)
2093
- prior = manifest["tables"].get(name, {})
2094
  if metadata["sha256"] == prior.get("sha256") and prior.get("key"):
2095
  continue
2096
  key = _content_key(name, ".parquet", metadata)
@@ -4325,6 +4481,62 @@ def mirror_selftest(log=print, check=None):
4325
  _purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)])
4326
  ck("purge deletes every stored version including the orphan", lambda:
4327
  orphans and not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4328
  finally:
4329
  forget()
4330
  con.close()
 
1702
  #: restores; v2 names each blob's CONTENT key, so a manifest only ever names complete blobs and a
1703
  #: key it names is never overwritten. `keys.json` is the write-ahead ledger of every key ever
1704
  #: uploaded: the store has no list-by-prefix, so without it an orphan is unpurgeable.
1705
+ #: ⭐⭐ v3 (D-2639, owner 2026-09-29 "make sure the chunking works"): a table larger than
1706
+ #: `MIRROR_PART_MAX_BYTES` is stored as PARTS, one per month of its date column (split further
1707
+ #: when a month is still too big), each under its own content key. A checkpoint re-hashes every
1708
+ #: part of a changed table and uploads only the parts whose bytes changed, so a day of new orders
1709
+ #: rewrites the current month's part instead of the whole 89 MB `amzn_transactions` value that
1710
+ #: swap-thrashed the DigitalOcean node. Tables under the cap keep the v2 single-blob entry.
1711
+ #: ⚠ A build older than v3 cannot read a v3 manifest and would ask for a first load.
1712
+ MANIFEST_VERSIONS = (1, 2, 3)
1713
+ MANIFEST_VERSION = 3
1714
  _MANIFEST_KEY = "manifest.json"
1715
  _KEY_LEDGER = "keys.json"
1716
  _TABLE_BLOB_RE = re.compile(r"(amzn_[a-z0-9_]+)(?:\.([0-9a-f]{16}))?\.parquet")
1717
+ #: v3: ONE part of a chunked table, always content-keyed: `amzn_<t>.p<part>.<sha16>.parquet`.
1718
+ _PART_BLOB_RE = re.compile(r"(amzn_[a-z0-9_]+)\.p([0-9a-z-]{1,48})\.([0-9a-f]{16})\.parquet")
1719
+ #: No single mirror blob the checkpoint writes exceeds this (D-2639's exit names 8 MB).
1720
+ MIRROR_PART_MAX_BYTES = 8 * 1024 * 1024
1721
+ #: The column a chunked table is split by, first match wins; a table with none of them is split
1722
+ #: into a fixed number of hash buckets instead (bounded, though every bucket then changes).
1723
+ _PART_COLUMNS = ("_period_start", "month", "_snapshot_date", "posted_date", "purchase_date",
1724
+ "shipment_date", "date", "snapshot_date", "last_updated_date", "settlement_start_date")
1725
  _STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?\.json")
1726
  #: A ledger key no manifest names is a killed checkpoint's orphan once it is this old. A live
1727
  #: checkpoint publishes within minutes, so the grace never races an in-flight writer.
 
1749
  return key
1750
 
1751
 
1752
+ def _part_blob_key(name, part, *, strict=True):
1753
+ """The content key of ONE part of a chunked table (v3)."""
1754
+ if not isinstance(part, dict):
1755
+ raise ValueError("Amazon mirror manifest names an invalid table part")
1756
+ key = str(part.get("key") or "")
1757
+ match = _PART_BLOB_RE.fullmatch(key)
1758
+ if not match or match.group(1) != name or (
1759
+ strict and not str(part.get("sha256") or "").startswith(match.group(3))):
1760
+ raise ValueError("Amazon mirror manifest names an invalid table part key")
1761
+ return key
1762
+
1763
+
1764
+ def _entry_blob_keys(name, metadata, *, strict=True):
1765
+ """[(key, metadata)] for one manifest table entry: its parts (v3) or its one blob."""
1766
+ if isinstance(metadata, dict) and "parts" in metadata:
1767
+ parts = metadata["parts"]
1768
+ if not isinstance(parts, list) or not parts:
1769
+ raise ValueError("Amazon mirror manifest names an empty part list")
1770
+ return [(_part_blob_key(name, part, strict=strict), part) for part in parts]
1771
+ return [(_table_blob_key(name, metadata, strict=strict), metadata)]
1772
+
1773
+
1774
  def _state_blob_key(metadata, *, strict=True):
1775
  key = str(metadata.get("key") or "sync_state.json")
1776
  match = _STATE_BLOB_RE.fullmatch(key)
 
1781
 
1782
 
1783
  def _is_mirror_blob_key(key):
1784
+ return bool(_TABLE_BLOB_RE.fullmatch(str(key)) or _PART_BLOB_RE.fullmatch(str(key))
1785
+ or _STATE_BLOB_RE.fullmatch(str(key)))
1786
 
1787
 
1788
  def _manifest_keys(manifest, *, strict=True):
 
1793
  tables = manifest.get("tables")
1794
  for name, metadata in (tables.items() if isinstance(tables, dict) else ()):
1795
  try:
1796
+ for key, blob in _entry_blob_keys(str(name), metadata, strict=strict):
1797
+ out[key] = blob.get("sha256")
1798
  except (ValueError, AttributeError, TypeError):
1799
  continue
1800
  state = manifest.get("sync_state")
 
1859
  for name, metadata in manifest["tables"].items():
1860
  if not re.fullmatch(r"amzn_[a-z0-9_]+", name) or not isinstance(metadata, dict):
1861
  raise ValueError("Amazon mirror manifest contains an invalid table name")
1862
+ files, count = [], 0
1863
+ for index, (key, blob) in enumerate(_entry_blob_keys(name, metadata)):
1864
+ data = _checked_mirror_blob(rt, key, blob)
1865
+ path = Path(tmp) / f"{name}.{index:04d}.parquet"
1866
+ path.write_bytes(data)
1867
+ try:
1868
+ part_rows = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0]
1869
+ except Exception as exc: # noqa: BLE001
1870
+ raise ValueError("Amazon mirror parquet is unreadable") from exc
1871
+ if "parts" in metadata and (type(blob.get("rows")) is not int or part_rows != blob["rows"]):
1872
+ raise ValueError("Amazon mirror parquet part row count mismatch")
1873
+ files.append(str(path))
1874
+ count += part_rows
1875
  if type(metadata.get("rows")) is not int or count != metadata["rows"]:
1876
  raise ValueError("Amazon mirror parquet row count mismatch")
1877
+ paths[name] = files
1878
  total += count
1879
  return manifest, rows, paths, total
1880
 
1881
 
1882
+ def _parquet_source(files):
1883
+ """A `read_parquet` source over one or many validated local files (our own temp paths)."""
1884
+ quoted = ", ".join("'" + str(path).replace("'", "''") + "'" for path in files)
1885
+ return f"read_parquet([{quoted}], union_by_name=true)"
1886
+
1887
+
1888
  def _snapshot_tables():
1889
  """Current-state tables: every report or API the client declares `snapshot`."""
1890
  cx = _client()
 
1995
  manifest, rows, paths, total = _validated_snapshot(tenant_key, rt, con, raw, tmp)
1996
  con.execute("BEGIN TRANSACTION")
1997
  try:
1998
+ for name, files in paths.items():
1999
  con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
2000
+ f"SELECT * FROM {_parquet_source(files)}")
2001
  con.executemany("INSERT INTO _sync_state "
2002
  "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)", rows)
2003
  con.execute("COMMIT")
 
2098
  return 0
2099
 
2100
 
2101
+ def _part_slug(value):
2102
+ slug = re.sub(r"[^0-9a-z-]", "-", str(value).lower())[:40].strip("-")
2103
+ return slug or "none"
2104
+
2105
+
2106
+ def _sql_literal(value):
2107
+ return "'" + str(value).replace("'", "''") + "'"
2108
+
2109
+
2110
+ def _copy_part(con, name, where, path):
2111
+ """One part, sorted so unchanged rows give byte-identical parquet. `where` is fully literal:
2112
+ a parameter inside `COPY (subquery)` is not portable, and the values are escaped."""
2113
+ con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)} AS t WHERE {where} ORDER BY ALL) "
2114
+ "TO ? (FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
2115
+ return path.stat().st_size
2116
+
2117
+
2118
+ def _table_parts(con, name, tmp):
2119
+ """Write `name` as bounded parts under `tmp`. -> [(slug, path, rows)], every file under the cap.
2120
+
2121
+ ⭐ SPLIT BY MONTH OF THE TABLE'S DATE COLUMN, because Amazon's incremental windows land in the
2122
+ newest months: yesterday's orders change the current month's part and leave history's bytes
2123
+ identical. `ORDER BY ALL` makes an unchanged part byte-identical from one checkpoint to the
2124
+ next, which is what lets its content key be reused rather than re-uploaded. A month still over
2125
+ the cap is split into `hash(row) % k` sub-parts (k a power of two, so the split is stable).
2126
+ """
2127
+ cols = [row[0] for row in con.execute(
2128
+ "SELECT column_name FROM information_schema.columns WHERE table_name = ? "
2129
+ "ORDER BY ordinal_position", [name]).fetchall()]
2130
+ column = next((c for c in _PART_COLUMNS if c in cols), None)
2131
+ if column:
2132
+ expr = f"coalesce(substr(CAST(t.{_quoted_identifier(column)} AS VARCHAR), 1, 7), 'none')"
2133
+ groups = [row[0] for row in con.execute(
2134
+ f"SELECT DISTINCT {expr} AS g FROM {_quoted_identifier(name)} AS t ORDER BY g").fetchall()]
2135
+ groups = groups or ["none"] # an emptied table is still one (empty) part
2136
+ else:
2137
+ total = max(1, con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0])
2138
+ buckets = 2
2139
+ while buckets < 4096 and total / buckets > 50000:
2140
+ buckets *= 2
2141
+ expr = f"CAST(hash(t) % {buckets} AS VARCHAR)"
2142
+ groups = [str(b) for b in range(buckets)]
2143
+ out = []
2144
+ for group in groups:
2145
+ slug = _part_slug(group)
2146
+ path = tmp / f"{name}.p{slug}.parquet"
2147
+ size = _copy_part(con, name, f"{expr} = {_sql_literal(group)}", path)
2148
+ if size <= MIRROR_PART_MAX_BYTES:
2149
+ out.append((slug, path, None))
2150
+ continue
2151
+ k = 2
2152
+ while True:
2153
+ files, fits = [], True
2154
+ for j in range(k):
2155
+ sub = tmp / f"{name}.p{slug}-s{j}.parquet"
2156
+ files.append((f"{slug}-s{j}", sub))
2157
+ if _copy_part(con, name, f"{expr} = {_sql_literal(group)} AND hash(t) % {k} = {j}",
2158
+ sub) > MIRROR_PART_MAX_BYTES:
2159
+ fits = False
2160
+ if fits or k >= 1024:
2161
+ out.extend((s, f, None) for s, f in files)
2162
+ break
2163
+ k *= 2
2164
+ return [(slug, path, con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0])
2165
+ for slug, path, _rows in out]
2166
+
2167
+
2168
+ def _chunked_entry(con, name, tmp, prior, named):
2169
+ """A v3 manifest entry for `name`, plus [(key, path)] of the parts that must be uploaded.
2170
+
2171
+ Returns (None, []) when every part's bytes equal the published entry's (nothing to write).
2172
+ A part whose sha the previous entry already names keeps that key and is not uploaded again.
2173
+ """
2174
+ known = {}
2175
+ for part in (prior.get("parts") or []) if isinstance(prior, dict) else []:
2176
+ if isinstance(part, dict) and part.get("sha256") and part.get("key"):
2177
+ known[part["sha256"]] = part["key"]
2178
+ parts, fresh = [], []
2179
+ for slug, path, rows in _table_parts(con, name, tmp):
2180
+ metadata = _blob_metadata(path.read_bytes())
2181
+ key = known.get(metadata["sha256"]) or _content_key(f"{name}.p{slug}", ".parquet", metadata)
2182
+ parts.append({"part": slug, "key": key, "rows": int(rows), **metadata})
2183
+ if key not in known.values() and key not in named:
2184
+ fresh.append((key, path))
2185
+ combined = hashlib.sha256("".join(p["sha256"] for p in parts).encode()).hexdigest()
2186
+ if prior.get("parts") and prior.get("sha256") == combined:
2187
+ return None, []
2188
+ entry = {"parts": parts, "rows": sum(p["rows"] for p in parts), "sha256": combined,
2189
+ "bytes": sum(p["bytes"] for p in parts),
2190
+ "written_at": _iso(datetime.now(timezone.utc))}
2191
+ return entry, fresh
2192
+
2193
+
2194
  def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print):
2195
  """Checkpoint changed tables and all Amazon markers, publishing the manifest last.
2196
 
 
2225
  changed, pending = [], []
2226
  with tempfile.TemporaryDirectory(prefix="amazon-persist-") as tmp:
2227
  # A legacy entry (no content key) is a candidate too: the first checkpoint migrates.
2228
+ legacy = {name for name, meta in manifest["tables"].items()
2229
+ if not meta.get("key") and "parts" not in meta}
2230
  for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables):
2231
  if not re.fullmatch(r"amzn_[a-z0-9_]+", name):
2232
  raise ValueError("Amazon mirror table name is invalid")
2233
+ prior = manifest["tables"].get(name, {})
2234
  path = Path(tmp) / (name + ".parquet")
2235
+ size = None
2236
+ if "parts" not in prior:
2237
+ con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
2238
+ "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
2239
+ size = path.stat().st_size
2240
+ if size is None or size > MIRROR_PART_MAX_BYTES:
2241
+ entry, fresh = _chunked_entry(con, name, Path(tmp), prior, named)
2242
+ if entry is None:
2243
+ continue
2244
+ manifest["tables"][name] = entry
2245
+ changed.append(name)
2246
+ pending.extend(fresh)
2247
+ continue
2248
  data = path.read_bytes()
2249
  metadata = _blob_metadata(data)
 
2250
  if metadata["sha256"] == prior.get("sha256") and prior.get("key"):
2251
  continue
2252
  key = _content_key(name, ".parquet", metadata)
 
4481
  _purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)])
4482
  ck("purge deletes every stored version including the orphan", lambda:
4483
  orphans and not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"})
4484
+
4485
+ # ⭐⭐ D-2639 (owner 2026-09-29 "make sure the chunking works"): a table over the cap is
4486
+ # stored as month parts, none over the cap, and a checkpoint uploads only changed parts.
4487
+ module = sys.modules[__name__]
4488
+ con.execute("CREATE OR REPLACE TABLE amzn_transactions AS SELECT "
4489
+ "'2026-0' || (1 + i % 3) || '-15' AS posted_date, i AS id, md5(i::VARCHAR) AS a, "
4490
+ "md5((i * 7)::VARCHAR) AS b FROM range(9000) t(i)")
4491
+ cap = 60000
4492
+ with patch.object(module, "MIRROR_PART_MAX_BYTES", 10 ** 9):
4493
+ persist({name for name, _kind in _amazon_objects(con)})
4494
+ whole = json.loads(rt.blobs[manifest_path])["tables"]["amzn_transactions"]
4495
+ ck("NC: under a huge cap the same table is one blob, and that blob is over the real cap",
4496
+ lambda: "parts" not in whole and whole["bytes"] > cap)
4497
+
4498
+ def is_part(path):
4499
+ return bool(re.search(r"\.p[0-9a-z-]+\.[0-9a-f]{16}\.parquet$", path))
4500
+
4501
+ def txn_keys():
4502
+ entry = json.loads(rt.blobs[manifest_path])["tables"]["amzn_transactions"]
4503
+ return [part["key"] for part in entry.get("parts", [])]
4504
+
4505
+ with patch.object(module, "MIRROR_PART_MAX_BYTES", cap):
4506
+ start = len(rt.uploads)
4507
+ chunk_report = persist({"amzn_transactions"})
4508
+ if chunk_report["problems"]:
4509
+ log(" chunked persist problems: " + " | ".join(chunk_report["problems"]))
4510
+ saved = json.loads(rt.blobs[manifest_path])
4511
+ entry = saved["tables"]["amzn_transactions"]
4512
+ written = [p for p in rt.uploads[start:] if is_part(p)]
4513
+ ck("a table over the cap is saved as parts, none over the cap, rows intact (v3)", lambda:
4514
+ saved["version"] == 3 and len(entry["parts"]) >= 3 and written
4515
+ and all(len(rt.blobs[p]) <= cap for p in written)
4516
+ and entry["rows"] == 9000 == sum(part["rows"] for part in entry["parts"])
4517
+ and {p["part"][:7] for p in entry["parts"]} == {"2026-01", "2026-02", "2026-03"})
4518
+ before = txn_keys()
4519
+ start = len(rt.uploads)
4520
+ persist({"amzn_transactions"})
4521
+ ck("an unchanged chunked table uploads no part again", lambda:
4522
+ not [p for p in rt.uploads[start:] if is_part(p)] and txn_keys() == before)
4523
+ con.execute("INSERT INTO amzn_transactions SELECT '2026-03-20', 9000 + i, "
4524
+ "md5(('x' || i)::VARCHAR), 'new' FROM range(50) t(i)")
4525
+ start = len(rt.uploads)
4526
+ persist({"amzn_transactions"})
4527
+ moved = [p for p in rt.uploads[start:] if is_part(p)]
4528
+ ck("new rows in one month upload only that month's parts", lambda:
4529
+ moved and all(".p2026-03" in p for p in moved)
4530
+ and set(txn_keys()) - set(before) == {p[len(MIRROR_PREFIX):] for p in moved}
4531
+ and [k for k in before if ".p2026-01" in k] == [k for k in txn_keys() if ".p2026-01" in k])
4532
+ expected = dump()
4533
+ wipe()
4534
+ restored = restore()
4535
+ ck("a chunked snapshot restores every row exactly", lambda:
4536
+ restored["state"] == "restored" and dump() == expected)
4537
+ _purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)])
4538
+ ck("purge deletes every part of a chunked table", lambda:
4539
+ not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"})
4540
  finally:
4541
  forget()
4542
  con.close()
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 283087dedfe0488f85ba23beaaf1222010922133299bb9d8171e2722c8f59598
 
1
+ 8a504ee35407fb8885493ae89b253f1a55072c7b026dd7e7861a13023e5a9182