fsanyoto commited on
Commit
e0b5c28
·
verified ·
1 Parent(s): 1bee71d

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "99129353b8d3",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "1443cd50c99f",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 99129353b8d3
 
1
+ 1443cd50c99f
platform/harness/amazon_store.py CHANGED
@@ -732,6 +732,10 @@ BRAND_EMPTY_STOP = 12
732
  ROLLING_FLOOR_MARGIN = timedelta(days=1)
733
  #: Per-table marker whose detail is the ISO UTC end of the last successfully landed window.
734
  WATERMARK_PREFIX = "sync_mark_"
 
 
 
 
735
  #: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one.
736
  MAX_WINDOW_DAYS = 30
737
  #: R18: the most recent month lands before any history, so a newly connected account has usable
@@ -9704,13 +9708,53 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9704
  invalid_proofs=invalid_proofs, reopened_tables=reopened)
9705
  prior_refusals = {row["table"]: row for row in _refused_reports(con)}
9706
  exhausted = {table for table in want if prior_refusals.get(table, {}).get("state") == "failed"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9707
 
9708
  def current_table(table):
9709
  # A cadence wait can reuse completed rows, never a failed/bounded checkpoint
9710
  # or a legacy month whose completion proof still needs repair.
9711
  return (table not in repairs and table not in invalid_proofs
9712
  and (_read_state(con, entity_key(table)) or {}).get("state") == "done"
9713
- and bool(read_watermark(con, table) or table in completed))
 
9714
  if first_load:
9715
  # D-11622: each loaded table's own floor decides its refresh, whatever `refresh_loaded` says.
9716
  loaded = [table for table in want if table not in exhausted
@@ -9753,7 +9797,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9753
  # bounded; deferring it too froze every loaded table behind one refused report.
9754
  for table in want:
9755
  # An entity snapshot owes no history, so the first load never holds it.
9756
- if ((read_watermark(con, table) and (not refresh_loaded or table not in repairs))
9757
  or table in completed or table in FEEDBACK_TABLES
9758
  or (table == "settlements" and not refresh_loaded)):
9759
  continue
@@ -9769,6 +9813,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9769
  and (not refresh_loaded or table not in repairs))
9770
  or table in completed or table in FEEDBACK_TABLES
9771
  or (table == "settlements" and not refresh_loaded)]
 
 
9772
  if scheduled and not first_load:
9773
  ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
9774
  # A table standing rule 8 held is asked again at this pass, with its pair, inside Amazon's own floor.
@@ -9965,6 +10011,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9965
  else:
9966
  for table in pending:
9967
  mark = read_watermark(con, table)
 
 
 
9968
  if table == "settlements" and since is None:
9969
  # Empty discovery earns a request cutoff, never a stored source watermark.
9970
  tick = _read_state(con, entity_key("sync_tick_" + table))
@@ -10610,7 +10659,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10610
  and read_watermark(con, table) is not None):
10611
  # A reopened history pass never rewinds the recent data the scheduled refresh landed.
10612
  mark = max(mark, read_watermark(con, table))
10613
- if mark is not None and has_window:
 
 
 
10614
  record_watermark(con, table, mark, total_rows)
10615
  if not first_load and not resync:
10616
  completed_plan = ({**plan, "incremental": True}
@@ -13513,6 +13565,77 @@ def group_a_selftest(log=print, check=None):
13513
  return len(requests) == 3 and len(rows) == 2 and requests[1]["createdBefore"] == requests[2]["createdAfter"]
13514
  ck("Listings search splits date ranges at the real thousand-SKU ceiling",
13515
  not partition_probe(False) and partition_probe(True))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13516
  ck("catalog Parent ASIN map fixtures and red-first controls", catalog_parent_selftest(log, check) == 0)
13517
  return 1 if failures else 0
13518
 
 
732
  ROLLING_FLOOR_MARGIN = timedelta(days=1)
733
  #: Per-table marker whose detail is the ISO UTC end of the last successfully landed window.
734
  WATERMARK_PREFIX = "sync_mark_"
735
+ #: D-17409: the end of a table's last refresh while no first-load job serves its tenant. Never a watermark.
736
+ CATCH_UP_PREFIX = "catchup_"
737
+ #: A first load whose job lease, chain record and own record have not moved for this long serves nobody.
738
+ FIRST_LOAD_IDLE_SECONDS = 24 * 3600
739
  #: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one.
740
  MAX_WINDOW_DAYS = 30
741
  #: R18: the most recent month lands before any history, so a newly connected account has usable
 
9708
  invalid_proofs=invalid_proofs, reopened_tables=reopened)
9709
  prior_refusals = {row["table"]: row for row in _refused_reports(con)}
9710
  exhausted = {table for table in want if prior_refusals.get(table, {}).get("state") == "failed"}
9711
+ # D-17409: only a first-load job pulls history now. With none serving this tenant, a table the first
9712
+ # load owes refreshes from its last refresh, else the frozen first-load end, and keeps its history owed.
9713
+ catch_up = {}
9714
+
9715
+ def first_load_served():
9716
+ if first_load_job_live(rt):
9717
+ return True
9718
+ try:
9719
+ # A first-load pass rewrites this record, so a recent stamp means a producer is running.
9720
+ if (end - _utc(load["updated_at"])).total_seconds() <= FIRST_LOAD_IDLE_SECONDS:
9721
+ return True
9722
+ except (KeyError, TypeError, ValueError, AttributeError, OverflowError):
9723
+ pass
9724
+ chain = _doc(rt, FIRST_LOAD_GENERATIONS_KEY)
9725
+ try:
9726
+ idle = (end - _utc(chain["updatedAt"])).total_seconds()
9727
+ return int(chain.get("remaining") or 0) > 0 and idle <= FIRST_LOAD_IDLE_SECONDS
9728
+ except (KeyError, TypeError, ValueError, AttributeError, OverflowError):
9729
+ return False
9730
+
9731
+ if (scheduled and not first_load and not refresh_loaded and load and load["state"] != "done"
9732
+ and _store_documents(rt) and not first_load_served()):
9733
+ try:
9734
+ cohort = min(_utc(load["detail"]), end) if load.get("detail") else None
9735
+ except (TypeError, ValueError, AttributeError, OverflowError):
9736
+ cohort = None
9737
+ cohort = cohort or end - timedelta(days=FIRST_LOAD_RECENT_DAYS)
9738
+ for table in want:
9739
+ if (read_watermark(con, table) or table in completed or table in FEEDBACK_TABLES
9740
+ or table == "settlements"):
9741
+ continue
9742
+ landed = (_read_state(con, entity_key(CATCH_UP_PREFIX + table)) or {}).get("detail")
9743
+ try:
9744
+ catch_up[table] = min(_utc(landed), end) if landed else cohort
9745
+ except (TypeError, ValueError, AttributeError, OverflowError):
9746
+ catch_up[table] = cohort
9747
+ if catch_up:
9748
+ log(f"no first-load job serves {tenant_key}: {', '.join(catch_up)} refresh from their last "
9749
+ "landing; their history stays owed to the first-load job")
9750
 
9751
  def current_table(table):
9752
  # A cadence wait can reuse completed rows, never a failed/bounded checkpoint
9753
  # or a legacy month whose completion proof still needs repair.
9754
  return (table not in repairs and table not in invalid_proofs
9755
  and (_read_state(con, entity_key(table)) or {}).get("state") == "done"
9756
+ and bool(read_watermark(con, table) or table in completed
9757
+ or (table in catch_up and _read_state(con, entity_key(CATCH_UP_PREFIX + table)))))
9758
  if first_load:
9759
  # D-11622: each loaded table's own floor decides its refresh, whatever `refresh_loaded` says.
9760
  loaded = [table for table in want if table not in exhausted
 
9797
  # bounded; deferring it too froze every loaded table behind one refused report.
9798
  for table in want:
9799
  # An entity snapshot owes no history, so the first load never holds it.
9800
+ if (table in catch_up or (read_watermark(con, table) and (not refresh_loaded or table not in repairs))
9801
  or table in completed or table in FEEDBACK_TABLES
9802
  or (table == "settlements" and not refresh_loaded)):
9803
  continue
 
9813
  and (not refresh_loaded or table not in repairs))
9814
  or table in completed or table in FEEDBACK_TABLES
9815
  or (table == "settlements" and not refresh_loaded)]
9816
+ # A catch-up table stays in this pass; the planner below decides when it is due.
9817
+ want += [table for table in catch_up if table not in want]
9818
  if scheduled and not first_load:
9819
  ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
9820
  # A table standing rule 8 held is asked again at this pass, with its pair, inside Amazon's own floor.
 
10011
  else:
10012
  for table in pending:
10013
  mark = read_watermark(con, table)
10014
+ if not mark and since is None and table in catch_up:
10015
+ # An incremental window keeps the table's rows; its history stays the first load's.
10016
+ mark = catch_up[table]
10017
  if table == "settlements" and since is None:
10018
  # Empty discovery earns a request cutoff, never a stored source watermark.
10019
  tick = _read_state(con, entity_key("sync_tick_" + table))
 
10659
  and read_watermark(con, table) is not None):
10660
  # A reopened history pass never rewinds the recent data the scheduled refresh landed.
10661
  mark = max(mark, read_watermark(con, table))
10662
+ if mark is not None and has_window and table in catch_up and not read_watermark(con, table):
10663
+ # A catch-up landed recent data only, so the history watermark stays unwritten.
10664
+ _record_state(con, entity_key(CATCH_UP_PREFIX + table), "done", total_rows, _iso(_utc(mark)))
10665
+ elif mark is not None and has_window:
10666
  record_watermark(con, table, mark, total_rows)
10667
  if not first_load and not resync:
10668
  completed_plan = ({**plan, "incremental": True}
 
13565
  return len(requests) == 3 and len(rows) == 2 and requests[1]["createdBefore"] == requests[2]["createdAfter"]
13566
  ck("Listings search splits date ranges at the real thousand-SKU ceiling",
13567
  not partition_probe(False) and partition_probe(True))
13568
+
13569
+ # D-17409: a table the first load owes refreshes in the scheduled pass while no first-load job runs.
13570
+ class CatchUpClock(datetime):
13571
+ instant = datetime(2026, 11, 17, 12, 0, tzinfo=timezone.utc)
13572
+
13573
+ @classmethod
13574
+ def now(cls, tz=None):
13575
+ return cls.instant.astimezone(tz)
13576
+
13577
+ def catch_up_pass(served):
13578
+ # served: None = a first load stalled since its frozen end, "lease" = a job holds the Amazon
13579
+ # lease, "fresh" = a first-load pass wrote its record an hour ago.
13580
+ con = duckdb.connect(":memory:")
13581
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
13582
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
13583
+ table, cohort = "fba_customer_returns", _utc("2026-09-29T00:00:00Z")
13584
+ mirror = table_name(table)
13585
+ con.execute(f"CREATE TABLE {mirror} (return_date VARCHAR, order_id VARCHAR, sku VARCHAR, asin VARCHAR, "
13586
+ "fnsku VARCHAR)")
13587
+ con.execute(f"INSERT INTO {mirror} VALUES ('2026-08-01T10:00:00+00:00', '111-0000000-0000009', 'OLD-1', "
13588
+ "'B00000000A', 'X00000000A')")
13589
+ rt = _FixtureSyncRuntime("group-a-catch-up", (table,), "group-a-account")
13590
+ asked = []
13591
+
13592
+ def returns(_fields, report, **kwargs):
13593
+ asked.append(kwargs)
13594
+ return ("return-date\torder-id\tsku\tasin\tfnsku\tquantity\tdetailed-disposition\n"
13595
+ "2026-10-01T10:00:00+00:00\t111-0000000-0000001\tSKU-1\tB000000001\tX000000001\t1\tSELLABLE\n",
13596
+ {"reportDocumentId": "fixture-catch-up", "createdTime": "2026-11-17T11:00:00Z"})
13597
+ try:
13598
+ with _fixture_sync_store(con, rt), \
13599
+ patch.object(module, "datetime", CatchUpClock), \
13600
+ patch.object(module, "credential", return_value={"fixture": True}), \
13601
+ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
13602
+ patch.object(module, "_sync_event"), \
13603
+ patch.object(module, "_ensure_seller_first_month"), \
13604
+ patch.object(cx, "report_document", side_effect=returns), \
13605
+ patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \
13606
+ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")):
13607
+ now = datetime(2026, 11, 17, 12, 0, tzinfo=timezone.utc)
13608
+ CatchUpClock.instant = now - timedelta(hours=1) if served == "fresh" else cohort
13609
+ with _fixture_account_state(con, rt, write=True) as account:
13610
+ record_watermark(account, "orders", cohort)
13611
+ _record_state(account, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(cohort))
13612
+ CatchUpClock.instant = now
13613
+ if served == "lease":
13614
+ acquire_amazon_lease(rt, "group-a-first-load-job")
13615
+ first = (sync(rt.key, tables=(table,), rt=rt, log=lambda *_: None, scheduled=True)
13616
+ .get("tables") or {}).get(table) or {}
13617
+ first_asked = list(asked)
13618
+ second = sync(rt.key, tables=(table,), rt=rt, log=lambda *_: None, scheduled=True)
13619
+ with _fixture_account_state(con, rt) as account:
13620
+ marks = (read_watermark(account, table),
13621
+ _read_state(account, entity_key(CATCH_UP_PREFIX + table)),
13622
+ _read_state(account, entity_key(PHASE_HOLD_PREFIX + table)))
13623
+ kept = con.execute(f"SELECT count(*) FROM {mirror} WHERE sku = 'OLD-1'").fetchone()[0]
13624
+ holds = module._MIRROR_STORE._publication_holds(second, ()) or {}
13625
+ return first, first_asked, (second.get("tables") or {}).get(table) or {}, marks, kept, holds
13626
+ finally:
13627
+ con.close()
13628
+ first, first_asked, second, marks, kept, holds = catch_up_pass(None)
13629
+ controls = [catch_up_pass(mode) for mode in ("lease", "fresh")]
13630
+ ck("with no first-load job a scheduled pass refreshes a table the first load owes and leaves it current",
13631
+ first.get("state") not in ("deferred", "failed", "throttled") and first.get("rows") == 2
13632
+ and "since watermark 2026-09-29T00:00:00Z" in first.get("window", "")
13633
+ and [str(a.get("start", ""))[:10] for a in first_asked] == ["2026-09-18", "2026-10-18"]
13634
+ and marks[0] is None and (marks[1] or {}).get("state") == "done" and marks[2] is None
13635
+ and kept == 1 and second.get("current") is True
13636
+ and "fba_customer_returns" not in (holds.get("held") or {})
13637
+ and all(nc[0].get("state") == "deferred" and "first load" in nc[0].get("reason", "")
13638
+ and not nc[1] and nc[3][0] is None and nc[3][1] is None for nc in controls))
13639
  ck("catalog Parent ASIN map fixtures and red-first controls", catalog_parent_selftest(log, check) == 0)
13640
  return 1 if failures else 0
13641
 
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- afc4252504d8e071ec0937b2e6a7ad64c1e00860885ad0596112eaca89d79eea
 
1
+ caf50b7cd08f7db147501d1945d38430751ce78c8ff308df354e5978316f1dd7