Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- platform/harness/amazon_store.py +126 -3
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "1443cd50c99f",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
1443cd50c99f
|
platform/harness/amazon_store.py
CHANGED
|
@@ -732,6 +732,10 @@ BRAND_EMPTY_STOP = 12
|
|
| 732 |
ROLLING_FLOOR_MARGIN = timedelta(days=1)
|
| 733 |
#: Per-table marker whose detail is the ISO UTC end of the last successfully landed window.
|
| 734 |
WATERMARK_PREFIX = "sync_mark_"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 735 |
#: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one.
|
| 736 |
MAX_WINDOW_DAYS = 30
|
| 737 |
#: R18: the most recent month lands before any history, so a newly connected account has usable
|
|
@@ -9704,13 +9708,53 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9704 |
invalid_proofs=invalid_proofs, reopened_tables=reopened)
|
| 9705 |
prior_refusals = {row["table"]: row for row in _refused_reports(con)}
|
| 9706 |
exhausted = {table for table in want if prior_refusals.get(table, {}).get("state") == "failed"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9707 |
|
| 9708 |
def current_table(table):
|
| 9709 |
# A cadence wait can reuse completed rows, never a failed/bounded checkpoint
|
| 9710 |
# or a legacy month whose completion proof still needs repair.
|
| 9711 |
return (table not in repairs and table not in invalid_proofs
|
| 9712 |
and (_read_state(con, entity_key(table)) or {}).get("state") == "done"
|
| 9713 |
-
and bool(read_watermark(con, table) or table in completed
|
|
|
|
| 9714 |
if first_load:
|
| 9715 |
# D-11622: each loaded table's own floor decides its refresh, whatever `refresh_loaded` says.
|
| 9716 |
loaded = [table for table in want if table not in exhausted
|
|
@@ -9753,7 +9797,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9753 |
# bounded; deferring it too froze every loaded table behind one refused report.
|
| 9754 |
for table in want:
|
| 9755 |
# An entity snapshot owes no history, so the first load never holds it.
|
| 9756 |
-
if ((read_watermark(con, table) and (not refresh_loaded or table not in repairs))
|
| 9757 |
or table in completed or table in FEEDBACK_TABLES
|
| 9758 |
or (table == "settlements" and not refresh_loaded)):
|
| 9759 |
continue
|
|
@@ -9769,6 +9813,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9769 |
and (not refresh_loaded or table not in repairs))
|
| 9770 |
or table in completed or table in FEEDBACK_TABLES
|
| 9771 |
or (table == "settlements" and not refresh_loaded)]
|
|
|
|
|
|
|
| 9772 |
if scheduled and not first_load:
|
| 9773 |
ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
|
| 9774 |
# A table standing rule 8 held is asked again at this pass, with its pair, inside Amazon's own floor.
|
|
@@ -9965,6 +10011,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9965 |
else:
|
| 9966 |
for table in pending:
|
| 9967 |
mark = read_watermark(con, table)
|
|
|
|
|
|
|
|
|
|
| 9968 |
if table == "settlements" and since is None:
|
| 9969 |
# Empty discovery earns a request cutoff, never a stored source watermark.
|
| 9970 |
tick = _read_state(con, entity_key("sync_tick_" + table))
|
|
@@ -10610,7 +10659,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 10610 |
and read_watermark(con, table) is not None):
|
| 10611 |
# A reopened history pass never rewinds the recent data the scheduled refresh landed.
|
| 10612 |
mark = max(mark, read_watermark(con, table))
|
| 10613 |
-
if mark is not None and has_window:
|
|
|
|
|
|
|
|
|
|
| 10614 |
record_watermark(con, table, mark, total_rows)
|
| 10615 |
if not first_load and not resync:
|
| 10616 |
completed_plan = ({**plan, "incremental": True}
|
|
@@ -13513,6 +13565,77 @@ def group_a_selftest(log=print, check=None):
|
|
| 13513 |
return len(requests) == 3 and len(rows) == 2 and requests[1]["createdBefore"] == requests[2]["createdAfter"]
|
| 13514 |
ck("Listings search splits date ranges at the real thousand-SKU ceiling",
|
| 13515 |
not partition_probe(False) and partition_probe(True))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13516 |
ck("catalog Parent ASIN map fixtures and red-first controls", catalog_parent_selftest(log, check) == 0)
|
| 13517 |
return 1 if failures else 0
|
| 13518 |
|
|
|
|
| 732 |
ROLLING_FLOOR_MARGIN = timedelta(days=1)
|
| 733 |
#: Per-table marker whose detail is the ISO UTC end of the last successfully landed window.
|
| 734 |
WATERMARK_PREFIX = "sync_mark_"
|
| 735 |
+
#: D-17409: the end of a table's last refresh while no first-load job serves its tenant. Never a watermark.
|
| 736 |
+
CATCH_UP_PREFIX = "catchup_"
|
| 737 |
+
#: A first load whose job lease, chain record and own record have not moved for this long serves nobody.
|
| 738 |
+
FIRST_LOAD_IDLE_SECONDS = 24 * 3600
|
| 739 |
#: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one.
|
| 740 |
MAX_WINDOW_DAYS = 30
|
| 741 |
#: R18: the most recent month lands before any history, so a newly connected account has usable
|
|
|
|
| 9708 |
invalid_proofs=invalid_proofs, reopened_tables=reopened)
|
| 9709 |
prior_refusals = {row["table"]: row for row in _refused_reports(con)}
|
| 9710 |
exhausted = {table for table in want if prior_refusals.get(table, {}).get("state") == "failed"}
|
| 9711 |
+
# D-17409: only a first-load job pulls history now. With none serving this tenant, a table the first
|
| 9712 |
+
# load owes refreshes from its last refresh, else the frozen first-load end, and keeps its history owed.
|
| 9713 |
+
catch_up = {}
|
| 9714 |
+
|
| 9715 |
+
def first_load_served():
|
| 9716 |
+
if first_load_job_live(rt):
|
| 9717 |
+
return True
|
| 9718 |
+
try:
|
| 9719 |
+
# A first-load pass rewrites this record, so a recent stamp means a producer is running.
|
| 9720 |
+
if (end - _utc(load["updated_at"])).total_seconds() <= FIRST_LOAD_IDLE_SECONDS:
|
| 9721 |
+
return True
|
| 9722 |
+
except (KeyError, TypeError, ValueError, AttributeError, OverflowError):
|
| 9723 |
+
pass
|
| 9724 |
+
chain = _doc(rt, FIRST_LOAD_GENERATIONS_KEY)
|
| 9725 |
+
try:
|
| 9726 |
+
idle = (end - _utc(chain["updatedAt"])).total_seconds()
|
| 9727 |
+
return int(chain.get("remaining") or 0) > 0 and idle <= FIRST_LOAD_IDLE_SECONDS
|
| 9728 |
+
except (KeyError, TypeError, ValueError, AttributeError, OverflowError):
|
| 9729 |
+
return False
|
| 9730 |
+
|
| 9731 |
+
if (scheduled and not first_load and not refresh_loaded and load and load["state"] != "done"
|
| 9732 |
+
and _store_documents(rt) and not first_load_served()):
|
| 9733 |
+
try:
|
| 9734 |
+
cohort = min(_utc(load["detail"]), end) if load.get("detail") else None
|
| 9735 |
+
except (TypeError, ValueError, AttributeError, OverflowError):
|
| 9736 |
+
cohort = None
|
| 9737 |
+
cohort = cohort or end - timedelta(days=FIRST_LOAD_RECENT_DAYS)
|
| 9738 |
+
for table in want:
|
| 9739 |
+
if (read_watermark(con, table) or table in completed or table in FEEDBACK_TABLES
|
| 9740 |
+
or table == "settlements"):
|
| 9741 |
+
continue
|
| 9742 |
+
landed = (_read_state(con, entity_key(CATCH_UP_PREFIX + table)) or {}).get("detail")
|
| 9743 |
+
try:
|
| 9744 |
+
catch_up[table] = min(_utc(landed), end) if landed else cohort
|
| 9745 |
+
except (TypeError, ValueError, AttributeError, OverflowError):
|
| 9746 |
+
catch_up[table] = cohort
|
| 9747 |
+
if catch_up:
|
| 9748 |
+
log(f"no first-load job serves {tenant_key}: {', '.join(catch_up)} refresh from their last "
|
| 9749 |
+
"landing; their history stays owed to the first-load job")
|
| 9750 |
|
| 9751 |
def current_table(table):
|
| 9752 |
# A cadence wait can reuse completed rows, never a failed/bounded checkpoint
|
| 9753 |
# or a legacy month whose completion proof still needs repair.
|
| 9754 |
return (table not in repairs and table not in invalid_proofs
|
| 9755 |
and (_read_state(con, entity_key(table)) or {}).get("state") == "done"
|
| 9756 |
+
and bool(read_watermark(con, table) or table in completed
|
| 9757 |
+
or (table in catch_up and _read_state(con, entity_key(CATCH_UP_PREFIX + table)))))
|
| 9758 |
if first_load:
|
| 9759 |
# D-11622: each loaded table's own floor decides its refresh, whatever `refresh_loaded` says.
|
| 9760 |
loaded = [table for table in want if table not in exhausted
|
|
|
|
| 9797 |
# bounded; deferring it too froze every loaded table behind one refused report.
|
| 9798 |
for table in want:
|
| 9799 |
# An entity snapshot owes no history, so the first load never holds it.
|
| 9800 |
+
if (table in catch_up or (read_watermark(con, table) and (not refresh_loaded or table not in repairs))
|
| 9801 |
or table in completed or table in FEEDBACK_TABLES
|
| 9802 |
or (table == "settlements" and not refresh_loaded)):
|
| 9803 |
continue
|
|
|
|
| 9813 |
and (not refresh_loaded or table not in repairs))
|
| 9814 |
or table in completed or table in FEEDBACK_TABLES
|
| 9815 |
or (table == "settlements" and not refresh_loaded)]
|
| 9816 |
+
# A catch-up table stays in this pass; the planner below decides when it is due.
|
| 9817 |
+
want += [table for table in catch_up if table not in want]
|
| 9818 |
if scheduled and not first_load:
|
| 9819 |
ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
|
| 9820 |
# A table standing rule 8 held is asked again at this pass, with its pair, inside Amazon's own floor.
|
|
|
|
| 10011 |
else:
|
| 10012 |
for table in pending:
|
| 10013 |
mark = read_watermark(con, table)
|
| 10014 |
+
if not mark and since is None and table in catch_up:
|
| 10015 |
+
# An incremental window keeps the table's rows; its history stays the first load's.
|
| 10016 |
+
mark = catch_up[table]
|
| 10017 |
if table == "settlements" and since is None:
|
| 10018 |
# Empty discovery earns a request cutoff, never a stored source watermark.
|
| 10019 |
tick = _read_state(con, entity_key("sync_tick_" + table))
|
|
|
|
| 10659 |
and read_watermark(con, table) is not None):
|
| 10660 |
# A reopened history pass never rewinds the recent data the scheduled refresh landed.
|
| 10661 |
mark = max(mark, read_watermark(con, table))
|
| 10662 |
+
if mark is not None and has_window and table in catch_up and not read_watermark(con, table):
|
| 10663 |
+
# A catch-up landed recent data only, so the history watermark stays unwritten.
|
| 10664 |
+
_record_state(con, entity_key(CATCH_UP_PREFIX + table), "done", total_rows, _iso(_utc(mark)))
|
| 10665 |
+
elif mark is not None and has_window:
|
| 10666 |
record_watermark(con, table, mark, total_rows)
|
| 10667 |
if not first_load and not resync:
|
| 10668 |
completed_plan = ({**plan, "incremental": True}
|
|
|
|
| 13565 |
return len(requests) == 3 and len(rows) == 2 and requests[1]["createdBefore"] == requests[2]["createdAfter"]
|
| 13566 |
ck("Listings search splits date ranges at the real thousand-SKU ceiling",
|
| 13567 |
not partition_probe(False) and partition_probe(True))
|
| 13568 |
+
|
| 13569 |
+
# D-17409: a table the first load owes refreshes in the scheduled pass while no first-load job runs.
|
| 13570 |
+
class CatchUpClock(datetime):
|
| 13571 |
+
instant = datetime(2026, 11, 17, 12, 0, tzinfo=timezone.utc)
|
| 13572 |
+
|
| 13573 |
+
@classmethod
|
| 13574 |
+
def now(cls, tz=None):
|
| 13575 |
+
return cls.instant.astimezone(tz)
|
| 13576 |
+
|
| 13577 |
+
def catch_up_pass(served):
|
| 13578 |
+
# served: None = a first load stalled since its frozen end, "lease" = a job holds the Amazon
|
| 13579 |
+
# lease, "fresh" = a first-load pass wrote its record an hour ago.
|
| 13580 |
+
con = duckdb.connect(":memory:")
|
| 13581 |
+
con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
|
| 13582 |
+
"cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
|
| 13583 |
+
table, cohort = "fba_customer_returns", _utc("2026-09-29T00:00:00Z")
|
| 13584 |
+
mirror = table_name(table)
|
| 13585 |
+
con.execute(f"CREATE TABLE {mirror} (return_date VARCHAR, order_id VARCHAR, sku VARCHAR, asin VARCHAR, "
|
| 13586 |
+
"fnsku VARCHAR)")
|
| 13587 |
+
con.execute(f"INSERT INTO {mirror} VALUES ('2026-08-01T10:00:00+00:00', '111-0000000-0000009', 'OLD-1', "
|
| 13588 |
+
"'B00000000A', 'X00000000A')")
|
| 13589 |
+
rt = _FixtureSyncRuntime("group-a-catch-up", (table,), "group-a-account")
|
| 13590 |
+
asked = []
|
| 13591 |
+
|
| 13592 |
+
def returns(_fields, report, **kwargs):
|
| 13593 |
+
asked.append(kwargs)
|
| 13594 |
+
return ("return-date\torder-id\tsku\tasin\tfnsku\tquantity\tdetailed-disposition\n"
|
| 13595 |
+
"2026-10-01T10:00:00+00:00\t111-0000000-0000001\tSKU-1\tB000000001\tX000000001\t1\tSELLABLE\n",
|
| 13596 |
+
{"reportDocumentId": "fixture-catch-up", "createdTime": "2026-11-17T11:00:00Z"})
|
| 13597 |
+
try:
|
| 13598 |
+
with _fixture_sync_store(con, rt), \
|
| 13599 |
+
patch.object(module, "datetime", CatchUpClock), \
|
| 13600 |
+
patch.object(module, "credential", return_value={"fixture": True}), \
|
| 13601 |
+
patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
|
| 13602 |
+
patch.object(module, "_sync_event"), \
|
| 13603 |
+
patch.object(module, "_ensure_seller_first_month"), \
|
| 13604 |
+
patch.object(cx, "report_document", side_effect=returns), \
|
| 13605 |
+
patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \
|
| 13606 |
+
patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")):
|
| 13607 |
+
now = datetime(2026, 11, 17, 12, 0, tzinfo=timezone.utc)
|
| 13608 |
+
CatchUpClock.instant = now - timedelta(hours=1) if served == "fresh" else cohort
|
| 13609 |
+
with _fixture_account_state(con, rt, write=True) as account:
|
| 13610 |
+
record_watermark(account, "orders", cohort)
|
| 13611 |
+
_record_state(account, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(cohort))
|
| 13612 |
+
CatchUpClock.instant = now
|
| 13613 |
+
if served == "lease":
|
| 13614 |
+
acquire_amazon_lease(rt, "group-a-first-load-job")
|
| 13615 |
+
first = (sync(rt.key, tables=(table,), rt=rt, log=lambda *_: None, scheduled=True)
|
| 13616 |
+
.get("tables") or {}).get(table) or {}
|
| 13617 |
+
first_asked = list(asked)
|
| 13618 |
+
second = sync(rt.key, tables=(table,), rt=rt, log=lambda *_: None, scheduled=True)
|
| 13619 |
+
with _fixture_account_state(con, rt) as account:
|
| 13620 |
+
marks = (read_watermark(account, table),
|
| 13621 |
+
_read_state(account, entity_key(CATCH_UP_PREFIX + table)),
|
| 13622 |
+
_read_state(account, entity_key(PHASE_HOLD_PREFIX + table)))
|
| 13623 |
+
kept = con.execute(f"SELECT count(*) FROM {mirror} WHERE sku = 'OLD-1'").fetchone()[0]
|
| 13624 |
+
holds = module._MIRROR_STORE._publication_holds(second, ()) or {}
|
| 13625 |
+
return first, first_asked, (second.get("tables") or {}).get(table) or {}, marks, kept, holds
|
| 13626 |
+
finally:
|
| 13627 |
+
con.close()
|
| 13628 |
+
first, first_asked, second, marks, kept, holds = catch_up_pass(None)
|
| 13629 |
+
controls = [catch_up_pass(mode) for mode in ("lease", "fresh")]
|
| 13630 |
+
ck("with no first-load job a scheduled pass refreshes a table the first load owes and leaves it current",
|
| 13631 |
+
first.get("state") not in ("deferred", "failed", "throttled") and first.get("rows") == 2
|
| 13632 |
+
and "since watermark 2026-09-29T00:00:00Z" in first.get("window", "")
|
| 13633 |
+
and [str(a.get("start", ""))[:10] for a in first_asked] == ["2026-09-18", "2026-10-18"]
|
| 13634 |
+
and marks[0] is None and (marks[1] or {}).get("state") == "done" and marks[2] is None
|
| 13635 |
+
and kept == 1 and second.get("current") is True
|
| 13636 |
+
and "fba_customer_returns" not in (holds.get("held") or {})
|
| 13637 |
+
and all(nc[0].get("state") == "deferred" and "first load" in nc[0].get("reason", "")
|
| 13638 |
+
and not nc[1] and nc[3][0] is None and nc[3][1] is None for nc in controls))
|
| 13639 |
ck("catalog Parent ASIN map fixtures and red-first controls", catalog_parent_selftest(log, check) == 0)
|
| 13640 |
return 1 if failures else 0
|
| 13641 |
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
caf50b7cd08f7db147501d1945d38430751ce78c8ff308df354e5978316f1dd7
|