fsanyoto commited on
Commit
6417f0b
·
verified ·
1 Parent(s): 78749d4

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "f4881fc7",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "20c03dc2",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- f4881fc7
 
1
+ 20c03dc2
api/main.py CHANGED
@@ -880,18 +880,18 @@ def _pull_amazon(why, vendor=True):
880
  receipt = _connector_sync.request_sync("amazon_sp", tenant_key, requested_by="first load")
881
  print(f"[aios-api] amazon sync ({why}): first load "
882
  + ("queued on the sync desk" if receipt.get("accepted") else "already queued or running"))
883
- return True
884
  if not vendor:
885
  print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
886
  return True
887
  rep = _amazon.sync(tenant_key, rt=rt, scheduled=True,
 
888
  log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}"))
889
  _rot.invalidate_sources()
890
  _rot._sources()
891
  for problem in rep.get("problems") or []:
892
  print(f"[aios-api] amazon sync PROBLEM ({why}): {problem}")
893
  print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
894
- f"{table}={row.get('state', str(row.get('rows', 0)) + ' rows')}"
895
  for table, row in rep["tables"].items()))
896
  return not bool(rep.get("problems"))
897
  except Exception as exc: # noqa: BLE001
@@ -1105,8 +1105,13 @@ def _run_sync_requests(outcome=None):
1105
  if not amazon.credential(tenant, rt=rt):
1106
  raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
1107
  first_load = amazon.needs_first_load(tenant, rt=rt)
1108
- options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS} if first_load else {}
1109
- report = amazon.sync(tenant, rt=rt, scheduled=False, **options, log=lambda *a, **k: None)
 
 
 
 
 
1110
  from connectors_amazon import assert_safe_status
1111
  first_load_progress = assert_safe_status(report.get("firstLoad"))
1112
  continuing = report.get("state") == "partial"
 
880
  receipt = _connector_sync.request_sync("amazon_sp", tenant_key, requested_by="first load")
881
  print(f"[aios-api] amazon sync ({why}): first load "
882
  + ("queued on the sync desk" if receipt.get("accepted") else "already queued or running"))
 
883
  if not vendor:
884
  print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
885
  return True
886
  rep = _amazon.sync(tenant_key, rt=rt, scheduled=True,
887
+ refresh_loaded=first_load_needed,
888
  log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}"))
889
  _rot.invalidate_sources()
890
  _rot._sources()
891
  for problem in rep.get("problems") or []:
892
  print(f"[aios-api] amazon sync PROBLEM ({why}): {problem}")
893
  print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
894
+ f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
895
  for table, row in rep["tables"].items()))
896
  return not bool(rep.get("problems"))
897
  except Exception as exc: # noqa: BLE001
 
1105
  if not amazon.credential(tenant, rt=rt):
1106
  raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
1107
  first_load = amazon.needs_first_load(tenant, rt=rt)
1108
+ options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS,
1109
+ "refresh_loaded": True} if first_load else {}
1110
+ report = amazon.sync(tenant, rt=rt, scheduled=receipt.get("requestedBy") == "first load",
1111
+ **options, log=lambda *a, **k: None)
1112
+ print("[aios-api] amazon sync done (sync desk): " + ", ".join(
1113
+ f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
1114
+ for table, row in (report.get("tables") or {}).items()))
1115
  from connectors_amazon import assert_safe_status
1116
  first_load_progress = assert_safe_status(report.get("firstLoad"))
1117
  continuing = report.get("state") == "partial"
platform/harness/amazon_store.py CHANGED
@@ -2845,7 +2845,7 @@ def refused_reports(states):
2845
  return rows
2846
 
2847
 
2848
- def needs_first_load(tenant_key, rt=None):
2849
  """A fresh account or an unfinished durable load uses bounded passes, even after one table finishes."""
2850
  rt = rt if rt is not None else _runtime_for(tenant_key)
2851
  restored = restore_mirror(tenant_key, rt)
@@ -2860,26 +2860,62 @@ def needs_first_load(tenant_key, rt=None):
2860
  # a 403 for a report it does not offer this seller, a document with no row root) never
2861
  # earns a watermark, so it kept this answer True and re-queued a first-load pass on every
2862
  # boot. Its refusal is on screen; the scheduled rotation retries it.
2863
- refused = {table for table in added
2864
- if (_read_state(con, entity_key(table)) or {}).get("state") == "refused"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2865
  # ⛔ HOTFIX 2026-09-29 (D-3496): the same rule for a load that STOPPED. A load left
2866
  # "failed" only because Amazon refused a report (a 403 for a report it does not offer this
2867
  # seller) stayed "not done" forever, so every restart queued it again AND the scheduled
2868
  # producer deferred every table to it: live's Amazon tables stopped refreshing. A stopped
2869
- # load is still owed only while an enabled table has no watermark and was not refused; a
2870
- # load still CONTINUING ("partial") is owed until a pass closes it.
 
 
 
 
 
2871
  missing = [table for table in TABLES
2872
  if enabled.get(amazon_schema.DATABASE_PREFIX + table)
2873
  and not read_watermark(con, table)
2874
- and (_read_state(con, entity_key(table)) or {}).get("state") != "refused"]
2875
- owed = bool(load and load["state"] != "done"
2876
- and (load["state"] != "failed" or missing))
2877
  partially_refused = {name[len(PREFIX):] for name in restored.get("refused_tables", ())}
2878
  return (bool(partially_refused.intersection(missing))
2879
  or owed or any(_undocumented_month(con, table) for table in TABLES
2880
  if enabled.get(amazon_schema.DATABASE_PREFIX + table))
2881
- or not any(read_watermark(con, table) for table in TABLES)
2882
- or any(not read_watermark(con, table) for table in added if table not in refused))
2883
  finally:
2884
  con.close()
2885
 
@@ -3491,11 +3527,12 @@ def _totals_match(expected, actual):
3491
 
3492
 
3493
  def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
3494
- max_wait=900, *, scheduled=False, first_load=False, report_budget=None):
3495
  """Pull every declared table into the tenant's mirror. -> a report dict.
3496
 
3497
  Idempotent: a re-sync replaces each window rather than appending. Refuses, loudly and with the
3498
  reason, when the tenant has stored no credential.
 
3499
  """
3500
  rt = rt if rt is not None else _runtime_for(tenant_key)
3501
  restored = restore_mirror(tenant_key, rt, log=log)
@@ -3525,7 +3562,7 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
3525
  "(`connectors_amazon.py --check`) and sync only a PRODUCTION credential.")
3526
 
3527
  want = [t for t in (TABLES if tables is None else tables) if t in TABLES]
3528
- if scheduled and want and "settlements" not in want:
3529
  want.append("settlements")
3530
  end = datetime.now(timezone.utc)
3531
  report = {"tenant": tenant_key, "tables": {}, "started": _iso(end),
@@ -3557,7 +3594,23 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
3557
  con.close()
3558
  raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data")
3559
  load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
 
 
 
3560
  if first_load:
 
 
 
 
 
 
 
 
 
 
 
 
 
3561
  due = _restore_first_load_wait(tenant_key, con)
3562
  if due and due > end:
3563
  progress = first_load_status(tenant_key, rt)
@@ -3565,20 +3618,25 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
3565
  report.update(state="partial", landedTasks=0, nextDueSeconds=(due - end).total_seconds(),
3566
  firstLoad=dict(progress or {}), finished=_iso(end))
3567
  con.close()
3568
- return report # No vendor call or mirror publication while the durable wait is pending.
3569
- if scheduled and load and load["state"] != "done":
3570
  # The pinned inline producer still calls sync(scheduled=True). Do not let it start an
3571
  # unbounded history pull between the resident writer's bounded first-load passes.
3572
  # HOTFIX 2026-09-29 (D-3496): only a table WITHOUT a watermark belongs to the first load.
3573
  # A watermark is written when a table's history is complete, so its next window is
3574
  # bounded; deferring it too froze every loaded table behind one refused report.
3575
  for table in want:
3576
- if table == "settlements" or read_watermark(con, table):
 
 
3577
  continue
3578
  report["tables"][table] = {"state": "deferred", "reason":
3579
  "The first load is continuing in bounded passes on the resident sync thread."}
3580
- want = [table for table in want if table == "settlements" or read_watermark(con, table)]
3581
- if scheduled:
 
 
 
3582
  plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table))
3583
  for table in TABLES}, want, now=end)
3584
  want = plan["tables"]
@@ -3680,8 +3738,15 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
3680
  work = [task for task in work if task[0] != sibling]
3681
  else:
3682
  for table in pending:
 
 
 
 
 
 
 
 
3683
  if table == SQP_TABLE:
3684
- plan = plans[table] = sync_window(table, read_watermark(con, table), now=end, since=since)
3685
  months = _month_windows(plan["start"], plan["end"])
3686
  if not plan["incremental"]:
3687
  months.reverse()
@@ -3718,8 +3783,7 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
3718
  continue
3719
  if table in handled:
3720
  continue
3721
- plan = plans[table] if first_load else sync_window(
3722
- table, read_watermark(con, table), now=end, since=since)
3723
  rtype = plan["report"]
3724
  key, mirror = entity_key(table), table_name(table)
3725
  if not rtype:
@@ -3735,7 +3799,7 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
3735
  sibling = paired.get(table)
3736
  sibling_plan = plans.get(sibling)
3737
  elif sibling in want and sibling not in handled:
3738
- sibling_plan = sync_window(sibling, read_watermark(con, sibling), now=end, since=since)
3739
  differs = [part for part in ("start", "end", "report")
3740
  if plan[part] != sibling_plan[part]]
3741
  if differs:
@@ -3809,7 +3873,9 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
3809
  report["tables"][table] = {"rows": 0, "operations": operations,
3810
  "state": "empty", "reason": "No issued settlement reports found."}
3811
  landed.add((table, task_start, task_end, task_batch))
3812
- # No documents means no new cursor, attempt stamp or completion marker.
 
 
3813
  continue
3814
  if first_load and settlement_remaining:
3815
  break
 
2845
  return rows
2846
 
2847
 
2848
+ def needs_first_load(tenant_key, rt=None, *, completed_tables=None):
2849
  """A fresh account or an unfinished durable load uses bounded passes, even after one table finishes."""
2850
  rt = rt if rt is not None else _runtime_for(tenant_key)
2851
  restored = restore_mirror(tenant_key, rt)
 
2860
  # a 403 for a report it does not offer this seller, a document with no row root) never
2861
  # earns a watermark, so it kept this answer True and re-queued a first-load pass on every
2862
  # boot. Its refusal is on screen; the scheduled rotation retries it.
2863
+ def terminal(table):
2864
+ state = _read_state(con, entity_key(table)) or {}
2865
+ if state.get("state") == "refused":
2866
+ return True
2867
+ if state.get("state") != "done":
2868
+ return False
2869
+ if table == "settlements":
2870
+ return state.get("detail") == "No issued settlement reports found."
2871
+ spec = _client().REPORT_TYPES.get(table) or {}
2872
+ proof = _read_state(con, entity_key("reconcile_" + table)) or {}
2873
+ if spec.get("monthly") or spec.get("snapshot"):
2874
+ if proof.get("state") != "done" or not proof.get("detail"):
2875
+ return False
2876
+ try:
2877
+ parsed = json.loads(proof["detail"])
2878
+ if not isinstance(parsed, dict) or not parsed.get("start") or not parsed.get("end"):
2879
+ return False
2880
+ bounds = []
2881
+ for value in (parsed["start"], parsed["end"]):
2882
+ bound = datetime.fromisoformat(value + "-01" if len(value) == 7 else value.replace("Z", "+00:00"))
2883
+ bounds.append(bound.replace(tzinfo=timezone.utc) if bound.tzinfo is None
2884
+ else bound.astimezone(timezone.utc))
2885
+ if bounds[0] > bounds[1]:
2886
+ return False
2887
+ except (ValueError, TypeError, AttributeError, OverflowError):
2888
+ return False
2889
+ if spec.get("snapshot"):
2890
+ return True
2891
+ floor = read_backfill_floor(con, table)
2892
+ try:
2893
+ cohort = _utc(load["detail"]) if load and load.get("detail") else datetime.now(timezone.utc)
2894
+ except (ValueError, TypeError, AttributeError, OverflowError):
2895
+ return False
2896
+ return floor is not None and floor <= sync_window(table, None, now=cohort)["start"]
2897
  # ⛔ HOTFIX 2026-09-29 (D-3496): the same rule for a load that STOPPED. A load left
2898
  # "failed" only because Amazon refused a report (a 403 for a report it does not offer this
2899
  # seller) stayed "not done" forever, so every restart queued it again AND the scheduled
2900
  # producer deferred every table to it: live's Amazon tables stopped refreshing. A stopped
2901
+ # load is still owed only while an enabled table lacks a watermark and completion evidence.
2902
+ # A continuing load remains owed while its tables still lack completion evidence.
2903
+ completed = {table for table in TABLES
2904
+ if enabled.get(amazon_schema.DATABASE_PREFIX + table)
2905
+ and not _undocumented_month(con, table) and terminal(table)}
2906
+ if completed_tables is not None:
2907
+ completed_tables.update(completed)
2908
  missing = [table for table in TABLES
2909
  if enabled.get(amazon_schema.DATABASE_PREFIX + table)
2910
  and not read_watermark(con, table)
2911
+ and table not in completed]
2912
+ owed = bool(load and load["state"] != "done" and missing)
 
2913
  partially_refused = {name[len(PREFIX):] for name in restored.get("refused_tables", ())}
2914
  return (bool(partially_refused.intersection(missing))
2915
  or owed or any(_undocumented_month(con, table) for table in TABLES
2916
  if enabled.get(amazon_schema.DATABASE_PREFIX + table))
2917
+ or (bool(missing) and not any(read_watermark(con, table) for table in TABLES))
2918
+ or any(table in missing for table in added))
2919
  finally:
2920
  con.close()
2921
 
 
3527
 
3528
 
3529
  def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
3530
+ max_wait=900, *, scheduled=False, first_load=False, report_budget=None, refresh_loaded=False):
3531
  """Pull every declared table into the tenant's mirror. -> a report dict.
3532
 
3533
  Idempotent: a re-sync replaces each window rather than appending. Refuses, loudly and with the
3534
  reason, when the tenant has stored no credential.
3535
+ `refresh_loaded` refreshes completed tables before bounded history and its durable waits.
3536
  """
3537
  rt = rt if rt is not None else _runtime_for(tenant_key)
3538
  restored = restore_mirror(tenant_key, rt, log=log)
 
3562
  "(`connectors_amazon.py --check`) and sync only a PRODUCTION credential.")
3563
 
3564
  want = [t for t in (TABLES if tables is None else tables) if t in TABLES]
3565
+ if scheduled and not first_load and want and "settlements" not in want:
3566
  want.append("settlements")
3567
  end = datetime.now(timezone.utc)
3568
  report = {"tenant": tenant_key, "tables": {}, "started": _iso(end),
 
3594
  con.close()
3595
  raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data")
3596
  load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
3597
+ # Table completion evidence also admits empty histories behind a stale aggregate marker.
3598
+ completed = set()
3599
+ needs_first_load(tenant_key, rt=rt, completed_tables=completed)
3600
  if first_load:
3601
+ if refresh_loaded:
3602
+ loaded = [table for table in want if (read_watermark(con, table) or table in completed)
3603
+ and not _undocumented_month(con, table)]
3604
+ if loaded:
3605
+ # Refreshes use their own floors; the history partition retains its bounded budget.
3606
+ con.close()
3607
+ refreshed = sync(tenant_key, tables=loaded, log=log, rt=rt, max_wait=max_wait,
3608
+ scheduled=scheduled, refresh_loaded=True)
3609
+ report["tables"].update(refreshed.get("tables") or {})
3610
+ for key in ("problems", "audit_problems"):
3611
+ report[key].extend(refreshed.get(key) or [])
3612
+ con, _path = _connect_tenant_mirror(tenant_key, rt)
3613
+ want = [table for table in want if table not in loaded]
3614
  due = _restore_first_load_wait(tenant_key, con)
3615
  if due and due > end:
3616
  progress = first_load_status(tenant_key, rt)
 
3618
  report.update(state="partial", landedTasks=0, nextDueSeconds=(due - end).total_seconds(),
3619
  firstLoad=dict(progress or {}), finished=_iso(end))
3620
  con.close()
3621
+ return report # The history partition cannot call the vendor before its durable due time.
3622
+ if scheduled and not first_load and (refresh_loaded or (load and load["state"] != "done")):
3623
  # The pinned inline producer still calls sync(scheduled=True). Do not let it start an
3624
  # unbounded history pull between the resident writer's bounded first-load passes.
3625
  # HOTFIX 2026-09-29 (D-3496): only a table WITHOUT a watermark belongs to the first load.
3626
  # A watermark is written when a table's history is complete, so its next window is
3627
  # bounded; deferring it too froze every loaded table behind one refused report.
3628
  for table in want:
3629
+ if ((read_watermark(con, table) and (not refresh_loaded or not _undocumented_month(con, table)))
3630
+ or table in completed
3631
+ or (table == "settlements" and not refresh_loaded)):
3632
  continue
3633
  report["tables"][table] = {"state": "deferred", "reason":
3634
  "The first load is continuing in bounded passes on the resident sync thread."}
3635
+ want = [table for table in want if (read_watermark(con, table)
3636
+ and (not refresh_loaded or not _undocumented_month(con, table)))
3637
+ or table in completed
3638
+ or (table == "settlements" and not refresh_loaded)]
3639
+ if scheduled and not first_load:
3640
  plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table))
3641
  for table in TABLES}, want, now=end)
3642
  want = plan["tables"]
 
3738
  work = [task for task in work if task[0] != sibling]
3739
  else:
3740
  for table in pending:
3741
+ mark = read_watermark(con, table)
3742
+ if not mark and since is None and table != "settlements" and table in completed:
3743
+ # The completed cohort bounds this incremental request without inventing a stored watermark.
3744
+ try:
3745
+ mark = _utc(load["detail"]) if load and load.get("detail") else end - timedelta(days=FIRST_LOAD_RECENT_DAYS)
3746
+ except (ValueError, TypeError, AttributeError, OverflowError):
3747
+ mark = end - timedelta(days=FIRST_LOAD_RECENT_DAYS)
3748
+ plan = plans[table] = sync_window(table, mark, now=end, since=since)
3749
  if table == SQP_TABLE:
 
3750
  months = _month_windows(plan["start"], plan["end"])
3751
  if not plan["incremental"]:
3752
  months.reverse()
 
3783
  continue
3784
  if table in handled:
3785
  continue
3786
+ plan = plans[table]
 
3787
  rtype = plan["report"]
3788
  key, mirror = entity_key(table), table_name(table)
3789
  if not rtype:
 
3799
  sibling = paired.get(table)
3800
  sibling_plan = plans.get(sibling)
3801
  elif sibling in want and sibling not in handled:
3802
+ sibling_plan = plans[sibling]
3803
  differs = [part for part in ("start", "end", "report")
3804
  if plan[part] != sibling_plan[part]]
3805
  if differs:
 
3873
  report["tables"][table] = {"rows": 0, "operations": operations,
3874
  "state": "empty", "reason": "No issued settlement reports found."}
3875
  landed.add((table, task_start, task_end, task_batch))
3876
+ if first_load:
3877
+ _record_state(con, key, "done", 0, "No issued settlement reports found.")
3878
+ # Empty discovery earns no cursor, attempt stamp or document marker.
3879
  continue
3880
  if first_load and settlement_remaining:
3881
  break
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 834b29edc66ff8f408620ad32c2f5682dd6876cf090ba3d98668ce8de1638523
 
1
+ b24ae0a3ec2803b04b38ddb08174ab63dca7909b304d9b61622f3d9c16b05a3f