fsanyoto commited on
Commit
87761df
·
verified ·
1 Parent(s): bb71a16

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "0046a8388",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "7636cd8b3",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 0046a8388
 
1
+ 7636cd8b3
platform/harness/amazon_store.py CHANGED
@@ -717,6 +717,11 @@ FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS
717
  FIRST_LOAD_REPORTS_PER_PASS = 15
718
  #: A generation's repeated other-tables pass: about an hour of reports at Amazon's rate, then it publishes.
719
  FIRST_LOAD_DELEGATE_REPORTS = 60
 
 
 
 
 
720
  #: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap:
721
  #: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25
722
  #: minutes each) never hold the sync desk for hours (D-7081).
@@ -5601,6 +5606,23 @@ def job_prefetchable(table):
5601
  and table not in LEGACY_SNAPSHOT_TABLES and table not in ("settlements", SQP_TABLE))
5602
 
5603
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5604
  def first_load_task(account, fields, table, w0, w1, batch=None, *, landed=False):
5605
  """One owed first-load task. `key` is the report identity, shared by tables that share a report."""
5606
  ident = json.dumps([str(account), table, _iso(w0) if w0 else None, _iso(w1) if w1 else None,
@@ -6310,7 +6332,9 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6310
  return task["id"] not in finished and not task.get("landed") and task["id"] not in failed
6311
 
6312
  record["remainingBefore"] = sum(owed(t) for t in tasks)
6313
- mine = [t for t in tasks if t.get("prefetch")]
 
 
6314
  own_tables = sorted({t["table"] for t in mine if owed(t)})
6315
  ready = {}
6316
  next_token = started
@@ -6339,7 +6363,7 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6339
  if not acquire_amazon_lease(docs, holder, now=stamp):
6340
  record["reason"] = "Another holder took the Amazon writer lease."
6341
  break
6342
- pending = [t for t in mine if owed(t)]
6343
  # Request: one report per identity, round robin across tables, while the ledger has tokens.
6344
  if stamp < request_until and stamp >= next_token:
6345
  queue, seen, by_table = [], set(opened), {}
@@ -6383,6 +6407,9 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6383
  for key_, row in list(opened.items()):
6384
  if key_ in ready or held(key_, stamp):
6385
  continue
 
 
 
6386
  task = next((t for t in mine if t["key"] == key_ and owed(t)), None)
6387
  if task is None or stamp - float(row.get("createdAt") or 0) >= REPORT_REUSE_SECONDS:
6388
  opened.pop(key_)
@@ -6455,13 +6482,16 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6455
  record["windowsWritten"] += 1
6456
  if unit not in record["tablesPublished"]:
6457
  record["tablesPublished"].append(unit)
6458
- for t in prefix:
 
 
 
6459
  if t["id"] in finished or t["key"] not in result["served"]:
6460
  ready.pop(t["key"], None)
6461
  if not any(owed(o) and o["key"] == t["key"] for o in mine):
6462
  opened.pop(t["key"], None)
6463
  save(lambda doc: doc.update(finished=finished, open=opened))
6464
- pending = [t for t in mine if owed(t)]
6465
  if not pending:
6466
  break
6467
  waiting_on = [t for t in pending if t["key"] in opened or t["key"] in ready]
@@ -6718,6 +6748,12 @@ def history_bound(con, table, end, *, now=None):
6718
  if observed and observed[0] > floor:
6719
  floor, cause = observed[0], observed[1] or (
6720
  f"Amazon keeps this data back to {observed[0].date().isoformat()}; older history is not available.")
 
 
 
 
 
 
6721
  return floor, cause
6722
 
6723
 
 
717
  FIRST_LOAD_REPORTS_PER_PASS = 15
718
  #: A generation's repeated other-tables pass: about an hour of reports at Amazon's rate, then it publishes.
719
  FIRST_LOAD_DELEGATE_REPORTS = 60
720
+ #: Weekly history weeks (a 3 GB search terms report each) one generation takes before its other-tables pass.
721
+ FIRST_LOAD_WEEKLY_HISTORY_PER_GENERATION = 2
722
+ #: Older weeks of a weekly table are not planned: the weekly writer defers a history week by its cadence, and a
723
+ #: generation that wrote one never finished it (staging 2026-11-14). The latest week stays first, as R18.
724
+ WEEKLY_HISTORY_PLANNED = False
725
  #: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap:
726
  #: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25
727
  #: minutes each) never hold the sync desk for hours (D-7081).
 
5606
  and table not in LEGACY_SNAPSHOT_TABLES and table not in ("settlements", SQP_TABLE))
5607
 
5608
 
5609
+ def _pace_weekly_history(tasks, owed, limit=None):
5610
+ """Keep the newest `limit` owed weeks of each weekly table in this generation; later generations take the rest.
5611
+
5612
+ The job runs its own reports before the other-tables pass, so unpaced weekly history held SQP back for hours.
5613
+ """
5614
+ limit = FIRST_LOAD_WEEKLY_HISTORY_PER_GENERATION if limit is None else max(1, int(limit))
5615
+ weekly = {}
5616
+ for task in tasks:
5617
+ if owed(task) and _client().REPORT_TYPES.get(task["table"], {}).get("weekly"):
5618
+ weekly.setdefault(task["table"], []).append(task)
5619
+ deferred = set()
5620
+ for rows in weekly.values():
5621
+ rows.sort(key=lambda t: str(t.get("start") or ""), reverse=True)
5622
+ deferred.update(t["id"] for t in rows[limit:])
5623
+ return [t for t in tasks if t["id"] not in deferred]
5624
+
5625
+
5626
  def first_load_task(account, fields, table, w0, w1, batch=None, *, landed=False):
5627
  """One owed first-load task. `key` is the report identity, shared by tables that share a report."""
5628
  ident = json.dumps([str(account), table, _iso(w0) if w0 else None, _iso(w1) if w1 else None,
 
6332
  return task["id"] not in finished and not task.get("landed") and task["id"] not in failed
6333
 
6334
  record["remainingBefore"] = sum(owed(t) for t in tasks)
6335
+ mine = _pace_weekly_history([t for t in tasks if t.get("prefetch")], owed)
6336
+ # A window the writer served but did not finish (deferred, held) waits for the next generation.
6337
+ stalled = set()
6338
  own_tables = sorted({t["table"] for t in mine if owed(t)})
6339
  ready = {}
6340
  next_token = started
 
6363
  if not acquire_amazon_lease(docs, holder, now=stamp):
6364
  record["reason"] = "Another holder took the Amazon writer lease."
6365
  break
6366
+ pending = [t for t in mine if owed(t) and t["id"] not in stalled]
6367
  # Request: one report per identity, round robin across tables, while the ledger has tokens.
6368
  if stamp < request_until and stamp >= next_token:
6369
  queue, seen, by_table = [], set(opened), {}
 
6407
  for key_, row in list(opened.items()):
6408
  if key_ in ready or held(key_, stamp):
6409
  continue
6410
+ keyed = [t for t in mine if t["key"] == key_ and owed(t)]
6411
+ if keyed and all(t["id"] in stalled for t in keyed):
6412
+ continue # Its reportId stays in the cursor for the next generation.
6413
  task = next((t for t in mine if t["key"] == key_ and owed(t)), None)
6414
  if task is None or stamp - float(row.get("createdAt") or 0) >= REPORT_REUSE_SECONDS:
6415
  opened.pop(key_)
 
6482
  record["windowsWritten"] += 1
6483
  if unit not in record["tablesPublished"]:
6484
  record["tablesPublished"].append(unit)
6485
+ for t in prefix + pair:
6486
+ if t["id"] not in finished and t["key"] in result["served"]:
6487
+ stalled.add(t["id"])
6488
+ ready.pop(t["key"], None)
6489
  if t["id"] in finished or t["key"] not in result["served"]:
6490
  ready.pop(t["key"], None)
6491
  if not any(owed(o) and o["key"] == t["key"] for o in mine):
6492
  opened.pop(t["key"], None)
6493
  save(lambda doc: doc.update(finished=finished, open=opened))
6494
+ pending = [t for t in mine if owed(t) and t["id"] not in stalled]
6495
  if not pending:
6496
  break
6497
  waiting_on = [t for t in pending if t["key"] in opened or t["key"] in ready]
 
6748
  if observed and observed[0] > floor:
6749
  floor, cause = observed[0], observed[1] or (
6750
  f"Amazon keeps this data back to {observed[0].date().isoformat()}; older history is not available.")
6751
+ if spec.get("weekly") and not WEEKLY_HISTORY_PLANNED:
6752
+ latest = _week_start(_utc(end)) - timedelta(days=7)
6753
+ if latest > floor:
6754
+ older = f" Amazon serves weeks back to {floor.date().isoformat()}." if bound is not None else ""
6755
+ floor, cause = latest, ("The latest completed week loads first; older weeks are not loaded yet, "
6756
+ "because the weekly sync defers a past week until its next weekly turn." + older)
6757
  return floor, cause
6758
 
6759
 
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- a316de715da32a0bac5e51bcf06514188b87ecfbe38015edf489a6cd7fa04652
 
1
+ 84729cb9d4d622164742fedd32b15e46bdb613c2ffe7e4e1187b1767855629b5