fsanyoto commited on
Commit
53bf512
·
verified ·
1 Parent(s): c4a1388

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "a1bdf2c861",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "bf1d0c4db",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- a1bdf2c861
 
1
+ bf1d0c4db
api/amazon_ads_desk.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """amazon_ads_desk.py: one Amazon Ads pass for the sync desk, under the Amazon mirror writer lease."""
2
+ import contextlib
3
+ import re
4
+ import time
5
+
6
+ #: One Ads pass stops starting reports after this many seconds. A partial pass requeues itself.
7
+ ADS_PASS_SECONDS = 600
8
+ #: A pass that meets another Amazon writer waits this long before it tries again.
9
+ LEASE_RETRY_SECONDS = 600
10
+ #: "Sync finished. " plus this many characters fits the 299 character receipt.
11
+ DETAIL_LIMIT = 284
12
+
13
+ NO_CREDENTIAL = "This tenant has no stored Amazon Ads credential. Connect Amazon Ads before syncing."
14
+ NOT_RESIDENT = "This tenant's mirror is not resident on this server. Sync requires its tenant worker."
15
+ FIRST_LOAD_HELD = "The Amazon first load holds the Amazon writer lease. Run Amazon Ads sync after it ends."
16
+ LEASE_WAIT = "Another Amazon job holds the writer lease. Amazon Ads tries again in 10 minutes."
17
+ PENDING_WAIT = "Amazon Ads reports are pending. The next pass collects them."
18
+
19
+ _TABLE = re.compile(r"[a-z][a-z0-9_]{0,79}")
20
+
21
+
22
+ class Refused(Exception):
23
+ """An authored sentence that ends the pass before any Ads request."""
24
+
25
+
26
+ class LeaseWait(Refused):
27
+ """Another job holds the Amazon writer lease. `retry_at` is the epoch to try again."""
28
+
29
+ def __init__(self, retry_at):
30
+ super().__init__(LEASE_WAIT)
31
+ self.retry_at = retry_at
32
+
33
+
34
+ def has_credential(rt):
35
+ """True when the tenant keychain holds an `amazon_ads` entry. Fixture mode never counts."""
36
+ from core import keychain
37
+ try:
38
+ return any(row.get("type") == "amazon_ads" for row in keychain.list_entries(rt))
39
+ except Exception: # noqa: BLE001
40
+ return False
41
+
42
+
43
+ def _resident(rt, tenant):
44
+ """The process-global mirror must already be this tenant's file: never repoint it under readers."""
45
+ from harness import datastore
46
+ current = datastore.DB_PATH.resolve()
47
+ return (rt.datastore_path.resolve() == current
48
+ and datastore.path_for(tenant).resolve() == current)
49
+
50
+
51
+ def _log(*parts, **_kwargs):
52
+ text = " ".join(str(part) for part in parts).strip()
53
+ if text:
54
+ print(f"[aios-api] amazon_ads sync desk: {text[:220]}", flush=True)
55
+
56
+
57
+ def receipt_detail(report, *, partial=False, after=None):
58
+ """Rows per Ads table in the mirror, plus a pending sentence. At most DETAIL_LIMIT characters."""
59
+ parts = [f"{name} {int(row.get('in_mirror') or 0)}"
60
+ for name, row in (report.get("tables") or {}).items()
61
+ if isinstance(row, dict) and _TABLE.fullmatch(str(name)) and row.get("state") != "not_offered"]
62
+ tail = ""
63
+ if partial:
64
+ minutes = max(1, round(float(after or ADS_PASS_SECONDS) / 60))
65
+ tail = (" More Amazon Ads reports are pending. The next pass runs in "
66
+ f"{minutes} minute{'s' if minutes != 1 else ''}.")
67
+ if not parts:
68
+ return tail.strip()
69
+ head = "Amazon Ads rows: "
70
+ budget = DETAIL_LIMIT - len(head) - len(tail) - 1
71
+ shown = []
72
+ for part in parts:
73
+ left = len(parts) - len(shown) - 1
74
+ if len(", ".join(shown + [part]) + (f" and {left} more tables" if left else "")) > budget:
75
+ break
76
+ shown.append(part)
77
+ left = len(parts) - len(shown)
78
+ return head + ", ".join(shown) + (f" and {left} more tables" if left else "") + "." + tail
79
+
80
+
81
+ def run_pass(rt, tenant, *, log=None, now=time.time):
82
+ """Pull due Ads reports into the tenant's Amazon mirror. -> the desk's view of the report.
83
+
84
+ Raises `Refused` before any Ads request, and `LeaseWait` when another Amazon writer runs.
85
+ The store persists through `amazon_store.persist_mirror` because this call lends no connection.
86
+ """
87
+ from harness import amazon_ads_store as ads, amazon_store as amazon
88
+ import process_role
89
+ if not _resident(rt, tenant):
90
+ raise Refused(NOT_RESIDENT)
91
+ if not has_credential(rt):
92
+ raise Refused(NO_CREDENTIAL)
93
+ if amazon.first_load_job_live(rt):
94
+ raise Refused(FIRST_LOAD_HELD)
95
+ with contextlib.ExitStack() as stack:
96
+ try:
97
+ stack.enter_context(process_role.connector_writer(rt, "amazon"))
98
+ except RuntimeError as exc:
99
+ if "holds the amazon writer lease" not in str(exc):
100
+ raise
101
+ raise LeaseWait(now() + LEASE_RETRY_SECONDS) from None
102
+ report = ads.sync(rt.key, rt=rt, log=log or _log, max_seconds=ADS_PASS_SECONDS)
103
+ waits = list(report.get("waits") or [])
104
+ problems = [problem for problem in report.get("problems") or [] if problem not in waits]
105
+ partial = bool(report.get("partial")) and not problems
106
+ after = report.get("continueAfterSeconds")
107
+ if not (isinstance(after, (int, float)) and not isinstance(after, bool) and after > 0):
108
+ after = ADS_PASS_SECONDS
109
+ return {"report": report, "problems": problems, "partial": partial,
110
+ "continueAt": now() + float(after) if partial else None,
111
+ "detail": receipt_detail(report, partial=partial, after=after)}
api/main.py CHANGED
@@ -1819,10 +1819,13 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
1819
  # A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
1820
  hold = during_odoo if hold_first_load is None else hold_first_load
1821
  # Claim only work this turn executes. A later provider remains queued with an honest wait.
1822
- excluded = ({"odoo", "meta_ads"} if amazon_only else
1823
- ({"odoo"} if during_odoo else set()))
1824
  if not amazon_only and globals().get("_AMAZON_DESK", {}).get("managed"):
1825
  excluded.add("amazon_sp")
 
 
 
1826
  pending = connector_sync.take_sync_requests(limit=1, exclude_providers=excluded,
1827
  refresh_remote=True,
1828
  exclude_requested_by=("first load",) if hold else ())
@@ -1841,6 +1844,7 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
1841
  nothing_due, first_load = None, False
1842
  progress = ()
1843
  amazon_detail, amazon_failure = "", None
 
1844
  status_rt, status_cid = None, None
1845
  try:
1846
  from harness import runtime, datastore, mirror_store
@@ -1943,6 +1947,18 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
1943
  problems.append("The pull hit the page cap.")
1944
  if not problems and not report.get("tables"):
1945
  raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.")
 
 
 
 
 
 
 
 
 
 
 
 
1946
  else:
1947
  raise _SyncNowRefused("This connector has no Sync now runner.")
1948
  if nothing_due:
@@ -1984,7 +2000,8 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
1984
  first_load=first_load_progress, request_id=receipt.get("id"))
1985
  else:
1986
  connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, request_id=receipt.get("id"))
1987
- pass_outcome = ("waiting" if nothing_due and ok else "continuing" if (continuing and ok)
 
1988
  else ("done" if ok else "failed"))
1989
  _status_write = globals().get("_publish_connector_status")
1990
  if status_rt is not None and _status_write:
@@ -1999,6 +2016,13 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
1999
  import time as _time
2000
  print(f"[aios-api] amazon first load had nothing due; it waits "
2001
  f"{int(nothing_due - _time.time())}s for the next report window", flush=True)
 
 
 
 
 
 
 
2002
  if during_odoo and ok and not continuing and not nothing_due:
2003
  _project_desk({provider: "done"}, tenant)
2004
  if continuing and ok:
@@ -3118,8 +3142,10 @@ def _project_desk(outcome, tenant):
3118
 
3119
  The desk has already pulled, so this is the relational half alone. A continuing first load
3120
  projects nothing until it finishes; the scheduled pass still projects at its cadence.
 
3121
  """
3122
- finished = [provider for provider, state in (outcome or {}).items() if state == "done"]
 
3123
  if not finished:
3124
  return
3125
  try:
@@ -3127,7 +3153,8 @@ def _project_desk(outcome, tenant):
3127
  rt = _runtime.get_runtime(tenant or "royal-imports")
3128
  if "meta_ads" in finished:
3129
  _rebuild_meta_relational("sync now", rt)
3130
- if "amazon_sp" in finished:
 
3131
  _rebuild_amazon_relational("sync now", rt)
3132
  except Exception as e: # noqa: BLE001
3133
  print(f"[aios-api] sync now projection failed: {type(e).__name__}: {e}")
@@ -3387,6 +3414,21 @@ def _amazon_intraday_pass(tenant_key, previous_fingerprint):
3387
  return None
3388
 
3389
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3390
  def _amazon_desk_pass(tenant, *, project=True):
3391
  """Run one Amazon desk claim off the slot clock and collect its projection after publication."""
3392
  import process_role
@@ -3937,6 +3979,11 @@ def _store_resync_loop():
3937
  _pulls["amazon"] = _t.time()
3938
  _resync_memory_write(last_good_pass_at, last_relational_success_at,
3939
  last_source_fingerprints, dict(_pulls))
 
 
 
 
 
3940
  # A zero Odoo report says nothing about Amazon either, so its projection runs after its pull in both branches.
3941
  try:
3942
  from harness import runtime as _runtime
 
1819
  # A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
1820
  hold = during_odoo if hold_first_load is None else hold_first_load
1821
  # Claim only work this turn executes. A later provider remains queued with an honest wait.
1822
+ excluded = ({"odoo", "meta_ads", "amazon_ads"} if amazon_only else
1823
+ ({"odoo", "amazon_ads"} if during_odoo else set()))
1824
  if not amazon_only and globals().get("_AMAZON_DESK", {}).get("managed"):
1825
  excluded.add("amazon_sp")
1826
+ # Amazon Ads shares the Amazon mirror lease: it waits while the Amazon desk turn holds it.
1827
+ if not amazon_only and globals().get("_AMAZON_DESK", {}).get("job") is not None:
1828
+ excluded.add("amazon_ads")
1829
  pending = connector_sync.take_sync_requests(limit=1, exclude_providers=excluded,
1830
  refresh_remote=True,
1831
  exclude_requested_by=("first load",) if hold else ())
 
1844
  nothing_due, first_load = None, False
1845
  progress = ()
1846
  amazon_detail, amazon_failure = "", None
1847
+ ads_continue_at, ads_retry_at = None, None
1848
  status_rt, status_cid = None, None
1849
  try:
1850
  from harness import runtime, datastore, mirror_store
 
1947
  problems.append("The pull hit the page cap.")
1948
  if not problems and not report.get("tables"):
1949
  raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.")
1950
+ elif provider == "amazon_ads":
1951
+ import amazon_ads_desk as _ads_desk
1952
+ try:
1953
+ ads = _ads_desk.run_pass(rt, tenant)
1954
+ except _ads_desk.LeaseWait as wait:
1955
+ ads_retry_at = wait.retry_at
1956
+ raise _SyncNowRefused(str(wait)) from None
1957
+ except _ads_desk.Refused as exc:
1958
+ raise _SyncNowRefused(str(exc)) from None
1959
+ # A partial pass is not `continuing`: that path reads first-load windows Ads has not.
1960
+ report, problems, amazon_detail = ads["report"], ads["problems"], ads["detail"]
1961
+ ads_continue_at = ads["continueAt"]
1962
  else:
1963
  raise _SyncNowRefused("This connector has no Sync now runner.")
1964
  if nothing_due:
 
2000
  first_load=first_load_progress, request_id=receipt.get("id"))
2001
  else:
2002
  connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, request_id=receipt.get("id"))
2003
+ pass_outcome = ("waiting" if (nothing_due and ok) or ads_retry_at is not None
2004
+ else "continuing" if ((continuing or ads_continue_at is not None) and ok)
2005
  else ("done" if ok else "failed"))
2006
  _status_write = globals().get("_publish_connector_status")
2007
  if status_rt is not None and _status_write:
 
2016
  import time as _time
2017
  print(f"[aios-api] amazon first load had nothing due; it waits "
2018
  f"{int(nothing_due - _time.time())}s for the next report window", flush=True)
2019
+ # Finish first: the desk refuses a second in-flight intent for the same pair.
2020
+ if ads_continue_at is not None and ok:
2021
+ connector_sync.request_sync("amazon_ads", tenant, requested_by="pending report",
2022
+ not_before=ads_continue_at, wait_cause=_ads_desk.PENDING_WAIT)
2023
+ if ads_retry_at is not None:
2024
+ connector_sync.request_sync("amazon_ads", tenant, requested_by=receipt.get("requestedBy") or "schedule",
2025
+ not_before=ads_retry_at, wait_cause=detail)
2026
  if during_odoo and ok and not continuing and not nothing_due:
2027
  _project_desk({provider: "done"}, tenant)
2028
  if continuing and ok:
 
3142
 
3143
  The desk has already pulled, so this is the relational half alone. A continuing first load
3144
  projects nothing until it finishes; the scheduled pass still projects at its cadence.
3145
+ A continuing Amazon Ads pass projects the rows it landed, so a backfill publishes as it goes.
3146
  """
3147
+ finished = [provider for provider, state in (outcome or {}).items()
3148
+ if state == "done" or (provider == "amazon_ads" and state == "continuing")]
3149
  if not finished:
3150
  return
3151
  try:
 
3153
  rt = _runtime.get_runtime(tenant or "royal-imports")
3154
  if "meta_ads" in finished:
3155
  _rebuild_meta_relational("sync now", rt)
3156
+ # One rebuild serves both: SP and Ads share the Amazon mirror and its `ut_amzn_*` projection.
3157
+ if "amazon_sp" in finished or "amazon_ads" in finished:
3158
  _rebuild_amazon_relational("sync now", rt)
3159
  except Exception as e: # noqa: BLE001
3160
  print(f"[aios-api] sync now projection failed: {type(e).__name__}: {e}")
 
3414
  return None
3415
 
3416
 
3417
+ def _request_amazon_ads_schedule(tenant):
3418
+ """Queue the daily Amazon Ads pass when the tenant's keychain holds an Ads entry. Never raises."""
3419
+ try:
3420
+ import amazon_ads_desk
3421
+ from core import connector_sync
3422
+ from harness import runtime
3423
+ rt = runtime.get_runtime(tenant)
3424
+ if not amazon_ads_desk.has_credential(rt):
3425
+ return False
3426
+ return bool(connector_sync.request_sync("amazon_ads", rt.key, requested_by="schedule").get("accepted"))
3427
+ except Exception as exc: # noqa: BLE001
3428
+ print(f"[aios-api] amazon ads schedule not queued ({type(exc).__name__})", flush=True)
3429
+ return False
3430
+
3431
+
3432
  def _amazon_desk_pass(tenant, *, project=True):
3433
  """Run one Amazon desk claim off the slot clock and collect its projection after publication."""
3434
  import process_role
 
3979
  _pulls["amazon"] = _t.time()
3980
  _resync_memory_write(last_good_pass_at, last_relational_success_at,
3981
  last_source_fingerprints, dict(_pulls))
3982
+ # A boot restore is not a sync. A scheduled pass queues Ads on the desk, which owns its lease wait.
3983
+ # Looked up by name: several gates compile this loop with a fixed set of main.py names.
3984
+ _ads_schedule = globals().get("_request_amazon_ads_schedule")
3985
+ if _ads_schedule and not _resumed.get("amazon"):
3986
+ _ads_schedule(_cad["tenant"])
3987
  # A zero Odoo report says nothing about Amazon either, so its projection runs after its pull in both branches.
3988
  try:
3989
  from harness import runtime as _runtime
api/routes_connectors.py CHANGED
@@ -3034,8 +3034,11 @@ def _amazon_completed_sync(rt):
3034
  return stored if stored is not None else _amazon_mirror_completion(rt)
3035
 
3036
 
3037
- def _amazon_mirror_completion(rt):
3038
- """Project completion markers without counting business rows or building database detail."""
 
 
 
3039
  import duckdb
3040
  from harness import amazon_store as az, datastore
3041
 
@@ -3044,7 +3047,12 @@ def _amazon_mirror_completion(rt):
3044
  "WHERE starts_with(entity, ?)", [az.PREFIX]).fetchall()
3045
  markers = {str(entity): (phase, _sync_stamp(at)) for entity, phase, at in rows}
3046
  stamps = []
3047
- for table in az.TABLES:
 
 
 
 
 
3048
  landed = markers.get(az.entity_key(table), (None, None))
3049
  watermark = markers.get(az.entity_key(az.WATERMARK_PREFIX + table), (None, None))
3050
  # A refused pass also closes its tick. Only its saved watermark proves success.
@@ -3157,9 +3165,9 @@ def _daily_completed_sync(rt, connector_key):
3157
  """Read the stored last good sync; never create a mirror or contact a provider."""
3158
  if connector_key == "amazon_sp":
3159
  return _amazon_completed_sync(rt)
3160
- if connector_key == "meta_ads":
3161
- stored = _stored_completed_sync(rt, "meta_ads")
3162
- return stored if stored is not None else _mirror_completed_sync(rt, "meta_ads")
3163
  return None, False
3164
 
3165
 
@@ -3167,6 +3175,8 @@ def _mirror_completed_sync(rt, connector_key):
3167
  """The container mirror's completed markers. Writers seed C-status from it; cards read the store."""
3168
  if connector_key == "amazon_sp":
3169
  return _amazon_mirror_completion(rt)
 
 
3170
  if connector_key == "meta_ads":
3171
  from pathlib import Path
3172
  from harness import datastore
@@ -3268,7 +3278,7 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
3268
  receipt = odoo_health_receipt(rt, resolved_connection, jobs=jobs)
3269
  if not answer.get("lastGoodSyncAt"):
3270
  answer["lastSyncAt"] = _odoo_last_sync_at(rt, receipt, connection_id)
3271
- if connector_key in ("amazon_sp", "meta_ads"):
3272
  if answer.get("lastCompletedKnown") is True:
3273
  completed, known = answer.get("lastGoodSyncAt"), True
3274
  elif connector_key == "amazon_sp" and (str(resolved_connection or "").strip() or "default") != "default":
 
3034
  return stored if stored is not None else _amazon_mirror_completion(rt)
3035
 
3036
 
3037
+ def _amazon_mirror_completion(rt, *, ads=False):
3038
+ """Project completion markers without counting business rows or building database detail.
3039
+
3040
+ `ads=True` reads the Amazon Ads markers in the same mirror instead of the Amazon SP ones.
3041
+ """
3042
  import duckdb
3043
  from harness import amazon_store as az, datastore
3044
 
 
3047
  "WHERE starts_with(entity, ?)", [az.PREFIX]).fetchall()
3048
  markers = {str(entity): (phase, _sync_stamp(at)) for entity, phase, at in rows}
3049
  stamps = []
3050
+ if ads:
3051
+ from harness import amazon_ads_store as ads_store
3052
+ tables = tuple(ads_store._marker_table(table) for table in ads_store.TABLES)
3053
+ else:
3054
+ tables = az.TABLES
3055
+ for table in tables:
3056
  landed = markers.get(az.entity_key(table), (None, None))
3057
  watermark = markers.get(az.entity_key(az.WATERMARK_PREFIX + table), (None, None))
3058
  # A refused pass also closes its tick. Only its saved watermark proves success.
 
3165
  """Read the stored last good sync; never create a mirror or contact a provider."""
3166
  if connector_key == "amazon_sp":
3167
  return _amazon_completed_sync(rt)
3168
+ if connector_key in ("meta_ads", "amazon_ads"):
3169
+ stored = _stored_completed_sync(rt, connector_key)
3170
+ return stored if stored is not None else _mirror_completed_sync(rt, connector_key)
3171
  return None, False
3172
 
3173
 
 
3175
  """The container mirror's completed markers. Writers seed C-status from it; cards read the store."""
3176
  if connector_key == "amazon_sp":
3177
  return _amazon_mirror_completion(rt)
3178
+ if connector_key == "amazon_ads":
3179
+ return _amazon_mirror_completion(rt, ads=True)
3180
  if connector_key == "meta_ads":
3181
  from pathlib import Path
3182
  from harness import datastore
 
3278
  receipt = odoo_health_receipt(rt, resolved_connection, jobs=jobs)
3279
  if not answer.get("lastGoodSyncAt"):
3280
  answer["lastSyncAt"] = _odoo_last_sync_at(rt, receipt, connection_id)
3281
+ if connector_key in ("amazon_sp", "meta_ads", "amazon_ads"):
3282
  if answer.get("lastCompletedKnown") is True:
3283
  completed, known = answer.get("lastGoodSyncAt"), True
3284
  elif connector_key == "amazon_sp" and (str(resolved_connection or "").strip() or "default") != "default":
platform/core/connector_sync.py CHANGED
@@ -1168,3 +1168,5 @@ register_provider(
1168
  )
1169
  register_provider("meta_ads", label="Meta Ads",
1170
  retry_hint="Use Sync now to retry the Meta Ads pull.")
 
 
 
1168
  )
1169
  register_provider("meta_ads", label="Meta Ads",
1170
  retry_hint="Use Sync now to retry the Meta Ads pull.")
1171
+ register_provider("amazon_ads", label="Amazon Ads",
1172
+ retry_hint="Use Sync now to retry the Amazon Ads pull.")
platform/harness/amazon_ads_client.py CHANGED
@@ -213,7 +213,11 @@ class AdsReportUnavailable(AdsError):
213
 
214
 
215
  class AdsReportPending(AdsError):
216
- """A report did not finish inside the wait limit; the next sync asks again."""
 
 
 
 
217
 
218
 
219
  def fixture_mode():
@@ -584,8 +588,11 @@ def _report_lock(tenant_key, profile_id):
584
 
585
 
586
  def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, connection_id=None, sleep=time.sleep,
587
- now=time.monotonic):
588
- """Create one Reporting v1 report, wait for it, and read every row page. -> (report_id, rows)."""
 
 
 
589
  from harness import amazon_ads_schema as schema
590
  spec = schema.TABLE_SPECS[schema.short_name(table)]
591
  if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
@@ -593,11 +600,15 @@ def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, conne
593
  body = {"reportTypeId": spec["report_type"], "adProduct": spec["ad_type"].upper(),
594
  "startDate": start.isoformat(), "endDate": end.isoformat(),
595
  "timeUnit": "DAILY", "columns": list(schema.vendor_columns(table)), "format": "JSON"}
 
596
  # One report in flight per advertiser (F11): the lock serialises this process's reports.
597
  with _report_lock(tenant_key, profile_id):
598
- created = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1], body=body,
599
- rt=rt, profile_id=profile_id, connection_id=connection_id)
600
- report_id = str(created.get("reportId") or "")
 
 
 
601
  if not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", report_id):
602
  raise AdsError("Amazon Ads returned no report id")
603
  started = now()
@@ -612,9 +623,13 @@ def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, conne
612
  if re.search(r"not (?:supported|offered|available)", reason, re.I):
613
  raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}")
614
  raise AdsError(f"Amazon Ads report {report_id} ended {state}: {reason}")
615
- if now() - started > REPORT_MAX_WAIT_SECONDS:
616
- raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours")
617
- sleep(0 if fixture_mode() else REPORT_POLL_SECONDS)
 
 
 
 
618
  rows, token = [], None
619
  rows_path = REPORT_OPERATIONS["getReportRows"][1].replace("{reportId}", report_id)
620
  while True:
 
213
 
214
 
215
  class AdsReportPending(AdsError):
216
+ """A report did not finish inside the wait limit. `report_id` names it so a later pass can resume it."""
217
+
218
+ def __init__(self, message, report_id=None):
219
+ super().__init__(message)
220
+ self.report_id = report_id
221
 
222
 
223
  def fixture_mode():
 
588
 
589
 
590
  def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, connection_id=None, sleep=time.sleep,
591
+ now=time.monotonic, max_wait=None, report_id=None):
592
+ """Create one Reporting v1 report, wait for it, and read every row page. -> (report_id, rows).
593
+
594
+ `max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without createReport.
595
+ """
596
  from harness import amazon_ads_schema as schema
597
  spec = schema.TABLE_SPECS[schema.short_name(table)]
598
  if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
 
600
  body = {"reportTypeId": spec["report_type"], "adProduct": spec["ad_type"].upper(),
601
  "startDate": start.isoformat(), "endDate": end.isoformat(),
602
  "timeUnit": "DAILY", "columns": list(schema.vendor_columns(table)), "format": "JSON"}
603
+ limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
604
  # One report in flight per advertiser (F11): the lock serialises this process's reports.
605
  with _report_lock(tenant_key, profile_id):
606
+ if report_id is None:
607
+ created = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1], body=body,
608
+ rt=rt, profile_id=profile_id, connection_id=connection_id)
609
+ report_id = str(created.get("reportId") or "")
610
+ else:
611
+ report_id = str(report_id)
612
  if not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", report_id):
613
  raise AdsError("Amazon Ads returned no report id")
614
  started = now()
 
623
  if re.search(r"not (?:supported|offered|available)", reason, re.I):
624
  raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}")
625
  raise AdsError(f"Amazon Ads report {report_id} ended {state}: {reason}")
626
+ elapsed = now() - started
627
+ if elapsed > REPORT_MAX_WAIT_SECONDS:
628
+ raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
629
+ if limit is not None and elapsed >= limit:
630
+ raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
631
+ pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
632
+ sleep(pause if limit is None else min(pause, limit - elapsed))
633
  rows, token = [], None
634
  rows_path = REPORT_OPERATIONS["getReportRows"][1].replace("{reportId}", report_id)
635
  while True:
platform/harness/amazon_ads_store.py CHANGED
@@ -25,6 +25,7 @@ import logging
25
  import sys
26
  import tempfile
27
  import threading
 
28
  import uuid
29
  from datetime import date, datetime, timedelta, timezone
30
  from pathlib import Path
@@ -51,6 +52,14 @@ CONNECT_DAY_KEY = amazon_store.entity_key("ads_connect_day")
51
  BUDGET_KEY = amazon_store.entity_key("ads_report_budget")
52
  INSERT_BATCH_ROWS = 500
53
  ORIGIN_COLUMNS = ("_connection_id", "store")
 
 
 
 
 
 
 
 
54
 
55
 
56
  def table_name(table):
@@ -330,9 +339,58 @@ def _refresh_origin(con, table, connection_id, label, legacy):
330
  return changed
331
 
332
 
333
- def _run_report(key, table, start, end, rt, profile_id, connection_id):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
334
  """Require named transport support instead of sending a sibling's credential."""
335
- options = {"rt": rt, "profile_id": profile_id}
336
  if connection_id:
337
  options["connection_id"] = connection_id
338
  try:
@@ -344,14 +402,17 @@ def _run_report(key, table, start, end, rt, profile_id, connection_id):
344
 
345
 
346
  def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
347
- profile_id=None):
348
  """Pull every offered Ads report into the tenant's Amazon mirror. -> a report dict.
349
 
350
  When this call opens the connection and has a runtime, it saves landed windows and cadence
351
  through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
 
 
352
  """
353
  key = amazon_store._tenant_key(tenant_key)
354
- report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": []}
 
355
  if rt is None and not client.fixture_mode():
356
  rt = _runtime_for(key)
357
  if rt is not None and str(getattr(rt, "key", "") or "").strip().lower() != key:
@@ -364,10 +425,21 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
364
  own = con is None
365
  if own:
366
  con, _path = amazon_store._connect_tenant_mirror(key, rt)
 
 
 
 
367
  try:
368
  datastore._ensure_sync_state(con)
369
  now = _utc_now(now)
370
  today = _utc_today(now)
 
 
 
 
 
 
 
371
  synced = []
372
  checkpointed = []
373
  for cid, label, profile, legacy in connections:
@@ -376,58 +448,120 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
376
  entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
377
  report["connections"].append(entry)
378
  used = _budget(con, today, cid, legacy=legacy)
379
- for table in (tables or TABLES):
380
- table = schema.short_name(table)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
381
  spec = schema.TABLE_SPECS[table]
382
  _ensure_table(con, table)
383
  origin_changed = _refresh_origin(con, table, cid, label, legacy)
384
  pulled, state, mark, reason = 0, "done", _marker_table(table), ""
385
- if not spec["offered"]:
 
 
 
 
 
 
386
  state, reason = "not_offered", spec["not_offered_reason"]
387
  elif not _cadence_due(con, table, now, cid):
388
  state = "skipped"
 
 
 
389
  else:
390
  for start, end in windows(con, table, connect_day, today, cid):
 
 
391
  if used >= client.REPORTS_PER_DAY:
392
  state = "deferred"
393
  reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
394
  f"reports are used, so days from {start.isoformat()} load on the next sync.")
395
  report["problems"].append(f"{label}: {spec['label']}: waiting for tomorrow's report limit")
 
 
 
 
 
396
  break
397
  used += 1
398
  _spend_budget(con, today, used, cid)
 
399
  try:
400
- report_id, rows = _run_report(key, table, start, end, rt, profile, cid)
 
 
 
 
401
  except client.AdsReportUnavailable as exc:
402
  state = "not_offered"
403
- reason = (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
404
- f"({str(exc)[:160]}). The table keeps the days it has. To ask for it, "
405
- f"contact Amazon Ads support with the report type {spec['report_type']}.")
406
  break
407
  except client.AdsError as exc:
 
 
 
 
 
 
 
 
 
408
  state = "failed"
409
  reason = str(exc)[:200]
410
  report["problems"].append(f"{label}: {spec['label']}: {reason}")
411
  break
412
- stored = []
413
- for raw in rows:
414
- row = schema.row_from_vendor(table, raw, profile_id=profile, report_id=report_id)
415
- row.update(_connection_id=cid, store=label,
416
- _row_key=_row_key(cid, row["_row_key"]))
417
- stored.append(row)
418
- pulled += _upsert(con, table, stored)
419
  if table_name(table) not in checkpointed:
420
  checkpointed.append(table_name(table))
421
- # Only this connection's landed window advances its continuation.
422
- for prefix, day, later in ((amazon_store.WATERMARK_PREFIX, end, True),
423
- (amazon_store.BACKFILL_FLOOR_PREFIX, start, False)):
424
- public = amazon_store.entity_key(prefix + mark)
425
- for marker in dict.fromkeys((_state_key(public, cid), public)):
426
- previous = _day_marker(con, marker)
427
- if previous is None or (day > previous if later else day < previous):
428
- amazon_store._record_state(con, marker, "done", _count(con, table), day.isoformat())
429
  if state == "done":
430
  _record_cadence(con, table, now, cid)
 
 
431
  if reason:
432
  _record_limit(con, table, reason, cid, state=state)
433
  shown = f"{label}: {reason}" if cid else reason
@@ -457,6 +591,10 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
457
  _record_limit(con, table, report["limits"][table], state=state["state"])
458
  else:
459
  _clear_limit(con, table)
 
 
 
 
460
  if own and rt is not None and (synced or checkpointed):
461
  _persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
462
  finally:
@@ -466,6 +604,12 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
466
  return report
467
 
468
 
 
 
 
 
 
 
469
  def _persist(tenant_key, rt, con, synced, report, log):
470
  """Save the Ads tables that landed rows, with their markers. A failure is a problem, never a raise."""
471
  try:
 
25
  import sys
26
  import tempfile
27
  import threading
28
+ import time
29
  import uuid
30
  from datetime import date, datetime, timedelta, timezone
31
  from pathlib import Path
 
52
  BUDGET_KEY = amazon_store.entity_key("ads_report_budget")
53
  INSERT_BATCH_ROWS = 500
54
  ORIGIN_COLUMNS = ("_connection_id", "store")
55
+ #: One `_sync_state` entity per (connection, table) holds a report Amazon is still preparing.
56
+ PENDING_PREFIX = "ads_pending_"
57
+ PREPARING_REASON = "Amazon is still preparing this report. The next sync reads it."
58
+ TIME_LIMIT_REASON = "This sync reached its time limit. The next sync continues."
59
+ #: A partial pass asks the caller to run the next pass after this many seconds.
60
+ CONTINUE_AFTER_SECONDS = 600.0
61
+ #: The pass clock for `max_seconds`. A gate patches it to stand in for Amazon's preparing time.
62
+ _clock = time.monotonic
63
 
64
 
65
  def table_name(table):
 
339
  return changed
340
 
341
 
342
+ def _pending_key(table, connection_id=None):
343
+ return _state_key(amazon_store.entity_key(PENDING_PREFIX + _marker_table(table)), connection_id)
344
+
345
+
346
+ def _read_pending(con, table, connection_id=None):
347
+ """The report this connection left preparing for `table`, or None."""
348
+ state = amazon_store._read_state(con, _pending_key(table, connection_id))
349
+ if not state or state.get("state") != "pending" or not state.get("detail"):
350
+ return None
351
+ try:
352
+ entry = json.loads(state["detail"])
353
+ return {"report_id": str(entry["report_id"]), "start": date.fromisoformat(entry["start"]),
354
+ "end": date.fromisoformat(entry["end"]), "created": amazon_store._utc(entry["created"])}
355
+ except (ValueError, TypeError, KeyError, AttributeError, OverflowError):
356
+ return None
357
+
358
+
359
+ def _write_pending(con, table, connection_id, report_id, start, end, created):
360
+ amazon_store._record_state(con, _pending_key(table, connection_id), "pending", detail=json.dumps(
361
+ {"report_id": report_id, "start": start.isoformat(), "end": end.isoformat(),
362
+ "created": created.isoformat()}))
363
+
364
+
365
+ def _clear_pending(con, table, connection_id=None):
366
+ """Keep the row with no detail, so a store that merges state rows cannot bring the entry back."""
367
+ amazon_store._record_state(con, _pending_key(table, connection_id), "done")
368
+
369
+
370
+ def _land(con, table, rows, report_id, start, end, profile, connection_id, label):
371
+ """Upsert one finished report window and advance this connection's markers. -> rows stored."""
372
+ stored = []
373
+ for raw in rows:
374
+ row = schema.row_from_vendor(table, raw, profile_id=profile, report_id=report_id)
375
+ row.update(_connection_id=connection_id, store=label,
376
+ _row_key=_row_key(connection_id, row["_row_key"]))
377
+ stored.append(row)
378
+ pulled = _upsert(con, table, stored)
379
+ mark = _marker_table(table)
380
+ # Only this connection's landed window advances its continuation.
381
+ for prefix, day, later in ((amazon_store.WATERMARK_PREFIX, end, True),
382
+ (amazon_store.BACKFILL_FLOOR_PREFIX, start, False)):
383
+ public = amazon_store.entity_key(prefix + mark)
384
+ for marker in dict.fromkeys((_state_key(public, connection_id), public)):
385
+ previous = _day_marker(con, marker)
386
+ if previous is None or (day > previous if later else day < previous):
387
+ amazon_store._record_state(con, marker, "done", _count(con, table), day.isoformat())
388
+ return pulled
389
+
390
+
391
+ def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
392
  """Require named transport support instead of sending a sibling's credential."""
393
+ options = {"rt": rt, "profile_id": profile_id, **extra}
394
  if connection_id:
395
  options["connection_id"] = connection_id
396
  try:
 
402
 
403
 
404
  def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
405
+ profile_id=None, max_seconds=None):
406
  """Pull every offered Ads report into the tenant's Amazon mirror. -> a report dict.
407
 
408
  When this call opens the connection and has a runtime, it saves landed windows and cadence
409
  through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
410
+ `max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
411
+ pending ledger, and the next pass reads it before it creates any other report.
412
  """
413
  key = amazon_store._tenant_key(tenant_key)
414
+ report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
415
+ "pendingReports": 0, "partial": False, "continueAfterSeconds": None, "waits": []}
416
  if rt is None and not client.fixture_mode():
417
  rt = _runtime_for(key)
418
  if rt is not None and str(getattr(rt, "key", "") or "").strip().lower() != key:
 
425
  own = con is None
426
  if own:
427
  con, _path = amazon_store._connect_tenant_mirror(key, rt)
428
+ bounded = max_seconds is not None
429
+ started = _clock()
430
+ deadline = started + float(max_seconds) if bounded else None
431
+ waits = report["waits"]
432
  try:
433
  datastore._ensure_sync_state(con)
434
  now = _utc_now(now)
435
  today = _utc_today(now)
436
+
437
+ def stamp():
438
+ return now + timedelta(seconds=max(0.0, _clock() - started))
439
+
440
+ def wait(label, spec, reason):
441
+ waits.append(f"{label}: {spec['label']}: {reason}")
442
+
443
  synced = []
444
  checkpointed = []
445
  for cid, label, profile, legacy in connections:
 
448
  entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
449
  report["connections"].append(entry)
450
  used = _budget(con, today, cid, legacy=legacy)
451
+ # Resume every report this connection left preparing before it creates another one.
452
+ resumed, blocked = {}, False
453
+ for table in TABLES:
454
+ pending = _read_pending(con, table, cid)
455
+ if pending is None:
456
+ continue
457
+ spec = schema.TABLE_SPECS[table]
458
+ _ensure_table(con, table)
459
+ if table_name(table) not in checkpointed:
460
+ checkpointed.append(table_name(table))
461
+ age = (stamp() - pending["created"]).total_seconds()
462
+ expired = (f"Amazon Ads report {pending['report_id']} did not finish in 3 hours")
463
+ if age >= client.REPORT_MAX_WAIT_SECONDS:
464
+ _clear_pending(con, table, cid)
465
+ report["problems"].append(f"{label}: {spec['label']}: {expired}")
466
+ resumed[table] = ("failed", expired, 0, None)
467
+ continue
468
+ limit = client.REPORT_MAX_WAIT_SECONDS - age
469
+ if bounded:
470
+ limit = min(limit, max(0.0, deadline - _clock()))
471
+ try:
472
+ report_id, rows = _run_report(key, table, pending["start"], pending["end"], rt, profile, cid,
473
+ max_wait=limit, report_id=pending["report_id"], now=_clock)
474
+ except client.AdsReportUnavailable as exc:
475
+ _clear_pending(con, table, cid)
476
+ resumed[table] = ("not_offered", _not_offered_reason(spec, exc), 0, None)
477
+ continue
478
+ except client.AdsReportPending:
479
+ if (stamp() - pending["created"]).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS:
480
+ _clear_pending(con, table, cid)
481
+ report["problems"].append(f"{label}: {spec['label']}: {expired}")
482
+ resumed[table] = ("failed", expired, 0, None)
483
+ else:
484
+ blocked = True
485
+ wait(label, spec, PREPARING_REASON)
486
+ resumed[table] = ("deferred", PREPARING_REASON, 0, None)
487
+ continue
488
+ except client.AdsError as exc:
489
+ _clear_pending(con, table, cid)
490
+ reason = str(exc)[:200]
491
+ report["problems"].append(f"{label}: {spec['label']}: {reason}")
492
+ resumed[table] = ("failed", reason, 0, None)
493
+ continue
494
+ _clear_pending(con, table, cid)
495
+ pulled = _land(con, table, rows, report_id, pending["start"], pending["end"], profile, cid, label)
496
+ resumed[table] = ("done", "", pulled, (pending["start"], pending["end"]))
497
+ requested = [schema.short_name(t) for t in (tables or TABLES)]
498
+ for table in list(dict.fromkeys(requested + list(resumed))):
499
  spec = schema.TABLE_SPECS[table]
500
  _ensure_table(con, table)
501
  origin_changed = _refresh_origin(con, table, cid, label, legacy)
502
  pulled, state, mark, reason = 0, "done", _marker_table(table), ""
503
+ prior = resumed.get(table)
504
+ landed = None
505
+ if prior is not None:
506
+ state, reason, pulled, landed = prior
507
+ if state != "done" or table not in requested:
508
+ pass
509
+ elif not spec["offered"]:
510
  state, reason = "not_offered", spec["not_offered_reason"]
511
  elif not _cadence_due(con, table, now, cid):
512
  state = "skipped"
513
+ elif blocked:
514
+ state, reason = "deferred", PREPARING_REASON
515
+ wait(label, spec, reason)
516
  else:
517
  for start, end in windows(con, table, connect_day, today, cid):
518
+ if landed and landed[0] <= start and end <= landed[1]:
519
+ continue # The resumed report just read these days.
520
  if used >= client.REPORTS_PER_DAY:
521
  state = "deferred"
522
  reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
523
  f"reports are used, so days from {start.isoformat()} load on the next sync.")
524
  report["problems"].append(f"{label}: {spec['label']}: waiting for tomorrow's report limit")
525
+ waits.append(report["problems"][-1])
526
+ break
527
+ if bounded and _clock() >= deadline:
528
+ state, reason = "deferred", TIME_LIMIT_REASON
529
+ wait(label, spec, reason)
530
  break
531
  used += 1
532
  _spend_budget(con, today, used, cid)
533
+ created = stamp()
534
  try:
535
+ if bounded:
536
+ report_id, rows = _run_report(key, table, start, end, rt, profile, cid,
537
+ max_wait=max(0.0, deadline - _clock()), now=_clock)
538
+ else:
539
+ report_id, rows = _run_report(key, table, start, end, rt, profile, cid)
540
  except client.AdsReportUnavailable as exc:
541
  state = "not_offered"
542
+ reason = _not_offered_reason(spec, exc)
 
 
543
  break
544
  except client.AdsError as exc:
545
+ if (bounded and isinstance(exc, client.AdsReportPending) and exc.report_id
546
+ and (stamp() - created).total_seconds() < client.REPORT_MAX_WAIT_SECONDS):
547
+ _write_pending(con, table, cid, exc.report_id, start, end, created)
548
+ if table_name(table) not in checkpointed:
549
+ checkpointed.append(table_name(table))
550
+ blocked = True
551
+ state, reason = "deferred", PREPARING_REASON
552
+ wait(label, spec, reason)
553
+ break
554
  state = "failed"
555
  reason = str(exc)[:200]
556
  report["problems"].append(f"{label}: {spec['label']}: {reason}")
557
  break
558
+ pulled += _land(con, table, rows, report_id, start, end, profile, cid, label)
 
 
 
 
 
 
559
  if table_name(table) not in checkpointed:
560
  checkpointed.append(table_name(table))
 
 
 
 
 
 
 
 
561
  if state == "done":
562
  _record_cadence(con, table, now, cid)
563
+ if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
564
+ report["partial"] = True
565
  if reason:
566
  _record_limit(con, table, reason, cid, state=state)
567
  shown = f"{label}: {reason}" if cid else reason
 
591
  _record_limit(con, table, report["limits"][table], state=state["state"])
592
  else:
593
  _clear_limit(con, table)
594
+ report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
595
+ for cid, *_ in connections for table in TABLES)
596
+ if report["partial"]:
597
+ report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
598
  if own and rt is not None and (synced or checkpointed):
599
  _persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
600
  finally:
 
604
  return report
605
 
606
 
607
+ def _not_offered_reason(spec, exc):
608
+ return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
609
+ f"({str(exc)[:160]}). The table keeps the days it has. To ask for it, "
610
+ f"contact Amazon Ads support with the report type {spec['report_type']}.")
611
+
612
+
613
  def _persist(tenant_key, rt, con, synced, report, log):
614
  """Save the Ads tables that landed rows, with their markers. A failure is a problem, never a raise."""
615
  try:
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- b4c8cbaebc5ccbde2b3f83cb6c84b606c16ce361e5f5a31816633a233a008f92
 
1
+ 605d3040010f8ea96127add3e8ca62666d1aa9c7de9da187968a211a2f5dc158