fsanyoto commited on
Commit
3dbbea8
·
verified ·
1 Parent(s): a97d6f1

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "24b7ac5bbe",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "0085702694",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 24b7ac5bbe
 
1
+ 0085702694
api/routes_shares.py CHANGED
@@ -1300,7 +1300,7 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
1300
  # would be a second answer to a question this file already answers once, and would leave
1301
  # the new grantee with an editor and no people to put in it. A `view` grantee is still
1302
  # `may_admin=False` here and still gets `[]`.
1303
- "people": _people(session.tenant) if may_admin else [],
1304
  }
1305
 
1306
 
@@ -1309,6 +1309,15 @@ def _people(tenant):
1309
  return sorted(users.assignable_identities(tenant), key=lambda p: p["name"].lower())
1310
 
1311
 
 
 
 
 
 
 
 
 
 
1312
  def _report_entries_or_400(entries):
1313
  """A report grants download to named people alone."""
1314
  for entry in entries:
 
1300
  # would be a second answer to a question this file already answers once, and would leave
1301
  # the new grantee with an editor and no people to put in it. A `view` grantee is still
1302
  # `may_admin=False` here and still gets `[]`.
1303
+ "people": _picker_people(session.tenant) if may_admin else [],
1304
  }
1305
 
1306
 
 
1309
  return sorted(users.assignable_identities(tenant), key=lambda p: p["name"].lower())
1310
 
1311
 
1312
+ def _picker_people(tenant):
1313
+ """The share picker's people: `_people` without the QA logins (D-11165).
1314
+
1315
+ The write door keeps `_people`, so a stored grant to a QA login still saves when the dialog
1316
+ sends it back.
1317
+ """
1318
+ return [p for p in _people(tenant) if not users.is_qa_login(p["username"])]
1319
+
1320
+
1321
  def _report_entries_or_400(entries):
1322
  """A report grants download to named people alone."""
1323
  for entry in entries:
platform/core/connector_spec.py CHANGED
@@ -3785,15 +3785,15 @@ AMAZON_ADS_REPORTS = ("campaigns", "targeting", "advertised_products", "search_t
3785
  _ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
3786
  _ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting",
3787
  "advertised_products": "advertised products", "search_terms": "search terms"}
3788
- #: `native` is the Ads report type id. Unknown: the exact v1 spelling; the first credentialed
3789
- #: report settles it (PRD P17), and each id is one literal here.
3790
  _ADS_NATIVE = {
3791
- "sp": {"campaigns": "spCampaigns", "targeting": "spTargeting",
3792
- "advertised_products": "spAdvertisedProduct", "search_terms": "spSearchTerm"},
3793
- "sb": {"campaigns": "sbCampaigns", "targeting": "sbTargeting",
3794
- "advertised_products": "sbAds", "search_terms": "sbSearchTerm"},
3795
- "sd": {"campaigns": "sdCampaigns", "targeting": "sdTargeting",
3796
- "advertised_products": "sdAdvertisedProduct", "search_terms": "sdSearchTerm"},
3797
  }
3798
  _ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting",
3799
  "advertised_products": "asin", "search_terms": "search_term"}
@@ -3822,21 +3822,22 @@ AMAZON_ADS_TABLES = tuple(
3822
  #: which verify_connectors holds equal to the `"amazon"` registration.
3823
  AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES}
3824
 
3825
- #: Days the first sync reaches back, per table: the longest window Amazon documents per ad type
3826
- #: (SP 95, SB 60, SD 65, search terms 60). Unknown: v1 retention (primer F12). The first
3827
- #: credentialed sync measures the oldest day Amazon returns.
3828
- AMAZON_ADS_BACKFILL_DAYS = {
3829
- spec.key: (60 if spec.key.endswith("_search_terms")
3830
- else {"sp": 95, "sb": 60, "sd": 65}[spec.key[len(_ADS):len(_ADS) + 2]])
3831
- for spec in AMAZON_ADS_TABLES}
3832
 
3833
  #: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen.
3834
  AMAZON_ADS_THROUGHPUT = (
3835
- "Amazon Reporting v1 runs one report at a time per advertiser and about 100 reports a day. "
3836
- "Sponsored Products, Brands and Display need 11 report types, and one report can take up to "
3837
- "3 hours, so one day of refresh may not finish in a day. Loopable runs the reports one at a "
3838
- "time and keeps every day it already has. If the refresh falls behind, ask Amazon Ads support "
3839
- "to raise the report limit for this advertiser.")
 
 
 
 
 
3840
 
3841
  #: Standing rule 1: the report v1 does not offer, with cause and fix. On screen.
3842
  AMAZON_ADS_NOT_OFFERED = {
@@ -3859,9 +3860,10 @@ def amazon_ads_description(key):
3859
  if unoffered:
3860
  return f"{unoffered} {AMAZON_ADS_THROUGHPUT}"
3861
  days = AMAZON_ADS_BACKFILL_DAYS[spec.key]
3862
- return (f"{spec.label} by day, from Amazon Ads Reporting v1. History is kept from the day you "
3863
- f"connect, and the first sync reaches back {days} days, the longest Amazon keeps. "
3864
- f"{AMAZON_ADS_THROUGHPUT}")
 
3865
 
3866
 
3867
  def table_group(key):
 
3785
  _ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
3786
  _ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting",
3787
  "advertised_products": "advertised products", "search_terms": "search terms"}
3788
+ #: `native` is the Reporting v1 report group that fills the table (`amazon_ads_schema.REPORT_GROUPS`).
3789
+ #: v1 has no report types. SD search terms has no group, because v1 does not offer it.
3790
  _ADS_NATIVE = {
3791
+ "sp": {"campaigns": "campaigns", "targeting": "targeting",
3792
+ "advertised_products": "advertised_products", "search_terms": "search_terms"},
3793
+ "sb": {"campaigns": "campaigns", "targeting": "targeting",
3794
+ "advertised_products": "advertised_products_sb", "search_terms": "search_terms"},
3795
+ "sd": {"campaigns": "campaigns", "targeting": "targeting",
3796
+ "advertised_products": "advertised_products", "search_terms": "not_offered"},
3797
  }
3798
  _ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting",
3799
  "advertised_products": "asin", "search_terms": "search_term"}
 
3822
  #: which verify_connectors holds equal to the `"amazon"` registration.
3823
  AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES}
3824
 
3825
+ #: Days the first sync reaches back: Reporting v1 keeps 24 months of daily data, and standing
3826
+ #: rule 1 takes all of it.
3827
+ AMAZON_ADS_BACKFILL_DAYS = {spec.key: 730 for spec in AMAZON_ADS_TABLES}
 
 
 
 
3828
 
3829
  #: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen.
3830
  AMAZON_ADS_THROUGHPUT = (
3831
+ "Amazon Ads runs one report at a time for each Loopable app login (its client ID) and about "
3832
+ "100 reports a day, shared by every advertiser on that login. Sponsored Products, Brands and "
3833
+ "Display need 5 reports for each refresh, and one report can take up to 3 hours, so one day of "
3834
+ "refresh may not finish in a day. Loopable runs the reports one at a time and keeps every day "
3835
+ "it already has. If the refresh falls behind, ask Amazon Ads support to raise the report limit "
3836
+ "for this client ID.")
3837
+
3838
+ #: On screen: the attribution window Amazon applies to Sponsored Products seller metrics.
3839
+ AMAZON_ADS_SP_ATTRIBUTION = ("Sponsored Products sales, orders and units count purchases within 7 "
3840
+ "days of a click, Amazon's default for sellers.")
3841
 
3842
  #: Standing rule 1: the report v1 does not offer, with cause and fix. On screen.
3843
  AMAZON_ADS_NOT_OFFERED = {
 
3860
  if unoffered:
3861
  return f"{unoffered} {AMAZON_ADS_THROUGHPUT}"
3862
  days = AMAZON_ADS_BACKFILL_DAYS[spec.key]
3863
+ attribution = AMAZON_ADS_SP_ATTRIBUTION + " " if spec.key.startswith(_ADS + "sp_") else ""
3864
+ return (f"{spec.label} by day, from Amazon Ads Reporting v1. {attribution}History is kept from "
3865
+ f"the day you connect, and the first sync reaches back {days} days, the longest Amazon "
3866
+ f"keeps. {AMAZON_ADS_THROUGHPUT}")
3867
 
3868
 
3869
  def table_group(key):
platform/core/users.py CHANGED
@@ -1203,6 +1203,16 @@ def is_test_account(record):
1203
  return isinstance(record, dict) and isinstance(record.get('test_account'), dict)
1204
 
1205
 
 
 
 
 
 
 
 
 
 
 
1206
  #: The keys of one database entry that `set_database_access` writes.
1207
  DATABASE_ENTRY_KEYS = ('access', 'filter', 'hiddenFields')
1208
 
 
1203
  return isinstance(record, dict) and isinstance(record.get('test_account'), dict)
1204
 
1205
 
1206
+ #: The QA sign-in logins: slot 1, slots 2 to 9 and the old sandbox runner. They carry no
1207
+ #: `test_account` marker, so a picker that must leave them out asks this (D-11165).
1208
+ _QA_LOGIN = re.compile(r"qa-(?:bot|member)(?:-[2-9])?|qa-runner")
1209
+
1210
+
1211
+ def is_qa_login(username):
1212
+ """True for a QA sign-in account, which a share picker never offers as a person."""
1213
+ return bool(_QA_LOGIN.fullmatch(str(username or "").strip().lower()))
1214
+
1215
+
1216
  #: The keys of one database entry that `set_database_access` writes.
1217
  DATABASE_ENTRY_KEYS = ('access', 'filter', 'hiddenFields')
1218
 
platform/harness/amazon_ads_client.py CHANGED
@@ -4,7 +4,7 @@
4
  read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the
5
  refusal lives here, in the request builder, before any credential read, token exchange or socket:
6
 
7
- reports create, status and row pages on Reporting v1 (R8)
8
  streams create and list subscriptions for the dedicated V1 destination
9
  named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in
10
  `bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant
@@ -17,8 +17,11 @@ harness module never imports `modules` at load). A missing approval source refus
17
  made until Royal's Direct Advertiser credential arrives (R7, R17).
18
 
19
  NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is
20
- stored with the credential, so the client needs no profile listing call.
 
 
21
  """
 
22
  import importlib
23
  import json
24
  import logging
@@ -31,6 +34,7 @@ import urllib.parse
31
  import urllib.request
32
  from contextlib import contextmanager
33
  from datetime import date, datetime, timedelta, timezone
 
34
  from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
35
 
36
  KEY = "amazon_ads"
@@ -46,22 +50,28 @@ DEFAULT_REGION = "na"
46
  LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
47
  CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id")
48
 
49
- #: Reporting v1 limits (primer F11, F12): one report in flight per advertiser, about 100 a day,
50
- #: up to 3 hours per report, at most 31 days per request.
51
  REPORT_CONCURRENCY = 1
52
  REPORTS_PER_DAY = 100
53
  REPORT_MAX_WAIT_SECONDS = 3 * 3600
54
- MAX_DAYS_PER_REQUEST = 31
55
- REPORT_POLL_SECONDS = 30.0
 
 
56
  _TIMEOUT = 60
57
 
58
- #: The only non-write calls: Reporting v1 create, status and row pages. Unknown: the exact v1
59
- #: paths; the first credentialed call settles them, and each is one literal here.
60
  REPORT_OPERATIONS = {
61
- "createReport": ("POST", "reporting/v1/reports"),
62
- "getReport": ("GET", "reporting/v1/reports/{reportId}"),
63
- "getReportRows": ("GET", "reporting/v1/reports/{reportId}/rows"),
64
  }
 
 
 
 
65
 
66
  # Subscription management never admits an advertising write.
67
  STREAM_OPERATIONS = {
@@ -208,7 +218,15 @@ class AdsWriteRefused(AdsError):
208
  f"the named writes: bid, budget, negative keyword and scheduled change.")
209
 
210
 
211
- class AdsReportUnavailable(AdsError):
 
 
 
 
 
 
 
 
212
  """Amazon does not offer this report to this advertiser."""
213
 
214
 
@@ -225,25 +243,10 @@ def fixture_mode():
225
  return os.environ.get(FIXTURE_ENV) == "1"
226
 
227
 
228
- def _template(path):
229
- return tuple(path.strip("/").split("/"))
230
-
231
-
232
- def _matches(template, path):
233
- parts = str(path or "").strip("/").split("/")
234
- if len(parts) != len(template):
235
- return False
236
- return all((bool(re.fullmatch(r"[A-Za-z0-9_-]{1,128}", part)))
237
- if seg.startswith("{") and seg.endswith("}") else part == seg
238
- for part, seg in zip(parts, template))
239
-
240
-
241
  def report_operation(method, path):
242
- """The Reporting v1 operation name for one call, or None."""
243
- for name, (verb, tmpl) in REPORT_OPERATIONS.items():
244
- if str(method).upper() == verb and _matches(_template(tmpl), path):
245
- return name
246
- return None
247
 
248
 
249
  class _Prepared:
@@ -503,12 +506,17 @@ def _send(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
503
  if not _admitted(prepared):
504
  raise AdsWriteRefused("the request did not come from the request builder")
505
  if fixture_mode():
506
- return FIXTURE.answer(prepared, tenant_key, profile_id)
 
507
  return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id)
508
 
509
 
510
  # --------------------------------------------------------------------------- live transport
511
 
 
 
 
 
512
  def credential(tenant_key, rt=None, connection_id=None):
513
  """The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source."""
514
  try:
@@ -538,11 +546,50 @@ def _token(fields):
538
  raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None
539
 
540
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
541
  def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None):
542
  fields = credential(tenant_key, rt, connection_id=connection_id)
543
  if fields is None:
544
- raise AdsError("No Amazon Ads credential is stored for this tenant. "
545
- "Connect Amazon Ads under Amazon Ads Setup.")
546
  if prepared.operation in STREAM_OPERATIONS and (
547
  str(profile_id or fields["profile_id"]) != str(fields["profile_id"])
548
  or str(fields.get("region") or DEFAULT_REGION).lower() != "na"):
@@ -551,13 +598,7 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
551
  url = f"{host}/{prepared.path}"
552
  if prepared.query:
553
  url += "?" + urllib.parse.urlencode(prepared.query)
554
- headers = {"Authorization": f"Bearer {_token(fields)}",
555
- "Amazon-Advertising-API-ClientId": fields["client_id"],
556
- "Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]),
557
- "Content-Type": "application/json", "Accept": "application/json"}
558
- if prepared.operation in STREAM_OPERATIONS:
559
- headers["Content-Type"] = STREAM_CONTENT_TYPE
560
- headers["Accept"] = STREAM_CONTENT_TYPE
561
  data = json.dumps(prepared.body).encode() if prepared.body is not None else None
562
  req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers)
563
  try:
@@ -566,12 +607,10 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
566
  except urllib.error.HTTPError as exc:
567
  try:
568
  detail = json.loads(exc.read().decode("utf-8", "replace") or "{}")
569
- message = str(detail.get("details") or detail.get("message") or "")[:200]
570
- except Exception: # noqa: BLE001
571
- message = ""
572
- if exc.code == 400 and re.search(r"not (?:supported|offered|available)", message, re.I):
573
- raise AdsReportUnavailable(f"Amazon does not offer this report: {message}") from None
574
- raise AdsError(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}: {message}") from None
575
  except Exception as exc: # noqa: BLE001
576
  raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None
577
 
@@ -580,89 +619,276 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
580
 
581
  _REPORT_LOCKS = {}
582
  _REPORT_LOCKS_GUARD = threading.Lock()
 
 
 
 
 
 
 
 
 
 
 
 
583
 
584
 
585
- def _report_lock(tenant_key, profile_id):
 
586
  with _REPORT_LOCKS_GUARD:
587
- return _REPORT_LOCKS.setdefault((tenant_key, str(profile_id or "")), threading.Lock())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
588
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
589
 
590
- def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, connection_id=None, sleep=time.sleep,
591
- now=time.monotonic, max_wait=None, report_id=None):
592
- """Create one Reporting v1 report, wait for it, and read every row page. -> (report_id, rows).
593
 
594
- `max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without createReport.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
595
  """
596
  from harness import amazon_ads_schema as schema
597
- spec = schema.TABLE_SPECS[schema.short_name(table)]
598
  if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
599
- raise ValueError("a report window is at most 31 days")
600
- body = {"reportTypeId": spec["report_type"], "adProduct": spec["ad_type"].upper(),
601
- "startDate": start.isoformat(), "endDate": end.isoformat(),
602
- "timeUnit": "DAILY", "columns": list(schema.vendor_columns(table)), "format": "JSON"}
603
  limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
604
- # One report in flight per advertiser (F11): the lock serialises this process's reports.
605
- with _report_lock(tenant_key, profile_id):
606
  if report_id is None:
607
- created = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1], body=body,
608
- rt=rt, profile_id=profile_id, connection_id=connection_id)
609
- report_id = str(created.get("reportId") or "")
610
- else:
611
- report_id = str(report_id)
612
- if not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", report_id):
613
- raise AdsError("Amazon Ads returned no report id")
614
- started = now()
615
- status_path = REPORT_OPERATIONS["getReport"][1].replace("{reportId}", report_id)
616
- while True:
617
- status = request(tenant_key, "GET", status_path, rt=rt, profile_id=profile_id, connection_id=connection_id)
618
- state = str(status.get("status") or "").upper()
619
- if state == "COMPLETED":
620
- break
621
- if state in ("FAILED", "FAILURE", "CANCELLED"):
622
- reason = str(status.get("failureReason") or state)[:200]
623
- if re.search(r"not (?:supported|offered|available)", reason, re.I):
624
- raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}")
625
- raise AdsError(f"Amazon Ads report {report_id} ended {state}: {reason}")
626
- elapsed = now() - started
627
- if elapsed > REPORT_MAX_WAIT_SECONDS:
628
- raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
629
- if limit is not None and elapsed >= limit:
630
- raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
631
- pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
632
- sleep(pause if limit is None else min(pause, limit - elapsed))
633
- rows, token = [], None
634
- rows_path = REPORT_OPERATIONS["getReportRows"][1].replace("{reportId}", report_id)
635
- while True:
636
- page = request(tenant_key, "GET", rows_path, query={"nextToken": token} if token else None,
637
- rt=rt, profile_id=profile_id, connection_id=connection_id)
638
- rows.extend(page.get("rows") or [])
639
- token = page.get("nextToken")
640
- if not token:
641
- return report_id, rows
642
 
643
 
644
  # --------------------------------------------------------------------------- recording fixture
645
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
646
  class _Fixture:
647
- """The recording fixture `AMAZON_ADS_FIXTURE=1` routes to. Records each call; answers reports
648
- with deterministic rows, two report polls, and paged rows."""
649
 
650
- PAGE_SIZE = 25
 
 
 
 
 
651
 
652
  def __init__(self):
653
  self.reset()
654
 
655
  def reset(self):
656
- self.calls, self.reports = [], {}
657
- self.subscriptions = {}
658
  self.in_flight = self.max_in_flight = 0
659
- self.drop_days, self.unsupported = set(), set()
 
660
  self.responder = None
661
 
662
- def answer(self, prepared, tenant_key, profile_id):
 
 
 
 
663
  self.calls.append({"method": prepared.method, "path": prepared.path,
664
  "operation": prepared.operation, "body": prepared.body,
665
- "query": dict(prepared.query), "tenant": tenant_key,
666
  "named": dict(prepared.named) if prepared.named else None})
667
  if self.responder is not None:
668
  return self.responder(prepared)
@@ -678,88 +904,174 @@ class _Fixture:
678
  if prepared.named is not None:
679
  items = next(iter(prepared.body.values()))
680
  return {"success": [{"index": i} for i in range(len(items))], "error": []}
 
 
681
  if prepared.operation == "createReport":
682
- report_id = f"fx{len(self.reports) + 1:05d}"
683
- self.in_flight += 1
684
- self.max_in_flight = max(self.max_in_flight, self.in_flight)
685
- self.reports[report_id] = {"body": prepared.body, "polls": 0, "rows": None}
686
- return {"reportId": report_id, "status": "PENDING"}
687
- report_id = prepared.path.split("/")[3]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
688
  report = self.reports[report_id]
689
- if prepared.operation == "getReport":
690
- report["polls"] += 1
691
- if report["body"]["reportTypeId"] in self.unsupported:
692
- self.in_flight -= 1
693
- return {"status": "FAILED", "failureReason": "Report type is not supported for this advertiser"}
694
- if report["polls"] < 2:
695
- return {"status": "PROCESSING"}
696
- return {"status": "COMPLETED"}
697
- if report["rows"] is None:
698
- report["rows"] = fixture_rows(report["body"], drop_days=self.drop_days)
699
- start = int(prepared.query.get("nextToken") or 0)
700
- page = report["rows"][start:start + self.PAGE_SIZE]
701
- end = start + len(page)
702
- if end >= len(report["rows"]):
 
 
 
 
 
 
703
  self.in_flight -= 1
704
- return {"rows": page}
705
- return {"rows": page, "nextToken": str(end)}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
706
 
707
 
708
  def fixture_rows(body, *, drop_days=()):
709
- """Deterministic Ads report rows for one request body, every day in its window."""
710
- report_type, columns = body["reportTypeId"], body["columns"]
711
- start, end = date.fromisoformat(body["startDate"]), date.fromisoformat(body["endDate"])
712
- ad = report_type[:2]
713
- out, day = [], start
714
- while day <= end:
715
- if day.isoformat() not in drop_days:
716
- for c in (1, 2):
717
- for g in ((0,) if report_type.endswith("Campaigns") else (1, 2)):
718
- n = day.toordinal() % 7 + c + g
719
- row = {
720
- "date": day.isoformat(), "campaignId": f"{ad.encode().hex()}10{c}",
721
- "campaignName": f"{ad.upper()} campaign {c}", "campaignStatus": "ENABLED",
722
- "campaignBudgetAmount": 25.0 * c, "campaignBudgetCurrencyCode": "USD",
723
- "adGroupId": f"20{c}{g}", "adGroupName": f"Ad group {c}.{g}",
724
- "keywordId": f"30{c}{g}", "targetingId": f"30{c}{g}",
725
- "targeting": f"term {c}{g}", "targetingExpression": f"asin is B00FX{c}{g}",
726
- "matchType": "exact", "keywordBid": 0.5 + g / 10,
727
- "adId": f"40{c}{g}", "advertisedSku": f"SKU-{c}{g}",
728
- "advertisedAsin": f"B00FX0{c}{g}0", "promotedSku": f"SKU-{c}{g}",
729
- "promotedAsin": f"B00FX0{c}{g}0", "searchTerm": f"search {c}{g}",
730
- "impressions": 100 * n, "clicks": 3 * n, "cost": round(1.25 * n, 2),
731
- "sales": round(9.5 * n, 2), "sales14d": round(9.5 * n, 2),
732
- "purchases": n, "purchases14d": n, "unitsSold": n, "unitsSoldClicks14d": n,
733
- }
734
- out.append({k: row.get(k) for k in columns})
735
- day += timedelta(days=1)
736
  return out
737
 
738
 
739
  FIXTURE = _Fixture()
740
 
741
 
 
 
 
 
 
 
742
  @contextmanager
743
  def recording_fixture():
744
- """Route every call to a fresh recording fixture for the block. -> the fixture."""
745
  previous = os.environ.get(FIXTURE_ENV)
746
  os.environ[FIXTURE_ENV] = "1"
747
  FIXTURE.reset()
 
748
  try:
749
  yield FIXTURE
750
  finally:
751
  FIXTURE.reset()
 
752
  if previous is None:
753
  os.environ.pop(FIXTURE_ENV, None)
754
  else:
755
  os.environ[FIXTURE_ENV] = previous
756
 
757
 
758
- __all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "NAMED_WRITES",
759
- "STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation",
760
  "FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE",
761
  "SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY",
762
- "MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsReportUnavailable",
763
- "AdsReportPending", "fixture_mode", "build_request", "report_operation",
764
- "send_named_write", "request", "credential", "run_report", "FIXTURE",
765
- "recording_fixture", "fixture_rows"]
 
4
  read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the
5
  refusal lives here, in the request builder, before any credential read, token exchange or socket:
6
 
7
+ reports Reporting v1: create, retrieve, the advertiser account query, and the report files
8
  streams create and list subscriptions for the dedicated V1 destination
9
  named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in
10
  `bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant
 
17
  made until Royal's Direct Advertiser credential arrives (R7, R17).
18
 
19
  NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is
20
+ stored with the credential. v1 names the account by `advertiserAccountId`, which the client looks
21
+ up once per connection from `query/advertiserAccounts`. Every path, header and field comes from
22
+ Amazon's saved contracts in `AutoWiki/research/amazon-ads-v1-schema/`.
23
  """
24
+ import gzip
25
  import importlib
26
  import json
27
  import logging
 
34
  import urllib.request
35
  from contextlib import contextmanager
36
  from datetime import date, datetime, timedelta, timezone
37
+ from typing import NamedTuple
38
  from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
39
 
40
  KEY = "amazon_ads"
 
50
  LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
51
  CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id")
52
 
53
+ #: Reporting v1 limits (limits.md, dates.md): one report in flight per client ID, about 100 a day,
54
+ #: up to 3 hours per report, at most 120 days per request.
55
  REPORT_CONCURRENCY = 1
56
  REPORTS_PER_DAY = 100
57
  REPORT_MAX_WAIT_SECONDS = 3 * 3600
58
+ MAX_DAYS_PER_REQUEST = 120
59
+ REPORT_POLL_SECONDS = 60.0
60
+ #: Amazon's error text reaches the sync report verbatim, up to this many characters.
61
+ ERROR_TEXT_LIMIT = 600
62
  _TIMEOUT = 60
63
 
64
+ #: The only non-write calls: the Reporting v1 operations, each a POST to one literal path from the
65
+ #: Reports and AdvertiserAccounts contracts. Every other `adsApi` path refuses, a GET included.
66
  REPORT_OPERATIONS = {
67
+ "createReport": ("POST", "adsApi/v1/create/reports"),
68
+ "retrieveReport": ("POST", "adsApi/v1/retrieve/reports"),
69
+ "queryAdvertiserAccounts": ("POST", "adsApi/v1/query/advertiserAccounts"),
70
  }
71
+ #: The contract names `Amazon-Ads-ClientId`. The limits page names the older header. v1 sends both.
72
+ V1_CLIENT_HEADERS = ("Amazon-Ads-ClientId", "Amazon-Advertising-API-ClientId")
73
+ #: A report file is a presigned S3 URL: https, on this host suffix, GET with no Ads header.
74
+ REPORT_FILE_HOST_SUFFIX = ".amazonaws.com"
75
 
76
  # Subscription management never admits an advertising write.
77
  STREAM_OPERATIONS = {
 
218
  f"the named writes: bid, budget, negative keyword and scheduled change.")
219
 
220
 
221
+ class AdsRejected(AdsError):
222
+ """Amazon answered an error. `errors` keeps each entry's code, message and fieldLocation."""
223
+
224
+ def __init__(self, message, errors=()):
225
+ super().__init__(message)
226
+ self.errors = list(errors)
227
+
228
+
229
+ class AdsReportUnavailable(AdsRejected):
230
  """Amazon does not offer this report to this advertiser."""
231
 
232
 
 
243
  return os.environ.get(FIXTURE_ENV) == "1"
244
 
245
 
 
 
 
 
 
 
 
 
 
 
 
 
 
246
  def report_operation(method, path):
247
+ """The Reporting v1 operation name for one call, or None. A path matches only its literal."""
248
+ return next((name for name, (verb, endpoint) in REPORT_OPERATIONS.items()
249
+ if str(method).upper() == verb and str(path or "").strip("/") == endpoint), None)
 
 
250
 
251
 
252
  class _Prepared:
 
506
  if not _admitted(prepared):
507
  raise AdsWriteRefused("the request did not come from the request builder")
508
  if fixture_mode():
509
+ headers = _headers(prepared, FIXTURE.credential(tenant_key), FIXTURE_TOKEN, profile_id)
510
+ return FIXTURE.answer(prepared, tenant_key, profile_id, headers)
511
  return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id)
512
 
513
 
514
  # --------------------------------------------------------------------------- live transport
515
 
516
+ _NO_CREDENTIAL = ("No Amazon Ads credential is stored for this tenant. "
517
+ "Connect Amazon Ads under Amazon Ads Setup.")
518
+
519
+
520
  def credential(tenant_key, rt=None, connection_id=None):
521
  """The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source."""
522
  try:
 
546
  raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None
547
 
548
 
549
+ def _headers(prepared, fields, token, profile_id=None):
550
+ """The headers one admitted call sends. A v1 call names its account in the body, not a scope."""
551
+ if prepared.operation in REPORT_OPERATIONS:
552
+ return {"Authorization": f"Bearer {token}", **dict.fromkeys(V1_CLIENT_HEADERS, fields["client_id"]),
553
+ "Content-Type": "application/json", "Accept": "application/json"}
554
+ headers = {"Authorization": f"Bearer {token}",
555
+ "Amazon-Advertising-API-ClientId": fields["client_id"],
556
+ "Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]),
557
+ "Content-Type": "application/json", "Accept": "application/json"}
558
+ if prepared.operation in STREAM_OPERATIONS:
559
+ headers["Content-Type"] = STREAM_CONTENT_TYPE
560
+ headers["Accept"] = STREAM_CONTENT_TYPE
561
+ return headers
562
+
563
+
564
+ def _errors(detail):
565
+ """Amazon's error entries from an error body, each with its code, message and fieldLocation."""
566
+ if not isinstance(detail, dict):
567
+ return []
568
+ entries = detail["errors"] if isinstance(detail.get("errors"), list) else [detail]
569
+ return [{"code": str(e.get("code") or ""), "message": str(e.get("details") or e.get("message") or ""),
570
+ "fieldLocation": str(e.get("fieldLocation") or "")}
571
+ for e in entries if isinstance(e, dict) and (e.get("code") or e.get("details") or e.get("message"))]
572
+
573
+
574
+ def _error_text(errors):
575
+ """Amazon's errors as one line a person reads, verbatim, cut to `ERROR_TEXT_LIMIT`."""
576
+ return " ".join(" ".join(filter(None, (e["code"] + ":" if e["code"] else "", e["message"],
577
+ f"(field {e['fieldLocation']})" if e["fieldLocation"] else "")))
578
+ for e in errors)[:ERROR_TEXT_LIMIT]
579
+
580
+
581
+ def _rejection(prefix, errors, *, may_be_unoffered=True):
582
+ """The exception for Amazon's errors: not offered when Amazon says so, else rejected."""
583
+ text = _error_text(errors)
584
+ if may_be_unoffered and re.search(r"not (?:supported|offered|available)", text, re.I):
585
+ return AdsReportUnavailable(f"Amazon does not offer this report: {text}", errors)
586
+ return AdsRejected(f"{prefix}: {text}" if text else prefix, errors)
587
+
588
+
589
  def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None):
590
  fields = credential(tenant_key, rt, connection_id=connection_id)
591
  if fields is None:
592
+ raise AdsError(_NO_CREDENTIAL)
 
593
  if prepared.operation in STREAM_OPERATIONS and (
594
  str(profile_id or fields["profile_id"]) != str(fields["profile_id"])
595
  or str(fields.get("region") or DEFAULT_REGION).lower() != "na"):
 
598
  url = f"{host}/{prepared.path}"
599
  if prepared.query:
600
  url += "?" + urllib.parse.urlencode(prepared.query)
601
+ headers = _headers(prepared, fields, _token(fields), profile_id)
 
 
 
 
 
 
602
  data = json.dumps(prepared.body).encode() if prepared.body is not None else None
603
  req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers)
604
  try:
 
607
  except urllib.error.HTTPError as exc:
608
  try:
609
  detail = json.loads(exc.read().decode("utf-8", "replace") or "{}")
610
+ except Exception: # noqa: BLE001 an unreadable body still raises its status
611
+ detail = {}
612
+ raise _rejection(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}", _errors(detail),
613
+ may_be_unoffered=exc.code == 400) from None
 
 
614
  except Exception as exc: # noqa: BLE001
615
  raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None
616
 
 
619
 
620
  _REPORT_LOCKS = {}
621
  _REPORT_LOCKS_GUARD = threading.Lock()
622
+ #: Process caches: the advertiser account per connection, the optional fields Amazon refused per
623
+ #: (client id, group) with Amazon's text, and the report file URLs a retrieve returned.
624
+ _ACCOUNTS = {}
625
+ _REFUSED = {}
626
+ _ISSUED_FILES = set()
627
+
628
+
629
+ class GroupReport(NamedTuple):
630
+ """One finished group report: its id, flat rows per member table, and notes a person reads."""
631
+ report_id: str
632
+ rows: dict
633
+ notes: tuple
634
 
635
 
636
+ def _report_lock(client_id):
637
+ """One lock per client ID: Amazon runs one report at a time for each app login."""
638
  with _REPORT_LOCKS_GUARD:
639
+ return _REPORT_LOCKS.setdefault(str(client_id), threading.Lock())
640
+
641
+
642
+ def _report_credential(tenant_key, rt, connection_id):
643
+ """The credential a report runs under: its client id keys the lock, its profile names the account."""
644
+ if fixture_mode():
645
+ return FIXTURE.credential(tenant_key)
646
+ fields = credential(tenant_key, rt, connection_id=connection_id)
647
+ if fields is None:
648
+ raise AdsError(_NO_CREDENTIAL)
649
+ return fields
650
+
651
+
652
+ def _lists_profile(account, profile_id):
653
+ return (isinstance(account, dict) and bool(account.get("advertiserAccountId"))
654
+ and any(isinstance(alt, dict) and str(alt.get("profileId")) == str(profile_id)
655
+ for alt in account.get("alternateIds") or ()))
656
+
657
 
658
+ def _advertiser_account(tenant_key, *, rt, profile_id, connection_id):
659
+ """The v1 `advertiserAccountId` whose alternate ids carry the stored profile id, cached per connection."""
660
+ key, token = (tenant_key, connection_id or "", str(profile_id)), None
661
+ while key not in _ACCOUNTS:
662
+ page = request(tenant_key, "POST", REPORT_OPERATIONS["queryAdvertiserAccounts"][1],
663
+ body={"maxResults": 100, **({"nextToken": token} if token else {})},
664
+ rt=rt, profile_id=profile_id, connection_id=connection_id)
665
+ match = next((a for a in page.get("advertiserAccounts") or () if _lists_profile(a, profile_id)), None)
666
+ token = page.get("nextToken")
667
+ if match is not None:
668
+ _ACCOUNTS[key] = str(match["advertiserAccountId"])
669
+ elif not token:
670
+ raise AdsError(f"No Amazon Ads advertiser account lists profile {profile_id}. "
671
+ f"Check the profile ID on this Amazon Ads connection.")
672
+ return _ACCOUNTS[key]
673
 
 
 
 
674
 
675
+ def _create_body(group, start, end, account, fields):
676
+ """A CreateReportRequest for one group and window, filtered to the group's ad products."""
677
+ from harness import amazon_ads_schema as schema
678
+ return {"accessRequestedAccounts": [{"advertiserAccountId": account}],
679
+ "reports": [{"format": "GZIP_JSON",
680
+ "periods": [{"datePeriod": {"startDate": start.isoformat(), "endDate": end.isoformat()}}],
681
+ "query": {"fields": list(fields),
682
+ "filter": {"on": {"field": schema.AD_PRODUCT_FIELD, "comparisonOperator": "IN",
683
+ "not": False, "values": list(group.tables)}}}}]}
684
+
685
+
686
+ def _first_report(reply, operation):
687
+ """The one report a 207 multi-status answer carries, or its errors raised verbatim."""
688
+ errors = [e for entry in reply.get("error") or () if isinstance(entry, dict)
689
+ for e in _errors({"errors": entry.get("errors") or []})]
690
+ if errors or reply.get("error"):
691
+ raise _rejection(f"Amazon Ads refused {operation}", errors)
692
+ for entry in reply.get("success") or ():
693
+ report = entry.get("report") if isinstance(entry, dict) else None
694
+ if isinstance(report, dict) and re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", str(report.get("reportId") or "")):
695
+ return report
696
+ raise AdsError(f"Amazon Ads returned no report on {operation}")
697
+
698
+
699
+ def _named_fields(exc, fields):
700
+ """The requested fields an error names, by id in its text or by index in its field location."""
701
+ text = " ".join([str(exc)] + [e["fieldLocation"] for e in exc.errors])
702
+ named = {f for f in fields if re.search(rf"(?<![\w.]){re.escape(f)}(?!\w)", text)}
703
+ named.update(fields[int(i)] for i in re.findall(r"fields\[(\d+)\]", text) if int(i) < len(fields))
704
+ return named
705
+
706
+
707
+ def _create(tenant_key, group, start, end, account, refused, opts):
708
+ """Create one group report. An error naming only optional fields drops them and creates again."""
709
+ while True:
710
+ fields = group.fields(drop=refused)
711
+ try:
712
+ reply = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1],
713
+ body=_create_body(group, start, end, account, fields), **opts)
714
+ return _first_report(reply, "createReport")
715
+ except AdsRejected as exc:
716
+ named = _named_fields(exc, fields)
717
+ if not named or named & set(group.core):
718
+ raise
719
+ refused.update(dict.fromkeys(sorted(named), _error_text(exc.errors) or str(exc)))
720
+
721
+
722
+ def _await(tenant_key, report_id, opts, *, sleep, now, limit):
723
+ """Poll retrieve about once a minute until the report completes. -> its report file URLs."""
724
+ started = now()
725
+ while True:
726
+ report = _first_report(request(tenant_key, "POST", REPORT_OPERATIONS["retrieveReport"][1],
727
+ body={"reportIds": [report_id]}, **opts), "retrieveReport")
728
+ state = str(report.get("status") or "").upper()
729
+ if state == "COMPLETED":
730
+ urls = [str(part.get("url") or "") for part in report.get("completedReportParts") or ()
731
+ if isinstance(part, dict)]
732
+ _ISSUED_FILES.update(urls)
733
+ return urls
734
+ if state == "FAILED":
735
+ reason = " ".join(filter(None, (str(report.get("failureCode") or ""),
736
+ str(report.get("failureReason") or ""))))[:ERROR_TEXT_LIMIT] or state
737
+ if re.search(r"not (?:supported|offered|available)", reason, re.I):
738
+ raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}")
739
+ raise AdsError(f"Amazon Ads report {report_id} ended FAILED: {reason}")
740
+ elapsed = now() - started
741
+ if elapsed > REPORT_MAX_WAIT_SECONDS:
742
+ raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
743
+ if limit is not None and elapsed >= limit:
744
+ raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
745
+ pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
746
+ sleep(pause if limit is None else min(pause, limit - elapsed))
747
+
748
+
749
+ def _flat(row, prefix=""):
750
+ """A nested v1 row -> flat dotted field ids. A flat row stays as it is."""
751
+ out = {}
752
+ for key, value in row.items():
753
+ if isinstance(value, dict):
754
+ out.update(_flat(value, f"{prefix}{key}."))
755
+ else:
756
+ out[f"{prefix}{key}"] = value
757
+ return out
758
+
759
+
760
+ def _part_rows(raw):
761
+ """One report file -> flat rows. GZIP or plain, a JSON array or JSON lines."""
762
+ try:
763
+ text = (gzip.decompress(raw) if raw[:2] == b"\x1f\x8b" else raw).decode("utf-8", "replace").strip()
764
+ if not text:
765
+ return []
766
+ try:
767
+ data = json.loads(text)
768
+ except ValueError:
769
+ data = [json.loads(line) for line in text.splitlines() if line.strip()]
770
+ except (OSError, EOFError, ValueError) as exc:
771
+ raise AdsError(f"Amazon Ads sent a report file Loopable cannot read ({type(exc).__name__})") from None
772
+ return [_flat(row) for row in (data if isinstance(data, list) else [data]) if isinstance(row, dict)]
773
+
774
+
775
+ def _read_file(url):
776
+ """GET one report file with no Ads header. Only an https S3 URL a retrieve returned is admitted."""
777
+ parts = urllib.parse.urlsplit(str(url or ""))
778
+ host = parts.hostname or ""
779
+ if parts.scheme != "https" or not host.endswith(REPORT_FILE_HOST_SUFFIX) or url not in _ISSUED_FILES:
780
+ raise AdsWriteRefused(f"{parts.scheme or 'no scheme'} {_safe_path(host)} is not a report file Amazon returned")
781
+ _ISSUED_FILES.discard(url)
782
+ headers = {}
783
+ if fixture_mode():
784
+ return _part_rows(FIXTURE.download(url, headers))
785
+ try:
786
+ with urllib.request.urlopen(urllib.request.Request(url, method="GET", headers=headers),
787
+ timeout=_TIMEOUT) as r:
788
+ raw = r.read()
789
+ except Exception as exc: # noqa: BLE001 the URL carries a signature: never echo it
790
+ raise AdsError(f"Amazon Ads report file could not be read ({type(exc).__name__})") from None
791
+ return _part_rows(raw)
792
+
793
+
794
+ def _split(group, rows):
795
+ """Flat rows -> {member table: rows} by `adProduct.value`, and a note for rows no member takes."""
796
+ from harness import amazon_ads_schema as schema
797
+ out, tables, stray = {table: [] for table in group.members}, group.tables, 0
798
+ for row in rows:
799
+ table = tables.get(str(row.get(schema.AD_PRODUCT_FIELD) or "").upper())
800
+ if table is None or not row.get(schema.vendor_name("date")):
801
+ stray += 1
802
+ else:
803
+ out[table].append(row)
804
+ note = (f"Amazon Ads sent {stray} rows with no date or with another ad product. Loopable left them out.",)
805
+ return out, note if stray else ()
806
+
807
+
808
+ def run_group_report(tenant_key, group, start, end, *, rt=None, profile_id=None, connection_id=None,
809
+ sleep=time.sleep, now=time.monotonic, max_wait=None, report_id=None):
810
+ """Create one Reporting v1 group report, wait for it, and read every file. -> GroupReport.
811
+
812
+ `max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without a create.
813
  """
814
  from harness import amazon_ads_schema as schema
815
+ group = schema.GROUPS[group] if isinstance(group, str) else group
816
  if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
817
+ raise ValueError(f"a report window is at most {MAX_DAYS_PER_REQUEST} days")
818
+ fields = _report_credential(tenant_key, rt, connection_id)
819
+ opts = {"rt": rt, "profile_id": str(profile_id or fields["profile_id"]), "connection_id": connection_id}
820
+ refused = _REFUSED.setdefault((str(fields["client_id"]), group.name), {})
821
  limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
822
+ with _report_lock(fields["client_id"]):
 
823
  if report_id is None:
824
+ account = _advertiser_account(tenant_key, **opts)
825
+ report_id = _create(tenant_key, group, start, end, account, refused, opts)["reportId"]
826
+ urls = _await(tenant_key, str(report_id), opts, sleep=sleep, now=now, limit=limit)
827
+ rows = [row for url in urls for row in _read_file(url)]
828
+ split, stray = _split(group, rows)
829
+ notes = tuple(f"Amazon Ads refused the field {field} on this report, so Loopable reads it without that "
830
+ f"field. Amazon said: {text}" for field, text in refused.items())
831
+ return GroupReport(str(report_id), split, notes + stray)
832
+
833
+
834
+ def run_report(tenant_key, table, start, end, **options):
835
+ """One table's rows from its group report. -> (report_id, rows). The store reads `run_group_report`."""
836
+ from harness import amazon_ads_schema as schema
837
+ group = schema.group_of(table)
838
+ if group is None:
839
+ raise AdsReportUnavailable(f"Amazon Reporting v1 offers no report for {schema.short_name(table)}")
840
+ result = run_group_report(tenant_key, group, start, end, **options)
841
+ return result.report_id, result.rows[schema.short_name(table)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
842
 
843
 
844
  # --------------------------------------------------------------------------- recording fixture
845
 
846
+ FIXTURE_TOKEN = "fixture-access-token"
847
+ FIXTURE_CREDENTIAL = {"client_id": "amzn1.application-oa2-client.fixture", "client_secret": "fixture",
848
+ "refresh_token": "fixture", "profile_id": "1000000001", "region": "na"}
849
+ FIXTURE_FILE_HOST = "https://fixture-ads-reports.s3.amazonaws.com"
850
+ FIXTURE_CREATED = "2026-10-20T09:00:00Z"
851
+ _FIXTURE_PREFIX = {"SPONSORED_PRODUCTS": ("71", "SP"), "SPONSORED_BRANDS": ("72", "SB"),
852
+ "SPONSORED_DISPLAY": ("73", "SD")}
853
+ #: Per campaign and day: the metric totals every finer row splits evenly.
854
+ _FIXTURE_TOTALS = {"metric.impressions": 400, "metric.clicks": 12, "metric.totalCost": 5.0,
855
+ "metric.sales": 38.0, "metric.purchases": 4, "metric.unitsSold": 4}
856
+
857
+
858
+ def _multi_status(report=None, errors=None):
859
+ """A ReportMultiStatusResponse with one success or one error entry at index 0."""
860
+ return {"success": [{"index": 0, "report": report}] if report else [],
861
+ "error": [{"index": 0, "errors": errors}] if errors else []}
862
+
863
+
864
  class _Fixture:
865
+ """The recording fixture `AMAZON_ADS_FIXTURE=1` routes to, in the saved contracts' v1 shapes.
 
866
 
867
+ It records method, path, operation, headers and body of every call, report file downloads
868
+ included. Toggles: `drop_days`, `unsupported` (group names that end FAILED), `refuse_fields`
869
+ (a create naming one answers 400), `nested` (files carry nested JSON lines), `responder`.
870
+ """
871
+
872
+ FILES = 2
873
 
874
  def __init__(self):
875
  self.reset()
876
 
877
  def reset(self):
878
+ self.calls, self.reports, self.subscriptions, self.credentials = [], {}, {}, {}
 
879
  self.in_flight = self.max_in_flight = 0
880
+ self.drop_days, self.unsupported, self.refuse_fields = set(), set(), set()
881
+ self.nested = False
882
  self.responder = None
883
 
884
+ def credential(self, tenant_key):
885
+ """The fixture credential for one tenant. `credentials[tenant]` overrides its fields."""
886
+ return {**FIXTURE_CREDENTIAL, **self.credentials.get(tenant_key, {})}
887
+
888
+ def answer(self, prepared, tenant_key, profile_id, headers=None):
889
  self.calls.append({"method": prepared.method, "path": prepared.path,
890
  "operation": prepared.operation, "body": prepared.body,
891
+ "query": dict(prepared.query), "tenant": tenant_key, "headers": dict(headers or {}),
892
  "named": dict(prepared.named) if prepared.named else None})
893
  if self.responder is not None:
894
  return self.responder(prepared)
 
904
  if prepared.named is not None:
905
  items = next(iter(prepared.body.values()))
906
  return {"success": [{"index": i} for i in range(len(items))], "error": []}
907
+ if prepared.operation == "queryAdvertiserAccounts":
908
+ return self._accounts(prepared.body, profile_id)
909
  if prepared.operation == "createReport":
910
+ return self._create(prepared.body)
911
+ return self._retrieve(prepared.body["reportIds"][0])
912
+
913
+ def _accounts(self, body, profile_id):
914
+ """Two pages: an unrelated account, then the account that lists the caller's profile."""
915
+ if not body.get("nextToken"):
916
+ return {"advertiserAccounts": [{"advertiserAccountId": "amzn1.ads-account.g.fxother",
917
+ "alternateIds": [{"profileId": "999", "countryCode": "CA"}]}],
918
+ "nextToken": "fixture-page-2"}
919
+ return {"advertiserAccounts": [{"advertiserAccountId": f"amzn1.ads-account.g.fx{profile_id}",
920
+ "alternateIds": [{"profileId": str(profile_id), "countryCode": "US",
921
+ "entityId": "ENTITYFX"}]}]}
922
+
923
+ def _create(self, body):
924
+ if not (body.get("accessRequestedAccounts") or [{}])[0].get("advertiserAccountId"):
925
+ return _multi_status(errors=[{"code": "400001", "message": "An advertiser account is required.",
926
+ "fieldLocation": "accessRequestedAccounts"}])
927
+ refused = [f for f in body["reports"][0]["query"]["fields"] if f in self.refuse_fields]
928
+ if refused:
929
+ raise _rejection("Amazon Ads answered HTTP 400 on createReport",
930
+ [{"code": "400004", "message": f"Field {refused[0]} is invalid for this request.",
931
+ "fieldLocation": ""}], may_be_unoffered=False)
932
+ report_id = f"fx{len(self.reports) + 1:05d}"
933
+ self.in_flight += 1
934
+ self.max_in_flight = max(self.max_in_flight, self.in_flight)
935
+ self.reports[report_id] = {"body": body, "polls": 0, "files": None, "done": False}
936
+ return _multi_status(self._record(report_id, "PENDING"))
937
+
938
+ def _record(self, report_id, status, **extra):
939
+ """A contract `Report`: every required key, the request's format, periods and query echoed."""
940
+ request = self.reports[report_id]["body"]["reports"][0]
941
+ return {"reportId": report_id, "status": status, "format": request["format"],
942
+ "periods": request["periods"], "query": request["query"],
943
+ "creationDateTime": FIXTURE_CREATED, "lastUpdatedDateTime": FIXTURE_CREATED, **extra}
944
+
945
+ def _retrieve(self, report_id):
946
+ from harness import amazon_ads_schema as schema
947
  report = self.reports[report_id]
948
+ report["polls"] += 1
949
+ query = report["body"]["reports"][0]["query"]
950
+ group = schema.group_for(query["fields"], query["filter"]["on"]["values"])
951
+ if group is not None and group.name in self.unsupported:
952
+ self._finish(report)
953
+ return _multi_status(self._record(report_id, "FAILED", failureCode="REPORT_NOT_SUPPORTED",
954
+ failureReason="This report is not supported for this advertiser."))
955
+ if report["polls"] < 2:
956
+ return _multi_status(self._record(report_id, "PROCESSING"))
957
+ if report["files"] is None:
958
+ rows = fixture_rows(report["body"], drop_days=self.drop_days)
959
+ report["files"] = {f"{FIXTURE_FILE_HOST}/{report_id}/part-{i}.json.gz?X-Amz-Signature=fx{report_id}{i}":
960
+ _fixture_file(rows[i::self.FILES], self.nested) for i in range(self.FILES)}
961
+ return _multi_status(self._record(report_id, "COMPLETED", completedDateTime=FIXTURE_CREATED, completedReportParts=[
962
+ {"url": url, "sizeInBytes": len(raw), "urlExpirationDateTime": "2026-12-31T00:00:00Z"}
963
+ for url, raw in report["files"].items()]))
964
+
965
+ def _finish(self, report):
966
+ if not report["done"]:
967
+ report["done"] = True
968
  self.in_flight -= 1
969
+
970
+ def download(self, url, headers):
971
+ """Serve one report file and record the GET with the headers the client sent."""
972
+ self.calls.append({"method": "GET", "path": url, "operation": "downloadReportPart", "body": None,
973
+ "query": {}, "tenant": None, "headers": dict(headers), "named": None})
974
+ report = self.reports[urllib.parse.urlsplit(url).path.split("/")[1]]
975
+ raw = report["files"][url]
976
+ report.setdefault("read", set()).add(url)
977
+ if report["read"] == set(report["files"]):
978
+ self._finish(report)
979
+ return raw
980
+
981
+
982
+ def _fixture_file(rows, nested):
983
+ """GZIP JSON: an array of flat rows, or nested objects one per line."""
984
+ def nest(row):
985
+ out = {}
986
+ for field, value in row.items():
987
+ head, _, tail = field.partition(".")
988
+ out.setdefault(head, {})[tail] = value
989
+ return out
990
+ text = "\n".join(json.dumps(nest(r)) for r in rows) if nested else json.dumps(rows)
991
+ return gzip.compress(text.encode("utf-8"))
992
+
993
+
994
+ def _fixture_query(body):
995
+ report = body["reports"][0]
996
+ period = report["periods"][0]["datePeriod"]
997
+ return (report["query"]["fields"], report["query"]["filter"]["on"]["values"],
998
+ date.fromisoformat(period["startDate"]), date.fromisoformat(period["endDate"]))
999
+
1000
+
1001
+ def _fixture_days(start, end, drop_days):
1002
+ return [start + timedelta(days=i) for i in range((end - start).days + 1)
1003
+ if (start + timedelta(days=i)).isoformat() not in drop_days]
1004
+
1005
+
1006
+ def fixture_totals(body, *, drop_days=()):
1007
+ """The fixture's own totals for one create body. -> {(ad product, date, campaign id): {field: total}}."""
1008
+ _fields, products, start, end = _fixture_query(body)
1009
+ out = {}
1010
+ for day in _fixture_days(start, end, drop_days):
1011
+ for index, product in enumerate(products):
1012
+ for c in (1, 2):
1013
+ n = day.toordinal() % 7 + c + index
1014
+ out[(product, day.isoformat(), f"{_FIXTURE_PREFIX[product][0]}10{c}")] = {
1015
+ field: total * n for field, total in _FIXTURE_TOTALS.items()}
1016
+ return out
1017
 
1018
 
1019
  def fixture_rows(body, *, drop_days=()):
1020
+ """Deterministic v1 rows for one create body: one row per distinct value of every requested
1021
+ dimension and day, only the requested fields and ad products. Two ad groups split a campaign."""
1022
+ fields, _products, start, end = _fixture_query(body)
1023
+ groups = (1, 2) if "adGroup.id" in fields else (0,)
1024
+ out = []
1025
+ for (product, day, campaign), totals in fixture_totals(body, drop_days=drop_days).items():
1026
+ prefix, short = _FIXTURE_PREFIX[product]
1027
+ c = campaign[-1]
1028
+ for g in groups:
1029
+ row = {"date.value": day, "adProduct.value": product, "campaign.id": campaign,
1030
+ "campaign.name": f"{short} campaign {c}", "campaign.deliveryStatus": "DELIVERING",
1031
+ "campaign.budgetAmount": 25 * int(c), "budgetCurrency.value": "USD",
1032
+ "adGroup.id": f"{prefix}20{c}{g}", "adGroup.name": f"Ad group {c}.{g}",
1033
+ "adGroupTarget.id": f"{prefix}30{c}{g}", "target.value": f"term {c}{g}",
1034
+ "target.matchType": "EXACT", "adGroupTarget.bid": str(0.5 + g / 10),
1035
+ "ad.id": f"{prefix}40{c}{g}", "advertisedProduct.id": f"B0FX{prefix}{c}{g}00",
1036
+ "advertisedProductMarketplace.value": "ATVPDKIKX0DER", "advertisedProductSku.value": f"SKU-{c}{g}",
1037
+ "searchTerm.value": f"search {c}{g}",
1038
+ **{f: (t // len(groups) if isinstance(t, int) else t / len(groups)) for f, t in totals.items()}}
1039
+ out.append({f: row[f] for f in fields})
 
 
 
 
 
 
 
1040
  return out
1041
 
1042
 
1043
  FIXTURE = _Fixture()
1044
 
1045
 
1046
+ def _reset_caches():
1047
+ _ACCOUNTS.clear()
1048
+ _REFUSED.clear()
1049
+ _ISSUED_FILES.clear()
1050
+
1051
+
1052
  @contextmanager
1053
  def recording_fixture():
1054
+ """Route every call to a fresh recording fixture and fresh report caches for the block. -> the fixture."""
1055
  previous = os.environ.get(FIXTURE_ENV)
1056
  os.environ[FIXTURE_ENV] = "1"
1057
  FIXTURE.reset()
1058
+ _reset_caches()
1059
  try:
1060
  yield FIXTURE
1061
  finally:
1062
  FIXTURE.reset()
1063
+ _reset_caches()
1064
  if previous is None:
1065
  os.environ.pop(FIXTURE_ENV, None)
1066
  else:
1067
  os.environ[FIXTURE_ENV] = previous
1068
 
1069
 
1070
+ __all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "V1_CLIENT_HEADERS",
1071
+ "NAMED_WRITES", "STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation",
1072
  "FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE",
1073
  "SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY",
1074
+ "MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsRejected", "AdsReportUnavailable",
1075
+ "AdsReportPending", "GroupReport", "fixture_mode", "build_request", "report_operation",
1076
+ "send_named_write", "request", "credential", "run_group_report", "run_report", "FIXTURE",
1077
+ "recording_fixture", "fixture_rows", "fixture_totals"]
platform/harness/amazon_ads_schema.py CHANGED
@@ -1,16 +1,17 @@
1
  """amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6).
2
 
3
- One entry per ad type and report: the report type, the columns we store, the Ads column each one
4
- is read from, the row identity and the backfill window. The table keys, labels and backfill days
5
- come from `core.connector_spec`, so the two files cannot disagree on a name.
6
 
7
- Unknown: the exact Reporting v1 column spellings. The vendor names below are the Ads reporting
8
- vocabulary the primer records (F11, F12). The first credentialed report settles them (PRD P17),
9
- and a rename is one literal in `_VENDOR`.
10
  """
11
  import hashlib
12
  import sys
13
  from pathlib import Path
 
14
 
15
  _HERE = Path(__file__).resolve().parent
16
  if str(_HERE.parent) not in sys.path:
@@ -22,8 +23,6 @@ AD_TYPES = _cs.AMAZON_ADS_AD_TYPES
22
  REPORTS = _cs.AMAZON_ADS_REPORTS
23
  DATABASE_PREFIX = "ut_amzn_"
24
  MIRROR_PREFIX = "amzn_ads_"
25
- #: Amazon accepts at most 31 days per report request.
26
- MAX_DAYS_PER_REQUEST = 31
27
 
28
  #: The C6 columns every Ads table carries, in this order.
29
  C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date",
@@ -54,22 +53,76 @@ _EXTRA = {
54
  }
55
  _TAIL = ("profile_id", "currency")
56
 
57
- #: Our column -> the Ads column, shared by all three ad types unless `_VENDOR_BY_TYPE` says not.
58
  _VENDOR = {
59
- "campaign_id": "campaignId", "campaign_name": "campaignName",
60
- "campaign_status": "campaignStatus", "campaign_budget_amount": "campaignBudgetAmount",
61
- "ad_group_id": "adGroupId", "ad_group_name": "adGroupName", "target_id": "keywordId",
62
- "targeting": "targeting", "match_type": "matchType", "bid": "keywordBid",
63
- "ad_id": "adId", "advertised_sku": "advertisedSku", "asin": "advertisedAsin",
64
- "search_term": "searchTerm", "date": "date", "impressions": "impressions",
65
- "clicks": "clicks", "cost": "cost", "sales": "sales", "orders": "purchases",
66
- "units": "unitsSold", "currency": "campaignBudgetCurrencyCode",
67
- }
68
- _VENDOR_BY_TYPE = {
69
- "sp": {"sales": "sales14d", "orders": "purchases14d", "units": "unitsSoldClicks14d"},
70
- "sd": {"target_id": "targetingId", "targeting": "targetingExpression",
71
- "advertised_sku": "promotedSku", "asin": "promotedAsin"},
72
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
 
74
 
75
  def _short(ad_type, report):
@@ -89,25 +142,24 @@ def coltype(name):
89
  return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR"
90
 
91
 
92
- def vendor_name(ad_type, column):
93
- """The Ads column one of our columns is read from, or None when the vendor has none."""
94
- if column == "profile_id":
95
- return None
96
- return _VENDOR_BY_TYPE.get(ad_type, {}).get(column) or _VENDOR.get(column)
97
 
98
 
99
  TABLE_SPECS = {}
100
  for _spec in _cs.AMAZON_ADS_TABLES:
101
  _ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1)
 
102
  TABLE_SPECS[_short(_ad_type, _report)] = {
103
  "key": _spec.key, "source": _spec.source, "label": _spec.label,
104
- "ad_type": _ad_type, "report": _report, "report_type": _spec.native,
105
  "columns": _columns(_report), "row_identity": tuple(_spec.identity),
106
  "backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key],
107
  "offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED,
108
  "not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key),
109
  }
110
- del _spec, _ad_type, _report
111
 
112
 
113
  def short_name(table):
@@ -127,11 +179,16 @@ def columns_for(table):
127
  return TABLE_SPECS[short_name(table)]["columns"]
128
 
129
 
130
- def vendor_columns(table):
131
- """The Ads columns one report asks for, in our column order, each once."""
132
- spec = TABLE_SPECS[short_name(table)]
133
- names = (vendor_name(spec["ad_type"], c) for c in spec["columns"])
134
- return tuple(dict.fromkeys(n for n in names if n))
 
 
 
 
 
135
 
136
 
137
  def _cell(name, value):
@@ -149,11 +206,11 @@ def _cell(name, value):
149
 
150
 
151
  def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None):
152
- """One Ads report row -> one stored row, with its stable `_row_key` and report origin."""
153
  spec = TABLE_SPECS[short_name(table)]
154
  out = {}
155
  for column in spec["columns"]:
156
- source = vendor_name(spec["ad_type"], column)
157
  value = vendor_row.get(source) if source else None
158
  if column == "profile_id":
159
  value = profile_id
@@ -213,7 +270,8 @@ def stream_coltype(name):
213
  return "DOUBLE"
214
  return "VARCHAR"
215
 
216
- __all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "MAX_DAYS_PER_REQUEST",
 
217
  "ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for",
218
- "vendor_columns", "row_from_vendor", "row_key", "STREAM_WINDOWS", "STREAM_METRICS", "STREAM_GRAIN",
219
  "STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"]
 
1
  """amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6).
2
 
3
+ One entry per ad type and report: the columns we store, the Reporting v1 field each one is read
4
+ from, the row identity and the backfill window. The table keys, labels and backfill days come from
5
+ `core.connector_spec`, so the two files cannot disagree on a name.
6
 
7
+ Field ids come from Amazon's v1 field library, saved at
8
+ `AutoWiki/research/amazon-ads-v1-schema/reporting_v1_fields.json`. v1 has no report types: one
9
+ report per `REPORT_GROUPS` row covers several ad types, and its rows split by `adProduct.value`.
10
  """
11
  import hashlib
12
  import sys
13
  from pathlib import Path
14
+ from typing import NamedTuple
15
 
16
  _HERE = Path(__file__).resolve().parent
17
  if str(_HERE.parent) not in sys.path:
 
23
  REPORTS = _cs.AMAZON_ADS_REPORTS
24
  DATABASE_PREFIX = "ut_amzn_"
25
  MIRROR_PREFIX = "amzn_ads_"
 
 
26
 
27
  #: The C6 columns every Ads table carries, in this order.
28
  C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date",
 
53
  }
54
  _TAIL = ("profile_id", "currency")
55
 
56
+ #: Our column -> its Reporting v1 field id, one map for every ad type. The store stamps profile_id.
57
  _VENDOR = {
58
+ "date": "date.value", "campaign_id": "campaign.id", "campaign_name": "campaign.name",
59
+ "campaign_status": "campaign.deliveryStatus", "campaign_budget_amount": "campaign.budgetAmount",
60
+ "ad_group_id": "adGroup.id", "ad_group_name": "adGroup.name", "target_id": "adGroupTarget.id",
61
+ "targeting": "target.value", "match_type": "target.matchType", "bid": "adGroupTarget.bid",
62
+ "ad_id": "ad.id", "advertised_sku": "advertisedProductSku.value", "asin": "advertisedProduct.id",
63
+ "search_term": "searchTerm.value", "impressions": "metric.impressions", "clicks": "metric.clicks",
64
+ "cost": "metric.totalCost", "sales": "metric.sales", "orders": "metric.purchases",
65
+ "units": "metric.unitsSold", "currency": "budgetCurrency.value",
 
 
 
 
 
66
  }
67
+ #: The v1 field that names each row's ad type, and its value per ad type.
68
+ AD_PRODUCT_FIELD = "adProduct.value"
69
+ AD_PRODUCTS = {"sp": "SPONSORED_PRODUCTS", "sb": "SPONSORED_BRANDS", "sd": "SPONSORED_DISPLAY"}
70
+ #: The name the v1 field library lists each ad type under.
71
+ LIBRARY_AD_PRODUCTS = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
72
+
73
+ METRICS = ("metric.impressions", "metric.clicks", "metric.totalCost", "metric.sales",
74
+ "metric.purchases", "metric.unitsSold")
75
+ _HEAD = ("date.value", AD_PRODUCT_FIELD, "campaign.id", "campaign.name")
76
+ _AD_GROUP = ("adGroup.id", "adGroup.name")
77
+ _TAIL_FIELDS = ("budgetCurrency.value",) + METRICS
78
+ #: ⛔ Not `targeting.value` or `targetingMatchType.value`: the library lists them for no ad product.
79
+ #: The live probe of 2026-11-18 (`probe-2026-11-18.json`) proved these names, absent from the library.
80
+ _TARGET_TEXT = ("target.value", "target.matchType")
81
+ #: Fields a live Reporting v1 create accepted that the saved field library does not list.
82
+ PROBE_PROVEN = frozenset(_TARGET_TEXT)
83
+
84
+
85
+ class ReportGroup(NamedTuple):
86
+ """One Reporting v1 report and the member tables its rows split into by `adProduct.value`.
87
+
88
+ `optional` fields list no ad product in Amazon's library, or only a live probe proved them:
89
+ a create that refuses one drops it.
90
+ """
91
+ name: str
92
+ members: tuple
93
+ core: tuple
94
+ optional: tuple = ()
95
+
96
+ @property
97
+ def tables(self):
98
+ """`adProduct.value` -> the member table its rows land in."""
99
+ return {AD_PRODUCTS[table.split("_", 1)[0]]: table for table in self.members}
100
+
101
+ def fields(self, drop=()):
102
+ """The fields one create asks for, without the optional fields Amazon refused."""
103
+ return tuple(f for f in self.core + self.optional if f not in drop)
104
+
105
+
106
+ #: ⛔ Every requested dimension is a group-by and sets the row grain. A group asks only for its
107
+ #: tables' identity dimensions, attributes of those, metrics, the ad product and required partners.
108
+ #: Every core field lists every ad product of its group in the library: SB lacks the SKU field and
109
+ #: the marketplace field `advertisedProduct.id` requires, so SB advertised products is its own report.
110
+ REPORT_GROUPS = (
111
+ ReportGroup("campaigns", ("sp_campaigns", "sb_campaigns", "sd_campaigns"),
112
+ _HEAD + ("campaign.deliveryStatus", "campaign.budgetAmount") + _TAIL_FIELDS),
113
+ ReportGroup("targeting", ("sp_targeting", "sb_targeting", "sd_targeting"),
114
+ _HEAD + _AD_GROUP + ("adGroupTarget.id",) + _TAIL_FIELDS,
115
+ _TARGET_TEXT + ("adGroupTarget.bid",)),
116
+ ReportGroup("advertised_products", ("sp_advertised_products", "sd_advertised_products"),
117
+ _HEAD + _AD_GROUP + ("ad.id", "advertisedProduct.id", "advertisedProductMarketplace.value",
118
+ "advertisedProductSku.value") + _TAIL_FIELDS),
119
+ ReportGroup("advertised_products_sb", ("sb_advertised_products",),
120
+ _HEAD + _AD_GROUP + ("ad.id",) + _TAIL_FIELDS),
121
+ ReportGroup("search_terms", ("sp_search_terms", "sb_search_terms"),
122
+ _HEAD + _AD_GROUP + ("adGroupTarget.id", "searchTerm.value") + _TAIL_FIELDS, _TARGET_TEXT),
123
+ )
124
+ GROUPS = {group.name: group for group in REPORT_GROUPS}
125
+ _GROUP_OF = {table: group for group in REPORT_GROUPS for table in group.members}
126
 
127
 
128
  def _short(ad_type, report):
 
142
  return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR"
143
 
144
 
145
+ def vendor_name(column):
146
+ """The v1 field one of our columns is read from, or None when the vendor has none."""
147
+ return _VENDOR.get(column)
 
 
148
 
149
 
150
  TABLE_SPECS = {}
151
  for _spec in _cs.AMAZON_ADS_TABLES:
152
  _ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1)
153
+ _group = _GROUP_OF.get(_short(_ad_type, _report))
154
  TABLE_SPECS[_short(_ad_type, _report)] = {
155
  "key": _spec.key, "source": _spec.source, "label": _spec.label,
156
+ "ad_type": _ad_type, "report": _report, "group": _group.name if _group else None,
157
  "columns": _columns(_report), "row_identity": tuple(_spec.identity),
158
  "backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key],
159
  "offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED,
160
  "not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key),
161
  }
162
+ del _spec, _ad_type, _report, _group
163
 
164
 
165
  def short_name(table):
 
179
  return TABLE_SPECS[short_name(table)]["columns"]
180
 
181
 
182
+ def group_of(table):
183
+ """The report group that fills one Ads table, or None for a table v1 does not offer."""
184
+ return _GROUP_OF.get(short_name(table))
185
+
186
+
187
+ def group_for(fields, ad_products):
188
+ """The report group a create request asks for, by its fields and ad product filter, or None."""
189
+ fields, ad_products = set(fields), set(ad_products)
190
+ return next((g for g in REPORT_GROUPS if set(g.tables) == ad_products
191
+ and set(g.core) <= fields <= set(g.core + g.optional)), None)
192
 
193
 
194
  def _cell(name, value):
 
206
 
207
 
208
  def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None):
209
+ """One flat v1 report row (dotted field ids) -> one stored row, with its `_row_key` and report origin."""
210
  spec = TABLE_SPECS[short_name(table)]
211
  out = {}
212
  for column in spec["columns"]:
213
+ source = vendor_name(column)
214
  value = vendor_row.get(source) if source else None
215
  if column == "profile_id":
216
  value = profile_id
 
270
  return "DOUBLE"
271
  return "VARCHAR"
272
 
273
+ __all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "AD_PRODUCT_FIELD", "AD_PRODUCTS",
274
+ "LIBRARY_AD_PRODUCTS", "METRICS", "ReportGroup", "REPORT_GROUPS", "GROUPS",
275
  "ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for",
276
+ "group_of", "group_for", "row_from_vendor", "row_key", "STREAM_WINDOWS", "STREAM_METRICS", "STREAM_GRAIN",
277
  "STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"]
platform/harness/amazon_ads_store.py CHANGED
@@ -10,8 +10,8 @@ back the longest window Amazon documents per table (`connector_spec.AMAZON_ADS_B
10
  A later sync re-reads the table's daily or intraday cadence window and upserts by row identity.
11
  No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here.
12
 
13
- ⛔ STANDING RULE 1. Every report is read whole and paged. The limits that cannot be removed (one
14
- report at a time, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
15
  `amzn_limit_ads_<table>` with the reason a person reads, and stated on the table description.
16
 
17
  Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_<table>` is the newest landed day,
@@ -178,7 +178,7 @@ def _day_marker(con, key):
178
 
179
 
180
  def windows(con, table, connect_day, today, connection_id=None):
181
- """The report windows one sync asks for, oldest first, each at most 31 days."""
182
  table = schema.short_name(table)
183
  spec = schema.TABLE_SPECS[table]
184
  cadence = amazon_client.CADENCE["ads_" + table]
@@ -237,13 +237,13 @@ def _upsert(con, table, rows):
237
  cols = _columns_for(table)
238
  by_key = {r["_row_key"]: r for r in rows}
239
  keys = list(by_key)
 
240
  for i in range(0, len(keys), INSERT_BATCH_ROWS):
241
  part = keys[i:i + INSERT_BATCH_ROWS]
242
  con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part)
243
- con.executemany(
244
- f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) '
245
- f'VALUES ({", ".join("?" for _ in cols)})',
246
- [[by_key[k].get(c) for c in cols] for k in part])
247
  return len(by_key)
248
 
249
 
@@ -389,16 +389,158 @@ def _land(con, table, rows, report_id, start, end, profile, connection_id, label
389
 
390
 
391
  def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
392
- """Require named transport support instead of sending a sibling's credential."""
393
  options = {"rt": rt, "profile_id": profile_id, **extra}
394
  if connection_id:
395
  options["connection_id"] = connection_id
396
- try:
397
- return client.run_report(key, table, start, end, **options)
398
- except TypeError as exc:
399
- if connection_id and "unexpected keyword argument 'connection_id'" in str(exc):
400
- raise client.AdsError("The Amazon Ads client needs named connection support before this connection can sync.") from None
401
- raise
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
402
 
403
 
404
  def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
@@ -409,6 +551,7 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
409
  through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
410
  `max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
411
  pending ledger, and the next pass reads it before it creates any other report.
 
412
  """
413
  key = amazon_store._tenant_key(tenant_key)
414
  report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
@@ -425,10 +568,9 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
425
  own = con is None
426
  if own:
427
  con, _path = amazon_store._connect_tenant_mirror(key, rt)
428
- bounded = max_seconds is not None
429
  started = _clock()
430
- deadline = started + float(max_seconds) if bounded else None
431
- waits = report["waits"]
432
  try:
433
  datastore._ensure_sync_state(con)
434
  now = _utc_now(now)
@@ -437,164 +579,24 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
437
  def stamp():
438
  return now + timedelta(seconds=max(0.0, _clock() - started))
439
 
440
- def wait(label, spec, reason):
441
- waits.append(f"{label}: {spec['label']}: {reason}")
442
-
443
- synced = []
444
- checkpointed = []
445
- for cid, label, profile, legacy in connections:
446
  connect_day = _connect_day(con, today, cid, legacy=legacy)
447
  report.setdefault("connectDay", connect_day.isoformat())
448
  entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
449
  report["connections"].append(entry)
450
- used = _budget(con, today, cid, legacy=legacy)
451
- # Resume every report this connection left preparing before it creates another one.
452
- resumed, blocked = {}, False
453
- for table in TABLES:
454
- pending = _read_pending(con, table, cid)
455
- if pending is None:
456
- continue
457
- spec = schema.TABLE_SPECS[table]
458
- _ensure_table(con, table)
459
- if table_name(table) not in checkpointed:
460
- checkpointed.append(table_name(table))
461
- age = (stamp() - pending["created"]).total_seconds()
462
- expired = (f"Amazon Ads report {pending['report_id']} did not finish in 3 hours")
463
- if age >= client.REPORT_MAX_WAIT_SECONDS:
464
- _clear_pending(con, table, cid)
465
- report["problems"].append(f"{label}: {spec['label']}: {expired}")
466
- resumed[table] = ("failed", expired, 0, None)
467
- continue
468
- limit = client.REPORT_MAX_WAIT_SECONDS - age
469
- if bounded:
470
- limit = min(limit, max(0.0, deadline - _clock()))
471
- try:
472
- report_id, rows = _run_report(key, table, pending["start"], pending["end"], rt, profile, cid,
473
- max_wait=limit, report_id=pending["report_id"], now=_clock)
474
- except client.AdsReportUnavailable as exc:
475
- _clear_pending(con, table, cid)
476
- resumed[table] = ("not_offered", _not_offered_reason(spec, exc), 0, None)
477
- continue
478
- except client.AdsReportPending:
479
- if (stamp() - pending["created"]).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS:
480
- _clear_pending(con, table, cid)
481
- report["problems"].append(f"{label}: {spec['label']}: {expired}")
482
- resumed[table] = ("failed", expired, 0, None)
483
- else:
484
- blocked = True
485
- wait(label, spec, PREPARING_REASON)
486
- resumed[table] = ("deferred", PREPARING_REASON, 0, None)
487
- continue
488
- except client.AdsError as exc:
489
- _clear_pending(con, table, cid)
490
- reason = str(exc)[:200]
491
- report["problems"].append(f"{label}: {spec['label']}: {reason}")
492
- resumed[table] = ("failed", reason, 0, None)
493
- continue
494
- _clear_pending(con, table, cid)
495
- pulled = _land(con, table, rows, report_id, pending["start"], pending["end"], profile, cid, label)
496
- resumed[table] = ("done", "", pulled, (pending["start"], pending["end"]))
497
- requested = [schema.short_name(t) for t in (tables or TABLES)]
498
  for table in list(dict.fromkeys(requested + list(resumed))):
499
- spec = schema.TABLE_SPECS[table]
500
- _ensure_table(con, table)
501
- origin_changed = _refresh_origin(con, table, cid, label, legacy)
502
- pulled, state, mark, reason = 0, "done", _marker_table(table), ""
503
- prior = resumed.get(table)
504
- landed = None
505
- if prior is not None:
506
- state, reason, pulled, landed = prior
507
- if state != "done" or table not in requested:
508
- pass
509
- elif not spec["offered"]:
510
- state, reason = "not_offered", spec["not_offered_reason"]
511
- elif not _cadence_due(con, table, now, cid):
512
- state = "skipped"
513
- elif blocked:
514
- state, reason = "deferred", PREPARING_REASON
515
- wait(label, spec, reason)
516
- else:
517
- for start, end in windows(con, table, connect_day, today, cid):
518
- if landed and landed[0] <= start and end <= landed[1]:
519
- continue # The resumed report just read these days.
520
- if used >= client.REPORTS_PER_DAY:
521
- state = "deferred"
522
- reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
523
- f"reports are used, so days from {start.isoformat()} load on the next sync.")
524
- report["problems"].append(f"{label}: {spec['label']}: waiting for tomorrow's report limit")
525
- waits.append(report["problems"][-1])
526
- break
527
- if bounded and _clock() >= deadline:
528
- state, reason = "deferred", TIME_LIMIT_REASON
529
- wait(label, spec, reason)
530
- break
531
- used += 1
532
- _spend_budget(con, today, used, cid)
533
- created = stamp()
534
- try:
535
- if bounded:
536
- report_id, rows = _run_report(key, table, start, end, rt, profile, cid,
537
- max_wait=max(0.0, deadline - _clock()), now=_clock)
538
- else:
539
- report_id, rows = _run_report(key, table, start, end, rt, profile, cid)
540
- except client.AdsReportUnavailable as exc:
541
- state = "not_offered"
542
- reason = _not_offered_reason(spec, exc)
543
- break
544
- except client.AdsError as exc:
545
- if (bounded and isinstance(exc, client.AdsReportPending) and exc.report_id
546
- and (stamp() - created).total_seconds() < client.REPORT_MAX_WAIT_SECONDS):
547
- _write_pending(con, table, cid, exc.report_id, start, end, created)
548
- if table_name(table) not in checkpointed:
549
- checkpointed.append(table_name(table))
550
- blocked = True
551
- state, reason = "deferred", PREPARING_REASON
552
- wait(label, spec, reason)
553
- break
554
- state = "failed"
555
- reason = str(exc)[:200]
556
- report["problems"].append(f"{label}: {spec['label']}: {reason}")
557
- break
558
- pulled += _land(con, table, rows, report_id, start, end, profile, cid, label)
559
- if table_name(table) not in checkpointed:
560
- checkpointed.append(table_name(table))
561
- if state == "done":
562
- _record_cadence(con, table, now, cid)
563
- if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
564
- report["partial"] = True
565
- if reason:
566
- _record_limit(con, table, reason, cid, state=state)
567
- shown = f"{label}: {reason}" if cid else reason
568
- report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown)))
569
- elif state == "done":
570
- _clear_limit(con, table, cid)
571
- total = _count(con, table)
572
- entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
573
- if state != "skipped":
574
- amazon_store._record_state(con, _state_key(amazon_store.entity_key(mark), cid),
575
- "done" if state == "done" else "partial", _count(con, table, cid), state)
576
- previous = report["tables"].get(table, {})
577
- severity = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
578
- aggregate = max((state, previous.get("state", state)), key=severity.get)
579
- report["tables"][table] = {"state": aggregate, "pulled": previous.get("pulled", 0) + pulled,
580
- "in_mirror": total}
581
- if (pulled or origin_changed) and table_name(table) not in synced:
582
  synced.append(table_name(table))
583
- log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}")
584
- for table, state in report["tables"].items():
585
- if state["state"] == "skipped":
586
- continue
587
- amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
588
- "done" if state["state"] == "done" else "partial",
589
- state["in_mirror"], state["state"])
590
- if table in report["limits"]:
591
- _record_limit(con, table, report["limits"][table], state=state["state"])
592
- else:
593
- _clear_limit(con, table)
594
- report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
595
- for cid, *_ in connections for table in TABLES)
596
- if report["partial"]:
597
- report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
598
  if own and rt is not None and (synced or checkpointed):
599
  _persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
600
  finally:
@@ -604,10 +606,75 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
604
  return report
605
 
606
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
607
  def _not_offered_reason(spec, exc):
608
  return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
609
- f"({str(exc)[:160]}). The table keeps the days it has. To ask for it, "
610
- f"contact Amazon Ads support with the report type {spec['report_type']}.")
611
 
612
 
613
  def _persist(tenant_key, rt, con, synced, report, log):
 
10
  A later sync re-reads the table's daily or intraday cadence window and upserts by row identity.
11
  No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here.
12
 
13
+ ⛔ STANDING RULE 1. Every report is read whole, every file of it. The limits that cannot be removed
14
+ (one report at a time per client ID, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
15
  `amzn_limit_ads_<table>` with the reason a person reads, and stated on the table description.
16
 
17
  Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_<table>` is the newest landed day,
 
178
 
179
 
180
  def windows(con, table, connect_day, today, connection_id=None):
181
+ """The report windows one sync asks for, oldest first, each at most 120 days (`client.MAX_DAYS_PER_REQUEST`)."""
182
  table = schema.short_name(table)
183
  spec = schema.TABLE_SPECS[table]
184
  cadence = amazon_client.CADENCE["ads_" + table]
 
237
  cols = _columns_for(table)
238
  by_key = {r["_row_key"]: r for r in rows}
239
  keys = list(by_key)
240
+ one = f'({", ".join("?" for _ in cols)})'
241
  for i in range(0, len(keys), INSERT_BATCH_ROWS):
242
  part = keys[i:i + INSERT_BATCH_ROWS]
243
  con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part)
244
+ # One multi-row INSERT per batch: DuckDB's executemany is about 5 times slower at 24 months a pass.
245
+ con.execute(f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) '
246
+ f'VALUES {", ".join(one for _ in part)}', [by_key[k].get(c) for k in part for c in cols])
 
247
  return len(by_key)
248
 
249
 
 
389
 
390
 
391
  def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
392
+ """Run the group report that fills `table` under this connection's own credential. -> GroupReport."""
393
  options = {"rt": rt, "profile_id": profile_id, **extra}
394
  if connection_id:
395
  options["connection_id"] = connection_id
396
+ return client.run_group_report(key, schema.group_of(table), start, end, **options)
397
+
398
+
399
+ class _Pass:
400
+ """One connection's sync pass: its budget, clock, and what its group reports already covered.
401
+
402
+ `eligible` holds the tables a group report may land into this pass: requested, offered, due and
403
+ not yet finished. `covered` holds each table's spans landed this pass, `pulled` the rows a
404
+ sibling's report landed, `outcome` a group's failure, `notes` Amazon's text per table.
405
+ """
406
+
407
+ def __init__(self, con, key, rt, connection, report, *, today, used, deadline, stamp, checkpointed):
408
+ self.con, self.key, self.rt, self.report = con, key, rt, report
409
+ self.cid, self.label, self.profile = connection[:3]
410
+ self.today, self.used, self.deadline, self.stamp = today, used, deadline, stamp
411
+ self.checkpointed = checkpointed
412
+ self.eligible, self.covered, self.pulled, self.outcome, self.notes = set(), {}, {}, {}, {}
413
+ self.blocked = False
414
+
415
+ def wait(self, spec, reason):
416
+ self.report["waits"].append(f"{self.label}: {spec['label']}: {reason}")
417
+
418
+ def checkpoint(self, table):
419
+ if table_name(table) not in self.checkpointed:
420
+ self.checkpointed.append(table_name(table))
421
+
422
+ def covers(self, table, start, end):
423
+ """True when reports this pass already read every day from `start` to `end` for `table`."""
424
+ spans = self.covered.get(table, ())
425
+ return all(any(lo <= start + timedelta(days=i) <= hi for lo, hi in spans)
426
+ for i in range((end - start).days + 1))
427
+
428
+ def bounds(self):
429
+ """The poll limit for one new report: none, or the time left in this bounded pass."""
430
+ return {} if self.deadline is None else {"max_wait": max(0.0, self.deadline - _clock()), "now": _clock}
431
+
432
+ def fail(self, table, exc):
433
+ """Record a group report's failure for every member. -> (state, reason) for `table`."""
434
+ self.outcome[schema.group_of(table).name] = exc
435
+ return self.failure(table, exc)
436
+
437
+ def failure(self, table, exc):
438
+ spec = schema.TABLE_SPECS[table]
439
+ if isinstance(exc, client.AdsReportUnavailable):
440
+ return "not_offered", _not_offered_reason(spec, exc)
441
+ reason = str(exc)[:client.ERROR_TEXT_LIMIT]
442
+ self.report["problems"].append(f"{self.label}: {spec['label']}: {reason}")
443
+ return "failed", reason
444
+
445
+ def land(self, table, result, start, end):
446
+ """Land one group report into `table` and each eligible sibling, covering the span. -> rows into `table`."""
447
+ own = 0
448
+ for member in schema.group_of(table).members:
449
+ if member != table and member not in self.eligible:
450
+ continue
451
+ _ensure_table(self.con, member)
452
+ rows = _land(self.con, member, result.rows.get(member, []), result.report_id, start, end,
453
+ self.profile, self.cid, self.label)
454
+ self.covered.setdefault(member, []).append((start, end))
455
+ self.notes[member] = list(dict.fromkeys(self.notes.get(member, []) + list(result.notes)))
456
+ self.checkpoint(member)
457
+ if member == table:
458
+ own = rows
459
+ else:
460
+ self.pulled[member] = self.pulled.get(member, 0) + rows
461
+ for note in result.notes:
462
+ if f"{self.label}: {note}" not in self.report["problems"]:
463
+ self.report["problems"].append(f"{self.label}: {note}")
464
+ return own
465
+
466
+
467
+ def _resume(p):
468
+ """Read every report this connection left preparing before it creates another. -> {table: outcome}."""
469
+ resumed = {}
470
+ for table in TABLES:
471
+ pending = _read_pending(p.con, table, p.cid)
472
+ if pending is None:
473
+ continue
474
+ spec = schema.TABLE_SPECS[table]
475
+ _ensure_table(p.con, table)
476
+ p.checkpoint(table)
477
+ expired = f"Amazon Ads report {pending['report_id']} did not finish in 3 hours"
478
+ age = (p.stamp() - pending["created"]).total_seconds()
479
+ if age >= client.REPORT_MAX_WAIT_SECONDS:
480
+ _clear_pending(p.con, table, p.cid)
481
+ resumed[table] = (*p.failure(table, client.AdsError(expired)), 0)
482
+ continue
483
+ limit = client.REPORT_MAX_WAIT_SECONDS - age
484
+ if p.deadline is not None:
485
+ limit = min(limit, max(0.0, p.deadline - _clock()))
486
+ try:
487
+ result = _run_report(p.key, table, pending["start"], pending["end"], p.rt, p.profile, p.cid,
488
+ max_wait=limit, report_id=pending["report_id"], now=_clock)
489
+ except client.AdsReportPending:
490
+ if (p.stamp() - pending["created"]).total_seconds() < client.REPORT_MAX_WAIT_SECONDS:
491
+ p.blocked = True
492
+ p.wait(spec, PREPARING_REASON)
493
+ resumed[table] = ("deferred", PREPARING_REASON, 0)
494
+ continue
495
+ _clear_pending(p.con, table, p.cid)
496
+ resumed[table] = (*p.failure(table, client.AdsError(expired)), 0)
497
+ continue
498
+ except client.AdsError as exc:
499
+ _clear_pending(p.con, table, p.cid)
500
+ resumed[table] = (*p.fail(table, exc), 0)
501
+ continue
502
+ _clear_pending(p.con, table, p.cid)
503
+ resumed[table] = ("done", "", p.land(table, result, pending["start"], pending["end"]))
504
+ return resumed
505
+
506
+
507
+ def _run_windows(p, table, connect_day):
508
+ """Run the group report for each window of `table` no report covered yet. -> (state, reason, rows)."""
509
+ spec = schema.TABLE_SPECS[table]
510
+ failed = p.outcome.get(schema.group_of(table).name)
511
+ if failed is not None:
512
+ return (*p.failure(table, failed), 0)
513
+ pulled = 0
514
+ for start, end in windows(p.con, table, connect_day, p.today, p.cid):
515
+ if p.covers(table, start, end):
516
+ continue
517
+ if p.used >= client.REPORTS_PER_DAY:
518
+ reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
519
+ f"reports are used, so days from {start.isoformat()} load on the next sync.")
520
+ p.report["problems"].append(f"{p.label}: {spec['label']}: waiting for tomorrow's report limit")
521
+ p.report["waits"].append(p.report["problems"][-1])
522
+ return "deferred", reason, pulled
523
+ if p.deadline is not None and _clock() >= p.deadline:
524
+ p.wait(spec, TIME_LIMIT_REASON)
525
+ return "deferred", TIME_LIMIT_REASON, pulled
526
+ p.used += 1
527
+ _spend_budget(p.con, p.today, p.used, p.cid)
528
+ created = p.stamp()
529
+ try:
530
+ result = _run_report(p.key, table, start, end, p.rt, p.profile, p.cid, **p.bounds())
531
+ except client.AdsReportPending as exc:
532
+ if (p.deadline is None or not exc.report_id
533
+ or (p.stamp() - created).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS):
534
+ return (*p.failure(table, exc), pulled)
535
+ _write_pending(p.con, table, p.cid, exc.report_id, start, end, created)
536
+ p.checkpoint(table)
537
+ p.blocked = True
538
+ p.wait(spec, PREPARING_REASON)
539
+ return "deferred", PREPARING_REASON, pulled
540
+ except client.AdsError as exc:
541
+ return (*p.fail(table, exc), pulled)
542
+ pulled += p.land(table, result, start, end)
543
+ return "done", "", pulled
544
 
545
 
546
  def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
 
551
  through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
552
  `max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
553
  pending ledger, and the next pass reads it before it creates any other report.
554
+ One group report fills every requested member table, so a sibling's covered days spend no report.
555
  """
556
  key = amazon_store._tenant_key(tenant_key)
557
  report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
 
568
  own = con is None
569
  if own:
570
  con, _path = amazon_store._connect_tenant_mirror(key, rt)
 
571
  started = _clock()
572
+ deadline = None if max_seconds is None else started + float(max_seconds)
573
+ synced, checkpointed = [], []
574
  try:
575
  datastore._ensure_sync_state(con)
576
  now = _utc_now(now)
 
579
  def stamp():
580
  return now + timedelta(seconds=max(0.0, _clock() - started))
581
 
582
+ requested = [schema.short_name(t) for t in (tables or TABLES)]
583
+ for connection in connections:
584
+ cid, label, _profile, legacy = connection
 
 
 
585
  connect_day = _connect_day(con, today, cid, legacy=legacy)
586
  report.setdefault("connectDay", connect_day.isoformat())
587
  entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
588
  report["connections"].append(entry)
589
+ p = _Pass(con, key, rt, connection, report, today=today, used=_budget(con, today, cid, legacy=legacy),
590
+ deadline=deadline, stamp=stamp, checkpointed=checkpointed)
591
+ p.eligible = {t for t in requested if schema.TABLE_SPECS[t]["offered"] and _cadence_due(con, t, now, cid)}
592
+ resumed = _resume(p)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
593
  for table in list(dict.fromkeys(requested + list(resumed))):
594
+ landed = _sync_table(p, table, entry, connect_day, now, legacy, table in requested,
595
+ resumed.get(table), log)
596
+ p.eligible.discard(table)
597
+ if landed and table_name(table) not in synced:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
598
  synced.append(table_name(table))
599
+ _record_tables(con, report, connections)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
600
  if own and rt is not None and (synced or checkpointed):
601
  _persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
602
  finally:
 
606
  return report
607
 
608
 
609
+ _SEVERITY = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
610
+
611
+
612
+ def _sync_table(p, table, entry, connect_day, now, legacy, requested, prior, log):
613
+ """One table's turn in one connection's pass: run or skip it, record its state. -> True if it changed."""
614
+ con, cid, label, report = p.con, p.cid, p.label, p.report
615
+ spec = schema.TABLE_SPECS[table]
616
+ _ensure_table(con, table)
617
+ origin_changed = _refresh_origin(con, table, cid, label, legacy)
618
+ state, reason, pulled = prior or ("done", "", 0)
619
+ pulled += p.pulled.pop(table, 0)
620
+ if state != "done" or not requested:
621
+ pass
622
+ elif not spec["offered"]:
623
+ state, reason = "not_offered", spec["not_offered_reason"]
624
+ elif not _cadence_due(con, table, now, cid):
625
+ state = "skipped"
626
+ elif p.blocked:
627
+ state, reason = "deferred", PREPARING_REASON
628
+ p.wait(spec, reason)
629
+ else:
630
+ state, reason, rows = _run_windows(p, table, connect_day)
631
+ pulled += rows
632
+ if state == "done":
633
+ _record_cadence(con, table, now, cid)
634
+ if state == "done" and not reason and p.notes.get(table):
635
+ reason = " ".join(p.notes[table])
636
+ if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
637
+ report["partial"] = True
638
+ if reason:
639
+ _record_limit(con, table, reason, cid, state=state)
640
+ shown = f"{label}: {reason}" if cid else reason
641
+ report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown)))
642
+ elif state == "done":
643
+ _clear_limit(con, table, cid)
644
+ total = _count(con, table)
645
+ entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
646
+ if state != "skipped":
647
+ amazon_store._record_state(con, _state_key(amazon_store.entity_key(_marker_table(table)), cid),
648
+ "done" if state == "done" else "partial", _count(con, table, cid), state)
649
+ previous = report["tables"].get(table, {})
650
+ report["tables"][table] = {"state": max((state, previous.get("state", state)), key=_SEVERITY.get),
651
+ "pulled": previous.get("pulled", 0) + pulled, "in_mirror": total}
652
+ log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}")
653
+ return bool(pulled or origin_changed)
654
+
655
+
656
+ def _record_tables(con, report, connections):
657
+ """Write each table's aggregate state and limit across connections, and the pass's pending count."""
658
+ for table, state in report["tables"].items():
659
+ if state["state"] == "skipped":
660
+ continue
661
+ amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
662
+ "done" if state["state"] == "done" else "partial",
663
+ state["in_mirror"], state["state"])
664
+ if table in report["limits"]:
665
+ _record_limit(con, table, report["limits"][table], state=state["state"])
666
+ else:
667
+ _clear_limit(con, table)
668
+ report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
669
+ for cid, *_ in connections for table in TABLES)
670
+ if report["partial"]:
671
+ report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
672
+
673
+
674
  def _not_offered_reason(spec, exc):
675
  return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
676
+ f"({str(exc)[:client.ERROR_TEXT_LIMIT]}). The table keeps the days it has. To ask for it, "
677
+ f"contact Amazon Ads support and quote Amazon's message.")
678
 
679
 
680
  def _persist(tenant_key, rt, con, synced, report, log):
platform/harness/amazon_client.py CHANGED
@@ -1127,7 +1127,7 @@ for _tables, _passes, _daily, _intraday in (
1127
  from harness import amazon_ads_schema as _ads_schema
1128
 
1129
  CADENCE.update({
1130
- "ads_" + table: {"perDay": 4, "lookbackDays": 21, "intradayLookbackDays": 5, "period": "day"}
1131
  for table in _ads_schema.TABLE_SPECS
1132
  })
1133
  del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
 
1127
  from harness import amazon_ads_schema as _ads_schema
1128
 
1129
  CADENCE.update({
1130
+ "ads_" + table: {"perDay": 4, "lookbackDays": 60, "intradayLookbackDays": 5, "period": "day"}
1131
  for table in _ads_schema.TABLE_SPECS
1132
  })
1133
  del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 7b3fa972918f4775d1a39386caef6dd64b506f516c56895555ce39a2bdf027c1
 
1
+ c62b077954cb16ddf668e8e5aee89d6e4cb852718d0b57b94aa3412a210747e6