Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/routes_shares.py +10 -1
- platform/core/connector_spec.py +25 -23
- platform/core/users.py +10 -0
- platform/harness/amazon_ads_client.py +470 -158
- platform/harness/amazon_ads_schema.py +96 -38
- platform/harness/amazon_ads_store.py +238 -171
- platform/harness/amazon_client.py +1 -1
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "0085702694",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
0085702694
|
api/routes_shares.py
CHANGED
|
@@ -1300,7 +1300,7 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
|
| 1300 |
# would be a second answer to a question this file already answers once, and would leave
|
| 1301 |
# the new grantee with an editor and no people to put in it. A `view` grantee is still
|
| 1302 |
# `may_admin=False` here and still gets `[]`.
|
| 1303 |
-
"people":
|
| 1304 |
}
|
| 1305 |
|
| 1306 |
|
|
@@ -1309,6 +1309,15 @@ def _people(tenant):
|
|
| 1309 |
return sorted(users.assignable_identities(tenant), key=lambda p: p["name"].lower())
|
| 1310 |
|
| 1311 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1312 |
def _report_entries_or_400(entries):
|
| 1313 |
"""A report grants download to named people alone."""
|
| 1314 |
for entry in entries:
|
|
|
|
| 1300 |
# would be a second answer to a question this file already answers once, and would leave
|
| 1301 |
# the new grantee with an editor and no people to put in it. A `view` grantee is still
|
| 1302 |
# `may_admin=False` here and still gets `[]`.
|
| 1303 |
+
"people": _picker_people(session.tenant) if may_admin else [],
|
| 1304 |
}
|
| 1305 |
|
| 1306 |
|
|
|
|
| 1309 |
return sorted(users.assignable_identities(tenant), key=lambda p: p["name"].lower())
|
| 1310 |
|
| 1311 |
|
| 1312 |
+
def _picker_people(tenant):
|
| 1313 |
+
"""The share picker's people: `_people` without the QA logins (D-11165).
|
| 1314 |
+
|
| 1315 |
+
The write door keeps `_people`, so a stored grant to a QA login still saves when the dialog
|
| 1316 |
+
sends it back.
|
| 1317 |
+
"""
|
| 1318 |
+
return [p for p in _people(tenant) if not users.is_qa_login(p["username"])]
|
| 1319 |
+
|
| 1320 |
+
|
| 1321 |
def _report_entries_or_400(entries):
|
| 1322 |
"""A report grants download to named people alone."""
|
| 1323 |
for entry in entries:
|
platform/core/connector_spec.py
CHANGED
|
@@ -3785,15 +3785,15 @@ AMAZON_ADS_REPORTS = ("campaigns", "targeting", "advertised_products", "search_t
|
|
| 3785 |
_ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
|
| 3786 |
_ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting",
|
| 3787 |
"advertised_products": "advertised products", "search_terms": "search terms"}
|
| 3788 |
-
#: `native` is the
|
| 3789 |
-
#: report
|
| 3790 |
_ADS_NATIVE = {
|
| 3791 |
-
"sp": {"campaigns": "
|
| 3792 |
-
"advertised_products": "
|
| 3793 |
-
"sb": {"campaigns": "
|
| 3794 |
-
"advertised_products": "
|
| 3795 |
-
"sd": {"campaigns": "
|
| 3796 |
-
"advertised_products": "
|
| 3797 |
}
|
| 3798 |
_ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting",
|
| 3799 |
"advertised_products": "asin", "search_terms": "search_term"}
|
|
@@ -3822,21 +3822,22 @@ AMAZON_ADS_TABLES = tuple(
|
|
| 3822 |
#: which verify_connectors holds equal to the `"amazon"` registration.
|
| 3823 |
AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES}
|
| 3824 |
|
| 3825 |
-
#: Days the first sync reaches back
|
| 3826 |
-
#:
|
| 3827 |
-
|
| 3828 |
-
AMAZON_ADS_BACKFILL_DAYS = {
|
| 3829 |
-
spec.key: (60 if spec.key.endswith("_search_terms")
|
| 3830 |
-
else {"sp": 95, "sb": 60, "sd": 65}[spec.key[len(_ADS):len(_ADS) + 2]])
|
| 3831 |
-
for spec in AMAZON_ADS_TABLES}
|
| 3832 |
|
| 3833 |
#: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen.
|
| 3834 |
AMAZON_ADS_THROUGHPUT = (
|
| 3835 |
-
"Amazon
|
| 3836 |
-
"
|
| 3837 |
-
"
|
| 3838 |
-
"
|
| 3839 |
-
"to raise the report limit
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3840 |
|
| 3841 |
#: Standing rule 1: the report v1 does not offer, with cause and fix. On screen.
|
| 3842 |
AMAZON_ADS_NOT_OFFERED = {
|
|
@@ -3859,9 +3860,10 @@ def amazon_ads_description(key):
|
|
| 3859 |
if unoffered:
|
| 3860 |
return f"{unoffered} {AMAZON_ADS_THROUGHPUT}"
|
| 3861 |
days = AMAZON_ADS_BACKFILL_DAYS[spec.key]
|
| 3862 |
-
|
| 3863 |
-
|
| 3864 |
-
f"{
|
|
|
|
| 3865 |
|
| 3866 |
|
| 3867 |
def table_group(key):
|
|
|
|
| 3785 |
_ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
|
| 3786 |
_ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting",
|
| 3787 |
"advertised_products": "advertised products", "search_terms": "search terms"}
|
| 3788 |
+
#: `native` is the Reporting v1 report group that fills the table (`amazon_ads_schema.REPORT_GROUPS`).
|
| 3789 |
+
#: v1 has no report types. SD search terms has no group, because v1 does not offer it.
|
| 3790 |
_ADS_NATIVE = {
|
| 3791 |
+
"sp": {"campaigns": "campaigns", "targeting": "targeting",
|
| 3792 |
+
"advertised_products": "advertised_products", "search_terms": "search_terms"},
|
| 3793 |
+
"sb": {"campaigns": "campaigns", "targeting": "targeting",
|
| 3794 |
+
"advertised_products": "advertised_products_sb", "search_terms": "search_terms"},
|
| 3795 |
+
"sd": {"campaigns": "campaigns", "targeting": "targeting",
|
| 3796 |
+
"advertised_products": "advertised_products", "search_terms": "not_offered"},
|
| 3797 |
}
|
| 3798 |
_ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting",
|
| 3799 |
"advertised_products": "asin", "search_terms": "search_term"}
|
|
|
|
| 3822 |
#: which verify_connectors holds equal to the `"amazon"` registration.
|
| 3823 |
AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES}
|
| 3824 |
|
| 3825 |
+
#: Days the first sync reaches back: Reporting v1 keeps 24 months of daily data, and standing
|
| 3826 |
+
#: rule 1 takes all of it.
|
| 3827 |
+
AMAZON_ADS_BACKFILL_DAYS = {spec.key: 730 for spec in AMAZON_ADS_TABLES}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3828 |
|
| 3829 |
#: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen.
|
| 3830 |
AMAZON_ADS_THROUGHPUT = (
|
| 3831 |
+
"Amazon Ads runs one report at a time for each Loopable app login (its client ID) and about "
|
| 3832 |
+
"100 reports a day, shared by every advertiser on that login. Sponsored Products, Brands and "
|
| 3833 |
+
"Display need 5 reports for each refresh, and one report can take up to 3 hours, so one day of "
|
| 3834 |
+
"refresh may not finish in a day. Loopable runs the reports one at a time and keeps every day "
|
| 3835 |
+
"it already has. If the refresh falls behind, ask Amazon Ads support to raise the report limit "
|
| 3836 |
+
"for this client ID.")
|
| 3837 |
+
|
| 3838 |
+
#: On screen: the attribution window Amazon applies to Sponsored Products seller metrics.
|
| 3839 |
+
AMAZON_ADS_SP_ATTRIBUTION = ("Sponsored Products sales, orders and units count purchases within 7 "
|
| 3840 |
+
"days of a click, Amazon's default for sellers.")
|
| 3841 |
|
| 3842 |
#: Standing rule 1: the report v1 does not offer, with cause and fix. On screen.
|
| 3843 |
AMAZON_ADS_NOT_OFFERED = {
|
|
|
|
| 3860 |
if unoffered:
|
| 3861 |
return f"{unoffered} {AMAZON_ADS_THROUGHPUT}"
|
| 3862 |
days = AMAZON_ADS_BACKFILL_DAYS[spec.key]
|
| 3863 |
+
attribution = AMAZON_ADS_SP_ATTRIBUTION + " " if spec.key.startswith(_ADS + "sp_") else ""
|
| 3864 |
+
return (f"{spec.label} by day, from Amazon Ads Reporting v1. {attribution}History is kept from "
|
| 3865 |
+
f"the day you connect, and the first sync reaches back {days} days, the longest Amazon "
|
| 3866 |
+
f"keeps. {AMAZON_ADS_THROUGHPUT}")
|
| 3867 |
|
| 3868 |
|
| 3869 |
def table_group(key):
|
platform/core/users.py
CHANGED
|
@@ -1203,6 +1203,16 @@ def is_test_account(record):
|
|
| 1203 |
return isinstance(record, dict) and isinstance(record.get('test_account'), dict)
|
| 1204 |
|
| 1205 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1206 |
#: The keys of one database entry that `set_database_access` writes.
|
| 1207 |
DATABASE_ENTRY_KEYS = ('access', 'filter', 'hiddenFields')
|
| 1208 |
|
|
|
|
| 1203 |
return isinstance(record, dict) and isinstance(record.get('test_account'), dict)
|
| 1204 |
|
| 1205 |
|
| 1206 |
+
#: The QA sign-in logins: slot 1, slots 2 to 9 and the old sandbox runner. They carry no
|
| 1207 |
+
#: `test_account` marker, so a picker that must leave them out asks this (D-11165).
|
| 1208 |
+
_QA_LOGIN = re.compile(r"qa-(?:bot|member)(?:-[2-9])?|qa-runner")
|
| 1209 |
+
|
| 1210 |
+
|
| 1211 |
+
def is_qa_login(username):
|
| 1212 |
+
"""True for a QA sign-in account, which a share picker never offers as a person."""
|
| 1213 |
+
return bool(_QA_LOGIN.fullmatch(str(username or "").strip().lower()))
|
| 1214 |
+
|
| 1215 |
+
|
| 1216 |
#: The keys of one database entry that `set_database_access` writes.
|
| 1217 |
DATABASE_ENTRY_KEYS = ('access', 'filter', 'hiddenFields')
|
| 1218 |
|
platform/harness/amazon_ads_client.py
CHANGED
|
@@ -4,7 +4,7 @@
|
|
| 4 |
read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the
|
| 5 |
refusal lives here, in the request builder, before any credential read, token exchange or socket:
|
| 6 |
|
| 7 |
-
reports create,
|
| 8 |
streams create and list subscriptions for the dedicated V1 destination
|
| 9 |
named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in
|
| 10 |
`bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant
|
|
@@ -17,8 +17,11 @@ harness module never imports `modules` at load). A missing approval source refus
|
|
| 17 |
made until Royal's Direct Advertiser credential arrives (R7, R17).
|
| 18 |
|
| 19 |
NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is
|
| 20 |
-
stored with the credential
|
|
|
|
|
|
|
| 21 |
"""
|
|
|
|
| 22 |
import importlib
|
| 23 |
import json
|
| 24 |
import logging
|
|
@@ -31,6 +34,7 @@ import urllib.parse
|
|
| 31 |
import urllib.request
|
| 32 |
from contextlib import contextmanager
|
| 33 |
from datetime import date, datetime, timedelta, timezone
|
|
|
|
| 34 |
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
| 35 |
|
| 36 |
KEY = "amazon_ads"
|
|
@@ -46,22 +50,28 @@ DEFAULT_REGION = "na"
|
|
| 46 |
LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
|
| 47 |
CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id")
|
| 48 |
|
| 49 |
-
#: Reporting v1 limits (
|
| 50 |
-
#: up to 3 hours per report, at most
|
| 51 |
REPORT_CONCURRENCY = 1
|
| 52 |
REPORTS_PER_DAY = 100
|
| 53 |
REPORT_MAX_WAIT_SECONDS = 3 * 3600
|
| 54 |
-
MAX_DAYS_PER_REQUEST =
|
| 55 |
-
REPORT_POLL_SECONDS =
|
|
|
|
|
|
|
| 56 |
_TIMEOUT = 60
|
| 57 |
|
| 58 |
-
#: The only non-write calls: Reporting v1
|
| 59 |
-
#:
|
| 60 |
REPORT_OPERATIONS = {
|
| 61 |
-
"createReport": ("POST", "
|
| 62 |
-
"
|
| 63 |
-
"
|
| 64 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
|
| 66 |
# Subscription management never admits an advertising write.
|
| 67 |
STREAM_OPERATIONS = {
|
|
@@ -208,7 +218,15 @@ class AdsWriteRefused(AdsError):
|
|
| 208 |
f"the named writes: bid, budget, negative keyword and scheduled change.")
|
| 209 |
|
| 210 |
|
| 211 |
-
class
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
"""Amazon does not offer this report to this advertiser."""
|
| 213 |
|
| 214 |
|
|
@@ -225,25 +243,10 @@ def fixture_mode():
|
|
| 225 |
return os.environ.get(FIXTURE_ENV) == "1"
|
| 226 |
|
| 227 |
|
| 228 |
-
def _template(path):
|
| 229 |
-
return tuple(path.strip("/").split("/"))
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
def _matches(template, path):
|
| 233 |
-
parts = str(path or "").strip("/").split("/")
|
| 234 |
-
if len(parts) != len(template):
|
| 235 |
-
return False
|
| 236 |
-
return all((bool(re.fullmatch(r"[A-Za-z0-9_-]{1,128}", part)))
|
| 237 |
-
if seg.startswith("{") and seg.endswith("}") else part == seg
|
| 238 |
-
for part, seg in zip(parts, template))
|
| 239 |
-
|
| 240 |
-
|
| 241 |
def report_operation(method, path):
|
| 242 |
-
"""The Reporting v1 operation name for one call, or None."""
|
| 243 |
-
for name, (verb,
|
| 244 |
-
|
| 245 |
-
return name
|
| 246 |
-
return None
|
| 247 |
|
| 248 |
|
| 249 |
class _Prepared:
|
|
@@ -503,12 +506,17 @@ def _send(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
|
|
| 503 |
if not _admitted(prepared):
|
| 504 |
raise AdsWriteRefused("the request did not come from the request builder")
|
| 505 |
if fixture_mode():
|
| 506 |
-
|
|
|
|
| 507 |
return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id)
|
| 508 |
|
| 509 |
|
| 510 |
# --------------------------------------------------------------------------- live transport
|
| 511 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 512 |
def credential(tenant_key, rt=None, connection_id=None):
|
| 513 |
"""The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source."""
|
| 514 |
try:
|
|
@@ -538,11 +546,50 @@ def _token(fields):
|
|
| 538 |
raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None
|
| 539 |
|
| 540 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 541 |
def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None):
|
| 542 |
fields = credential(tenant_key, rt, connection_id=connection_id)
|
| 543 |
if fields is None:
|
| 544 |
-
raise AdsError(
|
| 545 |
-
"Connect Amazon Ads under Amazon Ads Setup.")
|
| 546 |
if prepared.operation in STREAM_OPERATIONS and (
|
| 547 |
str(profile_id or fields["profile_id"]) != str(fields["profile_id"])
|
| 548 |
or str(fields.get("region") or DEFAULT_REGION).lower() != "na"):
|
|
@@ -551,13 +598,7 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
|
|
| 551 |
url = f"{host}/{prepared.path}"
|
| 552 |
if prepared.query:
|
| 553 |
url += "?" + urllib.parse.urlencode(prepared.query)
|
| 554 |
-
headers =
|
| 555 |
-
"Amazon-Advertising-API-ClientId": fields["client_id"],
|
| 556 |
-
"Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]),
|
| 557 |
-
"Content-Type": "application/json", "Accept": "application/json"}
|
| 558 |
-
if prepared.operation in STREAM_OPERATIONS:
|
| 559 |
-
headers["Content-Type"] = STREAM_CONTENT_TYPE
|
| 560 |
-
headers["Accept"] = STREAM_CONTENT_TYPE
|
| 561 |
data = json.dumps(prepared.body).encode() if prepared.body is not None else None
|
| 562 |
req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers)
|
| 563 |
try:
|
|
@@ -566,12 +607,10 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
|
|
| 566 |
except urllib.error.HTTPError as exc:
|
| 567 |
try:
|
| 568 |
detail = json.loads(exc.read().decode("utf-8", "replace") or "{}")
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
raise AdsReportUnavailable(f"Amazon does not offer this report: {message}") from None
|
| 574 |
-
raise AdsError(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}: {message}") from None
|
| 575 |
except Exception as exc: # noqa: BLE001
|
| 576 |
raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None
|
| 577 |
|
|
@@ -580,89 +619,276 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
|
|
| 580 |
|
| 581 |
_REPORT_LOCKS = {}
|
| 582 |
_REPORT_LOCKS_GUARD = threading.Lock()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 583 |
|
| 584 |
|
| 585 |
-
def _report_lock(
|
|
|
|
| 586 |
with _REPORT_LOCKS_GUARD:
|
| 587 |
-
return _REPORT_LOCKS.setdefault(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 588 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 589 |
|
| 590 |
-
def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, connection_id=None, sleep=time.sleep,
|
| 591 |
-
now=time.monotonic, max_wait=None, report_id=None):
|
| 592 |
-
"""Create one Reporting v1 report, wait for it, and read every row page. -> (report_id, rows).
|
| 593 |
|
| 594 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 595 |
"""
|
| 596 |
from harness import amazon_ads_schema as schema
|
| 597 |
-
|
| 598 |
if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
|
| 599 |
-
raise ValueError("a report window is at most
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
|
| 604 |
-
|
| 605 |
-
with _report_lock(tenant_key, profile_id):
|
| 606 |
if report_id is None:
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
raise AdsError(f"Amazon Ads report {report_id} ended {state}: {reason}")
|
| 626 |
-
elapsed = now() - started
|
| 627 |
-
if elapsed > REPORT_MAX_WAIT_SECONDS:
|
| 628 |
-
raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
|
| 629 |
-
if limit is not None and elapsed >= limit:
|
| 630 |
-
raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
|
| 631 |
-
pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
|
| 632 |
-
sleep(pause if limit is None else min(pause, limit - elapsed))
|
| 633 |
-
rows, token = [], None
|
| 634 |
-
rows_path = REPORT_OPERATIONS["getReportRows"][1].replace("{reportId}", report_id)
|
| 635 |
-
while True:
|
| 636 |
-
page = request(tenant_key, "GET", rows_path, query={"nextToken": token} if token else None,
|
| 637 |
-
rt=rt, profile_id=profile_id, connection_id=connection_id)
|
| 638 |
-
rows.extend(page.get("rows") or [])
|
| 639 |
-
token = page.get("nextToken")
|
| 640 |
-
if not token:
|
| 641 |
-
return report_id, rows
|
| 642 |
|
| 643 |
|
| 644 |
# --------------------------------------------------------------------------- recording fixture
|
| 645 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 646 |
class _Fixture:
|
| 647 |
-
"""The recording fixture `AMAZON_ADS_FIXTURE=1` routes to
|
| 648 |
-
with deterministic rows, two report polls, and paged rows."""
|
| 649 |
|
| 650 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 651 |
|
| 652 |
def __init__(self):
|
| 653 |
self.reset()
|
| 654 |
|
| 655 |
def reset(self):
|
| 656 |
-
self.calls, self.reports = [], {}
|
| 657 |
-
self.subscriptions = {}
|
| 658 |
self.in_flight = self.max_in_flight = 0
|
| 659 |
-
self.drop_days, self.unsupported = set(), set()
|
|
|
|
| 660 |
self.responder = None
|
| 661 |
|
| 662 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
| 663 |
self.calls.append({"method": prepared.method, "path": prepared.path,
|
| 664 |
"operation": prepared.operation, "body": prepared.body,
|
| 665 |
-
"query": dict(prepared.query), "tenant": tenant_key,
|
| 666 |
"named": dict(prepared.named) if prepared.named else None})
|
| 667 |
if self.responder is not None:
|
| 668 |
return self.responder(prepared)
|
|
@@ -678,88 +904,174 @@ class _Fixture:
|
|
| 678 |
if prepared.named is not None:
|
| 679 |
items = next(iter(prepared.body.values()))
|
| 680 |
return {"success": [{"index": i} for i in range(len(items))], "error": []}
|
|
|
|
|
|
|
| 681 |
if prepared.operation == "createReport":
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 688 |
report = self.reports[report_id]
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 703 |
self.in_flight -= 1
|
| 704 |
-
|
| 705 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 706 |
|
| 707 |
|
| 708 |
def fixture_rows(body, *, drop_days=()):
|
| 709 |
-
"""Deterministic
|
| 710 |
-
|
| 711 |
-
start, end =
|
| 712 |
-
|
| 713 |
-
out
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
"promotedAsin": f"B00FX0{c}{g}0", "searchTerm": f"search {c}{g}",
|
| 730 |
-
"impressions": 100 * n, "clicks": 3 * n, "cost": round(1.25 * n, 2),
|
| 731 |
-
"sales": round(9.5 * n, 2), "sales14d": round(9.5 * n, 2),
|
| 732 |
-
"purchases": n, "purchases14d": n, "unitsSold": n, "unitsSoldClicks14d": n,
|
| 733 |
-
}
|
| 734 |
-
out.append({k: row.get(k) for k in columns})
|
| 735 |
-
day += timedelta(days=1)
|
| 736 |
return out
|
| 737 |
|
| 738 |
|
| 739 |
FIXTURE = _Fixture()
|
| 740 |
|
| 741 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 742 |
@contextmanager
|
| 743 |
def recording_fixture():
|
| 744 |
-
"""Route every call to a fresh recording fixture for the block. -> the fixture."""
|
| 745 |
previous = os.environ.get(FIXTURE_ENV)
|
| 746 |
os.environ[FIXTURE_ENV] = "1"
|
| 747 |
FIXTURE.reset()
|
|
|
|
| 748 |
try:
|
| 749 |
yield FIXTURE
|
| 750 |
finally:
|
| 751 |
FIXTURE.reset()
|
|
|
|
| 752 |
if previous is None:
|
| 753 |
os.environ.pop(FIXTURE_ENV, None)
|
| 754 |
else:
|
| 755 |
os.environ[FIXTURE_ENV] = previous
|
| 756 |
|
| 757 |
|
| 758 |
-
__all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "
|
| 759 |
-
"STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation",
|
| 760 |
"FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE",
|
| 761 |
"SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY",
|
| 762 |
-
"MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsReportUnavailable",
|
| 763 |
-
"AdsReportPending", "fixture_mode", "build_request", "report_operation",
|
| 764 |
-
"send_named_write", "request", "credential", "run_report", "FIXTURE",
|
| 765 |
-
"recording_fixture", "fixture_rows"]
|
|
|
|
| 4 |
read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the
|
| 5 |
refusal lives here, in the request builder, before any credential read, token exchange or socket:
|
| 6 |
|
| 7 |
+
reports Reporting v1: create, retrieve, the advertiser account query, and the report files
|
| 8 |
streams create and list subscriptions for the dedicated V1 destination
|
| 9 |
named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in
|
| 10 |
`bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant
|
|
|
|
| 17 |
made until Royal's Direct Advertiser credential arrives (R7, R17).
|
| 18 |
|
| 19 |
NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is
|
| 20 |
+
stored with the credential. v1 names the account by `advertiserAccountId`, which the client looks
|
| 21 |
+
up once per connection from `query/advertiserAccounts`. Every path, header and field comes from
|
| 22 |
+
Amazon's saved contracts in `AutoWiki/research/amazon-ads-v1-schema/`.
|
| 23 |
"""
|
| 24 |
+
import gzip
|
| 25 |
import importlib
|
| 26 |
import json
|
| 27 |
import logging
|
|
|
|
| 34 |
import urllib.request
|
| 35 |
from contextlib import contextmanager
|
| 36 |
from datetime import date, datetime, timedelta, timezone
|
| 37 |
+
from typing import NamedTuple
|
| 38 |
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
| 39 |
|
| 40 |
KEY = "amazon_ads"
|
|
|
|
| 50 |
LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
|
| 51 |
CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id")
|
| 52 |
|
| 53 |
+
#: Reporting v1 limits (limits.md, dates.md): one report in flight per client ID, about 100 a day,
|
| 54 |
+
#: up to 3 hours per report, at most 120 days per request.
|
| 55 |
REPORT_CONCURRENCY = 1
|
| 56 |
REPORTS_PER_DAY = 100
|
| 57 |
REPORT_MAX_WAIT_SECONDS = 3 * 3600
|
| 58 |
+
MAX_DAYS_PER_REQUEST = 120
|
| 59 |
+
REPORT_POLL_SECONDS = 60.0
|
| 60 |
+
#: Amazon's error text reaches the sync report verbatim, up to this many characters.
|
| 61 |
+
ERROR_TEXT_LIMIT = 600
|
| 62 |
_TIMEOUT = 60
|
| 63 |
|
| 64 |
+
#: The only non-write calls: the Reporting v1 operations, each a POST to one literal path from the
|
| 65 |
+
#: Reports and AdvertiserAccounts contracts. Every other `adsApi` path refuses, a GET included.
|
| 66 |
REPORT_OPERATIONS = {
|
| 67 |
+
"createReport": ("POST", "adsApi/v1/create/reports"),
|
| 68 |
+
"retrieveReport": ("POST", "adsApi/v1/retrieve/reports"),
|
| 69 |
+
"queryAdvertiserAccounts": ("POST", "adsApi/v1/query/advertiserAccounts"),
|
| 70 |
}
|
| 71 |
+
#: The contract names `Amazon-Ads-ClientId`. The limits page names the older header. v1 sends both.
|
| 72 |
+
V1_CLIENT_HEADERS = ("Amazon-Ads-ClientId", "Amazon-Advertising-API-ClientId")
|
| 73 |
+
#: A report file is a presigned S3 URL: https, on this host suffix, GET with no Ads header.
|
| 74 |
+
REPORT_FILE_HOST_SUFFIX = ".amazonaws.com"
|
| 75 |
|
| 76 |
# Subscription management never admits an advertising write.
|
| 77 |
STREAM_OPERATIONS = {
|
|
|
|
| 218 |
f"the named writes: bid, budget, negative keyword and scheduled change.")
|
| 219 |
|
| 220 |
|
| 221 |
+
class AdsRejected(AdsError):
|
| 222 |
+
"""Amazon answered an error. `errors` keeps each entry's code, message and fieldLocation."""
|
| 223 |
+
|
| 224 |
+
def __init__(self, message, errors=()):
|
| 225 |
+
super().__init__(message)
|
| 226 |
+
self.errors = list(errors)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
class AdsReportUnavailable(AdsRejected):
|
| 230 |
"""Amazon does not offer this report to this advertiser."""
|
| 231 |
|
| 232 |
|
|
|
|
| 243 |
return os.environ.get(FIXTURE_ENV) == "1"
|
| 244 |
|
| 245 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
def report_operation(method, path):
|
| 247 |
+
"""The Reporting v1 operation name for one call, or None. A path matches only its literal."""
|
| 248 |
+
return next((name for name, (verb, endpoint) in REPORT_OPERATIONS.items()
|
| 249 |
+
if str(method).upper() == verb and str(path or "").strip("/") == endpoint), None)
|
|
|
|
|
|
|
| 250 |
|
| 251 |
|
| 252 |
class _Prepared:
|
|
|
|
| 506 |
if not _admitted(prepared):
|
| 507 |
raise AdsWriteRefused("the request did not come from the request builder")
|
| 508 |
if fixture_mode():
|
| 509 |
+
headers = _headers(prepared, FIXTURE.credential(tenant_key), FIXTURE_TOKEN, profile_id)
|
| 510 |
+
return FIXTURE.answer(prepared, tenant_key, profile_id, headers)
|
| 511 |
return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id)
|
| 512 |
|
| 513 |
|
| 514 |
# --------------------------------------------------------------------------- live transport
|
| 515 |
|
| 516 |
+
_NO_CREDENTIAL = ("No Amazon Ads credential is stored for this tenant. "
|
| 517 |
+
"Connect Amazon Ads under Amazon Ads Setup.")
|
| 518 |
+
|
| 519 |
+
|
| 520 |
def credential(tenant_key, rt=None, connection_id=None):
|
| 521 |
"""The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source."""
|
| 522 |
try:
|
|
|
|
| 546 |
raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None
|
| 547 |
|
| 548 |
|
| 549 |
+
def _headers(prepared, fields, token, profile_id=None):
|
| 550 |
+
"""The headers one admitted call sends. A v1 call names its account in the body, not a scope."""
|
| 551 |
+
if prepared.operation in REPORT_OPERATIONS:
|
| 552 |
+
return {"Authorization": f"Bearer {token}", **dict.fromkeys(V1_CLIENT_HEADERS, fields["client_id"]),
|
| 553 |
+
"Content-Type": "application/json", "Accept": "application/json"}
|
| 554 |
+
headers = {"Authorization": f"Bearer {token}",
|
| 555 |
+
"Amazon-Advertising-API-ClientId": fields["client_id"],
|
| 556 |
+
"Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]),
|
| 557 |
+
"Content-Type": "application/json", "Accept": "application/json"}
|
| 558 |
+
if prepared.operation in STREAM_OPERATIONS:
|
| 559 |
+
headers["Content-Type"] = STREAM_CONTENT_TYPE
|
| 560 |
+
headers["Accept"] = STREAM_CONTENT_TYPE
|
| 561 |
+
return headers
|
| 562 |
+
|
| 563 |
+
|
| 564 |
+
def _errors(detail):
|
| 565 |
+
"""Amazon's error entries from an error body, each with its code, message and fieldLocation."""
|
| 566 |
+
if not isinstance(detail, dict):
|
| 567 |
+
return []
|
| 568 |
+
entries = detail["errors"] if isinstance(detail.get("errors"), list) else [detail]
|
| 569 |
+
return [{"code": str(e.get("code") or ""), "message": str(e.get("details") or e.get("message") or ""),
|
| 570 |
+
"fieldLocation": str(e.get("fieldLocation") or "")}
|
| 571 |
+
for e in entries if isinstance(e, dict) and (e.get("code") or e.get("details") or e.get("message"))]
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
def _error_text(errors):
|
| 575 |
+
"""Amazon's errors as one line a person reads, verbatim, cut to `ERROR_TEXT_LIMIT`."""
|
| 576 |
+
return " ".join(" ".join(filter(None, (e["code"] + ":" if e["code"] else "", e["message"],
|
| 577 |
+
f"(field {e['fieldLocation']})" if e["fieldLocation"] else "")))
|
| 578 |
+
for e in errors)[:ERROR_TEXT_LIMIT]
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
def _rejection(prefix, errors, *, may_be_unoffered=True):
|
| 582 |
+
"""The exception for Amazon's errors: not offered when Amazon says so, else rejected."""
|
| 583 |
+
text = _error_text(errors)
|
| 584 |
+
if may_be_unoffered and re.search(r"not (?:supported|offered|available)", text, re.I):
|
| 585 |
+
return AdsReportUnavailable(f"Amazon does not offer this report: {text}", errors)
|
| 586 |
+
return AdsRejected(f"{prefix}: {text}" if text else prefix, errors)
|
| 587 |
+
|
| 588 |
+
|
| 589 |
def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None):
|
| 590 |
fields = credential(tenant_key, rt, connection_id=connection_id)
|
| 591 |
if fields is None:
|
| 592 |
+
raise AdsError(_NO_CREDENTIAL)
|
|
|
|
| 593 |
if prepared.operation in STREAM_OPERATIONS and (
|
| 594 |
str(profile_id or fields["profile_id"]) != str(fields["profile_id"])
|
| 595 |
or str(fields.get("region") or DEFAULT_REGION).lower() != "na"):
|
|
|
|
| 598 |
url = f"{host}/{prepared.path}"
|
| 599 |
if prepared.query:
|
| 600 |
url += "?" + urllib.parse.urlencode(prepared.query)
|
| 601 |
+
headers = _headers(prepared, fields, _token(fields), profile_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 602 |
data = json.dumps(prepared.body).encode() if prepared.body is not None else None
|
| 603 |
req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers)
|
| 604 |
try:
|
|
|
|
| 607 |
except urllib.error.HTTPError as exc:
|
| 608 |
try:
|
| 609 |
detail = json.loads(exc.read().decode("utf-8", "replace") or "{}")
|
| 610 |
+
except Exception: # noqa: BLE001 an unreadable body still raises its status
|
| 611 |
+
detail = {}
|
| 612 |
+
raise _rejection(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}", _errors(detail),
|
| 613 |
+
may_be_unoffered=exc.code == 400) from None
|
|
|
|
|
|
|
| 614 |
except Exception as exc: # noqa: BLE001
|
| 615 |
raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None
|
| 616 |
|
|
|
|
| 619 |
|
| 620 |
_REPORT_LOCKS = {}
|
| 621 |
_REPORT_LOCKS_GUARD = threading.Lock()
|
| 622 |
+
#: Process caches: the advertiser account per connection, the optional fields Amazon refused per
|
| 623 |
+
#: (client id, group) with Amazon's text, and the report file URLs a retrieve returned.
|
| 624 |
+
_ACCOUNTS = {}
|
| 625 |
+
_REFUSED = {}
|
| 626 |
+
_ISSUED_FILES = set()
|
| 627 |
+
|
| 628 |
+
|
| 629 |
+
class GroupReport(NamedTuple):
|
| 630 |
+
"""One finished group report: its id, flat rows per member table, and notes a person reads."""
|
| 631 |
+
report_id: str
|
| 632 |
+
rows: dict
|
| 633 |
+
notes: tuple
|
| 634 |
|
| 635 |
|
| 636 |
+
def _report_lock(client_id):
|
| 637 |
+
"""One lock per client ID: Amazon runs one report at a time for each app login."""
|
| 638 |
with _REPORT_LOCKS_GUARD:
|
| 639 |
+
return _REPORT_LOCKS.setdefault(str(client_id), threading.Lock())
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def _report_credential(tenant_key, rt, connection_id):
|
| 643 |
+
"""The credential a report runs under: its client id keys the lock, its profile names the account."""
|
| 644 |
+
if fixture_mode():
|
| 645 |
+
return FIXTURE.credential(tenant_key)
|
| 646 |
+
fields = credential(tenant_key, rt, connection_id=connection_id)
|
| 647 |
+
if fields is None:
|
| 648 |
+
raise AdsError(_NO_CREDENTIAL)
|
| 649 |
+
return fields
|
| 650 |
+
|
| 651 |
+
|
| 652 |
+
def _lists_profile(account, profile_id):
|
| 653 |
+
return (isinstance(account, dict) and bool(account.get("advertiserAccountId"))
|
| 654 |
+
and any(isinstance(alt, dict) and str(alt.get("profileId")) == str(profile_id)
|
| 655 |
+
for alt in account.get("alternateIds") or ()))
|
| 656 |
+
|
| 657 |
|
| 658 |
+
def _advertiser_account(tenant_key, *, rt, profile_id, connection_id):
|
| 659 |
+
"""The v1 `advertiserAccountId` whose alternate ids carry the stored profile id, cached per connection."""
|
| 660 |
+
key, token = (tenant_key, connection_id or "", str(profile_id)), None
|
| 661 |
+
while key not in _ACCOUNTS:
|
| 662 |
+
page = request(tenant_key, "POST", REPORT_OPERATIONS["queryAdvertiserAccounts"][1],
|
| 663 |
+
body={"maxResults": 100, **({"nextToken": token} if token else {})},
|
| 664 |
+
rt=rt, profile_id=profile_id, connection_id=connection_id)
|
| 665 |
+
match = next((a for a in page.get("advertiserAccounts") or () if _lists_profile(a, profile_id)), None)
|
| 666 |
+
token = page.get("nextToken")
|
| 667 |
+
if match is not None:
|
| 668 |
+
_ACCOUNTS[key] = str(match["advertiserAccountId"])
|
| 669 |
+
elif not token:
|
| 670 |
+
raise AdsError(f"No Amazon Ads advertiser account lists profile {profile_id}. "
|
| 671 |
+
f"Check the profile ID on this Amazon Ads connection.")
|
| 672 |
+
return _ACCOUNTS[key]
|
| 673 |
|
|
|
|
|
|
|
|
|
|
| 674 |
|
| 675 |
+
def _create_body(group, start, end, account, fields):
|
| 676 |
+
"""A CreateReportRequest for one group and window, filtered to the group's ad products."""
|
| 677 |
+
from harness import amazon_ads_schema as schema
|
| 678 |
+
return {"accessRequestedAccounts": [{"advertiserAccountId": account}],
|
| 679 |
+
"reports": [{"format": "GZIP_JSON",
|
| 680 |
+
"periods": [{"datePeriod": {"startDate": start.isoformat(), "endDate": end.isoformat()}}],
|
| 681 |
+
"query": {"fields": list(fields),
|
| 682 |
+
"filter": {"on": {"field": schema.AD_PRODUCT_FIELD, "comparisonOperator": "IN",
|
| 683 |
+
"not": False, "values": list(group.tables)}}}}]}
|
| 684 |
+
|
| 685 |
+
|
| 686 |
+
def _first_report(reply, operation):
|
| 687 |
+
"""The one report a 207 multi-status answer carries, or its errors raised verbatim."""
|
| 688 |
+
errors = [e for entry in reply.get("error") or () if isinstance(entry, dict)
|
| 689 |
+
for e in _errors({"errors": entry.get("errors") or []})]
|
| 690 |
+
if errors or reply.get("error"):
|
| 691 |
+
raise _rejection(f"Amazon Ads refused {operation}", errors)
|
| 692 |
+
for entry in reply.get("success") or ():
|
| 693 |
+
report = entry.get("report") if isinstance(entry, dict) else None
|
| 694 |
+
if isinstance(report, dict) and re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", str(report.get("reportId") or "")):
|
| 695 |
+
return report
|
| 696 |
+
raise AdsError(f"Amazon Ads returned no report on {operation}")
|
| 697 |
+
|
| 698 |
+
|
| 699 |
+
def _named_fields(exc, fields):
|
| 700 |
+
"""The requested fields an error names, by id in its text or by index in its field location."""
|
| 701 |
+
text = " ".join([str(exc)] + [e["fieldLocation"] for e in exc.errors])
|
| 702 |
+
named = {f for f in fields if re.search(rf"(?<![\w.]){re.escape(f)}(?!\w)", text)}
|
| 703 |
+
named.update(fields[int(i)] for i in re.findall(r"fields\[(\d+)\]", text) if int(i) < len(fields))
|
| 704 |
+
return named
|
| 705 |
+
|
| 706 |
+
|
| 707 |
+
def _create(tenant_key, group, start, end, account, refused, opts):
|
| 708 |
+
"""Create one group report. An error naming only optional fields drops them and creates again."""
|
| 709 |
+
while True:
|
| 710 |
+
fields = group.fields(drop=refused)
|
| 711 |
+
try:
|
| 712 |
+
reply = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1],
|
| 713 |
+
body=_create_body(group, start, end, account, fields), **opts)
|
| 714 |
+
return _first_report(reply, "createReport")
|
| 715 |
+
except AdsRejected as exc:
|
| 716 |
+
named = _named_fields(exc, fields)
|
| 717 |
+
if not named or named & set(group.core):
|
| 718 |
+
raise
|
| 719 |
+
refused.update(dict.fromkeys(sorted(named), _error_text(exc.errors) or str(exc)))
|
| 720 |
+
|
| 721 |
+
|
| 722 |
+
def _await(tenant_key, report_id, opts, *, sleep, now, limit):
|
| 723 |
+
"""Poll retrieve about once a minute until the report completes. -> its report file URLs."""
|
| 724 |
+
started = now()
|
| 725 |
+
while True:
|
| 726 |
+
report = _first_report(request(tenant_key, "POST", REPORT_OPERATIONS["retrieveReport"][1],
|
| 727 |
+
body={"reportIds": [report_id]}, **opts), "retrieveReport")
|
| 728 |
+
state = str(report.get("status") or "").upper()
|
| 729 |
+
if state == "COMPLETED":
|
| 730 |
+
urls = [str(part.get("url") or "") for part in report.get("completedReportParts") or ()
|
| 731 |
+
if isinstance(part, dict)]
|
| 732 |
+
_ISSUED_FILES.update(urls)
|
| 733 |
+
return urls
|
| 734 |
+
if state == "FAILED":
|
| 735 |
+
reason = " ".join(filter(None, (str(report.get("failureCode") or ""),
|
| 736 |
+
str(report.get("failureReason") or ""))))[:ERROR_TEXT_LIMIT] or state
|
| 737 |
+
if re.search(r"not (?:supported|offered|available)", reason, re.I):
|
| 738 |
+
raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}")
|
| 739 |
+
raise AdsError(f"Amazon Ads report {report_id} ended FAILED: {reason}")
|
| 740 |
+
elapsed = now() - started
|
| 741 |
+
if elapsed > REPORT_MAX_WAIT_SECONDS:
|
| 742 |
+
raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
|
| 743 |
+
if limit is not None and elapsed >= limit:
|
| 744 |
+
raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
|
| 745 |
+
pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
|
| 746 |
+
sleep(pause if limit is None else min(pause, limit - elapsed))
|
| 747 |
+
|
| 748 |
+
|
| 749 |
+
def _flat(row, prefix=""):
|
| 750 |
+
"""A nested v1 row -> flat dotted field ids. A flat row stays as it is."""
|
| 751 |
+
out = {}
|
| 752 |
+
for key, value in row.items():
|
| 753 |
+
if isinstance(value, dict):
|
| 754 |
+
out.update(_flat(value, f"{prefix}{key}."))
|
| 755 |
+
else:
|
| 756 |
+
out[f"{prefix}{key}"] = value
|
| 757 |
+
return out
|
| 758 |
+
|
| 759 |
+
|
| 760 |
+
def _part_rows(raw):
|
| 761 |
+
"""One report file -> flat rows. GZIP or plain, a JSON array or JSON lines."""
|
| 762 |
+
try:
|
| 763 |
+
text = (gzip.decompress(raw) if raw[:2] == b"\x1f\x8b" else raw).decode("utf-8", "replace").strip()
|
| 764 |
+
if not text:
|
| 765 |
+
return []
|
| 766 |
+
try:
|
| 767 |
+
data = json.loads(text)
|
| 768 |
+
except ValueError:
|
| 769 |
+
data = [json.loads(line) for line in text.splitlines() if line.strip()]
|
| 770 |
+
except (OSError, EOFError, ValueError) as exc:
|
| 771 |
+
raise AdsError(f"Amazon Ads sent a report file Loopable cannot read ({type(exc).__name__})") from None
|
| 772 |
+
return [_flat(row) for row in (data if isinstance(data, list) else [data]) if isinstance(row, dict)]
|
| 773 |
+
|
| 774 |
+
|
| 775 |
+
def _read_file(url):
|
| 776 |
+
"""GET one report file with no Ads header. Only an https S3 URL a retrieve returned is admitted."""
|
| 777 |
+
parts = urllib.parse.urlsplit(str(url or ""))
|
| 778 |
+
host = parts.hostname or ""
|
| 779 |
+
if parts.scheme != "https" or not host.endswith(REPORT_FILE_HOST_SUFFIX) or url not in _ISSUED_FILES:
|
| 780 |
+
raise AdsWriteRefused(f"{parts.scheme or 'no scheme'} {_safe_path(host)} is not a report file Amazon returned")
|
| 781 |
+
_ISSUED_FILES.discard(url)
|
| 782 |
+
headers = {}
|
| 783 |
+
if fixture_mode():
|
| 784 |
+
return _part_rows(FIXTURE.download(url, headers))
|
| 785 |
+
try:
|
| 786 |
+
with urllib.request.urlopen(urllib.request.Request(url, method="GET", headers=headers),
|
| 787 |
+
timeout=_TIMEOUT) as r:
|
| 788 |
+
raw = r.read()
|
| 789 |
+
except Exception as exc: # noqa: BLE001 the URL carries a signature: never echo it
|
| 790 |
+
raise AdsError(f"Amazon Ads report file could not be read ({type(exc).__name__})") from None
|
| 791 |
+
return _part_rows(raw)
|
| 792 |
+
|
| 793 |
+
|
| 794 |
+
def _split(group, rows):
|
| 795 |
+
"""Flat rows -> {member table: rows} by `adProduct.value`, and a note for rows no member takes."""
|
| 796 |
+
from harness import amazon_ads_schema as schema
|
| 797 |
+
out, tables, stray = {table: [] for table in group.members}, group.tables, 0
|
| 798 |
+
for row in rows:
|
| 799 |
+
table = tables.get(str(row.get(schema.AD_PRODUCT_FIELD) or "").upper())
|
| 800 |
+
if table is None or not row.get(schema.vendor_name("date")):
|
| 801 |
+
stray += 1
|
| 802 |
+
else:
|
| 803 |
+
out[table].append(row)
|
| 804 |
+
note = (f"Amazon Ads sent {stray} rows with no date or with another ad product. Loopable left them out.",)
|
| 805 |
+
return out, note if stray else ()
|
| 806 |
+
|
| 807 |
+
|
| 808 |
+
def run_group_report(tenant_key, group, start, end, *, rt=None, profile_id=None, connection_id=None,
|
| 809 |
+
sleep=time.sleep, now=time.monotonic, max_wait=None, report_id=None):
|
| 810 |
+
"""Create one Reporting v1 group report, wait for it, and read every file. -> GroupReport.
|
| 811 |
+
|
| 812 |
+
`max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without a create.
|
| 813 |
"""
|
| 814 |
from harness import amazon_ads_schema as schema
|
| 815 |
+
group = schema.GROUPS[group] if isinstance(group, str) else group
|
| 816 |
if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
|
| 817 |
+
raise ValueError(f"a report window is at most {MAX_DAYS_PER_REQUEST} days")
|
| 818 |
+
fields = _report_credential(tenant_key, rt, connection_id)
|
| 819 |
+
opts = {"rt": rt, "profile_id": str(profile_id or fields["profile_id"]), "connection_id": connection_id}
|
| 820 |
+
refused = _REFUSED.setdefault((str(fields["client_id"]), group.name), {})
|
| 821 |
limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
|
| 822 |
+
with _report_lock(fields["client_id"]):
|
|
|
|
| 823 |
if report_id is None:
|
| 824 |
+
account = _advertiser_account(tenant_key, **opts)
|
| 825 |
+
report_id = _create(tenant_key, group, start, end, account, refused, opts)["reportId"]
|
| 826 |
+
urls = _await(tenant_key, str(report_id), opts, sleep=sleep, now=now, limit=limit)
|
| 827 |
+
rows = [row for url in urls for row in _read_file(url)]
|
| 828 |
+
split, stray = _split(group, rows)
|
| 829 |
+
notes = tuple(f"Amazon Ads refused the field {field} on this report, so Loopable reads it without that "
|
| 830 |
+
f"field. Amazon said: {text}" for field, text in refused.items())
|
| 831 |
+
return GroupReport(str(report_id), split, notes + stray)
|
| 832 |
+
|
| 833 |
+
|
| 834 |
+
def run_report(tenant_key, table, start, end, **options):
|
| 835 |
+
"""One table's rows from its group report. -> (report_id, rows). The store reads `run_group_report`."""
|
| 836 |
+
from harness import amazon_ads_schema as schema
|
| 837 |
+
group = schema.group_of(table)
|
| 838 |
+
if group is None:
|
| 839 |
+
raise AdsReportUnavailable(f"Amazon Reporting v1 offers no report for {schema.short_name(table)}")
|
| 840 |
+
result = run_group_report(tenant_key, group, start, end, **options)
|
| 841 |
+
return result.report_id, result.rows[schema.short_name(table)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 842 |
|
| 843 |
|
| 844 |
# --------------------------------------------------------------------------- recording fixture
|
| 845 |
|
| 846 |
+
FIXTURE_TOKEN = "fixture-access-token"
|
| 847 |
+
FIXTURE_CREDENTIAL = {"client_id": "amzn1.application-oa2-client.fixture", "client_secret": "fixture",
|
| 848 |
+
"refresh_token": "fixture", "profile_id": "1000000001", "region": "na"}
|
| 849 |
+
FIXTURE_FILE_HOST = "https://fixture-ads-reports.s3.amazonaws.com"
|
| 850 |
+
FIXTURE_CREATED = "2026-10-20T09:00:00Z"
|
| 851 |
+
_FIXTURE_PREFIX = {"SPONSORED_PRODUCTS": ("71", "SP"), "SPONSORED_BRANDS": ("72", "SB"),
|
| 852 |
+
"SPONSORED_DISPLAY": ("73", "SD")}
|
| 853 |
+
#: Per campaign and day: the metric totals every finer row splits evenly.
|
| 854 |
+
_FIXTURE_TOTALS = {"metric.impressions": 400, "metric.clicks": 12, "metric.totalCost": 5.0,
|
| 855 |
+
"metric.sales": 38.0, "metric.purchases": 4, "metric.unitsSold": 4}
|
| 856 |
+
|
| 857 |
+
|
| 858 |
+
def _multi_status(report=None, errors=None):
|
| 859 |
+
"""A ReportMultiStatusResponse with one success or one error entry at index 0."""
|
| 860 |
+
return {"success": [{"index": 0, "report": report}] if report else [],
|
| 861 |
+
"error": [{"index": 0, "errors": errors}] if errors else []}
|
| 862 |
+
|
| 863 |
+
|
| 864 |
class _Fixture:
|
| 865 |
+
"""The recording fixture `AMAZON_ADS_FIXTURE=1` routes to, in the saved contracts' v1 shapes.
|
|
|
|
| 866 |
|
| 867 |
+
It records method, path, operation, headers and body of every call, report file downloads
|
| 868 |
+
included. Toggles: `drop_days`, `unsupported` (group names that end FAILED), `refuse_fields`
|
| 869 |
+
(a create naming one answers 400), `nested` (files carry nested JSON lines), `responder`.
|
| 870 |
+
"""
|
| 871 |
+
|
| 872 |
+
FILES = 2
|
| 873 |
|
| 874 |
def __init__(self):
|
| 875 |
self.reset()
|
| 876 |
|
| 877 |
def reset(self):
|
| 878 |
+
self.calls, self.reports, self.subscriptions, self.credentials = [], {}, {}, {}
|
|
|
|
| 879 |
self.in_flight = self.max_in_flight = 0
|
| 880 |
+
self.drop_days, self.unsupported, self.refuse_fields = set(), set(), set()
|
| 881 |
+
self.nested = False
|
| 882 |
self.responder = None
|
| 883 |
|
| 884 |
+
def credential(self, tenant_key):
|
| 885 |
+
"""The fixture credential for one tenant. `credentials[tenant]` overrides its fields."""
|
| 886 |
+
return {**FIXTURE_CREDENTIAL, **self.credentials.get(tenant_key, {})}
|
| 887 |
+
|
| 888 |
+
def answer(self, prepared, tenant_key, profile_id, headers=None):
|
| 889 |
self.calls.append({"method": prepared.method, "path": prepared.path,
|
| 890 |
"operation": prepared.operation, "body": prepared.body,
|
| 891 |
+
"query": dict(prepared.query), "tenant": tenant_key, "headers": dict(headers or {}),
|
| 892 |
"named": dict(prepared.named) if prepared.named else None})
|
| 893 |
if self.responder is not None:
|
| 894 |
return self.responder(prepared)
|
|
|
|
| 904 |
if prepared.named is not None:
|
| 905 |
items = next(iter(prepared.body.values()))
|
| 906 |
return {"success": [{"index": i} for i in range(len(items))], "error": []}
|
| 907 |
+
if prepared.operation == "queryAdvertiserAccounts":
|
| 908 |
+
return self._accounts(prepared.body, profile_id)
|
| 909 |
if prepared.operation == "createReport":
|
| 910 |
+
return self._create(prepared.body)
|
| 911 |
+
return self._retrieve(prepared.body["reportIds"][0])
|
| 912 |
+
|
| 913 |
+
def _accounts(self, body, profile_id):
|
| 914 |
+
"""Two pages: an unrelated account, then the account that lists the caller's profile."""
|
| 915 |
+
if not body.get("nextToken"):
|
| 916 |
+
return {"advertiserAccounts": [{"advertiserAccountId": "amzn1.ads-account.g.fxother",
|
| 917 |
+
"alternateIds": [{"profileId": "999", "countryCode": "CA"}]}],
|
| 918 |
+
"nextToken": "fixture-page-2"}
|
| 919 |
+
return {"advertiserAccounts": [{"advertiserAccountId": f"amzn1.ads-account.g.fx{profile_id}",
|
| 920 |
+
"alternateIds": [{"profileId": str(profile_id), "countryCode": "US",
|
| 921 |
+
"entityId": "ENTITYFX"}]}]}
|
| 922 |
+
|
| 923 |
+
def _create(self, body):
|
| 924 |
+
if not (body.get("accessRequestedAccounts") or [{}])[0].get("advertiserAccountId"):
|
| 925 |
+
return _multi_status(errors=[{"code": "400001", "message": "An advertiser account is required.",
|
| 926 |
+
"fieldLocation": "accessRequestedAccounts"}])
|
| 927 |
+
refused = [f for f in body["reports"][0]["query"]["fields"] if f in self.refuse_fields]
|
| 928 |
+
if refused:
|
| 929 |
+
raise _rejection("Amazon Ads answered HTTP 400 on createReport",
|
| 930 |
+
[{"code": "400004", "message": f"Field {refused[0]} is invalid for this request.",
|
| 931 |
+
"fieldLocation": ""}], may_be_unoffered=False)
|
| 932 |
+
report_id = f"fx{len(self.reports) + 1:05d}"
|
| 933 |
+
self.in_flight += 1
|
| 934 |
+
self.max_in_flight = max(self.max_in_flight, self.in_flight)
|
| 935 |
+
self.reports[report_id] = {"body": body, "polls": 0, "files": None, "done": False}
|
| 936 |
+
return _multi_status(self._record(report_id, "PENDING"))
|
| 937 |
+
|
| 938 |
+
def _record(self, report_id, status, **extra):
|
| 939 |
+
"""A contract `Report`: every required key, the request's format, periods and query echoed."""
|
| 940 |
+
request = self.reports[report_id]["body"]["reports"][0]
|
| 941 |
+
return {"reportId": report_id, "status": status, "format": request["format"],
|
| 942 |
+
"periods": request["periods"], "query": request["query"],
|
| 943 |
+
"creationDateTime": FIXTURE_CREATED, "lastUpdatedDateTime": FIXTURE_CREATED, **extra}
|
| 944 |
+
|
| 945 |
+
def _retrieve(self, report_id):
|
| 946 |
+
from harness import amazon_ads_schema as schema
|
| 947 |
report = self.reports[report_id]
|
| 948 |
+
report["polls"] += 1
|
| 949 |
+
query = report["body"]["reports"][0]["query"]
|
| 950 |
+
group = schema.group_for(query["fields"], query["filter"]["on"]["values"])
|
| 951 |
+
if group is not None and group.name in self.unsupported:
|
| 952 |
+
self._finish(report)
|
| 953 |
+
return _multi_status(self._record(report_id, "FAILED", failureCode="REPORT_NOT_SUPPORTED",
|
| 954 |
+
failureReason="This report is not supported for this advertiser."))
|
| 955 |
+
if report["polls"] < 2:
|
| 956 |
+
return _multi_status(self._record(report_id, "PROCESSING"))
|
| 957 |
+
if report["files"] is None:
|
| 958 |
+
rows = fixture_rows(report["body"], drop_days=self.drop_days)
|
| 959 |
+
report["files"] = {f"{FIXTURE_FILE_HOST}/{report_id}/part-{i}.json.gz?X-Amz-Signature=fx{report_id}{i}":
|
| 960 |
+
_fixture_file(rows[i::self.FILES], self.nested) for i in range(self.FILES)}
|
| 961 |
+
return _multi_status(self._record(report_id, "COMPLETED", completedDateTime=FIXTURE_CREATED, completedReportParts=[
|
| 962 |
+
{"url": url, "sizeInBytes": len(raw), "urlExpirationDateTime": "2026-12-31T00:00:00Z"}
|
| 963 |
+
for url, raw in report["files"].items()]))
|
| 964 |
+
|
| 965 |
+
def _finish(self, report):
|
| 966 |
+
if not report["done"]:
|
| 967 |
+
report["done"] = True
|
| 968 |
self.in_flight -= 1
|
| 969 |
+
|
| 970 |
+
def download(self, url, headers):
|
| 971 |
+
"""Serve one report file and record the GET with the headers the client sent."""
|
| 972 |
+
self.calls.append({"method": "GET", "path": url, "operation": "downloadReportPart", "body": None,
|
| 973 |
+
"query": {}, "tenant": None, "headers": dict(headers), "named": None})
|
| 974 |
+
report = self.reports[urllib.parse.urlsplit(url).path.split("/")[1]]
|
| 975 |
+
raw = report["files"][url]
|
| 976 |
+
report.setdefault("read", set()).add(url)
|
| 977 |
+
if report["read"] == set(report["files"]):
|
| 978 |
+
self._finish(report)
|
| 979 |
+
return raw
|
| 980 |
+
|
| 981 |
+
|
| 982 |
+
def _fixture_file(rows, nested):
|
| 983 |
+
"""GZIP JSON: an array of flat rows, or nested objects one per line."""
|
| 984 |
+
def nest(row):
|
| 985 |
+
out = {}
|
| 986 |
+
for field, value in row.items():
|
| 987 |
+
head, _, tail = field.partition(".")
|
| 988 |
+
out.setdefault(head, {})[tail] = value
|
| 989 |
+
return out
|
| 990 |
+
text = "\n".join(json.dumps(nest(r)) for r in rows) if nested else json.dumps(rows)
|
| 991 |
+
return gzip.compress(text.encode("utf-8"))
|
| 992 |
+
|
| 993 |
+
|
| 994 |
+
def _fixture_query(body):
|
| 995 |
+
report = body["reports"][0]
|
| 996 |
+
period = report["periods"][0]["datePeriod"]
|
| 997 |
+
return (report["query"]["fields"], report["query"]["filter"]["on"]["values"],
|
| 998 |
+
date.fromisoformat(period["startDate"]), date.fromisoformat(period["endDate"]))
|
| 999 |
+
|
| 1000 |
+
|
| 1001 |
+
def _fixture_days(start, end, drop_days):
|
| 1002 |
+
return [start + timedelta(days=i) for i in range((end - start).days + 1)
|
| 1003 |
+
if (start + timedelta(days=i)).isoformat() not in drop_days]
|
| 1004 |
+
|
| 1005 |
+
|
| 1006 |
+
def fixture_totals(body, *, drop_days=()):
|
| 1007 |
+
"""The fixture's own totals for one create body. -> {(ad product, date, campaign id): {field: total}}."""
|
| 1008 |
+
_fields, products, start, end = _fixture_query(body)
|
| 1009 |
+
out = {}
|
| 1010 |
+
for day in _fixture_days(start, end, drop_days):
|
| 1011 |
+
for index, product in enumerate(products):
|
| 1012 |
+
for c in (1, 2):
|
| 1013 |
+
n = day.toordinal() % 7 + c + index
|
| 1014 |
+
out[(product, day.isoformat(), f"{_FIXTURE_PREFIX[product][0]}10{c}")] = {
|
| 1015 |
+
field: total * n for field, total in _FIXTURE_TOTALS.items()}
|
| 1016 |
+
return out
|
| 1017 |
|
| 1018 |
|
| 1019 |
def fixture_rows(body, *, drop_days=()):
|
| 1020 |
+
"""Deterministic v1 rows for one create body: one row per distinct value of every requested
|
| 1021 |
+
dimension and day, only the requested fields and ad products. Two ad groups split a campaign."""
|
| 1022 |
+
fields, _products, start, end = _fixture_query(body)
|
| 1023 |
+
groups = (1, 2) if "adGroup.id" in fields else (0,)
|
| 1024 |
+
out = []
|
| 1025 |
+
for (product, day, campaign), totals in fixture_totals(body, drop_days=drop_days).items():
|
| 1026 |
+
prefix, short = _FIXTURE_PREFIX[product]
|
| 1027 |
+
c = campaign[-1]
|
| 1028 |
+
for g in groups:
|
| 1029 |
+
row = {"date.value": day, "adProduct.value": product, "campaign.id": campaign,
|
| 1030 |
+
"campaign.name": f"{short} campaign {c}", "campaign.deliveryStatus": "DELIVERING",
|
| 1031 |
+
"campaign.budgetAmount": 25 * int(c), "budgetCurrency.value": "USD",
|
| 1032 |
+
"adGroup.id": f"{prefix}20{c}{g}", "adGroup.name": f"Ad group {c}.{g}",
|
| 1033 |
+
"adGroupTarget.id": f"{prefix}30{c}{g}", "target.value": f"term {c}{g}",
|
| 1034 |
+
"target.matchType": "EXACT", "adGroupTarget.bid": str(0.5 + g / 10),
|
| 1035 |
+
"ad.id": f"{prefix}40{c}{g}", "advertisedProduct.id": f"B0FX{prefix}{c}{g}00",
|
| 1036 |
+
"advertisedProductMarketplace.value": "ATVPDKIKX0DER", "advertisedProductSku.value": f"SKU-{c}{g}",
|
| 1037 |
+
"searchTerm.value": f"search {c}{g}",
|
| 1038 |
+
**{f: (t // len(groups) if isinstance(t, int) else t / len(groups)) for f, t in totals.items()}}
|
| 1039 |
+
out.append({f: row[f] for f in fields})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1040 |
return out
|
| 1041 |
|
| 1042 |
|
| 1043 |
FIXTURE = _Fixture()
|
| 1044 |
|
| 1045 |
|
| 1046 |
+
def _reset_caches():
|
| 1047 |
+
_ACCOUNTS.clear()
|
| 1048 |
+
_REFUSED.clear()
|
| 1049 |
+
_ISSUED_FILES.clear()
|
| 1050 |
+
|
| 1051 |
+
|
| 1052 |
@contextmanager
|
| 1053 |
def recording_fixture():
|
| 1054 |
+
"""Route every call to a fresh recording fixture and fresh report caches for the block. -> the fixture."""
|
| 1055 |
previous = os.environ.get(FIXTURE_ENV)
|
| 1056 |
os.environ[FIXTURE_ENV] = "1"
|
| 1057 |
FIXTURE.reset()
|
| 1058 |
+
_reset_caches()
|
| 1059 |
try:
|
| 1060 |
yield FIXTURE
|
| 1061 |
finally:
|
| 1062 |
FIXTURE.reset()
|
| 1063 |
+
_reset_caches()
|
| 1064 |
if previous is None:
|
| 1065 |
os.environ.pop(FIXTURE_ENV, None)
|
| 1066 |
else:
|
| 1067 |
os.environ[FIXTURE_ENV] = previous
|
| 1068 |
|
| 1069 |
|
| 1070 |
+
__all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "V1_CLIENT_HEADERS",
|
| 1071 |
+
"NAMED_WRITES", "STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation",
|
| 1072 |
"FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE",
|
| 1073 |
"SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY",
|
| 1074 |
+
"MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsRejected", "AdsReportUnavailable",
|
| 1075 |
+
"AdsReportPending", "GroupReport", "fixture_mode", "build_request", "report_operation",
|
| 1076 |
+
"send_named_write", "request", "credential", "run_group_report", "run_report", "FIXTURE",
|
| 1077 |
+
"recording_fixture", "fixture_rows", "fixture_totals"]
|
platform/harness/amazon_ads_schema.py
CHANGED
|
@@ -1,16 +1,17 @@
|
|
| 1 |
"""amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6).
|
| 2 |
|
| 3 |
-
One entry per ad type and report: the
|
| 4 |
-
|
| 5 |
-
|
| 6 |
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
"""
|
| 11 |
import hashlib
|
| 12 |
import sys
|
| 13 |
from pathlib import Path
|
|
|
|
| 14 |
|
| 15 |
_HERE = Path(__file__).resolve().parent
|
| 16 |
if str(_HERE.parent) not in sys.path:
|
|
@@ -22,8 +23,6 @@ AD_TYPES = _cs.AMAZON_ADS_AD_TYPES
|
|
| 22 |
REPORTS = _cs.AMAZON_ADS_REPORTS
|
| 23 |
DATABASE_PREFIX = "ut_amzn_"
|
| 24 |
MIRROR_PREFIX = "amzn_ads_"
|
| 25 |
-
#: Amazon accepts at most 31 days per report request.
|
| 26 |
-
MAX_DAYS_PER_REQUEST = 31
|
| 27 |
|
| 28 |
#: The C6 columns every Ads table carries, in this order.
|
| 29 |
C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date",
|
|
@@ -54,22 +53,76 @@ _EXTRA = {
|
|
| 54 |
}
|
| 55 |
_TAIL = ("profile_id", "currency")
|
| 56 |
|
| 57 |
-
#: Our column ->
|
| 58 |
_VENDOR = {
|
| 59 |
-
"campaign_id": "
|
| 60 |
-
"campaign_status": "
|
| 61 |
-
"ad_group_id": "
|
| 62 |
-
"targeting": "
|
| 63 |
-
"ad_id": "
|
| 64 |
-
"search_term": "searchTerm", "
|
| 65 |
-
"
|
| 66 |
-
"units": "unitsSold", "currency": "
|
| 67 |
-
}
|
| 68 |
-
_VENDOR_BY_TYPE = {
|
| 69 |
-
"sp": {"sales": "sales14d", "orders": "purchases14d", "units": "unitsSoldClicks14d"},
|
| 70 |
-
"sd": {"target_id": "targetingId", "targeting": "targetingExpression",
|
| 71 |
-
"advertised_sku": "promotedSku", "asin": "promotedAsin"},
|
| 72 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
|
| 74 |
|
| 75 |
def _short(ad_type, report):
|
|
@@ -89,25 +142,24 @@ def coltype(name):
|
|
| 89 |
return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR"
|
| 90 |
|
| 91 |
|
| 92 |
-
def vendor_name(
|
| 93 |
-
"""The
|
| 94 |
-
|
| 95 |
-
return None
|
| 96 |
-
return _VENDOR_BY_TYPE.get(ad_type, {}).get(column) or _VENDOR.get(column)
|
| 97 |
|
| 98 |
|
| 99 |
TABLE_SPECS = {}
|
| 100 |
for _spec in _cs.AMAZON_ADS_TABLES:
|
| 101 |
_ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1)
|
|
|
|
| 102 |
TABLE_SPECS[_short(_ad_type, _report)] = {
|
| 103 |
"key": _spec.key, "source": _spec.source, "label": _spec.label,
|
| 104 |
-
"ad_type": _ad_type, "report": _report, "
|
| 105 |
"columns": _columns(_report), "row_identity": tuple(_spec.identity),
|
| 106 |
"backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key],
|
| 107 |
"offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED,
|
| 108 |
"not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key),
|
| 109 |
}
|
| 110 |
-
del _spec, _ad_type, _report
|
| 111 |
|
| 112 |
|
| 113 |
def short_name(table):
|
|
@@ -127,11 +179,16 @@ def columns_for(table):
|
|
| 127 |
return TABLE_SPECS[short_name(table)]["columns"]
|
| 128 |
|
| 129 |
|
| 130 |
-
def
|
| 131 |
-
"""The
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
|
| 136 |
|
| 137 |
def _cell(name, value):
|
|
@@ -149,11 +206,11 @@ def _cell(name, value):
|
|
| 149 |
|
| 150 |
|
| 151 |
def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None):
|
| 152 |
-
"""One
|
| 153 |
spec = TABLE_SPECS[short_name(table)]
|
| 154 |
out = {}
|
| 155 |
for column in spec["columns"]:
|
| 156 |
-
source = vendor_name(
|
| 157 |
value = vendor_row.get(source) if source else None
|
| 158 |
if column == "profile_id":
|
| 159 |
value = profile_id
|
|
@@ -213,7 +270,8 @@ def stream_coltype(name):
|
|
| 213 |
return "DOUBLE"
|
| 214 |
return "VARCHAR"
|
| 215 |
|
| 216 |
-
__all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "
|
|
|
|
| 217 |
"ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for",
|
| 218 |
-
"
|
| 219 |
"STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"]
|
|
|
|
| 1 |
"""amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6).
|
| 2 |
|
| 3 |
+
One entry per ad type and report: the columns we store, the Reporting v1 field each one is read
|
| 4 |
+
from, the row identity and the backfill window. The table keys, labels and backfill days come from
|
| 5 |
+
`core.connector_spec`, so the two files cannot disagree on a name.
|
| 6 |
|
| 7 |
+
Field ids come from Amazon's v1 field library, saved at
|
| 8 |
+
`AutoWiki/research/amazon-ads-v1-schema/reporting_v1_fields.json`. v1 has no report types: one
|
| 9 |
+
report per `REPORT_GROUPS` row covers several ad types, and its rows split by `adProduct.value`.
|
| 10 |
"""
|
| 11 |
import hashlib
|
| 12 |
import sys
|
| 13 |
from pathlib import Path
|
| 14 |
+
from typing import NamedTuple
|
| 15 |
|
| 16 |
_HERE = Path(__file__).resolve().parent
|
| 17 |
if str(_HERE.parent) not in sys.path:
|
|
|
|
| 23 |
REPORTS = _cs.AMAZON_ADS_REPORTS
|
| 24 |
DATABASE_PREFIX = "ut_amzn_"
|
| 25 |
MIRROR_PREFIX = "amzn_ads_"
|
|
|
|
|
|
|
| 26 |
|
| 27 |
#: The C6 columns every Ads table carries, in this order.
|
| 28 |
C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date",
|
|
|
|
| 53 |
}
|
| 54 |
_TAIL = ("profile_id", "currency")
|
| 55 |
|
| 56 |
+
#: Our column -> its Reporting v1 field id, one map for every ad type. The store stamps profile_id.
|
| 57 |
_VENDOR = {
|
| 58 |
+
"date": "date.value", "campaign_id": "campaign.id", "campaign_name": "campaign.name",
|
| 59 |
+
"campaign_status": "campaign.deliveryStatus", "campaign_budget_amount": "campaign.budgetAmount",
|
| 60 |
+
"ad_group_id": "adGroup.id", "ad_group_name": "adGroup.name", "target_id": "adGroupTarget.id",
|
| 61 |
+
"targeting": "target.value", "match_type": "target.matchType", "bid": "adGroupTarget.bid",
|
| 62 |
+
"ad_id": "ad.id", "advertised_sku": "advertisedProductSku.value", "asin": "advertisedProduct.id",
|
| 63 |
+
"search_term": "searchTerm.value", "impressions": "metric.impressions", "clicks": "metric.clicks",
|
| 64 |
+
"cost": "metric.totalCost", "sales": "metric.sales", "orders": "metric.purchases",
|
| 65 |
+
"units": "metric.unitsSold", "currency": "budgetCurrency.value",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
}
|
| 67 |
+
#: The v1 field that names each row's ad type, and its value per ad type.
|
| 68 |
+
AD_PRODUCT_FIELD = "adProduct.value"
|
| 69 |
+
AD_PRODUCTS = {"sp": "SPONSORED_PRODUCTS", "sb": "SPONSORED_BRANDS", "sd": "SPONSORED_DISPLAY"}
|
| 70 |
+
#: The name the v1 field library lists each ad type under.
|
| 71 |
+
LIBRARY_AD_PRODUCTS = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
|
| 72 |
+
|
| 73 |
+
METRICS = ("metric.impressions", "metric.clicks", "metric.totalCost", "metric.sales",
|
| 74 |
+
"metric.purchases", "metric.unitsSold")
|
| 75 |
+
_HEAD = ("date.value", AD_PRODUCT_FIELD, "campaign.id", "campaign.name")
|
| 76 |
+
_AD_GROUP = ("adGroup.id", "adGroup.name")
|
| 77 |
+
_TAIL_FIELDS = ("budgetCurrency.value",) + METRICS
|
| 78 |
+
#: ⛔ Not `targeting.value` or `targetingMatchType.value`: the library lists them for no ad product.
|
| 79 |
+
#: The live probe of 2026-11-18 (`probe-2026-11-18.json`) proved these names, absent from the library.
|
| 80 |
+
_TARGET_TEXT = ("target.value", "target.matchType")
|
| 81 |
+
#: Fields a live Reporting v1 create accepted that the saved field library does not list.
|
| 82 |
+
PROBE_PROVEN = frozenset(_TARGET_TEXT)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class ReportGroup(NamedTuple):
|
| 86 |
+
"""One Reporting v1 report and the member tables its rows split into by `adProduct.value`.
|
| 87 |
+
|
| 88 |
+
`optional` fields list no ad product in Amazon's library, or only a live probe proved them:
|
| 89 |
+
a create that refuses one drops it.
|
| 90 |
+
"""
|
| 91 |
+
name: str
|
| 92 |
+
members: tuple
|
| 93 |
+
core: tuple
|
| 94 |
+
optional: tuple = ()
|
| 95 |
+
|
| 96 |
+
@property
|
| 97 |
+
def tables(self):
|
| 98 |
+
"""`adProduct.value` -> the member table its rows land in."""
|
| 99 |
+
return {AD_PRODUCTS[table.split("_", 1)[0]]: table for table in self.members}
|
| 100 |
+
|
| 101 |
+
def fields(self, drop=()):
|
| 102 |
+
"""The fields one create asks for, without the optional fields Amazon refused."""
|
| 103 |
+
return tuple(f for f in self.core + self.optional if f not in drop)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
#: ⛔ Every requested dimension is a group-by and sets the row grain. A group asks only for its
|
| 107 |
+
#: tables' identity dimensions, attributes of those, metrics, the ad product and required partners.
|
| 108 |
+
#: Every core field lists every ad product of its group in the library: SB lacks the SKU field and
|
| 109 |
+
#: the marketplace field `advertisedProduct.id` requires, so SB advertised products is its own report.
|
| 110 |
+
REPORT_GROUPS = (
|
| 111 |
+
ReportGroup("campaigns", ("sp_campaigns", "sb_campaigns", "sd_campaigns"),
|
| 112 |
+
_HEAD + ("campaign.deliveryStatus", "campaign.budgetAmount") + _TAIL_FIELDS),
|
| 113 |
+
ReportGroup("targeting", ("sp_targeting", "sb_targeting", "sd_targeting"),
|
| 114 |
+
_HEAD + _AD_GROUP + ("adGroupTarget.id",) + _TAIL_FIELDS,
|
| 115 |
+
_TARGET_TEXT + ("adGroupTarget.bid",)),
|
| 116 |
+
ReportGroup("advertised_products", ("sp_advertised_products", "sd_advertised_products"),
|
| 117 |
+
_HEAD + _AD_GROUP + ("ad.id", "advertisedProduct.id", "advertisedProductMarketplace.value",
|
| 118 |
+
"advertisedProductSku.value") + _TAIL_FIELDS),
|
| 119 |
+
ReportGroup("advertised_products_sb", ("sb_advertised_products",),
|
| 120 |
+
_HEAD + _AD_GROUP + ("ad.id",) + _TAIL_FIELDS),
|
| 121 |
+
ReportGroup("search_terms", ("sp_search_terms", "sb_search_terms"),
|
| 122 |
+
_HEAD + _AD_GROUP + ("adGroupTarget.id", "searchTerm.value") + _TAIL_FIELDS, _TARGET_TEXT),
|
| 123 |
+
)
|
| 124 |
+
GROUPS = {group.name: group for group in REPORT_GROUPS}
|
| 125 |
+
_GROUP_OF = {table: group for group in REPORT_GROUPS for table in group.members}
|
| 126 |
|
| 127 |
|
| 128 |
def _short(ad_type, report):
|
|
|
|
| 142 |
return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR"
|
| 143 |
|
| 144 |
|
| 145 |
+
def vendor_name(column):
|
| 146 |
+
"""The v1 field one of our columns is read from, or None when the vendor has none."""
|
| 147 |
+
return _VENDOR.get(column)
|
|
|
|
|
|
|
| 148 |
|
| 149 |
|
| 150 |
TABLE_SPECS = {}
|
| 151 |
for _spec in _cs.AMAZON_ADS_TABLES:
|
| 152 |
_ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1)
|
| 153 |
+
_group = _GROUP_OF.get(_short(_ad_type, _report))
|
| 154 |
TABLE_SPECS[_short(_ad_type, _report)] = {
|
| 155 |
"key": _spec.key, "source": _spec.source, "label": _spec.label,
|
| 156 |
+
"ad_type": _ad_type, "report": _report, "group": _group.name if _group else None,
|
| 157 |
"columns": _columns(_report), "row_identity": tuple(_spec.identity),
|
| 158 |
"backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key],
|
| 159 |
"offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED,
|
| 160 |
"not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key),
|
| 161 |
}
|
| 162 |
+
del _spec, _ad_type, _report, _group
|
| 163 |
|
| 164 |
|
| 165 |
def short_name(table):
|
|
|
|
| 179 |
return TABLE_SPECS[short_name(table)]["columns"]
|
| 180 |
|
| 181 |
|
| 182 |
+
def group_of(table):
|
| 183 |
+
"""The report group that fills one Ads table, or None for a table v1 does not offer."""
|
| 184 |
+
return _GROUP_OF.get(short_name(table))
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def group_for(fields, ad_products):
|
| 188 |
+
"""The report group a create request asks for, by its fields and ad product filter, or None."""
|
| 189 |
+
fields, ad_products = set(fields), set(ad_products)
|
| 190 |
+
return next((g for g in REPORT_GROUPS if set(g.tables) == ad_products
|
| 191 |
+
and set(g.core) <= fields <= set(g.core + g.optional)), None)
|
| 192 |
|
| 193 |
|
| 194 |
def _cell(name, value):
|
|
|
|
| 206 |
|
| 207 |
|
| 208 |
def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None):
|
| 209 |
+
"""One flat v1 report row (dotted field ids) -> one stored row, with its `_row_key` and report origin."""
|
| 210 |
spec = TABLE_SPECS[short_name(table)]
|
| 211 |
out = {}
|
| 212 |
for column in spec["columns"]:
|
| 213 |
+
source = vendor_name(column)
|
| 214 |
value = vendor_row.get(source) if source else None
|
| 215 |
if column == "profile_id":
|
| 216 |
value = profile_id
|
|
|
|
| 270 |
return "DOUBLE"
|
| 271 |
return "VARCHAR"
|
| 272 |
|
| 273 |
+
__all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "AD_PRODUCT_FIELD", "AD_PRODUCTS",
|
| 274 |
+
"LIBRARY_AD_PRODUCTS", "METRICS", "ReportGroup", "REPORT_GROUPS", "GROUPS",
|
| 275 |
"ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for",
|
| 276 |
+
"group_of", "group_for", "row_from_vendor", "row_key", "STREAM_WINDOWS", "STREAM_METRICS", "STREAM_GRAIN",
|
| 277 |
"STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"]
|
platform/harness/amazon_ads_store.py
CHANGED
|
@@ -10,8 +10,8 @@ back the longest window Amazon documents per table (`connector_spec.AMAZON_ADS_B
|
|
| 10 |
A later sync re-reads the table's daily or intraday cadence window and upserts by row identity.
|
| 11 |
No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here.
|
| 12 |
|
| 13 |
-
⛔ STANDING RULE 1. Every report is read whole
|
| 14 |
-
report at a time, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
|
| 15 |
`amzn_limit_ads_<table>` with the reason a person reads, and stated on the table description.
|
| 16 |
|
| 17 |
Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_<table>` is the newest landed day,
|
|
@@ -178,7 +178,7 @@ def _day_marker(con, key):
|
|
| 178 |
|
| 179 |
|
| 180 |
def windows(con, table, connect_day, today, connection_id=None):
|
| 181 |
-
"""The report windows one sync asks for, oldest first, each at most
|
| 182 |
table = schema.short_name(table)
|
| 183 |
spec = schema.TABLE_SPECS[table]
|
| 184 |
cadence = amazon_client.CADENCE["ads_" + table]
|
|
@@ -237,13 +237,13 @@ def _upsert(con, table, rows):
|
|
| 237 |
cols = _columns_for(table)
|
| 238 |
by_key = {r["_row_key"]: r for r in rows}
|
| 239 |
keys = list(by_key)
|
|
|
|
| 240 |
for i in range(0, len(keys), INSERT_BATCH_ROWS):
|
| 241 |
part = keys[i:i + INSERT_BATCH_ROWS]
|
| 242 |
con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part)
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
[[by_key[k].get(c) for c in cols] for k in part])
|
| 247 |
return len(by_key)
|
| 248 |
|
| 249 |
|
|
@@ -389,16 +389,158 @@ def _land(con, table, rows, report_id, start, end, profile, connection_id, label
|
|
| 389 |
|
| 390 |
|
| 391 |
def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
|
| 392 |
-
"""
|
| 393 |
options = {"rt": rt, "profile_id": profile_id, **extra}
|
| 394 |
if connection_id:
|
| 395 |
options["connection_id"] = connection_id
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 402 |
|
| 403 |
|
| 404 |
def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
|
|
@@ -409,6 +551,7 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 409 |
through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
|
| 410 |
`max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
|
| 411 |
pending ledger, and the next pass reads it before it creates any other report.
|
|
|
|
| 412 |
"""
|
| 413 |
key = amazon_store._tenant_key(tenant_key)
|
| 414 |
report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
|
|
@@ -425,10 +568,9 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 425 |
own = con is None
|
| 426 |
if own:
|
| 427 |
con, _path = amazon_store._connect_tenant_mirror(key, rt)
|
| 428 |
-
bounded = max_seconds is not None
|
| 429 |
started = _clock()
|
| 430 |
-
deadline =
|
| 431 |
-
|
| 432 |
try:
|
| 433 |
datastore._ensure_sync_state(con)
|
| 434 |
now = _utc_now(now)
|
|
@@ -437,164 +579,24 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 437 |
def stamp():
|
| 438 |
return now + timedelta(seconds=max(0.0, _clock() - started))
|
| 439 |
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
synced = []
|
| 444 |
-
checkpointed = []
|
| 445 |
-
for cid, label, profile, legacy in connections:
|
| 446 |
connect_day = _connect_day(con, today, cid, legacy=legacy)
|
| 447 |
report.setdefault("connectDay", connect_day.isoformat())
|
| 448 |
entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
|
| 449 |
report["connections"].append(entry)
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
pending = _read_pending(con, table, cid)
|
| 455 |
-
if pending is None:
|
| 456 |
-
continue
|
| 457 |
-
spec = schema.TABLE_SPECS[table]
|
| 458 |
-
_ensure_table(con, table)
|
| 459 |
-
if table_name(table) not in checkpointed:
|
| 460 |
-
checkpointed.append(table_name(table))
|
| 461 |
-
age = (stamp() - pending["created"]).total_seconds()
|
| 462 |
-
expired = (f"Amazon Ads report {pending['report_id']} did not finish in 3 hours")
|
| 463 |
-
if age >= client.REPORT_MAX_WAIT_SECONDS:
|
| 464 |
-
_clear_pending(con, table, cid)
|
| 465 |
-
report["problems"].append(f"{label}: {spec['label']}: {expired}")
|
| 466 |
-
resumed[table] = ("failed", expired, 0, None)
|
| 467 |
-
continue
|
| 468 |
-
limit = client.REPORT_MAX_WAIT_SECONDS - age
|
| 469 |
-
if bounded:
|
| 470 |
-
limit = min(limit, max(0.0, deadline - _clock()))
|
| 471 |
-
try:
|
| 472 |
-
report_id, rows = _run_report(key, table, pending["start"], pending["end"], rt, profile, cid,
|
| 473 |
-
max_wait=limit, report_id=pending["report_id"], now=_clock)
|
| 474 |
-
except client.AdsReportUnavailable as exc:
|
| 475 |
-
_clear_pending(con, table, cid)
|
| 476 |
-
resumed[table] = ("not_offered", _not_offered_reason(spec, exc), 0, None)
|
| 477 |
-
continue
|
| 478 |
-
except client.AdsReportPending:
|
| 479 |
-
if (stamp() - pending["created"]).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS:
|
| 480 |
-
_clear_pending(con, table, cid)
|
| 481 |
-
report["problems"].append(f"{label}: {spec['label']}: {expired}")
|
| 482 |
-
resumed[table] = ("failed", expired, 0, None)
|
| 483 |
-
else:
|
| 484 |
-
blocked = True
|
| 485 |
-
wait(label, spec, PREPARING_REASON)
|
| 486 |
-
resumed[table] = ("deferred", PREPARING_REASON, 0, None)
|
| 487 |
-
continue
|
| 488 |
-
except client.AdsError as exc:
|
| 489 |
-
_clear_pending(con, table, cid)
|
| 490 |
-
reason = str(exc)[:200]
|
| 491 |
-
report["problems"].append(f"{label}: {spec['label']}: {reason}")
|
| 492 |
-
resumed[table] = ("failed", reason, 0, None)
|
| 493 |
-
continue
|
| 494 |
-
_clear_pending(con, table, cid)
|
| 495 |
-
pulled = _land(con, table, rows, report_id, pending["start"], pending["end"], profile, cid, label)
|
| 496 |
-
resumed[table] = ("done", "", pulled, (pending["start"], pending["end"]))
|
| 497 |
-
requested = [schema.short_name(t) for t in (tables or TABLES)]
|
| 498 |
for table in list(dict.fromkeys(requested + list(resumed))):
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
prior = resumed.get(table)
|
| 504 |
-
landed = None
|
| 505 |
-
if prior is not None:
|
| 506 |
-
state, reason, pulled, landed = prior
|
| 507 |
-
if state != "done" or table not in requested:
|
| 508 |
-
pass
|
| 509 |
-
elif not spec["offered"]:
|
| 510 |
-
state, reason = "not_offered", spec["not_offered_reason"]
|
| 511 |
-
elif not _cadence_due(con, table, now, cid):
|
| 512 |
-
state = "skipped"
|
| 513 |
-
elif blocked:
|
| 514 |
-
state, reason = "deferred", PREPARING_REASON
|
| 515 |
-
wait(label, spec, reason)
|
| 516 |
-
else:
|
| 517 |
-
for start, end in windows(con, table, connect_day, today, cid):
|
| 518 |
-
if landed and landed[0] <= start and end <= landed[1]:
|
| 519 |
-
continue # The resumed report just read these days.
|
| 520 |
-
if used >= client.REPORTS_PER_DAY:
|
| 521 |
-
state = "deferred"
|
| 522 |
-
reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
|
| 523 |
-
f"reports are used, so days from {start.isoformat()} load on the next sync.")
|
| 524 |
-
report["problems"].append(f"{label}: {spec['label']}: waiting for tomorrow's report limit")
|
| 525 |
-
waits.append(report["problems"][-1])
|
| 526 |
-
break
|
| 527 |
-
if bounded and _clock() >= deadline:
|
| 528 |
-
state, reason = "deferred", TIME_LIMIT_REASON
|
| 529 |
-
wait(label, spec, reason)
|
| 530 |
-
break
|
| 531 |
-
used += 1
|
| 532 |
-
_spend_budget(con, today, used, cid)
|
| 533 |
-
created = stamp()
|
| 534 |
-
try:
|
| 535 |
-
if bounded:
|
| 536 |
-
report_id, rows = _run_report(key, table, start, end, rt, profile, cid,
|
| 537 |
-
max_wait=max(0.0, deadline - _clock()), now=_clock)
|
| 538 |
-
else:
|
| 539 |
-
report_id, rows = _run_report(key, table, start, end, rt, profile, cid)
|
| 540 |
-
except client.AdsReportUnavailable as exc:
|
| 541 |
-
state = "not_offered"
|
| 542 |
-
reason = _not_offered_reason(spec, exc)
|
| 543 |
-
break
|
| 544 |
-
except client.AdsError as exc:
|
| 545 |
-
if (bounded and isinstance(exc, client.AdsReportPending) and exc.report_id
|
| 546 |
-
and (stamp() - created).total_seconds() < client.REPORT_MAX_WAIT_SECONDS):
|
| 547 |
-
_write_pending(con, table, cid, exc.report_id, start, end, created)
|
| 548 |
-
if table_name(table) not in checkpointed:
|
| 549 |
-
checkpointed.append(table_name(table))
|
| 550 |
-
blocked = True
|
| 551 |
-
state, reason = "deferred", PREPARING_REASON
|
| 552 |
-
wait(label, spec, reason)
|
| 553 |
-
break
|
| 554 |
-
state = "failed"
|
| 555 |
-
reason = str(exc)[:200]
|
| 556 |
-
report["problems"].append(f"{label}: {spec['label']}: {reason}")
|
| 557 |
-
break
|
| 558 |
-
pulled += _land(con, table, rows, report_id, start, end, profile, cid, label)
|
| 559 |
-
if table_name(table) not in checkpointed:
|
| 560 |
-
checkpointed.append(table_name(table))
|
| 561 |
-
if state == "done":
|
| 562 |
-
_record_cadence(con, table, now, cid)
|
| 563 |
-
if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
|
| 564 |
-
report["partial"] = True
|
| 565 |
-
if reason:
|
| 566 |
-
_record_limit(con, table, reason, cid, state=state)
|
| 567 |
-
shown = f"{label}: {reason}" if cid else reason
|
| 568 |
-
report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown)))
|
| 569 |
-
elif state == "done":
|
| 570 |
-
_clear_limit(con, table, cid)
|
| 571 |
-
total = _count(con, table)
|
| 572 |
-
entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
|
| 573 |
-
if state != "skipped":
|
| 574 |
-
amazon_store._record_state(con, _state_key(amazon_store.entity_key(mark), cid),
|
| 575 |
-
"done" if state == "done" else "partial", _count(con, table, cid), state)
|
| 576 |
-
previous = report["tables"].get(table, {})
|
| 577 |
-
severity = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
|
| 578 |
-
aggregate = max((state, previous.get("state", state)), key=severity.get)
|
| 579 |
-
report["tables"][table] = {"state": aggregate, "pulled": previous.get("pulled", 0) + pulled,
|
| 580 |
-
"in_mirror": total}
|
| 581 |
-
if (pulled or origin_changed) and table_name(table) not in synced:
|
| 582 |
synced.append(table_name(table))
|
| 583 |
-
|
| 584 |
-
for table, state in report["tables"].items():
|
| 585 |
-
if state["state"] == "skipped":
|
| 586 |
-
continue
|
| 587 |
-
amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
|
| 588 |
-
"done" if state["state"] == "done" else "partial",
|
| 589 |
-
state["in_mirror"], state["state"])
|
| 590 |
-
if table in report["limits"]:
|
| 591 |
-
_record_limit(con, table, report["limits"][table], state=state["state"])
|
| 592 |
-
else:
|
| 593 |
-
_clear_limit(con, table)
|
| 594 |
-
report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
|
| 595 |
-
for cid, *_ in connections for table in TABLES)
|
| 596 |
-
if report["partial"]:
|
| 597 |
-
report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
|
| 598 |
if own and rt is not None and (synced or checkpointed):
|
| 599 |
_persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
|
| 600 |
finally:
|
|
@@ -604,10 +606,75 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 604 |
return report
|
| 605 |
|
| 606 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 607 |
def _not_offered_reason(spec, exc):
|
| 608 |
return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
|
| 609 |
-
f"({str(exc)[:
|
| 610 |
-
f"contact Amazon Ads support
|
| 611 |
|
| 612 |
|
| 613 |
def _persist(tenant_key, rt, con, synced, report, log):
|
|
|
|
| 10 |
A later sync re-reads the table's daily or intraday cadence window and upserts by row identity.
|
| 11 |
No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here.
|
| 12 |
|
| 13 |
+
⛔ STANDING RULE 1. Every report is read whole, every file of it. The limits that cannot be removed
|
| 14 |
+
(one report at a time per client ID, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
|
| 15 |
`amzn_limit_ads_<table>` with the reason a person reads, and stated on the table description.
|
| 16 |
|
| 17 |
Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_<table>` is the newest landed day,
|
|
|
|
| 178 |
|
| 179 |
|
| 180 |
def windows(con, table, connect_day, today, connection_id=None):
|
| 181 |
+
"""The report windows one sync asks for, oldest first, each at most 120 days (`client.MAX_DAYS_PER_REQUEST`)."""
|
| 182 |
table = schema.short_name(table)
|
| 183 |
spec = schema.TABLE_SPECS[table]
|
| 184 |
cadence = amazon_client.CADENCE["ads_" + table]
|
|
|
|
| 237 |
cols = _columns_for(table)
|
| 238 |
by_key = {r["_row_key"]: r for r in rows}
|
| 239 |
keys = list(by_key)
|
| 240 |
+
one = f'({", ".join("?" for _ in cols)})'
|
| 241 |
for i in range(0, len(keys), INSERT_BATCH_ROWS):
|
| 242 |
part = keys[i:i + INSERT_BATCH_ROWS]
|
| 243 |
con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part)
|
| 244 |
+
# One multi-row INSERT per batch: DuckDB's executemany is about 5 times slower at 24 months a pass.
|
| 245 |
+
con.execute(f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) '
|
| 246 |
+
f'VALUES {", ".join(one for _ in part)}', [by_key[k].get(c) for k in part for c in cols])
|
|
|
|
| 247 |
return len(by_key)
|
| 248 |
|
| 249 |
|
|
|
|
| 389 |
|
| 390 |
|
| 391 |
def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
|
| 392 |
+
"""Run the group report that fills `table` under this connection's own credential. -> GroupReport."""
|
| 393 |
options = {"rt": rt, "profile_id": profile_id, **extra}
|
| 394 |
if connection_id:
|
| 395 |
options["connection_id"] = connection_id
|
| 396 |
+
return client.run_group_report(key, schema.group_of(table), start, end, **options)
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
class _Pass:
|
| 400 |
+
"""One connection's sync pass: its budget, clock, and what its group reports already covered.
|
| 401 |
+
|
| 402 |
+
`eligible` holds the tables a group report may land into this pass: requested, offered, due and
|
| 403 |
+
not yet finished. `covered` holds each table's spans landed this pass, `pulled` the rows a
|
| 404 |
+
sibling's report landed, `outcome` a group's failure, `notes` Amazon's text per table.
|
| 405 |
+
"""
|
| 406 |
+
|
| 407 |
+
def __init__(self, con, key, rt, connection, report, *, today, used, deadline, stamp, checkpointed):
|
| 408 |
+
self.con, self.key, self.rt, self.report = con, key, rt, report
|
| 409 |
+
self.cid, self.label, self.profile = connection[:3]
|
| 410 |
+
self.today, self.used, self.deadline, self.stamp = today, used, deadline, stamp
|
| 411 |
+
self.checkpointed = checkpointed
|
| 412 |
+
self.eligible, self.covered, self.pulled, self.outcome, self.notes = set(), {}, {}, {}, {}
|
| 413 |
+
self.blocked = False
|
| 414 |
+
|
| 415 |
+
def wait(self, spec, reason):
|
| 416 |
+
self.report["waits"].append(f"{self.label}: {spec['label']}: {reason}")
|
| 417 |
+
|
| 418 |
+
def checkpoint(self, table):
|
| 419 |
+
if table_name(table) not in self.checkpointed:
|
| 420 |
+
self.checkpointed.append(table_name(table))
|
| 421 |
+
|
| 422 |
+
def covers(self, table, start, end):
|
| 423 |
+
"""True when reports this pass already read every day from `start` to `end` for `table`."""
|
| 424 |
+
spans = self.covered.get(table, ())
|
| 425 |
+
return all(any(lo <= start + timedelta(days=i) <= hi for lo, hi in spans)
|
| 426 |
+
for i in range((end - start).days + 1))
|
| 427 |
+
|
| 428 |
+
def bounds(self):
|
| 429 |
+
"""The poll limit for one new report: none, or the time left in this bounded pass."""
|
| 430 |
+
return {} if self.deadline is None else {"max_wait": max(0.0, self.deadline - _clock()), "now": _clock}
|
| 431 |
+
|
| 432 |
+
def fail(self, table, exc):
|
| 433 |
+
"""Record a group report's failure for every member. -> (state, reason) for `table`."""
|
| 434 |
+
self.outcome[schema.group_of(table).name] = exc
|
| 435 |
+
return self.failure(table, exc)
|
| 436 |
+
|
| 437 |
+
def failure(self, table, exc):
|
| 438 |
+
spec = schema.TABLE_SPECS[table]
|
| 439 |
+
if isinstance(exc, client.AdsReportUnavailable):
|
| 440 |
+
return "not_offered", _not_offered_reason(spec, exc)
|
| 441 |
+
reason = str(exc)[:client.ERROR_TEXT_LIMIT]
|
| 442 |
+
self.report["problems"].append(f"{self.label}: {spec['label']}: {reason}")
|
| 443 |
+
return "failed", reason
|
| 444 |
+
|
| 445 |
+
def land(self, table, result, start, end):
|
| 446 |
+
"""Land one group report into `table` and each eligible sibling, covering the span. -> rows into `table`."""
|
| 447 |
+
own = 0
|
| 448 |
+
for member in schema.group_of(table).members:
|
| 449 |
+
if member != table and member not in self.eligible:
|
| 450 |
+
continue
|
| 451 |
+
_ensure_table(self.con, member)
|
| 452 |
+
rows = _land(self.con, member, result.rows.get(member, []), result.report_id, start, end,
|
| 453 |
+
self.profile, self.cid, self.label)
|
| 454 |
+
self.covered.setdefault(member, []).append((start, end))
|
| 455 |
+
self.notes[member] = list(dict.fromkeys(self.notes.get(member, []) + list(result.notes)))
|
| 456 |
+
self.checkpoint(member)
|
| 457 |
+
if member == table:
|
| 458 |
+
own = rows
|
| 459 |
+
else:
|
| 460 |
+
self.pulled[member] = self.pulled.get(member, 0) + rows
|
| 461 |
+
for note in result.notes:
|
| 462 |
+
if f"{self.label}: {note}" not in self.report["problems"]:
|
| 463 |
+
self.report["problems"].append(f"{self.label}: {note}")
|
| 464 |
+
return own
|
| 465 |
+
|
| 466 |
+
|
| 467 |
+
def _resume(p):
|
| 468 |
+
"""Read every report this connection left preparing before it creates another. -> {table: outcome}."""
|
| 469 |
+
resumed = {}
|
| 470 |
+
for table in TABLES:
|
| 471 |
+
pending = _read_pending(p.con, table, p.cid)
|
| 472 |
+
if pending is None:
|
| 473 |
+
continue
|
| 474 |
+
spec = schema.TABLE_SPECS[table]
|
| 475 |
+
_ensure_table(p.con, table)
|
| 476 |
+
p.checkpoint(table)
|
| 477 |
+
expired = f"Amazon Ads report {pending['report_id']} did not finish in 3 hours"
|
| 478 |
+
age = (p.stamp() - pending["created"]).total_seconds()
|
| 479 |
+
if age >= client.REPORT_MAX_WAIT_SECONDS:
|
| 480 |
+
_clear_pending(p.con, table, p.cid)
|
| 481 |
+
resumed[table] = (*p.failure(table, client.AdsError(expired)), 0)
|
| 482 |
+
continue
|
| 483 |
+
limit = client.REPORT_MAX_WAIT_SECONDS - age
|
| 484 |
+
if p.deadline is not None:
|
| 485 |
+
limit = min(limit, max(0.0, p.deadline - _clock()))
|
| 486 |
+
try:
|
| 487 |
+
result = _run_report(p.key, table, pending["start"], pending["end"], p.rt, p.profile, p.cid,
|
| 488 |
+
max_wait=limit, report_id=pending["report_id"], now=_clock)
|
| 489 |
+
except client.AdsReportPending:
|
| 490 |
+
if (p.stamp() - pending["created"]).total_seconds() < client.REPORT_MAX_WAIT_SECONDS:
|
| 491 |
+
p.blocked = True
|
| 492 |
+
p.wait(spec, PREPARING_REASON)
|
| 493 |
+
resumed[table] = ("deferred", PREPARING_REASON, 0)
|
| 494 |
+
continue
|
| 495 |
+
_clear_pending(p.con, table, p.cid)
|
| 496 |
+
resumed[table] = (*p.failure(table, client.AdsError(expired)), 0)
|
| 497 |
+
continue
|
| 498 |
+
except client.AdsError as exc:
|
| 499 |
+
_clear_pending(p.con, table, p.cid)
|
| 500 |
+
resumed[table] = (*p.fail(table, exc), 0)
|
| 501 |
+
continue
|
| 502 |
+
_clear_pending(p.con, table, p.cid)
|
| 503 |
+
resumed[table] = ("done", "", p.land(table, result, pending["start"], pending["end"]))
|
| 504 |
+
return resumed
|
| 505 |
+
|
| 506 |
+
|
| 507 |
+
def _run_windows(p, table, connect_day):
|
| 508 |
+
"""Run the group report for each window of `table` no report covered yet. -> (state, reason, rows)."""
|
| 509 |
+
spec = schema.TABLE_SPECS[table]
|
| 510 |
+
failed = p.outcome.get(schema.group_of(table).name)
|
| 511 |
+
if failed is not None:
|
| 512 |
+
return (*p.failure(table, failed), 0)
|
| 513 |
+
pulled = 0
|
| 514 |
+
for start, end in windows(p.con, table, connect_day, p.today, p.cid):
|
| 515 |
+
if p.covers(table, start, end):
|
| 516 |
+
continue
|
| 517 |
+
if p.used >= client.REPORTS_PER_DAY:
|
| 518 |
+
reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
|
| 519 |
+
f"reports are used, so days from {start.isoformat()} load on the next sync.")
|
| 520 |
+
p.report["problems"].append(f"{p.label}: {spec['label']}: waiting for tomorrow's report limit")
|
| 521 |
+
p.report["waits"].append(p.report["problems"][-1])
|
| 522 |
+
return "deferred", reason, pulled
|
| 523 |
+
if p.deadline is not None and _clock() >= p.deadline:
|
| 524 |
+
p.wait(spec, TIME_LIMIT_REASON)
|
| 525 |
+
return "deferred", TIME_LIMIT_REASON, pulled
|
| 526 |
+
p.used += 1
|
| 527 |
+
_spend_budget(p.con, p.today, p.used, p.cid)
|
| 528 |
+
created = p.stamp()
|
| 529 |
+
try:
|
| 530 |
+
result = _run_report(p.key, table, start, end, p.rt, p.profile, p.cid, **p.bounds())
|
| 531 |
+
except client.AdsReportPending as exc:
|
| 532 |
+
if (p.deadline is None or not exc.report_id
|
| 533 |
+
or (p.stamp() - created).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS):
|
| 534 |
+
return (*p.failure(table, exc), pulled)
|
| 535 |
+
_write_pending(p.con, table, p.cid, exc.report_id, start, end, created)
|
| 536 |
+
p.checkpoint(table)
|
| 537 |
+
p.blocked = True
|
| 538 |
+
p.wait(spec, PREPARING_REASON)
|
| 539 |
+
return "deferred", PREPARING_REASON, pulled
|
| 540 |
+
except client.AdsError as exc:
|
| 541 |
+
return (*p.fail(table, exc), pulled)
|
| 542 |
+
pulled += p.land(table, result, start, end)
|
| 543 |
+
return "done", "", pulled
|
| 544 |
|
| 545 |
|
| 546 |
def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
|
|
|
|
| 551 |
through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
|
| 552 |
`max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
|
| 553 |
pending ledger, and the next pass reads it before it creates any other report.
|
| 554 |
+
One group report fills every requested member table, so a sibling's covered days spend no report.
|
| 555 |
"""
|
| 556 |
key = amazon_store._tenant_key(tenant_key)
|
| 557 |
report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
|
|
|
|
| 568 |
own = con is None
|
| 569 |
if own:
|
| 570 |
con, _path = amazon_store._connect_tenant_mirror(key, rt)
|
|
|
|
| 571 |
started = _clock()
|
| 572 |
+
deadline = None if max_seconds is None else started + float(max_seconds)
|
| 573 |
+
synced, checkpointed = [], []
|
| 574 |
try:
|
| 575 |
datastore._ensure_sync_state(con)
|
| 576 |
now = _utc_now(now)
|
|
|
|
| 579 |
def stamp():
|
| 580 |
return now + timedelta(seconds=max(0.0, _clock() - started))
|
| 581 |
|
| 582 |
+
requested = [schema.short_name(t) for t in (tables or TABLES)]
|
| 583 |
+
for connection in connections:
|
| 584 |
+
cid, label, _profile, legacy = connection
|
|
|
|
|
|
|
|
|
|
| 585 |
connect_day = _connect_day(con, today, cid, legacy=legacy)
|
| 586 |
report.setdefault("connectDay", connect_day.isoformat())
|
| 587 |
entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
|
| 588 |
report["connections"].append(entry)
|
| 589 |
+
p = _Pass(con, key, rt, connection, report, today=today, used=_budget(con, today, cid, legacy=legacy),
|
| 590 |
+
deadline=deadline, stamp=stamp, checkpointed=checkpointed)
|
| 591 |
+
p.eligible = {t for t in requested if schema.TABLE_SPECS[t]["offered"] and _cadence_due(con, t, now, cid)}
|
| 592 |
+
resumed = _resume(p)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 593 |
for table in list(dict.fromkeys(requested + list(resumed))):
|
| 594 |
+
landed = _sync_table(p, table, entry, connect_day, now, legacy, table in requested,
|
| 595 |
+
resumed.get(table), log)
|
| 596 |
+
p.eligible.discard(table)
|
| 597 |
+
if landed and table_name(table) not in synced:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 598 |
synced.append(table_name(table))
|
| 599 |
+
_record_tables(con, report, connections)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 600 |
if own and rt is not None and (synced or checkpointed):
|
| 601 |
_persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
|
| 602 |
finally:
|
|
|
|
| 606 |
return report
|
| 607 |
|
| 608 |
|
| 609 |
+
_SEVERITY = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
|
| 610 |
+
|
| 611 |
+
|
| 612 |
+
def _sync_table(p, table, entry, connect_day, now, legacy, requested, prior, log):
|
| 613 |
+
"""One table's turn in one connection's pass: run or skip it, record its state. -> True if it changed."""
|
| 614 |
+
con, cid, label, report = p.con, p.cid, p.label, p.report
|
| 615 |
+
spec = schema.TABLE_SPECS[table]
|
| 616 |
+
_ensure_table(con, table)
|
| 617 |
+
origin_changed = _refresh_origin(con, table, cid, label, legacy)
|
| 618 |
+
state, reason, pulled = prior or ("done", "", 0)
|
| 619 |
+
pulled += p.pulled.pop(table, 0)
|
| 620 |
+
if state != "done" or not requested:
|
| 621 |
+
pass
|
| 622 |
+
elif not spec["offered"]:
|
| 623 |
+
state, reason = "not_offered", spec["not_offered_reason"]
|
| 624 |
+
elif not _cadence_due(con, table, now, cid):
|
| 625 |
+
state = "skipped"
|
| 626 |
+
elif p.blocked:
|
| 627 |
+
state, reason = "deferred", PREPARING_REASON
|
| 628 |
+
p.wait(spec, reason)
|
| 629 |
+
else:
|
| 630 |
+
state, reason, rows = _run_windows(p, table, connect_day)
|
| 631 |
+
pulled += rows
|
| 632 |
+
if state == "done":
|
| 633 |
+
_record_cadence(con, table, now, cid)
|
| 634 |
+
if state == "done" and not reason and p.notes.get(table):
|
| 635 |
+
reason = " ".join(p.notes[table])
|
| 636 |
+
if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
|
| 637 |
+
report["partial"] = True
|
| 638 |
+
if reason:
|
| 639 |
+
_record_limit(con, table, reason, cid, state=state)
|
| 640 |
+
shown = f"{label}: {reason}" if cid else reason
|
| 641 |
+
report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown)))
|
| 642 |
+
elif state == "done":
|
| 643 |
+
_clear_limit(con, table, cid)
|
| 644 |
+
total = _count(con, table)
|
| 645 |
+
entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
|
| 646 |
+
if state != "skipped":
|
| 647 |
+
amazon_store._record_state(con, _state_key(amazon_store.entity_key(_marker_table(table)), cid),
|
| 648 |
+
"done" if state == "done" else "partial", _count(con, table, cid), state)
|
| 649 |
+
previous = report["tables"].get(table, {})
|
| 650 |
+
report["tables"][table] = {"state": max((state, previous.get("state", state)), key=_SEVERITY.get),
|
| 651 |
+
"pulled": previous.get("pulled", 0) + pulled, "in_mirror": total}
|
| 652 |
+
log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}")
|
| 653 |
+
return bool(pulled or origin_changed)
|
| 654 |
+
|
| 655 |
+
|
| 656 |
+
def _record_tables(con, report, connections):
|
| 657 |
+
"""Write each table's aggregate state and limit across connections, and the pass's pending count."""
|
| 658 |
+
for table, state in report["tables"].items():
|
| 659 |
+
if state["state"] == "skipped":
|
| 660 |
+
continue
|
| 661 |
+
amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
|
| 662 |
+
"done" if state["state"] == "done" else "partial",
|
| 663 |
+
state["in_mirror"], state["state"])
|
| 664 |
+
if table in report["limits"]:
|
| 665 |
+
_record_limit(con, table, report["limits"][table], state=state["state"])
|
| 666 |
+
else:
|
| 667 |
+
_clear_limit(con, table)
|
| 668 |
+
report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
|
| 669 |
+
for cid, *_ in connections for table in TABLES)
|
| 670 |
+
if report["partial"]:
|
| 671 |
+
report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
|
| 672 |
+
|
| 673 |
+
|
| 674 |
def _not_offered_reason(spec, exc):
|
| 675 |
return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
|
| 676 |
+
f"({str(exc)[:client.ERROR_TEXT_LIMIT]}). The table keeps the days it has. To ask for it, "
|
| 677 |
+
f"contact Amazon Ads support and quote Amazon's message.")
|
| 678 |
|
| 679 |
|
| 680 |
def _persist(tenant_key, rt, con, synced, report, log):
|
platform/harness/amazon_client.py
CHANGED
|
@@ -1127,7 +1127,7 @@ for _tables, _passes, _daily, _intraday in (
|
|
| 1127 |
from harness import amazon_ads_schema as _ads_schema
|
| 1128 |
|
| 1129 |
CADENCE.update({
|
| 1130 |
-
"ads_" + table: {"perDay": 4, "lookbackDays":
|
| 1131 |
for table in _ads_schema.TABLE_SPECS
|
| 1132 |
})
|
| 1133 |
del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
|
|
|
|
| 1127 |
from harness import amazon_ads_schema as _ads_schema
|
| 1128 |
|
| 1129 |
CADENCE.update({
|
| 1130 |
+
"ads_" + table: {"perDay": 4, "lookbackDays": 60, "intradayLookbackDays": 5, "period": "day"}
|
| 1131 |
for table in _ads_schema.TABLE_SPECS
|
| 1132 |
})
|
| 1133 |
del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
c62b077954cb16ddf668e8e5aee89d6e4cb852718d0b57b94aa3412a210747e6
|