Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/routes_connectors.py +13 -1
- platform/core/connector_spec.py +98 -3
- platform/harness/amazon_client.py +176 -30
- platform/harness/amazon_schema.py +4 -2
- platform/harness/amazon_store.py +415 -61
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/connectors/ConnectorsPage.tsx +16 -3
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "b14f21b9e",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
b14f21b9e
|
api/routes_connectors.py
CHANGED
|
@@ -3167,6 +3167,13 @@ def _amazon_cause_text(text, limit=300):
|
|
| 3167 |
return text
|
| 3168 |
|
| 3169 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3170 |
def amazon_hold_cause(row=None, held="", *, table=""):
|
| 3171 |
"""One plain cause for an Amazon table a pass did not finish, from its report row or held text."""
|
| 3172 |
from harness.amazon_store import plain_refusal_reason
|
|
@@ -3187,6 +3194,10 @@ def amazon_hold_cause(row=None, held="", *, table=""):
|
|
| 3187 |
cause = "Amazon is still preparing this report."
|
| 3188 |
elif state == "deferred" or (not row and (not held or held.startswith("not reached"))):
|
| 3189 |
cause = "This pass ended before this table's turn. The next pass continues it."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3190 |
elif state == "never_run":
|
| 3191 |
# A table with no tick is due only in the minute after its daily slot, which a busy pass misses.
|
| 3192 |
cause = "This table has never synced. Its daily start window is too short."
|
|
@@ -3306,7 +3317,8 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
|
|
| 3306 |
reason = row_reason(table, state)
|
| 3307 |
# A source-internal mismatch published the table. Its warning stays until a clean reconcile.
|
| 3308 |
warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
|
| 3309 |
-
schedule.append({**row, "
|
|
|
|
| 3310 |
"lastSyncAt": (completed.get("updated_at")
|
| 3311 |
if completed.get("state") == "done" else None),
|
| 3312 |
"state": state, **({"reason": reason} if reason else {}),
|
|
|
|
| 3167 |
return text
|
| 3168 |
|
| 3169 |
|
| 3170 |
+
def _amazon_table_period(table):
|
| 3171 |
+
"""The table's refresh period from the client CADENCE: "day", "week" or "month"."""
|
| 3172 |
+
from harness import amazon_store as az
|
| 3173 |
+
period = (az._client().CADENCE.get(str(table or "")) or {}).get("period")
|
| 3174 |
+
return period if period in ("day", "week", "month") else "day"
|
| 3175 |
+
|
| 3176 |
+
|
| 3177 |
def amazon_hold_cause(row=None, held="", *, table=""):
|
| 3178 |
"""One plain cause for an Amazon table a pass did not finish, from its report row or held text."""
|
| 3179 |
from harness.amazon_store import plain_refusal_reason
|
|
|
|
| 3194 |
cause = "Amazon is still preparing this report."
|
| 3195 |
elif state == "deferred" or (not row and (not held or held.startswith("not reached"))):
|
| 3196 |
cause = "This pass ended before this table's turn. The next pass continues it."
|
| 3197 |
+
elif state == "never_run" and _amazon_table_period(table) in ("week", "month"):
|
| 3198 |
+
# A weekly or monthly table waits for its first calendar period, so a missing tick is expected.
|
| 3199 |
+
cause = (f"This table has not synced yet. Its first "
|
| 3200 |
+
f"{'weekly' if _amazon_table_period(table) == 'week' else 'monthly'} pass is scheduled.")
|
| 3201 |
elif state == "never_run":
|
| 3202 |
# A table with no tick is due only in the minute after its daily slot, which a busy pass misses.
|
| 3203 |
cause = "This table has never synced. Its daily start window is too short."
|
|
|
|
| 3317 |
reason = row_reason(table, state)
|
| 3318 |
# A source-internal mismatch published the table. Its warning stays until a clean reconcile.
|
| 3319 |
warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
|
| 3320 |
+
schedule.append({**row, "period": _amazon_table_period(table),
|
| 3321 |
+
"connectionId": cid, "account": entry["label"],
|
| 3322 |
"lastSyncAt": (completed.get("updated_at")
|
| 3323 |
if completed.get("state") == "done" else None),
|
| 3324 |
"state": state, **({"reason": reason} if reason else {}),
|
platform/core/connector_spec.py
CHANGED
|
@@ -815,7 +815,7 @@ AMAZON_TABLES = (
|
|
| 815 |
ConnectorTableSpec(
|
| 816 |
key=_AMZN + "review_topics", native="customerFeedback/2024-06-01/items/{asin}/reviews/topics",
|
| 817 |
source="amzn_review_topics", label="Amazon Listings: Review topics", pinned="asin",
|
| 818 |
-
identity=("asin", "marketplace_id", "sentiment", "topic"),
|
| 819 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 820 |
),
|
| 821 |
ConnectorTableSpec(
|
|
@@ -824,6 +824,13 @@ AMAZON_TABLES = (
|
|
| 824 |
identity=("asin", "marketplace_id", "sentiment", "topic", "month"),
|
| 825 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 826 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 827 |
ConnectorTableSpec(
|
| 828 |
key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance",
|
| 829 |
source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku",
|
|
@@ -3306,6 +3313,39 @@ AMAZON_LABELS = {
|
|
| 3306 |
"review_snippet_3": "Review snippet 3",
|
| 3307 |
"review_snippets": "Review snippets",
|
| 3308 |
"subtopics": "Subtopics",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3309 |
},
|
| 3310 |
_AMZN + "review_trends": {
|
| 3311 |
"_data_origin": "Data origin",
|
|
@@ -3321,6 +3361,57 @@ AMAZON_LABELS = {
|
|
| 3321 |
"date_range_start_date": "Review period start",
|
| 3322 |
"date_range_end_date": "Review period end",
|
| 3323 |
"asin_metrics_occurrence_percentage": "ASIN occurrence",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3324 |
},
|
| 3325 |
_AMZN + "replenishment": {
|
| 3326 |
"_data_origin": "Data origin",
|
|
@@ -3491,12 +3582,15 @@ AMAZON_DEFAULT_COLUMNS = {
|
|
| 3491 |
_AMZN + "replenishment": ("asin", "time_interval_start_date", "time_interval_end_date",
|
| 3492 |
"active_subscriptions", "shipped_subscription_units", "total_subscriptions_revenue",
|
| 3493 |
"revenue_penetration", "subscriber_retention_for30_days", "program_type"),
|
| 3494 |
-
_AMZN + "review_topics": ("sentiment", "topic", "asin_metrics_number_of_mentions",
|
| 3495 |
"asin_metrics_occurrence_percentage", "asin_metrics_star_rating_impact",
|
| 3496 |
"parent_asin_metrics_number_of_mentions", "browse_node_metrics_occurrence_percentage_all_products",
|
| 3497 |
"review_snippet_1", "review_snippet_2", "review_snippet_3"),
|
| 3498 |
_AMZN + "review_trends": ("sentiment", "topic", "month", "asin_metrics_occurrence_percentage",
|
| 3499 |
"date_range_start_date", "date_range_end_date"),
|
|
|
|
|
|
|
|
|
|
| 3500 |
_AMZN + "sku_keywords": ("sku", "asin", "month", "search_query", "impressions", "clicks",
|
| 3501 |
"cart_adds", "purchases", "backend_keywords", "source_basis"),
|
| 3502 |
_AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
|
|
@@ -3623,7 +3717,8 @@ AMAZON_GROUPS = {
|
|
| 3623 |
**{_AMZN + t: "Listings" for t in (
|
| 3624 |
"listings", "pricing", "inactive_listings", "cancelled_listings", "suppressed_listings",
|
| 3625 |
"listing_defects", "sold_listings", "listing_items", "catalog_items", "b2b_recommended",
|
| 3626 |
-
"b2b_not_yet_on_amazon", "geo_opportunities", "b2b_quotes", "review_topics", "review_trends"
|
|
|
|
| 3627 |
**{_AMZN + t: "Advertising" for t in (
|
| 3628 |
"brand_analytics", "search_query_performance", "search_catalog_performance",
|
| 3629 |
"market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
|
|
|
|
| 815 |
ConnectorTableSpec(
|
| 816 |
key=_AMZN + "review_topics", native="customerFeedback/2024-06-01/items/{asin}/reviews/topics",
|
| 817 |
source="amzn_review_topics", label="Amazon Listings: Review topics", pinned="asin",
|
| 818 |
+
identity=("asin", "marketplace_id", "sort_by", "sentiment", "topic"),
|
| 819 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 820 |
),
|
| 821 |
ConnectorTableSpec(
|
|
|
|
| 824 |
identity=("asin", "marketplace_id", "sentiment", "topic", "month"),
|
| 825 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 826 |
),
|
| 827 |
+
ConnectorTableSpec(
|
| 828 |
+
key=_AMZN + "browse_node_review_topics",
|
| 829 |
+
native="customerFeedback/2024-06-01/browseNodes/{browseNodeId}/reviews/topics",
|
| 830 |
+
source="amzn_browse_node_review_topics", label="Amazon Listings: Category review topics",
|
| 831 |
+
pinned="browse_node_name",
|
| 832 |
+
identity=("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"),
|
| 833 |
+
),
|
| 834 |
ConnectorTableSpec(
|
| 835 |
key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance",
|
| 836 |
source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku",
|
|
|
|
| 3313 |
"review_snippet_3": "Review snippet 3",
|
| 3314 |
"review_snippets": "Review snippets",
|
| 3315 |
"subtopics": "Subtopics",
|
| 3316 |
+
"sort_by": "Sorted by",
|
| 3317 |
+
"browse_node_id": "Browse node ID",
|
| 3318 |
+
"browse_node_name": "Category",
|
| 3319 |
+
"subtopic_1": "Subtopic 1",
|
| 3320 |
+
"subtopic_1_number_of_mentions": "Subtopic 1 mentions",
|
| 3321 |
+
"subtopic_1_occurrence_percentage": "Subtopic 1 occurrence",
|
| 3322 |
+
"subtopic_1_review_snippet_1": "First subtopic snippet 1",
|
| 3323 |
+
"subtopic_1_review_snippet_2": "First subtopic snippet 2",
|
| 3324 |
+
"subtopic_1_review_snippet_3": "First subtopic snippet 3",
|
| 3325 |
+
"subtopic_2": "Subtopic 2",
|
| 3326 |
+
"subtopic_2_number_of_mentions": "Subtopic 2 mentions",
|
| 3327 |
+
"subtopic_2_occurrence_percentage": "Subtopic 2 occurrence",
|
| 3328 |
+
"subtopic_2_review_snippet_1": "Second subtopic snippet 1",
|
| 3329 |
+
"subtopic_2_review_snippet_2": "Second subtopic snippet 2",
|
| 3330 |
+
"subtopic_2_review_snippet_3": "Second subtopic snippet 3",
|
| 3331 |
+
"subtopic_3": "Subtopic 3",
|
| 3332 |
+
"subtopic_3_number_of_mentions": "Subtopic 3 mentions",
|
| 3333 |
+
"subtopic_3_occurrence_percentage": "Subtopic 3 occurrence",
|
| 3334 |
+
"subtopic_3_review_snippet_1": "Third subtopic snippet 1",
|
| 3335 |
+
"subtopic_3_review_snippet_2": "Third subtopic snippet 2",
|
| 3336 |
+
"subtopic_3_review_snippet_3": "Third subtopic snippet 3",
|
| 3337 |
+
"subtopic_4": "Subtopic 4",
|
| 3338 |
+
"subtopic_4_number_of_mentions": "Subtopic 4 mentions",
|
| 3339 |
+
"subtopic_4_occurrence_percentage": "Subtopic 4 occurrence",
|
| 3340 |
+
"subtopic_4_review_snippet_1": "Fourth subtopic snippet 1",
|
| 3341 |
+
"subtopic_4_review_snippet_2": "Fourth subtopic snippet 2",
|
| 3342 |
+
"subtopic_4_review_snippet_3": "Fourth subtopic snippet 3",
|
| 3343 |
+
"subtopic_5": "Subtopic 5",
|
| 3344 |
+
"subtopic_5_number_of_mentions": "Subtopic 5 mentions",
|
| 3345 |
+
"subtopic_5_occurrence_percentage": "Subtopic 5 occurrence",
|
| 3346 |
+
"subtopic_5_review_snippet_1": "Fifth subtopic snippet 1",
|
| 3347 |
+
"subtopic_5_review_snippet_2": "Fifth subtopic snippet 2",
|
| 3348 |
+
"subtopic_5_review_snippet_3": "Fifth subtopic snippet 3",
|
| 3349 |
},
|
| 3350 |
_AMZN + "review_trends": {
|
| 3351 |
"_data_origin": "Data origin",
|
|
|
|
| 3361 |
"date_range_start_date": "Review period start",
|
| 3362 |
"date_range_end_date": "Review period end",
|
| 3363 |
"asin_metrics_occurrence_percentage": "ASIN occurrence",
|
| 3364 |
+
"parent_asin_metrics_occurrence_percentage": "Parent ASIN occurrence",
|
| 3365 |
+
"browse_node_metrics_occurrence_percentage_all_products": "Browse node occurrence",
|
| 3366 |
+
},
|
| 3367 |
+
_AMZN + "browse_node_review_topics": {
|
| 3368 |
+
"_data_origin": "Data origin",
|
| 3369 |
+
"_report_document_id": "Report document ID",
|
| 3370 |
+
"_report_row_number": "Report row",
|
| 3371 |
+
"_report_type": "Report type",
|
| 3372 |
+
"_snapshot_date": "Synced on",
|
| 3373 |
+
"browse_node_id": "Browse node ID",
|
| 3374 |
+
"browse_node_name": "Category",
|
| 3375 |
+
"marketplace_id": "Marketplace ID",
|
| 3376 |
+
"sort_by": "Sorted by",
|
| 3377 |
+
"sentiment": "Sentiment",
|
| 3378 |
+
"topic": "Topic",
|
| 3379 |
+
"date_range_start_date": "Review period start",
|
| 3380 |
+
"date_range_end_date": "Review period end",
|
| 3381 |
+
"browse_node_metrics_occurrence_percentage_all_products": "Category occurrence",
|
| 3382 |
+
"browse_node_metrics_occurrence_percentage_top_twenty_five_percent_products": "Top 25 percent occurrence",
|
| 3383 |
+
"browse_node_metrics_star_rating_impact_all_products": "Category star rating impact",
|
| 3384 |
+
"browse_node_metrics_star_rating_impact_top_twenty_five_percent_products": "Top 25 percent star rating impact",
|
| 3385 |
+
"review_snippet_1": "Review snippet 1",
|
| 3386 |
+
"review_snippet_2": "Review snippet 2",
|
| 3387 |
+
"review_snippet_3": "Review snippet 3",
|
| 3388 |
+
"subtopic_1": "Subtopic 1",
|
| 3389 |
+
"subtopic_1_occurrence_percentage": "Subtopic 1 occurrence",
|
| 3390 |
+
"subtopic_1_review_snippet_1": "First subtopic snippet 1",
|
| 3391 |
+
"subtopic_1_review_snippet_2": "First subtopic snippet 2",
|
| 3392 |
+
"subtopic_1_review_snippet_3": "First subtopic snippet 3",
|
| 3393 |
+
"subtopic_2": "Subtopic 2",
|
| 3394 |
+
"subtopic_2_occurrence_percentage": "Subtopic 2 occurrence",
|
| 3395 |
+
"subtopic_2_review_snippet_1": "Second subtopic snippet 1",
|
| 3396 |
+
"subtopic_2_review_snippet_2": "Second subtopic snippet 2",
|
| 3397 |
+
"subtopic_2_review_snippet_3": "Second subtopic snippet 3",
|
| 3398 |
+
"subtopic_3": "Subtopic 3",
|
| 3399 |
+
"subtopic_3_occurrence_percentage": "Subtopic 3 occurrence",
|
| 3400 |
+
"subtopic_3_review_snippet_1": "Third subtopic snippet 1",
|
| 3401 |
+
"subtopic_3_review_snippet_2": "Third subtopic snippet 2",
|
| 3402 |
+
"subtopic_3_review_snippet_3": "Third subtopic snippet 3",
|
| 3403 |
+
"subtopic_4": "Subtopic 4",
|
| 3404 |
+
"subtopic_4_occurrence_percentage": "Subtopic 4 occurrence",
|
| 3405 |
+
"subtopic_4_review_snippet_1": "Fourth subtopic snippet 1",
|
| 3406 |
+
"subtopic_4_review_snippet_2": "Fourth subtopic snippet 2",
|
| 3407 |
+
"subtopic_4_review_snippet_3": "Fourth subtopic snippet 3",
|
| 3408 |
+
"subtopic_5": "Subtopic 5",
|
| 3409 |
+
"subtopic_5_occurrence_percentage": "Subtopic 5 occurrence",
|
| 3410 |
+
"subtopic_5_review_snippet_1": "Fifth subtopic snippet 1",
|
| 3411 |
+
"subtopic_5_review_snippet_2": "Fifth subtopic snippet 2",
|
| 3412 |
+
"subtopic_5_review_snippet_3": "Fifth subtopic snippet 3",
|
| 3413 |
+
"our_asins": "Our ASINs",
|
| 3414 |
+
"our_asin_count": "Our ASIN count",
|
| 3415 |
},
|
| 3416 |
_AMZN + "replenishment": {
|
| 3417 |
"_data_origin": "Data origin",
|
|
|
|
| 3582 |
_AMZN + "replenishment": ("asin", "time_interval_start_date", "time_interval_end_date",
|
| 3583 |
"active_subscriptions", "shipped_subscription_units", "total_subscriptions_revenue",
|
| 3584 |
"revenue_penetration", "subscriber_retention_for30_days", "program_type"),
|
| 3585 |
+
_AMZN + "review_topics": ("sort_by", "sentiment", "topic", "browse_node_name", "asin_metrics_number_of_mentions",
|
| 3586 |
"asin_metrics_occurrence_percentage", "asin_metrics_star_rating_impact",
|
| 3587 |
"parent_asin_metrics_number_of_mentions", "browse_node_metrics_occurrence_percentage_all_products",
|
| 3588 |
"review_snippet_1", "review_snippet_2", "review_snippet_3"),
|
| 3589 |
_AMZN + "review_trends": ("sentiment", "topic", "month", "asin_metrics_occurrence_percentage",
|
| 3590 |
"date_range_start_date", "date_range_end_date"),
|
| 3591 |
+
_AMZN + "browse_node_review_topics": ("sort_by", "sentiment", "topic",
|
| 3592 |
+
"browse_node_metrics_occurrence_percentage_all_products",
|
| 3593 |
+
"browse_node_metrics_star_rating_impact_all_products", "our_asin_count", "review_snippet_1"),
|
| 3594 |
_AMZN + "sku_keywords": ("sku", "asin", "month", "search_query", "impressions", "clicks",
|
| 3595 |
"cart_adds", "purchases", "backend_keywords", "source_basis"),
|
| 3596 |
_AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
|
|
|
|
| 3717 |
**{_AMZN + t: "Listings" for t in (
|
| 3718 |
"listings", "pricing", "inactive_listings", "cancelled_listings", "suppressed_listings",
|
| 3719 |
"listing_defects", "sold_listings", "listing_items", "catalog_items", "b2b_recommended",
|
| 3720 |
+
"b2b_not_yet_on_amazon", "geo_opportunities", "b2b_quotes", "review_topics", "review_trends",
|
| 3721 |
+
"browse_node_review_topics")},
|
| 3722 |
**{_AMZN + t: "Advertising" for t in (
|
| 3723 |
"brand_analytics", "search_query_performance", "search_catalog_performance",
|
| 3724 |
"market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
|
platform/harness/amazon_client.py
CHANGED
|
@@ -117,6 +117,8 @@ _entity_pacers = {
|
|
| 117 |
"searchCatalogItems": {"last": None, "lock": threading.Lock()},
|
| 118 |
"getItemReviewTopics": {"last": None, "lock": threading.Lock()},
|
| 119 |
"getItemReviewTrends": {"last": None, "lock": threading.Lock()},
|
|
|
|
|
|
|
| 120 |
"getMyFeesEstimates": {"last": None, "lock": threading.Lock()},
|
| 121 |
"listAllFulfillmentOrders": {"last": None, "lock": threading.Lock()},
|
| 122 |
"getFulfillmentOrder": {"last": None, "lock": threading.Lock()},
|
|
@@ -937,6 +939,16 @@ GROUP_A_TABLES = {
|
|
| 937 |
"snapshot": True,
|
| 938 |
"validates_against": "Amazon raw trend point counts and occurrence percentages per ASIN, sentiment, topic and month vs stored rows for the same pass",
|
| 939 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 940 |
}
|
| 941 |
REPORT_TYPES.update(GROUP_A_TABLES)
|
| 942 |
|
|
@@ -1251,6 +1263,13 @@ OPERATIONS = {
|
|
| 1251 |
"getItemReviewTrends": {"name": "getItemReviewTrends",
|
| 1252 |
"roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
|
| 1253 |
"method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/reviews/trends"},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1254 |
"listAllFulfillmentOrders": {"name": "listAllFulfillmentOrders", "roles": ("Amazon Fulfillment",),
|
| 1255 |
"method": "GET", "path": "fulfillmentOutbound/2020-07-01/fulfillmentOrders"},
|
| 1256 |
"getFulfillmentOrder": {"name": "getFulfillmentOrder", "roles": ("Amazon Fulfillment",),
|
|
@@ -1883,7 +1902,10 @@ def transactions(fields, start, end, *, timeout=_TIMEOUT, sleep=time.sleep, now=
|
|
| 1883 |
ENTITY_INTERVALS = {"getOrderMetrics": 2.0, "getInventorySummaries": 0.5,
|
| 1884 |
"searchListingsItems": 0.2, "searchCatalogItems": 0.5,
|
| 1885 |
"getMyFeesEstimates": 2.0}
|
| 1886 |
-
FEEDBACK_INTERVALS = {"getItemReviewTopics": 1.0, "getItemReviewTrends": 1.0
|
|
|
|
|
|
|
|
|
|
| 1887 |
INBOUND_INTERVALS = {"listInboundPlans": 0.5, "getInboundPlan": 0.5, "listShipmentItems": 0.5,
|
| 1888 |
"getShipment": 0.5, "getShipmentItemsByShipmentId": 0.5, "getShipmentItems": 0.5}
|
| 1889 |
#: Amazon's 400 text for an Amazon Warehousing and Distribution plan in the FBA inbound list.
|
|
@@ -2242,46 +2264,168 @@ def inbound_shipment_items(fields, *, timeout=_TIMEOUT, sleep=time.sleep, now=ti
|
|
| 2242 |
return InboundRecords(records, skipped)
|
| 2243 |
|
| 2244 |
|
| 2245 |
-
def
|
| 2246 |
-
|
| 2247 |
-
"""Read every child ASIN, retaining known absences separately from invalid responses."""
|
| 2248 |
if operation not in FEEDBACK_INTERVALS:
|
| 2249 |
raise AmazonOperationRefused("unapproved Customer Feedback reader")
|
| 2250 |
if not set(OPERATIONS[operation]["roles"]).intersection(ROLES_TAKEN):
|
| 2251 |
raise AmazonOperationRefused("Customer Feedback needs Brand Analytics or Selling Partner Insights")
|
|
|
|
|
|
|
|
|
|
| 2252 |
identifiers = sorted(set(asins))
|
| 2253 |
if any(not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin) for asin in identifiers):
|
| 2254 |
raise AmazonError("Customer Feedback contains an invalid child ASIN. Correct the listings catalogue and retry.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2255 |
if not identifiers:
|
| 2256 |
return []
|
| 2257 |
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2258 |
-
|
| 2259 |
-
|
| 2260 |
-
|
| 2261 |
-
|
| 2262 |
-
|
| 2263 |
-
|
| 2264 |
-
|
| 2265 |
-
|
| 2266 |
-
|
| 2267 |
-
|
| 2268 |
-
|
| 2269 |
-
|
| 2270 |
-
|
| 2271 |
-
|
| 2272 |
-
|
| 2273 |
-
|
| 2274 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2275 |
continue
|
| 2276 |
-
if (not isinstance(body, dict) or
|
| 2277 |
-
or not isinstance(body.get(
|
| 2278 |
-
raise AmazonError(
|
| 2279 |
for key in ("positiveTopics", "negativeTopics"):
|
| 2280 |
-
topics = body[
|
| 2281 |
if topics is not None and (not isinstance(topics, list) or any(not isinstance(t, dict) for t in topics)):
|
| 2282 |
-
raise AmazonError(f"Amazon
|
| 2283 |
-
|
| 2284 |
-
return
|
| 2285 |
|
| 2286 |
|
| 2287 |
def group_a_records(fields, operation, *, start=None, end=None, asins=(),
|
|
@@ -2685,8 +2829,10 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
|
|
| 2685 |
if path.startswith("notifications/v1/"):
|
| 2686 |
reason = (detail or "Amazon refused this notification operation.").replace(token, "[redacted]")
|
| 2687 |
raise AmazonNotificationError(e.code, reason.replace("\u2014", ". ").replace("\u2013", " to ")) from None
|
| 2688 |
-
|
| 2689 |
-
|
|
|
|
|
|
|
| 2690 |
except Exception as e: # noqa: BLE001
|
| 2691 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
| 2692 |
|
|
|
|
| 117 |
"searchCatalogItems": {"last": None, "lock": threading.Lock()},
|
| 118 |
"getItemReviewTopics": {"last": None, "lock": threading.Lock()},
|
| 119 |
"getItemReviewTrends": {"last": None, "lock": threading.Lock()},
|
| 120 |
+
"getItemBrowseNode": {"last": None, "lock": threading.Lock()},
|
| 121 |
+
"getBrowseNodeReviewTopics": {"last": None, "lock": threading.Lock()},
|
| 122 |
"getMyFeesEstimates": {"last": None, "lock": threading.Lock()},
|
| 123 |
"listAllFulfillmentOrders": {"last": None, "lock": threading.Lock()},
|
| 124 |
"getFulfillmentOrder": {"last": None, "lock": threading.Lock()},
|
|
|
|
| 939 |
"snapshot": True,
|
| 940 |
"validates_against": "Amazon raw trend point counts and occurrence percentages per ASIN, sentiment, topic and month vs stored rows for the same pass",
|
| 941 |
},
|
| 942 |
+
"browse_node_review_topics": {
|
| 943 |
+
"label": "Amazon category review topics",
|
| 944 |
+
"api": "getBrowseNodeReviewTopics",
|
| 945 |
+
"role": "Brand Analytics / Selling Partner Insights",
|
| 946 |
+
"role_match": "any",
|
| 947 |
+
"min_interval_s": 7 * 86400,
|
| 948 |
+
"cadence_note": "Amazon refreshes the top 10 positive and negative category topics weekly. Every browse node of every active child ASIN is requested, sorted by mentions and by star rating impact.",
|
| 949 |
+
"snapshot": True,
|
| 950 |
+
"validates_against": "Amazon raw positive and negative topic counts and metrics per browse node and sort vs stored topics for the same pass",
|
| 951 |
+
},
|
| 952 |
}
|
| 953 |
REPORT_TYPES.update(GROUP_A_TABLES)
|
| 954 |
|
|
|
|
| 1263 |
"getItemReviewTrends": {"name": "getItemReviewTrends",
|
| 1264 |
"roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
|
| 1265 |
"method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/reviews/trends"},
|
| 1266 |
+
"getItemBrowseNode": {"name": "getItemBrowseNode",
|
| 1267 |
+
"roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
|
| 1268 |
+
"method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/browseNode"},
|
| 1269 |
+
"getBrowseNodeReviewTopics": {"name": "getBrowseNodeReviewTopics",
|
| 1270 |
+
"roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
|
| 1271 |
+
"method": "GET",
|
| 1272 |
+
"path": "customerFeedback/2024-06-01/browseNodes/{browseNodeId}/reviews/topics"},
|
| 1273 |
"listAllFulfillmentOrders": {"name": "listAllFulfillmentOrders", "roles": ("Amazon Fulfillment",),
|
| 1274 |
"method": "GET", "path": "fulfillmentOutbound/2020-07-01/fulfillmentOrders"},
|
| 1275 |
"getFulfillmentOrder": {"name": "getFulfillmentOrder", "roles": ("Amazon Fulfillment",),
|
|
|
|
| 1902 |
ENTITY_INTERVALS = {"getOrderMetrics": 2.0, "getInventorySummaries": 0.5,
|
| 1903 |
"searchListingsItems": 0.2, "searchCatalogItems": 0.5,
|
| 1904 |
"getMyFeesEstimates": 2.0}
|
| 1905 |
+
FEEDBACK_INTERVALS = {"getItemReviewTopics": 1.0, "getItemReviewTrends": 1.0,
|
| 1906 |
+
"getItemBrowseNode": 1.0, "getBrowseNodeReviewTopics": 1.0}
|
| 1907 |
+
#: getItemReviewTopics and getBrowseNodeReviewTopics accept exactly these sortBy values.
|
| 1908 |
+
FEEDBACK_SORTS = ("MENTIONS", "STAR_RATING_IMPACT")
|
| 1909 |
INBOUND_INTERVALS = {"listInboundPlans": 0.5, "getInboundPlan": 0.5, "listShipmentItems": 0.5,
|
| 1910 |
"getShipment": 0.5, "getShipmentItemsByShipmentId": 0.5, "getShipmentItems": 0.5}
|
| 1911 |
#: Amazon's 400 text for an Amazon Warehousing and Distribution plan in the FBA inbound list.
|
|
|
|
| 2264 |
return InboundRecords(records, skipped)
|
| 2265 |
|
| 2266 |
|
| 2267 |
+
def _feedback_admitted(operation):
|
| 2268 |
+
"""Refuse an unapproved Customer Feedback operation or a missing role before any transport."""
|
|
|
|
| 2269 |
if operation not in FEEDBACK_INTERVALS:
|
| 2270 |
raise AmazonOperationRefused("unapproved Customer Feedback reader")
|
| 2271 |
if not set(OPERATIONS[operation]["roles"]).intersection(ROLES_TAKEN):
|
| 2272 |
raise AmazonOperationRefused("Customer Feedback needs Brand Analytics or Selling Partner Insights")
|
| 2273 |
+
|
| 2274 |
+
|
| 2275 |
+
def _feedback_child_asins(asins):
|
| 2276 |
identifiers = sorted(set(asins))
|
| 2277 |
if any(not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin) for asin in identifiers):
|
| 2278 |
raise AmazonError("Customer Feedback contains an invalid child ASIN. Correct the listings catalogue and retry.")
|
| 2279 |
+
return identifiers
|
| 2280 |
+
|
| 2281 |
+
|
| 2282 |
+
def _feedback_sort(sort_by):
|
| 2283 |
+
if sort_by not in FEEDBACK_SORTS:
|
| 2284 |
+
raise AmazonError("Customer Feedback sortBy must be MENTIONS or STAR_RATING_IMPACT")
|
| 2285 |
+
return sort_by
|
| 2286 |
+
|
| 2287 |
+
|
| 2288 |
+
def _feedback_reads(fields, keys, reads, *, subject, timeout, sleep, now):
|
| 2289 |
+
"""Yield `(key, label, body, reason)` per key and read, interleaved so each operation paces its own bucket.
|
| 2290 |
+
|
| 2291 |
+
A 204, 400 or 404 answers one key: body None and a reason. A 403, an exhausted 429 or any other refusal raises.
|
| 2292 |
+
"""
|
| 2293 |
+
token, issued = access_token(fields, timeout=timeout), now()
|
| 2294 |
+
for key in keys:
|
| 2295 |
+
for label, operation, path, params in reads(key):
|
| 2296 |
+
_pace_entity(operation, FEEDBACK_INTERVALS[operation], sleep=sleep, now=now)
|
| 2297 |
+
if now() - issued >= 3000:
|
| 2298 |
+
token, issued = access_token(fields, timeout=timeout), now()
|
| 2299 |
+
try:
|
| 2300 |
+
body = _request(path, token, region=str(fields.get("region") or DEFAULT_REGION),
|
| 2301 |
+
sandbox=is_sandbox(fields), timeout=timeout,
|
| 2302 |
+
throttle_fix="continue every child ASIN at the Customer Feedback rate limit",
|
| 2303 |
+
**params)
|
| 2304 |
+
except AmazonError as exc:
|
| 2305 |
+
status = getattr(exc, "status", None)
|
| 2306 |
+
if isinstance(exc, (AmazonThrottled, AmazonOperationRefused)) or status not in (400, 404):
|
| 2307 |
+
raise
|
| 2308 |
+
detail = str(getattr(exc, "vendor_message", "") or "").replace("\u2014", ". ").replace("\u2013", " to ")
|
| 2309 |
+
detail = re.sub(r"[.\s]+$", "", detail.strip())
|
| 2310 |
+
yield key, label, None, (f"Amazon {operation} answered {status} for this {subject} and marketplace: "
|
| 2311 |
+
+ (detail or "no reason given") + ".")
|
| 2312 |
+
continue
|
| 2313 |
+
if body is None:
|
| 2314 |
+
yield key, label, None, f"Amazon {operation} returned 204 No Content for this {subject} and marketplace."
|
| 2315 |
+
continue
|
| 2316 |
+
yield key, label, body, None
|
| 2317 |
+
|
| 2318 |
+
|
| 2319 |
+
def _feedback_record(operation, asin, marketplace, body, reason):
|
| 2320 |
+
"""One validated item response, or its absence record."""
|
| 2321 |
+
if reason is not None:
|
| 2322 |
+
return {"asin": asin, "marketplaceId": marketplace, "_no_feedback_reason": reason}
|
| 2323 |
+
root = "topics" if operation == "getItemReviewTopics" else "reviewTrends"
|
| 2324 |
+
if (not isinstance(body, dict) or body.get("asin") != asin or body.get("marketplaceId") != marketplace
|
| 2325 |
+
or not isinstance(body.get(root), dict)):
|
| 2326 |
+
raise AmazonError(f"Amazon {operation} returned an invalid response for the requested child ASIN and marketplace. Retry after Amazon restores its response.")
|
| 2327 |
+
for key in ("positiveTopics", "negativeTopics"):
|
| 2328 |
+
topics = body[root].get(key)
|
| 2329 |
+
if topics is not None and (not isinstance(topics, list) or any(not isinstance(t, dict) for t in topics)):
|
| 2330 |
+
raise AmazonError(f"Amazon {operation} returned invalid {key}. No partial snapshot was stored.")
|
| 2331 |
+
return body
|
| 2332 |
+
|
| 2333 |
+
|
| 2334 |
+
def _browse_node_record(body, reason):
|
| 2335 |
+
"""`{"browse_node_id", "browse_node_name"}` from getItemBrowseNode, or its absence."""
|
| 2336 |
+
if reason is not None:
|
| 2337 |
+
return {"_no_feedback_reason": reason}
|
| 2338 |
+
node, name = (body.get(k) if isinstance(body, dict) else None for k in ("browseNodeId", "displayName"))
|
| 2339 |
+
if not isinstance(node, str) or not re.fullmatch(r"\d{1,20}", node) or (name is not None and not isinstance(name, str)):
|
| 2340 |
+
raise AmazonError("Amazon getItemBrowseNode returned no valid browse node. No partial snapshot was stored.")
|
| 2341 |
+
return {"browse_node_id": node, "browse_node_name": name}
|
| 2342 |
+
|
| 2343 |
+
|
| 2344 |
+
def item_review_feedback(fields, asins, *, operation="getItemReviewTopics", sort_by="MENTIONS", timeout=_TIMEOUT,
|
| 2345 |
+
sleep=time.sleep, now=time.monotonic):
|
| 2346 |
+
"""Read every child ASIN, retaining known absences separately from invalid responses."""
|
| 2347 |
+
if operation not in ("getItemReviewTopics", "getItemReviewTrends"):
|
| 2348 |
+
raise AmazonOperationRefused("unapproved Customer Feedback reader")
|
| 2349 |
+
_feedback_admitted(operation)
|
| 2350 |
+
params = {"sortBy": _feedback_sort(sort_by)} if operation == "getItemReviewTopics" else {}
|
| 2351 |
+
identifiers = _feedback_child_asins(asins)
|
| 2352 |
if not identifiers:
|
| 2353 |
return []
|
| 2354 |
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2355 |
+
path = OPERATIONS[operation]["path"]
|
| 2356 |
+
reads = lambda asin: [(None, operation, path.replace("{asin}", asin), {"marketplaceId": marketplace, **params})]
|
| 2357 |
+
return [_feedback_record(operation, asin, marketplace, body, reason) for asin, _label, body, reason in
|
| 2358 |
+
_feedback_reads(fields, identifiers, reads, subject="ASIN", timeout=timeout, sleep=sleep, now=now)]
|
| 2359 |
+
|
| 2360 |
+
|
| 2361 |
+
def item_browse_nodes(fields, asins, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
|
| 2362 |
+
"""`{asin: {"browse_node_id", "browse_node_name"}}` for every child ASIN; an absence carries `_no_feedback_reason`."""
|
| 2363 |
+
_feedback_admitted("getItemBrowseNode")
|
| 2364 |
+
identifiers = _feedback_child_asins(asins)
|
| 2365 |
+
if not identifiers:
|
| 2366 |
+
return {}
|
| 2367 |
+
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2368 |
+
path = OPERATIONS["getItemBrowseNode"]["path"]
|
| 2369 |
+
reads = lambda asin: [(None, "getItemBrowseNode", path.replace("{asin}", asin), {"marketplaceId": marketplace})]
|
| 2370 |
+
return {asin: _browse_node_record(body, reason) for asin, _label, body, reason in
|
| 2371 |
+
_feedback_reads(fields, identifiers, reads, subject="ASIN", timeout=timeout, sleep=sleep, now=now)}
|
| 2372 |
+
|
| 2373 |
+
|
| 2374 |
+
def item_review_topics(fields, asins, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
|
| 2375 |
+
"""Per child ASIN: its browse node and its topics by both sorts, interleaved on one token.
|
| 2376 |
+
|
| 2377 |
+
Returns `{"MENTIONS": [record], "STAR_RATING_IMPACT": [record], "browse_nodes": {asin: node}}`.
|
| 2378 |
+
"""
|
| 2379 |
+
for operation in ("getItemBrowseNode", "getItemReviewTopics"):
|
| 2380 |
+
_feedback_admitted(operation)
|
| 2381 |
+
identifiers = _feedback_child_asins(asins)
|
| 2382 |
+
out = {**{sort: [] for sort in FEEDBACK_SORTS}, "browse_nodes": {}}
|
| 2383 |
+
if not identifiers:
|
| 2384 |
+
return out
|
| 2385 |
+
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2386 |
+
node_path, topic_path = (OPERATIONS[name]["path"] for name in ("getItemBrowseNode", "getItemReviewTopics"))
|
| 2387 |
+
|
| 2388 |
+
def reads(asin):
|
| 2389 |
+
return [("browse_node", "getItemBrowseNode", node_path.replace("{asin}", asin), {"marketplaceId": marketplace}),
|
| 2390 |
+
*((sort, "getItemReviewTopics", topic_path.replace("{asin}", asin),
|
| 2391 |
+
{"marketplaceId": marketplace, "sortBy": sort}) for sort in FEEDBACK_SORTS)]
|
| 2392 |
+
for asin, label, body, reason in _feedback_reads(fields, identifiers, reads, subject="ASIN",
|
| 2393 |
+
timeout=timeout, sleep=sleep, now=now):
|
| 2394 |
+
if label == "browse_node":
|
| 2395 |
+
out["browse_nodes"][asin] = _browse_node_record(body, reason)
|
| 2396 |
+
else:
|
| 2397 |
+
out[label].append(_feedback_record("getItemReviewTopics", asin, marketplace, body, reason))
|
| 2398 |
+
return out
|
| 2399 |
+
|
| 2400 |
+
|
| 2401 |
+
def browse_node_review_topics(fields, node_ids, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
|
| 2402 |
+
"""Each browse node's top category topics by both sorts: `{sort: [record]}`; an absence carries `_no_feedback_reason`."""
|
| 2403 |
+
_feedback_admitted("getBrowseNodeReviewTopics")
|
| 2404 |
+
nodes = sorted(set(node_ids))
|
| 2405 |
+
# The id becomes a path segment, so only a numeric browse node id is admitted.
|
| 2406 |
+
if any(not isinstance(node, str) or not re.fullmatch(r"\d{1,20}", node) for node in nodes):
|
| 2407 |
+
raise AmazonError("Customer Feedback contains an invalid browse node ID. Retry after Amazon returns a numeric browse node.")
|
| 2408 |
+
out = {sort: [] for sort in FEEDBACK_SORTS}
|
| 2409 |
+
if not nodes:
|
| 2410 |
+
return out
|
| 2411 |
+
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2412 |
+
path = OPERATIONS["getBrowseNodeReviewTopics"]["path"]
|
| 2413 |
+
reads = lambda node: [(sort, "getBrowseNodeReviewTopics", path.replace("{browseNodeId}", node),
|
| 2414 |
+
{"marketplaceId": marketplace, "sortBy": sort}) for sort in FEEDBACK_SORTS]
|
| 2415 |
+
for node, sort, body, reason in _feedback_reads(fields, nodes, reads, subject="browse node",
|
| 2416 |
+
timeout=timeout, sleep=sleep, now=now):
|
| 2417 |
+
if reason is not None:
|
| 2418 |
+
out[sort].append({"browseNodeId": node, "marketplaceId": marketplace, "_no_feedback_reason": reason})
|
| 2419 |
continue
|
| 2420 |
+
if (not isinstance(body, dict) or str(body.get("browseNodeId")) != node
|
| 2421 |
+
or body.get("marketplaceId") != marketplace or not isinstance(body.get("topics"), dict)):
|
| 2422 |
+
raise AmazonError("Amazon getBrowseNodeReviewTopics returned an invalid response for the requested browse node and marketplace. Retry after Amazon restores its response.")
|
| 2423 |
for key in ("positiveTopics", "negativeTopics"):
|
| 2424 |
+
topics = body["topics"].get(key)
|
| 2425 |
if topics is not None and (not isinstance(topics, list) or any(not isinstance(t, dict) for t in topics)):
|
| 2426 |
+
raise AmazonError(f"Amazon getBrowseNodeReviewTopics returned invalid {key}. No partial snapshot was stored.")
|
| 2427 |
+
out[sort].append(body)
|
| 2428 |
+
return out
|
| 2429 |
|
| 2430 |
|
| 2431 |
def group_a_records(fields, operation, *, start=None, end=None, asins=(),
|
|
|
|
| 2829 |
if path.startswith("notifications/v1/"):
|
| 2830 |
reason = (detail or "Amazon refused this notification operation.").replace(token, "[redacted]")
|
| 2831 |
raise AmazonNotificationError(e.code, reason.replace("\u2014", ". ").replace("\u2013", " to ")) from None
|
| 2832 |
+
refusal = AmazonError(f"{_safe(url)} answered {e.code}" + (f": {detail}" if detail else ""))
|
| 2833 |
+
# Callers branch on the status attribute, never on the message text.
|
| 2834 |
+
refusal.status, refusal.vendor_message = e.code, detail
|
| 2835 |
+
raise refusal from None
|
| 2836 |
except Exception as e: # noqa: BLE001
|
| 2837 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
| 2838 |
|
platform/harness/amazon_schema.py
CHANGED
|
@@ -563,10 +563,12 @@ TABLE_SPECS.update({
|
|
| 563 |
for column, target, field, cardinality in _inbound_spec.links), grain="inbound_shipment_item"),
|
| 564 |
"replenishment": _table_spec("replenishment", ("metric_id",),
|
| 565 |
(("sku", "listings", "seller_sku", "many_to_one"),), grain="replenishment_metric"),
|
| 566 |
-
"review_topics": _table_spec("review_topics", ("asin", "marketplace_id", "sentiment", "topic"),
|
| 567 |
-
(("asin", "listings", "asin1", "many_to_one"),), grain="
|
| 568 |
"review_trends": _table_spec("review_trends", ("asin", "marketplace_id", "sentiment", "topic", "month"),
|
| 569 |
(("asin", "listings", "asin1", "many_to_one"),), grain="asin_sentiment_topic_month"),
|
|
|
|
|
|
|
| 570 |
"sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
|
| 571 |
(("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"),
|
| 572 |
"keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
|
|
|
|
| 563 |
for column, target, field, cardinality in _inbound_spec.links), grain="inbound_shipment_item"),
|
| 564 |
"replenishment": _table_spec("replenishment", ("metric_id",),
|
| 565 |
(("sku", "listings", "seller_sku", "many_to_one"),), grain="replenishment_metric"),
|
| 566 |
+
"review_topics": _table_spec("review_topics", ("asin", "marketplace_id", "sort_by", "sentiment", "topic"),
|
| 567 |
+
(("asin", "listings", "asin1", "many_to_one"),), grain="asin_sort_sentiment_topic"),
|
| 568 |
"review_trends": _table_spec("review_trends", ("asin", "marketplace_id", "sentiment", "topic", "month"),
|
| 569 |
(("asin", "listings", "asin1", "many_to_one"),), grain="asin_sentiment_topic_month"),
|
| 570 |
+
"browse_node_review_topics": _table_spec("browse_node_review_topics",
|
| 571 |
+
("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"),
|
| 572 |
"sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
|
| 573 |
(("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"),
|
| 574 |
"keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
|
platform/harness/amazon_store.py
CHANGED
|
@@ -85,6 +85,8 @@ SQP_PASS_REPORTS = 15
|
|
| 85 |
SQP_RESUME_SECONDS = 900
|
| 86 |
GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
|
| 87 |
TABLES += GROUP_A_TABLES
|
|
|
|
|
|
|
| 88 |
DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
|
| 89 |
#: Snapshot reports whose changes are kept as versions in `amzn_<t>_history`.
|
| 90 |
HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES)
|
|
@@ -5143,8 +5145,9 @@ def plain_refusal_reason(reason, *, status="", table="", operation=""):
|
|
| 5143 |
return "The reconciliation check could not run. The last good rows are kept until it runs."
|
| 5144 |
return (f"Reconciliation did not tie: {detail}. The last good rows are kept." if detail
|
| 5145 |
else "Reconciliation did not tie. The last good rows are kept.")
|
| 5146 |
-
feedback = table in
|
| 5147 |
-
subject = "
|
|
|
|
| 5148 |
if feedback and (str(status) in ("403", "NOT_OFFERED")
|
| 5149 |
or re.search(r"answered\s+403\b|\(403\)", text)):
|
| 5150 |
vendor = re.search(r"answered 403(?:: (.*?))?(?:; fix:|$)", text)
|
|
@@ -7034,7 +7037,7 @@ def _sync_problem(exc, table, operation, tenant, rt, report, log):
|
|
| 7034 |
held = all(r.strip() in cx.ROLES_TAKEN
|
| 7035 |
for r in str(role or "").replace(" / ", " + ").split(" + ") if r.strip())
|
| 7036 |
held = held and not re.search(r"missing.*role|not granted|has not granted", str(exc), re.I)
|
| 7037 |
-
fix = ("review Customer Feedback seller and marketplace eligibility and either Brand Analytics or Selling Partner Insights access, then reconnect Amazon" if table in
|
| 7038 |
"review Amazon Fulfillment access for this seller account and reconnect Amazon" if table in ("outbound_fulfillment", "inbound_shipments") else
|
| 7039 |
"review the Pricing role's access for this seller account and reconnect Amazon" if table == "fee_estimates_api" else
|
| 7040 |
f"needs the {role} role; this application already holds it, so Amazon does not "
|
|
@@ -7380,6 +7383,8 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
|
|
| 7380 |
declared.update({name: kind for name, kind in REVIEW_TOPIC_TYPES.items() if name in columns})
|
| 7381 |
if table == "review_trends":
|
| 7382 |
declared.update({name: kind for name, kind in REVIEW_TREND_TYPES.items() if name in columns})
|
|
|
|
|
|
|
| 7383 |
types = _ensure(con, mirror, columns, rows, declared=declared, preserve_rows=True)
|
| 7384 |
predicate = f'"{column}" >= ? AND "{column}" < ?' if end else f'"{column}" = ?'
|
| 7385 |
if column == "posted_date" and end:
|
|
@@ -8039,23 +8044,125 @@ REVIEW_TOPIC_METRICS = (
|
|
| 8039 |
("child_asin_metrics_most_mentions_number_of_mentions", ("childAsinMetrics", "mostMentions", "numberOfMentions")),
|
| 8040 |
("child_asin_metrics_least_mentions_number_of_mentions", ("childAsinMetrics", "leastMentions", "numberOfMentions")),
|
| 8041 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8042 |
REVIEW_TOPIC_TYPES = {
|
| 8043 |
-
**dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "review_snippets", "subtopics",
|
| 8044 |
-
"review_snippet_1", "review_snippet_2", "review_snippet_3"
|
|
|
|
| 8045 |
**{name: "BIGINT" if name.endswith("_number_of_mentions") else "DOUBLE"
|
| 8046 |
for name, _path in REVIEW_TOPIC_METRICS},
|
|
|
|
| 8047 |
}
|
| 8048 |
|
| 8049 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8050 |
def _review_topic_evidence(records, asins, marketplace):
|
| 8051 |
"""Count raw vendor topics and retain their metrics before the row parser runs."""
|
| 8052 |
import math
|
| 8053 |
-
|
|
|
|
|
|
|
| 8054 |
for record in records:
|
| 8055 |
-
asin = record.get("asin")
|
| 8056 |
-
if asin not in expected or asin in seen or record.get("marketplaceId") != marketplace:
|
| 8057 |
raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign ASIN or marketplace")
|
| 8058 |
-
seen.add(asin)
|
| 8059 |
no_content = record.get("_no_feedback_reason")
|
| 8060 |
topics = record.get("topics")
|
| 8061 |
if not no_content and not isinstance(topics, dict):
|
|
@@ -8066,13 +8173,13 @@ def _review_topic_evidence(records, asins, marketplace):
|
|
| 8066 |
group = [] if group is None else group
|
| 8067 |
if not isinstance(group, list):
|
| 8068 |
raise ValueError("Amazon getItemReviewTopics returned an invalid sentiment topic list")
|
| 8069 |
-
counts[json.dumps([asin, sentiment], separators=(",", ":"))] = len(group)
|
| 8070 |
total += len(group)
|
| 8071 |
for topic in group:
|
| 8072 |
if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip()
|
| 8073 |
or not isinstance(topic.get("asinMetrics"), dict)):
|
| 8074 |
raise ValueError("Amazon review topic requires a topic name and asinMetrics object")
|
| 8075 |
-
identity = json.dumps([asin, marketplace, sentiment, topic["topic"]], separators=(",", ":"))
|
| 8076 |
if identity in metrics:
|
| 8077 |
raise ValueError("Amazon returned duplicate topics for one ASIN and sentiment")
|
| 8078 |
values = []
|
|
@@ -8092,7 +8199,8 @@ def _review_topic_evidence(records, asins, marketplace):
|
|
| 8092 |
values.append(value)
|
| 8093 |
metrics[identity] = values
|
| 8094 |
if not total:
|
| 8095 |
-
absent.append({"asin": asin, "
|
|
|
|
| 8096 |
"reason": no_content or "Amazon returned no positive or negative review topics for this ASIN and marketplace."})
|
| 8097 |
if seen != expected:
|
| 8098 |
raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN")
|
|
@@ -8100,50 +8208,52 @@ def _review_topic_evidence(records, asins, marketplace):
|
|
| 8100 |
"marketplace_id": marketplace, "absent_asins": absent}
|
| 8101 |
|
| 8102 |
|
| 8103 |
-
def review_topic_rows(records, document):
|
| 8104 |
"""Flatten positive and negative topics while retaining null metrics and vendor percentage units."""
|
| 8105 |
rows = []
|
| 8106 |
for record in records:
|
|
|
|
| 8107 |
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8108 |
for topic in (record.get("topics") or {}).get(key) or []:
|
| 8109 |
row = dict.fromkeys(REVIEW_TOPIC_TYPES)
|
| 8110 |
row.update(_flatten_row(topic))
|
| 8111 |
-
row.update(asin=record["asin"], marketplace_id=record["marketplaceId"], sentiment=sentiment
|
|
|
|
|
|
|
| 8112 |
row.update(_flatten_row({k: record[k] for k in ("itemName", "countryCode", "dateRange") if k in record}))
|
| 8113 |
-
|
| 8114 |
-
|
| 8115 |
-
|
| 8116 |
-
|
| 8117 |
-
for index, snippet in enumerate(snippets or [], 1):
|
| 8118 |
row[f"review_snippet_{index}"] = snippet
|
|
|
|
| 8119 |
rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTopics"))
|
| 8120 |
return rows
|
| 8121 |
|
| 8122 |
|
| 8123 |
def _sync_review_topics(con, fields):
|
| 8124 |
-
"""Replace the complete
|
| 8125 |
-
|
| 8126 |
-
asins = ([row[0] for row in con.execute(
|
| 8127 |
-
'SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), \'\') IS NOT NULL ORDER BY 1').fetchall()]
|
| 8128 |
-
if "asin1" in columns else [])
|
| 8129 |
marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
|
| 8130 |
-
|
|
|
|
|
|
|
| 8131 |
evidence = _review_topic_evidence(records, asins, marketplace)
|
| 8132 |
document = "operation:" + uuid.uuid4().hex
|
| 8133 |
-
rows = review_topic_rows(records, document)
|
| 8134 |
period = datetime.now(timezone.utc).date().isoformat()
|
| 8135 |
for row in rows:
|
| 8136 |
row["_snapshot_date"] = period
|
| 8137 |
reason = ("Amazon Customer Feedback returned no review topics for this marketplace. "
|
| 8138 |
"The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
|
| 8139 |
-
if asins else
|
| 8140 |
marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
|
| 8141 |
-
"empty_reason": reason if not rows else ""}
|
| 8142 |
count = _replace_period(con, "review_topics", rows,
|
| 8143 |
sorted(set(REVIEW_TOPIC_TYPES) | {key for row in rows for key in row}),
|
| 8144 |
"_snapshot_date", period, marker=marker, snapshot=True)
|
| 8145 |
_record_state(con, entity_key("limit_review_topics"), "done", detail=json.dumps({
|
| 8146 |
-
"reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"]}))
|
| 8147 |
return count
|
| 8148 |
|
| 8149 |
|
|
@@ -8151,12 +8261,50 @@ REVIEW_TREND_TYPES = {
|
|
| 8151 |
**dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "month",
|
| 8152 |
"date_range_start_date", "date_range_end_date"), "VARCHAR"),
|
| 8153 |
"asin_metrics_occurrence_percentage": "DOUBLE",
|
|
|
|
|
|
|
| 8154 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8155 |
|
| 8156 |
|
| 8157 |
def _review_trend_evidence(records, asins, marketplace):
|
| 8158 |
"""Count vendor trend points and retain source values before flattening or storing rows."""
|
| 8159 |
-
import math
|
| 8160 |
expected, seen, counts, metrics, absent = set(asins), set(), {}, {}, []
|
| 8161 |
for record in records:
|
| 8162 |
asin = record.get("asin")
|
|
@@ -8189,22 +8337,12 @@ def _review_trend_evidence(records, asins, marketplace):
|
|
| 8189 |
if (not isinstance(point, dict) or not isinstance(point.get("dateRange"), dict)
|
| 8190 |
or not isinstance(point.get("asinMetrics"), dict)):
|
| 8191 |
raise ValueError("Amazon review trend requires dateRange and asinMetrics objects")
|
| 8192 |
-
start, end = (point["dateRange"]
|
| 8193 |
-
|
| 8194 |
-
for value in (start, end)):
|
| 8195 |
-
raise ValueError("Amazon review trend requires ISO calendar dates")
|
| 8196 |
-
first, last = datetime.strptime(start, "%Y-%m-%d"), datetime.strptime(end, "%Y-%m-%d")
|
| 8197 |
-
if first > last or start[:7] != end[:7]:
|
| 8198 |
-
raise ValueError("Amazon review trend dateRange must identify one calendar month")
|
| 8199 |
-
identity = json.dumps([asin, marketplace, sentiment, topic["topic"], start[:7]],
|
| 8200 |
separators=(",", ":"))
|
| 8201 |
if identity in metrics:
|
| 8202 |
raise ValueError("Amazon returned duplicate trend points for one ASIN, topic and month")
|
| 8203 |
-
|
| 8204 |
-
if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
|
| 8205 |
-
or not math.isfinite(value) or not 0 <= value <= 100):
|
| 8206 |
-
raise ValueError("Amazon review trend returned an invalid occurrence percentage")
|
| 8207 |
-
metrics[identity] = [start, end, value]
|
| 8208 |
total += counts[group]
|
| 8209 |
if not total:
|
| 8210 |
absent.append({"asin": asin, "code": "NO_CONTENT" if no_content else "NO_TRENDS",
|
|
@@ -8222,23 +8360,21 @@ def review_trend_rows(records, document):
|
|
| 8222 |
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8223 |
for topic in (record.get("reviewTrends") or {}).get(key) or []:
|
| 8224 |
for point in topic["trendMetrics"]:
|
| 8225 |
-
|
| 8226 |
row = {"asin": record["asin"], "marketplace_id": record["marketplaceId"],
|
| 8227 |
-
"sentiment": sentiment, "topic": topic["topic"], "month":
|
| 8228 |
-
"date_range_start_date":
|
| 8229 |
-
|
| 8230 |
rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTrends"))
|
| 8231 |
return rows
|
| 8232 |
|
| 8233 |
|
| 8234 |
def _sync_review_trends(con, fields):
|
| 8235 |
"""Replace the full six-month review snapshot with independently counted source evidence."""
|
| 8236 |
-
|
| 8237 |
-
asins = ([row[0] for row in con.execute(
|
| 8238 |
-
'SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), \'\') IS NOT NULL ORDER BY 1').fetchall()]
|
| 8239 |
-
if "asin1" in columns else [])
|
| 8240 |
marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
|
| 8241 |
records = _client().item_review_feedback(fields, asins, operation="getItemReviewTrends")
|
|
|
|
| 8242 |
evidence = _review_trend_evidence(records, asins, marketplace)
|
| 8243 |
rows = review_trend_rows(records, "operation:" + uuid.uuid4().hex)
|
| 8244 |
period = datetime.now(timezone.utc).date().isoformat()
|
|
@@ -8246,13 +8382,144 @@ def _sync_review_trends(con, fields):
|
|
| 8246 |
row["_snapshot_date"] = period
|
| 8247 |
reason = ("Amazon Customer Feedback returned no review trends for this marketplace. "
|
| 8248 |
"The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
|
| 8249 |
-
if asins else
|
| 8250 |
marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
|
| 8251 |
-
"empty_reason": reason if not rows else ""}
|
| 8252 |
count = _replace_period(con, "review_trends", rows, REVIEW_TREND_TYPES,
|
| 8253 |
"_snapshot_date", period, marker=marker, snapshot=True)
|
| 8254 |
_record_state(con, entity_key("limit_review_trends"), "done", detail=json.dumps({
|
| 8255 |
-
"reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"]}))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8256 |
return count
|
| 8257 |
|
| 8258 |
|
|
@@ -8264,6 +8531,8 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log, operations
|
|
| 8264 |
return _sync_review_topics(con, fields)
|
| 8265 |
if table == "review_trends":
|
| 8266 |
return _sync_review_trends(con, fields)
|
|
|
|
|
|
|
| 8267 |
if table == "economics":
|
| 8268 |
start, end = _utc(w0).date().isoformat(), _utc(w1).date().isoformat()
|
| 8269 |
if start >= end:
|
|
@@ -8673,7 +8942,7 @@ def _validate_margin(con, table):
|
|
| 8673 |
identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sentiment"),
|
| 8674 |
row.get("topic"), row.get("month")], separators=(",", ":"))
|
| 8675 |
metrics[identity] = [row.get("date_range_start_date"), row.get("date_range_end_date"),
|
| 8676 |
-
row.get(
|
| 8677 |
total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
|
| 8678 |
trends_match = (marker["count"] == len(rows) == len(metrics) == total
|
| 8679 |
and counts == marker["source_counts"] and metrics == marker["source_metrics"])
|
|
@@ -8684,10 +8953,10 @@ def _validate_margin(con, table):
|
|
| 8684 |
counts = dict.fromkeys(marker["source_counts"], 0)
|
| 8685 |
metrics = {}
|
| 8686 |
for row in rows:
|
| 8687 |
-
group = json.dumps([row.get("asin"), row.get("sentiment")], separators=(",", ":"))
|
| 8688 |
counts[group] = counts.get(group, 0) + 1
|
| 8689 |
-
identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("
|
| 8690 |
-
|
| 8691 |
metrics[identity] = [row.get(name) for name, _path in REVIEW_TOPIC_METRICS]
|
| 8692 |
total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
|
| 8693 |
matches = (marker["count"] == len(rows) == len(metrics) == total
|
|
@@ -8695,6 +8964,22 @@ def _validate_margin(con, table):
|
|
| 8695 |
return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows),
|
| 8696 |
"reason": (marker["empty_reason"] if matches and not rows else
|
| 8697 |
f"Amazon returned {marker['count']} review topics. Stored {len(rows)} topics with per-ASIN, sentiment and metric reconciliation.")}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8698 |
if table == "fee_estimates_api":
|
| 8699 |
expected, actual = marker["count"], len(rows)
|
| 8700 |
identities = {row.get("estimate_id") for row in rows}
|
|
@@ -9428,8 +9713,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9428 |
# A watermark is written when a table's history is complete, so its next window is
|
| 9429 |
# bounded; deferring it too froze every loaded table behind one refused report.
|
| 9430 |
for table in want:
|
|
|
|
| 9431 |
if ((read_watermark(con, table) and (not refresh_loaded or table not in repairs))
|
| 9432 |
-
or table in completed
|
| 9433 |
or (table == "settlements" and not refresh_loaded)):
|
| 9434 |
continue
|
| 9435 |
report["tables"][table] = {"state": "deferred", "reason":
|
|
@@ -9442,7 +9728,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9442 |
_record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail"))
|
| 9443 |
want = [table for table in want if (read_watermark(con, table)
|
| 9444 |
and (not refresh_loaded or table not in repairs))
|
| 9445 |
-
or table in completed
|
| 9446 |
or (table == "settlements" and not refresh_loaded)]
|
| 9447 |
if scheduled and not first_load:
|
| 9448 |
ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
|
|
@@ -10580,7 +10866,7 @@ def _validate_account(tenant_key="royal-imports", rt=None, log=print):
|
|
| 10580 |
out[table] = {"state": "unproven", "reason": "table has never synced",
|
| 10581 |
"reconciles": spec.get("validates_against")}
|
| 10582 |
continue
|
| 10583 |
-
if table in ("fee_estimates_api", "replenishment", "inbound_shipments",
|
| 10584 |
out[table] = {"rows": n, "reconciles": spec.get("validates_against"),
|
| 10585 |
**_validate_margin(con, table)}
|
| 10586 |
continue
|
|
@@ -11266,6 +11552,24 @@ def snapshot_sync_selftest(log=print, check=None):
|
|
| 11266 |
"numberOfMentions": generation[0] + 2, "occurrencePercentage": 12.5,
|
| 11267 |
"starRatingImpact": 0.25}}], "negativeTopics": None}} for asin in asins]
|
| 11268 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11269 |
def snapshot_counts(requests):
|
| 11270 |
return Counter(table for table, _kwargs in requests if table in snapshots)
|
| 11271 |
|
|
@@ -11305,6 +11609,9 @@ def snapshot_sync_selftest(log=print, check=None):
|
|
| 11305 |
patch.object(cx, "outbound_fulfillment_orders", side_effect=outbound_records),
|
| 11306 |
patch.object(cx, "inbound_shipment_items", side_effect=inbound_records),
|
| 11307 |
patch.object(cx, "item_review_feedback", side_effect=feedback_records),
|
|
|
|
|
|
|
|
|
|
| 11308 |
patch.object(cx, "replenishment_metrics", side_effect=replenishment_records),
|
| 11309 |
patch.object(cx, "awd_inventory", return_value=[]),
|
| 11310 |
patch.object(cx, "access_token", side_effect=AssertionError("offline snapshot sync fixture")),
|
|
@@ -12797,6 +13104,16 @@ def group_a_selftest(log=print, check=None):
|
|
| 12797 |
"occurrencePercentage": 12.5, "starRatingImpact": 0.2}}],
|
| 12798 |
"negativeTopics": [{"topic": "Fixture assembly", "asinMetrics": {"numberOfMentions": 2,
|
| 12799 |
"occurrencePercentage": 6.25, "starRatingImpact": -0.1}}]}}]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12800 |
|
| 12801 |
try:
|
| 12802 |
with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \
|
|
@@ -12807,6 +13124,9 @@ def group_a_selftest(log=print, check=None):
|
|
| 12807 |
patch.object(cx, "inbound_shipment_items", return_value=inbound), \
|
| 12808 |
patch.object(cx, "replenishment_metrics", return_value=replenishment), \
|
| 12809 |
patch.object(cx, "item_review_feedback", return_value=feedback), \
|
|
|
|
|
|
|
|
|
|
| 12810 |
patch.object(module, "_fee_estimate_inputs", return_value=[fee_input]), \
|
| 12811 |
patch.object(cx, "fee_estimates", side_effect=lambda _f, estimates: _fixture_fee_records(estimates)):
|
| 12812 |
for table in GROUP_A_TABLES:
|
|
@@ -13037,6 +13357,40 @@ def group_a_selftest(log=print, check=None):
|
|
| 13037 |
finally:
|
| 13038 |
con.close()
|
| 13039 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13040 |
# Client pagination: an empty middle page must not end the traversal.
|
| 13041 |
for operation, interval in cx.ENTITY_INTERVALS.items():
|
| 13042 |
if operation == "getMyFeesEstimates":
|
|
|
|
| 85 |
SQP_RESUME_SECONDS = 900
|
| 86 |
GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
|
| 87 |
TABLES += GROUP_A_TABLES
|
| 88 |
+
#: Customer Feedback entity snapshots: weekly current state, no history window.
|
| 89 |
+
FEEDBACK_TABLES = ("review_topics", "review_trends", "browse_node_review_topics")
|
| 90 |
DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
|
| 91 |
#: Snapshot reports whose changes are kept as versions in `amzn_<t>_history`.
|
| 92 |
HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES)
|
|
|
|
| 5145 |
return "The reconciliation check could not run. The last good rows are kept until it runs."
|
| 5146 |
return (f"Reconciliation did not tie: {detail}. The last good rows are kept." if detail
|
| 5147 |
else "Reconciliation did not tie. The last good rows are kept.")
|
| 5148 |
+
feedback = table in FEEDBACK_TABLES
|
| 5149 |
+
subject = {"review_trends": "review trends",
|
| 5150 |
+
"browse_node_review_topics": "category review topics"}.get(table, "review topics")
|
| 5151 |
if feedback and (str(status) in ("403", "NOT_OFFERED")
|
| 5152 |
or re.search(r"answered\s+403\b|\(403\)", text)):
|
| 5153 |
vendor = re.search(r"answered 403(?:: (.*?))?(?:; fix:|$)", text)
|
|
|
|
| 7037 |
held = all(r.strip() in cx.ROLES_TAKEN
|
| 7038 |
for r in str(role or "").replace(" / ", " + ").split(" + ") if r.strip())
|
| 7039 |
held = held and not re.search(r"missing.*role|not granted|has not granted", str(exc), re.I)
|
| 7040 |
+
fix = ("review Customer Feedback seller and marketplace eligibility and either Brand Analytics or Selling Partner Insights access, then reconnect Amazon" if table in FEEDBACK_TABLES else
|
| 7041 |
"review Amazon Fulfillment access for this seller account and reconnect Amazon" if table in ("outbound_fulfillment", "inbound_shipments") else
|
| 7042 |
"review the Pricing role's access for this seller account and reconnect Amazon" if table == "fee_estimates_api" else
|
| 7043 |
f"needs the {role} role; this application already holds it, so Amazon does not "
|
|
|
|
| 7383 |
declared.update({name: kind for name, kind in REVIEW_TOPIC_TYPES.items() if name in columns})
|
| 7384 |
if table == "review_trends":
|
| 7385 |
declared.update({name: kind for name, kind in REVIEW_TREND_TYPES.items() if name in columns})
|
| 7386 |
+
if table == "browse_node_review_topics":
|
| 7387 |
+
declared.update({name: kind for name, kind in BROWSE_NODE_TOPIC_TYPES.items() if name in columns})
|
| 7388 |
types = _ensure(con, mirror, columns, rows, declared=declared, preserve_rows=True)
|
| 7389 |
predicate = f'"{column}" >= ? AND "{column}" < ?' if end else f'"{column}" = ?'
|
| 7390 |
if column == "posted_date" and end:
|
|
|
|
| 8044 |
("child_asin_metrics_most_mentions_number_of_mentions", ("childAsinMetrics", "mostMentions", "numberOfMentions")),
|
| 8045 |
("child_asin_metrics_least_mentions_number_of_mentions", ("childAsinMetrics", "leastMentions", "numberOfMentions")),
|
| 8046 |
)
|
| 8047 |
+
#: Amazon returns at most five subtopics per topic and three snippets per topic or subtopic.
|
| 8048 |
+
REVIEW_SUBTOPICS, REVIEW_SNIPPETS = 5, 3
|
| 8049 |
+
_SUBTOPIC_TYPES = {
|
| 8050 |
+
**{f"subtopic_{i}": "VARCHAR" for i in range(1, REVIEW_SUBTOPICS + 1)},
|
| 8051 |
+
**{f"subtopic_{i}_number_of_mentions": "BIGINT" for i in range(1, REVIEW_SUBTOPICS + 1)},
|
| 8052 |
+
**{f"subtopic_{i}_occurrence_percentage": "DOUBLE" for i in range(1, REVIEW_SUBTOPICS + 1)},
|
| 8053 |
+
**{f"subtopic_{i}_review_snippet_{j}": "VARCHAR"
|
| 8054 |
+
for i in range(1, REVIEW_SUBTOPICS + 1) for j in range(1, REVIEW_SNIPPETS + 1)},
|
| 8055 |
+
}
|
| 8056 |
REVIEW_TOPIC_TYPES = {
|
| 8057 |
+
**dict.fromkeys(("asin", "marketplace_id", "sort_by", "sentiment", "topic", "review_snippets", "subtopics",
|
| 8058 |
+
"review_snippet_1", "review_snippet_2", "review_snippet_3",
|
| 8059 |
+
"browse_node_id", "browse_node_name"), "VARCHAR"),
|
| 8060 |
**{name: "BIGINT" if name.endswith("_number_of_mentions") else "DOUBLE"
|
| 8061 |
for name, _path in REVIEW_TOPIC_METRICS},
|
| 8062 |
+
**_SUBTOPIC_TYPES,
|
| 8063 |
}
|
| 8064 |
|
| 8065 |
|
| 8066 |
+
def _feedback_asins(con):
|
| 8067 |
+
"""The tenant's active child ASINs: active listings minus every parent in the latest catalogue snapshot."""
|
| 8068 |
+
columns = _columns_of(con, "amzn_listings")
|
| 8069 |
+
if "asin1" not in columns:
|
| 8070 |
+
return []
|
| 8071 |
+
active = " AND LOWER(TRIM(CAST(status AS VARCHAR))) = 'active'" if "status" in columns else ""
|
| 8072 |
+
asins = [row[0] for row in con.execute(
|
| 8073 |
+
"SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), '') IS NOT NULL"
|
| 8074 |
+
+ active + " ORDER BY 1").fetchall()]
|
| 8075 |
+
catalog = _columns_of(con, "amzn_catalog_items")
|
| 8076 |
+
if "parent_asin" not in catalog:
|
| 8077 |
+
return asins
|
| 8078 |
+
other = " AND TRIM(CAST(parent_asin AS VARCHAR)) <> TRIM(CAST(asin AS VARCHAR))" if "asin" in catalog else ""
|
| 8079 |
+
parents = {row[0] for row in con.execute(
|
| 8080 |
+
"SELECT DISTINCT TRIM(CAST(parent_asin AS VARCHAR)) FROM ("
|
| 8081 |
+
+ _latest_account_snapshot_sql(catalog, "amzn_catalog_items")
|
| 8082 |
+
+ ") AS current_catalog WHERE NULLIF(TRIM(CAST(parent_asin AS VARCHAR)), '') IS NOT NULL" + other).fetchall()}
|
| 8083 |
+
return [asin for asin in asins if asin not in parents]
|
| 8084 |
+
|
| 8085 |
+
|
| 8086 |
+
FEEDBACK_NO_ASINS = ("No source rows: the Amazon listings catalogue has no active child ASINs. "
|
| 8087 |
+
"Sync Amazon listings first.")
|
| 8088 |
+
|
| 8089 |
+
|
| 8090 |
+
def _feedback_number(value, *, count=False, percentage=False):
|
| 8091 |
+
"""A Customer Feedback metric: null, or a finite number; a count is a nonnegative integer."""
|
| 8092 |
+
import math
|
| 8093 |
+
if value is None:
|
| 8094 |
+
return None
|
| 8095 |
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
| 8096 |
+
raise ValueError("Amazon review topic returned a nonnumeric metric")
|
| 8097 |
+
if count and (type(value) is not int or value < 0):
|
| 8098 |
+
raise ValueError("Amazon review topic returned an invalid mention count")
|
| 8099 |
+
if percentage and not 0 <= value <= 100:
|
| 8100 |
+
raise ValueError("Amazon review topic returned an invalid occurrence percentage")
|
| 8101 |
+
return value
|
| 8102 |
+
|
| 8103 |
+
|
| 8104 |
+
def _feedback_snippets(snippets):
|
| 8105 |
+
if snippets is not None and (not isinstance(snippets, list) or len(snippets) > REVIEW_SNIPPETS
|
| 8106 |
+
or any(not isinstance(snippet, str) for snippet in snippets)):
|
| 8107 |
+
raise ValueError("Amazon review snippets must be null or a list of up to three strings")
|
| 8108 |
+
return snippets or []
|
| 8109 |
+
|
| 8110 |
+
|
| 8111 |
+
def _subtopic_columns(topic, *, mentions=True):
|
| 8112 |
+
"""Up to five subtopics as columns: name, metrics and up to three snippets each."""
|
| 8113 |
+
subtopics = topic.get("subtopics")
|
| 8114 |
+
if subtopics is not None and (not isinstance(subtopics, list) or len(subtopics) > REVIEW_SUBTOPICS
|
| 8115 |
+
or any(not isinstance(sub, dict) for sub in subtopics)):
|
| 8116 |
+
raise ValueError("Amazon review subtopics must be null or a list of up to five objects")
|
| 8117 |
+
out = {}
|
| 8118 |
+
for i, sub in enumerate(subtopics or [], 1):
|
| 8119 |
+
metrics = sub.get("metrics")
|
| 8120 |
+
if (not isinstance(sub.get("subtopic"), str) or not sub["subtopic"].strip()
|
| 8121 |
+
or (metrics is not None and not isinstance(metrics, dict))):
|
| 8122 |
+
raise ValueError("Amazon review subtopic requires a subtopic name and a metrics object")
|
| 8123 |
+
metrics = metrics or {}
|
| 8124 |
+
out[f"subtopic_{i}"] = sub["subtopic"]
|
| 8125 |
+
if mentions:
|
| 8126 |
+
out[f"subtopic_{i}_number_of_mentions"] = _feedback_number(metrics.get("numberOfMentions"), count=True)
|
| 8127 |
+
out[f"subtopic_{i}_occurrence_percentage"] = _feedback_number(metrics.get("occurrencePercentage"))
|
| 8128 |
+
for j, snippet in enumerate(_feedback_snippets(sub.get("reviewSnippets")), 1):
|
| 8129 |
+
out[f"subtopic_{i}_review_snippet_{j}"] = snippet
|
| 8130 |
+
return out
|
| 8131 |
+
|
| 8132 |
+
|
| 8133 |
+
def _null_unresolved_metrics(records):
|
| 8134 |
+
"""Read an unfilled Amazon placeholder (a "$" string inside a metrics object) as null; return how many."""
|
| 8135 |
+
count = 0
|
| 8136 |
+
|
| 8137 |
+
def walk(node, inside):
|
| 8138 |
+
nonlocal count
|
| 8139 |
+
items = node.items() if isinstance(node, dict) else enumerate(node) if isinstance(node, list) else ()
|
| 8140 |
+
for key, value in list(items):
|
| 8141 |
+
if inside and isinstance(value, str) and value.startswith("$"):
|
| 8142 |
+
node[key] = None
|
| 8143 |
+
count += 1
|
| 8144 |
+
elif isinstance(value, (dict, list)):
|
| 8145 |
+
walk(value, inside or (isinstance(key, str) and key.endswith(("Metrics", "metrics"))))
|
| 8146 |
+
walk(records, False)
|
| 8147 |
+
return count
|
| 8148 |
+
|
| 8149 |
+
|
| 8150 |
+
def _sorted_feedback(read):
|
| 8151 |
+
"""Stamp each record with the sort it answered, in FEEDBACK_SORTS order."""
|
| 8152 |
+
return [{**record, "_sort_by": sort} for sort in _client().FEEDBACK_SORTS for record in read.get(sort) or []]
|
| 8153 |
+
|
| 8154 |
+
|
| 8155 |
def _review_topic_evidence(records, asins, marketplace):
|
| 8156 |
"""Count raw vendor topics and retain their metrics before the row parser runs."""
|
| 8157 |
import math
|
| 8158 |
+
sorts = _client().FEEDBACK_SORTS
|
| 8159 |
+
expected = {(asin, sort) for asin in asins for sort in sorts}
|
| 8160 |
+
counts, metrics, absent, seen = {}, {}, [], set()
|
| 8161 |
for record in records:
|
| 8162 |
+
asin, sort = record.get("asin"), record.get("_sort_by")
|
| 8163 |
+
if (asin, sort) not in expected or (asin, sort) in seen or record.get("marketplaceId") != marketplace:
|
| 8164 |
raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign ASIN or marketplace")
|
| 8165 |
+
seen.add((asin, sort))
|
| 8166 |
no_content = record.get("_no_feedback_reason")
|
| 8167 |
topics = record.get("topics")
|
| 8168 |
if not no_content and not isinstance(topics, dict):
|
|
|
|
| 8173 |
group = [] if group is None else group
|
| 8174 |
if not isinstance(group, list):
|
| 8175 |
raise ValueError("Amazon getItemReviewTopics returned an invalid sentiment topic list")
|
| 8176 |
+
counts[json.dumps([asin, sort, sentiment], separators=(",", ":"))] = len(group)
|
| 8177 |
total += len(group)
|
| 8178 |
for topic in group:
|
| 8179 |
if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip()
|
| 8180 |
or not isinstance(topic.get("asinMetrics"), dict)):
|
| 8181 |
raise ValueError("Amazon review topic requires a topic name and asinMetrics object")
|
| 8182 |
+
identity = json.dumps([asin, marketplace, sort, sentiment, topic["topic"]], separators=(",", ":"))
|
| 8183 |
if identity in metrics:
|
| 8184 |
raise ValueError("Amazon returned duplicate topics for one ASIN and sentiment")
|
| 8185 |
values = []
|
|
|
|
| 8199 |
values.append(value)
|
| 8200 |
metrics[identity] = values
|
| 8201 |
if not total:
|
| 8202 |
+
absent.append({"asin": asin, "sort_by": sort,
|
| 8203 |
+
"code": "NO_CONTENT" if no_content else "NO_TOPICS",
|
| 8204 |
"reason": no_content or "Amazon returned no positive or negative review topics for this ASIN and marketplace."})
|
| 8205 |
if seen != expected:
|
| 8206 |
raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN")
|
|
|
|
| 8208 |
"marketplace_id": marketplace, "absent_asins": absent}
|
| 8209 |
|
| 8210 |
|
| 8211 |
+
def review_topic_rows(records, document, browse_nodes=None):
|
| 8212 |
"""Flatten positive and negative topics while retaining null metrics and vendor percentage units."""
|
| 8213 |
rows = []
|
| 8214 |
for record in records:
|
| 8215 |
+
node = (browse_nodes or {}).get(record["asin"]) or {}
|
| 8216 |
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8217 |
for topic in (record.get("topics") or {}).get(key) or []:
|
| 8218 |
row = dict.fromkeys(REVIEW_TOPIC_TYPES)
|
| 8219 |
row.update(_flatten_row(topic))
|
| 8220 |
+
row.update(asin=record["asin"], marketplace_id=record["marketplaceId"], sentiment=sentiment,
|
| 8221 |
+
sort_by=record.get("_sort_by"), browse_node_id=node.get("browse_node_id"),
|
| 8222 |
+
browse_node_name=node.get("browse_node_name"))
|
| 8223 |
row.update(_flatten_row({k: record[k] for k in ("itemName", "countryCode", "dateRange") if k in record}))
|
| 8224 |
+
# All three Customer Feedback tables store the review period as a calendar date.
|
| 8225 |
+
for name in ("date_range_start_date", "date_range_end_date"):
|
| 8226 |
+
row[name] = str(row[name])[:10] if row.get(name) is not None else None
|
| 8227 |
+
for index, snippet in enumerate(_feedback_snippets(topic.get("reviewSnippets")), 1):
|
|
|
|
| 8228 |
row[f"review_snippet_{index}"] = snippet
|
| 8229 |
+
row.update(_subtopic_columns(topic))
|
| 8230 |
rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTopics"))
|
| 8231 |
return rows
|
| 8232 |
|
| 8233 |
|
| 8234 |
def _sync_review_topics(con, fields):
|
| 8235 |
+
"""Replace the complete active child ASIN topic snapshot, both sorts, with same-pass vendor reconciliation."""
|
| 8236 |
+
asins = _feedback_asins(con)
|
|
|
|
|
|
|
|
|
|
| 8237 |
marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
|
| 8238 |
+
read = _client().item_review_topics(fields, asins)
|
| 8239 |
+
records = _sorted_feedback(read)
|
| 8240 |
+
unresolved = _null_unresolved_metrics(records)
|
| 8241 |
evidence = _review_topic_evidence(records, asins, marketplace)
|
| 8242 |
document = "operation:" + uuid.uuid4().hex
|
| 8243 |
+
rows = review_topic_rows(records, document, read.get("browse_nodes"))
|
| 8244 |
period = datetime.now(timezone.utc).date().isoformat()
|
| 8245 |
for row in rows:
|
| 8246 |
row["_snapshot_date"] = period
|
| 8247 |
reason = ("Amazon Customer Feedback returned no review topics for this marketplace. "
|
| 8248 |
"The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
|
| 8249 |
+
if asins else FEEDBACK_NO_ASINS)
|
| 8250 |
marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
|
| 8251 |
+
"unresolved_metrics": unresolved, "empty_reason": reason if not rows else ""}
|
| 8252 |
count = _replace_period(con, "review_topics", rows,
|
| 8253 |
sorted(set(REVIEW_TOPIC_TYPES) | {key for row in rows for key in row}),
|
| 8254 |
"_snapshot_date", period, marker=marker, snapshot=True)
|
| 8255 |
_record_state(con, entity_key("limit_review_topics"), "done", detail=json.dumps({
|
| 8256 |
+
"reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"], "unresolved_metrics": unresolved}))
|
| 8257 |
return count
|
| 8258 |
|
| 8259 |
|
|
|
|
| 8261 |
**dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "month",
|
| 8262 |
"date_range_start_date", "date_range_end_date"), "VARCHAR"),
|
| 8263 |
"asin_metrics_occurrence_percentage": "DOUBLE",
|
| 8264 |
+
"parent_asin_metrics_occurrence_percentage": "DOUBLE",
|
| 8265 |
+
"browse_node_metrics_occurrence_percentage_all_products": "DOUBLE",
|
| 8266 |
}
|
| 8267 |
+
_TREND_METRICS = (("asin_metrics_occurrence_percentage", ("asinMetrics", "occurrencePercentage")),
|
| 8268 |
+
("parent_asin_metrics_occurrence_percentage", ("parentAsinMetrics", "occurrencePercentage")),
|
| 8269 |
+
("browse_node_metrics_occurrence_percentage_all_products",
|
| 8270 |
+
("browseNodeMetrics", "occurrencePercentage", "allProducts")))
|
| 8271 |
+
|
| 8272 |
+
|
| 8273 |
+
def _trend_window(date_range):
|
| 8274 |
+
"""`(start, end, month)` of one trend point: ISO dates, start before end, month the last one inside the window."""
|
| 8275 |
+
start, end = (date_range.get(name) for name in ("startDate", "endDate"))
|
| 8276 |
+
days = []
|
| 8277 |
+
for value in (start, end):
|
| 8278 |
+
if not isinstance(value, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}(T[0-9:.]+(Z|[+-]\d{2}:\d{2})?)?", value):
|
| 8279 |
+
raise ValueError("Amazon review trend requires ISO calendar dates")
|
| 8280 |
+
try:
|
| 8281 |
+
days.append(datetime.strptime(value[:10], "%Y-%m-%d"))
|
| 8282 |
+
if len(value) > 10:
|
| 8283 |
+
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
| 8284 |
+
except ValueError:
|
| 8285 |
+
raise ValueError("Amazon review trend requires ISO calendar dates") from None
|
| 8286 |
+
if days[0] >= days[1]:
|
| 8287 |
+
raise ValueError("Amazon review trend dateRange must start before it ends")
|
| 8288 |
+
return start[:10], end[:10], (days[1] - timedelta(days=1)).strftime("%Y-%m")
|
| 8289 |
+
|
| 8290 |
+
|
| 8291 |
+
def _trend_values(point):
|
| 8292 |
+
"""The ASIN, parent and browse node occurrence percentages of one trend point."""
|
| 8293 |
+
import math
|
| 8294 |
+
values = []
|
| 8295 |
+
for _name, path in _TREND_METRICS:
|
| 8296 |
+
value = point
|
| 8297 |
+
for segment in path:
|
| 8298 |
+
value = value.get(segment) if isinstance(value, dict) else None
|
| 8299 |
+
if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
|
| 8300 |
+
or not math.isfinite(value) or not 0 <= value <= 100):
|
| 8301 |
+
raise ValueError("Amazon review trend returned an invalid occurrence percentage")
|
| 8302 |
+
values.append(value)
|
| 8303 |
+
return values
|
| 8304 |
|
| 8305 |
|
| 8306 |
def _review_trend_evidence(records, asins, marketplace):
|
| 8307 |
"""Count vendor trend points and retain source values before flattening or storing rows."""
|
|
|
|
| 8308 |
expected, seen, counts, metrics, absent = set(asins), set(), {}, {}, []
|
| 8309 |
for record in records:
|
| 8310 |
asin = record.get("asin")
|
|
|
|
| 8337 |
if (not isinstance(point, dict) or not isinstance(point.get("dateRange"), dict)
|
| 8338 |
or not isinstance(point.get("asinMetrics"), dict)):
|
| 8339 |
raise ValueError("Amazon review trend requires dateRange and asinMetrics objects")
|
| 8340 |
+
start, end, month = _trend_window(point["dateRange"])
|
| 8341 |
+
identity = json.dumps([asin, marketplace, sentiment, topic["topic"], month],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8342 |
separators=(",", ":"))
|
| 8343 |
if identity in metrics:
|
| 8344 |
raise ValueError("Amazon returned duplicate trend points for one ASIN, topic and month")
|
| 8345 |
+
metrics[identity] = [start, end, *_trend_values(point)]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8346 |
total += counts[group]
|
| 8347 |
if not total:
|
| 8348 |
absent.append({"asin": asin, "code": "NO_CONTENT" if no_content else "NO_TRENDS",
|
|
|
|
| 8360 |
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8361 |
for topic in (record.get("reviewTrends") or {}).get(key) or []:
|
| 8362 |
for point in topic["trendMetrics"]:
|
| 8363 |
+
start, end, month = _trend_window(point["dateRange"])
|
| 8364 |
row = {"asin": record["asin"], "marketplace_id": record["marketplaceId"],
|
| 8365 |
+
"sentiment": sentiment, "topic": topic["topic"], "month": month,
|
| 8366 |
+
"date_range_start_date": start, "date_range_end_date": end,
|
| 8367 |
+
**dict(zip((name for name, _path in _TREND_METRICS), _trend_values(point)))}
|
| 8368 |
rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTrends"))
|
| 8369 |
return rows
|
| 8370 |
|
| 8371 |
|
| 8372 |
def _sync_review_trends(con, fields):
|
| 8373 |
"""Replace the full six-month review snapshot with independently counted source evidence."""
|
| 8374 |
+
asins = _feedback_asins(con)
|
|
|
|
|
|
|
|
|
|
| 8375 |
marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
|
| 8376 |
records = _client().item_review_feedback(fields, asins, operation="getItemReviewTrends")
|
| 8377 |
+
unresolved = _null_unresolved_metrics(records)
|
| 8378 |
evidence = _review_trend_evidence(records, asins, marketplace)
|
| 8379 |
rows = review_trend_rows(records, "operation:" + uuid.uuid4().hex)
|
| 8380 |
period = datetime.now(timezone.utc).date().isoformat()
|
|
|
|
| 8382 |
row["_snapshot_date"] = period
|
| 8383 |
reason = ("Amazon Customer Feedback returned no review trends for this marketplace. "
|
| 8384 |
"The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
|
| 8385 |
+
if asins else FEEDBACK_NO_ASINS)
|
| 8386 |
marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
|
| 8387 |
+
"unresolved_metrics": unresolved, "empty_reason": reason if not rows else ""}
|
| 8388 |
count = _replace_period(con, "review_trends", rows, REVIEW_TREND_TYPES,
|
| 8389 |
"_snapshot_date", period, marker=marker, snapshot=True)
|
| 8390 |
_record_state(con, entity_key("limit_review_trends"), "done", detail=json.dumps({
|
| 8391 |
+
"reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"], "unresolved_metrics": unresolved}))
|
| 8392 |
+
return count
|
| 8393 |
+
|
| 8394 |
+
|
| 8395 |
+
BROWSE_NODE_TOPIC_METRICS = (
|
| 8396 |
+
("browse_node_metrics_occurrence_percentage_all_products", ("occurrencePercentage", "allProducts")),
|
| 8397 |
+
("browse_node_metrics_occurrence_percentage_top_twenty_five_percent_products",
|
| 8398 |
+
("occurrencePercentage", "topTwentyFivePercentProducts")),
|
| 8399 |
+
("browse_node_metrics_star_rating_impact_all_products", ("starRatingImpact", "allProducts")),
|
| 8400 |
+
("browse_node_metrics_star_rating_impact_top_twenty_five_percent_products",
|
| 8401 |
+
("starRatingImpact", "topTwentyFivePercentProducts")),
|
| 8402 |
+
)
|
| 8403 |
+
BROWSE_NODE_TOPIC_TYPES = {
|
| 8404 |
+
**dict.fromkeys(("browse_node_id", "browse_node_name", "marketplace_id", "sort_by", "sentiment", "topic",
|
| 8405 |
+
"date_range_start_date", "date_range_end_date", "review_snippet_1", "review_snippet_2",
|
| 8406 |
+
"review_snippet_3", "our_asins"), "VARCHAR"),
|
| 8407 |
+
**dict.fromkeys((name for name, _path in BROWSE_NODE_TOPIC_METRICS), "DOUBLE"),
|
| 8408 |
+
**{name: kind for name, kind in _SUBTOPIC_TYPES.items() if "_number_of_mentions" not in name},
|
| 8409 |
+
"our_asin_count": "BIGINT",
|
| 8410 |
+
}
|
| 8411 |
+
|
| 8412 |
+
|
| 8413 |
+
def _browse_node_topic_evidence(records, nodes, marketplace):
|
| 8414 |
+
"""Count raw category topics per node and sort and retain their metrics before the row parser runs."""
|
| 8415 |
+
sorts = _client().FEEDBACK_SORTS
|
| 8416 |
+
expected = {(node, sort) for node in nodes for sort in sorts}
|
| 8417 |
+
counts, metrics, absent, seen = {}, {}, [], set()
|
| 8418 |
+
for record in records:
|
| 8419 |
+
node, sort = str(record.get("browseNodeId")), record.get("_sort_by")
|
| 8420 |
+
if (node, sort) not in expected or (node, sort) in seen or record.get("marketplaceId") != marketplace:
|
| 8421 |
+
raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign browse node or marketplace")
|
| 8422 |
+
seen.add((node, sort))
|
| 8423 |
+
no_content = record.get("_no_feedback_reason")
|
| 8424 |
+
topics = record.get("topics")
|
| 8425 |
+
if not no_content and not isinstance(topics, dict):
|
| 8426 |
+
raise ValueError("Amazon getBrowseNodeReviewTopics returned no valid topics object")
|
| 8427 |
+
total = 0
|
| 8428 |
+
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8429 |
+
group = None if no_content else topics.get(key)
|
| 8430 |
+
group = [] if group is None else group
|
| 8431 |
+
if not isinstance(group, list):
|
| 8432 |
+
raise ValueError("Amazon getBrowseNodeReviewTopics returned an invalid sentiment topic list")
|
| 8433 |
+
counts[json.dumps([node, sort, sentiment], separators=(",", ":"))] = len(group)
|
| 8434 |
+
total += len(group)
|
| 8435 |
+
for topic in group:
|
| 8436 |
+
if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip()
|
| 8437 |
+
or not isinstance(topic.get("browseNodeMetrics"), dict)):
|
| 8438 |
+
raise ValueError("Amazon category review topic requires a topic name and browseNodeMetrics object")
|
| 8439 |
+
identity = json.dumps([node, marketplace, sort, sentiment, topic["topic"]], separators=(",", ":"))
|
| 8440 |
+
if identity in metrics:
|
| 8441 |
+
raise ValueError("Amazon returned duplicate topics for one browse node and sentiment")
|
| 8442 |
+
values = []
|
| 8443 |
+
for _name, path in BROWSE_NODE_TOPIC_METRICS:
|
| 8444 |
+
value = topic["browseNodeMetrics"]
|
| 8445 |
+
for segment in path:
|
| 8446 |
+
if value is not None and not isinstance(value, dict):
|
| 8447 |
+
raise ValueError("Amazon category review topic returned invalid nested metrics")
|
| 8448 |
+
value = value.get(segment) if value is not None else None
|
| 8449 |
+
values.append(_feedback_number(value))
|
| 8450 |
+
metrics[identity] = values
|
| 8451 |
+
if not total:
|
| 8452 |
+
absent.append({"browse_node_id": node, "sort_by": sort,
|
| 8453 |
+
"code": "NO_CONTENT" if no_content else "NO_TOPICS",
|
| 8454 |
+
"reason": no_content or "Amazon returned no positive or negative category topics for this browse node and marketplace."})
|
| 8455 |
+
if seen != expected:
|
| 8456 |
+
raise ValueError("Amazon Customer Feedback did not return a response for every browse node")
|
| 8457 |
+
return {"count": sum(counts.values()), "source_counts": counts, "source_metrics": metrics,
|
| 8458 |
+
"marketplace_id": marketplace, "absent_browse_nodes": absent}
|
| 8459 |
+
|
| 8460 |
+
|
| 8461 |
+
def browse_node_topic_rows(records, document, our_asins):
|
| 8462 |
+
"""Flatten each browse node's category topics with the tenant's own ASINs in that node."""
|
| 8463 |
+
rows = []
|
| 8464 |
+
for record in records:
|
| 8465 |
+
node = str(record["browseNodeId"])
|
| 8466 |
+
dates = record.get("dateRange") if isinstance(record.get("dateRange"), dict) else {}
|
| 8467 |
+
ours = sorted(our_asins.get(node, {}).get("asins", ()))
|
| 8468 |
+
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8469 |
+
for topic in (record.get("topics") or {}).get(key) or []:
|
| 8470 |
+
row = dict.fromkeys(BROWSE_NODE_TOPIC_TYPES)
|
| 8471 |
+
row.update(browse_node_id=node, marketplace_id=record["marketplaceId"], sort_by=record["_sort_by"],
|
| 8472 |
+
browse_node_name=record.get("displayName") or our_asins.get(node, {}).get("name"),
|
| 8473 |
+
sentiment=sentiment, topic=topic["topic"],
|
| 8474 |
+
date_range_start_date=str(dates.get("startDate") or "")[:10] or None,
|
| 8475 |
+
date_range_end_date=str(dates.get("endDate") or "")[:10] or None,
|
| 8476 |
+
our_asins=json.dumps(ours, separators=(",", ":")), our_asin_count=len(ours))
|
| 8477 |
+
for name, path in BROWSE_NODE_TOPIC_METRICS:
|
| 8478 |
+
value = topic["browseNodeMetrics"]
|
| 8479 |
+
for segment in path:
|
| 8480 |
+
value = value.get(segment) if isinstance(value, dict) else None
|
| 8481 |
+
row[name] = value
|
| 8482 |
+
for index, snippet in enumerate(_feedback_snippets(topic.get("reviewSnippets")), 1):
|
| 8483 |
+
row[f"review_snippet_{index}"] = snippet
|
| 8484 |
+
row.update(_subtopic_columns(topic, mentions=False))
|
| 8485 |
+
rows.append(_origin(row, document, len(rows) + 1, "getBrowseNodeReviewTopics"))
|
| 8486 |
+
return rows
|
| 8487 |
+
|
| 8488 |
+
|
| 8489 |
+
def _sync_browse_node_review_topics(con, fields):
|
| 8490 |
+
"""Replace the category topic snapshot of every browse node the active child ASINs sit in, both sorts."""
|
| 8491 |
+
asins = _feedback_asins(con)
|
| 8492 |
+
marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
|
| 8493 |
+
located = _client().item_browse_nodes(fields, asins)
|
| 8494 |
+
nodes, unplaced = {}, []
|
| 8495 |
+
for asin in asins:
|
| 8496 |
+
node = located.get(asin) or {}
|
| 8497 |
+
if node.get("browse_node_id"):
|
| 8498 |
+
entry = nodes.setdefault(node["browse_node_id"], {"name": node.get("browse_node_name"), "asins": set()})
|
| 8499 |
+
entry["asins"].add(asin)
|
| 8500 |
+
else:
|
| 8501 |
+
unplaced.append({"asin": asin, "code": "NO_BROWSE_NODE",
|
| 8502 |
+
"reason": node.get("_no_feedback_reason") or "Amazon returned no browse node for this ASIN."})
|
| 8503 |
+
records = _sorted_feedback(_client().browse_node_review_topics(fields, sorted(nodes)))
|
| 8504 |
+
unresolved = _null_unresolved_metrics(records)
|
| 8505 |
+
evidence = _browse_node_topic_evidence(records, sorted(nodes), marketplace)
|
| 8506 |
+
rows = browse_node_topic_rows(records, "operation:" + uuid.uuid4().hex, nodes)
|
| 8507 |
+
period = datetime.now(timezone.utc).date().isoformat()
|
| 8508 |
+
for row in rows:
|
| 8509 |
+
row["_snapshot_date"] = period
|
| 8510 |
+
reason = (FEEDBACK_NO_ASINS if not asins else
|
| 8511 |
+
"Amazon returned no browse node for any active child ASIN. The per-ASIN reasons are recorded in the sync evidence."
|
| 8512 |
+
if not nodes else
|
| 8513 |
+
"Amazon Customer Feedback returned no category review topics for these browse nodes. "
|
| 8514 |
+
"The per-node reasons are recorded in the sync evidence. Amazon refreshes this data weekly.")
|
| 8515 |
+
marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence, "absent_asins": unplaced,
|
| 8516 |
+
"unresolved_metrics": unresolved, "empty_reason": reason if not rows else ""}
|
| 8517 |
+
count = _replace_period(con, "browse_node_review_topics", rows,
|
| 8518 |
+
sorted(set(BROWSE_NODE_TOPIC_TYPES) | {key for row in rows for key in row}),
|
| 8519 |
+
"_snapshot_date", period, marker=marker, snapshot=True)
|
| 8520 |
+
_record_state(con, entity_key("limit_browse_node_review_topics"), "done", detail=json.dumps({
|
| 8521 |
+
"reason": marker["empty_reason"], "absent_asins": unplaced,
|
| 8522 |
+
"absent_browse_nodes": evidence["absent_browse_nodes"], "unresolved_metrics": unresolved}))
|
| 8523 |
return count
|
| 8524 |
|
| 8525 |
|
|
|
|
| 8531 |
return _sync_review_topics(con, fields)
|
| 8532 |
if table == "review_trends":
|
| 8533 |
return _sync_review_trends(con, fields)
|
| 8534 |
+
if table == "browse_node_review_topics":
|
| 8535 |
+
return _sync_browse_node_review_topics(con, fields)
|
| 8536 |
if table == "economics":
|
| 8537 |
start, end = _utc(w0).date().isoformat(), _utc(w1).date().isoformat()
|
| 8538 |
if start >= end:
|
|
|
|
| 8942 |
identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sentiment"),
|
| 8943 |
row.get("topic"), row.get("month")], separators=(",", ":"))
|
| 8944 |
metrics[identity] = [row.get("date_range_start_date"), row.get("date_range_end_date"),
|
| 8945 |
+
*(row.get(name) for name, _path in _TREND_METRICS)]
|
| 8946 |
total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
|
| 8947 |
trends_match = (marker["count"] == len(rows) == len(metrics) == total
|
| 8948 |
and counts == marker["source_counts"] and metrics == marker["source_metrics"])
|
|
|
|
| 8953 |
counts = dict.fromkeys(marker["source_counts"], 0)
|
| 8954 |
metrics = {}
|
| 8955 |
for row in rows:
|
| 8956 |
+
group = json.dumps([row.get("asin"), row.get("sort_by"), row.get("sentiment")], separators=(",", ":"))
|
| 8957 |
counts[group] = counts.get(group, 0) + 1
|
| 8958 |
+
identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sort_by"),
|
| 8959 |
+
row.get("sentiment"), row.get("topic")], separators=(",", ":"))
|
| 8960 |
metrics[identity] = [row.get(name) for name, _path in REVIEW_TOPIC_METRICS]
|
| 8961 |
total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
|
| 8962 |
matches = (marker["count"] == len(rows) == len(metrics) == total
|
|
|
|
| 8964 |
return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows),
|
| 8965 |
"reason": (marker["empty_reason"] if matches and not rows else
|
| 8966 |
f"Amazon returned {marker['count']} review topics. Stored {len(rows)} topics with per-ASIN, sentiment and metric reconciliation.")}
|
| 8967 |
+
if table == "browse_node_review_topics":
|
| 8968 |
+
counts = dict.fromkeys(marker["source_counts"], 0)
|
| 8969 |
+
metrics = {}
|
| 8970 |
+
for row in rows:
|
| 8971 |
+
group = json.dumps([row.get("browse_node_id"), row.get("sort_by"), row.get("sentiment")],
|
| 8972 |
+
separators=(",", ":"))
|
| 8973 |
+
counts[group] = counts.get(group, 0) + 1
|
| 8974 |
+
identity = json.dumps([row.get("browse_node_id"), row.get("marketplace_id"), row.get("sort_by"),
|
| 8975 |
+
row.get("sentiment"), row.get("topic")], separators=(",", ":"))
|
| 8976 |
+
metrics[identity] = [row.get(name) for name, _path in BROWSE_NODE_TOPIC_METRICS]
|
| 8977 |
+
total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
|
| 8978 |
+
matches = (marker["count"] == len(rows) == len(metrics) == total
|
| 8979 |
+
and counts == marker["source_counts"] and metrics == marker["source_metrics"])
|
| 8980 |
+
return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows),
|
| 8981 |
+
"reason": (marker["empty_reason"] if matches and not rows else
|
| 8982 |
+
f"Amazon returned {marker['count']} category review topics. Stored {len(rows)} topics with per-node, sort, sentiment and metric reconciliation.")}
|
| 8983 |
if table == "fee_estimates_api":
|
| 8984 |
expected, actual = marker["count"], len(rows)
|
| 8985 |
identities = {row.get("estimate_id") for row in rows}
|
|
|
|
| 9713 |
# A watermark is written when a table's history is complete, so its next window is
|
| 9714 |
# bounded; deferring it too froze every loaded table behind one refused report.
|
| 9715 |
for table in want:
|
| 9716 |
+
# An entity snapshot owes no history, so the first load never holds it.
|
| 9717 |
if ((read_watermark(con, table) and (not refresh_loaded or table not in repairs))
|
| 9718 |
+
or table in completed or table in FEEDBACK_TABLES
|
| 9719 |
or (table == "settlements" and not refresh_loaded)):
|
| 9720 |
continue
|
| 9721 |
report["tables"][table] = {"state": "deferred", "reason":
|
|
|
|
| 9728 |
_record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail"))
|
| 9729 |
want = [table for table in want if (read_watermark(con, table)
|
| 9730 |
and (not refresh_loaded or table not in repairs))
|
| 9731 |
+
or table in completed or table in FEEDBACK_TABLES
|
| 9732 |
or (table == "settlements" and not refresh_loaded)]
|
| 9733 |
if scheduled and not first_load:
|
| 9734 |
ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
|
|
|
|
| 10866 |
out[table] = {"state": "unproven", "reason": "table has never synced",
|
| 10867 |
"reconciles": spec.get("validates_against")}
|
| 10868 |
continue
|
| 10869 |
+
if table in ("fee_estimates_api", "replenishment", "inbound_shipments", *FEEDBACK_TABLES):
|
| 10870 |
out[table] = {"rows": n, "reconciles": spec.get("validates_against"),
|
| 10871 |
**_validate_margin(con, table)}
|
| 10872 |
continue
|
|
|
|
| 11552 |
"numberOfMentions": generation[0] + 2, "occurrencePercentage": 12.5,
|
| 11553 |
"starRatingImpact": 0.25}}], "negativeTopics": None}} for asin in asins]
|
| 11554 |
|
| 11555 |
+
def browse_nodes(_fields, asins):
|
| 11556 |
+
return {asin: {"browse_node_id": "1000", "browse_node_name": "Fixture node"} for asin in asins}
|
| 11557 |
+
|
| 11558 |
+
def review_topic_records(fields, asins):
|
| 11559 |
+
empty = [{"asin": asin, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": {"positiveTopics": None}}
|
| 11560 |
+
for asin in asins]
|
| 11561 |
+
return {"MENTIONS": feedback_records(fields, asins), "STAR_RATING_IMPACT": empty,
|
| 11562 |
+
"browse_nodes": browse_nodes(fields, asins)}
|
| 11563 |
+
|
| 11564 |
+
def category_records(_fields, nodes):
|
| 11565 |
+
calls.append(("browse_node_review_topics", {}))
|
| 11566 |
+
topics = [{"topic": f"Fixture category {generation[0]} {i}", "browseNodeMetrics": {
|
| 11567 |
+
"occurrencePercentage": {"allProducts": 12.5}}} for i in range(size("browse_node_review_topics"))]
|
| 11568 |
+
return {"MENTIONS": [{"browseNodeId": node, "marketplaceId": cx.US_MARKETPLACE_ID,
|
| 11569 |
+
"topics": {"positiveTopics": topics}} for node in nodes],
|
| 11570 |
+
"STAR_RATING_IMPACT": [{"browseNodeId": node, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": {}}
|
| 11571 |
+
for node in nodes]}
|
| 11572 |
+
|
| 11573 |
def snapshot_counts(requests):
|
| 11574 |
return Counter(table for table, _kwargs in requests if table in snapshots)
|
| 11575 |
|
|
|
|
| 11609 |
patch.object(cx, "outbound_fulfillment_orders", side_effect=outbound_records),
|
| 11610 |
patch.object(cx, "inbound_shipment_items", side_effect=inbound_records),
|
| 11611 |
patch.object(cx, "item_review_feedback", side_effect=feedback_records),
|
| 11612 |
+
patch.object(cx, "item_review_topics", side_effect=review_topic_records),
|
| 11613 |
+
patch.object(cx, "item_browse_nodes", side_effect=browse_nodes),
|
| 11614 |
+
patch.object(cx, "browse_node_review_topics", side_effect=category_records),
|
| 11615 |
patch.object(cx, "replenishment_metrics", side_effect=replenishment_records),
|
| 11616 |
patch.object(cx, "awd_inventory", return_value=[]),
|
| 11617 |
patch.object(cx, "access_token", side_effect=AssertionError("offline snapshot sync fixture")),
|
|
|
|
| 13104 |
"occurrencePercentage": 12.5, "starRatingImpact": 0.2}}],
|
| 13105 |
"negativeTopics": [{"topic": "Fixture assembly", "asinMetrics": {"numberOfMentions": 2,
|
| 13106 |
"occurrencePercentage": 6.25, "starRatingImpact": -0.1}}]}}]
|
| 13107 |
+
node = {"browse_node_id": "3734491", "browse_node_name": "Fixture node"}
|
| 13108 |
+
review_topics = {**{sort: feedback for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {"B000000001": node}}
|
| 13109 |
+
category = {sort: [{"browseNodeId": "3734491", "displayName": "Fixture node", "marketplaceId": cx.US_MARKETPLACE_ID,
|
| 13110 |
+
"dateRange": {"startDate": "2026-05-06T00:00:00.000Z", "endDate": "2026-11-06T00:00:00.000Z"},
|
| 13111 |
+
"topics": {"positiveTopics": [{"topic": "Fixture glow", "browseNodeMetrics": {
|
| 13112 |
+
"occurrencePercentage": {"allProducts": 18.15, "topTwentyFivePercentProducts": 22.56},
|
| 13113 |
+
"starRatingImpact": {"allProducts": 1.58, "topTwentyFivePercentProducts": 1.15}}}],
|
| 13114 |
+
"negativeTopics": [{"topic": "Fixture size", "browseNodeMetrics": {
|
| 13115 |
+
"occurrencePercentage": {"allProducts": 14.31}}}]}}]
|
| 13116 |
+
for sort in cx.FEEDBACK_SORTS}
|
| 13117 |
|
| 13118 |
try:
|
| 13119 |
with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \
|
|
|
|
| 13124 |
patch.object(cx, "inbound_shipment_items", return_value=inbound), \
|
| 13125 |
patch.object(cx, "replenishment_metrics", return_value=replenishment), \
|
| 13126 |
patch.object(cx, "item_review_feedback", return_value=feedback), \
|
| 13127 |
+
patch.object(cx, "item_review_topics", return_value=review_topics), \
|
| 13128 |
+
patch.object(cx, "item_browse_nodes", return_value={"B000000001": node}), \
|
| 13129 |
+
patch.object(cx, "browse_node_review_topics", return_value=category), \
|
| 13130 |
patch.object(module, "_fee_estimate_inputs", return_value=[fee_input]), \
|
| 13131 |
patch.object(cx, "fee_estimates", side_effect=lambda _f, estimates: _fixture_fee_records(estimates)):
|
| 13132 |
for table in GROUP_A_TABLES:
|
|
|
|
| 13357 |
finally:
|
| 13358 |
con.close()
|
| 13359 |
|
| 13360 |
+
# A scheduled pass during an unfinished first load still runs a Customer Feedback snapshot.
|
| 13361 |
+
def feedback_pass(exempt):
|
| 13362 |
+
con = duckdb.connect(":memory:")
|
| 13363 |
+
con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
|
| 13364 |
+
"cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
|
| 13365 |
+
rt = _FixtureSyncRuntime("group-a-feedback-fixture", ("review_topics",), "group-a-account")
|
| 13366 |
+
asked = []
|
| 13367 |
+
try:
|
| 13368 |
+
with _fixture_sync_store(con, rt), \
|
| 13369 |
+
patch.object(module, "datetime", Clock), \
|
| 13370 |
+
patch.object(module, "FEEDBACK_TABLES", FEEDBACK_TABLES if exempt else ()), \
|
| 13371 |
+
patch.object(module, "credential", return_value={"fixture": True}), \
|
| 13372 |
+
patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
|
| 13373 |
+
patch.object(module, "_sync_event"), \
|
| 13374 |
+
patch.object(cx, "item_review_topics", side_effect=lambda _f, asins: asked.append(asins) or {
|
| 13375 |
+
**{sort: [] for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {}}), \
|
| 13376 |
+
patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \
|
| 13377 |
+
patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")):
|
| 13378 |
+
with _fixture_account_state(con, rt, write=True) as account:
|
| 13379 |
+
record_watermark(account, "orders", end)
|
| 13380 |
+
_record_state(account, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end))
|
| 13381 |
+
result = sync(rt.key, tables=("review_topics",), rt=rt, log=lambda *_: None, scheduled=True)
|
| 13382 |
+
with _fixture_account_state(con, rt) as account:
|
| 13383 |
+
tick = _read_state(account, entity_key("sync_tick_review_topics"))
|
| 13384 |
+
return asked, (result.get("tables") or {}).get("review_topics") or {}, tick
|
| 13385 |
+
finally:
|
| 13386 |
+
con.close()
|
| 13387 |
+
asked, row, tick = feedback_pass(True)
|
| 13388 |
+
red_asked, red_row, red_tick = feedback_pass(False)
|
| 13389 |
+
ck("a scheduled pass during an unfinished first load runs review topics and records its tick",
|
| 13390 |
+
asked == [[]] and (tick or {}).get("state") == "done" and row.get("state") != "deferred"
|
| 13391 |
+
and not red_asked and not red_tick and red_row.get("state") == "deferred"
|
| 13392 |
+
and "first load" in red_row.get("reason", ""))
|
| 13393 |
+
|
| 13394 |
# Client pagination: an empty middle page must not end the traversal.
|
| 13395 |
for operation, interval in cx.ENTITY_INTERVALS.items():
|
| 13396 |
if operation == "getMyFeesEstimates":
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
2c6ee1c3bc024a5040186bf9860ffadbba1c94b96e8db285633e643b65b87f09
|
web/src/connectors/ConnectorsPage.tsx
CHANGED
|
@@ -263,12 +263,23 @@ export interface AmazonTableSync {
|
|
| 263 |
lastSyncAt: string | null;
|
| 264 |
nextSyncAt: string;
|
| 265 |
perDay: number;
|
|
|
|
|
|
|
| 266 |
state: string;
|
| 267 |
reason?: string;
|
| 268 |
/** A source-internal mismatch: the table published, and both counts are named. */
|
| 269 |
warning?: string;
|
| 270 |
}
|
| 271 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 272 |
export interface AmazonSyncScheduleReading {
|
| 273 |
state: "ready" | "empty" | "unavailable";
|
| 274 |
timezone: string;
|
|
@@ -295,7 +306,9 @@ export function parseAmazonSyncSchedule(body: Record<string, unknown>): AmazonSy
|
|
| 295 |
|| typeof row.account !== "string" || typeof row.state !== "string"
|
| 296 |
|| (row.lastSyncAt !== null && (typeof row.lastSyncAt !== "string" || !Number.isFinite(Date.parse(row.lastSyncAt))))
|
| 297 |
|| typeof row.nextSyncAt !== "string" || !Number.isFinite(Date.parse(row.nextSyncAt))
|
| 298 |
-
|| typeof row.perDay !== "number" || !Number.isInteger(row.perDay) || row.perDay < 1
|
|
|
|
|
|
|
| 299 |
const identity = JSON.stringify([row.connectionId, row.table]);
|
| 300 |
if (identities.has(identity)) return unavailable;
|
| 301 |
identities.add(identity);
|
|
@@ -336,7 +349,7 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
|
|
| 336 |
<table className="conn-settings-databases" aria-label="Amazon table sync schedule">
|
| 337 |
<thead><tr><th scope="col">Table</th>{multipleAccounts ? <th scope="col">Account</th> : null}
|
| 338 |
<th scope="col">Last sync</th><th scope="col">Next sync</th>
|
| 339 |
-
<th scope="col"
|
| 340 |
<tbody>{reading.tables.map((row) => <tr key={JSON.stringify([row.connectionId, row.table])}
|
| 341 |
data-amazon-table={row.table} data-connection-id={row.connectionId} data-table-sync-state={row.state}>
|
| 342 |
<th scope="row">{row.table.replace(/_/g, " ")}</th>
|
|
@@ -351,7 +364,7 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
|
|
| 351 |
: row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
|
| 352 |
: ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
|
| 353 |
</td>
|
| 354 |
-
<td
|
| 355 |
</tr>)}</tbody>
|
| 356 |
</table>
|
| 357 |
</div>
|
|
|
|
| 263 |
lastSyncAt: string | null;
|
| 264 |
nextSyncAt: string;
|
| 265 |
perDay: number;
|
| 266 |
+
/** The CADENCE period. A row without one refreshes daily. */
|
| 267 |
+
period?: AmazonRefreshPeriod;
|
| 268 |
state: string;
|
| 269 |
reason?: string;
|
| 270 |
/** A source-internal mismatch: the table published, and both counts are named. */
|
| 271 |
warning?: string;
|
| 272 |
}
|
| 273 |
|
| 274 |
+
export type AmazonRefreshPeriod = "day" | "week" | "month";
|
| 275 |
+
|
| 276 |
+
/** One plain word for how often a table refreshes. */
|
| 277 |
+
export function amazonRefreshLabel(row: Pick<AmazonTableSync, "perDay" | "period">): string {
|
| 278 |
+
if (row.period === "week") return "Weekly";
|
| 279 |
+
if (row.period === "month") return "Monthly";
|
| 280 |
+
return row.perDay > 1 ? `${row.perDay} times a day` : "Daily";
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
export interface AmazonSyncScheduleReading {
|
| 284 |
state: "ready" | "empty" | "unavailable";
|
| 285 |
timezone: string;
|
|
|
|
| 306 |
|| typeof row.account !== "string" || typeof row.state !== "string"
|
| 307 |
|| (row.lastSyncAt !== null && (typeof row.lastSyncAt !== "string" || !Number.isFinite(Date.parse(row.lastSyncAt))))
|
| 308 |
|| typeof row.nextSyncAt !== "string" || !Number.isFinite(Date.parse(row.nextSyncAt))
|
| 309 |
+
|| typeof row.perDay !== "number" || !Number.isInteger(row.perDay) || row.perDay < 1
|
| 310 |
+
|| (row.period !== undefined
|
| 311 |
+
&& (typeof row.period !== "string" || !["day", "week", "month"].includes(row.period)))) return unavailable;
|
| 312 |
const identity = JSON.stringify([row.connectionId, row.table]);
|
| 313 |
if (identities.has(identity)) return unavailable;
|
| 314 |
identities.add(identity);
|
|
|
|
| 349 |
<table className="conn-settings-databases" aria-label="Amazon table sync schedule">
|
| 350 |
<thead><tr><th scope="col">Table</th>{multipleAccounts ? <th scope="col">Account</th> : null}
|
| 351 |
<th scope="col">Last sync</th><th scope="col">Next sync</th>
|
| 352 |
+
<th scope="col">Refresh</th></tr></thead>
|
| 353 |
<tbody>{reading.tables.map((row) => <tr key={JSON.stringify([row.connectionId, row.table])}
|
| 354 |
data-amazon-table={row.table} data-connection-id={row.connectionId} data-table-sync-state={row.state}>
|
| 355 |
<th scope="row">{row.table.replace(/_/g, " ")}</th>
|
|
|
|
| 364 |
: row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
|
| 365 |
: ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
|
| 366 |
</td>
|
| 367 |
+
<td data-amazon-refresh={row.period ?? "day"}>{amazonRefreshLabel(row)}</td>
|
| 368 |
</tr>)}</tbody>
|
| 369 |
</table>
|
| 370 |
</div>
|