Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +7 -1
- VERSION +1 -1
- api/registry_source.py +15 -4
- api/routes_admin.py +114 -23
- api/routes_automation.py +20 -1
- api/routes_connectors.py +13 -1
- platform/core/connector_spec.py +98 -3
- platform/core/shares.py +13 -1
- platform/core/store_pg.py +2 -1
- platform/harness/amazon_client.py +176 -30
- platform/harness/amazon_schema.py +4 -2
- platform/harness/amazon_store.py +494 -65
- platform/modules/product_data.py +42 -11
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/connectors/ConnectorsPage.tsx +16 -3
RELEASES.json
CHANGED
|
@@ -1,6 +1,12 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
{
|
| 5 |
"version": "v86",
|
| 6 |
"sha": "108bd7510dcd2",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "9507685b7700 (9507685b7700c)",
|
| 3 |
"releases": [
|
| 4 |
+
{
|
| 5 |
+
"version": "v87",
|
| 6 |
+
"sha": "288b6b8793041",
|
| 7 |
+
"date": "2026-11-12",
|
| 8 |
+
"subject": "release(sdlc85): release lessons and log line"
|
| 9 |
+
},
|
| 10 |
{
|
| 11 |
"version": "v86",
|
| 12 |
"sha": "108bd7510dcd2",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
9507685b7700 (9507685b7700c)
|
api/registry_source.py
CHANGED
|
@@ -535,6 +535,7 @@ class CanonicalStore(BatchStore):
|
|
| 535 |
return super().get_projection(name, drop=drop)
|
| 536 |
|
| 537 |
def get(self, name, fresh=False):
|
|
|
|
| 538 |
if name in self.docs and not fresh:
|
| 539 |
return copy.deepcopy(self.docs[name])
|
| 540 |
if name.endswith('_cohorts'):
|
|
@@ -557,10 +558,18 @@ class CanonicalStore(BatchStore):
|
|
| 557 |
value['routes'], value['zones'] = [], []
|
| 558 |
for index, route in saved['routes']:
|
| 559 |
if isinstance(route, dict):
|
| 560 |
-
|
| 561 |
-
|
|
|
|
| 562 |
value['routes'].append(route)
|
| 563 |
value['zones'] = copy.deepcopy(saved['zones'])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 564 |
elif name == 'table_zones':
|
| 565 |
value = {self.table_key: []}
|
| 566 |
for batch in array_values(self._runtime, name, [self.table_key], drop=(
|
|
@@ -877,12 +886,14 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
|
|
| 877 |
raw = [field for field in raw if field.get('key') in required]
|
| 878 |
import routes_keychain
|
| 879 |
import time
|
| 880 |
-
paused = routes_keychain.odoo_paused(session.runtime)
|
| 881 |
pool_key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
|
| 882 |
cached = session.runtime.pool_cache.get(pool_key)
|
|
|
|
|
|
|
|
|
|
| 883 |
# One product assembly per drill: its store reads cost per call, not per row (sdlc85:23).
|
| 884 |
size = None if table_key == 'product_data' else BATCH_ROWS
|
| 885 |
-
if cached and (paused or
|
| 886 |
import itertools
|
| 887 |
def borrowed():
|
| 888 |
source = (r for r in cached[1] if ids is None or r['pid'] in ids)
|
|
|
|
| 535 |
return super().get_projection(name, drop=drop)
|
| 536 |
|
| 537 |
def get(self, name, fresh=False):
|
| 538 |
+
from core import user_tables
|
| 539 |
if name in self.docs and not fresh:
|
| 540 |
return copy.deepcopy(self.docs[name])
|
| 541 |
if name.endswith('_cohorts'):
|
|
|
|
| 558 |
value['routes'], value['zones'] = [], []
|
| 559 |
for index, route in saved['routes']:
|
| 560 |
if isinstance(route, dict):
|
| 561 |
+
# A stop names a customer pid, so a product batch reads no stop membership.
|
| 562 |
+
route = dict(route, stops=[] if self.table_key == 'product_data' else self._selected_json(
|
| 563 |
+
name, ['routes', index, 'stops'], array=True, id_key='customerPid'))
|
| 564 |
value['routes'].append(route)
|
| 565 |
value['zones'] = copy.deepcopy(saved['zones'])
|
| 566 |
+
elif name == user_tables.ODOO_SCHEMA_SNAPSHOT_KEY and self.table_key == 'product_data':
|
| 567 |
+
# Product assembly reads the product.product schema only, never the whole snapshot.
|
| 568 |
+
meta_key = ('product_schema', name)
|
| 569 |
+
if meta_key not in self.metadata or fresh:
|
| 570 |
+
model = paths(self._runtime, name, [['models', 'product.product']])[0]
|
| 571 |
+
self.metadata[meta_key] = {'models': {'product.product': model}} if isinstance(model, dict) else {}
|
| 572 |
+
value = self.metadata[meta_key]
|
| 573 |
elif name == 'table_zones':
|
| 574 |
value = {self.table_key: []}
|
| 575 |
for batch in array_values(self._runtime, name, [self.table_key], drop=(
|
|
|
|
| 886 |
raw = [field for field in raw if field.get('key') in required]
|
| 887 |
import routes_keychain
|
| 888 |
import time
|
|
|
|
| 889 |
pool_key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
|
| 890 |
cached = session.runtime.pool_cache.get(pool_key)
|
| 891 |
+
fresh = bool(cached) and time.time() - cached[0] < 900
|
| 892 |
+
# A fresh pool serves paused or not, so only a stale or absent one reads the connector state.
|
| 893 |
+
paused = not fresh and routes_keychain.odoo_paused(session.runtime)
|
| 894 |
# One product assembly per drill: its store reads cost per call, not per row (sdlc85:23).
|
| 895 |
size = None if table_key == 'product_data' else BATCH_ROWS
|
| 896 |
+
if cached and (paused or fresh):
|
| 897 |
import itertools
|
| 898 |
def borrowed():
|
| 899 |
source = (r for r in cached[1] if ids is None or r['pid'] in ids)
|
api/routes_admin.py
CHANGED
|
@@ -1717,9 +1717,13 @@ def _row_wall_blind_keys(key, session=None, nodes=(), reasons=None, verdicts=Non
|
|
| 1717 |
return blind
|
| 1718 |
|
| 1719 |
|
| 1720 |
-
def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=None
|
|
|
|
| 1721 |
"""Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient.
|
| 1722 |
|
|
|
|
|
|
|
|
|
|
| 1723 |
⭐⭐ W46-T35 — `stored` AND `stale_out` ARE OPTIONAL AND THE DEFAULT IS THE OLD BEHAVIOUR.
|
| 1724 |
`stored` is the block this account ALREADY holds, handed down by `put_perms` from the
|
| 1725 |
registry read it has already made — not a second read at a second moment, which is what the
|
|
@@ -1807,7 +1811,9 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non
|
|
| 1807 |
# measured, not feared. `clean_filter_tree` takes bare strings and cannot see the flag,
|
| 1808 |
# so the narrowing has to happen HERE, at the call.
|
| 1809 |
# Reuse submitted verdicts in the picker vocabulary so each leaf is judged once.
|
| 1810 |
-
fields_here = _module_fields(key, session=session,
|
|
|
|
|
|
|
| 1811 |
valid_keys = {f["key"] for f in fields_here}
|
| 1812 |
filter_keys = {f["key"] for f in fields_here if f.get("filterable")}
|
| 1813 |
# ⭐⭐ W41-T09 / R8 — THE COLUMNS THE ADMIN-OWNER RULE TOOK OUT OF THE BUILDER, READ OFF
|
|
@@ -2240,6 +2246,25 @@ def _prune_to_module(tree, valid_keys):
|
|
| 2240 |
return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None
|
| 2241 |
|
| 2242 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2243 |
@router.get("/admin/users/{username}/perms")
|
| 2244 |
def get_perms(username: str, session: Session = Depends(admin_gate)):
|
| 2245 |
"""This account's permission block plus the field schema the editor needs to render it."""
|
|
@@ -2250,26 +2275,19 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2250 |
# 404, never 403, because "that account exists in another company" is itself a leak.
|
| 2251 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 2252 |
import core.perm_scope as perm_scope
|
| 2253 |
-
from
|
| 2254 |
-
from core import user_tables, shares, freeze_notice
|
| 2255 |
-
from routes_customers import _AssemblyStore
|
| 2256 |
|
| 2257 |
try:
|
| 2258 |
shares.reconcile_database_grants(session.runtime, tenant=session.tenant)
|
| 2259 |
except freeze_notice.DataInputFrozen:
|
| 2260 |
raise
|
| 2261 |
except Exception as exc:
|
| 2262 |
-
|
|
|
|
|
|
|
| 2263 |
|
| 2264 |
-
#
|
| 2265 |
-
|
| 2266 |
-
# writes and their required post-write readback below continue through the original store.
|
| 2267 |
-
session = copy(session)
|
| 2268 |
-
session._permission_roster = reg
|
| 2269 |
-
try:
|
| 2270 |
-
session.runtime = user_tables.lend_defs(_AssemblyStore(session.runtime, schema_only=True))
|
| 2271 |
-
except Exception:
|
| 2272 |
-
raise err(503, "schema_unavailable", "The permission field definitions are unavailable. Try again.")
|
| 2273 |
|
| 2274 |
rec = reg[uname]
|
| 2275 |
stored = rec.get("perms") or {}
|
|
@@ -2342,6 +2360,7 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2342 |
if _gone:
|
| 2343 |
stale_hidden_by_module[k] = _gone
|
| 2344 |
perms_out = {}
|
|
|
|
| 2345 |
for k in governed:
|
| 2346 |
e = stored.get(k)
|
| 2347 |
# ⛔⛔ W36-T22 — THE DEFAULT IS `may_read`, NOT `may_access`, AND THE DIFFERENCE IS AN
|
|
@@ -2376,7 +2395,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2376 |
perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
|
| 2377 |
if (str(k).startswith("ut_") and isinstance(e, dict)
|
| 2378 |
and e.get("access") is True and not _default):
|
| 2379 |
-
|
|
|
|
| 2380 |
# ⭐⭐ W38-T19 — `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN
|
| 2381 |
# BEFORE THE KEY EXISTED. The wire is TOTAL on purpose: the client's own parse already
|
| 2382 |
# reads an absent key as granted, so this line changes no rendering — what it changes is
|
|
@@ -2424,6 +2444,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2424 |
# `is_migrated` slip away from consuming `_fold_legacy_scope`'s one chance, which is
|
| 2425 |
# the BU leak the guard above exists for.
|
| 2426 |
"stale_hidden_by_module": stale_hidden_by_module,
|
|
|
|
|
|
|
| 2427 |
# ⭐⭐ W46-T34 — `{module: {count, refused, status, reason}}`, the metric census
|
| 2428 |
# beside the field list it summarises. `status` is `ok` | `none` | `unavailable`,
|
| 2429 |
# and the third is the one that had no way to be said before: an empty metric list
|
|
@@ -2519,9 +2541,49 @@ def migrate_perms(body: dict = Body(default=None), session: Session = Depends(ad
|
|
| 2519 |
if not report["reconciliation"]["ok"]]}
|
| 2520 |
|
| 2521 |
|
| 2522 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2523 |
"""R15/C5: explicit ut_* access grants editor authority; explicit off removes the share.
|
| 2524 |
|
|
|
|
|
|
|
|
|
|
| 2525 |
Two doors answer two questions: the access rule decides WHICH rows, a `core.shares` `database`
|
| 2526 |
grant decides IF the member sees and edits it, so the editor's toggle writes both. Re-saving
|
| 2527 |
an enabled entry repairs old view-only grants. Other principals, owners and provenance stay
|
|
@@ -2561,10 +2623,25 @@ def _sync_database_shares(session, uname, before, after):
|
|
| 2561 |
if now:
|
| 2562 |
entries.append({**(held[0] if held else {}), "user": uname, "role": "edit"})
|
| 2563 |
try:
|
| 2564 |
-
shares.set_grants("database", key, entries, st=session.runtime,
|
| 2565 |
-
|
| 2566 |
except shares.GrantsChanged as exc:
|
| 2567 |
raise err(409, "shares_changed", "Database sharing changed. Save access again.") from exc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2568 |
|
| 2569 |
|
| 2570 |
def _set_database_request_share(session, key, entries, held, snapshot):
|
|
@@ -2835,9 +2912,14 @@ def put_perms(username: str, body: dict = Body(default=None),
|
|
| 2835 |
# control for it: a report that lives only in a client this ticket does not own would be a
|
| 2836 |
# feature nobody ships [[two-lanes-one-contract-dead-feature]].
|
| 2837 |
_stale = {}
|
|
|
|
|
|
|
|
|
|
| 2838 |
cleaned = _clean_perms(body.get("perms"),
|
| 2839 |
governed_keys={m["key"] for m in _mods}, session=session,
|
| 2840 |
-
stored=reg[uname].get("perms"), stale_out=_stale
|
|
|
|
|
|
|
| 2841 |
governed = {m["key"] for m in _mods}
|
| 2842 |
opaque = {key: entry for key, entry in cleaned.items() if key not in governed}
|
| 2843 |
known = _fold_legacy_scope(reg[uname],
|
|
@@ -2845,6 +2927,7 @@ def put_perms(username: str, body: dict = Body(default=None),
|
|
| 2845 |
session=session)
|
| 2846 |
_guard_bu_widening(reg[uname], known, bool(body.get("confirm_widen")))
|
| 2847 |
cleaned = dict(known, **opaque)
|
|
|
|
| 2848 |
users.set_access(uname, perms=cleaned)
|
| 2849 |
fresh = _registry()
|
| 2850 |
rec = fresh.get(uname) or {}
|
|
@@ -2853,9 +2936,17 @@ def put_perms(username: str, body: dict = Body(default=None),
|
|
| 2853 |
# the wall is up when it is not — the one direction this must never fail in.
|
| 2854 |
raise err(503, "store_unavailable",
|
| 2855 |
"the permissions were not saved; the account is UNCHANGED")
|
| 2856 |
-
|
| 2857 |
-
|
| 2858 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2859 |
return {"username": uname, "perms": rec.get("perms") or {},
|
| 2860 |
"perms_v": int(rec.get("perms_v") or 0),
|
| 2861 |
# W46-T35 — `{module: [key, ...]}`, empty on an ordinary save. What the save cleared
|
|
|
|
| 1717 |
return blind
|
| 1718 |
|
| 1719 |
|
| 1720 |
+
def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=None,
|
| 1721 |
+
fields_session=None, vocabulary_memo=None):
|
| 1722 |
"""Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient.
|
| 1723 |
|
| 1724 |
+
`fields_session` and `vocabulary_memo` are optional: `put_perms` passes the snapshot and memo
|
| 1725 |
+
`get_perms` builds, so the field vocabulary reads each store document once per save.
|
| 1726 |
+
|
| 1727 |
⭐⭐ W46-T35 — `stored` AND `stale_out` ARE OPTIONAL AND THE DEFAULT IS THE OLD BEHAVIOUR.
|
| 1728 |
`stored` is the block this account ALREADY holds, handed down by `put_perms` from the
|
| 1729 |
registry read it has already made — not a second read at a second moment, which is what the
|
|
|
|
| 1811 |
# measured, not feared. `clean_filter_tree` takes bare strings and cannot see the flag,
|
| 1812 |
# so the narrowing has to happen HERE, at the call.
|
| 1813 |
# Reuse submitted verdicts in the picker vocabulary so each leaf is judged once.
|
| 1814 |
+
fields_here = _module_fields(key, session=fields_session or session,
|
| 1815 |
+
wall_verdicts=wall_verdicts,
|
| 1816 |
+
vocabulary_memo=vocabulary_memo)
|
| 1817 |
valid_keys = {f["key"] for f in fields_here}
|
| 1818 |
filter_keys = {f["key"] for f in fields_here if f.get("filterable")}
|
| 1819 |
# ⭐⭐ W41-T09 / R8 — THE COLUMNS THE ADMIN-OWNER RULE TOOK OUT OF THE BUILDER, READ OFF
|
|
|
|
| 2246 |
return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None
|
| 2247 |
|
| 2248 |
|
| 2249 |
+
def _perms_read_snapshot(session, roster):
|
| 2250 |
+
"""A request-only copy of `session` that reads definitions, grants and the roster once.
|
| 2251 |
+
|
| 2252 |
+
The editor's picker and the save's validator both build their field vocabulary through it.
|
| 2253 |
+
It never enters a write door or outlives the request.
|
| 2254 |
+
"""
|
| 2255 |
+
from copy import copy
|
| 2256 |
+
from core import user_tables
|
| 2257 |
+
from routes_customers import _AssemblyStore
|
| 2258 |
+
|
| 2259 |
+
snapshot = copy(session)
|
| 2260 |
+
snapshot._permission_roster = roster
|
| 2261 |
+
try:
|
| 2262 |
+
snapshot.runtime = user_tables.lend_defs(_AssemblyStore(session.runtime, schema_only=True))
|
| 2263 |
+
except Exception:
|
| 2264 |
+
raise err(503, "schema_unavailable", "The permission field definitions are unavailable. Try again.")
|
| 2265 |
+
return snapshot
|
| 2266 |
+
|
| 2267 |
+
|
| 2268 |
@router.get("/admin/users/{username}/perms")
|
| 2269 |
def get_perms(username: str, session: Session = Depends(admin_gate)):
|
| 2270 |
"""This account's permission block plus the field schema the editor needs to render it."""
|
|
|
|
| 2275 |
# 404, never 403, because "that account exists in another company" is itself a leak.
|
| 2276 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 2277 |
import core.perm_scope as perm_scope
|
| 2278 |
+
from core import shares, freeze_notice
|
|
|
|
|
|
|
| 2279 |
|
| 2280 |
try:
|
| 2281 |
shares.reconcile_database_grants(session.runtime, tenant=session.tenant)
|
| 2282 |
except freeze_notice.DataInputFrozen:
|
| 2283 |
raise
|
| 2284 |
except Exception as exc:
|
| 2285 |
+
# Boot retries the repair. The page still opens and names each tick left without a grant.
|
| 2286 |
+
import logging
|
| 2287 |
+
logging.getLogger(__name__).warning("Database grant repair deferred: %s", type(exc).__name__)
|
| 2288 |
|
| 2289 |
+
# Global account writes and their required post-write readback below use the original store.
|
| 2290 |
+
session = _perms_read_snapshot(session, reg)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2291 |
|
| 2292 |
rec = reg[uname]
|
| 2293 |
stored = rec.get("perms") or {}
|
|
|
|
| 2360 |
if _gone:
|
| 2361 |
stale_hidden_by_module[k] = _gone
|
| 2362 |
perms_out = {}
|
| 2363 |
+
unsynced_access = []
|
| 2364 |
for k in governed:
|
| 2365 |
e = stored.get(k)
|
| 2366 |
# ⛔⛔ W36-T22 — THE DEFAULT IS `may_read`, NOT `may_access`, AND THE DIFFERENCE IS AN
|
|
|
|
| 2395 |
perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
|
| 2396 |
if (str(k).startswith("ut_") and isinstance(e, dict)
|
| 2397 |
and e.get("access") is True and not _default):
|
| 2398 |
+
# A tick whose share grant is missing still opens, so an admin can see it and re-save.
|
| 2399 |
+
unsynced_access.append(k)
|
| 2400 |
# ⭐⭐ W38-T19 — `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN
|
| 2401 |
# BEFORE THE KEY EXISTED. The wire is TOTAL on purpose: the client's own parse already
|
| 2402 |
# reads an absent key as granted, so this line changes no rendering — what it changes is
|
|
|
|
| 2444 |
# `is_migrated` slip away from consuming `_fold_legacy_scope`'s one chance, which is
|
| 2445 |
# the BU leak the guard above exists for.
|
| 2446 |
"stale_hidden_by_module": stale_hidden_by_module,
|
| 2447 |
+
# The `ut_*` keys stored as access while the account cannot open them: a save repairs them.
|
| 2448 |
+
"unsynced_access": unsynced_access,
|
| 2449 |
# ⭐⭐ W46-T34 — `{module: {count, refused, status, reason}}`, the metric census
|
| 2450 |
# beside the field list it summarises. `status` is `ok` | `none` | `unavailable`,
|
| 2451 |
# and the third is the one that had no way to be said before: an empty metric list
|
|
|
|
| 2541 |
if not report["reconciliation"]["ok"]]}
|
| 2542 |
|
| 2543 |
|
| 2544 |
+
# The share sync writes one grant per changed database, so a save starts writing only with this left.
|
| 2545 |
+
_PERMS_WRITE_RESERVE_S = 20.0
|
| 2546 |
+
|
| 2547 |
+
|
| 2548 |
+
def _require_write_budget():
|
| 2549 |
+
"""Refuse before the first write when the request deadline could fall between the two halves."""
|
| 2550 |
+
from core import request_timing
|
| 2551 |
+
try:
|
| 2552 |
+
left = request_timing.remaining_budget()
|
| 2553 |
+
except Exception:
|
| 2554 |
+
left = 0.0
|
| 2555 |
+
if left is not None and left < _PERMS_WRITE_RESERVE_S:
|
| 2556 |
+
raise err(503, "server_busy", "Access was not saved. Try again in a moment.")
|
| 2557 |
+
|
| 2558 |
+
|
| 2559 |
+
def _restore_perms(uname, prior, written):
|
| 2560 |
+
"""Put back the perms block a failed share sync left, unless another save replaced it since."""
|
| 2561 |
+
from core import account_rows
|
| 2562 |
+
|
| 2563 |
+
def _set(reg):
|
| 2564 |
+
current = reg.get(uname)
|
| 2565 |
+
if isinstance(current, dict) and current.get("perms") == written.get("perms"):
|
| 2566 |
+
for field in ("perms", "perms_v"):
|
| 2567 |
+
if field in prior:
|
| 2568 |
+
current[field] = deepcopy(prior[field])
|
| 2569 |
+
else:
|
| 2570 |
+
current.pop(field, None)
|
| 2571 |
+
return reg
|
| 2572 |
+
|
| 2573 |
+
try:
|
| 2574 |
+
account_rows.update(uname, _set, st=users.store)
|
| 2575 |
+
except Exception as exc:
|
| 2576 |
+
import logging
|
| 2577 |
+
logging.getLogger(__name__).warning("Access restore after a failed share sync failed: %s",
|
| 2578 |
+
type(exc).__name__)
|
| 2579 |
+
|
| 2580 |
+
|
| 2581 |
+
def _sync_database_shares(session, uname, before, after, applied=None):
|
| 2582 |
"""R15/C5: explicit ut_* access grants editor authority; explicit off removes the share.
|
| 2583 |
|
| 2584 |
+
`applied`, when given, collects `(key, entries before, entries stored)` per write, so a
|
| 2585 |
+
caller whose save fails later can revert exactly these grants.
|
| 2586 |
+
|
| 2587 |
Two doors answer two questions: the access rule decides WHICH rows, a `core.shares` `database`
|
| 2588 |
grant decides IF the member sees and edits it, so the editor's toggle writes both. Re-saving
|
| 2589 |
an enabled entry repairs old view-only grants. Other principals, owners and provenance stay
|
|
|
|
| 2623 |
if now:
|
| 2624 |
entries.append({**(held[0] if held else {}), "user": uname, "role": "edit"})
|
| 2625 |
try:
|
| 2626 |
+
stored = shares.set_grants("database", key, entries, st=session.runtime,
|
| 2627 |
+
granter=session.uname, expect=record["entries"])
|
| 2628 |
except shares.GrantsChanged as exc:
|
| 2629 |
raise err(409, "shares_changed", "Database sharing changed. Save access again.") from exc
|
| 2630 |
+
if applied is not None:
|
| 2631 |
+
applied.append((key, record["entries"], stored["entries"]))
|
| 2632 |
+
|
| 2633 |
+
|
| 2634 |
+
def _revert_shares(session, applied):
|
| 2635 |
+
"""Undo the grants a failed save wrote, newest first, each only while it still holds them."""
|
| 2636 |
+
from core import shares
|
| 2637 |
+
|
| 2638 |
+
for key, before, written in reversed(applied):
|
| 2639 |
+
try:
|
| 2640 |
+
shares.set_grants("database", key, before, st=session.runtime, expect=written)
|
| 2641 |
+
except Exception as exc:
|
| 2642 |
+
import logging
|
| 2643 |
+
logging.getLogger(__name__).warning("Grant revert after a failed save failed: %s",
|
| 2644 |
+
type(exc).__name__)
|
| 2645 |
|
| 2646 |
|
| 2647 |
def _set_database_request_share(session, key, entries, held, snapshot):
|
|
|
|
| 2912 |
# control for it: a report that lives only in a client this ticket does not own would be a
|
| 2913 |
# feature nobody ships [[two-lanes-one-contract-dead-feature]].
|
| 2914 |
_stale = {}
|
| 2915 |
+
prior = deepcopy(reg[uname])
|
| 2916 |
+
# Without the snapshot each database re-read the roster and schema: about 44 s on live, past
|
| 2917 |
+
# the 45 s request deadline, so the save failed after the perms block was written.
|
| 2918 |
cleaned = _clean_perms(body.get("perms"),
|
| 2919 |
governed_keys={m["key"] for m in _mods}, session=session,
|
| 2920 |
+
stored=reg[uname].get("perms"), stale_out=_stale,
|
| 2921 |
+
fields_session=_perms_read_snapshot(session, reg),
|
| 2922 |
+
vocabulary_memo={}) or {}
|
| 2923 |
governed = {m["key"] for m in _mods}
|
| 2924 |
opaque = {key: entry for key, entry in cleaned.items() if key not in governed}
|
| 2925 |
known = _fold_legacy_scope(reg[uname],
|
|
|
|
| 2927 |
session=session)
|
| 2928 |
_guard_bu_widening(reg[uname], known, bool(body.get("confirm_widen")))
|
| 2929 |
cleaned = dict(known, **opaque)
|
| 2930 |
+
_require_write_budget()
|
| 2931 |
users.set_access(uname, perms=cleaned)
|
| 2932 |
fresh = _registry()
|
| 2933 |
rec = fresh.get(uname) or {}
|
|
|
|
| 2936 |
# the wall is up when it is not — the one direction this must never fail in.
|
| 2937 |
raise err(503, "store_unavailable",
|
| 2938 |
"the permissions were not saved; the account is UNCHANGED")
|
| 2939 |
+
applied = []
|
| 2940 |
+
try:
|
| 2941 |
+
_sync_database_shares(session, uname, prior.get("perms") or {},
|
| 2942 |
+
{key: entry for key, entry in (rec.get("perms") or {}).items()
|
| 2943 |
+
if key in governed}, applied=applied)
|
| 2944 |
+
except Exception:
|
| 2945 |
+
# Both halves land or neither: undo this save's grants, then put the old block back. If
|
| 2946 |
+
# those writes fail too, the grant repair in `get_perms` and at boot follows the ticks.
|
| 2947 |
+
_revert_shares(session, applied)
|
| 2948 |
+
_restore_perms(uname, prior, rec)
|
| 2949 |
+
raise
|
| 2950 |
return {"username": uname, "perms": rec.get("perms") or {},
|
| 2951 |
"perms_v": int(rec.get("perms_v") or 0),
|
| 2952 |
# W46-T35 — `{module: [key, ...]}`, empty on an ordinary save. What the save cleared
|
api/routes_automation.py
CHANGED
|
@@ -2863,17 +2863,36 @@ if engine.grid_hook not in _ut_hooks.ROW_HOOKS:
|
|
| 2863 |
_ut_hooks.ROW_HOOKS.append(engine.grid_hook)
|
| 2864 |
|
| 2865 |
|
|
|
|
|
|
|
|
|
|
| 2866 |
def _received_projection(connector):
|
| 2867 |
"""Reuse each connected grid's canonical identities and field projection at publication."""
|
| 2868 |
def resolve(rt, cursor):
|
|
|
|
|
|
|
| 2869 |
from core import user_tables
|
| 2870 |
import routes_odoo_tables
|
| 2871 |
result = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2872 |
prefix = 'ut_amzn_' if connector == 'amazon' else 'ut_odoo_'
|
| 2873 |
for key in user_tables.all_defs(st=rt, schema_only=True):
|
| 2874 |
if not str(key).startswith(prefix):
|
| 2875 |
continue
|
| 2876 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2877 |
if spec and not spec.get('missing'):
|
| 2878 |
result[key] = spec
|
| 2879 |
return result
|
|
|
|
| 2863 |
_ut_hooks.ROW_HOOKS.append(engine.grid_hook)
|
| 2864 |
|
| 2865 |
|
| 2866 |
+
_RECEIVED_NOT_READY = frozenset({'amazon_mirror_not_loaded', 'store_not_ready'})
|
| 2867 |
+
|
| 2868 |
+
|
| 2869 |
def _received_projection(connector):
|
| 2870 |
"""Reuse each connected grid's canonical identities and field projection at publication."""
|
| 2871 |
def resolve(rt, cursor):
|
| 2872 |
+
import logging
|
| 2873 |
+
from fastapi import HTTPException
|
| 2874 |
from core import user_tables
|
| 2875 |
import routes_odoo_tables
|
| 2876 |
result = {}
|
| 2877 |
+
if connector == 'amazon':
|
| 2878 |
+
# Amazon specs raise BinderException on a publication's raw views and cost about 7 s
|
| 2879 |
+
# a table in the worker hand-off (sdlc85:88), so the projection waits for that fix.
|
| 2880 |
+
logging.getLogger(__name__).debug('received-change projection skips amazon until its specs bind')
|
| 2881 |
+
return result
|
| 2882 |
prefix = 'ut_amzn_' if connector == 'amazon' else 'ut_odoo_'
|
| 2883 |
for key in user_tables.all_defs(st=rt, schema_only=True):
|
| 2884 |
if not str(key).startswith(prefix):
|
| 2885 |
continue
|
| 2886 |
+
try:
|
| 2887 |
+
spec = routes_odoo_tables._source_for(cursor, key, allow_lookups=False)
|
| 2888 |
+
except HTTPException as exc:
|
| 2889 |
+
# A table still loading is skipped like a missing one, so it cannot drop its siblings.
|
| 2890 |
+
detail = exc.detail if isinstance(exc.detail, dict) else {}
|
| 2891 |
+
code = (detail.get('error') or {}).get('code')
|
| 2892 |
+
if code not in _RECEIVED_NOT_READY:
|
| 2893 |
+
raise
|
| 2894 |
+
logging.getLogger(__name__).debug('received-change projection skips %s (%s)', key, code)
|
| 2895 |
+
continue
|
| 2896 |
if spec and not spec.get('missing'):
|
| 2897 |
result[key] = spec
|
| 2898 |
return result
|
api/routes_connectors.py
CHANGED
|
@@ -3167,6 +3167,13 @@ def _amazon_cause_text(text, limit=300):
|
|
| 3167 |
return text
|
| 3168 |
|
| 3169 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3170 |
def amazon_hold_cause(row=None, held="", *, table=""):
|
| 3171 |
"""One plain cause for an Amazon table a pass did not finish, from its report row or held text."""
|
| 3172 |
from harness.amazon_store import plain_refusal_reason
|
|
@@ -3187,6 +3194,10 @@ def amazon_hold_cause(row=None, held="", *, table=""):
|
|
| 3187 |
cause = "Amazon is still preparing this report."
|
| 3188 |
elif state == "deferred" or (not row and (not held or held.startswith("not reached"))):
|
| 3189 |
cause = "This pass ended before this table's turn. The next pass continues it."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3190 |
elif state == "never_run":
|
| 3191 |
# A table with no tick is due only in the minute after its daily slot, which a busy pass misses.
|
| 3192 |
cause = "This table has never synced. Its daily start window is too short."
|
|
@@ -3306,7 +3317,8 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
|
|
| 3306 |
reason = row_reason(table, state)
|
| 3307 |
# A source-internal mismatch published the table. Its warning stays until a clean reconcile.
|
| 3308 |
warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
|
| 3309 |
-
schedule.append({**row, "
|
|
|
|
| 3310 |
"lastSyncAt": (completed.get("updated_at")
|
| 3311 |
if completed.get("state") == "done" else None),
|
| 3312 |
"state": state, **({"reason": reason} if reason else {}),
|
|
|
|
| 3167 |
return text
|
| 3168 |
|
| 3169 |
|
| 3170 |
+
def _amazon_table_period(table):
|
| 3171 |
+
"""The table's refresh period from the client CADENCE: "day", "week" or "month"."""
|
| 3172 |
+
from harness import amazon_store as az
|
| 3173 |
+
period = (az._client().CADENCE.get(str(table or "")) or {}).get("period")
|
| 3174 |
+
return period if period in ("day", "week", "month") else "day"
|
| 3175 |
+
|
| 3176 |
+
|
| 3177 |
def amazon_hold_cause(row=None, held="", *, table=""):
|
| 3178 |
"""One plain cause for an Amazon table a pass did not finish, from its report row or held text."""
|
| 3179 |
from harness.amazon_store import plain_refusal_reason
|
|
|
|
| 3194 |
cause = "Amazon is still preparing this report."
|
| 3195 |
elif state == "deferred" or (not row and (not held or held.startswith("not reached"))):
|
| 3196 |
cause = "This pass ended before this table's turn. The next pass continues it."
|
| 3197 |
+
elif state == "never_run" and _amazon_table_period(table) in ("week", "month"):
|
| 3198 |
+
# A weekly or monthly table waits for its first calendar period, so a missing tick is expected.
|
| 3199 |
+
cause = (f"This table has not synced yet. Its first "
|
| 3200 |
+
f"{'weekly' if _amazon_table_period(table) == 'week' else 'monthly'} pass is scheduled.")
|
| 3201 |
elif state == "never_run":
|
| 3202 |
# A table with no tick is due only in the minute after its daily slot, which a busy pass misses.
|
| 3203 |
cause = "This table has never synced. Its daily start window is too short."
|
|
|
|
| 3317 |
reason = row_reason(table, state)
|
| 3318 |
# A source-internal mismatch published the table. Its warning stays until a clean reconcile.
|
| 3319 |
warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
|
| 3320 |
+
schedule.append({**row, "period": _amazon_table_period(table),
|
| 3321 |
+
"connectionId": cid, "account": entry["label"],
|
| 3322 |
"lastSyncAt": (completed.get("updated_at")
|
| 3323 |
if completed.get("state") == "done" else None),
|
| 3324 |
"state": state, **({"reason": reason} if reason else {}),
|
platform/core/connector_spec.py
CHANGED
|
@@ -815,7 +815,7 @@ AMAZON_TABLES = (
|
|
| 815 |
ConnectorTableSpec(
|
| 816 |
key=_AMZN + "review_topics", native="customerFeedback/2024-06-01/items/{asin}/reviews/topics",
|
| 817 |
source="amzn_review_topics", label="Amazon Listings: Review topics", pinned="asin",
|
| 818 |
-
identity=("asin", "marketplace_id", "sentiment", "topic"),
|
| 819 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 820 |
),
|
| 821 |
ConnectorTableSpec(
|
|
@@ -824,6 +824,13 @@ AMAZON_TABLES = (
|
|
| 824 |
identity=("asin", "marketplace_id", "sentiment", "topic", "month"),
|
| 825 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 826 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 827 |
ConnectorTableSpec(
|
| 828 |
key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance",
|
| 829 |
source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku",
|
|
@@ -3306,6 +3313,39 @@ AMAZON_LABELS = {
|
|
| 3306 |
"review_snippet_3": "Review snippet 3",
|
| 3307 |
"review_snippets": "Review snippets",
|
| 3308 |
"subtopics": "Subtopics",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3309 |
},
|
| 3310 |
_AMZN + "review_trends": {
|
| 3311 |
"_data_origin": "Data origin",
|
|
@@ -3321,6 +3361,57 @@ AMAZON_LABELS = {
|
|
| 3321 |
"date_range_start_date": "Review period start",
|
| 3322 |
"date_range_end_date": "Review period end",
|
| 3323 |
"asin_metrics_occurrence_percentage": "ASIN occurrence",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3324 |
},
|
| 3325 |
_AMZN + "replenishment": {
|
| 3326 |
"_data_origin": "Data origin",
|
|
@@ -3491,12 +3582,15 @@ AMAZON_DEFAULT_COLUMNS = {
|
|
| 3491 |
_AMZN + "replenishment": ("asin", "time_interval_start_date", "time_interval_end_date",
|
| 3492 |
"active_subscriptions", "shipped_subscription_units", "total_subscriptions_revenue",
|
| 3493 |
"revenue_penetration", "subscriber_retention_for30_days", "program_type"),
|
| 3494 |
-
_AMZN + "review_topics": ("sentiment", "topic", "asin_metrics_number_of_mentions",
|
| 3495 |
"asin_metrics_occurrence_percentage", "asin_metrics_star_rating_impact",
|
| 3496 |
"parent_asin_metrics_number_of_mentions", "browse_node_metrics_occurrence_percentage_all_products",
|
| 3497 |
"review_snippet_1", "review_snippet_2", "review_snippet_3"),
|
| 3498 |
_AMZN + "review_trends": ("sentiment", "topic", "month", "asin_metrics_occurrence_percentage",
|
| 3499 |
"date_range_start_date", "date_range_end_date"),
|
|
|
|
|
|
|
|
|
|
| 3500 |
_AMZN + "sku_keywords": ("sku", "asin", "month", "search_query", "impressions", "clicks",
|
| 3501 |
"cart_adds", "purchases", "backend_keywords", "source_basis"),
|
| 3502 |
_AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
|
|
@@ -3623,7 +3717,8 @@ AMAZON_GROUPS = {
|
|
| 3623 |
**{_AMZN + t: "Listings" for t in (
|
| 3624 |
"listings", "pricing", "inactive_listings", "cancelled_listings", "suppressed_listings",
|
| 3625 |
"listing_defects", "sold_listings", "listing_items", "catalog_items", "b2b_recommended",
|
| 3626 |
-
"b2b_not_yet_on_amazon", "geo_opportunities", "b2b_quotes", "review_topics", "review_trends"
|
|
|
|
| 3627 |
**{_AMZN + t: "Advertising" for t in (
|
| 3628 |
"brand_analytics", "search_query_performance", "search_catalog_performance",
|
| 3629 |
"market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
|
|
|
|
| 815 |
ConnectorTableSpec(
|
| 816 |
key=_AMZN + "review_topics", native="customerFeedback/2024-06-01/items/{asin}/reviews/topics",
|
| 817 |
source="amzn_review_topics", label="Amazon Listings: Review topics", pinned="asin",
|
| 818 |
+
identity=("asin", "marketplace_id", "sort_by", "sentiment", "topic"),
|
| 819 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 820 |
),
|
| 821 |
ConnectorTableSpec(
|
|
|
|
| 824 |
identity=("asin", "marketplace_id", "sentiment", "topic", "month"),
|
| 825 |
links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
|
| 826 |
),
|
| 827 |
+
ConnectorTableSpec(
|
| 828 |
+
key=_AMZN + "browse_node_review_topics",
|
| 829 |
+
native="customerFeedback/2024-06-01/browseNodes/{browseNodeId}/reviews/topics",
|
| 830 |
+
source="amzn_browse_node_review_topics", label="Amazon Listings: Category review topics",
|
| 831 |
+
pinned="browse_node_name",
|
| 832 |
+
identity=("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"),
|
| 833 |
+
),
|
| 834 |
ConnectorTableSpec(
|
| 835 |
key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance",
|
| 836 |
source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku",
|
|
|
|
| 3313 |
"review_snippet_3": "Review snippet 3",
|
| 3314 |
"review_snippets": "Review snippets",
|
| 3315 |
"subtopics": "Subtopics",
|
| 3316 |
+
"sort_by": "Sorted by",
|
| 3317 |
+
"browse_node_id": "Browse node ID",
|
| 3318 |
+
"browse_node_name": "Category",
|
| 3319 |
+
"subtopic_1": "Subtopic 1",
|
| 3320 |
+
"subtopic_1_number_of_mentions": "Subtopic 1 mentions",
|
| 3321 |
+
"subtopic_1_occurrence_percentage": "Subtopic 1 occurrence",
|
| 3322 |
+
"subtopic_1_review_snippet_1": "First subtopic snippet 1",
|
| 3323 |
+
"subtopic_1_review_snippet_2": "First subtopic snippet 2",
|
| 3324 |
+
"subtopic_1_review_snippet_3": "First subtopic snippet 3",
|
| 3325 |
+
"subtopic_2": "Subtopic 2",
|
| 3326 |
+
"subtopic_2_number_of_mentions": "Subtopic 2 mentions",
|
| 3327 |
+
"subtopic_2_occurrence_percentage": "Subtopic 2 occurrence",
|
| 3328 |
+
"subtopic_2_review_snippet_1": "Second subtopic snippet 1",
|
| 3329 |
+
"subtopic_2_review_snippet_2": "Second subtopic snippet 2",
|
| 3330 |
+
"subtopic_2_review_snippet_3": "Second subtopic snippet 3",
|
| 3331 |
+
"subtopic_3": "Subtopic 3",
|
| 3332 |
+
"subtopic_3_number_of_mentions": "Subtopic 3 mentions",
|
| 3333 |
+
"subtopic_3_occurrence_percentage": "Subtopic 3 occurrence",
|
| 3334 |
+
"subtopic_3_review_snippet_1": "Third subtopic snippet 1",
|
| 3335 |
+
"subtopic_3_review_snippet_2": "Third subtopic snippet 2",
|
| 3336 |
+
"subtopic_3_review_snippet_3": "Third subtopic snippet 3",
|
| 3337 |
+
"subtopic_4": "Subtopic 4",
|
| 3338 |
+
"subtopic_4_number_of_mentions": "Subtopic 4 mentions",
|
| 3339 |
+
"subtopic_4_occurrence_percentage": "Subtopic 4 occurrence",
|
| 3340 |
+
"subtopic_4_review_snippet_1": "Fourth subtopic snippet 1",
|
| 3341 |
+
"subtopic_4_review_snippet_2": "Fourth subtopic snippet 2",
|
| 3342 |
+
"subtopic_4_review_snippet_3": "Fourth subtopic snippet 3",
|
| 3343 |
+
"subtopic_5": "Subtopic 5",
|
| 3344 |
+
"subtopic_5_number_of_mentions": "Subtopic 5 mentions",
|
| 3345 |
+
"subtopic_5_occurrence_percentage": "Subtopic 5 occurrence",
|
| 3346 |
+
"subtopic_5_review_snippet_1": "Fifth subtopic snippet 1",
|
| 3347 |
+
"subtopic_5_review_snippet_2": "Fifth subtopic snippet 2",
|
| 3348 |
+
"subtopic_5_review_snippet_3": "Fifth subtopic snippet 3",
|
| 3349 |
},
|
| 3350 |
_AMZN + "review_trends": {
|
| 3351 |
"_data_origin": "Data origin",
|
|
|
|
| 3361 |
"date_range_start_date": "Review period start",
|
| 3362 |
"date_range_end_date": "Review period end",
|
| 3363 |
"asin_metrics_occurrence_percentage": "ASIN occurrence",
|
| 3364 |
+
"parent_asin_metrics_occurrence_percentage": "Parent ASIN occurrence",
|
| 3365 |
+
"browse_node_metrics_occurrence_percentage_all_products": "Browse node occurrence",
|
| 3366 |
+
},
|
| 3367 |
+
_AMZN + "browse_node_review_topics": {
|
| 3368 |
+
"_data_origin": "Data origin",
|
| 3369 |
+
"_report_document_id": "Report document ID",
|
| 3370 |
+
"_report_row_number": "Report row",
|
| 3371 |
+
"_report_type": "Report type",
|
| 3372 |
+
"_snapshot_date": "Synced on",
|
| 3373 |
+
"browse_node_id": "Browse node ID",
|
| 3374 |
+
"browse_node_name": "Category",
|
| 3375 |
+
"marketplace_id": "Marketplace ID",
|
| 3376 |
+
"sort_by": "Sorted by",
|
| 3377 |
+
"sentiment": "Sentiment",
|
| 3378 |
+
"topic": "Topic",
|
| 3379 |
+
"date_range_start_date": "Review period start",
|
| 3380 |
+
"date_range_end_date": "Review period end",
|
| 3381 |
+
"browse_node_metrics_occurrence_percentage_all_products": "Category occurrence",
|
| 3382 |
+
"browse_node_metrics_occurrence_percentage_top_twenty_five_percent_products": "Top 25 percent occurrence",
|
| 3383 |
+
"browse_node_metrics_star_rating_impact_all_products": "Category star rating impact",
|
| 3384 |
+
"browse_node_metrics_star_rating_impact_top_twenty_five_percent_products": "Top 25 percent star rating impact",
|
| 3385 |
+
"review_snippet_1": "Review snippet 1",
|
| 3386 |
+
"review_snippet_2": "Review snippet 2",
|
| 3387 |
+
"review_snippet_3": "Review snippet 3",
|
| 3388 |
+
"subtopic_1": "Subtopic 1",
|
| 3389 |
+
"subtopic_1_occurrence_percentage": "Subtopic 1 occurrence",
|
| 3390 |
+
"subtopic_1_review_snippet_1": "First subtopic snippet 1",
|
| 3391 |
+
"subtopic_1_review_snippet_2": "First subtopic snippet 2",
|
| 3392 |
+
"subtopic_1_review_snippet_3": "First subtopic snippet 3",
|
| 3393 |
+
"subtopic_2": "Subtopic 2",
|
| 3394 |
+
"subtopic_2_occurrence_percentage": "Subtopic 2 occurrence",
|
| 3395 |
+
"subtopic_2_review_snippet_1": "Second subtopic snippet 1",
|
| 3396 |
+
"subtopic_2_review_snippet_2": "Second subtopic snippet 2",
|
| 3397 |
+
"subtopic_2_review_snippet_3": "Second subtopic snippet 3",
|
| 3398 |
+
"subtopic_3": "Subtopic 3",
|
| 3399 |
+
"subtopic_3_occurrence_percentage": "Subtopic 3 occurrence",
|
| 3400 |
+
"subtopic_3_review_snippet_1": "Third subtopic snippet 1",
|
| 3401 |
+
"subtopic_3_review_snippet_2": "Third subtopic snippet 2",
|
| 3402 |
+
"subtopic_3_review_snippet_3": "Third subtopic snippet 3",
|
| 3403 |
+
"subtopic_4": "Subtopic 4",
|
| 3404 |
+
"subtopic_4_occurrence_percentage": "Subtopic 4 occurrence",
|
| 3405 |
+
"subtopic_4_review_snippet_1": "Fourth subtopic snippet 1",
|
| 3406 |
+
"subtopic_4_review_snippet_2": "Fourth subtopic snippet 2",
|
| 3407 |
+
"subtopic_4_review_snippet_3": "Fourth subtopic snippet 3",
|
| 3408 |
+
"subtopic_5": "Subtopic 5",
|
| 3409 |
+
"subtopic_5_occurrence_percentage": "Subtopic 5 occurrence",
|
| 3410 |
+
"subtopic_5_review_snippet_1": "Fifth subtopic snippet 1",
|
| 3411 |
+
"subtopic_5_review_snippet_2": "Fifth subtopic snippet 2",
|
| 3412 |
+
"subtopic_5_review_snippet_3": "Fifth subtopic snippet 3",
|
| 3413 |
+
"our_asins": "Our ASINs",
|
| 3414 |
+
"our_asin_count": "Our ASIN count",
|
| 3415 |
},
|
| 3416 |
_AMZN + "replenishment": {
|
| 3417 |
"_data_origin": "Data origin",
|
|
|
|
| 3582 |
_AMZN + "replenishment": ("asin", "time_interval_start_date", "time_interval_end_date",
|
| 3583 |
"active_subscriptions", "shipped_subscription_units", "total_subscriptions_revenue",
|
| 3584 |
"revenue_penetration", "subscriber_retention_for30_days", "program_type"),
|
| 3585 |
+
_AMZN + "review_topics": ("sort_by", "sentiment", "topic", "browse_node_name", "asin_metrics_number_of_mentions",
|
| 3586 |
"asin_metrics_occurrence_percentage", "asin_metrics_star_rating_impact",
|
| 3587 |
"parent_asin_metrics_number_of_mentions", "browse_node_metrics_occurrence_percentage_all_products",
|
| 3588 |
"review_snippet_1", "review_snippet_2", "review_snippet_3"),
|
| 3589 |
_AMZN + "review_trends": ("sentiment", "topic", "month", "asin_metrics_occurrence_percentage",
|
| 3590 |
"date_range_start_date", "date_range_end_date"),
|
| 3591 |
+
_AMZN + "browse_node_review_topics": ("sort_by", "sentiment", "topic",
|
| 3592 |
+
"browse_node_metrics_occurrence_percentage_all_products",
|
| 3593 |
+
"browse_node_metrics_star_rating_impact_all_products", "our_asin_count", "review_snippet_1"),
|
| 3594 |
_AMZN + "sku_keywords": ("sku", "asin", "month", "search_query", "impressions", "clicks",
|
| 3595 |
"cart_adds", "purchases", "backend_keywords", "source_basis"),
|
| 3596 |
_AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
|
|
|
|
| 3717 |
**{_AMZN + t: "Listings" for t in (
|
| 3718 |
"listings", "pricing", "inactive_listings", "cancelled_listings", "suppressed_listings",
|
| 3719 |
"listing_defects", "sold_listings", "listing_items", "catalog_items", "b2b_recommended",
|
| 3720 |
+
"b2b_not_yet_on_amazon", "geo_opportunities", "b2b_quotes", "review_topics", "review_trends",
|
| 3721 |
+
"browse_node_review_topics")},
|
| 3722 |
**{_AMZN + t: "Advertising" for t in (
|
| 3723 |
"brand_analytics", "search_query_performance", "search_catalog_performance",
|
| 3724 |
"market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
|
platform/core/shares.py
CHANGED
|
@@ -598,7 +598,7 @@ def reconcile_database_grants(st=None, *, tenant=None, table_keys=None):
|
|
| 598 |
def _pending(data):
|
| 599 |
if not isinstance(data, dict) or not isinstance(data.get('database', {}), dict):
|
| 600 |
raise RuntimeError('database sharing is unavailable')
|
| 601 |
-
registry =
|
| 602 |
if not isinstance(registry, dict):
|
| 603 |
raise RuntimeError('the account registry is unavailable')
|
| 604 |
database = data.get('database', {})
|
|
@@ -692,6 +692,18 @@ def reconcile_database_grants(st=None, *, tenant=None, table_keys=None):
|
|
| 692 |
return repaired[0]
|
| 693 |
|
| 694 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 695 |
def repair_published_database_grants(st=None, *, table_keys):
|
| 696 |
"""Attempt one repair after publication, deferring failures to boot/editor except freeze refusal."""
|
| 697 |
from core import freeze_notice
|
|
|
|
| 598 |
def _pending(data):
|
| 599 |
if not isinstance(data, dict) or not isinstance(data.get('database', {}), dict):
|
| 600 |
raise RuntimeError('database sharing is unavailable')
|
| 601 |
+
registry = _strict_accounts()
|
| 602 |
if not isinstance(registry, dict):
|
| 603 |
raise RuntimeError('the account registry is unavailable')
|
| 604 |
database = data.get('database', {})
|
|
|
|
| 692 |
return repaired[0]
|
| 693 |
|
| 694 |
|
| 695 |
+
def _strict_accounts():
|
| 696 |
+
"""Every account record, read strictly: the account rows on PostgreSQL, else the users document."""
|
| 697 |
+
from core import account_rows, users
|
| 698 |
+
# On PostgreSQL the users document is a layout marker; reading it repairs no account at all.
|
| 699 |
+
if account_rows._pg_source(users.store):
|
| 700 |
+
return dict(account_rows._all(users.store))
|
| 701 |
+
registry = users.store._read_strict('users')
|
| 702 |
+
if account_rows.is_marker(registry):
|
| 703 |
+
raise RuntimeError('the account registry is unavailable')
|
| 704 |
+
return registry
|
| 705 |
+
|
| 706 |
+
|
| 707 |
def repair_published_database_grants(st=None, *, table_keys):
|
| 708 |
"""Attempt one repair after publication, deferring failures to boot/editor except freeze refusal."""
|
| 709 |
from core import freeze_notice
|
platform/core/store_pg.py
CHANGED
|
@@ -2719,9 +2719,10 @@ def download_bytes(path_in_repo, tenant_slug=None):
|
|
| 2719 |
"""Raw bytes, or None when absent — the same contract as the HF backend: a missing attachment
|
| 2720 |
degrades to "that file is gone", never to a broken page."""
|
| 2721 |
with _pool().connection() as con:
|
|
|
|
| 2722 |
row = con.execute(
|
| 2723 |
_q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
|
| 2724 |
-
(str(path_in_repo),)).fetchone()
|
| 2725 |
return bytes(row[0]) if row else None
|
| 2726 |
|
| 2727 |
|
|
|
|
| 2719 |
"""Raw bytes, or None when absent — the same contract as the HF backend: a missing attachment
|
| 2720 |
degrades to "that file is gone", never to a broken page."""
|
| 2721 |
with _pool().connection() as con:
|
| 2722 |
+
# Binary results: a text-format bytea arrives as hex and costs about 3x the transfer.
|
| 2723 |
row = con.execute(
|
| 2724 |
_q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
|
| 2725 |
+
(str(path_in_repo),), binary=True).fetchone()
|
| 2726 |
return bytes(row[0]) if row else None
|
| 2727 |
|
| 2728 |
|
platform/harness/amazon_client.py
CHANGED
|
@@ -117,6 +117,8 @@ _entity_pacers = {
|
|
| 117 |
"searchCatalogItems": {"last": None, "lock": threading.Lock()},
|
| 118 |
"getItemReviewTopics": {"last": None, "lock": threading.Lock()},
|
| 119 |
"getItemReviewTrends": {"last": None, "lock": threading.Lock()},
|
|
|
|
|
|
|
| 120 |
"getMyFeesEstimates": {"last": None, "lock": threading.Lock()},
|
| 121 |
"listAllFulfillmentOrders": {"last": None, "lock": threading.Lock()},
|
| 122 |
"getFulfillmentOrder": {"last": None, "lock": threading.Lock()},
|
|
@@ -937,6 +939,16 @@ GROUP_A_TABLES = {
|
|
| 937 |
"snapshot": True,
|
| 938 |
"validates_against": "Amazon raw trend point counts and occurrence percentages per ASIN, sentiment, topic and month vs stored rows for the same pass",
|
| 939 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 940 |
}
|
| 941 |
REPORT_TYPES.update(GROUP_A_TABLES)
|
| 942 |
|
|
@@ -1251,6 +1263,13 @@ OPERATIONS = {
|
|
| 1251 |
"getItemReviewTrends": {"name": "getItemReviewTrends",
|
| 1252 |
"roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
|
| 1253 |
"method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/reviews/trends"},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1254 |
"listAllFulfillmentOrders": {"name": "listAllFulfillmentOrders", "roles": ("Amazon Fulfillment",),
|
| 1255 |
"method": "GET", "path": "fulfillmentOutbound/2020-07-01/fulfillmentOrders"},
|
| 1256 |
"getFulfillmentOrder": {"name": "getFulfillmentOrder", "roles": ("Amazon Fulfillment",),
|
|
@@ -1883,7 +1902,10 @@ def transactions(fields, start, end, *, timeout=_TIMEOUT, sleep=time.sleep, now=
|
|
| 1883 |
ENTITY_INTERVALS = {"getOrderMetrics": 2.0, "getInventorySummaries": 0.5,
|
| 1884 |
"searchListingsItems": 0.2, "searchCatalogItems": 0.5,
|
| 1885 |
"getMyFeesEstimates": 2.0}
|
| 1886 |
-
FEEDBACK_INTERVALS = {"getItemReviewTopics": 1.0, "getItemReviewTrends": 1.0
|
|
|
|
|
|
|
|
|
|
| 1887 |
INBOUND_INTERVALS = {"listInboundPlans": 0.5, "getInboundPlan": 0.5, "listShipmentItems": 0.5,
|
| 1888 |
"getShipment": 0.5, "getShipmentItemsByShipmentId": 0.5, "getShipmentItems": 0.5}
|
| 1889 |
#: Amazon's 400 text for an Amazon Warehousing and Distribution plan in the FBA inbound list.
|
|
@@ -2242,46 +2264,168 @@ def inbound_shipment_items(fields, *, timeout=_TIMEOUT, sleep=time.sleep, now=ti
|
|
| 2242 |
return InboundRecords(records, skipped)
|
| 2243 |
|
| 2244 |
|
| 2245 |
-
def
|
| 2246 |
-
|
| 2247 |
-
"""Read every child ASIN, retaining known absences separately from invalid responses."""
|
| 2248 |
if operation not in FEEDBACK_INTERVALS:
|
| 2249 |
raise AmazonOperationRefused("unapproved Customer Feedback reader")
|
| 2250 |
if not set(OPERATIONS[operation]["roles"]).intersection(ROLES_TAKEN):
|
| 2251 |
raise AmazonOperationRefused("Customer Feedback needs Brand Analytics or Selling Partner Insights")
|
|
|
|
|
|
|
|
|
|
| 2252 |
identifiers = sorted(set(asins))
|
| 2253 |
if any(not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin) for asin in identifiers):
|
| 2254 |
raise AmazonError("Customer Feedback contains an invalid child ASIN. Correct the listings catalogue and retry.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2255 |
if not identifiers:
|
| 2256 |
return []
|
| 2257 |
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2258 |
-
|
| 2259 |
-
|
| 2260 |
-
|
| 2261 |
-
|
| 2262 |
-
|
| 2263 |
-
|
| 2264 |
-
|
| 2265 |
-
|
| 2266 |
-
|
| 2267 |
-
|
| 2268 |
-
|
| 2269 |
-
|
| 2270 |
-
|
| 2271 |
-
|
| 2272 |
-
|
| 2273 |
-
|
| 2274 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2275 |
continue
|
| 2276 |
-
if (not isinstance(body, dict) or
|
| 2277 |
-
or not isinstance(body.get(
|
| 2278 |
-
raise AmazonError(
|
| 2279 |
for key in ("positiveTopics", "negativeTopics"):
|
| 2280 |
-
topics = body[
|
| 2281 |
if topics is not None and (not isinstance(topics, list) or any(not isinstance(t, dict) for t in topics)):
|
| 2282 |
-
raise AmazonError(f"Amazon
|
| 2283 |
-
|
| 2284 |
-
return
|
| 2285 |
|
| 2286 |
|
| 2287 |
def group_a_records(fields, operation, *, start=None, end=None, asins=(),
|
|
@@ -2685,8 +2829,10 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
|
|
| 2685 |
if path.startswith("notifications/v1/"):
|
| 2686 |
reason = (detail or "Amazon refused this notification operation.").replace(token, "[redacted]")
|
| 2687 |
raise AmazonNotificationError(e.code, reason.replace("\u2014", ". ").replace("\u2013", " to ")) from None
|
| 2688 |
-
|
| 2689 |
-
|
|
|
|
|
|
|
| 2690 |
except Exception as e: # noqa: BLE001
|
| 2691 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
| 2692 |
|
|
|
|
| 117 |
"searchCatalogItems": {"last": None, "lock": threading.Lock()},
|
| 118 |
"getItemReviewTopics": {"last": None, "lock": threading.Lock()},
|
| 119 |
"getItemReviewTrends": {"last": None, "lock": threading.Lock()},
|
| 120 |
+
"getItemBrowseNode": {"last": None, "lock": threading.Lock()},
|
| 121 |
+
"getBrowseNodeReviewTopics": {"last": None, "lock": threading.Lock()},
|
| 122 |
"getMyFeesEstimates": {"last": None, "lock": threading.Lock()},
|
| 123 |
"listAllFulfillmentOrders": {"last": None, "lock": threading.Lock()},
|
| 124 |
"getFulfillmentOrder": {"last": None, "lock": threading.Lock()},
|
|
|
|
| 939 |
"snapshot": True,
|
| 940 |
"validates_against": "Amazon raw trend point counts and occurrence percentages per ASIN, sentiment, topic and month vs stored rows for the same pass",
|
| 941 |
},
|
| 942 |
+
"browse_node_review_topics": {
|
| 943 |
+
"label": "Amazon category review topics",
|
| 944 |
+
"api": "getBrowseNodeReviewTopics",
|
| 945 |
+
"role": "Brand Analytics / Selling Partner Insights",
|
| 946 |
+
"role_match": "any",
|
| 947 |
+
"min_interval_s": 7 * 86400,
|
| 948 |
+
"cadence_note": "Amazon refreshes the top 10 positive and negative category topics weekly. Every browse node of every active child ASIN is requested, sorted by mentions and by star rating impact.",
|
| 949 |
+
"snapshot": True,
|
| 950 |
+
"validates_against": "Amazon raw positive and negative topic counts and metrics per browse node and sort vs stored topics for the same pass",
|
| 951 |
+
},
|
| 952 |
}
|
| 953 |
REPORT_TYPES.update(GROUP_A_TABLES)
|
| 954 |
|
|
|
|
| 1263 |
"getItemReviewTrends": {"name": "getItemReviewTrends",
|
| 1264 |
"roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
|
| 1265 |
"method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/reviews/trends"},
|
| 1266 |
+
"getItemBrowseNode": {"name": "getItemBrowseNode",
|
| 1267 |
+
"roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
|
| 1268 |
+
"method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/browseNode"},
|
| 1269 |
+
"getBrowseNodeReviewTopics": {"name": "getBrowseNodeReviewTopics",
|
| 1270 |
+
"roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
|
| 1271 |
+
"method": "GET",
|
| 1272 |
+
"path": "customerFeedback/2024-06-01/browseNodes/{browseNodeId}/reviews/topics"},
|
| 1273 |
"listAllFulfillmentOrders": {"name": "listAllFulfillmentOrders", "roles": ("Amazon Fulfillment",),
|
| 1274 |
"method": "GET", "path": "fulfillmentOutbound/2020-07-01/fulfillmentOrders"},
|
| 1275 |
"getFulfillmentOrder": {"name": "getFulfillmentOrder", "roles": ("Amazon Fulfillment",),
|
|
|
|
| 1902 |
ENTITY_INTERVALS = {"getOrderMetrics": 2.0, "getInventorySummaries": 0.5,
|
| 1903 |
"searchListingsItems": 0.2, "searchCatalogItems": 0.5,
|
| 1904 |
"getMyFeesEstimates": 2.0}
|
| 1905 |
+
FEEDBACK_INTERVALS = {"getItemReviewTopics": 1.0, "getItemReviewTrends": 1.0,
|
| 1906 |
+
"getItemBrowseNode": 1.0, "getBrowseNodeReviewTopics": 1.0}
|
| 1907 |
+
#: getItemReviewTopics and getBrowseNodeReviewTopics accept exactly these sortBy values.
|
| 1908 |
+
FEEDBACK_SORTS = ("MENTIONS", "STAR_RATING_IMPACT")
|
| 1909 |
INBOUND_INTERVALS = {"listInboundPlans": 0.5, "getInboundPlan": 0.5, "listShipmentItems": 0.5,
|
| 1910 |
"getShipment": 0.5, "getShipmentItemsByShipmentId": 0.5, "getShipmentItems": 0.5}
|
| 1911 |
#: Amazon's 400 text for an Amazon Warehousing and Distribution plan in the FBA inbound list.
|
|
|
|
| 2264 |
return InboundRecords(records, skipped)
|
| 2265 |
|
| 2266 |
|
| 2267 |
+
def _feedback_admitted(operation):
|
| 2268 |
+
"""Refuse an unapproved Customer Feedback operation or a missing role before any transport."""
|
|
|
|
| 2269 |
if operation not in FEEDBACK_INTERVALS:
|
| 2270 |
raise AmazonOperationRefused("unapproved Customer Feedback reader")
|
| 2271 |
if not set(OPERATIONS[operation]["roles"]).intersection(ROLES_TAKEN):
|
| 2272 |
raise AmazonOperationRefused("Customer Feedback needs Brand Analytics or Selling Partner Insights")
|
| 2273 |
+
|
| 2274 |
+
|
| 2275 |
+
def _feedback_child_asins(asins):
|
| 2276 |
identifiers = sorted(set(asins))
|
| 2277 |
if any(not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin) for asin in identifiers):
|
| 2278 |
raise AmazonError("Customer Feedback contains an invalid child ASIN. Correct the listings catalogue and retry.")
|
| 2279 |
+
return identifiers
|
| 2280 |
+
|
| 2281 |
+
|
| 2282 |
+
def _feedback_sort(sort_by):
|
| 2283 |
+
if sort_by not in FEEDBACK_SORTS:
|
| 2284 |
+
raise AmazonError("Customer Feedback sortBy must be MENTIONS or STAR_RATING_IMPACT")
|
| 2285 |
+
return sort_by
|
| 2286 |
+
|
| 2287 |
+
|
| 2288 |
+
def _feedback_reads(fields, keys, reads, *, subject, timeout, sleep, now):
|
| 2289 |
+
"""Yield `(key, label, body, reason)` per key and read, interleaved so each operation paces its own bucket.
|
| 2290 |
+
|
| 2291 |
+
A 204, 400 or 404 answers one key: body None and a reason. A 403, an exhausted 429 or any other refusal raises.
|
| 2292 |
+
"""
|
| 2293 |
+
token, issued = access_token(fields, timeout=timeout), now()
|
| 2294 |
+
for key in keys:
|
| 2295 |
+
for label, operation, path, params in reads(key):
|
| 2296 |
+
_pace_entity(operation, FEEDBACK_INTERVALS[operation], sleep=sleep, now=now)
|
| 2297 |
+
if now() - issued >= 3000:
|
| 2298 |
+
token, issued = access_token(fields, timeout=timeout), now()
|
| 2299 |
+
try:
|
| 2300 |
+
body = _request(path, token, region=str(fields.get("region") or DEFAULT_REGION),
|
| 2301 |
+
sandbox=is_sandbox(fields), timeout=timeout,
|
| 2302 |
+
throttle_fix="continue every child ASIN at the Customer Feedback rate limit",
|
| 2303 |
+
**params)
|
| 2304 |
+
except AmazonError as exc:
|
| 2305 |
+
status = getattr(exc, "status", None)
|
| 2306 |
+
if isinstance(exc, (AmazonThrottled, AmazonOperationRefused)) or status not in (400, 404):
|
| 2307 |
+
raise
|
| 2308 |
+
detail = str(getattr(exc, "vendor_message", "") or "").replace("\u2014", ". ").replace("\u2013", " to ")
|
| 2309 |
+
detail = re.sub(r"[.\s]+$", "", detail.strip())
|
| 2310 |
+
yield key, label, None, (f"Amazon {operation} answered {status} for this {subject} and marketplace: "
|
| 2311 |
+
+ (detail or "no reason given") + ".")
|
| 2312 |
+
continue
|
| 2313 |
+
if body is None:
|
| 2314 |
+
yield key, label, None, f"Amazon {operation} returned 204 No Content for this {subject} and marketplace."
|
| 2315 |
+
continue
|
| 2316 |
+
yield key, label, body, None
|
| 2317 |
+
|
| 2318 |
+
|
| 2319 |
+
def _feedback_record(operation, asin, marketplace, body, reason):
|
| 2320 |
+
"""One validated item response, or its absence record."""
|
| 2321 |
+
if reason is not None:
|
| 2322 |
+
return {"asin": asin, "marketplaceId": marketplace, "_no_feedback_reason": reason}
|
| 2323 |
+
root = "topics" if operation == "getItemReviewTopics" else "reviewTrends"
|
| 2324 |
+
if (not isinstance(body, dict) or body.get("asin") != asin or body.get("marketplaceId") != marketplace
|
| 2325 |
+
or not isinstance(body.get(root), dict)):
|
| 2326 |
+
raise AmazonError(f"Amazon {operation} returned an invalid response for the requested child ASIN and marketplace. Retry after Amazon restores its response.")
|
| 2327 |
+
for key in ("positiveTopics", "negativeTopics"):
|
| 2328 |
+
topics = body[root].get(key)
|
| 2329 |
+
if topics is not None and (not isinstance(topics, list) or any(not isinstance(t, dict) for t in topics)):
|
| 2330 |
+
raise AmazonError(f"Amazon {operation} returned invalid {key}. No partial snapshot was stored.")
|
| 2331 |
+
return body
|
| 2332 |
+
|
| 2333 |
+
|
| 2334 |
+
def _browse_node_record(body, reason):
|
| 2335 |
+
"""`{"browse_node_id", "browse_node_name"}` from getItemBrowseNode, or its absence."""
|
| 2336 |
+
if reason is not None:
|
| 2337 |
+
return {"_no_feedback_reason": reason}
|
| 2338 |
+
node, name = (body.get(k) if isinstance(body, dict) else None for k in ("browseNodeId", "displayName"))
|
| 2339 |
+
if not isinstance(node, str) or not re.fullmatch(r"\d{1,20}", node) or (name is not None and not isinstance(name, str)):
|
| 2340 |
+
raise AmazonError("Amazon getItemBrowseNode returned no valid browse node. No partial snapshot was stored.")
|
| 2341 |
+
return {"browse_node_id": node, "browse_node_name": name}
|
| 2342 |
+
|
| 2343 |
+
|
| 2344 |
+
def item_review_feedback(fields, asins, *, operation="getItemReviewTopics", sort_by="MENTIONS", timeout=_TIMEOUT,
|
| 2345 |
+
sleep=time.sleep, now=time.monotonic):
|
| 2346 |
+
"""Read every child ASIN, retaining known absences separately from invalid responses."""
|
| 2347 |
+
if operation not in ("getItemReviewTopics", "getItemReviewTrends"):
|
| 2348 |
+
raise AmazonOperationRefused("unapproved Customer Feedback reader")
|
| 2349 |
+
_feedback_admitted(operation)
|
| 2350 |
+
params = {"sortBy": _feedback_sort(sort_by)} if operation == "getItemReviewTopics" else {}
|
| 2351 |
+
identifiers = _feedback_child_asins(asins)
|
| 2352 |
if not identifiers:
|
| 2353 |
return []
|
| 2354 |
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2355 |
+
path = OPERATIONS[operation]["path"]
|
| 2356 |
+
reads = lambda asin: [(None, operation, path.replace("{asin}", asin), {"marketplaceId": marketplace, **params})]
|
| 2357 |
+
return [_feedback_record(operation, asin, marketplace, body, reason) for asin, _label, body, reason in
|
| 2358 |
+
_feedback_reads(fields, identifiers, reads, subject="ASIN", timeout=timeout, sleep=sleep, now=now)]
|
| 2359 |
+
|
| 2360 |
+
|
| 2361 |
+
def item_browse_nodes(fields, asins, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
|
| 2362 |
+
"""`{asin: {"browse_node_id", "browse_node_name"}}` for every child ASIN; an absence carries `_no_feedback_reason`."""
|
| 2363 |
+
_feedback_admitted("getItemBrowseNode")
|
| 2364 |
+
identifiers = _feedback_child_asins(asins)
|
| 2365 |
+
if not identifiers:
|
| 2366 |
+
return {}
|
| 2367 |
+
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2368 |
+
path = OPERATIONS["getItemBrowseNode"]["path"]
|
| 2369 |
+
reads = lambda asin: [(None, "getItemBrowseNode", path.replace("{asin}", asin), {"marketplaceId": marketplace})]
|
| 2370 |
+
return {asin: _browse_node_record(body, reason) for asin, _label, body, reason in
|
| 2371 |
+
_feedback_reads(fields, identifiers, reads, subject="ASIN", timeout=timeout, sleep=sleep, now=now)}
|
| 2372 |
+
|
| 2373 |
+
|
| 2374 |
+
def item_review_topics(fields, asins, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
|
| 2375 |
+
"""Per child ASIN: its browse node and its topics by both sorts, interleaved on one token.
|
| 2376 |
+
|
| 2377 |
+
Returns `{"MENTIONS": [record], "STAR_RATING_IMPACT": [record], "browse_nodes": {asin: node}}`.
|
| 2378 |
+
"""
|
| 2379 |
+
for operation in ("getItemBrowseNode", "getItemReviewTopics"):
|
| 2380 |
+
_feedback_admitted(operation)
|
| 2381 |
+
identifiers = _feedback_child_asins(asins)
|
| 2382 |
+
out = {**{sort: [] for sort in FEEDBACK_SORTS}, "browse_nodes": {}}
|
| 2383 |
+
if not identifiers:
|
| 2384 |
+
return out
|
| 2385 |
+
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2386 |
+
node_path, topic_path = (OPERATIONS[name]["path"] for name in ("getItemBrowseNode", "getItemReviewTopics"))
|
| 2387 |
+
|
| 2388 |
+
def reads(asin):
|
| 2389 |
+
return [("browse_node", "getItemBrowseNode", node_path.replace("{asin}", asin), {"marketplaceId": marketplace}),
|
| 2390 |
+
*((sort, "getItemReviewTopics", topic_path.replace("{asin}", asin),
|
| 2391 |
+
{"marketplaceId": marketplace, "sortBy": sort}) for sort in FEEDBACK_SORTS)]
|
| 2392 |
+
for asin, label, body, reason in _feedback_reads(fields, identifiers, reads, subject="ASIN",
|
| 2393 |
+
timeout=timeout, sleep=sleep, now=now):
|
| 2394 |
+
if label == "browse_node":
|
| 2395 |
+
out["browse_nodes"][asin] = _browse_node_record(body, reason)
|
| 2396 |
+
else:
|
| 2397 |
+
out[label].append(_feedback_record("getItemReviewTopics", asin, marketplace, body, reason))
|
| 2398 |
+
return out
|
| 2399 |
+
|
| 2400 |
+
|
| 2401 |
+
def browse_node_review_topics(fields, node_ids, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
|
| 2402 |
+
"""Each browse node's top category topics by both sorts: `{sort: [record]}`; an absence carries `_no_feedback_reason`."""
|
| 2403 |
+
_feedback_admitted("getBrowseNodeReviewTopics")
|
| 2404 |
+
nodes = sorted(set(node_ids))
|
| 2405 |
+
# The id becomes a path segment, so only a numeric browse node id is admitted.
|
| 2406 |
+
if any(not isinstance(node, str) or not re.fullmatch(r"\d{1,20}", node) for node in nodes):
|
| 2407 |
+
raise AmazonError("Customer Feedback contains an invalid browse node ID. Retry after Amazon returns a numeric browse node.")
|
| 2408 |
+
out = {sort: [] for sort in FEEDBACK_SORTS}
|
| 2409 |
+
if not nodes:
|
| 2410 |
+
return out
|
| 2411 |
+
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2412 |
+
path = OPERATIONS["getBrowseNodeReviewTopics"]["path"]
|
| 2413 |
+
reads = lambda node: [(sort, "getBrowseNodeReviewTopics", path.replace("{browseNodeId}", node),
|
| 2414 |
+
{"marketplaceId": marketplace, "sortBy": sort}) for sort in FEEDBACK_SORTS]
|
| 2415 |
+
for node, sort, body, reason in _feedback_reads(fields, nodes, reads, subject="browse node",
|
| 2416 |
+
timeout=timeout, sleep=sleep, now=now):
|
| 2417 |
+
if reason is not None:
|
| 2418 |
+
out[sort].append({"browseNodeId": node, "marketplaceId": marketplace, "_no_feedback_reason": reason})
|
| 2419 |
continue
|
| 2420 |
+
if (not isinstance(body, dict) or str(body.get("browseNodeId")) != node
|
| 2421 |
+
or body.get("marketplaceId") != marketplace or not isinstance(body.get("topics"), dict)):
|
| 2422 |
+
raise AmazonError("Amazon getBrowseNodeReviewTopics returned an invalid response for the requested browse node and marketplace. Retry after Amazon restores its response.")
|
| 2423 |
for key in ("positiveTopics", "negativeTopics"):
|
| 2424 |
+
topics = body["topics"].get(key)
|
| 2425 |
if topics is not None and (not isinstance(topics, list) or any(not isinstance(t, dict) for t in topics)):
|
| 2426 |
+
raise AmazonError(f"Amazon getBrowseNodeReviewTopics returned invalid {key}. No partial snapshot was stored.")
|
| 2427 |
+
out[sort].append(body)
|
| 2428 |
+
return out
|
| 2429 |
|
| 2430 |
|
| 2431 |
def group_a_records(fields, operation, *, start=None, end=None, asins=(),
|
|
|
|
| 2829 |
if path.startswith("notifications/v1/"):
|
| 2830 |
reason = (detail or "Amazon refused this notification operation.").replace(token, "[redacted]")
|
| 2831 |
raise AmazonNotificationError(e.code, reason.replace("\u2014", ". ").replace("\u2013", " to ")) from None
|
| 2832 |
+
refusal = AmazonError(f"{_safe(url)} answered {e.code}" + (f": {detail}" if detail else ""))
|
| 2833 |
+
# Callers branch on the status attribute, never on the message text.
|
| 2834 |
+
refusal.status, refusal.vendor_message = e.code, detail
|
| 2835 |
+
raise refusal from None
|
| 2836 |
except Exception as e: # noqa: BLE001
|
| 2837 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
| 2838 |
|
platform/harness/amazon_schema.py
CHANGED
|
@@ -563,10 +563,12 @@ TABLE_SPECS.update({
|
|
| 563 |
for column, target, field, cardinality in _inbound_spec.links), grain="inbound_shipment_item"),
|
| 564 |
"replenishment": _table_spec("replenishment", ("metric_id",),
|
| 565 |
(("sku", "listings", "seller_sku", "many_to_one"),), grain="replenishment_metric"),
|
| 566 |
-
"review_topics": _table_spec("review_topics", ("asin", "marketplace_id", "sentiment", "topic"),
|
| 567 |
-
(("asin", "listings", "asin1", "many_to_one"),), grain="
|
| 568 |
"review_trends": _table_spec("review_trends", ("asin", "marketplace_id", "sentiment", "topic", "month"),
|
| 569 |
(("asin", "listings", "asin1", "many_to_one"),), grain="asin_sentiment_topic_month"),
|
|
|
|
|
|
|
| 570 |
"sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
|
| 571 |
(("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"),
|
| 572 |
"keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
|
|
|
|
| 563 |
for column, target, field, cardinality in _inbound_spec.links), grain="inbound_shipment_item"),
|
| 564 |
"replenishment": _table_spec("replenishment", ("metric_id",),
|
| 565 |
(("sku", "listings", "seller_sku", "many_to_one"),), grain="replenishment_metric"),
|
| 566 |
+
"review_topics": _table_spec("review_topics", ("asin", "marketplace_id", "sort_by", "sentiment", "topic"),
|
| 567 |
+
(("asin", "listings", "asin1", "many_to_one"),), grain="asin_sort_sentiment_topic"),
|
| 568 |
"review_trends": _table_spec("review_trends", ("asin", "marketplace_id", "sentiment", "topic", "month"),
|
| 569 |
(("asin", "listings", "asin1", "many_to_one"),), grain="asin_sentiment_topic_month"),
|
| 570 |
+
"browse_node_review_topics": _table_spec("browse_node_review_topics",
|
| 571 |
+
("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"),
|
| 572 |
"sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
|
| 573 |
(("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"),
|
| 574 |
"keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
|
platform/harness/amazon_store.py
CHANGED
|
@@ -85,6 +85,8 @@ SQP_PASS_REPORTS = 15
|
|
| 85 |
SQP_RESUME_SECONDS = 900
|
| 86 |
GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
|
| 87 |
TABLES += GROUP_A_TABLES
|
|
|
|
|
|
|
| 88 |
DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
|
| 89 |
#: Snapshot reports whose changes are kept as versions in `amzn_<t>_history`.
|
| 90 |
HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES)
|
|
@@ -5143,8 +5145,9 @@ def plain_refusal_reason(reason, *, status="", table="", operation=""):
|
|
| 5143 |
return "The reconciliation check could not run. The last good rows are kept until it runs."
|
| 5144 |
return (f"Reconciliation did not tie: {detail}. The last good rows are kept." if detail
|
| 5145 |
else "Reconciliation did not tie. The last good rows are kept.")
|
| 5146 |
-
feedback = table in
|
| 5147 |
-
subject = "
|
|
|
|
| 5148 |
if feedback and (str(status) in ("403", "NOT_OFFERED")
|
| 5149 |
or re.search(r"answered\s+403\b|\(403\)", text)):
|
| 5150 |
vendor = re.search(r"answered 403(?:: (.*?))?(?:; fix:|$)", text)
|
|
@@ -7034,7 +7037,7 @@ def _sync_problem(exc, table, operation, tenant, rt, report, log):
|
|
| 7034 |
held = all(r.strip() in cx.ROLES_TAKEN
|
| 7035 |
for r in str(role or "").replace(" / ", " + ").split(" + ") if r.strip())
|
| 7036 |
held = held and not re.search(r"missing.*role|not granted|has not granted", str(exc), re.I)
|
| 7037 |
-
fix = ("review Customer Feedback seller and marketplace eligibility and either Brand Analytics or Selling Partner Insights access, then reconnect Amazon" if table in
|
| 7038 |
"review Amazon Fulfillment access for this seller account and reconnect Amazon" if table in ("outbound_fulfillment", "inbound_shipments") else
|
| 7039 |
"review the Pricing role's access for this seller account and reconnect Amazon" if table == "fee_estimates_api" else
|
| 7040 |
f"needs the {role} role; this application already holds it, so Amazon does not "
|
|
@@ -7380,6 +7383,8 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
|
|
| 7380 |
declared.update({name: kind for name, kind in REVIEW_TOPIC_TYPES.items() if name in columns})
|
| 7381 |
if table == "review_trends":
|
| 7382 |
declared.update({name: kind for name, kind in REVIEW_TREND_TYPES.items() if name in columns})
|
|
|
|
|
|
|
| 7383 |
types = _ensure(con, mirror, columns, rows, declared=declared, preserve_rows=True)
|
| 7384 |
predicate = f'"{column}" >= ? AND "{column}" < ?' if end else f'"{column}" = ?'
|
| 7385 |
if column == "posted_date" and end:
|
|
@@ -8039,23 +8044,152 @@ REVIEW_TOPIC_METRICS = (
|
|
| 8039 |
("child_asin_metrics_most_mentions_number_of_mentions", ("childAsinMetrics", "mostMentions", "numberOfMentions")),
|
| 8040 |
("child_asin_metrics_least_mentions_number_of_mentions", ("childAsinMetrics", "leastMentions", "numberOfMentions")),
|
| 8041 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8042 |
REVIEW_TOPIC_TYPES = {
|
| 8043 |
-
**dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "review_snippets", "subtopics",
|
| 8044 |
-
"review_snippet_1", "review_snippet_2", "review_snippet_3"
|
|
|
|
| 8045 |
**{name: "BIGINT" if name.endswith("_number_of_mentions") else "DOUBLE"
|
| 8046 |
for name, _path in REVIEW_TOPIC_METRICS},
|
|
|
|
| 8047 |
}
|
| 8048 |
|
| 8049 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8050 |
def _review_topic_evidence(records, asins, marketplace):
|
| 8051 |
"""Count raw vendor topics and retain their metrics before the row parser runs."""
|
| 8052 |
import math
|
| 8053 |
-
|
|
|
|
|
|
|
|
|
|
| 8054 |
for record in records:
|
| 8055 |
-
asin = record.get("asin")
|
| 8056 |
-
if asin not in expected or asin in seen or record.get("marketplaceId") != marketplace:
|
| 8057 |
raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign ASIN or marketplace")
|
| 8058 |
-
seen.add(asin)
|
| 8059 |
no_content = record.get("_no_feedback_reason")
|
| 8060 |
topics = record.get("topics")
|
| 8061 |
if not no_content and not isinstance(topics, dict):
|
|
@@ -8066,13 +8200,15 @@ def _review_topic_evidence(records, asins, marketplace):
|
|
| 8066 |
group = [] if group is None else group
|
| 8067 |
if not isinstance(group, list):
|
| 8068 |
raise ValueError("Amazon getItemReviewTopics returned an invalid sentiment topic list")
|
| 8069 |
-
counts[json.dumps([asin, sentiment], separators=(",", ":"))] = len(group)
|
| 8070 |
total += len(group)
|
| 8071 |
for topic in group:
|
|
|
|
| 8072 |
if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip()
|
| 8073 |
-
or not isinstance(topic.get("asinMetrics"), dict)):
|
| 8074 |
-
raise ValueError("Amazon review topic requires a topic name and asinMetrics object")
|
| 8075 |
-
|
|
|
|
| 8076 |
if identity in metrics:
|
| 8077 |
raise ValueError("Amazon returned duplicate topics for one ASIN and sentiment")
|
| 8078 |
values = []
|
|
@@ -8092,58 +8228,63 @@ def _review_topic_evidence(records, asins, marketplace):
|
|
| 8092 |
values.append(value)
|
| 8093 |
metrics[identity] = values
|
| 8094 |
if not total:
|
| 8095 |
-
absent.append({"asin": asin, "
|
|
|
|
| 8096 |
"reason": no_content or "Amazon returned no positive or negative review topics for this ASIN and marketplace."})
|
| 8097 |
if seen != expected:
|
| 8098 |
raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN")
|
| 8099 |
-
return {"count": sum(counts.values()), "source_counts": counts, "source_metrics": metrics,
|
| 8100 |
"marketplace_id": marketplace, "absent_asins": absent}
|
| 8101 |
|
| 8102 |
|
| 8103 |
-
def review_topic_rows(records, document):
|
| 8104 |
"""Flatten positive and negative topics while retaining null metrics and vendor percentage units."""
|
| 8105 |
rows = []
|
| 8106 |
for record in records:
|
|
|
|
| 8107 |
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8108 |
for topic in (record.get("topics") or {}).get(key) or []:
|
| 8109 |
row = dict.fromkeys(REVIEW_TOPIC_TYPES)
|
| 8110 |
row.update(_flatten_row(topic))
|
| 8111 |
-
row.update(asin=record["asin"], marketplace_id=record["marketplaceId"], sentiment=sentiment
|
|
|
|
|
|
|
| 8112 |
row.update(_flatten_row({k: record[k] for k in ("itemName", "countryCode", "dateRange") if k in record}))
|
| 8113 |
-
|
| 8114 |
-
|
| 8115 |
-
|
| 8116 |
-
|
| 8117 |
-
for index, snippet in enumerate(snippets or [], 1):
|
| 8118 |
row[f"review_snippet_{index}"] = snippet
|
|
|
|
| 8119 |
rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTopics"))
|
| 8120 |
return rows
|
| 8121 |
|
| 8122 |
|
| 8123 |
def _sync_review_topics(con, fields):
|
| 8124 |
-
"""Replace the complete
|
| 8125 |
-
|
| 8126 |
-
asins = ([row[0] for row in con.execute(
|
| 8127 |
-
'SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), \'\') IS NOT NULL ORDER BY 1').fetchall()]
|
| 8128 |
-
if "asin1" in columns else [])
|
| 8129 |
marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
|
| 8130 |
-
|
|
|
|
|
|
|
|
|
|
| 8131 |
evidence = _review_topic_evidence(records, asins, marketplace)
|
| 8132 |
document = "operation:" + uuid.uuid4().hex
|
| 8133 |
-
rows = review_topic_rows(records, document)
|
| 8134 |
period = datetime.now(timezone.utc).date().isoformat()
|
| 8135 |
for row in rows:
|
| 8136 |
row["_snapshot_date"] = period
|
| 8137 |
reason = ("Amazon Customer Feedback returned no review topics for this marketplace. "
|
| 8138 |
"The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
|
| 8139 |
-
if asins else
|
| 8140 |
marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
|
|
|
|
| 8141 |
"empty_reason": reason if not rows else ""}
|
| 8142 |
count = _replace_period(con, "review_topics", rows,
|
| 8143 |
sorted(set(REVIEW_TOPIC_TYPES) | {key for row in rows for key in row}),
|
| 8144 |
"_snapshot_date", period, marker=marker, snapshot=True)
|
| 8145 |
_record_state(con, entity_key("limit_review_topics"), "done", detail=json.dumps({
|
| 8146 |
-
"reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"]}))
|
| 8147 |
return count
|
| 8148 |
|
| 8149 |
|
|
@@ -8151,13 +8292,52 @@ REVIEW_TREND_TYPES = {
|
|
| 8151 |
**dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "month",
|
| 8152 |
"date_range_start_date", "date_range_end_date"), "VARCHAR"),
|
| 8153 |
"asin_metrics_occurrence_percentage": "DOUBLE",
|
|
|
|
|
|
|
| 8154 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8155 |
|
| 8156 |
|
| 8157 |
def _review_trend_evidence(records, asins, marketplace):
|
| 8158 |
"""Count vendor trend points and retain source values before flattening or storing rows."""
|
| 8159 |
-
import math
|
| 8160 |
expected, seen, counts, metrics, absent = set(asins), set(), {}, {}, []
|
|
|
|
| 8161 |
for record in records:
|
| 8162 |
asin = record.get("asin")
|
| 8163 |
if asin not in expected or asin in seen or record.get("marketplaceId") != marketplace:
|
|
@@ -8187,31 +8367,22 @@ def _review_trend_evidence(records, asins, marketplace):
|
|
| 8187 |
counts[group] += len(topic["trendMetrics"])
|
| 8188 |
for point in topic["trendMetrics"]:
|
| 8189 |
if (not isinstance(point, dict) or not isinstance(point.get("dateRange"), dict)
|
| 8190 |
-
or not isinstance(point.get("asinMetrics"), dict)):
|
| 8191 |
-
raise ValueError("Amazon review trend requires dateRange and asinMetrics
|
| 8192 |
-
|
| 8193 |
-
|
| 8194 |
-
|
| 8195 |
-
raise ValueError("Amazon review trend requires ISO calendar dates")
|
| 8196 |
-
first, last = datetime.strptime(start, "%Y-%m-%d"), datetime.strptime(end, "%Y-%m-%d")
|
| 8197 |
-
if first > last or start[:7] != end[:7]:
|
| 8198 |
-
raise ValueError("Amazon review trend dateRange must identify one calendar month")
|
| 8199 |
-
identity = json.dumps([asin, marketplace, sentiment, topic["topic"], start[:7]],
|
| 8200 |
separators=(",", ":"))
|
| 8201 |
if identity in metrics:
|
| 8202 |
raise ValueError("Amazon returned duplicate trend points for one ASIN, topic and month")
|
| 8203 |
-
|
| 8204 |
-
if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
|
| 8205 |
-
or not math.isfinite(value) or not 0 <= value <= 100):
|
| 8206 |
-
raise ValueError("Amazon review trend returned an invalid occurrence percentage")
|
| 8207 |
-
metrics[identity] = [start, end, value]
|
| 8208 |
total += counts[group]
|
| 8209 |
if not total:
|
| 8210 |
absent.append({"asin": asin, "code": "NO_CONTENT" if no_content else "NO_TRENDS",
|
| 8211 |
"reason": no_content or "Amazon returned no positive or negative review trend points for this ASIN and marketplace."})
|
| 8212 |
if seen != expected:
|
| 8213 |
raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN")
|
| 8214 |
-
return {"count": sum(counts.values()), "source_counts": counts, "source_metrics": metrics,
|
| 8215 |
"marketplace_id": marketplace, "absent_asins": absent}
|
| 8216 |
|
| 8217 |
|
|
@@ -8222,23 +8393,22 @@ def review_trend_rows(records, document):
|
|
| 8222 |
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8223 |
for topic in (record.get("reviewTrends") or {}).get(key) or []:
|
| 8224 |
for point in topic["trendMetrics"]:
|
| 8225 |
-
|
| 8226 |
row = {"asin": record["asin"], "marketplace_id": record["marketplaceId"],
|
| 8227 |
-
"sentiment": sentiment, "topic": topic["topic"], "month":
|
| 8228 |
-
"date_range_start_date":
|
| 8229 |
-
|
| 8230 |
rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTrends"))
|
| 8231 |
return rows
|
| 8232 |
|
| 8233 |
|
| 8234 |
def _sync_review_trends(con, fields):
|
| 8235 |
"""Replace the full six-month review snapshot with independently counted source evidence."""
|
| 8236 |
-
|
| 8237 |
-
asins = ([row[0] for row in con.execute(
|
| 8238 |
-
'SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), \'\') IS NOT NULL ORDER BY 1').fetchall()]
|
| 8239 |
-
if "asin1" in columns else [])
|
| 8240 |
marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
|
| 8241 |
records = _client().item_review_feedback(fields, asins, operation="getItemReviewTrends")
|
|
|
|
|
|
|
| 8242 |
evidence = _review_trend_evidence(records, asins, marketplace)
|
| 8243 |
rows = review_trend_rows(records, "operation:" + uuid.uuid4().hex)
|
| 8244 |
period = datetime.now(timezone.utc).date().isoformat()
|
|
@@ -8246,13 +8416,149 @@ def _sync_review_trends(con, fields):
|
|
| 8246 |
row["_snapshot_date"] = period
|
| 8247 |
reason = ("Amazon Customer Feedback returned no review trends for this marketplace. "
|
| 8248 |
"The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
|
| 8249 |
-
if asins else
|
| 8250 |
marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
|
|
|
|
| 8251 |
"empty_reason": reason if not rows else ""}
|
| 8252 |
count = _replace_period(con, "review_trends", rows, REVIEW_TREND_TYPES,
|
| 8253 |
"_snapshot_date", period, marker=marker, snapshot=True)
|
| 8254 |
_record_state(con, entity_key("limit_review_trends"), "done", detail=json.dumps({
|
| 8255 |
-
"reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"]}))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8256 |
return count
|
| 8257 |
|
| 8258 |
|
|
@@ -8264,6 +8570,8 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log, operations
|
|
| 8264 |
return _sync_review_topics(con, fields)
|
| 8265 |
if table == "review_trends":
|
| 8266 |
return _sync_review_trends(con, fields)
|
|
|
|
|
|
|
| 8267 |
if table == "economics":
|
| 8268 |
start, end = _utc(w0).date().isoformat(), _utc(w1).date().isoformat()
|
| 8269 |
if start >= end:
|
|
@@ -8673,7 +8981,7 @@ def _validate_margin(con, table):
|
|
| 8673 |
identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sentiment"),
|
| 8674 |
row.get("topic"), row.get("month")], separators=(",", ":"))
|
| 8675 |
metrics[identity] = [row.get("date_range_start_date"), row.get("date_range_end_date"),
|
| 8676 |
-
row.get(
|
| 8677 |
total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
|
| 8678 |
trends_match = (marker["count"] == len(rows) == len(metrics) == total
|
| 8679 |
and counts == marker["source_counts"] and metrics == marker["source_metrics"])
|
|
@@ -8684,10 +8992,10 @@ def _validate_margin(con, table):
|
|
| 8684 |
counts = dict.fromkeys(marker["source_counts"], 0)
|
| 8685 |
metrics = {}
|
| 8686 |
for row in rows:
|
| 8687 |
-
group = json.dumps([row.get("asin"), row.get("sentiment")], separators=(",", ":"))
|
| 8688 |
counts[group] = counts.get(group, 0) + 1
|
| 8689 |
-
identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("
|
| 8690 |
-
|
| 8691 |
metrics[identity] = [row.get(name) for name, _path in REVIEW_TOPIC_METRICS]
|
| 8692 |
total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
|
| 8693 |
matches = (marker["count"] == len(rows) == len(metrics) == total
|
|
@@ -8695,6 +9003,22 @@ def _validate_margin(con, table):
|
|
| 8695 |
return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows),
|
| 8696 |
"reason": (marker["empty_reason"] if matches and not rows else
|
| 8697 |
f"Amazon returned {marker['count']} review topics. Stored {len(rows)} topics with per-ASIN, sentiment and metric reconciliation.")}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8698 |
if table == "fee_estimates_api":
|
| 8699 |
expected, actual = marker["count"], len(rows)
|
| 8700 |
identities = {row.get("estimate_id") for row in rows}
|
|
@@ -9428,8 +9752,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9428 |
# A watermark is written when a table's history is complete, so its next window is
|
| 9429 |
# bounded; deferring it too froze every loaded table behind one refused report.
|
| 9430 |
for table in want:
|
|
|
|
| 9431 |
if ((read_watermark(con, table) and (not refresh_loaded or table not in repairs))
|
| 9432 |
-
or table in completed
|
| 9433 |
or (table == "settlements" and not refresh_loaded)):
|
| 9434 |
continue
|
| 9435 |
report["tables"][table] = {"state": "deferred", "reason":
|
|
@@ -9442,7 +9767,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9442 |
_record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail"))
|
| 9443 |
want = [table for table in want if (read_watermark(con, table)
|
| 9444 |
and (not refresh_loaded or table not in repairs))
|
| 9445 |
-
or table in completed
|
| 9446 |
or (table == "settlements" and not refresh_loaded)]
|
| 9447 |
if scheduled and not first_load:
|
| 9448 |
ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
|
|
@@ -9470,6 +9795,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9470 |
elif not first_load:
|
| 9471 |
# The vendor's report floor also applies to an operator's Sync now click.
|
| 9472 |
for table in list(want):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9473 |
reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)),
|
| 9474 |
datetime.now(timezone.utc))
|
| 9475 |
if reason:
|
|
@@ -10580,7 +10909,7 @@ def _validate_account(tenant_key="royal-imports", rt=None, log=print):
|
|
| 10580 |
out[table] = {"state": "unproven", "reason": "table has never synced",
|
| 10581 |
"reconciles": spec.get("validates_against")}
|
| 10582 |
continue
|
| 10583 |
-
if table in ("fee_estimates_api", "replenishment", "inbound_shipments",
|
| 10584 |
out[table] = {"rows": n, "reconciles": spec.get("validates_against"),
|
| 10585 |
**_validate_margin(con, table)}
|
| 10586 |
continue
|
|
@@ -11266,6 +11595,24 @@ def snapshot_sync_selftest(log=print, check=None):
|
|
| 11266 |
"numberOfMentions": generation[0] + 2, "occurrencePercentage": 12.5,
|
| 11267 |
"starRatingImpact": 0.25}}], "negativeTopics": None}} for asin in asins]
|
| 11268 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11269 |
def snapshot_counts(requests):
|
| 11270 |
return Counter(table for table, _kwargs in requests if table in snapshots)
|
| 11271 |
|
|
@@ -11305,6 +11652,9 @@ def snapshot_sync_selftest(log=print, check=None):
|
|
| 11305 |
patch.object(cx, "outbound_fulfillment_orders", side_effect=outbound_records),
|
| 11306 |
patch.object(cx, "inbound_shipment_items", side_effect=inbound_records),
|
| 11307 |
patch.object(cx, "item_review_feedback", side_effect=feedback_records),
|
|
|
|
|
|
|
|
|
|
| 11308 |
patch.object(cx, "replenishment_metrics", side_effect=replenishment_records),
|
| 11309 |
patch.object(cx, "awd_inventory", return_value=[]),
|
| 11310 |
patch.object(cx, "access_token", side_effect=AssertionError("offline snapshot sync fixture")),
|
|
@@ -12797,6 +13147,16 @@ def group_a_selftest(log=print, check=None):
|
|
| 12797 |
"occurrencePercentage": 12.5, "starRatingImpact": 0.2}}],
|
| 12798 |
"negativeTopics": [{"topic": "Fixture assembly", "asinMetrics": {"numberOfMentions": 2,
|
| 12799 |
"occurrencePercentage": 6.25, "starRatingImpact": -0.1}}]}}]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12800 |
|
| 12801 |
try:
|
| 12802 |
with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \
|
|
@@ -12807,6 +13167,9 @@ def group_a_selftest(log=print, check=None):
|
|
| 12807 |
patch.object(cx, "inbound_shipment_items", return_value=inbound), \
|
| 12808 |
patch.object(cx, "replenishment_metrics", return_value=replenishment), \
|
| 12809 |
patch.object(cx, "item_review_feedback", return_value=feedback), \
|
|
|
|
|
|
|
|
|
|
| 12810 |
patch.object(module, "_fee_estimate_inputs", return_value=[fee_input]), \
|
| 12811 |
patch.object(cx, "fee_estimates", side_effect=lambda _f, estimates: _fixture_fee_records(estimates)):
|
| 12812 |
for table in GROUP_A_TABLES:
|
|
@@ -13037,6 +13400,72 @@ def group_a_selftest(log=print, check=None):
|
|
| 13037 |
finally:
|
| 13038 |
con.close()
|
| 13039 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13040 |
# Client pagination: an empty middle page must not end the traversal.
|
| 13041 |
for operation, interval in cx.ENTITY_INTERVALS.items():
|
| 13042 |
if operation == "getMyFeesEstimates":
|
|
|
|
| 85 |
SQP_RESUME_SECONDS = 900
|
| 86 |
GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
|
| 87 |
TABLES += GROUP_A_TABLES
|
| 88 |
+
#: Customer Feedback entity snapshots: weekly current state, no history window.
|
| 89 |
+
FEEDBACK_TABLES = ("review_topics", "review_trends", "browse_node_review_topics")
|
| 90 |
DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
|
| 91 |
#: Snapshot reports whose changes are kept as versions in `amzn_<t>_history`.
|
| 92 |
HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES)
|
|
|
|
| 5145 |
return "The reconciliation check could not run. The last good rows are kept until it runs."
|
| 5146 |
return (f"Reconciliation did not tie: {detail}. The last good rows are kept." if detail
|
| 5147 |
else "Reconciliation did not tie. The last good rows are kept.")
|
| 5148 |
+
feedback = table in FEEDBACK_TABLES
|
| 5149 |
+
subject = {"review_trends": "review trends",
|
| 5150 |
+
"browse_node_review_topics": "category review topics"}.get(table, "review topics")
|
| 5151 |
if feedback and (str(status) in ("403", "NOT_OFFERED")
|
| 5152 |
or re.search(r"answered\s+403\b|\(403\)", text)):
|
| 5153 |
vendor = re.search(r"answered 403(?:: (.*?))?(?:; fix:|$)", text)
|
|
|
|
| 7037 |
held = all(r.strip() in cx.ROLES_TAKEN
|
| 7038 |
for r in str(role or "").replace(" / ", " + ").split(" + ") if r.strip())
|
| 7039 |
held = held and not re.search(r"missing.*role|not granted|has not granted", str(exc), re.I)
|
| 7040 |
+
fix = ("review Customer Feedback seller and marketplace eligibility and either Brand Analytics or Selling Partner Insights access, then reconnect Amazon" if table in FEEDBACK_TABLES else
|
| 7041 |
"review Amazon Fulfillment access for this seller account and reconnect Amazon" if table in ("outbound_fulfillment", "inbound_shipments") else
|
| 7042 |
"review the Pricing role's access for this seller account and reconnect Amazon" if table == "fee_estimates_api" else
|
| 7043 |
f"needs the {role} role; this application already holds it, so Amazon does not "
|
|
|
|
| 7383 |
declared.update({name: kind for name, kind in REVIEW_TOPIC_TYPES.items() if name in columns})
|
| 7384 |
if table == "review_trends":
|
| 7385 |
declared.update({name: kind for name, kind in REVIEW_TREND_TYPES.items() if name in columns})
|
| 7386 |
+
if table == "browse_node_review_topics":
|
| 7387 |
+
declared.update({name: kind for name, kind in BROWSE_NODE_TOPIC_TYPES.items() if name in columns})
|
| 7388 |
types = _ensure(con, mirror, columns, rows, declared=declared, preserve_rows=True)
|
| 7389 |
predicate = f'"{column}" >= ? AND "{column}" < ?' if end else f'"{column}" = ?'
|
| 7390 |
if column == "posted_date" and end:
|
|
|
|
| 8044 |
("child_asin_metrics_most_mentions_number_of_mentions", ("childAsinMetrics", "mostMentions", "numberOfMentions")),
|
| 8045 |
("child_asin_metrics_least_mentions_number_of_mentions", ("childAsinMetrics", "leastMentions", "numberOfMentions")),
|
| 8046 |
)
|
| 8047 |
+
#: Amazon returns at most five subtopics per topic and three snippets per topic or subtopic.
|
| 8048 |
+
REVIEW_SUBTOPICS, REVIEW_SNIPPETS = 5, 3
|
| 8049 |
+
_SUBTOPIC_TYPES = {
|
| 8050 |
+
**{f"subtopic_{i}": "VARCHAR" for i in range(1, REVIEW_SUBTOPICS + 1)},
|
| 8051 |
+
**{f"subtopic_{i}_number_of_mentions": "BIGINT" for i in range(1, REVIEW_SUBTOPICS + 1)},
|
| 8052 |
+
**{f"subtopic_{i}_occurrence_percentage": "DOUBLE" for i in range(1, REVIEW_SUBTOPICS + 1)},
|
| 8053 |
+
**{f"subtopic_{i}_review_snippet_{j}": "VARCHAR"
|
| 8054 |
+
for i in range(1, REVIEW_SUBTOPICS + 1) for j in range(1, REVIEW_SNIPPETS + 1)},
|
| 8055 |
+
}
|
| 8056 |
REVIEW_TOPIC_TYPES = {
|
| 8057 |
+
**dict.fromkeys(("asin", "marketplace_id", "sort_by", "sentiment", "topic", "review_snippets", "subtopics",
|
| 8058 |
+
"review_snippet_1", "review_snippet_2", "review_snippet_3",
|
| 8059 |
+
"browse_node_id", "browse_node_name"), "VARCHAR"),
|
| 8060 |
**{name: "BIGINT" if name.endswith("_number_of_mentions") else "DOUBLE"
|
| 8061 |
for name, _path in REVIEW_TOPIC_METRICS},
|
| 8062 |
+
**_SUBTOPIC_TYPES,
|
| 8063 |
}
|
| 8064 |
|
| 8065 |
|
| 8066 |
+
def _feedback_asins(con):
|
| 8067 |
+
"""The tenant's active child ASINs: active listings minus every parent in the latest catalogue snapshot."""
|
| 8068 |
+
columns = _columns_of(con, "amzn_listings")
|
| 8069 |
+
if "asin1" not in columns:
|
| 8070 |
+
return []
|
| 8071 |
+
active = " AND LOWER(TRIM(CAST(status AS VARCHAR))) = 'active'" if "status" in columns else ""
|
| 8072 |
+
asins = [row[0] for row in con.execute(
|
| 8073 |
+
"SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), '') IS NOT NULL"
|
| 8074 |
+
+ active + " ORDER BY 1").fetchall()]
|
| 8075 |
+
catalog = _columns_of(con, "amzn_catalog_items")
|
| 8076 |
+
if "parent_asin" not in catalog:
|
| 8077 |
+
return asins
|
| 8078 |
+
other = " AND TRIM(CAST(parent_asin AS VARCHAR)) <> TRIM(CAST(asin AS VARCHAR))" if "asin" in catalog else ""
|
| 8079 |
+
parents = {row[0] for row in con.execute(
|
| 8080 |
+
"SELECT DISTINCT TRIM(CAST(parent_asin AS VARCHAR)) FROM ("
|
| 8081 |
+
+ _latest_account_snapshot_sql(catalog, "amzn_catalog_items")
|
| 8082 |
+
+ ") AS current_catalog WHERE NULLIF(TRIM(CAST(parent_asin AS VARCHAR)), '') IS NOT NULL" + other).fetchall()}
|
| 8083 |
+
return [asin for asin in asins if asin not in parents]
|
| 8084 |
+
|
| 8085 |
+
|
| 8086 |
+
FEEDBACK_NO_ASINS = ("No source rows: the Amazon listings catalogue has no active child ASINs. "
|
| 8087 |
+
"Sync Amazon listings first.")
|
| 8088 |
+
|
| 8089 |
+
|
| 8090 |
+
def _feedback_number(value, *, count=False, percentage=False):
|
| 8091 |
+
"""A Customer Feedback metric: null, or a finite number; a count is a nonnegative integer."""
|
| 8092 |
+
import math
|
| 8093 |
+
if value is None:
|
| 8094 |
+
return None
|
| 8095 |
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
| 8096 |
+
raise ValueError("Amazon review topic returned a nonnumeric metric")
|
| 8097 |
+
if count and (type(value) is not int or value < 0):
|
| 8098 |
+
raise ValueError("Amazon review topic returned an invalid mention count")
|
| 8099 |
+
if percentage and not 0 <= value <= 100:
|
| 8100 |
+
raise ValueError("Amazon review topic returned an invalid occurrence percentage")
|
| 8101 |
+
return value
|
| 8102 |
+
|
| 8103 |
+
|
| 8104 |
+
def _feedback_snippets(snippets):
|
| 8105 |
+
if snippets is not None and (not isinstance(snippets, list) or len(snippets) > REVIEW_SNIPPETS
|
| 8106 |
+
or any(not isinstance(snippet, str) for snippet in snippets)):
|
| 8107 |
+
raise ValueError("Amazon review snippets must be null or a list of up to three strings")
|
| 8108 |
+
return snippets or []
|
| 8109 |
+
|
| 8110 |
+
|
| 8111 |
+
def _subtopic_columns(topic, *, mentions=True):
|
| 8112 |
+
"""Up to five subtopics as columns: name, metrics and up to three snippets each."""
|
| 8113 |
+
subtopics = topic.get("subtopics")
|
| 8114 |
+
if subtopics is not None and (not isinstance(subtopics, list) or len(subtopics) > REVIEW_SUBTOPICS
|
| 8115 |
+
or any(not isinstance(sub, dict) for sub in subtopics)):
|
| 8116 |
+
raise ValueError("Amazon review subtopics must be null or a list of up to five objects")
|
| 8117 |
+
out = {}
|
| 8118 |
+
for i, sub in enumerate(subtopics or [], 1):
|
| 8119 |
+
metrics = sub.get("metrics")
|
| 8120 |
+
if (not isinstance(sub.get("subtopic"), str) or not sub["subtopic"].strip()
|
| 8121 |
+
or (metrics is not None and not isinstance(metrics, dict))):
|
| 8122 |
+
raise ValueError("Amazon review subtopic requires a subtopic name and a metrics object")
|
| 8123 |
+
metrics = metrics or {}
|
| 8124 |
+
out[f"subtopic_{i}"] = sub["subtopic"]
|
| 8125 |
+
if mentions:
|
| 8126 |
+
out[f"subtopic_{i}_number_of_mentions"] = _feedback_number(metrics.get("numberOfMentions"), count=True)
|
| 8127 |
+
out[f"subtopic_{i}_occurrence_percentage"] = _feedback_number(metrics.get("occurrencePercentage"))
|
| 8128 |
+
for j, snippet in enumerate(_feedback_snippets(sub.get("reviewSnippets")), 1):
|
| 8129 |
+
out[f"subtopic_{i}_review_snippet_{j}"] = snippet
|
| 8130 |
+
return out
|
| 8131 |
+
|
| 8132 |
+
|
| 8133 |
+
def _null_unresolved_metrics(records):
|
| 8134 |
+
"""Read an unfilled Amazon placeholder (a "$" string inside a metrics object) as null; return how many."""
|
| 8135 |
+
count = 0
|
| 8136 |
+
|
| 8137 |
+
def walk(node, inside):
|
| 8138 |
+
nonlocal count
|
| 8139 |
+
items = node.items() if isinstance(node, dict) else enumerate(node) if isinstance(node, list) else ()
|
| 8140 |
+
for key, value in list(items):
|
| 8141 |
+
if inside and isinstance(value, str) and value.startswith("$"):
|
| 8142 |
+
node[key] = None
|
| 8143 |
+
count += 1
|
| 8144 |
+
elif isinstance(value, (dict, list)):
|
| 8145 |
+
walk(value, inside or (isinstance(key, str) and key.endswith(("Metrics", "metrics"))))
|
| 8146 |
+
walk(records, False)
|
| 8147 |
+
return count
|
| 8148 |
+
|
| 8149 |
+
|
| 8150 |
+
def _drop_repeated_topics(records):
|
| 8151 |
+
"""Drop a topic Amazon repeats with identical metrics in one sorted answer; return how many. A conflicting repeat stays."""
|
| 8152 |
+
dropped = 0
|
| 8153 |
+
for record in records:
|
| 8154 |
+
topics = record.get("topics")
|
| 8155 |
+
if not isinstance(topics, dict):
|
| 8156 |
+
continue
|
| 8157 |
+
for key in ("positiveTopics", "negativeTopics"):
|
| 8158 |
+
group = topics.get(key)
|
| 8159 |
+
if not isinstance(group, list):
|
| 8160 |
+
continue
|
| 8161 |
+
kept, first = [], {}
|
| 8162 |
+
for topic in group:
|
| 8163 |
+
name = topic.get("topic") if isinstance(topic, dict) else None
|
| 8164 |
+
metrics = (json.dumps({k: v for k, v in topic.items() if k.endswith("Metrics")}, sort_keys=True, default=str)
|
| 8165 |
+
if isinstance(topic, dict) else None)
|
| 8166 |
+
if isinstance(name, str) and name in first and first[name] == metrics:
|
| 8167 |
+
dropped += 1
|
| 8168 |
+
continue
|
| 8169 |
+
if isinstance(name, str):
|
| 8170 |
+
first.setdefault(name, metrics)
|
| 8171 |
+
kept.append(topic)
|
| 8172 |
+
topics[key] = kept
|
| 8173 |
+
return dropped
|
| 8174 |
+
|
| 8175 |
+
|
| 8176 |
+
def _sorted_feedback(read):
|
| 8177 |
+
"""Stamp each record with the sort it answered, in FEEDBACK_SORTS order."""
|
| 8178 |
+
return [{**record, "_sort_by": sort} for sort in _client().FEEDBACK_SORTS for record in read.get(sort) or []]
|
| 8179 |
+
|
| 8180 |
+
|
| 8181 |
def _review_topic_evidence(records, asins, marketplace):
|
| 8182 |
"""Count raw vendor topics and retain their metrics before the row parser runs."""
|
| 8183 |
import math
|
| 8184 |
+
sorts = _client().FEEDBACK_SORTS
|
| 8185 |
+
expected = {(asin, sort) for asin in asins for sort in sorts}
|
| 8186 |
+
counts, metrics, absent, seen = {}, {}, [], set()
|
| 8187 |
+
metricless = 0
|
| 8188 |
for record in records:
|
| 8189 |
+
asin, sort = record.get("asin"), record.get("_sort_by")
|
| 8190 |
+
if (asin, sort) not in expected or (asin, sort) in seen or record.get("marketplaceId") != marketplace:
|
| 8191 |
raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign ASIN or marketplace")
|
| 8192 |
+
seen.add((asin, sort))
|
| 8193 |
no_content = record.get("_no_feedback_reason")
|
| 8194 |
topics = record.get("topics")
|
| 8195 |
if not no_content and not isinstance(topics, dict):
|
|
|
|
| 8200 |
group = [] if group is None else group
|
| 8201 |
if not isinstance(group, list):
|
| 8202 |
raise ValueError("Amazon getItemReviewTopics returned an invalid sentiment topic list")
|
| 8203 |
+
counts[json.dumps([asin, sort, sentiment], separators=(",", ":"))] = len(group)
|
| 8204 |
total += len(group)
|
| 8205 |
for topic in group:
|
| 8206 |
+
# Amazon omits asinMetrics on some real topics: the topic stays, its metrics read null and are counted.
|
| 8207 |
if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip()
|
| 8208 |
+
or not isinstance(topic.get("asinMetrics", {}), (dict, type(None)))):
|
| 8209 |
+
raise ValueError("Amazon review topic requires a topic name and an asinMetrics object or none")
|
| 8210 |
+
metricless += not isinstance(topic.get("asinMetrics"), dict)
|
| 8211 |
+
identity = json.dumps([asin, marketplace, sort, sentiment, topic["topic"]], separators=(",", ":"))
|
| 8212 |
if identity in metrics:
|
| 8213 |
raise ValueError("Amazon returned duplicate topics for one ASIN and sentiment")
|
| 8214 |
values = []
|
|
|
|
| 8228 |
values.append(value)
|
| 8229 |
metrics[identity] = values
|
| 8230 |
if not total:
|
| 8231 |
+
absent.append({"asin": asin, "sort_by": sort,
|
| 8232 |
+
"code": "NO_CONTENT" if no_content else "NO_TOPICS",
|
| 8233 |
"reason": no_content or "Amazon returned no positive or negative review topics for this ASIN and marketplace."})
|
| 8234 |
if seen != expected:
|
| 8235 |
raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN")
|
| 8236 |
+
return {"count": sum(counts.values()), "metricless": metricless, "source_counts": counts, "source_metrics": metrics,
|
| 8237 |
"marketplace_id": marketplace, "absent_asins": absent}
|
| 8238 |
|
| 8239 |
|
| 8240 |
+
def review_topic_rows(records, document, browse_nodes=None):
|
| 8241 |
"""Flatten positive and negative topics while retaining null metrics and vendor percentage units."""
|
| 8242 |
rows = []
|
| 8243 |
for record in records:
|
| 8244 |
+
node = (browse_nodes or {}).get(record["asin"]) or {}
|
| 8245 |
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8246 |
for topic in (record.get("topics") or {}).get(key) or []:
|
| 8247 |
row = dict.fromkeys(REVIEW_TOPIC_TYPES)
|
| 8248 |
row.update(_flatten_row(topic))
|
| 8249 |
+
row.update(asin=record["asin"], marketplace_id=record["marketplaceId"], sentiment=sentiment,
|
| 8250 |
+
sort_by=record.get("_sort_by"), browse_node_id=node.get("browse_node_id"),
|
| 8251 |
+
browse_node_name=node.get("browse_node_name"))
|
| 8252 |
row.update(_flatten_row({k: record[k] for k in ("itemName", "countryCode", "dateRange") if k in record}))
|
| 8253 |
+
# All three Customer Feedback tables store the review period as a calendar date.
|
| 8254 |
+
for name in ("date_range_start_date", "date_range_end_date"):
|
| 8255 |
+
row[name] = str(row[name])[:10] if row.get(name) is not None else None
|
| 8256 |
+
for index, snippet in enumerate(_feedback_snippets(topic.get("reviewSnippets")), 1):
|
|
|
|
| 8257 |
row[f"review_snippet_{index}"] = snippet
|
| 8258 |
+
row.update(_subtopic_columns(topic))
|
| 8259 |
rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTopics"))
|
| 8260 |
return rows
|
| 8261 |
|
| 8262 |
|
| 8263 |
def _sync_review_topics(con, fields):
|
| 8264 |
+
"""Replace the complete active child ASIN topic snapshot, both sorts, with same-pass vendor reconciliation."""
|
| 8265 |
+
asins = _feedback_asins(con)
|
|
|
|
|
|
|
|
|
|
| 8266 |
marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
|
| 8267 |
+
read = _client().item_review_topics(fields, asins)
|
| 8268 |
+
records = _sorted_feedback(read)
|
| 8269 |
+
unresolved = _null_unresolved_metrics(records)
|
| 8270 |
+
repeated = _drop_repeated_topics(records)
|
| 8271 |
evidence = _review_topic_evidence(records, asins, marketplace)
|
| 8272 |
document = "operation:" + uuid.uuid4().hex
|
| 8273 |
+
rows = review_topic_rows(records, document, read.get("browse_nodes"))
|
| 8274 |
period = datetime.now(timezone.utc).date().isoformat()
|
| 8275 |
for row in rows:
|
| 8276 |
row["_snapshot_date"] = period
|
| 8277 |
reason = ("Amazon Customer Feedback returned no review topics for this marketplace. "
|
| 8278 |
"The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
|
| 8279 |
+
if asins else FEEDBACK_NO_ASINS)
|
| 8280 |
marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
|
| 8281 |
+
"unresolved_metrics": unresolved, "repeated_topics": repeated,
|
| 8282 |
"empty_reason": reason if not rows else ""}
|
| 8283 |
count = _replace_period(con, "review_topics", rows,
|
| 8284 |
sorted(set(REVIEW_TOPIC_TYPES) | {key for row in rows for key in row}),
|
| 8285 |
"_snapshot_date", period, marker=marker, snapshot=True)
|
| 8286 |
_record_state(con, entity_key("limit_review_topics"), "done", detail=json.dumps({
|
| 8287 |
+
"reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"], "unresolved_metrics": unresolved, "repeated_topics": repeated}))
|
| 8288 |
return count
|
| 8289 |
|
| 8290 |
|
|
|
|
| 8292 |
**dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "month",
|
| 8293 |
"date_range_start_date", "date_range_end_date"), "VARCHAR"),
|
| 8294 |
"asin_metrics_occurrence_percentage": "DOUBLE",
|
| 8295 |
+
"parent_asin_metrics_occurrence_percentage": "DOUBLE",
|
| 8296 |
+
"browse_node_metrics_occurrence_percentage_all_products": "DOUBLE",
|
| 8297 |
}
|
| 8298 |
+
_TREND_METRICS = (("asin_metrics_occurrence_percentage", ("asinMetrics", "occurrencePercentage")),
|
| 8299 |
+
("parent_asin_metrics_occurrence_percentage", ("parentAsinMetrics", "occurrencePercentage")),
|
| 8300 |
+
("browse_node_metrics_occurrence_percentage_all_products",
|
| 8301 |
+
("browseNodeMetrics", "occurrencePercentage", "allProducts")))
|
| 8302 |
+
|
| 8303 |
+
|
| 8304 |
+
def _trend_window(date_range):
|
| 8305 |
+
"""`(start, end, month)` of one trend point: ISO dates, start before end, month the last one inside the window."""
|
| 8306 |
+
start, end = (date_range.get(name) for name in ("startDate", "endDate"))
|
| 8307 |
+
days = []
|
| 8308 |
+
for value in (start, end):
|
| 8309 |
+
if not isinstance(value, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}(T[0-9:.]+(Z|[+-]\d{2}:\d{2})?)?", value):
|
| 8310 |
+
raise ValueError("Amazon review trend requires ISO calendar dates")
|
| 8311 |
+
try:
|
| 8312 |
+
days.append(datetime.strptime(value[:10], "%Y-%m-%d"))
|
| 8313 |
+
if len(value) > 10:
|
| 8314 |
+
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
| 8315 |
+
except ValueError:
|
| 8316 |
+
raise ValueError("Amazon review trend requires ISO calendar dates") from None
|
| 8317 |
+
if days[0] >= days[1]:
|
| 8318 |
+
raise ValueError("Amazon review trend dateRange must start before it ends")
|
| 8319 |
+
return start[:10], end[:10], (days[1] - timedelta(days=1)).strftime("%Y-%m")
|
| 8320 |
+
|
| 8321 |
+
|
| 8322 |
+
def _trend_values(point):
|
| 8323 |
+
"""The ASIN, parent and browse node occurrence percentages of one trend point."""
|
| 8324 |
+
import math
|
| 8325 |
+
values = []
|
| 8326 |
+
for _name, path in _TREND_METRICS:
|
| 8327 |
+
value = point
|
| 8328 |
+
for segment in path:
|
| 8329 |
+
value = value.get(segment) if isinstance(value, dict) else None
|
| 8330 |
+
if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
|
| 8331 |
+
or not math.isfinite(value) or not 0 <= value <= 100):
|
| 8332 |
+
raise ValueError("Amazon review trend returned an invalid occurrence percentage")
|
| 8333 |
+
values.append(value)
|
| 8334 |
+
return values
|
| 8335 |
|
| 8336 |
|
| 8337 |
def _review_trend_evidence(records, asins, marketplace):
|
| 8338 |
"""Count vendor trend points and retain source values before flattening or storing rows."""
|
|
|
|
| 8339 |
expected, seen, counts, metrics, absent = set(asins), set(), {}, {}, []
|
| 8340 |
+
metricless = 0
|
| 8341 |
for record in records:
|
| 8342 |
asin = record.get("asin")
|
| 8343 |
if asin not in expected or asin in seen or record.get("marketplaceId") != marketplace:
|
|
|
|
| 8367 |
counts[group] += len(topic["trendMetrics"])
|
| 8368 |
for point in topic["trendMetrics"]:
|
| 8369 |
if (not isinstance(point, dict) or not isinstance(point.get("dateRange"), dict)
|
| 8370 |
+
or not isinstance(point.get("asinMetrics", {}), (dict, type(None)))):
|
| 8371 |
+
raise ValueError("Amazon review trend requires a dateRange object and an asinMetrics object or none")
|
| 8372 |
+
metricless += not isinstance(point.get("asinMetrics"), dict)
|
| 8373 |
+
start, end, month = _trend_window(point["dateRange"])
|
| 8374 |
+
identity = json.dumps([asin, marketplace, sentiment, topic["topic"], month],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8375 |
separators=(",", ":"))
|
| 8376 |
if identity in metrics:
|
| 8377 |
raise ValueError("Amazon returned duplicate trend points for one ASIN, topic and month")
|
| 8378 |
+
metrics[identity] = [start, end, *_trend_values(point)]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8379 |
total += counts[group]
|
| 8380 |
if not total:
|
| 8381 |
absent.append({"asin": asin, "code": "NO_CONTENT" if no_content else "NO_TRENDS",
|
| 8382 |
"reason": no_content or "Amazon returned no positive or negative review trend points for this ASIN and marketplace."})
|
| 8383 |
if seen != expected:
|
| 8384 |
raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN")
|
| 8385 |
+
return {"count": sum(counts.values()), "metricless": metricless, "source_counts": counts, "source_metrics": metrics,
|
| 8386 |
"marketplace_id": marketplace, "absent_asins": absent}
|
| 8387 |
|
| 8388 |
|
|
|
|
| 8393 |
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8394 |
for topic in (record.get("reviewTrends") or {}).get(key) or []:
|
| 8395 |
for point in topic["trendMetrics"]:
|
| 8396 |
+
start, end, month = _trend_window(point["dateRange"])
|
| 8397 |
row = {"asin": record["asin"], "marketplace_id": record["marketplaceId"],
|
| 8398 |
+
"sentiment": sentiment, "topic": topic["topic"], "month": month,
|
| 8399 |
+
"date_range_start_date": start, "date_range_end_date": end,
|
| 8400 |
+
**dict(zip((name for name, _path in _TREND_METRICS), _trend_values(point)))}
|
| 8401 |
rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTrends"))
|
| 8402 |
return rows
|
| 8403 |
|
| 8404 |
|
| 8405 |
def _sync_review_trends(con, fields):
|
| 8406 |
"""Replace the full six-month review snapshot with independently counted source evidence."""
|
| 8407 |
+
asins = _feedback_asins(con)
|
|
|
|
|
|
|
|
|
|
| 8408 |
marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
|
| 8409 |
records = _client().item_review_feedback(fields, asins, operation="getItemReviewTrends")
|
| 8410 |
+
unresolved = _null_unresolved_metrics(records)
|
| 8411 |
+
repeated = _drop_repeated_topics(records)
|
| 8412 |
evidence = _review_trend_evidence(records, asins, marketplace)
|
| 8413 |
rows = review_trend_rows(records, "operation:" + uuid.uuid4().hex)
|
| 8414 |
period = datetime.now(timezone.utc).date().isoformat()
|
|
|
|
| 8416 |
row["_snapshot_date"] = period
|
| 8417 |
reason = ("Amazon Customer Feedback returned no review trends for this marketplace. "
|
| 8418 |
"The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
|
| 8419 |
+
if asins else FEEDBACK_NO_ASINS)
|
| 8420 |
marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
|
| 8421 |
+
"unresolved_metrics": unresolved, "repeated_topics": repeated,
|
| 8422 |
"empty_reason": reason if not rows else ""}
|
| 8423 |
count = _replace_period(con, "review_trends", rows, REVIEW_TREND_TYPES,
|
| 8424 |
"_snapshot_date", period, marker=marker, snapshot=True)
|
| 8425 |
_record_state(con, entity_key("limit_review_trends"), "done", detail=json.dumps({
|
| 8426 |
+
"reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"], "unresolved_metrics": unresolved, "repeated_topics": repeated}))
|
| 8427 |
+
return count
|
| 8428 |
+
|
| 8429 |
+
|
| 8430 |
+
BROWSE_NODE_TOPIC_METRICS = (
|
| 8431 |
+
("browse_node_metrics_occurrence_percentage_all_products", ("occurrencePercentage", "allProducts")),
|
| 8432 |
+
("browse_node_metrics_occurrence_percentage_top_twenty_five_percent_products",
|
| 8433 |
+
("occurrencePercentage", "topTwentyFivePercentProducts")),
|
| 8434 |
+
("browse_node_metrics_star_rating_impact_all_products", ("starRatingImpact", "allProducts")),
|
| 8435 |
+
("browse_node_metrics_star_rating_impact_top_twenty_five_percent_products",
|
| 8436 |
+
("starRatingImpact", "topTwentyFivePercentProducts")),
|
| 8437 |
+
)
|
| 8438 |
+
BROWSE_NODE_TOPIC_TYPES = {
|
| 8439 |
+
**dict.fromkeys(("browse_node_id", "browse_node_name", "marketplace_id", "sort_by", "sentiment", "topic",
|
| 8440 |
+
"date_range_start_date", "date_range_end_date", "review_snippet_1", "review_snippet_2",
|
| 8441 |
+
"review_snippet_3", "our_asins"), "VARCHAR"),
|
| 8442 |
+
**dict.fromkeys((name for name, _path in BROWSE_NODE_TOPIC_METRICS), "DOUBLE"),
|
| 8443 |
+
**{name: kind for name, kind in _SUBTOPIC_TYPES.items() if "_number_of_mentions" not in name},
|
| 8444 |
+
"our_asin_count": "BIGINT",
|
| 8445 |
+
}
|
| 8446 |
+
|
| 8447 |
+
|
| 8448 |
+
def _browse_node_topic_evidence(records, nodes, marketplace):
|
| 8449 |
+
"""Count raw category topics per node and sort and retain their metrics before the row parser runs."""
|
| 8450 |
+
sorts = _client().FEEDBACK_SORTS
|
| 8451 |
+
expected = {(node, sort) for node in nodes for sort in sorts}
|
| 8452 |
+
counts, metrics, absent, seen = {}, {}, [], set()
|
| 8453 |
+
metricless = 0
|
| 8454 |
+
for record in records:
|
| 8455 |
+
node, sort = str(record.get("browseNodeId")), record.get("_sort_by")
|
| 8456 |
+
if (node, sort) not in expected or (node, sort) in seen or record.get("marketplaceId") != marketplace:
|
| 8457 |
+
raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign browse node or marketplace")
|
| 8458 |
+
seen.add((node, sort))
|
| 8459 |
+
no_content = record.get("_no_feedback_reason")
|
| 8460 |
+
topics = record.get("topics")
|
| 8461 |
+
if not no_content and not isinstance(topics, dict):
|
| 8462 |
+
raise ValueError("Amazon getBrowseNodeReviewTopics returned no valid topics object")
|
| 8463 |
+
total = 0
|
| 8464 |
+
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8465 |
+
group = None if no_content else topics.get(key)
|
| 8466 |
+
group = [] if group is None else group
|
| 8467 |
+
if not isinstance(group, list):
|
| 8468 |
+
raise ValueError("Amazon getBrowseNodeReviewTopics returned an invalid sentiment topic list")
|
| 8469 |
+
counts[json.dumps([node, sort, sentiment], separators=(",", ":"))] = len(group)
|
| 8470 |
+
total += len(group)
|
| 8471 |
+
for topic in group:
|
| 8472 |
+
if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip()
|
| 8473 |
+
or not isinstance(topic.get("browseNodeMetrics", {}), (dict, type(None)))):
|
| 8474 |
+
raise ValueError("Amazon category review topic requires a topic name and a browseNodeMetrics object or none")
|
| 8475 |
+
metricless += not isinstance(topic.get("browseNodeMetrics"), dict)
|
| 8476 |
+
identity = json.dumps([node, marketplace, sort, sentiment, topic["topic"]], separators=(",", ":"))
|
| 8477 |
+
if identity in metrics:
|
| 8478 |
+
raise ValueError("Amazon returned duplicate topics for one browse node and sentiment")
|
| 8479 |
+
values = []
|
| 8480 |
+
for _name, path in BROWSE_NODE_TOPIC_METRICS:
|
| 8481 |
+
value = topic.get("browseNodeMetrics")
|
| 8482 |
+
for segment in path:
|
| 8483 |
+
if value is not None and not isinstance(value, dict):
|
| 8484 |
+
raise ValueError("Amazon category review topic returned invalid nested metrics")
|
| 8485 |
+
value = value.get(segment) if value is not None else None
|
| 8486 |
+
values.append(_feedback_number(value))
|
| 8487 |
+
metrics[identity] = values
|
| 8488 |
+
if not total:
|
| 8489 |
+
absent.append({"browse_node_id": node, "sort_by": sort,
|
| 8490 |
+
"code": "NO_CONTENT" if no_content else "NO_TOPICS",
|
| 8491 |
+
"reason": no_content or "Amazon returned no positive or negative category topics for this browse node and marketplace."})
|
| 8492 |
+
if seen != expected:
|
| 8493 |
+
raise ValueError("Amazon Customer Feedback did not return a response for every browse node")
|
| 8494 |
+
return {"count": sum(counts.values()), "metricless": metricless, "source_counts": counts, "source_metrics": metrics,
|
| 8495 |
+
"marketplace_id": marketplace, "absent_browse_nodes": absent}
|
| 8496 |
+
|
| 8497 |
+
|
| 8498 |
+
def browse_node_topic_rows(records, document, our_asins):
|
| 8499 |
+
"""Flatten each browse node's category topics with the tenant's own ASINs in that node."""
|
| 8500 |
+
rows = []
|
| 8501 |
+
for record in records:
|
| 8502 |
+
node = str(record["browseNodeId"])
|
| 8503 |
+
dates = record.get("dateRange") if isinstance(record.get("dateRange"), dict) else {}
|
| 8504 |
+
ours = sorted(our_asins.get(node, {}).get("asins", ()))
|
| 8505 |
+
for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
|
| 8506 |
+
for topic in (record.get("topics") or {}).get(key) or []:
|
| 8507 |
+
row = dict.fromkeys(BROWSE_NODE_TOPIC_TYPES)
|
| 8508 |
+
row.update(browse_node_id=node, marketplace_id=record["marketplaceId"], sort_by=record["_sort_by"],
|
| 8509 |
+
browse_node_name=record.get("displayName") or our_asins.get(node, {}).get("name"),
|
| 8510 |
+
sentiment=sentiment, topic=topic["topic"],
|
| 8511 |
+
date_range_start_date=str(dates.get("startDate") or "")[:10] or None,
|
| 8512 |
+
date_range_end_date=str(dates.get("endDate") or "")[:10] or None,
|
| 8513 |
+
our_asins=json.dumps(ours, separators=(",", ":")), our_asin_count=len(ours))
|
| 8514 |
+
for name, path in BROWSE_NODE_TOPIC_METRICS:
|
| 8515 |
+
value = topic.get("browseNodeMetrics")
|
| 8516 |
+
for segment in path:
|
| 8517 |
+
value = value.get(segment) if isinstance(value, dict) else None
|
| 8518 |
+
row[name] = value
|
| 8519 |
+
for index, snippet in enumerate(_feedback_snippets(topic.get("reviewSnippets")), 1):
|
| 8520 |
+
row[f"review_snippet_{index}"] = snippet
|
| 8521 |
+
row.update(_subtopic_columns(topic, mentions=False))
|
| 8522 |
+
rows.append(_origin(row, document, len(rows) + 1, "getBrowseNodeReviewTopics"))
|
| 8523 |
+
return rows
|
| 8524 |
+
|
| 8525 |
+
|
| 8526 |
+
def _sync_browse_node_review_topics(con, fields):
|
| 8527 |
+
"""Replace the category topic snapshot of every browse node the active child ASINs sit in, both sorts."""
|
| 8528 |
+
asins = _feedback_asins(con)
|
| 8529 |
+
marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
|
| 8530 |
+
located = _client().item_browse_nodes(fields, asins)
|
| 8531 |
+
nodes, unplaced = {}, []
|
| 8532 |
+
for asin in asins:
|
| 8533 |
+
node = located.get(asin) or {}
|
| 8534 |
+
if node.get("browse_node_id"):
|
| 8535 |
+
entry = nodes.setdefault(node["browse_node_id"], {"name": node.get("browse_node_name"), "asins": set()})
|
| 8536 |
+
entry["asins"].add(asin)
|
| 8537 |
+
else:
|
| 8538 |
+
unplaced.append({"asin": asin, "code": "NO_BROWSE_NODE",
|
| 8539 |
+
"reason": node.get("_no_feedback_reason") or "Amazon returned no browse node for this ASIN."})
|
| 8540 |
+
records = _sorted_feedback(_client().browse_node_review_topics(fields, sorted(nodes)))
|
| 8541 |
+
unresolved = _null_unresolved_metrics(records)
|
| 8542 |
+
repeated = _drop_repeated_topics(records)
|
| 8543 |
+
evidence = _browse_node_topic_evidence(records, sorted(nodes), marketplace)
|
| 8544 |
+
rows = browse_node_topic_rows(records, "operation:" + uuid.uuid4().hex, nodes)
|
| 8545 |
+
period = datetime.now(timezone.utc).date().isoformat()
|
| 8546 |
+
for row in rows:
|
| 8547 |
+
row["_snapshot_date"] = period
|
| 8548 |
+
reason = (FEEDBACK_NO_ASINS if not asins else
|
| 8549 |
+
"Amazon returned no browse node for any active child ASIN. The per-ASIN reasons are recorded in the sync evidence."
|
| 8550 |
+
if not nodes else
|
| 8551 |
+
"Amazon Customer Feedback returned no category review topics for these browse nodes. "
|
| 8552 |
+
"The per-node reasons are recorded in the sync evidence. Amazon refreshes this data weekly.")
|
| 8553 |
+
marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence, "absent_asins": unplaced,
|
| 8554 |
+
"unresolved_metrics": unresolved, "repeated_topics": repeated,
|
| 8555 |
+
"empty_reason": reason if not rows else ""}
|
| 8556 |
+
count = _replace_period(con, "browse_node_review_topics", rows,
|
| 8557 |
+
sorted(set(BROWSE_NODE_TOPIC_TYPES) | {key for row in rows for key in row}),
|
| 8558 |
+
"_snapshot_date", period, marker=marker, snapshot=True)
|
| 8559 |
+
_record_state(con, entity_key("limit_browse_node_review_topics"), "done", detail=json.dumps({
|
| 8560 |
+
"reason": marker["empty_reason"], "absent_asins": unplaced,
|
| 8561 |
+
"absent_browse_nodes": evidence["absent_browse_nodes"], "unresolved_metrics": unresolved, "repeated_topics": repeated}))
|
| 8562 |
return count
|
| 8563 |
|
| 8564 |
|
|
|
|
| 8570 |
return _sync_review_topics(con, fields)
|
| 8571 |
if table == "review_trends":
|
| 8572 |
return _sync_review_trends(con, fields)
|
| 8573 |
+
if table == "browse_node_review_topics":
|
| 8574 |
+
return _sync_browse_node_review_topics(con, fields)
|
| 8575 |
if table == "economics":
|
| 8576 |
start, end = _utc(w0).date().isoformat(), _utc(w1).date().isoformat()
|
| 8577 |
if start >= end:
|
|
|
|
| 8981 |
identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sentiment"),
|
| 8982 |
row.get("topic"), row.get("month")], separators=(",", ":"))
|
| 8983 |
metrics[identity] = [row.get("date_range_start_date"), row.get("date_range_end_date"),
|
| 8984 |
+
*(row.get(name) for name, _path in _TREND_METRICS)]
|
| 8985 |
total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
|
| 8986 |
trends_match = (marker["count"] == len(rows) == len(metrics) == total
|
| 8987 |
and counts == marker["source_counts"] and metrics == marker["source_metrics"])
|
|
|
|
| 8992 |
counts = dict.fromkeys(marker["source_counts"], 0)
|
| 8993 |
metrics = {}
|
| 8994 |
for row in rows:
|
| 8995 |
+
group = json.dumps([row.get("asin"), row.get("sort_by"), row.get("sentiment")], separators=(",", ":"))
|
| 8996 |
counts[group] = counts.get(group, 0) + 1
|
| 8997 |
+
identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sort_by"),
|
| 8998 |
+
row.get("sentiment"), row.get("topic")], separators=(",", ":"))
|
| 8999 |
metrics[identity] = [row.get(name) for name, _path in REVIEW_TOPIC_METRICS]
|
| 9000 |
total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
|
| 9001 |
matches = (marker["count"] == len(rows) == len(metrics) == total
|
|
|
|
| 9003 |
return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows),
|
| 9004 |
"reason": (marker["empty_reason"] if matches and not rows else
|
| 9005 |
f"Amazon returned {marker['count']} review topics. Stored {len(rows)} topics with per-ASIN, sentiment and metric reconciliation.")}
|
| 9006 |
+
if table == "browse_node_review_topics":
|
| 9007 |
+
counts = dict.fromkeys(marker["source_counts"], 0)
|
| 9008 |
+
metrics = {}
|
| 9009 |
+
for row in rows:
|
| 9010 |
+
group = json.dumps([row.get("browse_node_id"), row.get("sort_by"), row.get("sentiment")],
|
| 9011 |
+
separators=(",", ":"))
|
| 9012 |
+
counts[group] = counts.get(group, 0) + 1
|
| 9013 |
+
identity = json.dumps([row.get("browse_node_id"), row.get("marketplace_id"), row.get("sort_by"),
|
| 9014 |
+
row.get("sentiment"), row.get("topic")], separators=(",", ":"))
|
| 9015 |
+
metrics[identity] = [row.get(name) for name, _path in BROWSE_NODE_TOPIC_METRICS]
|
| 9016 |
+
total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
|
| 9017 |
+
matches = (marker["count"] == len(rows) == len(metrics) == total
|
| 9018 |
+
and counts == marker["source_counts"] and metrics == marker["source_metrics"])
|
| 9019 |
+
return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows),
|
| 9020 |
+
"reason": (marker["empty_reason"] if matches and not rows else
|
| 9021 |
+
f"Amazon returned {marker['count']} category review topics. Stored {len(rows)} topics with per-node, sort, sentiment and metric reconciliation.")}
|
| 9022 |
if table == "fee_estimates_api":
|
| 9023 |
expected, actual = marker["count"], len(rows)
|
| 9024 |
identities = {row.get("estimate_id") for row in rows}
|
|
|
|
| 9752 |
# A watermark is written when a table's history is complete, so its next window is
|
| 9753 |
# bounded; deferring it too froze every loaded table behind one refused report.
|
| 9754 |
for table in want:
|
| 9755 |
+
# An entity snapshot owes no history, so the first load never holds it.
|
| 9756 |
if ((read_watermark(con, table) and (not refresh_loaded or table not in repairs))
|
| 9757 |
+
or table in completed or table in FEEDBACK_TABLES
|
| 9758 |
or (table == "settlements" and not refresh_loaded)):
|
| 9759 |
continue
|
| 9760 |
report["tables"][table] = {"state": "deferred", "reason":
|
|
|
|
| 9767 |
_record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail"))
|
| 9768 |
want = [table for table in want if (read_watermark(con, table)
|
| 9769 |
and (not refresh_loaded or table not in repairs))
|
| 9770 |
+
or table in completed or table in FEEDBACK_TABLES
|
| 9771 |
or (table == "settlements" and not refresh_loaded)]
|
| 9772 |
if scheduled and not first_load:
|
| 9773 |
ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
|
|
|
|
| 9795 |
elif not first_load:
|
| 9796 |
# The vendor's report floor also applies to an operator's Sync now click.
|
| 9797 |
for table in list(want):
|
| 9798 |
+
# A Customer Feedback refusal stored nothing, so its weekly floor counts from a pull that landed.
|
| 9799 |
+
if (table in FEEDBACK_TABLES
|
| 9800 |
+
and (_read_state(con, entity_key(table)) or {}).get("state") in ("refused", "failed")):
|
| 9801 |
+
continue
|
| 9802 |
reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)),
|
| 9803 |
datetime.now(timezone.utc))
|
| 9804 |
if reason:
|
|
|
|
| 10909 |
out[table] = {"state": "unproven", "reason": "table has never synced",
|
| 10910 |
"reconciles": spec.get("validates_against")}
|
| 10911 |
continue
|
| 10912 |
+
if table in ("fee_estimates_api", "replenishment", "inbound_shipments", *FEEDBACK_TABLES):
|
| 10913 |
out[table] = {"rows": n, "reconciles": spec.get("validates_against"),
|
| 10914 |
**_validate_margin(con, table)}
|
| 10915 |
continue
|
|
|
|
| 11595 |
"numberOfMentions": generation[0] + 2, "occurrencePercentage": 12.5,
|
| 11596 |
"starRatingImpact": 0.25}}], "negativeTopics": None}} for asin in asins]
|
| 11597 |
|
| 11598 |
+
def browse_nodes(_fields, asins):
|
| 11599 |
+
return {asin: {"browse_node_id": "1000", "browse_node_name": "Fixture node"} for asin in asins}
|
| 11600 |
+
|
| 11601 |
+
def review_topic_records(fields, asins):
|
| 11602 |
+
empty = [{"asin": asin, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": {"positiveTopics": None}}
|
| 11603 |
+
for asin in asins]
|
| 11604 |
+
return {"MENTIONS": feedback_records(fields, asins), "STAR_RATING_IMPACT": empty,
|
| 11605 |
+
"browse_nodes": browse_nodes(fields, asins)}
|
| 11606 |
+
|
| 11607 |
+
def category_records(_fields, nodes):
|
| 11608 |
+
calls.append(("browse_node_review_topics", {}))
|
| 11609 |
+
topics = [{"topic": f"Fixture category {generation[0]} {i}", "browseNodeMetrics": {
|
| 11610 |
+
"occurrencePercentage": {"allProducts": 12.5}}} for i in range(size("browse_node_review_topics"))]
|
| 11611 |
+
return {"MENTIONS": [{"browseNodeId": node, "marketplaceId": cx.US_MARKETPLACE_ID,
|
| 11612 |
+
"topics": {"positiveTopics": topics}} for node in nodes],
|
| 11613 |
+
"STAR_RATING_IMPACT": [{"browseNodeId": node, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": {}}
|
| 11614 |
+
for node in nodes]}
|
| 11615 |
+
|
| 11616 |
def snapshot_counts(requests):
|
| 11617 |
return Counter(table for table, _kwargs in requests if table in snapshots)
|
| 11618 |
|
|
|
|
| 11652 |
patch.object(cx, "outbound_fulfillment_orders", side_effect=outbound_records),
|
| 11653 |
patch.object(cx, "inbound_shipment_items", side_effect=inbound_records),
|
| 11654 |
patch.object(cx, "item_review_feedback", side_effect=feedback_records),
|
| 11655 |
+
patch.object(cx, "item_review_topics", side_effect=review_topic_records),
|
| 11656 |
+
patch.object(cx, "item_browse_nodes", side_effect=browse_nodes),
|
| 11657 |
+
patch.object(cx, "browse_node_review_topics", side_effect=category_records),
|
| 11658 |
patch.object(cx, "replenishment_metrics", side_effect=replenishment_records),
|
| 11659 |
patch.object(cx, "awd_inventory", return_value=[]),
|
| 11660 |
patch.object(cx, "access_token", side_effect=AssertionError("offline snapshot sync fixture")),
|
|
|
|
| 13147 |
"occurrencePercentage": 12.5, "starRatingImpact": 0.2}}],
|
| 13148 |
"negativeTopics": [{"topic": "Fixture assembly", "asinMetrics": {"numberOfMentions": 2,
|
| 13149 |
"occurrencePercentage": 6.25, "starRatingImpact": -0.1}}]}}]
|
| 13150 |
+
node = {"browse_node_id": "3734491", "browse_node_name": "Fixture node"}
|
| 13151 |
+
review_topics = {**{sort: feedback for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {"B000000001": node}}
|
| 13152 |
+
category = {sort: [{"browseNodeId": "3734491", "displayName": "Fixture node", "marketplaceId": cx.US_MARKETPLACE_ID,
|
| 13153 |
+
"dateRange": {"startDate": "2026-05-06T00:00:00.000Z", "endDate": "2026-11-06T00:00:00.000Z"},
|
| 13154 |
+
"topics": {"positiveTopics": [{"topic": "Fixture glow", "browseNodeMetrics": {
|
| 13155 |
+
"occurrencePercentage": {"allProducts": 18.15, "topTwentyFivePercentProducts": 22.56},
|
| 13156 |
+
"starRatingImpact": {"allProducts": 1.58, "topTwentyFivePercentProducts": 1.15}}}],
|
| 13157 |
+
"negativeTopics": [{"topic": "Fixture size", "browseNodeMetrics": {
|
| 13158 |
+
"occurrencePercentage": {"allProducts": 14.31}}}]}}]
|
| 13159 |
+
for sort in cx.FEEDBACK_SORTS}
|
| 13160 |
|
| 13161 |
try:
|
| 13162 |
with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \
|
|
|
|
| 13167 |
patch.object(cx, "inbound_shipment_items", return_value=inbound), \
|
| 13168 |
patch.object(cx, "replenishment_metrics", return_value=replenishment), \
|
| 13169 |
patch.object(cx, "item_review_feedback", return_value=feedback), \
|
| 13170 |
+
patch.object(cx, "item_review_topics", return_value=review_topics), \
|
| 13171 |
+
patch.object(cx, "item_browse_nodes", return_value={"B000000001": node}), \
|
| 13172 |
+
patch.object(cx, "browse_node_review_topics", return_value=category), \
|
| 13173 |
patch.object(module, "_fee_estimate_inputs", return_value=[fee_input]), \
|
| 13174 |
patch.object(cx, "fee_estimates", side_effect=lambda _f, estimates: _fixture_fee_records(estimates)):
|
| 13175 |
for table in GROUP_A_TABLES:
|
|
|
|
| 13400 |
finally:
|
| 13401 |
con.close()
|
| 13402 |
|
| 13403 |
+
# A scheduled pass during an unfinished first load still runs a Customer Feedback snapshot.
|
| 13404 |
+
def feedback_pass(exempt):
|
| 13405 |
+
con = duckdb.connect(":memory:")
|
| 13406 |
+
con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
|
| 13407 |
+
"cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
|
| 13408 |
+
rt = _FixtureSyncRuntime("group-a-feedback-fixture", ("review_topics",), "group-a-account")
|
| 13409 |
+
asked = []
|
| 13410 |
+
try:
|
| 13411 |
+
with _fixture_sync_store(con, rt), \
|
| 13412 |
+
patch.object(module, "datetime", Clock), \
|
| 13413 |
+
patch.object(module, "FEEDBACK_TABLES", FEEDBACK_TABLES if exempt else ()), \
|
| 13414 |
+
patch.object(module, "credential", return_value={"fixture": True}), \
|
| 13415 |
+
patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
|
| 13416 |
+
patch.object(module, "_sync_event"), \
|
| 13417 |
+
patch.object(cx, "item_review_topics", side_effect=lambda _f, asins: asked.append(asins) or {
|
| 13418 |
+
**{sort: [] for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {}}), \
|
| 13419 |
+
patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \
|
| 13420 |
+
patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")):
|
| 13421 |
+
with _fixture_account_state(con, rt, write=True) as account:
|
| 13422 |
+
record_watermark(account, "orders", end)
|
| 13423 |
+
_record_state(account, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end))
|
| 13424 |
+
result = sync(rt.key, tables=("review_topics",), rt=rt, log=lambda *_: None, scheduled=True)
|
| 13425 |
+
with _fixture_account_state(con, rt) as account:
|
| 13426 |
+
tick = _read_state(account, entity_key("sync_tick_review_topics"))
|
| 13427 |
+
return asked, (result.get("tables") or {}).get("review_topics") or {}, tick
|
| 13428 |
+
finally:
|
| 13429 |
+
con.close()
|
| 13430 |
+
asked, row, tick = feedback_pass(True)
|
| 13431 |
+
red_asked, red_row, red_tick = feedback_pass(False)
|
| 13432 |
+
ck("a scheduled pass during an unfinished first load runs review topics and records its tick",
|
| 13433 |
+
asked == [[]] and (tick or {}).get("state") == "done" and row.get("state") != "deferred"
|
| 13434 |
+
and not red_asked and not red_tick and red_row.get("state") == "deferred"
|
| 13435 |
+
and "first load" in red_row.get("reason", ""))
|
| 13436 |
+
|
| 13437 |
+
# A Sync now click retries a refused Customer Feedback table inside its weekly floor.
|
| 13438 |
+
def click_after(landed):
|
| 13439 |
+
con = duckdb.connect(":memory:")
|
| 13440 |
+
con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
|
| 13441 |
+
"cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
|
| 13442 |
+
rt = _FixtureSyncRuntime("group-a-feedback-click", ("review_topics",), "group-a-account")
|
| 13443 |
+
asked = []
|
| 13444 |
+
try:
|
| 13445 |
+
with _fixture_sync_store(con, rt), \
|
| 13446 |
+
patch.object(module, "datetime", Clock), \
|
| 13447 |
+
patch.object(module, "credential", return_value={"fixture": True}), \
|
| 13448 |
+
patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
|
| 13449 |
+
patch.object(module, "_sync_event"), \
|
| 13450 |
+
patch.object(cx, "item_review_topics", side_effect=lambda _f, asins: asked.append(asins) or {
|
| 13451 |
+
**{sort: [] for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {}}), \
|
| 13452 |
+
patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \
|
| 13453 |
+
patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")):
|
| 13454 |
+
with _fixture_account_state(con, rt, write=True) as account:
|
| 13455 |
+
record_watermark(account, "orders", end)
|
| 13456 |
+
_record_state(account, entity_key(FIRST_LOAD_END_KEY), "done", detail=_iso(end))
|
| 13457 |
+
_record_state(account, entity_key("sync_tick_review_topics"), "done")
|
| 13458 |
+
_record_state(account, entity_key("review_topics"), landed, 0, "fixture landed state")
|
| 13459 |
+
result = sync(rt.key, tables=("review_topics",), rt=rt, log=lambda *_: None)
|
| 13460 |
+
return asked, (result.get("tables") or {}).get("review_topics") or {}
|
| 13461 |
+
finally:
|
| 13462 |
+
con.close()
|
| 13463 |
+
retried, retried_row = click_after("refused")
|
| 13464 |
+
held, held_row = click_after("done")
|
| 13465 |
+
ck("Sync now retries a refused review table inside its weekly floor and defers a landed one",
|
| 13466 |
+
retried == [[]] and retried_row.get("state") != "deferred"
|
| 13467 |
+
and not held and held_row.get("state") == "deferred")
|
| 13468 |
+
|
| 13469 |
# Client pagination: an empty middle page must not end the traversal.
|
| 13470 |
for operation, interval in cx.ENTITY_INTERVALS.items():
|
| 13471 |
if operation == "getMyFeesEstimates":
|
platform/modules/product_data.py
CHANGED
|
@@ -741,7 +741,45 @@ def _prune_product_overlay_bounded(source, generation):
|
|
| 741 |
|
| 742 |
|
| 743 |
def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=()):
|
| 744 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 745 |
from harness.canonical_source import bounded, BATCH
|
| 746 |
with bounded(cur, 'identity', [], st) as identity:
|
| 747 |
def codes():
|
|
@@ -762,11 +800,9 @@ def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=(
|
|
| 762 |
f'FROM "{raw.table}" GROUP BY pid HAVING count(DISTINCT json_extract_string(body, \'/code\')) > 1 LIMIT 1').fetchone()
|
| 763 |
if collision:
|
| 764 |
raise ValueError(f'product_data: pid collision at {collision[0]}. Refusing the catalogue.')
|
| 765 |
-
selected = '' if ids is None else " WHERE try_cast(json_extract_string(body, '/pid') AS BIGINT) IN (SELECT unnest(?::BIGINT[]))"
|
| 766 |
-
params = [] if ids is None else [list(ids)]
|
| 767 |
population = identity.snapshot(())
|
| 768 |
cur.execute(f'INSERT INTO "{population.table}" SELECT min(ordinal), body FROM "{raw.table}"'
|
| 769 |
-
|
| 770 |
# Keep the tracked snapshot name while materialising its sorted, typed seek key.
|
| 771 |
cur.execute(f'CREATE OR REPLACE TEMP TABLE "{population.table}" AS '
|
| 772 |
f"SELECT ordinal, body, try_cast(json_extract_string(body, '/pid') AS BIGINT) pid "
|
|
@@ -778,14 +814,9 @@ def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=(
|
|
| 778 |
'WHERE pid > ? ORDER BY pid LIMIT ?', [last, BATCH]).fetchall()
|
| 779 |
if not codes:
|
| 780 |
return
|
| 781 |
-
|
| 782 |
-
rows = ([{'pid': pid, 'code': code} for pid, code in codes]
|
| 783 |
-
if needed is not None and not needed else
|
| 784 |
-
pool(team_id=team_id, raw_fields=raw_fields, needed=needed))
|
| 785 |
-
yield rows
|
| 786 |
last = codes[-1][0]
|
| 787 |
-
|
| 788 |
-
codes.clear()
|
| 789 |
|
| 790 |
|
| 791 |
def _fba_by_code(cat):
|
|
|
|
| 741 |
|
| 742 |
|
| 743 |
def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=()):
|
| 744 |
+
"""Project bounded code batches: the whole active catalogue, or only the page's `ids`."""
|
| 745 |
+
from contextlib import closing
|
| 746 |
+
from harness.canonical_source import bounded
|
| 747 |
+
with closing(_catalogue_codes(cur, st) if ids is None else _page_codes(cur, ids)) as batches:
|
| 748 |
+
for codes in batches:
|
| 749 |
+
with bounded(cur, 'product', [code for _, code in codes], st):
|
| 750 |
+
rows = ([{'pid': pid, 'code': code} for pid, code in codes]
|
| 751 |
+
if needed is not None and not needed else
|
| 752 |
+
pool(team_id=team_id, raw_fields=raw_fields, needed=needed))
|
| 753 |
+
yield rows
|
| 754 |
+
rows.clear()
|
| 755 |
+
codes.clear()
|
| 756 |
+
|
| 757 |
+
|
| 758 |
+
def _page_codes(cur, ids):
|
| 759 |
+
"""`[(pid, code)]` batches for the active products whose pid is in `ids`, sorted by pid.
|
| 760 |
+
|
| 761 |
+
A pid is a crc32 the mirror cannot compute, so each active code is hashed once and only the
|
| 762 |
+
selected ones are kept. The collision refusal covers the selected pids only.
|
| 763 |
+
"""
|
| 764 |
+
from harness.canonical_source import BATCH
|
| 765 |
+
wanted, found = {int(pid) for pid in ids}, {}
|
| 766 |
+
result = cur.execute('SELECT id, default_code FROM product_product WHERE active = TRUE')
|
| 767 |
+
while True:
|
| 768 |
+
page = result.fetchmany(BATCH)
|
| 769 |
+
if not page:
|
| 770 |
+
break
|
| 771 |
+
for product_id, default_code in page:
|
| 772 |
+
code = str(default_code).strip() if default_code else f"pid:{product_id}"
|
| 773 |
+
pid = sku_pid(code)
|
| 774 |
+
if pid in wanted and found.setdefault(pid, code) != code:
|
| 775 |
+
raise ValueError(f'product_data: pid collision at {pid}. Refusing the catalogue.')
|
| 776 |
+
selected = sorted(found.items())
|
| 777 |
+
for start in range(0, len(selected), BATCH):
|
| 778 |
+
yield selected[start:start + BATCH]
|
| 779 |
+
|
| 780 |
+
|
| 781 |
+
def _catalogue_codes(cur, st):
|
| 782 |
+
"""`[(pid, code)]` batches over every active product, refusing a pid collision anywhere."""
|
| 783 |
from harness.canonical_source import bounded, BATCH
|
| 784 |
with bounded(cur, 'identity', [], st) as identity:
|
| 785 |
def codes():
|
|
|
|
| 800 |
f'FROM "{raw.table}" GROUP BY pid HAVING count(DISTINCT json_extract_string(body, \'/code\')) > 1 LIMIT 1').fetchone()
|
| 801 |
if collision:
|
| 802 |
raise ValueError(f'product_data: pid collision at {collision[0]}. Refusing the catalogue.')
|
|
|
|
|
|
|
| 803 |
population = identity.snapshot(())
|
| 804 |
cur.execute(f'INSERT INTO "{population.table}" SELECT min(ordinal), body FROM "{raw.table}"'
|
| 805 |
+
' GROUP BY body')
|
| 806 |
# Keep the tracked snapshot name while materialising its sorted, typed seek key.
|
| 807 |
cur.execute(f'CREATE OR REPLACE TEMP TABLE "{population.table}" AS '
|
| 808 |
f"SELECT ordinal, body, try_cast(json_extract_string(body, '/pid') AS BIGINT) pid "
|
|
|
|
| 814 |
'WHERE pid > ? ORDER BY pid LIMIT ?', [last, BATCH]).fetchall()
|
| 815 |
if not codes:
|
| 816 |
return
|
| 817 |
+
# The consumer clears each batch, so the seek key is read before it is lent.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 818 |
last = codes[-1][0]
|
| 819 |
+
yield codes
|
|
|
|
| 820 |
|
| 821 |
|
| 822 |
def _fba_by_code(cat):
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
272331cb0e6d8111127b0284f29e5196f1b7043e895308d2b1c6612feef0ce08
|
web/src/connectors/ConnectorsPage.tsx
CHANGED
|
@@ -263,12 +263,23 @@ export interface AmazonTableSync {
|
|
| 263 |
lastSyncAt: string | null;
|
| 264 |
nextSyncAt: string;
|
| 265 |
perDay: number;
|
|
|
|
|
|
|
| 266 |
state: string;
|
| 267 |
reason?: string;
|
| 268 |
/** A source-internal mismatch: the table published, and both counts are named. */
|
| 269 |
warning?: string;
|
| 270 |
}
|
| 271 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 272 |
export interface AmazonSyncScheduleReading {
|
| 273 |
state: "ready" | "empty" | "unavailable";
|
| 274 |
timezone: string;
|
|
@@ -295,7 +306,9 @@ export function parseAmazonSyncSchedule(body: Record<string, unknown>): AmazonSy
|
|
| 295 |
|| typeof row.account !== "string" || typeof row.state !== "string"
|
| 296 |
|| (row.lastSyncAt !== null && (typeof row.lastSyncAt !== "string" || !Number.isFinite(Date.parse(row.lastSyncAt))))
|
| 297 |
|| typeof row.nextSyncAt !== "string" || !Number.isFinite(Date.parse(row.nextSyncAt))
|
| 298 |
-
|| typeof row.perDay !== "number" || !Number.isInteger(row.perDay) || row.perDay < 1
|
|
|
|
|
|
|
| 299 |
const identity = JSON.stringify([row.connectionId, row.table]);
|
| 300 |
if (identities.has(identity)) return unavailable;
|
| 301 |
identities.add(identity);
|
|
@@ -336,7 +349,7 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
|
|
| 336 |
<table className="conn-settings-databases" aria-label="Amazon table sync schedule">
|
| 337 |
<thead><tr><th scope="col">Table</th>{multipleAccounts ? <th scope="col">Account</th> : null}
|
| 338 |
<th scope="col">Last sync</th><th scope="col">Next sync</th>
|
| 339 |
-
<th scope="col"
|
| 340 |
<tbody>{reading.tables.map((row) => <tr key={JSON.stringify([row.connectionId, row.table])}
|
| 341 |
data-amazon-table={row.table} data-connection-id={row.connectionId} data-table-sync-state={row.state}>
|
| 342 |
<th scope="row">{row.table.replace(/_/g, " ")}</th>
|
|
@@ -351,7 +364,7 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
|
|
| 351 |
: row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
|
| 352 |
: ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
|
| 353 |
</td>
|
| 354 |
-
<td
|
| 355 |
</tr>)}</tbody>
|
| 356 |
</table>
|
| 357 |
</div>
|
|
|
|
| 263 |
lastSyncAt: string | null;
|
| 264 |
nextSyncAt: string;
|
| 265 |
perDay: number;
|
| 266 |
+
/** The CADENCE period. A row without one refreshes daily. */
|
| 267 |
+
period?: AmazonRefreshPeriod;
|
| 268 |
state: string;
|
| 269 |
reason?: string;
|
| 270 |
/** A source-internal mismatch: the table published, and both counts are named. */
|
| 271 |
warning?: string;
|
| 272 |
}
|
| 273 |
|
| 274 |
+
export type AmazonRefreshPeriod = "day" | "week" | "month";
|
| 275 |
+
|
| 276 |
+
/** One plain word for how often a table refreshes. */
|
| 277 |
+
export function amazonRefreshLabel(row: Pick<AmazonTableSync, "perDay" | "period">): string {
|
| 278 |
+
if (row.period === "week") return "Weekly";
|
| 279 |
+
if (row.period === "month") return "Monthly";
|
| 280 |
+
return row.perDay > 1 ? `${row.perDay} times a day` : "Daily";
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
export interface AmazonSyncScheduleReading {
|
| 284 |
state: "ready" | "empty" | "unavailable";
|
| 285 |
timezone: string;
|
|
|
|
| 306 |
|| typeof row.account !== "string" || typeof row.state !== "string"
|
| 307 |
|| (row.lastSyncAt !== null && (typeof row.lastSyncAt !== "string" || !Number.isFinite(Date.parse(row.lastSyncAt))))
|
| 308 |
|| typeof row.nextSyncAt !== "string" || !Number.isFinite(Date.parse(row.nextSyncAt))
|
| 309 |
+
|| typeof row.perDay !== "number" || !Number.isInteger(row.perDay) || row.perDay < 1
|
| 310 |
+
|| (row.period !== undefined
|
| 311 |
+
&& (typeof row.period !== "string" || !["day", "week", "month"].includes(row.period)))) return unavailable;
|
| 312 |
const identity = JSON.stringify([row.connectionId, row.table]);
|
| 313 |
if (identities.has(identity)) return unavailable;
|
| 314 |
identities.add(identity);
|
|
|
|
| 349 |
<table className="conn-settings-databases" aria-label="Amazon table sync schedule">
|
| 350 |
<thead><tr><th scope="col">Table</th>{multipleAccounts ? <th scope="col">Account</th> : null}
|
| 351 |
<th scope="col">Last sync</th><th scope="col">Next sync</th>
|
| 352 |
+
<th scope="col">Refresh</th></tr></thead>
|
| 353 |
<tbody>{reading.tables.map((row) => <tr key={JSON.stringify([row.connectionId, row.table])}
|
| 354 |
data-amazon-table={row.table} data-connection-id={row.connectionId} data-table-sync-state={row.state}>
|
| 355 |
<th scope="row">{row.table.replace(/_/g, " ")}</th>
|
|
|
|
| 364 |
: row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
|
| 365 |
: ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
|
| 366 |
</td>
|
| 367 |
+
<td data-amazon-refresh={row.period ?? "day"}>{amazonRefreshLabel(row)}</td>
|
| 368 |
</tr>)}</tbody>
|
| 369 |
</table>
|
| 370 |
</div>
|